huskies: merge 1211 story Deterministic crash notification when the agent PTY dies mid-turn

This commit is contained in:
Huskies Agent
2026-07-18 11:00:28 +00:00
parent 153333d055
commit 71f3fa09c4
8 changed files with 315 additions and 6 deletions
@@ -37,6 +37,8 @@ pub enum EventAction {
},
/// Post a disk-space recovery notification (story 1200).
DiskRecovery,
/// Post an agent-crashed notification (story 1211).
AgentCrashed,
/// Log server-side only; do not post to chat (e.g. hard rate-limit blocks).
LogOnly,
/// Reload the project configuration.
@@ -68,6 +70,7 @@ pub fn classify(event: &WatcherEvent) -> EventAction {
level: level.clone(),
},
WatcherEvent::DiskSpaceRecovered { .. } => EventAction::DiskRecovery,
WatcherEvent::AgentCrashed { .. } => EventAction::AgentCrashed,
_ => EventAction::Skip,
}
}
@@ -216,4 +219,15 @@ mod tests {
};
assert_eq!(classify(&event), EventAction::MergeAutoRetry);
}
#[test]
fn agent_crashed_is_classified_correctly() {
let event = WatcherEvent::AgentCrashed {
story_id: "1_story_foo".to_string(),
agent_name: "coder-1".to_string(),
exit_code: Some(134),
last_error_line: Some("thread panicked".to_string()),
};
assert_eq!(classify(&event), EventAction::AgentCrashed);
}
}
@@ -221,6 +221,50 @@ pub fn format_agent_completed_notification(
(plain, html)
}
/// Format an agent-crashed notification message (story 1211).
///
/// Sent when an agent's PTY child process dies before completing its
/// current turn (no `"result"` event observed) — a crash, kill, or
/// watchdog termination rather than a clean finish. Includes the exit
/// code and the last line emitted by the child when available.
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
pub fn format_agent_crashed_notification(
item_id: &str,
story_name: &str,
agent_name: &str,
exit_code: Option<u32>,
last_error_line: Option<&str>,
) -> (String, String) {
let number = extract_item_number(item_id).unwrap_or(item_id);
let effective_name = if story_name.is_empty() {
None
} else {
Some(story_name)
};
let name_plain = effective_name.map(|n| format!("{n} ")).unwrap_or_default();
let name_html = effective_name
.map(|n| format!("<em>{n}</em> "))
.unwrap_or_default();
let exit_code_display = exit_code
.map(|c| c.to_string())
.unwrap_or_else(|| "unknown".to_string());
let mut plain = format!(
"\u{1f480} #{number} {name_plain}\u{2014} {agent_name} crashed mid-turn \
(exit code: {exit_code_display})"
);
let mut html = format!(
"\u{1f480} <strong>#{number}</strong> {name_html}\u{2014} {agent_name} crashed mid-turn \
(exit code: {exit_code_display})"
);
if let Some(line) = last_error_line {
plain.push_str(&format!("\nLast line: {line}"));
html.push_str(&format!("<br>Last line: <code>{line}</code>"));
}
(plain, html)
}
/// Emoji used to badge a work item's kind in creation notifications.
fn item_type_emoji(item_type: &str) -> &'static str {
match item_type {
@@ -467,6 +467,44 @@ fn format_agent_completed_notification_falls_back_to_number() {
);
}
// ── format_agent_crashed_notification ─────────────────────────────────────
/// AC4: exit code and last error line both appear in the formatted text.
#[test]
fn format_agent_crashed_notification_includes_exit_code_and_last_line() {
let (plain, html) = format_agent_crashed_notification(
"1211_story_foo",
"My Feature",
"coder-1",
Some(134),
Some("assertion failed: output.write(&bytes).is_ok()"),
);
assert!(plain.contains("#1211"), "got: {plain}");
assert!(plain.contains("coder-1"), "got: {plain}");
assert!(plain.contains("crashed mid-turn"), "got: {plain}");
assert!(plain.contains("exit code: 134"), "got: {plain}");
assert!(
plain.contains("assertion failed: output.write(&bytes).is_ok()"),
"got: {plain}"
);
assert!(html.contains("exit code: 134"), "got: {html}");
assert!(
html.contains("<code>assertion failed: output.write(&bytes).is_ok()</code>"),
"got: {html}"
);
}
/// AC4: exit code and last error line are both optional — an unavailable
/// exit code renders as "unknown" and a missing last line is simply omitted.
#[test]
fn format_agent_crashed_notification_handles_missing_exit_code_and_line() {
let (plain, html) =
format_agent_crashed_notification("1211_story_foo", "", "coder-1", None, None);
assert!(plain.contains("exit code: unknown"), "got: {plain}");
assert!(!plain.contains("Last line"), "got: {plain}");
assert!(!html.contains("Last line"), "got: {html}");
}
// ── format_new_item_notification ──────────────────────────────────────────
#[test]
@@ -17,8 +17,9 @@ use super::super::filter::{
};
use super::super::format::{
MERGE_FAILURE_TAIL_LINES, NewItemInfo, format_agent_completed_notification,
format_agent_started_notification, format_blocked_notification,
format_disk_recovery_notification, format_disk_warning_notification, format_error_notification,
format_agent_crashed_notification, format_agent_started_notification,
format_blocked_notification, format_disk_recovery_notification,
format_disk_warning_notification, format_error_notification,
format_merge_auto_retry_notification, format_new_items_notification,
format_oauth_account_swapped, format_oauth_accounts_exhausted, format_rate_limit_notification,
truncate_gate_output,
@@ -423,6 +424,36 @@ pub fn spawn_notification_listener(
}
}
}
EventAction::AgentCrashed => {
if !config.status_push_enabled {
continue;
}
let WatcherEvent::AgentCrashed {
ref story_id,
ref agent_name,
exit_code,
ref last_error_line,
} = event
else {
continue;
};
let story_name = find_story_name_any_stage(&project_root, story_id);
let (plain, html) = format_agent_crashed_notification(
story_id,
&story_name,
agent_name,
exit_code,
last_error_line.as_deref(),
);
slog!("[bot] Sending agent-crashed notification: {plain}");
for room_id in &rooms_for_notification(&get_room_ids) {
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
slog!(
"[bot] Failed to send agent-crashed notification to {room_id}: {e}"
);
}
}
}
EventAction::DiskRecovery => {
if !config.status_push_enabled {
continue;
+2
View File
@@ -44,6 +44,8 @@ pub fn watcher_event_to_response(e: WatcherEvent) -> Option<WsResponse> {
// Disk-space events are forwarded to chat transports only; no WebSocket message (story 1200).
WatcherEvent::DiskSpaceWarning { .. } => None,
WatcherEvent::DiskSpaceRecovered { .. } => None,
// Crash notifications are forwarded to chat transports only; no WebSocket message (story 1211).
WatcherEvent::AgentCrashed { .. } => None,
}
}