huskies: merge 1211 story Deterministic crash notification when the agent PTY dies mid-turn
This commit is contained in:
@@ -37,6 +37,8 @@ pub enum EventAction {
|
||||
},
|
||||
/// Post a disk-space recovery notification (story 1200).
|
||||
DiskRecovery,
|
||||
/// Post an agent-crashed notification (story 1211).
|
||||
AgentCrashed,
|
||||
/// Log server-side only; do not post to chat (e.g. hard rate-limit blocks).
|
||||
LogOnly,
|
||||
/// Reload the project configuration.
|
||||
@@ -68,6 +70,7 @@ pub fn classify(event: &WatcherEvent) -> EventAction {
|
||||
level: level.clone(),
|
||||
},
|
||||
WatcherEvent::DiskSpaceRecovered { .. } => EventAction::DiskRecovery,
|
||||
WatcherEvent::AgentCrashed { .. } => EventAction::AgentCrashed,
|
||||
_ => EventAction::Skip,
|
||||
}
|
||||
}
|
||||
@@ -216,4 +219,15 @@ mod tests {
|
||||
};
|
||||
assert_eq!(classify(&event), EventAction::MergeAutoRetry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_crashed_is_classified_correctly() {
|
||||
let event = WatcherEvent::AgentCrashed {
|
||||
story_id: "1_story_foo".to_string(),
|
||||
agent_name: "coder-1".to_string(),
|
||||
exit_code: Some(134),
|
||||
last_error_line: Some("thread panicked".to_string()),
|
||||
};
|
||||
assert_eq!(classify(&event), EventAction::AgentCrashed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +221,50 @@ pub fn format_agent_completed_notification(
|
||||
(plain, html)
|
||||
}
|
||||
|
||||
/// Format an agent-crashed notification message (story 1211).
|
||||
///
|
||||
/// Sent when an agent's PTY child process dies before completing its
|
||||
/// current turn (no `"result"` event observed) — a crash, kill, or
|
||||
/// watchdog termination rather than a clean finish. Includes the exit
|
||||
/// code and the last line emitted by the child when available.
|
||||
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
|
||||
pub fn format_agent_crashed_notification(
|
||||
item_id: &str,
|
||||
story_name: &str,
|
||||
agent_name: &str,
|
||||
exit_code: Option<u32>,
|
||||
last_error_line: Option<&str>,
|
||||
) -> (String, String) {
|
||||
let number = extract_item_number(item_id).unwrap_or(item_id);
|
||||
let effective_name = if story_name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(story_name)
|
||||
};
|
||||
let name_plain = effective_name.map(|n| format!("{n} ")).unwrap_or_default();
|
||||
let name_html = effective_name
|
||||
.map(|n| format!("<em>{n}</em> "))
|
||||
.unwrap_or_default();
|
||||
|
||||
let exit_code_display = exit_code
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
let mut plain = format!(
|
||||
"\u{1f480} #{number} {name_plain}\u{2014} {agent_name} crashed mid-turn \
|
||||
(exit code: {exit_code_display})"
|
||||
);
|
||||
let mut html = format!(
|
||||
"\u{1f480} <strong>#{number}</strong> {name_html}\u{2014} {agent_name} crashed mid-turn \
|
||||
(exit code: {exit_code_display})"
|
||||
);
|
||||
if let Some(line) = last_error_line {
|
||||
plain.push_str(&format!("\nLast line: {line}"));
|
||||
html.push_str(&format!("<br>Last line: <code>{line}</code>"));
|
||||
}
|
||||
(plain, html)
|
||||
}
|
||||
|
||||
/// Emoji used to badge a work item's kind in creation notifications.
|
||||
fn item_type_emoji(item_type: &str) -> &'static str {
|
||||
match item_type {
|
||||
|
||||
@@ -467,6 +467,44 @@ fn format_agent_completed_notification_falls_back_to_number() {
|
||||
);
|
||||
}
|
||||
|
||||
// ── format_agent_crashed_notification ─────────────────────────────────────
|
||||
|
||||
/// AC4: exit code and last error line both appear in the formatted text.
|
||||
#[test]
|
||||
fn format_agent_crashed_notification_includes_exit_code_and_last_line() {
|
||||
let (plain, html) = format_agent_crashed_notification(
|
||||
"1211_story_foo",
|
||||
"My Feature",
|
||||
"coder-1",
|
||||
Some(134),
|
||||
Some("assertion failed: output.write(&bytes).is_ok()"),
|
||||
);
|
||||
assert!(plain.contains("#1211"), "got: {plain}");
|
||||
assert!(plain.contains("coder-1"), "got: {plain}");
|
||||
assert!(plain.contains("crashed mid-turn"), "got: {plain}");
|
||||
assert!(plain.contains("exit code: 134"), "got: {plain}");
|
||||
assert!(
|
||||
plain.contains("assertion failed: output.write(&bytes).is_ok()"),
|
||||
"got: {plain}"
|
||||
);
|
||||
assert!(html.contains("exit code: 134"), "got: {html}");
|
||||
assert!(
|
||||
html.contains("<code>assertion failed: output.write(&bytes).is_ok()</code>"),
|
||||
"got: {html}"
|
||||
);
|
||||
}
|
||||
|
||||
/// AC4: exit code and last error line are both optional — an unavailable
|
||||
/// exit code renders as "unknown" and a missing last line is simply omitted.
|
||||
#[test]
|
||||
fn format_agent_crashed_notification_handles_missing_exit_code_and_line() {
|
||||
let (plain, html) =
|
||||
format_agent_crashed_notification("1211_story_foo", "", "coder-1", None, None);
|
||||
assert!(plain.contains("exit code: unknown"), "got: {plain}");
|
||||
assert!(!plain.contains("Last line"), "got: {plain}");
|
||||
assert!(!html.contains("Last line"), "got: {html}");
|
||||
}
|
||||
|
||||
// ── format_new_item_notification ──────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -17,8 +17,9 @@ use super::super::filter::{
|
||||
};
|
||||
use super::super::format::{
|
||||
MERGE_FAILURE_TAIL_LINES, NewItemInfo, format_agent_completed_notification,
|
||||
format_agent_started_notification, format_blocked_notification,
|
||||
format_disk_recovery_notification, format_disk_warning_notification, format_error_notification,
|
||||
format_agent_crashed_notification, format_agent_started_notification,
|
||||
format_blocked_notification, format_disk_recovery_notification,
|
||||
format_disk_warning_notification, format_error_notification,
|
||||
format_merge_auto_retry_notification, format_new_items_notification,
|
||||
format_oauth_account_swapped, format_oauth_accounts_exhausted, format_rate_limit_notification,
|
||||
truncate_gate_output,
|
||||
@@ -423,6 +424,36 @@ pub fn spawn_notification_listener(
|
||||
}
|
||||
}
|
||||
}
|
||||
EventAction::AgentCrashed => {
|
||||
if !config.status_push_enabled {
|
||||
continue;
|
||||
}
|
||||
let WatcherEvent::AgentCrashed {
|
||||
ref story_id,
|
||||
ref agent_name,
|
||||
exit_code,
|
||||
ref last_error_line,
|
||||
} = event
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let story_name = find_story_name_any_stage(&project_root, story_id);
|
||||
let (plain, html) = format_agent_crashed_notification(
|
||||
story_id,
|
||||
&story_name,
|
||||
agent_name,
|
||||
exit_code,
|
||||
last_error_line.as_deref(),
|
||||
);
|
||||
slog!("[bot] Sending agent-crashed notification: {plain}");
|
||||
for room_id in &rooms_for_notification(&get_room_ids) {
|
||||
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
|
||||
slog!(
|
||||
"[bot] Failed to send agent-crashed notification to {room_id}: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
EventAction::DiskRecovery => {
|
||||
if !config.status_push_enabled {
|
||||
continue;
|
||||
|
||||
@@ -44,6 +44,8 @@ pub fn watcher_event_to_response(e: WatcherEvent) -> Option<WsResponse> {
|
||||
// Disk-space events are forwarded to chat transports only; no WebSocket message (story 1200).
|
||||
WatcherEvent::DiskSpaceWarning { .. } => None,
|
||||
WatcherEvent::DiskSpaceRecovered { .. } => None,
|
||||
// Crash notifications are forwarded to chat transports only; no WebSocket message (story 1211).
|
||||
WatcherEvent::AgentCrashed { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user