huskies: merge 1211 story Deterministic crash notification when the agent PTY dies mid-turn

server/src/agents/pty/mod.rs

@@ -601,4 +601,126 @@ mod tests {
             "Expected RateLimitWarning for status=allowed, got: {evt:?}"
         );
     }
+
+    // ── story 1211: deterministic crash notification on mid-turn death ──────
+
+    /// AC1/AC3/AC4/AC5: a child killed mid-turn (inactivity watchdog, never
+    /// emits a `"result"` event) produces exactly one `AgentCrashed` watcher
+    /// event, carrying the exit code and the last line the child printed
+    /// before it died.  The watchdog-kill path used to `return Err(...)`
+    /// directly from inside the receive loop, bypassing the bottom
+    /// cleanup/notification code entirely — this test guards against that
+    /// regression.
+    #[tokio::test]
+    async fn killed_mid_turn_sends_exactly_one_crash_notification() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let tmp = tempfile::tempdir().unwrap();
+        let script = tmp.path().join("crash_then_hang.sh");
+        std::fs::write(
+            &script,
+            "#!/bin/sh\nprintf '%s\\n' 'assertion failed: output.write(&bytes).is_ok()'\nsleep 5\n",
+        )
+        .unwrap();
+        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
+
+        let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
+        let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
+        let event_log = Arc::new(Mutex::new(Vec::new()));
+
+        let result = run_agent_pty_streaming(
+            "1211_story_crash",
+            "coder-1",
+            "sh",
+            &[script.to_string_lossy().to_string()],
+            "--",
+            "/tmp",
+            &tx,
+            &event_log,
+            None,
+            1, // inactivity_timeout_secs = 1s
+            watcher_tx,
+            None,
+            None,
+        )
+        .await;
+
+        assert!(
+            result.is_err(),
+            "a watchdog-killed agent must still return the inactivity timeout error"
+        );
+
+        let evt = watcher_rx
+            .try_recv()
+            .expect("Expected exactly one AgentCrashed watcher event");
+        match evt {
+            WatcherEvent::AgentCrashed {
+                story_id,
+                agent_name,
+                exit_code,
+                last_error_line,
+            } => {
+                assert_eq!(story_id, "1211_story_crash");
+                assert_eq!(agent_name, "coder-1");
+                assert!(exit_code.is_some(), "exit code should be captured");
+                assert_eq!(
+                    last_error_line.as_deref(),
+                    Some("assertion failed: output.write(&bytes).is_ok()"),
+                    "last line before death should be captured"
+                );
+            }
+            other => panic!("Expected AgentCrashed, got: {other:?}"),
+        }
+
+        // AC5: exactly one crash notification — no second event queued.
+        assert!(
+            watcher_rx.try_recv().is_err(),
+            "must not emit a duplicate AgentCrashed for the same death event"
+        );
+    }
+
+    /// AC2: a turn that ends cleanly (a `"result"` event observed) followed
+    /// by a normal child exit must produce zero `AgentCrashed` notifications.
+    #[tokio::test]
+    async fn clean_turn_end_sends_zero_crash_notifications() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let tmp = tempfile::tempdir().unwrap();
+        let script = tmp.path().join("clean_result.sh");
+        std::fs::write(
+            &script,
+            "#!/bin/sh\nprintf '%s\\n' '{\"type\":\"result\",\"subtype\":\"success\"}'\n",
+        )
+        .unwrap();
+        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
+
+        let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
+        let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
+        let event_log = Arc::new(Mutex::new(Vec::new()));
+
+        let result = run_agent_pty_streaming(
+            "1211_story_clean",
+            "coder-1",
+            "sh",
+            &[script.to_string_lossy().to_string()],
+            "--",
+            "/tmp",
+            &tx,
+            &event_log,
+            None,
+            0, // no inactivity timeout
+            watcher_tx,
+            None,
+            None,
+        )
+        .await;
+
+        assert!(result.is_ok(), "PTY run should succeed: {:?}", result.err());
+        assert!(result.unwrap().exit_ok, "clean exit should report exit_ok");
+
+        assert!(
+            watcher_rx.try_recv().is_err(),
+            "a clean turn end followed by normal exit must not send AgentCrashed"
+        );
+    }
 }

server/src/agents/pty/runner.rs

@@ -255,6 +255,20 @@ fn run_agent_pty_blocking(
     // tool_result clears the last in-flight id (story 1196).
     let mut tool_calls_in_flight: std::collections::HashSet<String> =
         std::collections::HashSet::new();
+    // Tracks whether a `"result"` event was observed — the CLI's signal that
+    // the current turn completed normally (see
+    // llm/providers/claude_code/events/mod.rs). If the child dies before
+    // this is set, the death is a mid-turn crash (AC1/AC2 of story 1211)
+    // rather than a clean completion.
+    let mut saw_result_event = false;
+    // Most recent non-empty line emitted by the child before it died — used
+    // as the "last error line" in the crash notification (AC4) since panics
+    // and assertion failures print to the PTY's combined stdout/stderr.
+    let mut last_line: Option<String> = None;
+    // Set when the inactivity watchdog kills the child (AC3 of story 1211):
+    // tracked instead of returning early so the death still flows through
+    // the single crash-detection funnel below before the function returns.
+    let mut timed_out = false;
 
     loop {
         let effective_timeout = if !tool_calls_in_flight.is_empty() {
@@ -291,10 +305,8 @@ fn run_agent_pty_blocking(
                      {inactivity_timeout_secs}s with no output. Killing process."
                 );
                 let _ = child.kill();
-                let _ = child.wait();
-                return Err(format!(
-                    "Agent inactivity timeout: no output received for {inactivity_timeout_secs}s"
-                ));
+                timed_out = true;
+                break;
             }
         };
 
@@ -302,6 +314,7 @@ fn run_agent_pty_blocking(
         if trimmed.is_empty() {
             continue;
         }
+        last_line = Some(trimmed.to_string());
 
         // Try to parse as JSON
         let json: serde_json::Value = match serde_json::from_str(trimmed) {
@@ -440,6 +453,10 @@ fn run_agent_pty_blocking(
                 }
             }
             "result" => {
+                // A "result" event signals the CLI turn completed — mark the
+                // turn as clean so the post-loop crash check (AC1/AC2 of
+                // story 1211) does not treat this exit as a mid-turn death.
+                saw_result_event = true;
                 // Extract token usage from the result event.
                 if let Some(usage) = TokenUsage::from_result_event(&json) {
                     slog!(
@@ -481,6 +498,7 @@ fn run_agent_pty_blocking(
             false
         }
     };
+    let exit_code = wait_result.as_ref().ok().map(|status| status.exit_code());
 
     // Wait for the reader thread to finish so it releases the cloned PTY
     // master fd before we return.  Without this, the next PTY spawn for the
@@ -489,6 +507,32 @@ fn run_agent_pty_blocking(
         slog!("[agent:{story_id}:{agent_name}] Reader thread panicked: {e:?}");
     }
 
+    // AC1-AC3 (story 1211): single crash-detection funnel. Every exit path
+    // above (EOF, reader disconnect, IO error, or watchdog kill on timeout)
+    // breaks out of the loop into this one spot instead of returning early,
+    // so a mid-turn death is detected and notified exactly once regardless
+    // of which path triggered it (AC5). A turn is "clean" once a `"result"`
+    // event has been observed; anything else was killed or crashed before
+    // finishing its turn.
+    if !saw_result_event {
+        slog_warn!(
+            "[agent:{story_id}:{agent_name}] Agent died mid-turn (no result event observed); \
+             exit_code={exit_code:?}, last_line={last_line:?}"
+        );
+        let _ = watcher_tx.send(WatcherEvent::AgentCrashed {
+            story_id: story_id.to_string(),
+            agent_name: agent_name.to_string(),
+            exit_code,
+            last_error_line: last_line.clone(),
+        });
+    }
+
+    if timed_out {
+        return Err(format!(
+            "Agent inactivity timeout: no output received for {inactivity_timeout_secs}s"
+        ));
+    }
+
     // Log whether session was created — Session: None indicates CLI died
     // before emitting any events (possible causes: rate limit, budget
     // exhaustion, PTY write failure, CLI crash).