huskies/server/src/agents/pool/process.rs

//! Process management — kills orphaned PTY child processes on server shutdown.
//!
//! As of story 1090 (2026-05-15), all process termination in this module uses
//! [`crate::process_kill::sigkill_pids_and_verify`] — SIGHUP-based killing via
//! `portable_pty::ChildKiller` has been removed entirely from the server.
//!
//! ## History
//!
//! Prior to commit `fe9804b3`, the watchdog and all kill paths sent SIGHUP via
//! `portable_pty::ChildKiller::kill()`.  Claude Code ignores SIGHUP, so agents
//! survived "kills" and ran concurrently with their replacements — the root cause
//! of the 2026-05-15 duplicate-spawn incident.  `fe9804b3` migrated the watchdog;
//! story 1090 completes the migration by rewriting `kill_all_children` and
//! `kill_child_for_key` (this file) to use `pids_matching` + `sigkill_pids_and_verify`.
use crate::process_kill::{pids_matching, sigkill_pids_and_verify};
use crate::slog;
use crate::slog_warn;

use super::AgentPool;

impl AgentPool {
    /// Kill all active PTY child processes.
    ///
    /// Called on server shutdown to prevent orphaned Claude Code processes from
    /// continuing to run after the server exits.  Collects each agent's worktree
    /// path, then SIGKILLs every process running inside that path and verifies
    /// termination before returning.
    pub async fn kill_all_children(&self) {
        let worktree_paths: Vec<(String, std::path::PathBuf)> = {
            let agents = self.agents.lock().await;
            agents
                .iter()
                .filter_map(|(key, agent)| {
                    agent
                        .worktree_info
                        .as_ref()
                        .map(|wt| (key.clone(), wt.path.clone()))
                })
                .collect()
        };

        for (key, path) in worktree_paths {
            let pattern = path.display().to_string();
            let pids = pids_matching(&pattern);
            if pids.is_empty() {
                slog!(
                    "[agents] No processes found in worktree {} for '{key}' on shutdown",
                    path.display()
                );
                continue;
            }
            match sigkill_pids_and_verify(&pids) {
                Ok(n) => slog!(
                    "[agents] SIGKILL'd {n} process(es) in worktree {} for '{key}' on shutdown",
                    path.display()
                ),
                Err(survivors) => slog_warn!(
                    "[agents] SIGKILL incomplete for '{key}' on shutdown: \
                     pids still alive: {survivors:?}"
                ),
            }
        }
    }

    /// Kill and deregister the child process for a specific agent key.
    ///
    /// Fallback used by `stop_agent` when no worktree path is recorded for the
    /// agent.  Also the primary kill path for any caller that has only a composite
    /// key and not a worktree path directly.
    pub(super) async fn kill_child_for_key(&self, key: &str) {
        let worktree_path = {
            let agents = self.agents.lock().await;
            agents
                .get(key)
                .and_then(|a| a.worktree_info.as_ref().map(|wt| wt.path.clone()))
        };

        let Some(path) = worktree_path else {
            slog_warn!(
                "[agents] No worktree path recorded for '{key}'; \
                 cannot SIGKILL via process_kill (no-op)"
            );
            return;
        };

        let pattern = path.display().to_string();
        let pids = pids_matching(&pattern);
        if pids.is_empty() {
            slog!(
                "[agents] No processes found in worktree {} for '{key}' on stop",
                path.display()
            );
            return;
        }
        match sigkill_pids_and_verify(&pids) {
            Ok(n) => slog!(
                "[agents] SIGKILL'd {n} process(es) in worktree {} for '{key}' on stop",
                path.display()
            ),
            Err(survivors) => slog_warn!(
                "[agents] SIGKILL incomplete for '{key}' on stop: \
                 pids still alive: {survivors:?}"
            ),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::super::AgentPool;
    use crate::agents::AgentStatus;
    use std::process::Command;

    /// Returns true if a process with the given PID is currently running.
    fn process_is_running(pid: u32) -> bool {
        Command::new("ps")
            .args(["-p", &pid.to_string()])
            .output()
            .map(|o| o.status.success())
            .unwrap_or(false)
    }

    #[tokio::test]
    async fn kill_all_children_is_safe_on_empty_pool() {
        let pool = AgentPool::new_test(3001);
        pool.kill_all_children().await; // must not panic
    }

    /// AC 4 — `kill_child_for_key` SIGKILLs the single agent's process and
    /// verifies it is gone within 2 s.  The sleeper has the worktree path in
    /// its argv[0] so `pgrep -f` can locate it, mirroring how claude-code is
    /// launched with `--directory <worktree>` in production.
    #[tokio::test]
    async fn kill_child_for_key_kills_real_process() {
        use std::os::unix::process::CommandExt;

        let pool = AgentPool::new_test(3002);
        let tmp = tempfile::tempdir().unwrap();
        let worktree = tmp.path();

        // argv[0] = worktree path → pgrep -f <path> finds this process.
        let mut child = Command::new("sleep")
            .arg0(worktree.to_string_lossy().as_ref())
            .arg("100")
            .spawn()
            .expect("spawn sleeper");
        let pid = child.id();

        // Give pgrep a moment to see the new process.
        std::thread::sleep(std::time::Duration::from_millis(100));

        pool.inject_test_agent_with_path(
            "story-1090-kill",
            "coder",
            AgentStatus::Running,
            worktree.to_path_buf(),
        );

        assert!(
            process_is_running(pid),
            "sleeper pid {pid} should be running before kill_child_for_key"
        );

        pool.kill_child_for_key("story-1090-kill:coder").await;
        let _ = child.wait(); // reap zombie so ps -p returns false

        assert!(
            !process_is_running(pid),
            "sleeper pid {pid} should be dead after kill_child_for_key"
        );
    }

    /// AC 5 — `kill_all_children` SIGKILLs all agents' processes.  Two agents
    /// with distinct worktree paths are injected; both must be gone after the call.
    #[tokio::test]
    async fn kill_all_children_kills_multiple_real_processes() {
        use std::os::unix::process::CommandExt;

        let pool = AgentPool::new_test(3003);

        let mut sleepers: Vec<(u32, std::process::Child, tempfile::TempDir)> = (0..2_u32)
            .map(|i| {
                let tmp = tempfile::tempdir().unwrap();
                let worktree = tmp.path();
                // argv[0] = worktree path for pgrep discoverability.
                let child = Command::new("sleep")
                    .arg0(worktree.to_string_lossy().as_ref())
                    .arg("100")
                    .spawn()
                    .expect("spawn sleeper");
                let pid = child.id();
                pool.inject_test_agent_with_path(
                    &format!("story-1090-all-{i}"),
                    "coder",
                    AgentStatus::Running,
                    worktree.to_path_buf(),
                );
                (pid, child, tmp)
            })
            .collect();

        // Give pgrep a moment to see the new processes.
        std::thread::sleep(std::time::Duration::from_millis(100));

        for (pid, _, _) in &sleepers {
            assert!(
                process_is_running(*pid),
                "pid {pid} should be running before kill_all_children"
            );
        }

        pool.kill_all_children().await;

        for (pid, child, _tmp) in &mut sleepers {
            let _ = child.wait(); // reap zombie
            assert!(
                !process_is_running(*pid),
                "pid {pid} should be dead after kill_all_children"
            );
        }
    }
}