From 01b24ff2ae005a542fa9c88844a31ad4e05d9b7f Mon Sep 17 00:00:00 2001 From: Timmy Date: Wed, 15 Jul 2026 16:54:55 +0100 Subject: [PATCH] =?UTF-8?q?Drop=20the=20drain=20check=20from=20upgrade=20?= =?UTF-8?q?=E2=80=94=20agent=20death=20is=20routine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents die all the time; the pipeline's retry machinery re-queues their work. Skipping busy sleds just created version skew and manual retries for no real protection. `upgrade all` now sweeps every sled unconditionally. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019fHdm92yjvguPi2LiXfLB9 --- .../chat/transport/matrix/project_rebuild.rs | 2 +- .../src/chat/transport/matrix/sled_upgrade.rs | 21 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/server/src/chat/transport/matrix/project_rebuild.rs b/server/src/chat/transport/matrix/project_rebuild.rs index 2673eea1..0e583f05 100644 --- a/server/src/chat/transport/matrix/project_rebuild.rs +++ b/server/src/chat/transport/matrix/project_rebuild.rs @@ -343,7 +343,7 @@ async fn wait_for_drain(container_name: &str, timeout_secs: u64) -> Option pgrep -f claude` — exits 0 with PID list when found, /// exits 1 when no matches (treated as 0 active processes). -pub(crate) async fn count_active_claude_processes(container_name: &str) -> Result { +async fn count_active_claude_processes(container_name: &str) -> Result { let out = tokio::process::Command::new("docker") .args(["exec", container_name, "pgrep", "-f", "claude"]) .output() diff --git a/server/src/chat/transport/matrix/sled_upgrade.rs b/server/src/chat/transport/matrix/sled_upgrade.rs index 3288c969..63a79f7b 100644 --- a/server/src/chat/transport/matrix/sled_upgrade.rs +++ b/server/src/chat/transport/matrix/sled_upgrade.rs @@ -7,8 +7,9 @@ //! //! The binary comes from the gateway's own artifact store //! (`~/.huskies/artifacts/`, published by the `release` command) — sleds never -//! download from anywhere but their gateway. Sleds with active agent -//! processes are skipped so an upgrade never kills in-flight work. +//! download from anywhere but their gateway. Agents running in a sled are +//! killed by the restart; the pipeline's retry machinery picks the work up +//! again, same as any other agent death. //! //! The gateway orchestrates each upgrade in four phases, streaming a marker to //! the chat room at each step: @@ -152,7 +153,6 @@ fn resolve_artifact_source(gateway_port: Option) -> Result<(String, Option< /// Upgrade every registered sled in sequence, streaming per-sled phase markers. /// -/// Sleds with active agent processes are skipped (reported, not failed). /// Returns a summary listing the outcome for each sled. pub async fn handle_upgrade_all( projects_store: &Arc>>, @@ -199,8 +199,8 @@ where /// begins. On any failure, an error message is returned and the previous /// binary remains active on the sled. /// -/// Sleds with active agent processes are skipped so an upgrade never kills -/// in-flight work. +/// Agents running in the sled are killed by the restart; the pipeline's +/// retry machinery re-queues their work. pub async fn handle_sled_upgrade( project: &str, projects_store: &Arc>>, @@ -236,17 +236,6 @@ where Err(e) => return e, }; - // ── Drain check ────────────────────────────────────────────────────────── - // Never kill in-flight agent work; the caller can retry once idle. - let container_name = format!("huskies-{project}"); - if let Ok(n) = super::project_rebuild::count_active_claude_processes(&container_name).await - && n > 0 - { - return format!( - "skipped — {n} active agent process(es) in `{container_name}`. Retry when idle." - ); - } - run_sled_upgrade(project, &sled_url, &source_url, expected_hash, send_phase).await }