Pre-check binary writability before accepting an upgrade request

/api/upgrade now verifies the target can be replaced (create + remove
the swap's tmp file) before returning 202. A sled that cannot write
its own binary — e.g. a container predating the /opt/huskies/bin
layout — fails phase 1 of `upgrade all` loudly instead of returning
202, staying healthy, and silently remaining on the old version, which
is exactly what happened on the first fleet deploy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019fHdm92yjvguPi2LiXfLB9

server/src/http/mod.rs

@@ -255,9 +255,18 @@ pub async fn upgrade_trigger_handler(
         }
     };
 
+    // Fail fast if the binary cannot be replaced — a background failure after
+    // a 202 looks like a healthy sled that silently stayed on the old version.
+    if let Err(e) = crate::upgrade::preflight_target_writable() {
+        return poem::Response::builder()
+            .status(StatusCode::INTERNAL_SERVER_ERROR)
+            .body(e);
+    }
+
     let project_root = ctx.state.get_project_root().unwrap_or_default();
 
-    // Spawn upgrade in background so we can return 202 before exec() fires.
+    // Spawn the download + swap in the background so we can return 202 before
+    // the process exits for its Docker restart.
     tokio::spawn(async move {
         if let Err(e) = crate::upgrade::upgrade_and_reexec(&source_url, &project_root).await {
             crate::slog!("[upgrade] Upgrade failed: {e}");
@@ -266,7 +275,7 @@ pub async fn upgrade_trigger_handler(
 
     poem::Response::builder()
         .status(StatusCode::ACCEPTED)
-        .body("Upgrade triggered. The sled will re-exec momentarily.")
+        .body("Upgrade triggered. The sled will restart momentarily.")
 }
 
 /// Canonical artifact filename for sled binaries on this deployment's platform.

server/src/upgrade.rs

@@ -60,11 +60,9 @@ pub async fn fetch_and_replace_binary(source_url: &str, target_path: &Path) -> R
 // ── Full server upgrade (called from the running process) ─────────────────
 
 /// Fetch a new binary, atomically replace the current executable, drain CRDT
-/// persistence, and exit so Docker restarts the container with the new binary.
-///
-/// The entrypoint script checks for a rebuilt binary at
-/// /app/target/release/huskies and prefers it over the image-baked
-/// /usr/local/bin/huskies.
+/// persistence, and exit so Docker restarts the container with the new binary
+/// (the entrypoint launches `huskies` from PATH, which resolves to the
+/// replaced file).
 ///
 /// On failure it returns `Err(message)` so the caller can report the error
 /// while keeping the original server running.
@@ -76,6 +74,28 @@ pub async fn upgrade_and_reexec(source_url: &str, project_root: &Path) -> Result
     crate::rebuild::drain_and_exit(project_root, "upgrade").await
 }
 
+/// Verify the upgrade target can actually be replaced, without downloading
+/// anything: create and remove the same sibling tmp file the real swap uses.
+///
+/// Called by the `/api/upgrade` handler BEFORE returning 202 so that a sled
+/// that cannot write its own binary (e.g. a container from an old image where
+/// the install dir is root-owned) fails the upgrade loudly and immediately
+/// instead of accepting the request and failing in the background.
+pub fn preflight_target_writable() -> Result<(), String> {
+    let target = resolve_target_path();
+    let tmp = sibling_tmp_path(&target)?;
+    std::fs::write(&tmp, b"").map_err(|e| {
+        format!(
+            "Upgrade target `{}` is not replaceable by this process: {e}. \
+             The container likely predates the /opt/huskies/bin layout — \
+             run `project-rebuild` on it once to migrate.",
+            target.display()
+        )
+    })?;
+    let _ = std::fs::remove_file(&tmp);
+    Ok(())
+}
+
 // ── Helpers ───────────────────────────────────────────────────────────────
 
 /// Resolve the path to replace with the new binary.