crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

Remove all alternate update paths — fleet redeploy is release + upgrade all

Browse Source 
Killed:
- rebuild_and_restart (in-container cargo self-compile): the MCP tool,
  the `rebuild` chat command in all four transports, the web-ui bot
  command, and the underlying function. This was the path that caused
  the exec() deadlocks.
- upgrade_sled gateway MCP tool: second entry point to sled upgrades,
  defaulted to serving the gateway's own macOS binary to Linux sleds.
- GET /api/huskies-binary (both sled and gateway route trees): served
  current_exe(), wrong platform when the gateway is macOS. Superseded
  by /api/artifacts/ which now also serves on the gateway route tree.
- `huskies upgrade` CLI subcommand and --source flag: third way of
  doing the same download-and-replace. Escape hatch for a bricked sled
  is `docker cp` + restart.

Kept, distinct jobs: `project-rebuild` (container/image updates),
`rebuild gateway` + script/local-release (gateway self-update).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019fHdm92yjvguPi2LiXfLB9
This commit is contained in:
Timmy
2026-07-15 16:46:11 +01:00
co-authored by Claude Fable 5
parent 83f941b77e
commit f39c4b7c4b
24 changed files with 36 additions and 824 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/rebuild.rs
+3 -138
View File
@@ -1,6 +1,5 @@
//! Server rebuild and restart logic shared between the MCP tool and Matrix bot command.
//! Shutdown notification and drain-and-exit sequence for server restarts.
use crate::agents::AgentPool;
use crate::chat::ChatTransport;
use crate::slog;
use std::path::Path;
@@ -77,8 +76,8 @@ impl BotShutdownNotifier {
// ── Shared shutdown sequence ─────────────────────────────────────────────
/// Flush CRDT persistence, remove port files, and exit so Docker restarts
/// the container with the new binary. Used by both `rebuild_and_restart`
/// and `upgrade_and_reexec`.
/// the container with the new binary. Called by `upgrade_and_reexec` after
/// the binary has been replaced on disk.
pub async fn drain_and_exit(project_root: &Path, label: &str) -> ! {
crate::crdt_state::flush_persistence(std::time::Duration::from_secs(5)).await;
@@ -95,140 +94,6 @@ pub async fn drain_and_exit(project_root: &Path, label: &str) -> ! {
std::process::exit(0)
}
// ── Rebuild ──────────────────────────────────────────────────────────────
/// Rebuild the server binary and exit for Docker restart.
///
/// 1. Gracefully stops all running agents (kills PTY children).
/// 2. Runs `cargo build [-p huskies]` from the workspace root, matching
/// the current build profile (debug or release).
/// 3. If the build fails, returns the build error (server stays up).
/// 4. If the build succeeds, sends a best-effort shutdown notification (if a
/// [`BotShutdownNotifier`] is provided), then calls [`drain_and_exit`] to
/// flush persistence and exit. Docker's restart policy brings the
/// container back up with the new binary.
pub async fn rebuild_and_restart(
agents: &AgentPool,
project_root: &Path,
notifier: Option<&BotShutdownNotifier>,
) -> Result<String, String> {
slog!("[rebuild] Rebuild and restart requested");
// 1. Gracefully stop all running agents.
let running_count = agents
.list_agents()
.await
.unwrap_or_default()
.iter()
.filter(|a| a.status == crate::agents::AgentStatus::Running)
.count();
if running_count > 0 {
slog!("[rebuild] Stopping {running_count} running agent(s) before rebuild");
}
agents.kill_all_children().await;
// 2. Find the workspace root (parent of the server binary's source).
// CARGO_MANIFEST_DIR at compile time points to the `server/` crate;
// the workspace root is its parent. When running inside Docker the
// compile-time path (/app) no longer exists — the source is bind-mounted
// at the project_root instead, so fall back to that.
let compile_time_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.filter(|p| p.join("Cargo.toml").exists());
let workspace_root = match compile_time_root {
Some(p) => p.to_path_buf(),
None => {
if project_root.join("Cargo.toml").exists() {
project_root.to_path_buf()
} else {
return Err(
"Cannot determine workspace root: neither CARGO_MANIFEST_DIR nor project_root contain Cargo.toml".to_string(),
);
}
}
};
let workspace_root = workspace_root.as_path();
slog!(
"[rebuild] Building server from workspace root: {}",
workspace_root.display()
);
// 3. Rebuild the frontend bundle so rust-embed picks up the latest assets.
let frontend_dir = workspace_root.join("frontend");
if frontend_dir.join("package.json").exists() {
slog!("[rebuild] Building frontend");
let fe_output = tokio::task::spawn_blocking({
let frontend_dir = frontend_dir.clone();
move || {
std::process::Command::new("npm")
.args(["run", "build"])
.current_dir(&frontend_dir)
.output()
}
})
.await
.map_err(|e| format!("Frontend build task panicked: {e}"))?
.map_err(|e| format!("Failed to run npm run build: {e}"))?;
if !fe_output.status.success() {
let stderr = String::from_utf8_lossy(&fe_output.stderr);
slog!("[rebuild] Frontend build failed:\n{stderr}");
return Err(format!("Frontend build failed:\n{stderr}"));
}
slog!("[rebuild] Frontend build succeeded");
}
// 4. Build the server binary, matching the current build profile so the
// re-exec via current_exe() picks up the new binary.
let build_args: Vec<&str> = if cfg!(debug_assertions) {
vec!["build", "-p", "huskies"]
} else {
vec!["build", "--release", "-p", "huskies"]
};
slog!("[rebuild] cargo {}", build_args.join(" "));
let output = tokio::task::spawn_blocking({
let workspace_root = workspace_root.to_path_buf();
move || {
std::process::Command::new("cargo")
.args(&build_args)
.current_dir(&workspace_root)
.output()
}
})
.await
.map_err(|e| format!("Build task panicked: {e}"))?
.map_err(|e| format!("Failed to run cargo build: {e}"))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
slog!("[rebuild] Build failed:\n{stderr}");
return Err(format!("Build failed:\n{stderr}"));
}
// Write the current git HEAD to a file so we can verify which commit is running.
if let Ok(head) = std::process::Command::new("git")
.args(["rev-parse", "--short", "HEAD"])
.current_dir(workspace_root)
.output()
{
let hash = String::from_utf8_lossy(&head.stdout).trim().to_string();
let _ = std::fs::write(workspace_root.join(".huskies/build_hash"), &hash);
slog!("[rebuild] Build succeeded (commit {hash}), re-execing with new binary");
} else {
slog!("[rebuild] Build succeeded, re-execing with new binary");
}
// 5. Send shutdown notification before replacing the process so that chat
// participants know the bot is going offline. Best-effort only — we
// do not abort the rebuild if the send fails.
if let Some(n) = notifier {
n.notify(ShutdownReason::Rebuild).await;
}
drain_and_exit(project_root, "rebuild").await
}
// ── Tests ────────────────────────────────────────────────────────────────
#[cfg(test)]