Remove all alternate update paths — fleet redeploy is release + upgrade all
Killed: - rebuild_and_restart (in-container cargo self-compile): the MCP tool, the `rebuild` chat command in all four transports, the web-ui bot command, and the underlying function. This was the path that caused the exec() deadlocks. - upgrade_sled gateway MCP tool: second entry point to sled upgrades, defaulted to serving the gateway's own macOS binary to Linux sleds. - GET /api/huskies-binary (both sled and gateway route trees): served current_exe(), wrong platform when the gateway is macOS. Superseded by /api/artifacts/ which now also serves on the gateway route tree. - `huskies upgrade` CLI subcommand and --source flag: third way of doing the same download-and-replace. Escape hatch for a bricked sled is `docker cp` + restart. Kept, distinct jobs: `project-rebuild` (container/image updates), `rebuild gateway` + script/local-release (gateway self-update). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019fHdm92yjvguPi2LiXfLB9
This commit is contained in:
Timmy
Claude Fable 5
@@ -57,8 +57,8 @@ pub struct AppContext {
|
||||
pub qa_app_process: Arc<std::sync::Mutex<Option<std::process::Child>>>,
|
||||
/// Best-effort shutdown notifier for active bot channels (Slack / WhatsApp).
|
||||
///
|
||||
/// When set, the MCP `rebuild_and_restart` tool uses this to announce the
|
||||
/// shutdown to configured channels before re-execing the server binary.
|
||||
/// When set, restart-inducing paths use this to announce the shutdown to
|
||||
/// configured channels before the process exits.
|
||||
/// `None` when no webhook-based bot transport is configured.
|
||||
pub bot_shutdown: Option<Arc<BotShutdownNotifier>>,
|
||||
/// Watch sender used to signal the Matrix bot task that the server is
|
||||
|
||||
@@ -27,8 +27,6 @@ const GATEWAY_TOOLS: &[&str] = &[
|
||||
// Handled at the gateway so the Matrix bot's perm_rx listener is used
|
||||
// rather than the container's (which has no interactive session attached).
|
||||
"prompt_permission",
|
||||
// Binary self-update: gateway serves its own binary and triggers upgrade on sleds.
|
||||
"upgrade_sled",
|
||||
// One-shot container rebuild: build fresh image, swap container, preserve state.
|
||||
"project_rebuild",
|
||||
];
|
||||
@@ -134,23 +132,6 @@ pub(crate) fn gateway_tool_definitions() -> Vec<Value> {
|
||||
"properties": {}
|
||||
}
|
||||
}),
|
||||
json!({
|
||||
"name": "upgrade_sled",
|
||||
"description": "Trigger a binary self-update on a project sled. The sled downloads the new binary from `source_url` (defaults to this gateway's /api/huskies-binary endpoint), atomically replaces its own executable, drains CRDT persistence so no ops are lost, and re-execs. Without `project`, upgrades the active project.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Name of the project sled to upgrade. Defaults to the currently active project."
|
||||
},
|
||||
"source_url": {
|
||||
"type": "string",
|
||||
"description": "HTTP URL of the binary to install (e.g. 'http://gateway:3000/api/huskies-binary'). Defaults to this gateway's own binary endpoint."
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
json!({
|
||||
"name": "project_rebuild",
|
||||
"description": "Rebuild a project's Docker image from its Dockerfile.fragment, swap the container, and preserve all CRDT and pipeline state. In-flight coder/merge work is drained before the swap; if not drainable within the timeout the command refuses. On success returns the new image hash and container ID.",
|
||||
@@ -438,7 +419,6 @@ async fn handle_gateway_tool(
|
||||
"aggregate_pipeline_status" => handle_aggregate_pipeline_status_tool(state, id).await,
|
||||
"agents.list" => handle_agents_list_tool(id),
|
||||
"prompt_permission" => handle_prompt_permission_tool(params, state, id).await,
|
||||
"upgrade_sled" => handle_upgrade_sled_tool(params, state, id).await,
|
||||
"project_rebuild" => handle_project_rebuild_tool(params, state, id).await,
|
||||
_ => JsonRpcResponse::error(id, -32601, format!("Unknown gateway tool: {tool_name}")),
|
||||
}
|
||||
@@ -893,93 +873,6 @@ fn handle_agents_list_tool(id: Option<Value>) -> JsonRpcResponse {
|
||||
)
|
||||
}
|
||||
|
||||
/// Handle the `upgrade_sled` gateway tool.
|
||||
///
|
||||
/// Posts `{"source_url": "<url>"}` to the target sled's `/api/upgrade` endpoint,
|
||||
/// which triggers the sled to download the new binary, drain CRDT persistence,
|
||||
/// and re-exec. Returns 202 text immediately — the sled connection will drop
|
||||
/// shortly after as `exec()` replaces the process.
|
||||
async fn handle_upgrade_sled_tool(
|
||||
params: &Value,
|
||||
state: &GatewayState,
|
||||
id: Option<Value>,
|
||||
) -> JsonRpcResponse {
|
||||
let args = params.get("arguments").unwrap_or(params);
|
||||
|
||||
// Resolve target project URL (explicit project arg or active project).
|
||||
let project_name = args.get("project").and_then(|v| v.as_str());
|
||||
let sled_url = if let Some(name) = project_name {
|
||||
let projects = state.projects.read().await;
|
||||
match projects.get(name).and_then(|e| e.url.clone()) {
|
||||
Some(u) => u,
|
||||
None => {
|
||||
return JsonRpcResponse::error(
|
||||
id,
|
||||
-32602,
|
||||
format!("Project '{name}' not found or has no URL configured"),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match state.active_url().await {
|
||||
Ok(u) => u,
|
||||
Err(e) => return JsonRpcResponse::error(id, -32603, e.to_string()),
|
||||
}
|
||||
};
|
||||
|
||||
// Build the binary source URL: caller-supplied or this gateway's own endpoint.
|
||||
let source_url = args
|
||||
.get("source_url")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| {
|
||||
// Default: the gateway serves its own binary at /api/huskies-binary.
|
||||
// Use the same host/port the gateway is bound to.
|
||||
std::env::var("HUSKIES_GATEWAY_BINARY_URL")
|
||||
.unwrap_or_else(|_| format!("http://gateway:{}/api/huskies-binary", state.port))
|
||||
});
|
||||
|
||||
let upgrade_url = format!("{sled_url}/api/upgrade");
|
||||
let body = serde_json::json!({ "source_url": source_url });
|
||||
|
||||
let active_name = project_name.map(|s| s.to_string()).unwrap_or_else(|| {
|
||||
state
|
||||
.active_project
|
||||
.try_read()
|
||||
.map(|g| g.clone())
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
match state.client.post(&upgrade_url).json(&body).send().await {
|
||||
Ok(resp) if resp.status().is_success() || resp.status().as_u16() == 202 => {
|
||||
JsonRpcResponse::success(
|
||||
id,
|
||||
json!({
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": format!(
|
||||
"Upgrade triggered on '{active_name}'. The sled is downloading the new binary from {source_url} and will re-exec momentarily."
|
||||
)
|
||||
}]
|
||||
}),
|
||||
)
|
||||
}
|
||||
Ok(resp) => JsonRpcResponse::error(
|
||||
id,
|
||||
-32603,
|
||||
format!(
|
||||
"Sled returned HTTP {} for upgrade request to {upgrade_url}",
|
||||
resp.status()
|
||||
),
|
||||
),
|
||||
Err(e) => JsonRpcResponse::error(
|
||||
id,
|
||||
-32603,
|
||||
format!("Failed to send upgrade request to {upgrade_url}: {e}"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle the `project_rebuild` gateway tool.
|
||||
///
|
||||
/// Rebuilds a project's Docker image, swaps the container, and preserves all
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use crate::agents::move_story_to_stage;
|
||||
use crate::http::context::AppContext;
|
||||
use crate::log_buffer;
|
||||
use crate::slog;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
mod permission;
|
||||
@@ -43,23 +42,6 @@ pub(crate) fn tool_get_server_logs(args: &Value) -> Result<String, String> {
|
||||
Ok(all_lines[start..].join("\n"))
|
||||
}
|
||||
|
||||
/// Rebuild the server binary and re-exec (delegates to `crate::rebuild`).
|
||||
pub(crate) async fn tool_rebuild_and_restart(ctx: &AppContext) -> Result<String, String> {
|
||||
slog!("[rebuild] Rebuild and restart requested via MCP tool");
|
||||
|
||||
// Signal the Matrix bot (if active) so it can send its own shutdown
|
||||
// announcement before the process is replaced. Best-effort: we wait up
|
||||
// to 1.5 s for the message to be delivered.
|
||||
if let Some(ref tx) = ctx.matrix_shutdown_tx {
|
||||
let _ = tx.send(Some(crate::rebuild::ShutdownReason::Rebuild));
|
||||
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||
}
|
||||
|
||||
let project_root = ctx.state.get_project_root().unwrap_or_default();
|
||||
let notifier = ctx.bot_shutdown.as_deref();
|
||||
crate::rebuild::rebuild_and_restart(&ctx.services.agents, &project_root, notifier).await
|
||||
}
|
||||
|
||||
/// MCP tool called by Claude Code via `--permission-prompt-tool`.
|
||||
///
|
||||
/// Forwards the permission request through the shared channel to the active
|
||||
|
||||
@@ -335,57 +335,6 @@ mod tests {
|
||||
assert_eq!(servers[0], "huskies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuild_and_restart_in_tools_list() {
|
||||
use super::super::super::tools_list::handle_tools_list;
|
||||
let resp = handle_tools_list(Some(json!(1)));
|
||||
let tools = resp.result.unwrap()["tools"].as_array().unwrap().clone();
|
||||
let tool = tools.iter().find(|t| t["name"] == "rebuild_and_restart");
|
||||
assert!(
|
||||
tool.is_some(),
|
||||
"rebuild_and_restart missing from tools list"
|
||||
);
|
||||
let t = tool.unwrap();
|
||||
assert!(t["description"].as_str().unwrap().contains("Rebuild"));
|
||||
assert!(t["inputSchema"].is_object());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebuild_and_restart_kills_agents_before_build() {
|
||||
// Verify that calling rebuild_and_restart on an empty pool doesn't
|
||||
// panic and proceeds to the build step. We can't test exec() in a
|
||||
// unit test, but we can verify the build attempt happens.
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let ctx = test_ctx(tmp.path());
|
||||
|
||||
// The build will succeed (we're running in the real workspace) and
|
||||
// then exec() will be called — which would replace our test process.
|
||||
// So we only test that the function *runs* without panicking up to
|
||||
// the agent-kill step. We do this by checking the pool is empty.
|
||||
assert_eq!(ctx.services.agents.list_agents().await.unwrap().len(), 0);
|
||||
ctx.services.agents.kill_all_children().await; // should not panic on empty pool
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuild_uses_matching_build_profile() {
|
||||
// The build must use the same profile (debug/release) as the running
|
||||
// binary, otherwise cargo build outputs to a different target dir and
|
||||
// current_exe() still points at the old binary.
|
||||
let build_args: Vec<&str> = if cfg!(debug_assertions) {
|
||||
vec!["build", "-p", "huskies"]
|
||||
} else {
|
||||
vec!["build", "--release", "-p", "huskies"]
|
||||
};
|
||||
|
||||
// Tests always run in debug mode, so --release must NOT be present.
|
||||
assert!(
|
||||
!build_args.contains(&"--release"),
|
||||
"In debug builds, rebuild must not pass --release (would put \
|
||||
the binary in target/release/ while current_exe() points to \
|
||||
target/debug/)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── move_story tool tests ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -81,8 +81,6 @@ pub async fn dispatch_tool_call(
|
||||
// Diagnostics
|
||||
"get_server_logs" => diagnostics::tool_get_server_logs(&args),
|
||||
"get_version" => diagnostics::tool_get_version(ctx),
|
||||
// Server lifecycle
|
||||
"rebuild_and_restart" => diagnostics::tool_rebuild_and_restart(ctx).await,
|
||||
// Permission bridge (Claude Code → frontend dialog)
|
||||
"prompt_permission" => diagnostics::tool_prompt_permission(&args, ctx).await,
|
||||
// Token usage
|
||||
|
||||
@@ -78,7 +78,6 @@ mod tests {
|
||||
assert!(names.contains(&"get_server_logs"));
|
||||
assert!(names.contains(&"prompt_permission"));
|
||||
assert!(names.contains(&"get_pipeline_status"));
|
||||
assert!(names.contains(&"rebuild_and_restart"));
|
||||
assert!(names.contains(&"get_token_usage"));
|
||||
assert!(names.contains(&"move_story"));
|
||||
assert!(names.contains(&"unblock_story"));
|
||||
@@ -117,7 +116,7 @@ mod tests {
|
||||
assert!(names.contains(&"convert_item_type"));
|
||||
assert!(names.contains(&"edit"));
|
||||
assert!(names.contains(&"write"));
|
||||
assert_eq!(tools.len(), 85);
|
||||
assert_eq!(tools.len(), 84);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -42,14 +42,6 @@ pub(super) fn system_tools() -> Vec<Value> {
|
||||
"properties": {}
|
||||
}
|
||||
}),
|
||||
json!({
|
||||
"name": "rebuild_and_restart",
|
||||
"description": "Rebuild the server binary from source and re-exec with the new binary. Gracefully stops all running agents before restart. If the build fails, the server stays up and returns the build error.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
}),
|
||||
json!({
|
||||
"name": "prompt_permission",
|
||||
"description": "Present a permission request to the user via the web UI. Used by Claude Code's --permission-prompt-tool to delegate permission decisions to the frontend dialog. Returns on approval; returns an error on denial.",
|
||||
|
||||
+2
-31
@@ -124,7 +124,6 @@ pub fn build_routes(
|
||||
|
||||
route = route
|
||||
.at("/api/upgrade", post(upgrade_trigger_handler))
|
||||
.at("/api/huskies-binary", get(serve_binary_handler))
|
||||
.at("/api/artifacts/:filename", get(serve_artifact_handler));
|
||||
|
||||
if let Some(wa_ctx) = whatsapp_ctx {
|
||||
@@ -234,9 +233,9 @@ pub fn debug_crdt_handler(req: &poem::Request) -> poem::Response {
|
||||
|
||||
/// `POST /api/upgrade` — trigger a self-update on the running sled.
|
||||
///
|
||||
/// Accepts `{"source_url": "http://gateway:3000/api/huskies-binary"}` and
|
||||
/// Accepts `{"source_url": "http://<gateway>/api/artifacts/<name>"}` and
|
||||
/// spawns the upgrade task in the background, returning 202 immediately.
|
||||
/// The connection will be dropped when `exec()` replaces the process.
|
||||
/// The sled exits after the binary swap; Docker restarts it.
|
||||
#[poem::handler]
|
||||
pub async fn upgrade_trigger_handler(
|
||||
body: poem::web::Json<serde_json::Value>,
|
||||
@@ -270,34 +269,6 @@ pub async fn upgrade_trigger_handler(
|
||||
.body("Upgrade triggered. The sled will re-exec momentarily.")
|
||||
}
|
||||
|
||||
/// `GET /api/huskies-binary` — serve the running binary so peer sleds can download it.
|
||||
///
|
||||
/// Streams `current_exe()` (the binary that is currently running) as an
|
||||
/// `application/octet-stream` download. Returns 500 if the path cannot be
|
||||
/// resolved or read.
|
||||
#[poem::handler]
|
||||
pub async fn serve_binary_handler() -> poem::Response {
|
||||
let exe = match std::env::current_exe() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
return poem::Response::builder()
|
||||
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(format!("Cannot resolve current executable: {e}"));
|
||||
}
|
||||
};
|
||||
|
||||
match tokio::fs::read(&exe).await {
|
||||
Ok(bytes) => poem::Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", "application/octet-stream")
|
||||
.header("Content-Disposition", "attachment; filename=\"huskies\"")
|
||||
.body(bytes),
|
||||
Err(e) => poem::Response::builder()
|
||||
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(format!("Cannot read binary at {}: {e}", exe.display())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical artifact filename for sled binaries on this deployment's platform.
|
||||
///
|
||||
/// Sleds run linux/arm64 under OrbStack on Apple Silicon. When amd64 hosts
|
||||
|
||||
Reference in New Issue
Block a user