diff --git a/server/src/chat/transport/matrix/bot/context.rs b/server/src/chat/transport/matrix/bot/context.rs index 720f57aa..e384fb52 100644 --- a/server/src/chat/transport/matrix/bot/context.rs +++ b/server/src/chat/transport/matrix/bot/context.rs @@ -486,6 +486,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, )]))); let ctx = test_bot_context(services, Some(Arc::clone(&active)), Some(store)); @@ -530,6 +531,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, ); diff --git a/server/src/chat/transport/matrix/bot/messages/on_room_message.rs b/server/src/chat/transport/matrix/bot/messages/on_room_message.rs index c651b221..812e4ac4 100644 --- a/server/src/chat/transport/matrix/bot/messages/on_room_message.rs +++ b/server/src/chat/transport/matrix/bot/messages/on_room_message.rs @@ -586,6 +586,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message( &project, store, ctx.gateway_port, + &ctx.services.project_root, |phase_msg| { let transport = Arc::clone(&transport); let bot_sent = Arc::clone(&bot_sent); @@ -633,6 +634,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message( let response = super::super::super::sled_upgrade::handle_upgrade_all( store, ctx.gateway_port, + &ctx.services.project_root, |phase_msg| { let transport = Arc::clone(&transport); let bot_sent = Arc::clone(&bot_sent); @@ -1189,6 +1191,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, ); @@ -1215,6 +1218,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, )])); diff --git a/server/src/chat/transport/matrix/health.rs b/server/src/chat/transport/matrix/health.rs index ebe35fa1..e812dace 100644 --- a/server/src/chat/transport/matrix/health.rs +++ b/server/src/chat/transport/matrix/health.rs @@ -227,10 +227,15 @@ async fn check_build_hash(project_root: &std::path::Path) -> HealthLine { /// Check each registered sled's `/health` endpoint with a 5-second timeout. /// /// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP -/// 2xx; FAIL when the request times out or returns an error status. Each line -/// includes a `relay=X` indicator derived from the gateway CRDT event log. +/// 2xx *and* its `/identity` probe matches the recorded `expected_node_id`; +/// FAIL when `/health` times out or errors, OR when the identity probe +/// reveals a mismatch or invalid signature — even if `/health` itself +/// reported ok, since a swapped-in container can still serve a healthy +/// `/health` response. Each line includes a `relay=X` indicator derived from +/// the gateway CRDT event log. async fn check_sleds( store: &tokio::sync::RwLock>, + config_dir: &std::path::Path, ) -> Vec { let entries: Vec<(String, Option)> = store .read() @@ -274,26 +279,30 @@ async fn check_sleds( Some(url) => { let health_url = format!("{}/health", url.trim_end_matches('/')); let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await; - match result { - Err(_) => { - HealthLine::fail(subsystem, "timed out", "check container is running") - .with_relay(relay) - } + let health_line = match result { + Err(_) => HealthLine::fail( + subsystem.clone(), + "timed out", + "check container is running", + ), Ok(Err(e)) => HealthLine::fail( - subsystem, + subsystem.clone(), format!("unreachable: {}", short_error(&e.to_string())), "check container is running", - ) - .with_relay(relay), + ), Ok(Ok(resp)) if resp.status().is_success() => { - HealthLine::pass(subsystem).with_relay(relay) + HealthLine::pass(subsystem.clone()) } Ok(Ok(resp)) => HealthLine::fail( - subsystem, + subsystem.clone(), format!("HTTP {}", resp.status().as_u16()), "check container logs", - ) - .with_relay(relay), + ), + }; + + match check_sled_identity(&client, &name, &url, store, config_dir).await { + Some(identity_fail) => identity_fail.with_relay(relay), + None => health_line.with_relay(relay), } } }; @@ -303,6 +312,69 @@ async fn check_sleds( lines } +/// Probe `{url}/identity`, verify the response against the `expected_node_id` +/// recorded for `name`, and capture a first-contact identity. +/// +/// Returns `Some(HealthLine::fail(...))` on a confirmed mismatch or invalid +/// signature — this overrides an otherwise-passing `/health` result. Returns +/// `None` when the identity is unverifiable (unreachable probe or a legacy +/// sled with no signature) or matches, in which case the `/health` result +/// stands on its own. +async fn check_sled_identity( + client: &reqwest::Client, + name: &str, + url: &str, + store: &tokio::sync::RwLock>, + config_dir: &std::path::Path, +) -> Option { + let subsystem = format!("sled:{name}"); + let nonce = crate::node_identity::generate_challenge(); + let response = timeout( + Duration::from_secs(5), + crate::service::gateway::probe_identity(client, url, &nonce), + ) + .await + .ok() + .flatten()?; + + let expected = store + .read() + .await + .get(name) + .and_then(|e| e.expected_node_id.clone()); + + match crate::service::gateway::check_identity(expected.as_deref(), &nonce, &response) { + crate::service::gateway::IdentityCheck::Match + | crate::service::gateway::IdentityCheck::MissingSignature => None, + crate::service::gateway::IdentityCheck::FirstContact { node_id } => { + { + let mut projects = store.write().await; + if let Some(entry) = projects.get_mut(name) { + entry.expected_node_id = Some(node_id); + } + } + let snapshot = store.read().await.clone(); + crate::service::gateway::io::save_config(&snapshot, config_dir).await; + None + } + crate::service::gateway::IdentityCheck::Mismatch { responder_node_id } => { + Some(HealthLine::fail( + subsystem, + format!( + "identity mismatch: expected `{}`, container answered as `{responder_node_id}`", + expected.unwrap_or_default() + ), + "verify the correct container is running — possible swap", + )) + } + crate::service::gateway::IdentityCheck::InvalidSignature => Some(HealthLine::fail( + subsystem, + "identity signature invalid", + "container's identity cannot be verified — check logs", + )), + } +} + /// Check the gateway process: pidfile validity and (on macOS) binary codesign. /// /// PASS when our PID is recorded in the pidfile. On macOS, also verifies that @@ -392,7 +464,7 @@ pub async fn run_health_check(ctx: &BotContext) -> String { if ctx.is_gateway() { lines.push(check_gateway_process()); if let Some(ref store) = ctx.gateway_projects_store { - lines.extend(check_sleds(store).await); + lines.extend(check_sleds(store, &ctx.services.project_root).await); } } @@ -652,6 +724,170 @@ mod tests { assert!(!cmd.description.is_empty()); } + // -- check_sled_identity --------------------------------------------------- + + /// Spawn a one-shot TCP listener that answers a single `GET + /// /identity?nonce=` request with a JSON body signed by `kp` over + /// whatever nonce the caller actually sent. + fn spawn_identity_responder( + listener: tokio::net::TcpListener, + kp: bft_json_crdt::keypair::Ed25519KeyPair, + ) { + tokio::spawn(async move { + if let Ok((mut stream, _)) = listener.accept().await { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = [0u8; 4096]; + let n = stream.read(&mut buf).await.unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..n]); + let nonce = req + .lines() + .next() + .unwrap_or("") + .split("nonce=") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or("") + .to_string(); + let pubkey = crate::node_identity::public_key_hex(&kp); + let sig = crate::node_identity::sign_challenge(&kp, &nonce); + let body = serde_json::json!({ + "node_id": pubkey, + "pubkey": pubkey, + "signature": sig, + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()).await; + } + }); + } + + #[tokio::test] + async fn check_sled_identity_first_contact_captures_and_persists() { + let kp = bft_json_crdt::keypair::make_keypair(); + let pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + crate::service::gateway::config::ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: None, + }, + ); + let store = tokio::sync::RwLock::new(map); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; + assert!( + result.is_none(), + "first contact must not fail the health line: {result:?}" + ); + + let captured = store + .read() + .await + .get("myapp") + .and_then(|e| e.expected_node_id.clone()); + assert_eq!( + captured, + Some(pubkey.clone()), + "expected_node_id should be captured on first contact" + ); + + let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await; + assert!( + toml_content.unwrap_or_default().contains(&pubkey), + "captured node_id should be persisted to projects.toml" + ); + } + + #[tokio::test] + async fn check_sled_identity_match_returns_none() { + let kp = bft_json_crdt::keypair::make_keypair(); + let pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + crate::service::gateway::config::ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: Some(pubkey), + }, + ); + let store = tokio::sync::RwLock::new(map); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; + assert!(result.is_none(), "matching identity must PASS: {result:?}"); + } + + #[tokio::test] + async fn check_sled_identity_mismatch_forces_fail_line() { + let kp = bft_json_crdt::keypair::make_keypair(); + let responder_pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + crate::service::gateway::config::ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: Some("ab".repeat(32)), + }, + ); + let store = tokio::sync::RwLock::new(map); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; + let line = result.expect("identity mismatch must force a FAIL health line"); + assert_eq!( + line.status, + Status::Fail, + "identity mismatch must FAIL even though /health would report ok" + ); + assert!( + line.detail + .as_deref() + .unwrap_or("") + .contains(&responder_pubkey), + "detail should name the node_id that actually answered: {line:?}" + ); + } + // -- Helper --------------------------------------------------------------- /// Build a minimal `BotContext` for testing purposes. diff --git a/server/src/chat/transport/matrix/new_project.rs b/server/src/chat/transport/matrix/new_project.rs index 91a4da70..1c1e50a6 100644 --- a/server/src/chat/transport/matrix/new_project.rs +++ b/server/src/chat/transport/matrix/new_project.rs @@ -803,6 +803,7 @@ async fn handle_adopt_project( auth_token: None, ssh_port: Some(ssh_port), host_path: Some(host_path.to_string_lossy().into_owned()), + expected_node_id: None, }, ); crate::service::gateway::io::save_config(&projects, config_dir).await; @@ -1204,6 +1205,7 @@ pub async fn handle_new_project( auth_token: None, ssh_port: Some(ssh_port), host_path: Some(host_path.to_string_lossy().into_owned()), + expected_node_id: None, }, ); crate::service::gateway::io::save_config(&projects, config_dir).await; diff --git a/server/src/chat/transport/matrix/project_rebuild.rs b/server/src/chat/transport/matrix/project_rebuild.rs index a703e78f..ca150434 100644 --- a/server/src/chat/transport/matrix/project_rebuild.rs +++ b/server/src/chat/transport/matrix/project_rebuild.rs @@ -525,6 +525,7 @@ mod tests { auth_token: None, ssh_port: Some(2201), host_path: None, + expected_node_id: None, }, )]); let dir = tempfile::tempdir().unwrap(); @@ -544,6 +545,7 @@ mod tests { auth_token: None, ssh_port: Some(2201), host_path: Some("/nonexistent/path/xyz123".into()), + expected_node_id: None, }, )]); let dir = tempfile::tempdir().unwrap(); @@ -578,6 +580,7 @@ mod tests { auth_token: Some("tok".into()), ssh_port: Some(2201), host_path: Some(host_dir.path().to_str().unwrap().to_string()), + expected_node_id: None, }, )]); let config_dir = tempfile::tempdir().unwrap(); diff --git a/server/src/chat/transport/matrix/projects.rs b/server/src/chat/transport/matrix/projects.rs index 79deb8e1..fbb658c4 100644 --- a/server/src/chat/transport/matrix/projects.rs +++ b/server/src/chat/transport/matrix/projects.rs @@ -114,6 +114,7 @@ mod tests { auth_token: None, ssh_port: Some(2203), host_path: Some("/home/user/workspace".into()), + expected_node_id: None, }, ); projects.insert( @@ -123,6 +124,7 @@ mod tests { auth_token: Some("tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); @@ -165,6 +167,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: Some("/home/user/adopted".into()), + expected_node_id: None, }, ); projects.insert("builtin".into(), ProjectEntry::with_url("http://b:3002")); diff --git a/server/src/chat/transport/matrix/release.rs b/server/src/chat/transport/matrix/release.rs index bc5d92c7..42bf6969 100644 --- a/server/src/chat/transport/matrix/release.rs +++ b/server/src/chat/transport/matrix/release.rs @@ -200,6 +200,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: host_path.map(String::from), + expected_node_id: None, } } diff --git a/server/src/chat/transport/matrix/sled_upgrade.rs b/server/src/chat/transport/matrix/sled_upgrade.rs index 63a79f7b..ab0732e1 100644 --- a/server/src/chat/transport/matrix/sled_upgrade.rs +++ b/server/src/chat/transport/matrix/sled_upgrade.rs @@ -27,12 +27,69 @@ //! on the sled restart. use crate::service::gateway::config::ProjectEntry; +use crate::service::gateway::{IdentityCheck, check_identity, probe_identity}; use std::collections::BTreeMap; use std::future::Future; +use std::path::Path; use std::sync::{Arc, OnceLock}; use std::time::Duration; use tokio::sync::{Mutex, RwLock}; +/// Probe `/identity` on `sled_url`, verify it against the `expected_node_id` +/// recorded for `project`, and capture a first-contact identity. +/// +/// Fails open (returns `Ok`) when the probe is unreachable or the responder +/// has no signature (legacy sled) — the existing `/health`-based liveness +/// check remains the primary reachability gate. Only a confirmed identity +/// mismatch or an invalid signature is treated as a hard failure, since both +/// mean a verifiably different — or untrustworthy — container answered. +async fn verify_sled_identity( + project: &str, + sled_url: &str, + projects_store: &Arc>>, + config_dir: &Path, + client: &reqwest::Client, +) -> Result<(), String> { + let container_name = format!("huskies-{project}"); + let nonce = crate::node_identity::generate_challenge(); + let Some(response) = probe_identity(client, sled_url, &nonce).await else { + return Ok(()); + }; + + let expected = { + let projects = projects_store.read().await; + projects + .get(project) + .and_then(|e| e.expected_node_id.clone()) + }; + + match check_identity(expected.as_deref(), &nonce, &response) { + IdentityCheck::Match | IdentityCheck::MissingSignature => Ok(()), + IdentityCheck::FirstContact { node_id } => { + { + let mut projects = projects_store.write().await; + if let Some(entry) = projects.get_mut(project) { + entry.expected_node_id = Some(node_id); + } + } + let snapshot = projects_store.read().await.clone(); + crate::service::gateway::io::save_config(&snapshot, config_dir).await; + Ok(()) + } + IdentityCheck::Mismatch { responder_node_id } => Err(format!( + "**identity mismatch** for `{container_name}` at `{sled_url}`: expected node_id \ + `{}`, but the container that answered identified as `{responder_node_id}`. \ + Refusing to proceed — this may not be the sled you expect.", + expected.unwrap_or_default() + )), + IdentityCheck::InvalidSignature => Err(format!( + "**identity verification failed** for `{container_name}` at `{sled_url}`: the \ + `/identity` response signature did not verify. Refusing to proceed — the \ + container's identity cannot be trusted." + )), + } +} + // ── Serial lock ──────────────────────────────────────────────────────────────── static UPGRADE_LOCK: OnceLock> = OnceLock::new(); @@ -157,6 +214,7 @@ fn resolve_artifact_source(gateway_port: Option) -> Result<(String, Option< pub async fn handle_upgrade_all( projects_store: &Arc>>, gateway_port: Option, + config_dir: &Path, send_phase: F, ) -> String where @@ -178,7 +236,7 @@ where let mut results: Vec = Vec::with_capacity(names.len()); for name in &names { - let outcome = handle_sled_upgrade(name, projects_store, gateway_port, |msg| { + let outcome = handle_sled_upgrade(name, projects_store, gateway_port, config_dir, |msg| { send_phase(format!("**{name}** {msg}")) }) .await; @@ -205,6 +263,7 @@ pub async fn handle_sled_upgrade( project: &str, projects_store: &Arc>>, gateway_port: Option, + config_dir: &Path, send_phase: F, ) -> String where @@ -236,17 +295,29 @@ where Err(e) => return e, }; - run_sled_upgrade(project, &sled_url, &source_url, expected_hash, send_phase).await + run_sled_upgrade( + project, + &sled_url, + &source_url, + expected_hash, + projects_store, + config_dir, + send_phase, + ) + .await } /// Run the four-phase upgrade against a sled whose source URL is already /// resolved. Split from [`handle_sled_upgrade`] so tests can drive the wire /// behaviour without a published artifact on the host. +#[allow(clippy::too_many_arguments)] async fn run_sled_upgrade( project: &str, sled_url: &str, source_url: &str, expected_hash: Option, + projects_store: &Arc>>, + config_dir: &Path, send_phase: F, ) -> String where @@ -263,6 +334,13 @@ where .build() .unwrap_or_default(); + // ── Verify identity before triggering ──────────────────────────────────── + if let Err(e) = + verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await + { + return e; + } + // ── Phase 1: downloading ───────────────────────────────────────────────── send_phase("[1/4] downloading\u{2026}".to_string()).await; @@ -316,6 +394,13 @@ where // ── Phase 4: reconnected ───────────────────────────────────────────────── send_phase("[4/4] reconnected to gateway".to_string()).await; + // ── Verify identity during convergence ─────────────────────────────────── + if let Err(e) = + verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await + { + return format!("upgraded and reconnected, but {e}"); + } + // ── Verify convergence ─────────────────────────────────────────────────── match fetch_sled_version(&client, sled_url).await { Some((version, git_hash)) => match expected_hash { @@ -446,6 +531,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, ); map.insert( @@ -455,6 +541,7 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, ); let store = Arc::new(RwLock::new(map)); @@ -470,10 +557,17 @@ mod tests { let store: Arc>> = Arc::new(RwLock::new(BTreeMap::new())); let phases: std::sync::Mutex> = std::sync::Mutex::new(vec![]); - let result = handle_sled_upgrade("nonexistent", &store, Some(3000), |msg| { - phases.lock().unwrap().push(msg); - async {} - }) + let config_dir = tempfile::tempdir().unwrap(); + let result = handle_sled_upgrade( + "nonexistent", + &store, + Some(3000), + config_dir.path(), + |msg| { + phases.lock().unwrap().push(msg); + async {} + }, + ) .await; assert!( result.contains("not found"), @@ -496,10 +590,19 @@ mod tests { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, }, ); let store = Arc::new(RwLock::new(map)); - let result = handle_sled_upgrade("myapp", &store, Some(3000), |_msg| async {}).await; + let config_dir = tempfile::tempdir().unwrap(); + let result = handle_sled_upgrade( + "myapp", + &store, + Some(3000), + config_dir.path(), + |_msg| async {}, + ) + .await; assert!( result.contains("not found"), "project with no URL should say not found: {result}" @@ -518,7 +621,8 @@ mod tests { async fn upgrade_all_empty_store_reports_no_projects() { let store: Arc>> = Arc::new(RwLock::new(BTreeMap::new())); - let msg = handle_upgrade_all(&store, Some(3000), |_msg| async {}).await; + let config_dir = tempfile::tempdir().unwrap(); + let msg = handle_upgrade_all(&store, Some(3000), config_dir.path(), |_msg| async {}).await; assert!( msg.contains("No projects"), "empty store should say no projects: {msg}" @@ -528,11 +632,16 @@ mod tests { #[tokio::test] async fn upgrade_unreachable_sled_reports_failure() { let phases: std::sync::Mutex> = std::sync::Mutex::new(vec![]); + let store: Arc>> = + Arc::new(RwLock::new(BTreeMap::new())); + let config_dir = tempfile::tempdir().unwrap(); let result = run_sled_upgrade( "myapp", "http://127.0.0.1:1", // port 1 is never listening "http://127.0.0.1:1/api/artifacts/huskies-linux-arm64", None, + &store, + config_dir.path(), |msg| { phases.lock().unwrap().push(msg); async {} @@ -590,4 +699,169 @@ mod tests { let ok = wait_for_health(&client, "http://127.0.0.1:1/health", 1).await; assert!(!ok, "should return false when health probe never succeeds"); } + + // ── verify_sled_identity ────────────────────────────────────────────────── + + /// Spawn a one-shot TCP listener that answers a single `GET + /// /identity?nonce=` request with a JSON body signed by `kp` over + /// whatever nonce the caller actually sent — mirrors the real + /// `/identity` handler without needing a full HTTP server. + fn spawn_identity_responder( + listener: tokio::net::TcpListener, + kp: bft_json_crdt::keypair::Ed25519KeyPair, + ) { + tokio::spawn(async move { + if let Ok((mut stream, _)) = listener.accept().await { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buf = [0u8; 4096]; + let n = stream.read(&mut buf).await.unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..n]); + let nonce = req + .lines() + .next() + .unwrap_or("") + .split("nonce=") + .nth(1) + .and_then(|s| s.split_whitespace().next()) + .unwrap_or("") + .to_string(); + let pubkey = crate::node_identity::public_key_hex(&kp); + let sig = crate::node_identity::sign_challenge(&kp, &nonce); + let body = serde_json::json!({ + "node_id": pubkey, + "pubkey": pubkey, + "signature": sig, + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()).await; + } + }); + } + + #[tokio::test] + async fn verify_sled_identity_first_contact_captures_and_persists() { + let kp = bft_json_crdt::keypair::make_keypair(); + let pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: None, + }, + ); + let store = Arc::new(RwLock::new(map)); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await; + assert!(result.is_ok(), "first contact should succeed: {result:?}"); + + let captured = store + .read() + .await + .get("myapp") + .and_then(|e| e.expected_node_id.clone()); + assert_eq!( + captured, + Some(pubkey.clone()), + "expected_node_id should be captured on first contact" + ); + + let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await; + assert!( + toml_content.unwrap_or_default().contains(&pubkey), + "captured node_id should be persisted to projects.toml" + ); + } + + #[tokio::test] + async fn verify_sled_identity_match_succeeds() { + let kp = bft_json_crdt::keypair::make_keypair(); + let pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: Some(pubkey), + }, + ); + let store = Arc::new(RwLock::new(map)); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await; + assert!( + result.is_ok(), + "matching identity should not fail the upgrade: {result:?}" + ); + } + + #[tokio::test] + async fn verify_sled_identity_mismatch_fails_loudly() { + let kp = bft_json_crdt::keypair::make_keypair(); + let responder_pubkey = crate::node_identity::public_key_hex(&kp); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + spawn_identity_responder(listener, kp); + + let sled_url = format!("http://127.0.0.1:{port}"); + let mut map = BTreeMap::new(); + map.insert( + "myapp".to_string(), + ProjectEntry { + url: Some(sled_url.clone()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: Some("ab".repeat(32)), + }, + ); + let store = Arc::new(RwLock::new(map)); + let config_dir = tempfile::tempdir().unwrap(); + let client = reqwest::Client::new(); + + let result = + verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await; + let err = result.expect_err("mismatched identity must fail loudly"); + assert!( + err.contains("identity mismatch"), + "error should name the failure class: {err}" + ); + assert!( + err.contains("huskies-myapp"), + "error should name the container: {err}" + ); + assert!( + err.contains(&responder_pubkey), + "error should name the node_id that actually answered: {err}" + ); + } } diff --git a/server/src/gateway/tests.rs b/server/src/gateway/tests.rs index e2a0d19e..6e505877 100644 --- a/server/src/gateway/tests.rs +++ b/server/src/gateway/tests.rs @@ -1177,6 +1177,7 @@ async fn ws_only_sled_handles_tools_list_and_tools_call() { auth_token: Some("secret".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); let config = GatewayConfig { @@ -1248,6 +1249,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() { auth_token: Some("alpha-tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); projects.insert( @@ -1257,6 +1259,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() { auth_token: Some("beta-tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); let config = GatewayConfig { diff --git a/server/src/http/identity.rs b/server/src/http/identity.rs index c35c99e1..8cf4b59c 100644 --- a/server/src/http/identity.rs +++ b/server/src/http/identity.rs @@ -3,10 +3,25 @@ //! `GET /identity` returns the node's ID and public key as JSON. No //! authentication is required; only the public half of the keypair is //! disclosed. +//! +//! `GET /identity?nonce=` (story 1173) additionally signs the nonce with +//! the node's CRDT signing key ([`crate::crdt_state::sign_challenge`]) so a +//! caller can verify the response actually came from the sled it expects, +//! not just whichever container answered on that URL. The signing call +//! acquires the CRDT state mutex exactly once (lock → sign → unlock) with no +//! nested CRDT calls while held, avoiding the bug-1170 self-deadlock class. use poem::handler; -use poem::web::Json; -use serde::Serialize; +use poem::web::{Json, Query}; +use serde::{Deserialize, Serialize}; + +/// Query parameters accepted by `GET /identity`. +#[derive(Deserialize)] +pub struct IdentityQuery { + /// Optional challenge nonce to sign with the node's CRDT signing key. + #[serde(default)] + pub nonce: Option, +} /// JSON response body for `GET /identity`. #[derive(Serialize)] @@ -15,22 +30,46 @@ pub struct IdentityResponse { pub node_id: String, /// Lowercase hex-encoding of the 32-byte Ed25519 public key. pub pubkey: String, + /// Ed25519 signature (hex) over `nonce`, made with the CRDT signing key. + /// `None` when no `nonce` was supplied, or before the CRDT layer has + /// initialised (in which case `node_id`/`pubkey` fall back to the + /// file-based node identity). + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option, } -/// `GET /identity` — return this node's Ed25519 public key. +/// `GET /identity` — return this node's Ed25519 public key, optionally +/// signing a caller-supplied nonce with the CRDT signing key. /// -/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>"}`. -/// No authentication required; the private key is never exposed. +/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>", "signature": "<128-hex>"}` +/// when a `nonce` query parameter is given and the CRDT layer is +/// initialised. Otherwise omits `signature` and falls back to the +/// file-based node identity (legacy shape). No authentication required; +/// private keys are never exposed. #[handler] -pub fn identity_handler() -> Json { +pub fn identity_handler( + Query(IdentityQuery { nonce }): Query, +) -> Json { + if let Some(ref nonce) = nonce + && let Some((pubkey_hex, signature_hex)) = crate::crdt_state::sign_challenge(nonce) + { + return Json(IdentityResponse { + node_id: pubkey_hex.clone(), + pubkey: pubkey_hex, + signature: Some(signature_hex), + }); + } + match crate::node_identity::get_identity() { Some(id) => Json(IdentityResponse { node_id: id.node_id.clone(), pubkey: id.pubkey_hex.clone(), + signature: None, }), None => Json(IdentityResponse { node_id: "uninitialized".to_string(), pubkey: "uninitialized".to_string(), + signature: None, }), } } @@ -58,5 +97,45 @@ mod tests { assert_eq!(node_id.len(), 64); assert!(node_id.chars().all(|c| c.is_ascii_hexdigit())); assert_eq!(node_id, pubkey); + assert!(body.get("signature").is_none()); + } + + #[tokio::test] + async fn identity_endpoint_signs_nonce_with_crdt_key_when_initialised() { + crate::crdt_state::init_for_test(); + + let app = Route::new().at("/identity", get(identity_handler)); + let cli = TestClient::new(app); + let resp = cli + .get("/identity") + .query("nonce", &"deadbeef") + .send() + .await; + resp.assert_status_is_ok(); + + let body: serde_json::Value = resp.json().await.value().deserialize(); + let node_id = body["node_id"].as_str().unwrap(); + let pubkey = body["pubkey"].as_str().unwrap(); + let signature = body["signature"].as_str().unwrap(); + assert_eq!(node_id, pubkey); + assert!( + crate::node_identity::verify_message_strict(pubkey, b"deadbeef", signature), + "signature must verify against the returned pubkey and nonce" + ); + // The signed node_id must be the CRDT node id, not the file-based one. + assert_eq!(node_id, crate::crdt_state::our_node_id().unwrap()); + } + + #[tokio::test] + async fn identity_endpoint_without_nonce_omits_signature_even_when_crdt_initialised() { + crate::crdt_state::init_for_test(); + + let app = Route::new().at("/identity", get(identity_handler)); + let cli = TestClient::new(app); + let resp = cli.get("/identity").send().await; + resp.assert_status_is_ok(); + + let body: serde_json::Value = resp.json().await.value().deserialize(); + assert!(body.get("signature").is_none()); } } diff --git a/server/src/service/gateway/config.rs b/server/src/service/gateway/config.rs index 2275630f..4d7c2d6b 100644 --- a/server/src/service/gateway/config.rs +++ b/server/src/service/gateway/config.rs @@ -40,6 +40,14 @@ pub struct ProjectEntry { /// commands can route to the correct directory without re-deriving it. #[serde(default, skip_serializing_if = "Option::is_none")] pub host_path: Option, + /// Ed25519 node ID (hex pubkey) this sled is expected to answer as. + /// + /// Captured automatically from the first successful `/identity` probe + /// (story 1173) when absent, then checked on every subsequent upgrade and + /// health probe so a container swapped out from under the gateway is + /// detected even when its `/health` endpoint still reports ok. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected_node_id: Option, } impl ProjectEntry { @@ -52,6 +60,7 @@ impl ProjectEntry { auth_token: None, ssh_port: None, host_path: None, + expected_node_id: None, } } @@ -223,6 +232,7 @@ auth_token = "secret" auth_token: Some("secret".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); let config = GatewayConfig { @@ -258,6 +268,7 @@ auth_token = "secret" auth_token: Some("tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); assert_eq!(validate_project_exists(&projects, "ws").unwrap(), ""); @@ -278,6 +289,7 @@ auth_token = "secret" auth_token: Some("tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }; assert!(!e.has_url()); } @@ -321,6 +333,7 @@ auth_token = "secret" auth_token: Some("mysecret".into()), ssh_port: None, host_path: None, + expected_node_id: None, }; let mut projects = BTreeMap::new(); projects.insert("myproj".into(), entry); @@ -347,6 +360,7 @@ auth_token = "secret" auth_token: None, ssh_port: Some(2201), host_path: None, + expected_node_id: None, }; let mut projects = BTreeMap::new(); projects.insert("myproj".into(), entry); @@ -364,6 +378,43 @@ auth_token = "secret" ); } + #[test] + fn expected_node_id_roundtrips_and_is_omitted_when_none() { + let with_id = ProjectEntry { + url: Some("http://127.0.0.1:3101".into()), + auth_token: None, + ssh_port: None, + host_path: None, + expected_node_id: Some("ab".repeat(32)), + }; + let mut projects = BTreeMap::new(); + projects.insert("p".into(), with_id); + let config = GatewayConfig { + projects, + sled_tokens: BTreeMap::new(), + }; + let toml_str = toml::to_string_pretty(&config).unwrap(); + assert!(toml_str.contains("expected_node_id")); + let parsed: GatewayConfig = toml::from_str(&toml_str).unwrap(); + assert_eq!( + parsed.projects["p"].expected_node_id.as_deref(), + Some("ab".repeat(32).as_str()) + ); + + let without_id = ProjectEntry::with_url("http://127.0.0.1:3101"); + let mut projects2 = BTreeMap::new(); + projects2.insert("p".into(), without_id); + let config2 = GatewayConfig { + projects: projects2, + sled_tokens: BTreeMap::new(), + }; + let toml_str2 = toml::to_string_pretty(&config2).unwrap(); + assert!( + !toml_str2.contains("expected_node_id"), + "expected_node_id should be omitted when None: {toml_str2}" + ); + } + #[test] fn ssh_port_none_is_omitted_from_toml() { let entry = ProjectEntry::with_url("http://127.0.0.1:3101"); diff --git a/server/src/service/gateway/identity.rs b/server/src/service/gateway/identity.rs new file mode 100644 index 00000000..2824c2b5 --- /dev/null +++ b/server/src/service/gateway/identity.rs @@ -0,0 +1,175 @@ +//! Pure identity-probe verification — no I/O (story 1173). +//! +//! [`check_identity`] compares a sled's `/identity` probe response against the +//! `expected_node_id` recorded for that project and reports one of: a first +//! contact (no expected id captured yet), a match, a mismatch, an invalid +//! signature, or a missing signature (a legacy sled predating this story). +//! Callers ([`super::super::super::chat::transport::matrix::sled_upgrade`] and +//! `health`) own all I/O — issuing the HTTP probe and persisting a captured +//! `expected_node_id` back to `projects.toml`. + +use serde::Deserialize; + +/// Deserialized body of a `GET /identity?nonce=...` response. +#[derive(Debug, Clone, Deserialize)] +pub struct IdentityProbeResponse { + /// The responder's self-reported node ID (hex Ed25519 pubkey). + pub node_id: String, + /// The responder's self-reported Ed25519 public key (hex). Equal to + /// `node_id` on every server built since this field was introduced. + pub pubkey: String, + /// Ed25519 signature (hex) over the probe nonce, made with the CRDT + /// signing key. `None` when the responder predates story 1173 (a legacy + /// sled) — the `/identity` endpoint did not sign nonces before then. + #[serde(default)] + pub signature: Option, +} + +/// Outcome of comparing an [`IdentityProbeResponse`] against an +/// `expected_node_id`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IdentityCheck { + /// No `expected_node_id` was recorded yet; the signature verified, so + /// `node_id` should be captured as the expected identity going forward. + FirstContact { + /// The node ID to capture. + node_id: String, + }, + /// The signature verified and the responder's node ID matches the + /// expected one. + Match, + /// The signature verified, but the responder's node ID does not match + /// the expected one — a different container answered. + Mismatch { + /// The verified node ID that actually answered. + responder_node_id: String, + }, + /// The response included a signature, but it did not verify against the + /// claimed pubkey/nonce — the responder's identity cannot be trusted. + InvalidSignature, + /// The response had no `signature` field — a legacy sled running a + /// pre-story-1173 binary. Identity cannot be verified either way. + MissingSignature, +} + +/// Verify `response` against `nonce` and compare the verified node ID to +/// `expected_node_id`. +/// +/// A signature verifies when [`crate::node_identity::verify_message_strict`] +/// confirms it was produced by the private key matching `response.pubkey` +/// over `nonce`'s UTF-8 bytes — mirroring how `/identity` signs on the server +/// side via `crdt_state::sign_challenge`. +pub fn check_identity( + expected_node_id: Option<&str>, + nonce: &str, + response: &IdentityProbeResponse, +) -> IdentityCheck { + let Some(signature) = response.signature.as_deref() else { + return IdentityCheck::MissingSignature; + }; + + if !crate::node_identity::verify_message_strict(&response.pubkey, nonce.as_bytes(), signature) { + return IdentityCheck::InvalidSignature; + } + + match expected_node_id { + None => IdentityCheck::FirstContact { + node_id: response.node_id.clone(), + }, + Some(expected) if expected == response.node_id => IdentityCheck::Match, + Some(_) => IdentityCheck::Mismatch { + responder_node_id: response.node_id.clone(), + }, + } +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use bft_json_crdt::keypair::make_keypair; + + fn signed_response(nonce: &str) -> (String, IdentityProbeResponse) { + let kp = make_keypair(); + let pubkey = crate::node_identity::public_key_hex(&kp); + let sig = crate::node_identity::sign_challenge(&kp, nonce); + ( + pubkey.clone(), + IdentityProbeResponse { + node_id: pubkey.clone(), + pubkey, + signature: Some(sig), + }, + ) + } + + #[test] + fn first_contact_when_no_expected_id() { + let nonce = "nonce-1"; + let (node_id, resp) = signed_response(nonce); + let result = check_identity(None, nonce, &resp); + assert_eq!(result, IdentityCheck::FirstContact { node_id }); + } + + #[test] + fn match_when_expected_id_equals_responder() { + let nonce = "nonce-2"; + let (node_id, resp) = signed_response(nonce); + let result = check_identity(Some(&node_id), nonce, &resp); + assert_eq!(result, IdentityCheck::Match); + } + + #[test] + fn mismatch_when_expected_id_differs() { + let nonce = "nonce-3"; + let (node_id, resp) = signed_response(nonce); + let result = check_identity(Some("some-other-node-id"), nonce, &resp); + assert_eq!( + result, + IdentityCheck::Mismatch { + responder_node_id: node_id + } + ); + } + + #[test] + fn invalid_signature_wrong_nonce() { + let nonce = "nonce-4"; + let (_node_id, resp) = signed_response(nonce); + // Verify against a different nonce than the one that was signed. + let result = check_identity(None, "different-nonce", &resp); + assert_eq!(result, IdentityCheck::InvalidSignature); + } + + #[test] + fn invalid_signature_wrong_key() { + let nonce = "nonce-5"; + let (_node_id, mut resp) = signed_response(nonce); + // Claim a different pubkey than the one that actually signed. + let other_kp = make_keypair(); + let other_pubkey = crate::node_identity::public_key_hex(&other_kp); + resp.node_id = other_pubkey.clone(); + resp.pubkey = other_pubkey; + let result = check_identity(None, nonce, &resp); + assert_eq!(result, IdentityCheck::InvalidSignature); + } + + #[test] + fn missing_signature_is_legacy_sled() { + let resp = IdentityProbeResponse { + node_id: "abc123".to_string(), + pubkey: "abc123".to_string(), + signature: None, + }; + let result = check_identity(Some("abc123"), "nonce-6", &resp); + assert_eq!(result, IdentityCheck::MissingSignature); + } + + #[test] + fn missing_signature_field_deserializes_from_json_without_it() { + let json = r#"{"node_id":"abc","pubkey":"abc"}"#; + let resp: IdentityProbeResponse = serde_json::from_str(json).unwrap(); + assert!(resp.signature.is_none()); + } +} diff --git a/server/src/service/gateway/io.rs b/server/src/service/gateway/io.rs index bbaa12dc..4dc60d71 100644 --- a/server/src/service/gateway/io.rs +++ b/server/src/service/gateway/io.rs @@ -94,6 +94,32 @@ pub fn write_bot_config(config_dir: &Path, content: &str) -> Result<(), String> std::fs::write(&path, content).map_err(|e| format!("cannot write bot.toml: {e}")) } +// ── Identity probe I/O ─────────────────────────────────────────────────────── + +/// `GET {sled_url}/identity?nonce=` and parse the JSON body. +/// +/// Returns `None` when the sled is unreachable or the response body doesn't +/// parse as [`super::identity::IdentityProbeResponse`] — callers treat that +/// the same as an unverifiable identity (distinct from a legacy sled, which +/// responds but omits the `signature` field). +pub async fn probe_identity( + client: &Client, + sled_url: &str, + nonce: &str, +) -> Option { + // `nonce` is always a hex string (see `node_identity::generate_challenge`), + // so no percent-encoding is needed for safe inclusion in the query string. + let url = format!("{}/identity?nonce={nonce}", sled_url.trim_end_matches('/')); + client + .get(&url) + .send() + .await + .ok()? + .json::() + .await + .ok() +} + // ── MCP proxy I/O ─────────────────────────────────────────────────────────── /// Proxy a raw MCP request body to the given project URL. diff --git a/server/src/service/gateway/mod.rs b/server/src/service/gateway/mod.rs index 69f7486c..38ced712 100644 --- a/server/src/service/gateway/mod.rs +++ b/server/src/service/gateway/mod.rs @@ -11,14 +11,17 @@ pub mod aggregation; /// Gateway configuration types and TOML parsing. pub mod config; +/// Pure identity-probe verification (match/mismatch/first-contact) — no I/O. +pub mod identity; pub(crate) mod io; /// Notification event polling for gateway-level broadcasts. pub mod polling; pub use aggregation::format_aggregate_status_compact; pub use config::{GatewayConfig, ProjectEntry}; +pub use identity::{IdentityCheck, check_identity}; pub use io::{ - fetch_all_project_pipeline_statuses, spawn_gateway_broadcaster_forwarder, + fetch_all_project_pipeline_statuses, probe_identity, spawn_gateway_broadcaster_forwarder, spawn_gateway_notification_poller, }; @@ -781,6 +784,7 @@ mod tests { auth_token: Some("tok".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); let config = GatewayConfig { @@ -921,6 +925,7 @@ mod tests { auth_token: Some("secret-token".into()), ssh_port: None, host_path: None, + expected_node_id: None, }, ); let config = GatewayConfig {