huskies: merge 1173 story Identity-aware fleet checks: cryptographic node identity in upgrade and health probes

This commit is contained in:
Huskies Agent
2026-07-16 13:26:03 +00:00
parent 489c415fd9
commit 0ac68afa4c
14 changed files with 894 additions and 30 deletions
@@ -486,6 +486,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
)])));
let ctx = test_bot_context(services, Some(Arc::clone(&active)), Some(store));
@@ -530,6 +531,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
@@ -586,6 +586,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
&project,
store,
ctx.gateway_port,
&ctx.services.project_root,
|phase_msg| {
let transport = Arc::clone(&transport);
let bot_sent = Arc::clone(&bot_sent);
@@ -633,6 +634,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
let response = super::super::super::sled_upgrade::handle_upgrade_all(
store,
ctx.gateway_port,
&ctx.services.project_root,
|phase_msg| {
let transport = Arc::clone(&transport);
let bot_sent = Arc::clone(&bot_sent);
@@ -1189,6 +1191,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
@@ -1215,6 +1218,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
)]));
+251 -15
View File
@@ -227,10 +227,15 @@ async fn check_build_hash(project_root: &std::path::Path) -> HealthLine {
/// Check each registered sled's `/health` endpoint with a 5-second timeout.
///
/// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP
/// 2xx; FAIL when the request times out or returns an error status. Each line
/// includes a `relay=X` indicator derived from the gateway CRDT event log.
/// 2xx *and* its `/identity` probe matches the recorded `expected_node_id`;
/// FAIL when `/health` times out or errors, OR when the identity probe
/// reveals a mismatch or invalid signature — even if `/health` itself
/// reported ok, since a swapped-in container can still serve a healthy
/// `/health` response. Each line includes a `relay=X` indicator derived from
/// the gateway CRDT event log.
async fn check_sleds(
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
config_dir: &std::path::Path,
) -> Vec<HealthLine> {
let entries: Vec<(String, Option<String>)> = store
.read()
@@ -274,26 +279,30 @@ async fn check_sleds(
Some(url) => {
let health_url = format!("{}/health", url.trim_end_matches('/'));
let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await;
match result {
Err(_) => {
HealthLine::fail(subsystem, "timed out", "check container is running")
.with_relay(relay)
}
let health_line = match result {
Err(_) => HealthLine::fail(
subsystem.clone(),
"timed out",
"check container is running",
),
Ok(Err(e)) => HealthLine::fail(
subsystem,
subsystem.clone(),
format!("unreachable: {}", short_error(&e.to_string())),
"check container is running",
)
.with_relay(relay),
),
Ok(Ok(resp)) if resp.status().is_success() => {
HealthLine::pass(subsystem).with_relay(relay)
HealthLine::pass(subsystem.clone())
}
Ok(Ok(resp)) => HealthLine::fail(
subsystem,
subsystem.clone(),
format!("HTTP {}", resp.status().as_u16()),
"check container logs",
)
.with_relay(relay),
),
};
match check_sled_identity(&client, &name, &url, store, config_dir).await {
Some(identity_fail) => identity_fail.with_relay(relay),
None => health_line.with_relay(relay),
}
}
};
@@ -303,6 +312,69 @@ async fn check_sleds(
lines
}
/// Probe `{url}/identity`, verify the response against the `expected_node_id`
/// recorded for `name`, and capture a first-contact identity.
///
/// Returns `Some(HealthLine::fail(...))` on a confirmed mismatch or invalid
/// signature — this overrides an otherwise-passing `/health` result. Returns
/// `None` when the identity is unverifiable (unreachable probe or a legacy
/// sled with no signature) or matches, in which case the `/health` result
/// stands on its own.
async fn check_sled_identity(
client: &reqwest::Client,
name: &str,
url: &str,
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
config_dir: &std::path::Path,
) -> Option<HealthLine> {
let subsystem = format!("sled:{name}");
let nonce = crate::node_identity::generate_challenge();
let response = timeout(
Duration::from_secs(5),
crate::service::gateway::probe_identity(client, url, &nonce),
)
.await
.ok()
.flatten()?;
let expected = store
.read()
.await
.get(name)
.and_then(|e| e.expected_node_id.clone());
match crate::service::gateway::check_identity(expected.as_deref(), &nonce, &response) {
crate::service::gateway::IdentityCheck::Match
| crate::service::gateway::IdentityCheck::MissingSignature => None,
crate::service::gateway::IdentityCheck::FirstContact { node_id } => {
{
let mut projects = store.write().await;
if let Some(entry) = projects.get_mut(name) {
entry.expected_node_id = Some(node_id);
}
}
let snapshot = store.read().await.clone();
crate::service::gateway::io::save_config(&snapshot, config_dir).await;
None
}
crate::service::gateway::IdentityCheck::Mismatch { responder_node_id } => {
Some(HealthLine::fail(
subsystem,
format!(
"identity mismatch: expected `{}`, container answered as `{responder_node_id}`",
expected.unwrap_or_default()
),
"verify the correct container is running — possible swap",
))
}
crate::service::gateway::IdentityCheck::InvalidSignature => Some(HealthLine::fail(
subsystem,
"identity signature invalid",
"container's identity cannot be verified — check logs",
)),
}
}
/// Check the gateway process: pidfile validity and (on macOS) binary codesign.
///
/// PASS when our PID is recorded in the pidfile. On macOS, also verifies that
@@ -392,7 +464,7 @@ pub async fn run_health_check(ctx: &BotContext) -> String {
if ctx.is_gateway() {
lines.push(check_gateway_process());
if let Some(ref store) = ctx.gateway_projects_store {
lines.extend(check_sleds(store).await);
lines.extend(check_sleds(store, &ctx.services.project_root).await);
}
}
@@ -652,6 +724,170 @@ mod tests {
assert!(!cmd.description.is_empty());
}
// -- check_sled_identity ---------------------------------------------------
/// Spawn a one-shot TCP listener that answers a single `GET
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
/// whatever nonce the caller actually sent.
fn spawn_identity_responder(
listener: tokio::net::TcpListener,
kp: bft_json_crdt::keypair::Ed25519KeyPair,
) {
tokio::spawn(async move {
if let Ok((mut stream, _)) = listener.accept().await {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 4096];
let n = stream.read(&mut buf).await.unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]);
let nonce = req
.lines()
.next()
.unwrap_or("")
.split("nonce=")
.nth(1)
.and_then(|s| s.split_whitespace().next())
.unwrap_or("")
.to_string();
let pubkey = crate::node_identity::public_key_hex(&kp);
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
let body = serde_json::json!({
"node_id": pubkey,
"pubkey": pubkey,
"signature": sig,
})
.to_string();
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(response.as_bytes()).await;
}
});
}
#[tokio::test]
async fn check_sled_identity_first_contact_captures_and_persists() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
assert!(
result.is_none(),
"first contact must not fail the health line: {result:?}"
);
let captured = store
.read()
.await
.get("myapp")
.and_then(|e| e.expected_node_id.clone());
assert_eq!(
captured,
Some(pubkey.clone()),
"expected_node_id should be captured on first contact"
);
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
assert!(
toml_content.unwrap_or_default().contains(&pubkey),
"captured node_id should be persisted to projects.toml"
);
}
#[tokio::test]
async fn check_sled_identity_match_returns_none() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some(pubkey),
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
assert!(result.is_none(), "matching identity must PASS: {result:?}");
}
#[tokio::test]
async fn check_sled_identity_mismatch_forces_fail_line() {
let kp = bft_json_crdt::keypair::make_keypair();
let responder_pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some("ab".repeat(32)),
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
let line = result.expect("identity mismatch must force a FAIL health line");
assert_eq!(
line.status,
Status::Fail,
"identity mismatch must FAIL even though /health would report ok"
);
assert!(
line.detail
.as_deref()
.unwrap_or("")
.contains(&responder_pubkey),
"detail should name the node_id that actually answered: {line:?}"
);
}
// -- Helper ---------------------------------------------------------------
/// Build a minimal `BotContext` for testing purposes.
@@ -803,6 +803,7 @@ async fn handle_adopt_project(
auth_token: None,
ssh_port: Some(ssh_port),
host_path: Some(host_path.to_string_lossy().into_owned()),
expected_node_id: None,
},
);
crate::service::gateway::io::save_config(&projects, config_dir).await;
@@ -1204,6 +1205,7 @@ pub async fn handle_new_project(
auth_token: None,
ssh_port: Some(ssh_port),
host_path: Some(host_path.to_string_lossy().into_owned()),
expected_node_id: None,
},
);
crate::service::gateway::io::save_config(&projects, config_dir).await;
@@ -525,6 +525,7 @@ mod tests {
auth_token: None,
ssh_port: Some(2201),
host_path: None,
expected_node_id: None,
},
)]);
let dir = tempfile::tempdir().unwrap();
@@ -544,6 +545,7 @@ mod tests {
auth_token: None,
ssh_port: Some(2201),
host_path: Some("/nonexistent/path/xyz123".into()),
expected_node_id: None,
},
)]);
let dir = tempfile::tempdir().unwrap();
@@ -578,6 +580,7 @@ mod tests {
auth_token: Some("tok".into()),
ssh_port: Some(2201),
host_path: Some(host_dir.path().to_str().unwrap().to_string()),
expected_node_id: None,
},
)]);
let config_dir = tempfile::tempdir().unwrap();
@@ -114,6 +114,7 @@ mod tests {
auth_token: None,
ssh_port: Some(2203),
host_path: Some("/home/user/workspace".into()),
expected_node_id: None,
},
);
projects.insert(
@@ -123,6 +124,7 @@ mod tests {
auth_token: Some("tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
@@ -165,6 +167,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: Some("/home/user/adopted".into()),
expected_node_id: None,
},
);
projects.insert("builtin".into(), ProjectEntry::with_url("http://b:3002"));
@@ -200,6 +200,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: host_path.map(String::from),
expected_node_id: None,
}
}
@@ -27,12 +27,69 @@
//! on the sled restart.
use crate::service::gateway::config::ProjectEntry;
use crate::service::gateway::{IdentityCheck, check_identity, probe_identity};
use std::collections::BTreeMap;
use std::future::Future;
use std::path::Path;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::sync::{Mutex, RwLock};
/// Probe `/identity` on `sled_url`, verify it against the `expected_node_id`
/// recorded for `project`, and capture a first-contact identity.
///
/// Fails open (returns `Ok`) when the probe is unreachable or the responder
/// has no signature (legacy sled) — the existing `/health`-based liveness
/// check remains the primary reachability gate. Only a confirmed identity
/// mismatch or an invalid signature is treated as a hard failure, since both
/// mean a verifiably different — or untrustworthy — container answered.
async fn verify_sled_identity(
project: &str,
sled_url: &str,
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
config_dir: &Path,
client: &reqwest::Client,
) -> Result<(), String> {
let container_name = format!("huskies-{project}");
let nonce = crate::node_identity::generate_challenge();
let Some(response) = probe_identity(client, sled_url, &nonce).await else {
return Ok(());
};
let expected = {
let projects = projects_store.read().await;
projects
.get(project)
.and_then(|e| e.expected_node_id.clone())
};
match check_identity(expected.as_deref(), &nonce, &response) {
IdentityCheck::Match | IdentityCheck::MissingSignature => Ok(()),
IdentityCheck::FirstContact { node_id } => {
{
let mut projects = projects_store.write().await;
if let Some(entry) = projects.get_mut(project) {
entry.expected_node_id = Some(node_id);
}
}
let snapshot = projects_store.read().await.clone();
crate::service::gateway::io::save_config(&snapshot, config_dir).await;
Ok(())
}
IdentityCheck::Mismatch { responder_node_id } => Err(format!(
"**identity mismatch** for `{container_name}` at `{sled_url}`: expected node_id \
`{}`, but the container that answered identified as `{responder_node_id}`. \
Refusing to proceed — this may not be the sled you expect.",
expected.unwrap_or_default()
)),
IdentityCheck::InvalidSignature => Err(format!(
"**identity verification failed** for `{container_name}` at `{sled_url}`: the \
`/identity` response signature did not verify. Refusing to proceed — the \
container's identity cannot be trusted."
)),
}
}
// ── Serial lock ────────────────────────────────────────────────────────────────
static UPGRADE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
@@ -157,6 +214,7 @@ fn resolve_artifact_source(gateway_port: Option<u16>) -> Result<(String, Option<
pub async fn handle_upgrade_all<F, Fut>(
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
gateway_port: Option<u16>,
config_dir: &Path,
send_phase: F,
) -> String
where
@@ -178,7 +236,7 @@ where
let mut results: Vec<String> = Vec::with_capacity(names.len());
for name in &names {
let outcome = handle_sled_upgrade(name, projects_store, gateway_port, |msg| {
let outcome = handle_sled_upgrade(name, projects_store, gateway_port, config_dir, |msg| {
send_phase(format!("**{name}** {msg}"))
})
.await;
@@ -205,6 +263,7 @@ pub async fn handle_sled_upgrade<F, Fut>(
project: &str,
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
gateway_port: Option<u16>,
config_dir: &Path,
send_phase: F,
) -> String
where
@@ -236,17 +295,29 @@ where
Err(e) => return e,
};
run_sled_upgrade(project, &sled_url, &source_url, expected_hash, send_phase).await
run_sled_upgrade(
project,
&sled_url,
&source_url,
expected_hash,
projects_store,
config_dir,
send_phase,
)
.await
}
/// Run the four-phase upgrade against a sled whose source URL is already
/// resolved. Split from [`handle_sled_upgrade`] so tests can drive the wire
/// behaviour without a published artifact on the host.
#[allow(clippy::too_many_arguments)]
async fn run_sled_upgrade<F, Fut>(
project: &str,
sled_url: &str,
source_url: &str,
expected_hash: Option<String>,
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
config_dir: &Path,
send_phase: F,
) -> String
where
@@ -263,6 +334,13 @@ where
.build()
.unwrap_or_default();
// ── Verify identity before triggering ────────────────────────────────────
if let Err(e) =
verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await
{
return e;
}
// ── Phase 1: downloading ─────────────────────────────────────────────────
send_phase("[1/4] downloading\u{2026}".to_string()).await;
@@ -316,6 +394,13 @@ where
// ── Phase 4: reconnected ─────────────────────────────────────────────────
send_phase("[4/4] reconnected to gateway".to_string()).await;
// ── Verify identity during convergence ───────────────────────────────────
if let Err(e) =
verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await
{
return format!("upgraded and reconnected, but {e}");
}
// ── Verify convergence ───────────────────────────────────────────────────
match fetch_sled_version(&client, sled_url).await {
Some((version, git_hash)) => match expected_hash {
@@ -446,6 +531,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
map.insert(
@@ -455,6 +541,7 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let store = Arc::new(RwLock::new(map));
@@ -470,10 +557,17 @@ mod tests {
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
Arc::new(RwLock::new(BTreeMap::new()));
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
let result = handle_sled_upgrade("nonexistent", &store, Some(3000), |msg| {
let config_dir = tempfile::tempdir().unwrap();
let result = handle_sled_upgrade(
"nonexistent",
&store,
Some(3000),
config_dir.path(),
|msg| {
phases.lock().unwrap().push(msg);
async {}
})
},
)
.await;
assert!(
result.contains("not found"),
@@ -496,10 +590,19 @@ mod tests {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let store = Arc::new(RwLock::new(map));
let result = handle_sled_upgrade("myapp", &store, Some(3000), |_msg| async {}).await;
let config_dir = tempfile::tempdir().unwrap();
let result = handle_sled_upgrade(
"myapp",
&store,
Some(3000),
config_dir.path(),
|_msg| async {},
)
.await;
assert!(
result.contains("not found"),
"project with no URL should say not found: {result}"
@@ -518,7 +621,8 @@ mod tests {
async fn upgrade_all_empty_store_reports_no_projects() {
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
Arc::new(RwLock::new(BTreeMap::new()));
let msg = handle_upgrade_all(&store, Some(3000), |_msg| async {}).await;
let config_dir = tempfile::tempdir().unwrap();
let msg = handle_upgrade_all(&store, Some(3000), config_dir.path(), |_msg| async {}).await;
assert!(
msg.contains("No projects"),
"empty store should say no projects: {msg}"
@@ -528,11 +632,16 @@ mod tests {
#[tokio::test]
async fn upgrade_unreachable_sled_reports_failure() {
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
Arc::new(RwLock::new(BTreeMap::new()));
let config_dir = tempfile::tempdir().unwrap();
let result = run_sled_upgrade(
"myapp",
"http://127.0.0.1:1", // port 1 is never listening
"http://127.0.0.1:1/api/artifacts/huskies-linux-arm64",
None,
&store,
config_dir.path(),
|msg| {
phases.lock().unwrap().push(msg);
async {}
@@ -590,4 +699,169 @@ mod tests {
let ok = wait_for_health(&client, "http://127.0.0.1:1/health", 1).await;
assert!(!ok, "should return false when health probe never succeeds");
}
// ── verify_sled_identity ──────────────────────────────────────────────────
/// Spawn a one-shot TCP listener that answers a single `GET
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
/// whatever nonce the caller actually sent — mirrors the real
/// `/identity` handler without needing a full HTTP server.
fn spawn_identity_responder(
listener: tokio::net::TcpListener,
kp: bft_json_crdt::keypair::Ed25519KeyPair,
) {
tokio::spawn(async move {
if let Ok((mut stream, _)) = listener.accept().await {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 4096];
let n = stream.read(&mut buf).await.unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]);
let nonce = req
.lines()
.next()
.unwrap_or("")
.split("nonce=")
.nth(1)
.and_then(|s| s.split_whitespace().next())
.unwrap_or("")
.to_string();
let pubkey = crate::node_identity::public_key_hex(&kp);
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
let body = serde_json::json!({
"node_id": pubkey,
"pubkey": pubkey,
"signature": sig,
})
.to_string();
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(response.as_bytes()).await;
}
});
}
#[tokio::test]
async fn verify_sled_identity_first_contact_captures_and_persists() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let store = Arc::new(RwLock::new(map));
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
assert!(result.is_ok(), "first contact should succeed: {result:?}");
let captured = store
.read()
.await
.get("myapp")
.and_then(|e| e.expected_node_id.clone());
assert_eq!(
captured,
Some(pubkey.clone()),
"expected_node_id should be captured on first contact"
);
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
assert!(
toml_content.unwrap_or_default().contains(&pubkey),
"captured node_id should be persisted to projects.toml"
);
}
#[tokio::test]
async fn verify_sled_identity_match_succeeds() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some(pubkey),
},
);
let store = Arc::new(RwLock::new(map));
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
assert!(
result.is_ok(),
"matching identity should not fail the upgrade: {result:?}"
);
}
#[tokio::test]
async fn verify_sled_identity_mismatch_fails_loudly() {
let kp = bft_json_crdt::keypair::make_keypair();
let responder_pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some("ab".repeat(32)),
},
);
let store = Arc::new(RwLock::new(map));
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
let err = result.expect_err("mismatched identity must fail loudly");
assert!(
err.contains("identity mismatch"),
"error should name the failure class: {err}"
);
assert!(
err.contains("huskies-myapp"),
"error should name the container: {err}"
);
assert!(
err.contains(&responder_pubkey),
"error should name the node_id that actually answered: {err}"
);
}
}
+3
View File
@@ -1177,6 +1177,7 @@ async fn ws_only_sled_handles_tools_list_and_tools_call() {
auth_token: Some("secret".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let config = GatewayConfig {
@@ -1248,6 +1249,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() {
auth_token: Some("alpha-tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
projects.insert(
@@ -1257,6 +1259,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() {
auth_token: Some("beta-tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let config = GatewayConfig {
+85 -6
View File
@@ -3,10 +3,25 @@
//! `GET /identity` returns the node's ID and public key as JSON. No
//! authentication is required; only the public half of the keypair is
//! disclosed.
//!
//! `GET /identity?nonce=<hex>` (story 1173) additionally signs the nonce with
//! the node's CRDT signing key ([`crate::crdt_state::sign_challenge`]) so a
//! caller can verify the response actually came from the sled it expects,
//! not just whichever container answered on that URL. The signing call
//! acquires the CRDT state mutex exactly once (lock → sign → unlock) with no
//! nested CRDT calls while held, avoiding the bug-1170 self-deadlock class.
use poem::handler;
use poem::web::Json;
use serde::Serialize;
use poem::web::{Json, Query};
use serde::{Deserialize, Serialize};
/// Query parameters accepted by `GET /identity`.
#[derive(Deserialize)]
pub struct IdentityQuery {
/// Optional challenge nonce to sign with the node's CRDT signing key.
#[serde(default)]
pub nonce: Option<String>,
}
/// JSON response body for `GET /identity`.
#[derive(Serialize)]
@@ -15,22 +30,46 @@ pub struct IdentityResponse {
pub node_id: String,
/// Lowercase hex-encoding of the 32-byte Ed25519 public key.
pub pubkey: String,
/// Ed25519 signature (hex) over `nonce`, made with the CRDT signing key.
/// `None` when no `nonce` was supplied, or before the CRDT layer has
/// initialised (in which case `node_id`/`pubkey` fall back to the
/// file-based node identity).
#[serde(skip_serializing_if = "Option::is_none")]
pub signature: Option<String>,
}
/// `GET /identity` — return this node's Ed25519 public key.
/// `GET /identity` — return this node's Ed25519 public key, optionally
/// signing a caller-supplied nonce with the CRDT signing key.
///
/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>"}`.
/// No authentication required; the private key is never exposed.
/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>", "signature": "<128-hex>"}`
/// when a `nonce` query parameter is given and the CRDT layer is
/// initialised. Otherwise omits `signature` and falls back to the
/// file-based node identity (legacy shape). No authentication required;
/// private keys are never exposed.
#[handler]
pub fn identity_handler() -> Json<IdentityResponse> {
pub fn identity_handler(
Query(IdentityQuery { nonce }): Query<IdentityQuery>,
) -> Json<IdentityResponse> {
if let Some(ref nonce) = nonce
&& let Some((pubkey_hex, signature_hex)) = crate::crdt_state::sign_challenge(nonce)
{
return Json(IdentityResponse {
node_id: pubkey_hex.clone(),
pubkey: pubkey_hex,
signature: Some(signature_hex),
});
}
match crate::node_identity::get_identity() {
Some(id) => Json(IdentityResponse {
node_id: id.node_id.clone(),
pubkey: id.pubkey_hex.clone(),
signature: None,
}),
None => Json(IdentityResponse {
node_id: "uninitialized".to_string(),
pubkey: "uninitialized".to_string(),
signature: None,
}),
}
}
@@ -58,5 +97,45 @@ mod tests {
assert_eq!(node_id.len(), 64);
assert!(node_id.chars().all(|c| c.is_ascii_hexdigit()));
assert_eq!(node_id, pubkey);
assert!(body.get("signature").is_none());
}
#[tokio::test]
async fn identity_endpoint_signs_nonce_with_crdt_key_when_initialised() {
crate::crdt_state::init_for_test();
let app = Route::new().at("/identity", get(identity_handler));
let cli = TestClient::new(app);
let resp = cli
.get("/identity")
.query("nonce", &"deadbeef")
.send()
.await;
resp.assert_status_is_ok();
let body: serde_json::Value = resp.json().await.value().deserialize();
let node_id = body["node_id"].as_str().unwrap();
let pubkey = body["pubkey"].as_str().unwrap();
let signature = body["signature"].as_str().unwrap();
assert_eq!(node_id, pubkey);
assert!(
crate::node_identity::verify_message_strict(pubkey, b"deadbeef", signature),
"signature must verify against the returned pubkey and nonce"
);
// The signed node_id must be the CRDT node id, not the file-based one.
assert_eq!(node_id, crate::crdt_state::our_node_id().unwrap());
}
#[tokio::test]
async fn identity_endpoint_without_nonce_omits_signature_even_when_crdt_initialised() {
crate::crdt_state::init_for_test();
let app = Route::new().at("/identity", get(identity_handler));
let cli = TestClient::new(app);
let resp = cli.get("/identity").send().await;
resp.assert_status_is_ok();
let body: serde_json::Value = resp.json().await.value().deserialize();
assert!(body.get("signature").is_none());
}
}
+51
View File
@@ -40,6 +40,14 @@ pub struct ProjectEntry {
/// commands can route to the correct directory without re-deriving it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub host_path: Option<String>,
/// Ed25519 node ID (hex pubkey) this sled is expected to answer as.
///
/// Captured automatically from the first successful `/identity` probe
/// (story 1173) when absent, then checked on every subsequent upgrade and
/// health probe so a container swapped out from under the gateway is
/// detected even when its `/health` endpoint still reports ok.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected_node_id: Option<String>,
}
impl ProjectEntry {
@@ -52,6 +60,7 @@ impl ProjectEntry {
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
}
}
@@ -223,6 +232,7 @@ auth_token = "secret"
auth_token: Some("secret".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let config = GatewayConfig {
@@ -258,6 +268,7 @@ auth_token = "secret"
auth_token: Some("tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
assert_eq!(validate_project_exists(&projects, "ws").unwrap(), "");
@@ -278,6 +289,7 @@ auth_token = "secret"
auth_token: Some("tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
};
assert!(!e.has_url());
}
@@ -321,6 +333,7 @@ auth_token = "secret"
auth_token: Some("mysecret".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
};
let mut projects = BTreeMap::new();
projects.insert("myproj".into(), entry);
@@ -347,6 +360,7 @@ auth_token = "secret"
auth_token: None,
ssh_port: Some(2201),
host_path: None,
expected_node_id: None,
};
let mut projects = BTreeMap::new();
projects.insert("myproj".into(), entry);
@@ -364,6 +378,43 @@ auth_token = "secret"
);
}
#[test]
fn expected_node_id_roundtrips_and_is_omitted_when_none() {
let with_id = ProjectEntry {
url: Some("http://127.0.0.1:3101".into()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some("ab".repeat(32)),
};
let mut projects = BTreeMap::new();
projects.insert("p".into(), with_id);
let config = GatewayConfig {
projects,
sled_tokens: BTreeMap::new(),
};
let toml_str = toml::to_string_pretty(&config).unwrap();
assert!(toml_str.contains("expected_node_id"));
let parsed: GatewayConfig = toml::from_str(&toml_str).unwrap();
assert_eq!(
parsed.projects["p"].expected_node_id.as_deref(),
Some("ab".repeat(32).as_str())
);
let without_id = ProjectEntry::with_url("http://127.0.0.1:3101");
let mut projects2 = BTreeMap::new();
projects2.insert("p".into(), without_id);
let config2 = GatewayConfig {
projects: projects2,
sled_tokens: BTreeMap::new(),
};
let toml_str2 = toml::to_string_pretty(&config2).unwrap();
assert!(
!toml_str2.contains("expected_node_id"),
"expected_node_id should be omitted when None: {toml_str2}"
);
}
#[test]
fn ssh_port_none_is_omitted_from_toml() {
let entry = ProjectEntry::with_url("http://127.0.0.1:3101");
+175
View File
@@ -0,0 +1,175 @@
//! Pure identity-probe verification — no I/O (story 1173).
//!
//! [`check_identity`] compares a sled's `/identity` probe response against the
//! `expected_node_id` recorded for that project and reports one of: a first
//! contact (no expected id captured yet), a match, a mismatch, an invalid
//! signature, or a missing signature (a legacy sled predating this story).
//! Callers ([`super::super::super::chat::transport::matrix::sled_upgrade`] and
//! `health`) own all I/O — issuing the HTTP probe and persisting a captured
//! `expected_node_id` back to `projects.toml`.
use serde::Deserialize;
/// Deserialized body of a `GET /identity?nonce=...` response.
#[derive(Debug, Clone, Deserialize)]
pub struct IdentityProbeResponse {
/// The responder's self-reported node ID (hex Ed25519 pubkey).
pub node_id: String,
/// The responder's self-reported Ed25519 public key (hex). Equal to
/// `node_id` on every server built since this field was introduced.
pub pubkey: String,
/// Ed25519 signature (hex) over the probe nonce, made with the CRDT
/// signing key. `None` when the responder predates story 1173 (a legacy
/// sled) — the `/identity` endpoint did not sign nonces before then.
#[serde(default)]
pub signature: Option<String>,
}
/// Outcome of comparing an [`IdentityProbeResponse`] against an
/// `expected_node_id`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum IdentityCheck {
/// No `expected_node_id` was recorded yet; the signature verified, so
/// `node_id` should be captured as the expected identity going forward.
FirstContact {
/// The node ID to capture.
node_id: String,
},
/// The signature verified and the responder's node ID matches the
/// expected one.
Match,
/// The signature verified, but the responder's node ID does not match
/// the expected one — a different container answered.
Mismatch {
/// The verified node ID that actually answered.
responder_node_id: String,
},
/// The response included a signature, but it did not verify against the
/// claimed pubkey/nonce — the responder's identity cannot be trusted.
InvalidSignature,
/// The response had no `signature` field — a legacy sled running a
/// pre-story-1173 binary. Identity cannot be verified either way.
MissingSignature,
}
/// Verify `response` against `nonce` and compare the verified node ID to
/// `expected_node_id`.
///
/// A signature verifies when [`crate::node_identity::verify_message_strict`]
/// confirms it was produced by the private key matching `response.pubkey`
/// over `nonce`'s UTF-8 bytes — mirroring how `/identity` signs on the server
/// side via `crdt_state::sign_challenge`.
pub fn check_identity(
expected_node_id: Option<&str>,
nonce: &str,
response: &IdentityProbeResponse,
) -> IdentityCheck {
let Some(signature) = response.signature.as_deref() else {
return IdentityCheck::MissingSignature;
};
if !crate::node_identity::verify_message_strict(&response.pubkey, nonce.as_bytes(), signature) {
return IdentityCheck::InvalidSignature;
}
match expected_node_id {
None => IdentityCheck::FirstContact {
node_id: response.node_id.clone(),
},
Some(expected) if expected == response.node_id => IdentityCheck::Match,
Some(_) => IdentityCheck::Mismatch {
responder_node_id: response.node_id.clone(),
},
}
}
// ── Tests ────────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
use bft_json_crdt::keypair::make_keypair;
fn signed_response(nonce: &str) -> (String, IdentityProbeResponse) {
let kp = make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let sig = crate::node_identity::sign_challenge(&kp, nonce);
(
pubkey.clone(),
IdentityProbeResponse {
node_id: pubkey.clone(),
pubkey,
signature: Some(sig),
},
)
}
#[test]
fn first_contact_when_no_expected_id() {
let nonce = "nonce-1";
let (node_id, resp) = signed_response(nonce);
let result = check_identity(None, nonce, &resp);
assert_eq!(result, IdentityCheck::FirstContact { node_id });
}
#[test]
fn match_when_expected_id_equals_responder() {
let nonce = "nonce-2";
let (node_id, resp) = signed_response(nonce);
let result = check_identity(Some(&node_id), nonce, &resp);
assert_eq!(result, IdentityCheck::Match);
}
#[test]
fn mismatch_when_expected_id_differs() {
let nonce = "nonce-3";
let (node_id, resp) = signed_response(nonce);
let result = check_identity(Some("some-other-node-id"), nonce, &resp);
assert_eq!(
result,
IdentityCheck::Mismatch {
responder_node_id: node_id
}
);
}
#[test]
fn invalid_signature_wrong_nonce() {
let nonce = "nonce-4";
let (_node_id, resp) = signed_response(nonce);
// Verify against a different nonce than the one that was signed.
let result = check_identity(None, "different-nonce", &resp);
assert_eq!(result, IdentityCheck::InvalidSignature);
}
#[test]
fn invalid_signature_wrong_key() {
let nonce = "nonce-5";
let (_node_id, mut resp) = signed_response(nonce);
// Claim a different pubkey than the one that actually signed.
let other_kp = make_keypair();
let other_pubkey = crate::node_identity::public_key_hex(&other_kp);
resp.node_id = other_pubkey.clone();
resp.pubkey = other_pubkey;
let result = check_identity(None, nonce, &resp);
assert_eq!(result, IdentityCheck::InvalidSignature);
}
#[test]
fn missing_signature_is_legacy_sled() {
let resp = IdentityProbeResponse {
node_id: "abc123".to_string(),
pubkey: "abc123".to_string(),
signature: None,
};
let result = check_identity(Some("abc123"), "nonce-6", &resp);
assert_eq!(result, IdentityCheck::MissingSignature);
}
#[test]
fn missing_signature_field_deserializes_from_json_without_it() {
let json = r#"{"node_id":"abc","pubkey":"abc"}"#;
let resp: IdentityProbeResponse = serde_json::from_str(json).unwrap();
assert!(resp.signature.is_none());
}
}
+26
View File
@@ -94,6 +94,32 @@ pub fn write_bot_config(config_dir: &Path, content: &str) -> Result<(), String>
std::fs::write(&path, content).map_err(|e| format!("cannot write bot.toml: {e}"))
}
// ── Identity probe I/O ───────────────────────────────────────────────────────
/// `GET {sled_url}/identity?nonce=<nonce>` and parse the JSON body.
///
/// Returns `None` when the sled is unreachable or the response body doesn't
/// parse as [`super::identity::IdentityProbeResponse`] — callers treat that
/// the same as an unverifiable identity (distinct from a legacy sled, which
/// responds but omits the `signature` field).
pub async fn probe_identity(
client: &Client,
sled_url: &str,
nonce: &str,
) -> Option<super::identity::IdentityProbeResponse> {
// `nonce` is always a hex string (see `node_identity::generate_challenge`),
// so no percent-encoding is needed for safe inclusion in the query string.
let url = format!("{}/identity?nonce={nonce}", sled_url.trim_end_matches('/'));
client
.get(&url)
.send()
.await
.ok()?
.json::<super::identity::IdentityProbeResponse>()
.await
.ok()
}
// ── MCP proxy I/O ───────────────────────────────────────────────────────────
/// Proxy a raw MCP request body to the given project URL.
+6 -1
View File
@@ -11,14 +11,17 @@
pub mod aggregation;
/// Gateway configuration types and TOML parsing.
pub mod config;
/// Pure identity-probe verification (match/mismatch/first-contact) — no I/O.
pub mod identity;
pub(crate) mod io;
/// Notification event polling for gateway-level broadcasts.
pub mod polling;
pub use aggregation::format_aggregate_status_compact;
pub use config::{GatewayConfig, ProjectEntry};
pub use identity::{IdentityCheck, check_identity};
pub use io::{
fetch_all_project_pipeline_statuses, spawn_gateway_broadcaster_forwarder,
fetch_all_project_pipeline_statuses, probe_identity, spawn_gateway_broadcaster_forwarder,
spawn_gateway_notification_poller,
};
@@ -781,6 +784,7 @@ mod tests {
auth_token: Some("tok".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let config = GatewayConfig {
@@ -921,6 +925,7 @@ mod tests {
auth_token: Some("secret-token".into()),
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let config = GatewayConfig {