crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

huskies: merge 1173 story Identity-aware fleet checks: cryptographic node identity in upgrade and health probes

Browse Source
This commit is contained in:
Huskies Agent
2026-07-16 13:26:03 +00:00
parent 489c415fd9
commit 0ac68afa4c
14 changed files with 894 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/chat/transport/matrix/health.rs
+251 -15
View File
@@ -227,10 +227,15 @@ async fn check_build_hash(project_root: &std::path::Path) -> HealthLine {
/// Check each registered sled's `/health` endpoint with a 5-second timeout.
///
/// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP
/// 2xx; FAIL when the request times out or returns an error status. Each line
/// includes a `relay=X` indicator derived from the gateway CRDT event log.
/// 2xx *and* its `/identity` probe matches the recorded `expected_node_id`;
/// FAIL when `/health` times out or errors, OR when the identity probe
/// reveals a mismatch or invalid signature — even if `/health` itself
/// reported ok, since a swapped-in container can still serve a healthy
/// `/health` response. Each line includes a `relay=X` indicator derived from
/// the gateway CRDT event log.
async fn check_sleds(
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
config_dir: &std::path::Path,
) -> Vec<HealthLine> {
let entries: Vec<(String, Option<String>)> = store
.read()
@@ -274,26 +279,30 @@ async fn check_sleds(
Some(url) => {
let health_url = format!("{}/health", url.trim_end_matches('/'));
let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await;
match result {
Err(_) => {
HealthLine::fail(subsystem, "timed out", "check container is running")
.with_relay(relay)
}
let health_line = match result {
Err(_) => HealthLine::fail(
subsystem.clone(),
"timed out",
"check container is running",
),
Ok(Err(e)) => HealthLine::fail(
subsystem,
subsystem.clone(),
format!("unreachable: {}", short_error(&e.to_string())),
"check container is running",
)
.with_relay(relay),
),
Ok(Ok(resp)) if resp.status().is_success() => {
HealthLine::pass(subsystem).with_relay(relay)
HealthLine::pass(subsystem.clone())
}
Ok(Ok(resp)) => HealthLine::fail(
subsystem,
subsystem.clone(),
format!("HTTP {}", resp.status().as_u16()),
"check container logs",
)
.with_relay(relay),
),
};
match check_sled_identity(&client, &name, &url, store, config_dir).await {
Some(identity_fail) => identity_fail.with_relay(relay),
None => health_line.with_relay(relay),
}
}
};
@@ -303,6 +312,69 @@ async fn check_sleds(
lines
}
/// Probe `{url}/identity`, verify the response against the `expected_node_id`
/// recorded for `name`, and capture a first-contact identity.
///
/// Returns `Some(HealthLine::fail(...))` on a confirmed mismatch or invalid
/// signature — this overrides an otherwise-passing `/health` result. Returns
/// `None` when the identity is unverifiable (unreachable probe or a legacy
/// sled with no signature) or matches, in which case the `/health` result
/// stands on its own.
async fn check_sled_identity(
client: &reqwest::Client,
name: &str,
url: &str,
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
config_dir: &std::path::Path,
) -> Option<HealthLine> {
let subsystem = format!("sled:{name}");
let nonce = crate::node_identity::generate_challenge();
let response = timeout(
Duration::from_secs(5),
crate::service::gateway::probe_identity(client, url, &nonce),
)
.await
.ok()
.flatten()?;
let expected = store
.read()
.await
.get(name)
.and_then(|e| e.expected_node_id.clone());
match crate::service::gateway::check_identity(expected.as_deref(), &nonce, &response) {
crate::service::gateway::IdentityCheck::Match
| crate::service::gateway::IdentityCheck::MissingSignature => None,
crate::service::gateway::IdentityCheck::FirstContact { node_id } => {
{
let mut projects = store.write().await;
if let Some(entry) = projects.get_mut(name) {
entry.expected_node_id = Some(node_id);
}
}
let snapshot = store.read().await.clone();
crate::service::gateway::io::save_config(&snapshot, config_dir).await;
None
}
crate::service::gateway::IdentityCheck::Mismatch { responder_node_id } => {
Some(HealthLine::fail(
subsystem,
format!(
"identity mismatch: expected `{}`, container answered as `{responder_node_id}`",
expected.unwrap_or_default()
),
"verify the correct container is running — possible swap",
))
}
crate::service::gateway::IdentityCheck::InvalidSignature => Some(HealthLine::fail(
subsystem,
"identity signature invalid",
"container's identity cannot be verified — check logs",
)),
}
}
/// Check the gateway process: pidfile validity and (on macOS) binary codesign.
///
/// PASS when our PID is recorded in the pidfile. On macOS, also verifies that
@@ -392,7 +464,7 @@ pub async fn run_health_check(ctx: &BotContext) -> String {
if ctx.is_gateway() {
lines.push(check_gateway_process());
if let Some(ref store) = ctx.gateway_projects_store {
lines.extend(check_sleds(store).await);
lines.extend(check_sleds(store, &ctx.services.project_root).await);
}
}
@@ -652,6 +724,170 @@ mod tests {
assert!(!cmd.description.is_empty());
}
// -- check_sled_identity ---------------------------------------------------
/// Spawn a one-shot TCP listener that answers a single `GET
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
/// whatever nonce the caller actually sent.
fn spawn_identity_responder(
listener: tokio::net::TcpListener,
kp: bft_json_crdt::keypair::Ed25519KeyPair,
) {
tokio::spawn(async move {
if let Ok((mut stream, _)) = listener.accept().await {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 4096];
let n = stream.read(&mut buf).await.unwrap_or(0);
let req = String::from_utf8_lossy(&buf[..n]);
let nonce = req
.lines()
.next()
.unwrap_or("")
.split("nonce=")
.nth(1)
.and_then(|s| s.split_whitespace().next())
.unwrap_or("")
.to_string();
let pubkey = crate::node_identity::public_key_hex(&kp);
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
let body = serde_json::json!({
"node_id": pubkey,
"pubkey": pubkey,
"signature": sig,
})
.to_string();
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(response.as_bytes()).await;
}
});
}
#[tokio::test]
async fn check_sled_identity_first_contact_captures_and_persists() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: None,
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
assert!(
result.is_none(),
"first contact must not fail the health line: {result:?}"
);
let captured = store
.read()
.await
.get("myapp")
.and_then(|e| e.expected_node_id.clone());
assert_eq!(
captured,
Some(pubkey.clone()),
"expected_node_id should be captured on first contact"
);
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
assert!(
toml_content.unwrap_or_default().contains(&pubkey),
"captured node_id should be persisted to projects.toml"
);
}
#[tokio::test]
async fn check_sled_identity_match_returns_none() {
let kp = bft_json_crdt::keypair::make_keypair();
let pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some(pubkey),
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
assert!(result.is_none(), "matching identity must PASS: {result:?}");
}
#[tokio::test]
async fn check_sled_identity_mismatch_forces_fail_line() {
let kp = bft_json_crdt::keypair::make_keypair();
let responder_pubkey = crate::node_identity::public_key_hex(&kp);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
spawn_identity_responder(listener, kp);
let sled_url = format!("http://127.0.0.1:{port}");
let mut map = BTreeMap::new();
map.insert(
"myapp".to_string(),
crate::service::gateway::config::ProjectEntry {
url: Some(sled_url.clone()),
auth_token: None,
ssh_port: None,
host_path: None,
expected_node_id: Some("ab".repeat(32)),
},
);
let store = tokio::sync::RwLock::new(map);
let config_dir = tempfile::tempdir().unwrap();
let client = reqwest::Client::new();
let result =
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
let line = result.expect("identity mismatch must force a FAIL health line");
assert_eq!(
line.status,
Status::Fail,
"identity mismatch must FAIL even though /health would report ok"
);
assert!(
line.detail
.as_deref()
.unwrap_or("")
.contains(&responder_pubkey),
"detail should name the node_id that actually answered: {line:?}"
);
}
// -- Helper ---------------------------------------------------------------
/// Build a minimal `BotContext` for testing purposes.