huskies: merge 1173 story Identity-aware fleet checks: cryptographic node identity in upgrade and health probes
This commit is contained in:
@@ -486,6 +486,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
)])));
|
||||
let ctx = test_bot_context(services, Some(Arc::clone(&active)), Some(store));
|
||||
@@ -530,6 +531,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -586,6 +586,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
||||
&project,
|
||||
store,
|
||||
ctx.gateway_port,
|
||||
&ctx.services.project_root,
|
||||
|phase_msg| {
|
||||
let transport = Arc::clone(&transport);
|
||||
let bot_sent = Arc::clone(&bot_sent);
|
||||
@@ -633,6 +634,7 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
||||
let response = super::super::super::sled_upgrade::handle_upgrade_all(
|
||||
store,
|
||||
ctx.gateway_port,
|
||||
&ctx.services.project_root,
|
||||
|phase_msg| {
|
||||
let transport = Arc::clone(&transport);
|
||||
let bot_sent = Arc::clone(&bot_sent);
|
||||
@@ -1189,6 +1191,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1215,6 +1218,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
)]));
|
||||
|
||||
|
||||
@@ -227,10 +227,15 @@ async fn check_build_hash(project_root: &std::path::Path) -> HealthLine {
|
||||
/// Check each registered sled's `/health` endpoint with a 5-second timeout.
|
||||
///
|
||||
/// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP
|
||||
/// 2xx; FAIL when the request times out or returns an error status. Each line
|
||||
/// includes a `relay=X` indicator derived from the gateway CRDT event log.
|
||||
/// 2xx *and* its `/identity` probe matches the recorded `expected_node_id`;
|
||||
/// FAIL when `/health` times out or errors, OR when the identity probe
|
||||
/// reveals a mismatch or invalid signature — even if `/health` itself
|
||||
/// reported ok, since a swapped-in container can still serve a healthy
|
||||
/// `/health` response. Each line includes a `relay=X` indicator derived from
|
||||
/// the gateway CRDT event log.
|
||||
async fn check_sleds(
|
||||
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
|
||||
config_dir: &std::path::Path,
|
||||
) -> Vec<HealthLine> {
|
||||
let entries: Vec<(String, Option<String>)> = store
|
||||
.read()
|
||||
@@ -274,26 +279,30 @@ async fn check_sleds(
|
||||
Some(url) => {
|
||||
let health_url = format!("{}/health", url.trim_end_matches('/'));
|
||||
let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await;
|
||||
match result {
|
||||
Err(_) => {
|
||||
HealthLine::fail(subsystem, "timed out", "check container is running")
|
||||
.with_relay(relay)
|
||||
}
|
||||
let health_line = match result {
|
||||
Err(_) => HealthLine::fail(
|
||||
subsystem.clone(),
|
||||
"timed out",
|
||||
"check container is running",
|
||||
),
|
||||
Ok(Err(e)) => HealthLine::fail(
|
||||
subsystem,
|
||||
subsystem.clone(),
|
||||
format!("unreachable: {}", short_error(&e.to_string())),
|
||||
"check container is running",
|
||||
)
|
||||
.with_relay(relay),
|
||||
),
|
||||
Ok(Ok(resp)) if resp.status().is_success() => {
|
||||
HealthLine::pass(subsystem).with_relay(relay)
|
||||
HealthLine::pass(subsystem.clone())
|
||||
}
|
||||
Ok(Ok(resp)) => HealthLine::fail(
|
||||
subsystem,
|
||||
subsystem.clone(),
|
||||
format!("HTTP {}", resp.status().as_u16()),
|
||||
"check container logs",
|
||||
)
|
||||
.with_relay(relay),
|
||||
),
|
||||
};
|
||||
|
||||
match check_sled_identity(&client, &name, &url, store, config_dir).await {
|
||||
Some(identity_fail) => identity_fail.with_relay(relay),
|
||||
None => health_line.with_relay(relay),
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -303,6 +312,69 @@ async fn check_sleds(
|
||||
lines
|
||||
}
|
||||
|
||||
/// Probe `{url}/identity`, verify the response against the `expected_node_id`
|
||||
/// recorded for `name`, and capture a first-contact identity.
|
||||
///
|
||||
/// Returns `Some(HealthLine::fail(...))` on a confirmed mismatch or invalid
|
||||
/// signature — this overrides an otherwise-passing `/health` result. Returns
|
||||
/// `None` when the identity is unverifiable (unreachable probe or a legacy
|
||||
/// sled with no signature) or matches, in which case the `/health` result
|
||||
/// stands on its own.
|
||||
async fn check_sled_identity(
|
||||
client: &reqwest::Client,
|
||||
name: &str,
|
||||
url: &str,
|
||||
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
|
||||
config_dir: &std::path::Path,
|
||||
) -> Option<HealthLine> {
|
||||
let subsystem = format!("sled:{name}");
|
||||
let nonce = crate::node_identity::generate_challenge();
|
||||
let response = timeout(
|
||||
Duration::from_secs(5),
|
||||
crate::service::gateway::probe_identity(client, url, &nonce),
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()?;
|
||||
|
||||
let expected = store
|
||||
.read()
|
||||
.await
|
||||
.get(name)
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
|
||||
match crate::service::gateway::check_identity(expected.as_deref(), &nonce, &response) {
|
||||
crate::service::gateway::IdentityCheck::Match
|
||||
| crate::service::gateway::IdentityCheck::MissingSignature => None,
|
||||
crate::service::gateway::IdentityCheck::FirstContact { node_id } => {
|
||||
{
|
||||
let mut projects = store.write().await;
|
||||
if let Some(entry) = projects.get_mut(name) {
|
||||
entry.expected_node_id = Some(node_id);
|
||||
}
|
||||
}
|
||||
let snapshot = store.read().await.clone();
|
||||
crate::service::gateway::io::save_config(&snapshot, config_dir).await;
|
||||
None
|
||||
}
|
||||
crate::service::gateway::IdentityCheck::Mismatch { responder_node_id } => {
|
||||
Some(HealthLine::fail(
|
||||
subsystem,
|
||||
format!(
|
||||
"identity mismatch: expected `{}`, container answered as `{responder_node_id}`",
|
||||
expected.unwrap_or_default()
|
||||
),
|
||||
"verify the correct container is running — possible swap",
|
||||
))
|
||||
}
|
||||
crate::service::gateway::IdentityCheck::InvalidSignature => Some(HealthLine::fail(
|
||||
subsystem,
|
||||
"identity signature invalid",
|
||||
"container's identity cannot be verified — check logs",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check the gateway process: pidfile validity and (on macOS) binary codesign.
|
||||
///
|
||||
/// PASS when our PID is recorded in the pidfile. On macOS, also verifies that
|
||||
@@ -392,7 +464,7 @@ pub async fn run_health_check(ctx: &BotContext) -> String {
|
||||
if ctx.is_gateway() {
|
||||
lines.push(check_gateway_process());
|
||||
if let Some(ref store) = ctx.gateway_projects_store {
|
||||
lines.extend(check_sleds(store).await);
|
||||
lines.extend(check_sleds(store, &ctx.services.project_root).await);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -652,6 +724,170 @@ mod tests {
|
||||
assert!(!cmd.description.is_empty());
|
||||
}
|
||||
|
||||
// -- check_sled_identity ---------------------------------------------------
|
||||
|
||||
/// Spawn a one-shot TCP listener that answers a single `GET
|
||||
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
|
||||
/// whatever nonce the caller actually sent.
|
||||
fn spawn_identity_responder(
|
||||
listener: tokio::net::TcpListener,
|
||||
kp: bft_json_crdt::keypair::Ed25519KeyPair,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let nonce = req
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split("nonce=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split_whitespace().next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
|
||||
let body = serde_json::json!({
|
||||
"node_id": pubkey,
|
||||
"pubkey": pubkey,
|
||||
"signature": sig,
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_sled_identity_first_contact_captures_and_persists() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
crate::service::gateway::config::ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = tokio::sync::RwLock::new(map);
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"first contact must not fail the health line: {result:?}"
|
||||
);
|
||||
|
||||
let captured = store
|
||||
.read()
|
||||
.await
|
||||
.get("myapp")
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
assert_eq!(
|
||||
captured,
|
||||
Some(pubkey.clone()),
|
||||
"expected_node_id should be captured on first contact"
|
||||
);
|
||||
|
||||
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
|
||||
assert!(
|
||||
toml_content.unwrap_or_default().contains(&pubkey),
|
||||
"captured node_id should be persisted to projects.toml"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_sled_identity_match_returns_none() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
crate::service::gateway::config::ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some(pubkey),
|
||||
},
|
||||
);
|
||||
let store = tokio::sync::RwLock::new(map);
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
|
||||
assert!(result.is_none(), "matching identity must PASS: {result:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn check_sled_identity_mismatch_forces_fail_line() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let responder_pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
crate::service::gateway::config::ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some("ab".repeat(32)),
|
||||
},
|
||||
);
|
||||
let store = tokio::sync::RwLock::new(map);
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await;
|
||||
let line = result.expect("identity mismatch must force a FAIL health line");
|
||||
assert_eq!(
|
||||
line.status,
|
||||
Status::Fail,
|
||||
"identity mismatch must FAIL even though /health would report ok"
|
||||
);
|
||||
assert!(
|
||||
line.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains(&responder_pubkey),
|
||||
"detail should name the node_id that actually answered: {line:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// -- Helper ---------------------------------------------------------------
|
||||
|
||||
/// Build a minimal `BotContext` for testing purposes.
|
||||
|
||||
@@ -803,6 +803,7 @@ async fn handle_adopt_project(
|
||||
auth_token: None,
|
||||
ssh_port: Some(ssh_port),
|
||||
host_path: Some(host_path.to_string_lossy().into_owned()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
crate::service::gateway::io::save_config(&projects, config_dir).await;
|
||||
@@ -1204,6 +1205,7 @@ pub async fn handle_new_project(
|
||||
auth_token: None,
|
||||
ssh_port: Some(ssh_port),
|
||||
host_path: Some(host_path.to_string_lossy().into_owned()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
crate::service::gateway::io::save_config(&projects, config_dir).await;
|
||||
|
||||
@@ -525,6 +525,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: Some(2201),
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
)]);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -544,6 +545,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: Some(2201),
|
||||
host_path: Some("/nonexistent/path/xyz123".into()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
)]);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -578,6 +580,7 @@ mod tests {
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: Some(2201),
|
||||
host_path: Some(host_dir.path().to_str().unwrap().to_string()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
)]);
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -114,6 +114,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: Some(2203),
|
||||
host_path: Some("/home/user/workspace".into()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
projects.insert(
|
||||
@@ -123,6 +124,7 @@ mod tests {
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -165,6 +167,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: Some("/home/user/adopted".into()),
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
projects.insert("builtin".into(), ProjectEntry::with_url("http://b:3002"));
|
||||
|
||||
@@ -200,6 +200,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: host_path.map(String::from),
|
||||
expected_node_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,12 +27,69 @@
|
||||
//! on the sled restart.
|
||||
|
||||
use crate::service::gateway::config::ProjectEntry;
|
||||
use crate::service::gateway::{IdentityCheck, check_identity, probe_identity};
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
|
||||
/// Probe `/identity` on `sled_url`, verify it against the `expected_node_id`
|
||||
/// recorded for `project`, and capture a first-contact identity.
|
||||
///
|
||||
/// Fails open (returns `Ok`) when the probe is unreachable or the responder
|
||||
/// has no signature (legacy sled) — the existing `/health`-based liveness
|
||||
/// check remains the primary reachability gate. Only a confirmed identity
|
||||
/// mismatch or an invalid signature is treated as a hard failure, since both
|
||||
/// mean a verifiably different — or untrustworthy — container answered.
|
||||
async fn verify_sled_identity(
|
||||
project: &str,
|
||||
sled_url: &str,
|
||||
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||
config_dir: &Path,
|
||||
client: &reqwest::Client,
|
||||
) -> Result<(), String> {
|
||||
let container_name = format!("huskies-{project}");
|
||||
let nonce = crate::node_identity::generate_challenge();
|
||||
let Some(response) = probe_identity(client, sled_url, &nonce).await else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let expected = {
|
||||
let projects = projects_store.read().await;
|
||||
projects
|
||||
.get(project)
|
||||
.and_then(|e| e.expected_node_id.clone())
|
||||
};
|
||||
|
||||
match check_identity(expected.as_deref(), &nonce, &response) {
|
||||
IdentityCheck::Match | IdentityCheck::MissingSignature => Ok(()),
|
||||
IdentityCheck::FirstContact { node_id } => {
|
||||
{
|
||||
let mut projects = projects_store.write().await;
|
||||
if let Some(entry) = projects.get_mut(project) {
|
||||
entry.expected_node_id = Some(node_id);
|
||||
}
|
||||
}
|
||||
let snapshot = projects_store.read().await.clone();
|
||||
crate::service::gateway::io::save_config(&snapshot, config_dir).await;
|
||||
Ok(())
|
||||
}
|
||||
IdentityCheck::Mismatch { responder_node_id } => Err(format!(
|
||||
"**identity mismatch** for `{container_name}` at `{sled_url}`: expected node_id \
|
||||
`{}`, but the container that answered identified as `{responder_node_id}`. \
|
||||
Refusing to proceed — this may not be the sled you expect.",
|
||||
expected.unwrap_or_default()
|
||||
)),
|
||||
IdentityCheck::InvalidSignature => Err(format!(
|
||||
"**identity verification failed** for `{container_name}` at `{sled_url}`: the \
|
||||
`/identity` response signature did not verify. Refusing to proceed — the \
|
||||
container's identity cannot be trusted."
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Serial lock ────────────────────────────────────────────────────────────────
|
||||
|
||||
static UPGRADE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
@@ -157,6 +214,7 @@ fn resolve_artifact_source(gateway_port: Option<u16>) -> Result<(String, Option<
|
||||
pub async fn handle_upgrade_all<F, Fut>(
|
||||
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||
gateway_port: Option<u16>,
|
||||
config_dir: &Path,
|
||||
send_phase: F,
|
||||
) -> String
|
||||
where
|
||||
@@ -178,7 +236,7 @@ where
|
||||
|
||||
let mut results: Vec<String> = Vec::with_capacity(names.len());
|
||||
for name in &names {
|
||||
let outcome = handle_sled_upgrade(name, projects_store, gateway_port, |msg| {
|
||||
let outcome = handle_sled_upgrade(name, projects_store, gateway_port, config_dir, |msg| {
|
||||
send_phase(format!("**{name}** {msg}"))
|
||||
})
|
||||
.await;
|
||||
@@ -205,6 +263,7 @@ pub async fn handle_sled_upgrade<F, Fut>(
|
||||
project: &str,
|
||||
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||
gateway_port: Option<u16>,
|
||||
config_dir: &Path,
|
||||
send_phase: F,
|
||||
) -> String
|
||||
where
|
||||
@@ -236,17 +295,29 @@ where
|
||||
Err(e) => return e,
|
||||
};
|
||||
|
||||
run_sled_upgrade(project, &sled_url, &source_url, expected_hash, send_phase).await
|
||||
run_sled_upgrade(
|
||||
project,
|
||||
&sled_url,
|
||||
&source_url,
|
||||
expected_hash,
|
||||
projects_store,
|
||||
config_dir,
|
||||
send_phase,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Run the four-phase upgrade against a sled whose source URL is already
|
||||
/// resolved. Split from [`handle_sled_upgrade`] so tests can drive the wire
|
||||
/// behaviour without a published artifact on the host.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_sled_upgrade<F, Fut>(
|
||||
project: &str,
|
||||
sled_url: &str,
|
||||
source_url: &str,
|
||||
expected_hash: Option<String>,
|
||||
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||
config_dir: &Path,
|
||||
send_phase: F,
|
||||
) -> String
|
||||
where
|
||||
@@ -263,6 +334,13 @@ where
|
||||
.build()
|
||||
.unwrap_or_default();
|
||||
|
||||
// ── Verify identity before triggering ────────────────────────────────────
|
||||
if let Err(e) =
|
||||
verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await
|
||||
{
|
||||
return e;
|
||||
}
|
||||
|
||||
// ── Phase 1: downloading ─────────────────────────────────────────────────
|
||||
send_phase("[1/4] downloading\u{2026}".to_string()).await;
|
||||
|
||||
@@ -316,6 +394,13 @@ where
|
||||
// ── Phase 4: reconnected ─────────────────────────────────────────────────
|
||||
send_phase("[4/4] reconnected to gateway".to_string()).await;
|
||||
|
||||
// ── Verify identity during convergence ───────────────────────────────────
|
||||
if let Err(e) =
|
||||
verify_sled_identity(project, sled_url, projects_store, config_dir, &client).await
|
||||
{
|
||||
return format!("upgraded and reconnected, but {e}");
|
||||
}
|
||||
|
||||
// ── Verify convergence ───────────────────────────────────────────────────
|
||||
match fetch_sled_version(&client, sled_url).await {
|
||||
Some((version, git_hash)) => match expected_hash {
|
||||
@@ -446,6 +531,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
map.insert(
|
||||
@@ -455,6 +541,7 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = Arc::new(RwLock::new(map));
|
||||
@@ -470,10 +557,17 @@ mod tests {
|
||||
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
|
||||
Arc::new(RwLock::new(BTreeMap::new()));
|
||||
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
|
||||
let result = handle_sled_upgrade("nonexistent", &store, Some(3000), |msg| {
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let result = handle_sled_upgrade(
|
||||
"nonexistent",
|
||||
&store,
|
||||
Some(3000),
|
||||
config_dir.path(),
|
||||
|msg| {
|
||||
phases.lock().unwrap().push(msg);
|
||||
async {}
|
||||
})
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
result.contains("not found"),
|
||||
@@ -496,10 +590,19 @@ mod tests {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = Arc::new(RwLock::new(map));
|
||||
let result = handle_sled_upgrade("myapp", &store, Some(3000), |_msg| async {}).await;
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let result = handle_sled_upgrade(
|
||||
"myapp",
|
||||
&store,
|
||||
Some(3000),
|
||||
config_dir.path(),
|
||||
|_msg| async {},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
result.contains("not found"),
|
||||
"project with no URL should say not found: {result}"
|
||||
@@ -518,7 +621,8 @@ mod tests {
|
||||
async fn upgrade_all_empty_store_reports_no_projects() {
|
||||
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
|
||||
Arc::new(RwLock::new(BTreeMap::new()));
|
||||
let msg = handle_upgrade_all(&store, Some(3000), |_msg| async {}).await;
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let msg = handle_upgrade_all(&store, Some(3000), config_dir.path(), |_msg| async {}).await;
|
||||
assert!(
|
||||
msg.contains("No projects"),
|
||||
"empty store should say no projects: {msg}"
|
||||
@@ -528,11 +632,16 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn upgrade_unreachable_sled_reports_failure() {
|
||||
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
|
||||
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
|
||||
Arc::new(RwLock::new(BTreeMap::new()));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let result = run_sled_upgrade(
|
||||
"myapp",
|
||||
"http://127.0.0.1:1", // port 1 is never listening
|
||||
"http://127.0.0.1:1/api/artifacts/huskies-linux-arm64",
|
||||
None,
|
||||
&store,
|
||||
config_dir.path(),
|
||||
|msg| {
|
||||
phases.lock().unwrap().push(msg);
|
||||
async {}
|
||||
@@ -590,4 +699,169 @@ mod tests {
|
||||
let ok = wait_for_health(&client, "http://127.0.0.1:1/health", 1).await;
|
||||
assert!(!ok, "should return false when health probe never succeeds");
|
||||
}
|
||||
|
||||
// ── verify_sled_identity ──────────────────────────────────────────────────
|
||||
|
||||
/// Spawn a one-shot TCP listener that answers a single `GET
|
||||
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
|
||||
/// whatever nonce the caller actually sent — mirrors the real
|
||||
/// `/identity` handler without needing a full HTTP server.
|
||||
fn spawn_identity_responder(
|
||||
listener: tokio::net::TcpListener,
|
||||
kp: bft_json_crdt::keypair::Ed25519KeyPair,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let nonce = req
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split("nonce=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split_whitespace().next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
|
||||
let body = serde_json::json!({
|
||||
"node_id": pubkey,
|
||||
"pubkey": pubkey,
|
||||
"signature": sig,
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verify_sled_identity_first_contact_captures_and_persists() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = Arc::new(RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
assert!(result.is_ok(), "first contact should succeed: {result:?}");
|
||||
|
||||
let captured = store
|
||||
.read()
|
||||
.await
|
||||
.get("myapp")
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
assert_eq!(
|
||||
captured,
|
||||
Some(pubkey.clone()),
|
||||
"expected_node_id should be captured on first contact"
|
||||
);
|
||||
|
||||
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
|
||||
assert!(
|
||||
toml_content.unwrap_or_default().contains(&pubkey),
|
||||
"captured node_id should be persisted to projects.toml"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verify_sled_identity_match_succeeds() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some(pubkey),
|
||||
},
|
||||
);
|
||||
let store = Arc::new(RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"matching identity should not fail the upgrade: {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verify_sled_identity_mismatch_fails_loudly() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let responder_pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some("ab".repeat(32)),
|
||||
},
|
||||
);
|
||||
let store = Arc::new(RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
verify_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
let err = result.expect_err("mismatched identity must fail loudly");
|
||||
assert!(
|
||||
err.contains("identity mismatch"),
|
||||
"error should name the failure class: {err}"
|
||||
);
|
||||
assert!(
|
||||
err.contains("huskies-myapp"),
|
||||
"error should name the container: {err}"
|
||||
);
|
||||
assert!(
|
||||
err.contains(&responder_pubkey),
|
||||
"error should name the node_id that actually answered: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1177,6 +1177,7 @@ async fn ws_only_sled_handles_tools_list_and_tools_call() {
|
||||
auth_token: Some("secret".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
@@ -1248,6 +1249,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() {
|
||||
auth_token: Some("alpha-tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
projects.insert(
|
||||
@@ -1257,6 +1259,7 @@ async fn two_concurrent_sleds_are_routed_by_active_project() {
|
||||
auth_token: Some("beta-tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
|
||||
@@ -3,10 +3,25 @@
|
||||
//! `GET /identity` returns the node's ID and public key as JSON. No
|
||||
//! authentication is required; only the public half of the keypair is
|
||||
//! disclosed.
|
||||
//!
|
||||
//! `GET /identity?nonce=<hex>` (story 1173) additionally signs the nonce with
|
||||
//! the node's CRDT signing key ([`crate::crdt_state::sign_challenge`]) so a
|
||||
//! caller can verify the response actually came from the sled it expects,
|
||||
//! not just whichever container answered on that URL. The signing call
|
||||
//! acquires the CRDT state mutex exactly once (lock → sign → unlock) with no
|
||||
//! nested CRDT calls while held, avoiding the bug-1170 self-deadlock class.
|
||||
|
||||
use poem::handler;
|
||||
use poem::web::Json;
|
||||
use serde::Serialize;
|
||||
use poem::web::{Json, Query};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Query parameters accepted by `GET /identity`.
|
||||
#[derive(Deserialize)]
|
||||
pub struct IdentityQuery {
|
||||
/// Optional challenge nonce to sign with the node's CRDT signing key.
|
||||
#[serde(default)]
|
||||
pub nonce: Option<String>,
|
||||
}
|
||||
|
||||
/// JSON response body for `GET /identity`.
|
||||
#[derive(Serialize)]
|
||||
@@ -15,22 +30,46 @@ pub struct IdentityResponse {
|
||||
pub node_id: String,
|
||||
/// Lowercase hex-encoding of the 32-byte Ed25519 public key.
|
||||
pub pubkey: String,
|
||||
/// Ed25519 signature (hex) over `nonce`, made with the CRDT signing key.
|
||||
/// `None` when no `nonce` was supplied, or before the CRDT layer has
|
||||
/// initialised (in which case `node_id`/`pubkey` fall back to the
|
||||
/// file-based node identity).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub signature: Option<String>,
|
||||
}
|
||||
|
||||
/// `GET /identity` — return this node's Ed25519 public key.
|
||||
/// `GET /identity` — return this node's Ed25519 public key, optionally
|
||||
/// signing a caller-supplied nonce with the CRDT signing key.
|
||||
///
|
||||
/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>"}`.
|
||||
/// No authentication required; the private key is never exposed.
|
||||
/// Returns `{"node_id": "<64-hex>", "pubkey": "<64-hex>", "signature": "<128-hex>"}`
|
||||
/// when a `nonce` query parameter is given and the CRDT layer is
|
||||
/// initialised. Otherwise omits `signature` and falls back to the
|
||||
/// file-based node identity (legacy shape). No authentication required;
|
||||
/// private keys are never exposed.
|
||||
#[handler]
|
||||
pub fn identity_handler() -> Json<IdentityResponse> {
|
||||
pub fn identity_handler(
|
||||
Query(IdentityQuery { nonce }): Query<IdentityQuery>,
|
||||
) -> Json<IdentityResponse> {
|
||||
if let Some(ref nonce) = nonce
|
||||
&& let Some((pubkey_hex, signature_hex)) = crate::crdt_state::sign_challenge(nonce)
|
||||
{
|
||||
return Json(IdentityResponse {
|
||||
node_id: pubkey_hex.clone(),
|
||||
pubkey: pubkey_hex,
|
||||
signature: Some(signature_hex),
|
||||
});
|
||||
}
|
||||
|
||||
match crate::node_identity::get_identity() {
|
||||
Some(id) => Json(IdentityResponse {
|
||||
node_id: id.node_id.clone(),
|
||||
pubkey: id.pubkey_hex.clone(),
|
||||
signature: None,
|
||||
}),
|
||||
None => Json(IdentityResponse {
|
||||
node_id: "uninitialized".to_string(),
|
||||
pubkey: "uninitialized".to_string(),
|
||||
signature: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -58,5 +97,45 @@ mod tests {
|
||||
assert_eq!(node_id.len(), 64);
|
||||
assert!(node_id.chars().all(|c| c.is_ascii_hexdigit()));
|
||||
assert_eq!(node_id, pubkey);
|
||||
assert!(body.get("signature").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identity_endpoint_signs_nonce_with_crdt_key_when_initialised() {
|
||||
crate::crdt_state::init_for_test();
|
||||
|
||||
let app = Route::new().at("/identity", get(identity_handler));
|
||||
let cli = TestClient::new(app);
|
||||
let resp = cli
|
||||
.get("/identity")
|
||||
.query("nonce", &"deadbeef")
|
||||
.send()
|
||||
.await;
|
||||
resp.assert_status_is_ok();
|
||||
|
||||
let body: serde_json::Value = resp.json().await.value().deserialize();
|
||||
let node_id = body["node_id"].as_str().unwrap();
|
||||
let pubkey = body["pubkey"].as_str().unwrap();
|
||||
let signature = body["signature"].as_str().unwrap();
|
||||
assert_eq!(node_id, pubkey);
|
||||
assert!(
|
||||
crate::node_identity::verify_message_strict(pubkey, b"deadbeef", signature),
|
||||
"signature must verify against the returned pubkey and nonce"
|
||||
);
|
||||
// The signed node_id must be the CRDT node id, not the file-based one.
|
||||
assert_eq!(node_id, crate::crdt_state::our_node_id().unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identity_endpoint_without_nonce_omits_signature_even_when_crdt_initialised() {
|
||||
crate::crdt_state::init_for_test();
|
||||
|
||||
let app = Route::new().at("/identity", get(identity_handler));
|
||||
let cli = TestClient::new(app);
|
||||
let resp = cli.get("/identity").send().await;
|
||||
resp.assert_status_is_ok();
|
||||
|
||||
let body: serde_json::Value = resp.json().await.value().deserialize();
|
||||
assert!(body.get("signature").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,14 @@ pub struct ProjectEntry {
|
||||
/// commands can route to the correct directory without re-deriving it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub host_path: Option<String>,
|
||||
/// Ed25519 node ID (hex pubkey) this sled is expected to answer as.
|
||||
///
|
||||
/// Captured automatically from the first successful `/identity` probe
|
||||
/// (story 1173) when absent, then checked on every subsequent upgrade and
|
||||
/// health probe so a container swapped out from under the gateway is
|
||||
/// detected even when its `/health` endpoint still reports ok.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_node_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ProjectEntry {
|
||||
@@ -52,6 +60,7 @@ impl ProjectEntry {
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,6 +232,7 @@ auth_token = "secret"
|
||||
auth_token: Some("secret".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
@@ -258,6 +268,7 @@ auth_token = "secret"
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
assert_eq!(validate_project_exists(&projects, "ws").unwrap(), "");
|
||||
@@ -278,6 +289,7 @@ auth_token = "secret"
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
};
|
||||
assert!(!e.has_url());
|
||||
}
|
||||
@@ -321,6 +333,7 @@ auth_token = "secret"
|
||||
auth_token: Some("mysecret".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
};
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert("myproj".into(), entry);
|
||||
@@ -347,6 +360,7 @@ auth_token = "secret"
|
||||
auth_token: None,
|
||||
ssh_port: Some(2201),
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
};
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert("myproj".into(), entry);
|
||||
@@ -364,6 +378,43 @@ auth_token = "secret"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_node_id_roundtrips_and_is_omitted_when_none() {
|
||||
let with_id = ProjectEntry {
|
||||
url: Some("http://127.0.0.1:3101".into()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some("ab".repeat(32)),
|
||||
};
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert("p".into(), with_id);
|
||||
let config = GatewayConfig {
|
||||
projects,
|
||||
sled_tokens: BTreeMap::new(),
|
||||
};
|
||||
let toml_str = toml::to_string_pretty(&config).unwrap();
|
||||
assert!(toml_str.contains("expected_node_id"));
|
||||
let parsed: GatewayConfig = toml::from_str(&toml_str).unwrap();
|
||||
assert_eq!(
|
||||
parsed.projects["p"].expected_node_id.as_deref(),
|
||||
Some("ab".repeat(32).as_str())
|
||||
);
|
||||
|
||||
let without_id = ProjectEntry::with_url("http://127.0.0.1:3101");
|
||||
let mut projects2 = BTreeMap::new();
|
||||
projects2.insert("p".into(), without_id);
|
||||
let config2 = GatewayConfig {
|
||||
projects: projects2,
|
||||
sled_tokens: BTreeMap::new(),
|
||||
};
|
||||
let toml_str2 = toml::to_string_pretty(&config2).unwrap();
|
||||
assert!(
|
||||
!toml_str2.contains("expected_node_id"),
|
||||
"expected_node_id should be omitted when None: {toml_str2}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssh_port_none_is_omitted_from_toml() {
|
||||
let entry = ProjectEntry::with_url("http://127.0.0.1:3101");
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Pure identity-probe verification — no I/O (story 1173).
|
||||
//!
|
||||
//! [`check_identity`] compares a sled's `/identity` probe response against the
|
||||
//! `expected_node_id` recorded for that project and reports one of: a first
|
||||
//! contact (no expected id captured yet), a match, a mismatch, an invalid
|
||||
//! signature, or a missing signature (a legacy sled predating this story).
|
||||
//! Callers ([`super::super::super::chat::transport::matrix::sled_upgrade`] and
|
||||
//! `health`) own all I/O — issuing the HTTP probe and persisting a captured
|
||||
//! `expected_node_id` back to `projects.toml`.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Deserialized body of a `GET /identity?nonce=...` response.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct IdentityProbeResponse {
|
||||
/// The responder's self-reported node ID (hex Ed25519 pubkey).
|
||||
pub node_id: String,
|
||||
/// The responder's self-reported Ed25519 public key (hex). Equal to
|
||||
/// `node_id` on every server built since this field was introduced.
|
||||
pub pubkey: String,
|
||||
/// Ed25519 signature (hex) over the probe nonce, made with the CRDT
|
||||
/// signing key. `None` when the responder predates story 1173 (a legacy
|
||||
/// sled) — the `/identity` endpoint did not sign nonces before then.
|
||||
#[serde(default)]
|
||||
pub signature: Option<String>,
|
||||
}
|
||||
|
||||
/// Outcome of comparing an [`IdentityProbeResponse`] against an
|
||||
/// `expected_node_id`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum IdentityCheck {
|
||||
/// No `expected_node_id` was recorded yet; the signature verified, so
|
||||
/// `node_id` should be captured as the expected identity going forward.
|
||||
FirstContact {
|
||||
/// The node ID to capture.
|
||||
node_id: String,
|
||||
},
|
||||
/// The signature verified and the responder's node ID matches the
|
||||
/// expected one.
|
||||
Match,
|
||||
/// The signature verified, but the responder's node ID does not match
|
||||
/// the expected one — a different container answered.
|
||||
Mismatch {
|
||||
/// The verified node ID that actually answered.
|
||||
responder_node_id: String,
|
||||
},
|
||||
/// The response included a signature, but it did not verify against the
|
||||
/// claimed pubkey/nonce — the responder's identity cannot be trusted.
|
||||
InvalidSignature,
|
||||
/// The response had no `signature` field — a legacy sled running a
|
||||
/// pre-story-1173 binary. Identity cannot be verified either way.
|
||||
MissingSignature,
|
||||
}
|
||||
|
||||
/// Verify `response` against `nonce` and compare the verified node ID to
|
||||
/// `expected_node_id`.
|
||||
///
|
||||
/// A signature verifies when [`crate::node_identity::verify_message_strict`]
|
||||
/// confirms it was produced by the private key matching `response.pubkey`
|
||||
/// over `nonce`'s UTF-8 bytes — mirroring how `/identity` signs on the server
|
||||
/// side via `crdt_state::sign_challenge`.
|
||||
pub fn check_identity(
|
||||
expected_node_id: Option<&str>,
|
||||
nonce: &str,
|
||||
response: &IdentityProbeResponse,
|
||||
) -> IdentityCheck {
|
||||
let Some(signature) = response.signature.as_deref() else {
|
||||
return IdentityCheck::MissingSignature;
|
||||
};
|
||||
|
||||
if !crate::node_identity::verify_message_strict(&response.pubkey, nonce.as_bytes(), signature) {
|
||||
return IdentityCheck::InvalidSignature;
|
||||
}
|
||||
|
||||
match expected_node_id {
|
||||
None => IdentityCheck::FirstContact {
|
||||
node_id: response.node_id.clone(),
|
||||
},
|
||||
Some(expected) if expected == response.node_id => IdentityCheck::Match,
|
||||
Some(_) => IdentityCheck::Mismatch {
|
||||
responder_node_id: response.node_id.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bft_json_crdt::keypair::make_keypair;
|
||||
|
||||
fn signed_response(nonce: &str) -> (String, IdentityProbeResponse) {
|
||||
let kp = make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
let sig = crate::node_identity::sign_challenge(&kp, nonce);
|
||||
(
|
||||
pubkey.clone(),
|
||||
IdentityProbeResponse {
|
||||
node_id: pubkey.clone(),
|
||||
pubkey,
|
||||
signature: Some(sig),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_contact_when_no_expected_id() {
|
||||
let nonce = "nonce-1";
|
||||
let (node_id, resp) = signed_response(nonce);
|
||||
let result = check_identity(None, nonce, &resp);
|
||||
assert_eq!(result, IdentityCheck::FirstContact { node_id });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_when_expected_id_equals_responder() {
|
||||
let nonce = "nonce-2";
|
||||
let (node_id, resp) = signed_response(nonce);
|
||||
let result = check_identity(Some(&node_id), nonce, &resp);
|
||||
assert_eq!(result, IdentityCheck::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mismatch_when_expected_id_differs() {
|
||||
let nonce = "nonce-3";
|
||||
let (node_id, resp) = signed_response(nonce);
|
||||
let result = check_identity(Some("some-other-node-id"), nonce, &resp);
|
||||
assert_eq!(
|
||||
result,
|
||||
IdentityCheck::Mismatch {
|
||||
responder_node_id: node_id
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_signature_wrong_nonce() {
|
||||
let nonce = "nonce-4";
|
||||
let (_node_id, resp) = signed_response(nonce);
|
||||
// Verify against a different nonce than the one that was signed.
|
||||
let result = check_identity(None, "different-nonce", &resp);
|
||||
assert_eq!(result, IdentityCheck::InvalidSignature);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_signature_wrong_key() {
|
||||
let nonce = "nonce-5";
|
||||
let (_node_id, mut resp) = signed_response(nonce);
|
||||
// Claim a different pubkey than the one that actually signed.
|
||||
let other_kp = make_keypair();
|
||||
let other_pubkey = crate::node_identity::public_key_hex(&other_kp);
|
||||
resp.node_id = other_pubkey.clone();
|
||||
resp.pubkey = other_pubkey;
|
||||
let result = check_identity(None, nonce, &resp);
|
||||
assert_eq!(result, IdentityCheck::InvalidSignature);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_signature_is_legacy_sled() {
|
||||
let resp = IdentityProbeResponse {
|
||||
node_id: "abc123".to_string(),
|
||||
pubkey: "abc123".to_string(),
|
||||
signature: None,
|
||||
};
|
||||
let result = check_identity(Some("abc123"), "nonce-6", &resp);
|
||||
assert_eq!(result, IdentityCheck::MissingSignature);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_signature_field_deserializes_from_json_without_it() {
|
||||
let json = r#"{"node_id":"abc","pubkey":"abc"}"#;
|
||||
let resp: IdentityProbeResponse = serde_json::from_str(json).unwrap();
|
||||
assert!(resp.signature.is_none());
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,32 @@ pub fn write_bot_config(config_dir: &Path, content: &str) -> Result<(), String>
|
||||
std::fs::write(&path, content).map_err(|e| format!("cannot write bot.toml: {e}"))
|
||||
}
|
||||
|
||||
// ── Identity probe I/O ───────────────────────────────────────────────────────
|
||||
|
||||
/// `GET {sled_url}/identity?nonce=<nonce>` and parse the JSON body.
|
||||
///
|
||||
/// Returns `None` when the sled is unreachable or the response body doesn't
|
||||
/// parse as [`super::identity::IdentityProbeResponse`] — callers treat that
|
||||
/// the same as an unverifiable identity (distinct from a legacy sled, which
|
||||
/// responds but omits the `signature` field).
|
||||
pub async fn probe_identity(
|
||||
client: &Client,
|
||||
sled_url: &str,
|
||||
nonce: &str,
|
||||
) -> Option<super::identity::IdentityProbeResponse> {
|
||||
// `nonce` is always a hex string (see `node_identity::generate_challenge`),
|
||||
// so no percent-encoding is needed for safe inclusion in the query string.
|
||||
let url = format!("{}/identity?nonce={nonce}", sled_url.trim_end_matches('/'));
|
||||
client
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.json::<super::identity::IdentityProbeResponse>()
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
// ── MCP proxy I/O ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Proxy a raw MCP request body to the given project URL.
|
||||
|
||||
@@ -11,14 +11,17 @@
|
||||
pub mod aggregation;
|
||||
/// Gateway configuration types and TOML parsing.
|
||||
pub mod config;
|
||||
/// Pure identity-probe verification (match/mismatch/first-contact) — no I/O.
|
||||
pub mod identity;
|
||||
pub(crate) mod io;
|
||||
/// Notification event polling for gateway-level broadcasts.
|
||||
pub mod polling;
|
||||
|
||||
pub use aggregation::format_aggregate_status_compact;
|
||||
pub use config::{GatewayConfig, ProjectEntry};
|
||||
pub use identity::{IdentityCheck, check_identity};
|
||||
pub use io::{
|
||||
fetch_all_project_pipeline_statuses, spawn_gateway_broadcaster_forwarder,
|
||||
fetch_all_project_pipeline_statuses, probe_identity, spawn_gateway_broadcaster_forwarder,
|
||||
spawn_gateway_notification_poller,
|
||||
};
|
||||
|
||||
@@ -781,6 +784,7 @@ mod tests {
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
@@ -921,6 +925,7 @@ mod tests {
|
||||
auth_token: Some("secret-token".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
|
||||
Reference in New Issue
Block a user