huskies: merge 1173 story Identity-aware fleet checks: cryptographic node identity in upgrade and health probes

server/src/service/gateway/config.rs

@@ -40,6 +40,14 @@ pub struct ProjectEntry {
     /// commands can route to the correct directory without re-deriving it.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub host_path: Option<String>,
+    /// Ed25519 node ID (hex pubkey) this sled is expected to answer as.
+    ///
+    /// Captured automatically from the first successful `/identity` probe
+    /// (story 1173) when absent, then checked on every subsequent upgrade and
+    /// health probe so a container swapped out from under the gateway is
+    /// detected even when its `/health` endpoint still reports ok.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub expected_node_id: Option<String>,
 }
 
 impl ProjectEntry {
@@ -52,6 +60,7 @@ impl ProjectEntry {
             auth_token: None,
             ssh_port: None,
             host_path: None,
+            expected_node_id: None,
         }
     }
 
@@ -223,6 +232,7 @@ auth_token = "secret"
                 auth_token: Some("secret".into()),
                 ssh_port: None,
                 host_path: None,
+                expected_node_id: None,
             },
         );
         let config = GatewayConfig {
@@ -258,6 +268,7 @@ auth_token = "secret"
                 auth_token: Some("tok".into()),
                 ssh_port: None,
                 host_path: None,
+                expected_node_id: None,
             },
         );
         assert_eq!(validate_project_exists(&projects, "ws").unwrap(), "");
@@ -278,6 +289,7 @@ auth_token = "secret"
             auth_token: Some("tok".into()),
             ssh_port: None,
             host_path: None,
+            expected_node_id: None,
         };
         assert!(!e.has_url());
     }
@@ -321,6 +333,7 @@ auth_token = "secret"
             auth_token: Some("mysecret".into()),
             ssh_port: None,
             host_path: None,
+            expected_node_id: None,
         };
         let mut projects = BTreeMap::new();
         projects.insert("myproj".into(), entry);
@@ -347,6 +360,7 @@ auth_token = "secret"
             auth_token: None,
             ssh_port: Some(2201),
             host_path: None,
+            expected_node_id: None,
         };
         let mut projects = BTreeMap::new();
         projects.insert("myproj".into(), entry);
@@ -364,6 +378,43 @@ auth_token = "secret"
         );
     }
 
+    #[test]
+    fn expected_node_id_roundtrips_and_is_omitted_when_none() {
+        let with_id = ProjectEntry {
+            url: Some("http://127.0.0.1:3101".into()),
+            auth_token: None,
+            ssh_port: None,
+            host_path: None,
+            expected_node_id: Some("ab".repeat(32)),
+        };
+        let mut projects = BTreeMap::new();
+        projects.insert("p".into(), with_id);
+        let config = GatewayConfig {
+            projects,
+            sled_tokens: BTreeMap::new(),
+        };
+        let toml_str = toml::to_string_pretty(&config).unwrap();
+        assert!(toml_str.contains("expected_node_id"));
+        let parsed: GatewayConfig = toml::from_str(&toml_str).unwrap();
+        assert_eq!(
+            parsed.projects["p"].expected_node_id.as_deref(),
+            Some("ab".repeat(32).as_str())
+        );
+
+        let without_id = ProjectEntry::with_url("http://127.0.0.1:3101");
+        let mut projects2 = BTreeMap::new();
+        projects2.insert("p".into(), without_id);
+        let config2 = GatewayConfig {
+            projects: projects2,
+            sled_tokens: BTreeMap::new(),
+        };
+        let toml_str2 = toml::to_string_pretty(&config2).unwrap();
+        assert!(
+            !toml_str2.contains("expected_node_id"),
+            "expected_node_id should be omitted when None: {toml_str2}"
+        );
+    }
+
     #[test]
     fn ssh_port_none_is_omitted_from_toml() {
         let entry = ProjectEntry::with_url("http://127.0.0.1:3101");

server/src/service/gateway/identity.rs

@@ -0,0 +1,175 @@
+//! Pure identity-probe verification — no I/O (story 1173).
+//!
+//! [`check_identity`] compares a sled's `/identity` probe response against the
+//! `expected_node_id` recorded for that project and reports one of: a first
+//! contact (no expected id captured yet), a match, a mismatch, an invalid
+//! signature, or a missing signature (a legacy sled predating this story).
+//! Callers ([`super::super::super::chat::transport::matrix::sled_upgrade`] and
+//! `health`) own all I/O — issuing the HTTP probe and persisting a captured
+//! `expected_node_id` back to `projects.toml`.
+
+use serde::Deserialize;
+
+/// Deserialized body of a `GET /identity?nonce=...` response.
+#[derive(Debug, Clone, Deserialize)]
+pub struct IdentityProbeResponse {
+    /// The responder's self-reported node ID (hex Ed25519 pubkey).
+    pub node_id: String,
+    /// The responder's self-reported Ed25519 public key (hex). Equal to
+    /// `node_id` on every server built since this field was introduced.
+    pub pubkey: String,
+    /// Ed25519 signature (hex) over the probe nonce, made with the CRDT
+    /// signing key. `None` when the responder predates story 1173 (a legacy
+    /// sled) — the `/identity` endpoint did not sign nonces before then.
+    #[serde(default)]
+    pub signature: Option<String>,
+}
+
+/// Outcome of comparing an [`IdentityProbeResponse`] against an
+/// `expected_node_id`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum IdentityCheck {
+    /// No `expected_node_id` was recorded yet; the signature verified, so
+    /// `node_id` should be captured as the expected identity going forward.
+    FirstContact {
+        /// The node ID to capture.
+        node_id: String,
+    },
+    /// The signature verified and the responder's node ID matches the
+    /// expected one.
+    Match,
+    /// The signature verified, but the responder's node ID does not match
+    /// the expected one — a different container answered.
+    Mismatch {
+        /// The verified node ID that actually answered.
+        responder_node_id: String,
+    },
+    /// The response included a signature, but it did not verify against the
+    /// claimed pubkey/nonce — the responder's identity cannot be trusted.
+    InvalidSignature,
+    /// The response had no `signature` field — a legacy sled running a
+    /// pre-story-1173 binary. Identity cannot be verified either way.
+    MissingSignature,
+}
+
+/// Verify `response` against `nonce` and compare the verified node ID to
+/// `expected_node_id`.
+///
+/// A signature verifies when [`crate::node_identity::verify_message_strict`]
+/// confirms it was produced by the private key matching `response.pubkey`
+/// over `nonce`'s UTF-8 bytes — mirroring how `/identity` signs on the server
+/// side via `crdt_state::sign_challenge`.
+pub fn check_identity(
+    expected_node_id: Option<&str>,
+    nonce: &str,
+    response: &IdentityProbeResponse,
+) -> IdentityCheck {
+    let Some(signature) = response.signature.as_deref() else {
+        return IdentityCheck::MissingSignature;
+    };
+
+    if !crate::node_identity::verify_message_strict(&response.pubkey, nonce.as_bytes(), signature) {
+        return IdentityCheck::InvalidSignature;
+    }
+
+    match expected_node_id {
+        None => IdentityCheck::FirstContact {
+            node_id: response.node_id.clone(),
+        },
+        Some(expected) if expected == response.node_id => IdentityCheck::Match,
+        Some(_) => IdentityCheck::Mismatch {
+            responder_node_id: response.node_id.clone(),
+        },
+    }
+}
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bft_json_crdt::keypair::make_keypair;
+
+    fn signed_response(nonce: &str) -> (String, IdentityProbeResponse) {
+        let kp = make_keypair();
+        let pubkey = crate::node_identity::public_key_hex(&kp);
+        let sig = crate::node_identity::sign_challenge(&kp, nonce);
+        (
+            pubkey.clone(),
+            IdentityProbeResponse {
+                node_id: pubkey.clone(),
+                pubkey,
+                signature: Some(sig),
+            },
+        )
+    }
+
+    #[test]
+    fn first_contact_when_no_expected_id() {
+        let nonce = "nonce-1";
+        let (node_id, resp) = signed_response(nonce);
+        let result = check_identity(None, nonce, &resp);
+        assert_eq!(result, IdentityCheck::FirstContact { node_id });
+    }
+
+    #[test]
+    fn match_when_expected_id_equals_responder() {
+        let nonce = "nonce-2";
+        let (node_id, resp) = signed_response(nonce);
+        let result = check_identity(Some(&node_id), nonce, &resp);
+        assert_eq!(result, IdentityCheck::Match);
+    }
+
+    #[test]
+    fn mismatch_when_expected_id_differs() {
+        let nonce = "nonce-3";
+        let (node_id, resp) = signed_response(nonce);
+        let result = check_identity(Some("some-other-node-id"), nonce, &resp);
+        assert_eq!(
+            result,
+            IdentityCheck::Mismatch {
+                responder_node_id: node_id
+            }
+        );
+    }
+
+    #[test]
+    fn invalid_signature_wrong_nonce() {
+        let nonce = "nonce-4";
+        let (_node_id, resp) = signed_response(nonce);
+        // Verify against a different nonce than the one that was signed.
+        let result = check_identity(None, "different-nonce", &resp);
+        assert_eq!(result, IdentityCheck::InvalidSignature);
+    }
+
+    #[test]
+    fn invalid_signature_wrong_key() {
+        let nonce = "nonce-5";
+        let (_node_id, mut resp) = signed_response(nonce);
+        // Claim a different pubkey than the one that actually signed.
+        let other_kp = make_keypair();
+        let other_pubkey = crate::node_identity::public_key_hex(&other_kp);
+        resp.node_id = other_pubkey.clone();
+        resp.pubkey = other_pubkey;
+        let result = check_identity(None, nonce, &resp);
+        assert_eq!(result, IdentityCheck::InvalidSignature);
+    }
+
+    #[test]
+    fn missing_signature_is_legacy_sled() {
+        let resp = IdentityProbeResponse {
+            node_id: "abc123".to_string(),
+            pubkey: "abc123".to_string(),
+            signature: None,
+        };
+        let result = check_identity(Some("abc123"), "nonce-6", &resp);
+        assert_eq!(result, IdentityCheck::MissingSignature);
+    }
+
+    #[test]
+    fn missing_signature_field_deserializes_from_json_without_it() {
+        let json = r#"{"node_id":"abc","pubkey":"abc"}"#;
+        let resp: IdentityProbeResponse = serde_json::from_str(json).unwrap();
+        assert!(resp.signature.is_none());
+    }
+}

server/src/service/gateway/io.rs

@@ -94,6 +94,32 @@ pub fn write_bot_config(config_dir: &Path, content: &str) -> Result<(), String>
     std::fs::write(&path, content).map_err(|e| format!("cannot write bot.toml: {e}"))
 }
 
+// ── Identity probe I/O ───────────────────────────────────────────────────────
+
+/// `GET {sled_url}/identity?nonce=<nonce>` and parse the JSON body.
+///
+/// Returns `None` when the sled is unreachable or the response body doesn't
+/// parse as [`super::identity::IdentityProbeResponse`] — callers treat that
+/// the same as an unverifiable identity (distinct from a legacy sled, which
+/// responds but omits the `signature` field).
+pub async fn probe_identity(
+    client: &Client,
+    sled_url: &str,
+    nonce: &str,
+) -> Option<super::identity::IdentityProbeResponse> {
+    // `nonce` is always a hex string (see `node_identity::generate_challenge`),
+    // so no percent-encoding is needed for safe inclusion in the query string.
+    let url = format!("{}/identity?nonce={nonce}", sled_url.trim_end_matches('/'));
+    client
+        .get(&url)
+        .send()
+        .await
+        .ok()?
+        .json::<super::identity::IdentityProbeResponse>()
+        .await
+        .ok()
+}
+
 // ── MCP proxy I/O ───────────────────────────────────────────────────────────
 
 /// Proxy a raw MCP request body to the given project URL.

server/src/service/gateway/mod.rs

@@ -11,14 +11,17 @@
 pub mod aggregation;
 /// Gateway configuration types and TOML parsing.
 pub mod config;
+/// Pure identity-probe verification (match/mismatch/first-contact) — no I/O.
+pub mod identity;
 pub(crate) mod io;
 /// Notification event polling for gateway-level broadcasts.
 pub mod polling;
 
 pub use aggregation::format_aggregate_status_compact;
 pub use config::{GatewayConfig, ProjectEntry};
+pub use identity::{IdentityCheck, check_identity};
 pub use io::{
-    fetch_all_project_pipeline_statuses, spawn_gateway_broadcaster_forwarder,
+    fetch_all_project_pipeline_statuses, probe_identity, spawn_gateway_broadcaster_forwarder,
     spawn_gateway_notification_poller,
 };
 
@@ -781,6 +784,7 @@ mod tests {
                 auth_token: Some("tok".into()),
                 ssh_port: None,
                 host_path: None,
+                expected_node_id: None,
             },
         );
         let config = GatewayConfig {
@@ -921,6 +925,7 @@ mod tests {
                 auth_token: Some("secret-token".into()),
                 ssh_port: None,
                 host_path: None,
+                expected_node_id: None,
             },
         );
         let config = GatewayConfig {