huskies: merge 1206 story fleet_identity MCP tool: read sled pins vs live signed identity, and re-pin via TOFU
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
//! transports (Matrix, Slack). Uses `service::pipeline::aggregate_pipeline_counts`
|
||||
//! for per-project parsing.
|
||||
|
||||
use super::identity::SledIdentityReport;
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -125,6 +126,54 @@ pub fn format_overview_compact(items_by_project: &BTreeMap<String, Value>) -> St
|
||||
format!("**Overview: Active Work**\n\n{}", sections.join("\n\n"))
|
||||
}
|
||||
|
||||
/// Format `fleet_identity` read-mode reports as Markdown, one line per sled.
|
||||
///
|
||||
/// Matches, first-contacts (no pin recorded yet), and unreachable sleds get a
|
||||
/// plain status line. A verified mismatch is called out with the same
|
||||
/// wording as the `upgrade` command's identity check
|
||||
/// (`chat::transport::matrix::sled_upgrade::verify_sled_identity`) so an
|
||||
/// operator sees one consistent message for "wrong container answered"
|
||||
/// regardless of which command surfaced it — including naming the container
|
||||
/// as `huskies-{project}`.
|
||||
pub fn format_identity_reports(reports: &[SledIdentityReport]) -> String {
|
||||
if reports.is_empty() {
|
||||
return "No projects registered.".to_string();
|
||||
}
|
||||
|
||||
let lines: Vec<String> = reports
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let container_name = format!("huskies-{}", r.project);
|
||||
let url = r.url.as_deref().unwrap_or("(no url configured)");
|
||||
|
||||
if !r.connected {
|
||||
return format!("\u{1F534} **{}** — unreachable at `{url}`", r.project);
|
||||
}
|
||||
|
||||
match (&r.expected_pin, &r.live_node_id, r.matched) {
|
||||
(_, _, true) => format!("\u{1F7E2} **{}** — matches pin `{url}`", r.project),
|
||||
(Some(expected), Some(live), false) => format!(
|
||||
"\u{1F7E0} **identity mismatch** for `{container_name}` at `{url}`: expected \
|
||||
node_id `{expected}`, but the container that answered identified as \
|
||||
`{live}`. Refusing to proceed — this may not be the sled you expect."
|
||||
),
|
||||
(None, Some(live), false) => format!(
|
||||
"\u{1F7E1} **{}** — no pin recorded yet at `{url}`; live node_id `{live}` \
|
||||
(re-pin to trust it)",
|
||||
r.project
|
||||
),
|
||||
(_, None, false) => format!(
|
||||
"\u{1F534} **identity verification failed** for `{container_name}` at `{url}`: \
|
||||
the `/identity` response's signature is missing or did not verify. \
|
||||
Refusing to proceed — the container's identity cannot be trusted."
|
||||
),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
format!("**Fleet Identity**\n\n{}", lines.join("\n"))
|
||||
}
|
||||
|
||||
/// Find the registered project whose pipeline contains a story with the
|
||||
/// given numeric ID prefix, searching `active`, `backlog`, and `archived`
|
||||
/// alike so gateway `status <n>` resolves regardless of which project is
|
||||
@@ -313,6 +362,95 @@ mod tests {
|
||||
assert!(output.find("alpha").unwrap() < output.find("beta").unwrap());
|
||||
}
|
||||
|
||||
// ── format_identity_reports (story 1206 AC3) ───────────────────────────
|
||||
|
||||
#[test]
|
||||
fn identity_report_mismatch_names_container_consistent_with_upgrade_sweep() {
|
||||
let reports = vec![SledIdentityReport {
|
||||
project: "myapp".to_string(),
|
||||
url: Some("http://sled:3001".to_string()),
|
||||
connected: true,
|
||||
expected_pin: Some("expected-id".to_string()),
|
||||
live_node_id: Some("different-id".to_string()),
|
||||
matched: false,
|
||||
}];
|
||||
let output = format_identity_reports(&reports);
|
||||
assert!(output.contains("**identity mismatch**"));
|
||||
assert!(
|
||||
output.contains("`huskies-myapp`"),
|
||||
"must name the container as huskies-<project>: {output}"
|
||||
);
|
||||
assert!(output.contains("expected node_id `expected-id`"));
|
||||
assert!(output.contains("identified as `different-id`"));
|
||||
assert!(output.contains("Refusing to proceed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_report_match_is_a_plain_status_line() {
|
||||
let reports = vec![SledIdentityReport {
|
||||
project: "myapp".to_string(),
|
||||
url: Some("http://sled:3001".to_string()),
|
||||
connected: true,
|
||||
expected_pin: Some("abc".to_string()),
|
||||
live_node_id: Some("abc".to_string()),
|
||||
matched: true,
|
||||
}];
|
||||
let output = format_identity_reports(&reports);
|
||||
assert!(output.contains("myapp"));
|
||||
assert!(output.contains("matches pin"));
|
||||
assert!(!output.contains("mismatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_report_unreachable_sled_is_marked() {
|
||||
let reports = vec![SledIdentityReport {
|
||||
project: "myapp".to_string(),
|
||||
url: Some("http://sled:3001".to_string()),
|
||||
connected: false,
|
||||
expected_pin: Some("abc".to_string()),
|
||||
live_node_id: None,
|
||||
matched: false,
|
||||
}];
|
||||
let output = format_identity_reports(&reports);
|
||||
assert!(output.contains("unreachable"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_report_no_pin_recorded_invites_a_repin() {
|
||||
let reports = vec![SledIdentityReport {
|
||||
project: "myapp".to_string(),
|
||||
url: Some("http://sled:3001".to_string()),
|
||||
connected: true,
|
||||
expected_pin: None,
|
||||
live_node_id: Some("some-id".to_string()),
|
||||
matched: false,
|
||||
}];
|
||||
let output = format_identity_reports(&reports);
|
||||
assert!(output.contains("no pin recorded"));
|
||||
assert!(output.contains("some-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_report_invalid_signature_is_flagged() {
|
||||
let reports = vec![SledIdentityReport {
|
||||
project: "myapp".to_string(),
|
||||
url: Some("http://sled:3001".to_string()),
|
||||
connected: true,
|
||||
expected_pin: Some("abc".to_string()),
|
||||
live_node_id: None,
|
||||
matched: false,
|
||||
}];
|
||||
let output = format_identity_reports(&reports);
|
||||
assert!(output.contains("**identity verification failed**"));
|
||||
assert!(output.contains("`huskies-myapp`"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_report_empty_list() {
|
||||
let output = format_identity_reports(&[]);
|
||||
assert_eq!(output, "No projects registered.");
|
||||
}
|
||||
|
||||
// ── find_project_containing_story ──────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -83,6 +83,63 @@ pub fn check_identity(
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fleet identity read report (story 1206) ────────────────────────────────
|
||||
|
||||
/// Per-sled identity report returned by the `fleet_identity` MCP tool's read
|
||||
/// mode: the recorded pin next to the live, cryptographically-verified
|
||||
/// identity.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct SledIdentityReport {
|
||||
/// The project/sled name.
|
||||
pub project: String,
|
||||
/// The sled's configured base URL, if any.
|
||||
pub url: Option<String>,
|
||||
/// Whether the `/identity` probe reached the sled at all.
|
||||
pub connected: bool,
|
||||
/// The pinned `expected_node_id` recorded in `projects.toml`, if any.
|
||||
pub expected_pin: Option<String>,
|
||||
/// The node ID from a signature-verified challenge-response, if the
|
||||
/// signature verified. Never populated from the unsigned `node_id`
|
||||
/// display field alone — an invalid or missing signature leaves this
|
||||
/// `None`.
|
||||
pub live_node_id: Option<String>,
|
||||
/// `true` only when the sled is connected, its signature verified, and
|
||||
/// the verified node ID equals `expected_pin`.
|
||||
pub matched: bool,
|
||||
}
|
||||
|
||||
/// Build a [`SledIdentityReport`] from the outcome of probing one sled.
|
||||
///
|
||||
/// `check` is `None` when the sled was unreachable (the probe never got a
|
||||
/// response). Otherwise it's the result of running [`check_identity`] against
|
||||
/// whatever response was received.
|
||||
pub fn build_identity_report(
|
||||
project: &str,
|
||||
url: Option<&str>,
|
||||
expected_node_id: Option<&str>,
|
||||
check: Option<IdentityCheck>,
|
||||
) -> SledIdentityReport {
|
||||
let connected = check.is_some();
|
||||
let (live_node_id, matched) = match &check {
|
||||
Some(IdentityCheck::Match) => (expected_node_id.map(str::to_string), true),
|
||||
Some(IdentityCheck::Mismatch { responder_node_id }) => {
|
||||
(Some(responder_node_id.clone()), false)
|
||||
}
|
||||
Some(IdentityCheck::FirstContact { node_id }) => (Some(node_id.clone()), false),
|
||||
Some(IdentityCheck::InvalidSignature) | Some(IdentityCheck::MissingSignature) | None => {
|
||||
(None, false)
|
||||
}
|
||||
};
|
||||
SledIdentityReport {
|
||||
project: project.to_string(),
|
||||
url: url.map(str::to_string),
|
||||
connected,
|
||||
expected_pin: expected_node_id.map(str::to_string),
|
||||
live_node_id,
|
||||
matched,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -172,4 +229,79 @@ mod tests {
|
||||
let resp: IdentityProbeResponse = serde_json::from_str(json).unwrap();
|
||||
assert!(resp.signature.is_none());
|
||||
}
|
||||
|
||||
// ── build_identity_report (story 1206) ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn report_matching_pin_is_connected_and_matched() {
|
||||
let report = build_identity_report(
|
||||
"myapp",
|
||||
Some("http://sled:3001"),
|
||||
Some("abc123"),
|
||||
Some(IdentityCheck::Match),
|
||||
);
|
||||
assert!(report.connected);
|
||||
assert!(report.matched);
|
||||
assert_eq!(report.expected_pin.as_deref(), Some("abc123"));
|
||||
assert_eq!(report.live_node_id.as_deref(), Some("abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_mismatch_names_the_responder_node_id_and_is_not_matched() {
|
||||
let report = build_identity_report(
|
||||
"myapp",
|
||||
Some("http://sled:3001"),
|
||||
Some("expected-id"),
|
||||
Some(IdentityCheck::Mismatch {
|
||||
responder_node_id: "different-id".to_string(),
|
||||
}),
|
||||
);
|
||||
assert!(report.connected);
|
||||
assert!(!report.matched);
|
||||
assert_eq!(report.expected_pin.as_deref(), Some("expected-id"));
|
||||
assert_eq!(report.live_node_id.as_deref(), Some("different-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_no_pin_recorded_is_never_matched_even_on_first_contact() {
|
||||
let report = build_identity_report(
|
||||
"myapp",
|
||||
Some("http://sled:3001"),
|
||||
None,
|
||||
Some(IdentityCheck::FirstContact {
|
||||
node_id: "some-id".to_string(),
|
||||
}),
|
||||
);
|
||||
assert!(report.connected);
|
||||
assert!(report.expected_pin.is_none());
|
||||
assert_eq!(report.live_node_id.as_deref(), Some("some-id"));
|
||||
assert!(
|
||||
!report.matched,
|
||||
"no pin recorded yet means there is nothing to match against"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_unreachable_sled_is_not_connected() {
|
||||
let report = build_identity_report("myapp", Some("http://sled:3001"), Some("abc123"), None);
|
||||
assert!(!report.connected);
|
||||
assert!(!report.matched);
|
||||
assert!(report.live_node_id.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_invalid_signature_is_connected_but_no_live_node_id() {
|
||||
let report = build_identity_report(
|
||||
"myapp",
|
||||
Some("http://sled:3001"),
|
||||
Some("abc123"),
|
||||
Some(IdentityCheck::InvalidSignature),
|
||||
);
|
||||
assert!(report.connected, "the probe did reach the sled");
|
||||
assert!(!report.matched);
|
||||
assert!(
|
||||
report.live_node_id.is_none(),
|
||||
"an unverified signature must never populate live_node_id"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,6 +128,98 @@ pub async fn probe_identity(
|
||||
.ok()
|
||||
}
|
||||
|
||||
// ── Fleet identity I/O (story 1206) ─────────────────────────────────────────
|
||||
|
||||
/// Probe one sled's `/identity` endpoint and build its
|
||||
/// [`super::identity::SledIdentityReport`].
|
||||
///
|
||||
/// Generates a fresh challenge nonce, probes `entry.url`, and compares the
|
||||
/// (signature-verified) response against `entry.expected_node_id`. When
|
||||
/// `entry.url` is unset the sled is reported unreachable without a network
|
||||
/// call.
|
||||
pub async fn probe_sled_identity_report(
|
||||
project: &str,
|
||||
entry: &ProjectEntry,
|
||||
client: &Client,
|
||||
) -> super::identity::SledIdentityReport {
|
||||
let Some(url) = entry.url.as_deref() else {
|
||||
return super::identity::build_identity_report(
|
||||
project,
|
||||
None,
|
||||
entry.expected_node_id.as_deref(),
|
||||
None,
|
||||
);
|
||||
};
|
||||
|
||||
let nonce = crate::node_identity::generate_challenge();
|
||||
let check = probe_identity(client, url, &nonce).await.map(|response| {
|
||||
super::check_identity(entry.expected_node_id.as_deref(), &nonce, &response)
|
||||
});
|
||||
|
||||
super::identity::build_identity_report(
|
||||
project,
|
||||
Some(url),
|
||||
entry.expected_node_id.as_deref(),
|
||||
check,
|
||||
)
|
||||
}
|
||||
|
||||
/// Re-pin a sled's expected identity via TOFU (trust-on-first-use).
|
||||
///
|
||||
/// Probes `sled_url`, and — only when the response's signature verifies —
|
||||
/// captures the verified node ID as the new pin, overwriting whatever was
|
||||
/// previously recorded (including a mismatched one; that's the point of an
|
||||
/// explicit re-pin). Persists the change to both the live `projects_store`
|
||||
/// and `projects.toml` at `config_dir`.
|
||||
///
|
||||
/// Refuses (returns `Err`) when the sled is unreachable, its response has no
|
||||
/// signature (a legacy sled), or the signature fails to verify — in none of
|
||||
/// those cases is a responder's claimed identity trustworthy enough to pin.
|
||||
pub async fn repin_sled_identity(
|
||||
project: &str,
|
||||
sled_url: &str,
|
||||
projects_store: &std::sync::Arc<tokio::sync::RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||
config_dir: &Path,
|
||||
client: &Client,
|
||||
) -> Result<String, String> {
|
||||
let nonce = crate::node_identity::generate_challenge();
|
||||
let Some(response) = probe_identity(client, sled_url, &nonce).await else {
|
||||
return Err(format!(
|
||||
"cannot re-pin `{project}` at `{sled_url}`: the sled is unreachable"
|
||||
));
|
||||
};
|
||||
|
||||
// Compare against `None` regardless of any existing pin — an explicit
|
||||
// re-pin always (re-)captures on a verified signature rather than
|
||||
// reporting a `Mismatch` against the old pin.
|
||||
match super::check_identity(None, &nonce, &response) {
|
||||
super::IdentityCheck::FirstContact { node_id } => {
|
||||
{
|
||||
let mut projects = projects_store.write().await;
|
||||
if let Some(entry) = projects.get_mut(project) {
|
||||
entry.expected_node_id = Some(node_id.clone());
|
||||
}
|
||||
}
|
||||
let snapshot = projects_store.read().await.clone();
|
||||
save_config(&snapshot, config_dir).await;
|
||||
Ok(node_id)
|
||||
}
|
||||
super::IdentityCheck::InvalidSignature => Err(format!(
|
||||
"refusing to pin `{project}` at `{sled_url}`: the `/identity` response signature \
|
||||
did not verify — the responder's identity cannot be trusted"
|
||||
)),
|
||||
super::IdentityCheck::MissingSignature => Err(format!(
|
||||
"refusing to pin `{project}` at `{sled_url}`: the response had no signature \
|
||||
(a legacy sled predating signed identity) — identity cannot be verified"
|
||||
)),
|
||||
super::IdentityCheck::Match | super::IdentityCheck::Mismatch { .. } => {
|
||||
unreachable!(
|
||||
"check_identity(None, ..) only returns FirstContact/InvalidSignature/MissingSignature"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Release channel I/O (story 1169) ────────────────────────────────────────
|
||||
|
||||
/// Fetch `{base_url}/manifest.json` and parse it into a
|
||||
@@ -928,4 +1020,277 @@ mod tests {
|
||||
};
|
||||
assert_eq!(dedupe_key(&transition), None);
|
||||
}
|
||||
|
||||
// ── fleet identity (story 1206) ──────────────────────────────────────────
|
||||
|
||||
/// Spawn a one-shot TCP listener that answers a single `GET
|
||||
/// /identity?nonce=<hex>` request with a JSON body signed by `kp` over
|
||||
/// whatever nonce the caller actually sent. Mirrors the responder used in
|
||||
/// `chat::transport::matrix::sled_upgrade`'s tests.
|
||||
fn spawn_identity_responder(
|
||||
listener: tokio::net::TcpListener,
|
||||
kp: bft_json_crdt::keypair::Ed25519KeyPair,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let nonce = req
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split("nonce=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split_whitespace().next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
|
||||
let body = serde_json::json!({
|
||||
"node_id": pubkey,
|
||||
"pubkey": pubkey,
|
||||
"signature": sig,
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Spawn a one-shot TCP listener that answers `GET /identity` with an
|
||||
/// unsigned body (no `signature` field) — a legacy sled.
|
||||
fn spawn_unsigned_identity_responder(listener: tokio::net::TcpListener) {
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let _ = stream.read(&mut buf).await;
|
||||
let body = serde_json::json!({
|
||||
"node_id": "legacy-node-id",
|
||||
"pubkey": "legacy-node-id",
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_sled_identity_report_matches_recorded_pin() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let entry = ProjectEntry {
|
||||
url: Some(format!("http://127.0.0.1:{port}")),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some(pubkey.clone()),
|
||||
};
|
||||
let client = reqwest::Client::new();
|
||||
let report = probe_sled_identity_report("myapp", &entry, &client).await;
|
||||
|
||||
assert!(report.connected);
|
||||
assert!(report.matched);
|
||||
assert_eq!(report.live_node_id.as_deref(), Some(pubkey.as_str()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_sled_identity_report_no_url_is_unreachable() {
|
||||
let entry = ProjectEntry {
|
||||
url: None,
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some("abc123".to_string()),
|
||||
};
|
||||
let client = reqwest::Client::new();
|
||||
let report = probe_sled_identity_report("myapp", &entry, &client).await;
|
||||
assert!(!report.connected);
|
||||
assert!(!report.matched);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repin_sled_identity_captures_verified_node_id_and_persists() {
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_identity_responder(listener, kp);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
// A stale/mismatched pin — re-pin must overwrite it.
|
||||
expected_node_id: Some("stale-node-id".to_string()),
|
||||
},
|
||||
);
|
||||
let store = std::sync::Arc::new(tokio::sync::RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
repin_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
assert_eq!(result, Ok(pubkey.clone()));
|
||||
|
||||
let captured = store
|
||||
.read()
|
||||
.await
|
||||
.get("myapp")
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
assert_eq!(captured, Some(pubkey.clone()));
|
||||
|
||||
let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await;
|
||||
assert!(
|
||||
toml_content.unwrap_or_default().contains(&pubkey),
|
||||
"re-pinned node_id should be persisted to projects.toml"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repin_sled_identity_refuses_invalid_signature() {
|
||||
// A responder that signs with a different key than it claims — an
|
||||
// untrustworthy identity that must never be pinned.
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let other_kp = bft_json_crdt::keypair::make_keypair();
|
||||
let claimed_pubkey = crate::node_identity::public_key_hex(&other_kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let nonce = req
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split("nonce=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split_whitespace().next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
// Sign with `kp` but claim `other_kp`'s pubkey — verification must fail.
|
||||
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
|
||||
let body = serde_json::json!({
|
||||
"node_id": claimed_pubkey,
|
||||
"pubkey": claimed_pubkey,
|
||||
"signature": sig,
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = std::sync::Arc::new(tokio::sync::RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
repin_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
let err = result.expect_err("invalid signature must refuse to pin");
|
||||
assert!(err.contains("refusing to pin"), "error: {err}");
|
||||
assert!(
|
||||
err.contains("myapp"),
|
||||
"error should name the project: {err}"
|
||||
);
|
||||
|
||||
let captured = store
|
||||
.read()
|
||||
.await
|
||||
.get("myapp")
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
assert!(captured.is_none(), "no pin should be captured on refusal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repin_sled_identity_refuses_missing_signature() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
spawn_unsigned_identity_responder(listener);
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut map = BTreeMap::new();
|
||||
map.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: Some(sled_url.clone()),
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let store = std::sync::Arc::new(tokio::sync::RwLock::new(map));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let result =
|
||||
repin_sled_identity("myapp", &sled_url, &store, config_dir.path(), &client).await;
|
||||
let err = result.expect_err("missing signature must refuse to pin");
|
||||
assert!(err.contains("refusing to pin"), "error: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repin_sled_identity_refuses_unreachable_sled() {
|
||||
let store = std::sync::Arc::new(tokio::sync::RwLock::new(BTreeMap::new()));
|
||||
let config_dir = tempfile::tempdir().unwrap();
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_millis(200))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let result = repin_sled_identity(
|
||||
"myapp",
|
||||
"http://127.0.0.1:1",
|
||||
&store,
|
||||
config_dir.path(),
|
||||
&client,
|
||||
)
|
||||
.await;
|
||||
let err = result.expect_err("unreachable sled must refuse to pin");
|
||||
assert!(err.contains("unreachable"), "error: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,11 @@ pub mod polling;
|
||||
pub mod release_manifest;
|
||||
|
||||
pub use aggregation::{
|
||||
find_project_containing_story, format_aggregate_status_compact, format_overview_compact,
|
||||
find_project_containing_story, format_aggregate_status_compact, format_identity_reports,
|
||||
format_overview_compact,
|
||||
};
|
||||
pub use config::{GatewayConfig, ProjectEntry};
|
||||
pub use identity::{IdentityCheck, check_identity};
|
||||
pub use identity::{IdentityCheck, SledIdentityReport, check_identity};
|
||||
pub use io::{
|
||||
fetch_all_project_pipeline_items, fetch_all_project_pipeline_statuses, probe_identity,
|
||||
spawn_gateway_broadcaster_forwarder,
|
||||
@@ -697,6 +698,59 @@ pub fn subscribe_status_events(
|
||||
state.event_tx.subscribe()
|
||||
}
|
||||
|
||||
// ── Fleet identity (story 1206) ─────────────────────────────────────────────
|
||||
|
||||
/// Read mode for the `fleet_identity` MCP tool: probe every registered
|
||||
/// project's `/identity` endpoint and report the recorded pin next to the
|
||||
/// live, signature-verified identity.
|
||||
///
|
||||
/// Projects are probed concurrently (bounded by however many are
|
||||
/// registered — gateways register at most a handful of sleds).
|
||||
pub async fn fleet_identity_read(state: &GatewayState) -> Vec<SledIdentityReport> {
|
||||
let projects = state.projects.read().await.clone();
|
||||
let probes = projects
|
||||
.iter()
|
||||
.map(|(name, entry)| io::probe_sled_identity_report(name, entry, &state.client));
|
||||
futures::future::join_all(probes).await
|
||||
}
|
||||
|
||||
/// Re-pin action for the `fleet_identity` MCP tool: capture `project`'s
|
||||
/// live, cryptographically-verified identity via TOFU and persist it as the
|
||||
/// new expected pin.
|
||||
///
|
||||
/// **Does this need a gateway restart?** No (verified against
|
||||
/// [`GatewayState`]'s design, AC 4). `GatewayState::projects` is the single
|
||||
/// `Arc<RwLock<BTreeMap<String, ProjectEntry>>>` that every gateway code path
|
||||
/// reads fresh on each call — [`GatewayState::active_url`], the MCP proxy,
|
||||
/// `/health` polling, and the `upgrade` command's own identity check
|
||||
/// (`verify_sled_identity`) all call `.read().await` on it rather than
|
||||
/// caching a snapshot at startup. [`io::repin_sled_identity`] writes through
|
||||
/// this same `Arc` and persists to `projects.toml`, so the next call from any
|
||||
/// of those paths — with no restart — sees the new pin. This mirrors how
|
||||
/// `verify_sled_identity`'s first-contact capture already updates the pin
|
||||
/// live during an `upgrade` without requiring a restart.
|
||||
pub async fn fleet_identity_repin(state: &GatewayState, project: &str) -> Result<String, Error> {
|
||||
let url = {
|
||||
let projects = state.projects.read().await;
|
||||
config::validate_project_exists(&projects, project).map_err(Error::ProjectNotFound)?
|
||||
};
|
||||
if url.is_empty() {
|
||||
return Err(Error::Config(format!(
|
||||
"project '{project}' has no URL configured; cannot probe it for re-pinning"
|
||||
)));
|
||||
}
|
||||
|
||||
io::repin_sled_identity(
|
||||
project,
|
||||
&url,
|
||||
&state.projects,
|
||||
&state.config_dir,
|
||||
&state.client,
|
||||
)
|
||||
.await
|
||||
.map_err(Error::Upstream)
|
||||
}
|
||||
|
||||
/// Save bot config and restart the bot.
|
||||
pub async fn save_bot_config_and_restart(state: &GatewayState, content: &str) -> Result<(), Error> {
|
||||
io::write_bot_config(&state.config_dir, content).map_err(Error::Config)?;
|
||||
@@ -958,4 +1012,132 @@ mod tests {
|
||||
"Per-project auth_token must be in reversed sled_tokens map"
|
||||
);
|
||||
}
|
||||
|
||||
// ── fleet_identity (story 1206) ─────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn fleet_identity_read_reports_unreachable_project_with_no_url() {
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert(
|
||||
"myapp".to_string(),
|
||||
ProjectEntry {
|
||||
url: None,
|
||||
auth_token: None,
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: Some("abc".to_string()),
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
projects,
|
||||
sled_tokens: BTreeMap::new(),
|
||||
release_channels: BTreeMap::new(),
|
||||
};
|
||||
let state = GatewayState::new(config, PathBuf::new(), 3000).unwrap();
|
||||
let reports = fleet_identity_read(&state).await;
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].project, "myapp");
|
||||
assert!(!reports[0].connected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fleet_identity_repin_unknown_project_fails() {
|
||||
let config = make_config(&[("alpha", "http://a:3001")]);
|
||||
let state = GatewayState::new(config, PathBuf::new(), 3000).unwrap();
|
||||
let result = fleet_identity_repin(&state, "nonexistent").await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fleet_identity_repin_project_without_url_fails() {
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert(
|
||||
"ws-only".to_string(),
|
||||
ProjectEntry {
|
||||
url: None,
|
||||
auth_token: Some("tok".into()),
|
||||
ssh_port: None,
|
||||
host_path: None,
|
||||
expected_node_id: None,
|
||||
},
|
||||
);
|
||||
let config = GatewayConfig {
|
||||
projects,
|
||||
sled_tokens: BTreeMap::new(),
|
||||
release_channels: BTreeMap::new(),
|
||||
};
|
||||
let state = GatewayState::new(config, PathBuf::new(), 3000).unwrap();
|
||||
let result = fleet_identity_repin(&state, "ws-only").await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fleet_identity_repin_updates_live_state_without_restart() {
|
||||
// Regression test for AC 4: re-pinning must be visible to a *second*,
|
||||
// independent read through the same `GatewayState` without any
|
||||
// restart step in between — proving the pin lives in the shared
|
||||
// in-memory `Arc<RwLock<..>>`, not some snapshot taken at startup.
|
||||
let kp = bft_json_crdt::keypair::make_keypair();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
tokio::spawn(async move {
|
||||
if let Ok((mut stream, _)) = listener.accept().await {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = stream.read(&mut buf).await.unwrap_or(0);
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let nonce = req
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split("nonce=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split_whitespace().next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let pubkey = crate::node_identity::public_key_hex(&kp);
|
||||
let sig = crate::node_identity::sign_challenge(&kp, &nonce);
|
||||
let body = serde_json::json!({
|
||||
"node_id": pubkey,
|
||||
"pubkey": pubkey,
|
||||
"signature": sig,
|
||||
})
|
||||
.to_string();
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
let _ = stream.write_all(response.as_bytes()).await;
|
||||
}
|
||||
});
|
||||
|
||||
let sled_url = format!("http://127.0.0.1:{port}");
|
||||
let mut projects = BTreeMap::new();
|
||||
projects.insert("myapp".to_string(), ProjectEntry::with_url(&sled_url));
|
||||
let config = GatewayConfig {
|
||||
projects,
|
||||
sled_tokens: BTreeMap::new(),
|
||||
release_channels: BTreeMap::new(),
|
||||
};
|
||||
let state = GatewayState::new(config, PathBuf::new(), 3000).unwrap();
|
||||
|
||||
let repinned = fleet_identity_repin(&state, "myapp").await.unwrap();
|
||||
assert_eq!(repinned, pubkey);
|
||||
|
||||
// No restart, no new GatewayState — read straight off the same `state`.
|
||||
let pin = state
|
||||
.projects
|
||||
.read()
|
||||
.await
|
||||
.get("myapp")
|
||||
.and_then(|e| e.expected_node_id.clone());
|
||||
assert_eq!(
|
||||
pin,
|
||||
Some(pubkey),
|
||||
"re-pinned identity must be visible immediately on the same live state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user