//! `health` chat command — surface gateway, sled, matrix, creds, and build-hash status. //! //! Runs one check per subsystem concurrently (each with a 5-second timeout) and //! returns a compact report: one line per subsystem with PASS / WARN / FAIL and a //! remediation hint on every non-PASS row. Output is capped at 20 lines; when //! more lines would be produced, the oldest WARN rows are dropped first. use crate::chat::transport::matrix::bot::context::BotContext; use std::collections::BTreeMap; use std::sync::atomic::Ordering; use std::time::Duration; use tokio::time::timeout; // ── Status ───────────────────────────────────────────────────────────────────── /// Health status for a single subsystem. #[derive(Debug, Clone, PartialEq)] enum Status { /// Subsystem is operating normally. Pass, /// Subsystem is degraded but not fully broken. Warn, /// Subsystem has failed and needs intervention. Fail, } // ── HealthLine ───────────────────────────────────────────────────────────────── /// One output row from the health check. #[derive(Debug, Clone)] struct HealthLine { subsystem: String, status: Status, /// Short description of why the check is non-PASS. detail: Option, /// Remediation hint shown after " — " on WARN/FAIL rows. hint: Option, } impl HealthLine { fn pass(subsystem: impl Into) -> Self { Self { subsystem: subsystem.into(), status: Status::Pass, detail: None, hint: None, } } fn warn( subsystem: impl Into, detail: impl Into, hint: impl Into, ) -> Self { Self { subsystem: subsystem.into(), status: Status::Warn, detail: Some(detail.into()), hint: Some(hint.into()), } } fn fail( subsystem: impl Into, detail: impl Into, hint: impl Into, ) -> Self { Self { subsystem: subsystem.into(), status: Status::Fail, detail: Some(detail.into()), hint: Some(hint.into()), } } /// Append `relay=X` information to this line's detail field. fn with_relay(mut self, relay: &str) -> Self { let relay_text = format!("relay={relay}"); self.detail = Some(match self.detail { Some(d) => format!("{d} | {relay_text}"), None => relay_text, }); self } /// Format as a single Markdown-friendly line. fn format(&self) -> String { let status = match self.status { Status::Pass => "PASS", Status::Warn => "WARN", Status::Fail => "FAIL", }; match (&self.detail, &self.hint) { (Some(d), Some(h)) => format!("{} {}: {} — {}", self.subsystem, status, d, h), (Some(d), None) => format!("{} {}: {}", self.subsystem, status, d), (None, None) => format!("{} {}", self.subsystem, status), (None, Some(h)) => format!("{} {}: — {}", self.subsystem, status, h), } } } // ── Truncation ──────────────────────────────────────────────────────────────── /// Maximum number of output lines before truncation. const MAX_LINES: usize = 20; /// Truncate to ≤ MAX_LINES by removing the oldest (first in order) WARN rows. fn truncate_lines(mut lines: Vec) -> Vec { while lines.len() > MAX_LINES { if let Some(pos) = lines.iter().position(|l| l.status == Status::Warn) { lines.remove(pos); } else { break; } } lines } // ── Individual checks ──────────────────────────────────────────────────────── /// Check the permission registry — PASS when at least one responder (e.g. the /// Matrix permission listener) is registered, FAIL when none is (listener has /// died or was never started). fn check_perm_rx(ctx: &BotContext) -> HealthLine { if ctx.services.permission_registry.is_empty() { HealthLine::fail("perm_rx", "no responder registered", "restart bot") } else { HealthLine::pass("perm_rx") } } /// Check the Matrix sync loop by measuring the age of the last received event. /// /// WARN after 60 s of silence, FAIL after 120 s. The timestamp is updated by /// `on_room_message` on every incoming event so receiving the health command /// itself resets the clock. fn check_matrix_sync(ctx: &BotContext) -> HealthLine { let last_ms = ctx.last_matrix_event_ms.load(Ordering::Relaxed); let age_secs = (chrono::Utc::now().timestamp_millis() - last_ms).max(0) / 1000; if age_secs < 60 { HealthLine::pass("matrix-sync") } else if age_secs < 120 { HealthLine::warn( "matrix-sync", format!("no events in {age_secs}s"), "check sync loop — may be a quiet room", ) } else { HealthLine::fail( "matrix-sync", format!("no events in {age_secs}s"), "sync loop may be dead — restart bot", ) } } /// Check LLM credentials (`~/.claude/.credentials.json`). /// /// FAIL if the file is missing or unreadable, FAIL if the access token is /// expired, WARN if it expires within the next 7 days. fn check_creds() -> HealthLine { match crate::llm::oauth::read_credentials() { Err(e) => HealthLine::fail("creds", e, "run `claude login`"), Ok(creds) => { let now_secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs(); let expires_at = creds.claude_ai_oauth.expires_at; if expires_at < now_secs { HealthLine::fail("creds", "token expired", "run `claude login` to refresh") } else { let days_left = (expires_at - now_secs) / 86400; if days_left < 7 { HealthLine::warn( "creds", format!("token expires in {days_left}d"), "run `claude login` to refresh", ) } else { HealthLine::pass("creds") } } } } } /// Compare the compile-time build hash against the current HEAD of the workspace. /// /// WARN when master has advanced past the running binary's commit (a rebuild is /// available but not urgent). PASS when hashes match or HEAD cannot be read. async fn check_build_hash(project_root: &std::path::Path) -> HealthLine { let running = option_env!("BUILD_GIT_HASH").unwrap_or("unknown"); // Read current HEAD from git (non-blocking, run in a spawn_blocking call). let repo_root = project_root.to_path_buf(); let head = tokio::task::spawn_blocking(move || { std::process::Command::new("git") .args(["rev-parse", "--short", "HEAD"]) .current_dir(&repo_root) .output() .ok() .filter(|o| o.status.success()) .and_then(|o| String::from_utf8(o.stdout).ok()) .map(|s| s.trim().to_string()) }) .await .unwrap_or(None); match head { None => HealthLine::pass("build-hash"), Some(ref head_hash) => { if running == "unknown" || head_hash == running { HealthLine::pass("build-hash") } else { HealthLine::warn( "build-hash", format!("running {running}, HEAD is {head_hash}"), "run `rebuild` to update", ) } } } } /// Check each registered sled's `/health` endpoint with a 5-second timeout. /// /// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP /// 2xx *and* its `/identity` probe matches the recorded `expected_node_id`; /// FAIL when `/health` times out or errors, OR when the identity probe /// reveals a mismatch or invalid signature — even if `/health` itself /// reported ok, since a swapped-in container can still serve a healthy /// `/health` response. Each line includes a `relay=X` indicator derived from /// the gateway CRDT event log. async fn check_sleds( store: &tokio::sync::RwLock>, config_dir: &std::path::Path, ) -> Vec { let entries: Vec<(String, Option)> = store .read() .await .iter() .map(|(n, e)| (n.clone(), e.url.clone())) .collect(); if entries.is_empty() { return vec![HealthLine::warn( "sled", "no sleds registered", "add projects to projects.toml", )]; } let event_entries = crate::crdt_state::read_all_event_log_entries(); let now_secs = chrono::Utc::now().timestamp() as f64; let client = reqwest::Client::new(); let mut lines = Vec::new(); for (name, url_opt) in entries { let subsystem = format!("sled:{name}"); let relay = { let latest = event_entries .iter() .filter(|e| e.sled_id == name) .map(|e| e.timestamp) .fold(f64::NEG_INFINITY, f64::max); if latest == f64::NEG_INFINITY { "never" } else if now_secs - latest <= crate::service::gateway::RELAY_MAX_AGE_SECS { "ok" } else { "silent" } }; let line = match url_opt { None => HealthLine::warn(subsystem, "no URL configured", "set url in projects.toml") .with_relay(relay), Some(url) => { let health_url = format!("{}/health", url.trim_end_matches('/')); let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await; let health_line = match result { Err(_) => HealthLine::fail( subsystem.clone(), "timed out", "check container is running", ), Ok(Err(e)) => HealthLine::fail( subsystem.clone(), format!("unreachable: {}", short_error(&e.to_string())), "check container is running", ), Ok(Ok(resp)) if resp.status().is_success() => { HealthLine::pass(subsystem.clone()) } Ok(Ok(resp)) => HealthLine::fail( subsystem.clone(), format!("HTTP {}", resp.status().as_u16()), "check container logs", ), }; match check_sled_identity(&client, &name, &url, store, config_dir).await { Some(identity_fail) => identity_fail.with_relay(relay), None => health_line.with_relay(relay), } } }; lines.push(line); } lines } /// Probe `{url}/identity`, verify the response against the `expected_node_id` /// recorded for `name`, and capture a first-contact identity. /// /// Returns `Some(HealthLine::fail(...))` on a confirmed mismatch or invalid /// signature — this overrides an otherwise-passing `/health` result. Returns /// `None` when the identity is unverifiable (unreachable probe or a legacy /// sled with no signature) or matches, in which case the `/health` result /// stands on its own. async fn check_sled_identity( client: &reqwest::Client, name: &str, url: &str, store: &tokio::sync::RwLock>, config_dir: &std::path::Path, ) -> Option { let subsystem = format!("sled:{name}"); let nonce = crate::node_identity::generate_challenge(); let response = timeout( Duration::from_secs(5), crate::service::gateway::probe_identity(client, url, &nonce), ) .await .ok() .flatten()?; let expected = store .read() .await .get(name) .and_then(|e| e.expected_node_id.clone()); match crate::service::gateway::check_identity(expected.as_deref(), &nonce, &response) { crate::service::gateway::IdentityCheck::Match | crate::service::gateway::IdentityCheck::MissingSignature => None, crate::service::gateway::IdentityCheck::FirstContact { node_id } => { { let mut projects = store.write().await; if let Some(entry) = projects.get_mut(name) { entry.expected_node_id = Some(node_id); } } let snapshot = store.read().await.clone(); crate::service::gateway::io::save_config(&snapshot, config_dir).await; None } crate::service::gateway::IdentityCheck::Mismatch { responder_node_id } => { Some(HealthLine::fail( subsystem, format!( "identity mismatch: expected `{}`, container answered as `{responder_node_id}`", expected.unwrap_or_default() ), "verify the correct container is running — possible swap", )) } crate::service::gateway::IdentityCheck::InvalidSignature => Some(HealthLine::fail( subsystem, "identity signature invalid", "container's identity cannot be verified — check logs", )), } } /// Check the gateway process: pidfile validity and (on macOS) binary codesign. /// /// PASS when our PID is recorded in the pidfile. On macOS, also verifies that /// `~/bin/huskies-bin` has a valid ad-hoc signature; FAIL with a `script/local-release` /// hint when it does not. fn check_gateway_process() -> HealthLine { // Verify that the pidfile records our PID (i.e. this IS the live gateway). let pidfile_ok = check_pidfile_matches_self(); // On macOS, verify the installed binary is codesigned. #[cfg(target_os = "macos")] { if !check_codesign_macos() { return HealthLine::fail( "gateway-process", "codesign invalid", "run `script/local-release`", ); } } if !pidfile_ok { return HealthLine::warn( "gateway-process", "pidfile missing or stale", "restart gateway with --gateway flag", ); } HealthLine::pass("gateway-process") } /// Return `true` when `$HOME/.huskies/gateway.pid` exists and contains our PID. fn check_pidfile_matches_self() -> bool { let home = homedir::my_home().ok().flatten(); let home = match home { Some(h) => h, None => return false, }; let path = home.join(".huskies").join("gateway.pid"); let content = std::fs::read_to_string(&path).unwrap_or_default(); content.trim().parse::().unwrap_or(0) == std::process::id() } /// On macOS, return `true` when `~/bin/huskies-bin` passes `codesign --verify`. /// /// Falls back to the current executable when `~/bin/huskies-bin` does not exist. /// Returns `true` (assume ok) if the `codesign` tool is unavailable. #[cfg(target_os = "macos")] fn check_codesign_macos() -> bool { let target = if let Ok(home) = std::env::var("HOME") { let installed = std::path::PathBuf::from(home) .join("bin") .join("huskies-bin"); if installed.exists() { installed } else { match std::env::current_exe() { Ok(p) => p, Err(_) => return true, } } } else { match std::env::current_exe() { Ok(p) => p, Err(_) => return true, } }; std::process::Command::new("codesign") .args(["--verify", "--quiet", target.to_str().unwrap_or("")]) .output() .map(|o| o.status.success()) .unwrap_or(true) } // ── Entry point ──────────────────────────────────────────────────────────────── /// Run all health checks and return a formatted Markdown report (≤ 20 lines). /// /// Gateway-specific checks (gateway-process, per-sled probes) are included /// only when running in gateway mode. All other checks run in every mode. pub async fn run_health_check(ctx: &BotContext) -> String { let mut lines: Vec = Vec::new(); // Gateway-only checks if ctx.is_gateway() { lines.push(check_gateway_process()); if let Some(ref store) = ctx.gateway_projects_store { lines.extend(check_sleds(store, &ctx.services.project_root).await); } } // Shared checks — run concurrently where possible. let perm_line = check_perm_rx(ctx); let sync_line = check_matrix_sync(ctx); let creds_line = check_creds(); let hash_line = check_build_hash(&ctx.services.project_root).await; lines.push(perm_line); lines.push(sync_line); lines.push(creds_line); lines.push(hash_line); lines.push(check_disk_space()); let lines = truncate_lines(lines); lines .iter() .map(|l| l.format()) .collect::>() .join("\n") } /// Report current free disk space on `/workspace` (story 1200 AC5). fn check_disk_space() -> HealthLine { match crate::service::disk_watch::io::free_space_bytes(std::path::Path::new("/workspace")) { Ok(free_bytes) => { let free_gb = free_bytes as f64 / 1_000_000_000.0; HealthLine { subsystem: "disk".to_string(), status: Status::Pass, detail: Some(format!("{free_gb:.1}GB free")), hint: None, } } Err(_) => HealthLine::warn( "disk", "unable to read free space", "check /workspace mount", ), } } // ── Utilities ──────────────────────────────────────────────────────────────── /// Shorten a long error string to the first 60 characters for compact display. fn short_error(s: &str) -> String { s.chars().take(60).collect() } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { use super::*; // -- HealthLine formatting ------------------------------------------------ #[test] fn pass_line_formats_without_detail() { let line = HealthLine::pass("perm_rx"); assert_eq!(line.format(), "perm_rx PASS"); } #[test] fn fail_line_formats_with_detail_and_hint() { let line = HealthLine::fail( "gateway-process", "codesign invalid", "run script/local-release", ); assert_eq!( line.format(), "gateway-process FAIL: codesign invalid — run script/local-release" ); } #[test] fn warn_line_formats_with_detail_and_hint() { let line = HealthLine::warn("build-hash", "running abc, HEAD is def", "run rebuild"); assert_eq!( line.format(), "build-hash WARN: running abc, HEAD is def — run rebuild" ); } // -- Truncation ----------------------------------------------------------- #[test] fn truncate_drops_oldest_warn_first() { let mut lines: Vec = (0..22) .map(|i| { if i % 3 == 0 { HealthLine::fail(format!("sled:{i}"), "down", "fix it") } else { HealthLine::warn(format!("check:{i}"), "slow", "investigate") } }) .collect(); // Manually insert a known WARN at position 0 and a FAIL at position 1 lines.insert(0, HealthLine::warn("oldest-warn", "stale", "restart")); lines.insert(1, HealthLine::fail("important-fail", "broken", "fix")); let result = truncate_lines(lines.clone()); assert!( result.len() <= MAX_LINES, "output must be ≤ {MAX_LINES} lines" ); // FAILs must be preserved. let fail_count = result.iter().filter(|l| l.status == Status::Fail).count(); let orig_fail_count = lines.iter().filter(|l| l.status == Status::Fail).count(); assert_eq!( fail_count, orig_fail_count.min(MAX_LINES), "all FAIL lines must be kept when they fit" ); } #[test] fn truncate_noop_when_under_limit() { let lines: Vec = (0..5).map(|i| HealthLine::pass(format!("s{i}"))).collect(); let result = truncate_lines(lines.clone()); assert_eq!(result.len(), 5); } #[test] fn truncate_stops_at_fails_when_no_warns_left() { // 25 FAIL lines — nothing to drop; output is clamped at MAX_LINES. let lines: Vec = (0..25) .map(|i| HealthLine::fail(format!("s{i}"), "broken", "fix")) .collect(); let result = truncate_lines(lines); // When only FAILs are present, truncation stops because no WARNs can be removed. assert_eq!(result.len(), 25, "FAILs are never dropped by truncation"); } // -- perm_rx check -------------------------------------------------------- #[tokio::test] async fn perm_rx_pass_when_locked() { use crate::service::permission_router::{PendingPermReplies, ResponderRegistry}; use crate::services::Services; use std::sync::Arc; let registry = ResponderRegistry::new(); // Register a responder to simulate the permission listener being active. let _guard_and_rx = registry.register(); let services = Arc::new(Services { project_root: std::path::PathBuf::from("/tmp"), agents: Arc::new(crate::agents::AgentPool::new_test(3000)), bot_name: "test".to_string(), bot_user_id: "@bot:test".to_string(), ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), permission_registry: registry, pending_perm_replies: PendingPermReplies::new(), permission_timeout_secs: 120, status: Arc::new(crate::service::status::StatusBroadcaster::new()), chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)), }); // Build a minimal BotContext just to pass services. let ctx = make_test_ctx(services); let line = check_perm_rx(&ctx); assert_eq!( line.status, Status::Pass, "perm_rx should PASS when a responder is registered" ); } #[tokio::test] async fn perm_rx_fail_when_unlocked() { use crate::service::permission_router::{PendingPermReplies, ResponderRegistry}; use crate::services::Services; use std::sync::Arc; // No responder registered. let services = Arc::new(Services { project_root: std::path::PathBuf::from("/tmp"), agents: Arc::new(crate::agents::AgentPool::new_test(3000)), bot_name: "test".to_string(), bot_user_id: "@bot:test".to_string(), ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())), permission_registry: ResponderRegistry::new(), pending_perm_replies: PendingPermReplies::new(), permission_timeout_secs: 120, status: Arc::new(crate::service::status::StatusBroadcaster::new()), chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)), }); let ctx = make_test_ctx(services); let line = check_perm_rx(&ctx); assert_eq!( line.status, Status::Fail, "perm_rx should FAIL when no responder is registered" ); } // -- matrix-sync check ---------------------------------------------------- #[tokio::test] async fn matrix_sync_pass_when_recent() { let services = crate::services::Services::new_test( std::path::PathBuf::from("/tmp"), "bot".to_string(), ); let ctx = make_test_ctx(services); // Set last event to just now. ctx.last_matrix_event_ms .store(chrono::Utc::now().timestamp_millis(), Ordering::Relaxed); let line = check_matrix_sync(&ctx); assert_eq!(line.status, Status::Pass); } #[tokio::test] async fn matrix_sync_fail_when_stale() { let services = crate::services::Services::new_test( std::path::PathBuf::from("/tmp"), "bot".to_string(), ); let ctx = make_test_ctx(services); // Simulate 200 seconds of silence. let old_ms = chrono::Utc::now().timestamp_millis() - 200_000; ctx.last_matrix_event_ms.store(old_ms, Ordering::Relaxed); let line = check_matrix_sync(&ctx); assert_eq!(line.status, Status::Fail); assert!( line.detail.as_deref().unwrap_or("").contains("200s") || line.detail.as_deref().unwrap_or("").contains("s"), "detail should mention age in seconds" ); } // -- creds check ---------------------------------------------------------- #[test] fn creds_fail_when_file_missing() { // In the test environment there is unlikely to be a ~/.claude/.credentials.json // with a valid non-expired token, so we just confirm the function returns a // HealthLine without panicking. let line = check_creds(); // We don't assert a specific status — the check should not panic. let _ = line.format(); } // -- build_hash check ----------------------------------------------------- #[tokio::test] async fn build_hash_pass_when_git_unavailable() { // In a test environment without a git repo at /tmp/nonexistent, the check // should gracefully return PASS rather than panicking. let line = check_build_hash(std::path::Path::new("/tmp/nonexistent")).await; // Should either PASS or produce a sensible result — must not panic. let _ = line.format(); } // -- health command registration ------------------------------------------ #[test] fn health_command_registered_in_commands() { let cmds = crate::chat::commands::commands(); assert!( cmds.iter().any(|c| c.name == "health"), "health must be registered in commands()" ); } #[test] fn health_command_has_description() { let cmds = crate::chat::commands::commands(); let cmd = cmds.iter().find(|c| c.name == "health").unwrap(); assert!(!cmd.description.is_empty()); } // -- check_sled_identity --------------------------------------------------- /// Spawn a one-shot TCP listener that answers a single `GET /// /identity?nonce=` request with a JSON body signed by `kp` over /// whatever nonce the caller actually sent. fn spawn_identity_responder( listener: tokio::net::TcpListener, kp: bft_json_crdt::keypair::Ed25519KeyPair, ) { tokio::spawn(async move { if let Ok((mut stream, _)) = listener.accept().await { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let mut buf = [0u8; 4096]; let n = stream.read(&mut buf).await.unwrap_or(0); let req = String::from_utf8_lossy(&buf[..n]); let nonce = req .lines() .next() .unwrap_or("") .split("nonce=") .nth(1) .and_then(|s| s.split_whitespace().next()) .unwrap_or("") .to_string(); let pubkey = crate::node_identity::public_key_hex(&kp); let sig = crate::node_identity::sign_challenge(&kp, &nonce); let body = serde_json::json!({ "node_id": pubkey, "pubkey": pubkey, "signature": sig, }) .to_string(); let response = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", body.len(), body ); let _ = stream.write_all(response.as_bytes()).await; } }); } #[tokio::test] async fn check_sled_identity_first_contact_captures_and_persists() { let kp = bft_json_crdt::keypair::make_keypair(); let pubkey = crate::node_identity::public_key_hex(&kp); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); spawn_identity_responder(listener, kp); let sled_url = format!("http://127.0.0.1:{port}"); let mut map = BTreeMap::new(); map.insert( "myapp".to_string(), crate::service::gateway::config::ProjectEntry { url: Some(sled_url.clone()), auth_token: None, ssh_port: None, host_path: None, expected_node_id: None, }, ); let store = tokio::sync::RwLock::new(map); let config_dir = tempfile::tempdir().unwrap(); let client = reqwest::Client::new(); let result = check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; assert!( result.is_none(), "first contact must not fail the health line: {result:?}" ); let captured = store .read() .await .get("myapp") .and_then(|e| e.expected_node_id.clone()); assert_eq!( captured, Some(pubkey.clone()), "expected_node_id should be captured on first contact" ); let toml_content = tokio::fs::read_to_string(config_dir.path().join("projects.toml")).await; assert!( toml_content.unwrap_or_default().contains(&pubkey), "captured node_id should be persisted to projects.toml" ); } #[tokio::test] async fn check_sled_identity_match_returns_none() { let kp = bft_json_crdt::keypair::make_keypair(); let pubkey = crate::node_identity::public_key_hex(&kp); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); spawn_identity_responder(listener, kp); let sled_url = format!("http://127.0.0.1:{port}"); let mut map = BTreeMap::new(); map.insert( "myapp".to_string(), crate::service::gateway::config::ProjectEntry { url: Some(sled_url.clone()), auth_token: None, ssh_port: None, host_path: None, expected_node_id: Some(pubkey), }, ); let store = tokio::sync::RwLock::new(map); let config_dir = tempfile::tempdir().unwrap(); let client = reqwest::Client::new(); let result = check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; assert!(result.is_none(), "matching identity must PASS: {result:?}"); } #[tokio::test] async fn check_sled_identity_mismatch_forces_fail_line() { let kp = bft_json_crdt::keypair::make_keypair(); let responder_pubkey = crate::node_identity::public_key_hex(&kp); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); spawn_identity_responder(listener, kp); let sled_url = format!("http://127.0.0.1:{port}"); let mut map = BTreeMap::new(); map.insert( "myapp".to_string(), crate::service::gateway::config::ProjectEntry { url: Some(sled_url.clone()), auth_token: None, ssh_port: None, host_path: None, expected_node_id: Some("ab".repeat(32)), }, ); let store = tokio::sync::RwLock::new(map); let config_dir = tempfile::tempdir().unwrap(); let client = reqwest::Client::new(); let result = check_sled_identity(&client, "myapp", &sled_url, &store, config_dir.path()).await; let line = result.expect("identity mismatch must force a FAIL health line"); assert_eq!( line.status, Status::Fail, "identity mismatch must FAIL even though /health would report ok" ); assert!( line.detail .as_deref() .unwrap_or("") .contains(&responder_pubkey), "detail should name the node_id that actually answered: {line:?}" ); } // -- Helper --------------------------------------------------------------- /// Build a minimal `BotContext` for testing purposes. fn make_test_ctx(services: std::sync::Arc) -> BotContext { use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::AtomicI64; use tokio::sync::Mutex as TokioMutex; BotContext { services, matrix_user_id: "@bot:example.com".parse().unwrap(), target_room_ids: vec![], allowed_users: vec![], history: Arc::new(TokioMutex::new(std::collections::HashMap::new())), history_size: 20, bot_sent_event_ids: Arc::new(TokioMutex::new(HashSet::new())), htop_sessions: Arc::new(TokioMutex::new(std::collections::HashMap::new())), transport: Arc::new(crate::chat::transport::whatsapp::WhatsAppTransport::new( "test-phone".to_string(), "test-token".to_string(), "pipeline_notification".to_string(), )), timer_store: Arc::new(crate::service::timer::TimerStore::load( std::path::PathBuf::from("/tmp/timers-health.json"), )), gateway_active_project: None, gateway_projects_store: None, gateway_channels_store: None, handled_incoming_event_ids: Arc::new(TokioMutex::new( crate::chat::transport::matrix::bot::context::SeenEventIds::new( crate::chat::transport::matrix::bot::context::SEEN_EVENT_IDS_CAP, ), )), gateway_port: None, last_matrix_event_ms: Arc::new(AtomicI64::new(chrono::Utc::now().timestamp_millis())), model: None, compact_seed_max_bytes: 8_000, cache_read_suggest_threshold: 50_000, compact_suggest_cooldown_secs: 3_600, digging_in_threshold_secs: 15, } } }