diff --git a/.claude/commands/coverage.md b/.claude/commands/coverage.md new file mode 100644 index 00000000..34815b24 --- /dev/null +++ b/.claude/commands/coverage.md @@ -0,0 +1,21 @@ +Show test coverage from the cached `.coverage_baseline` file, or rerun the full test suite with `$ARGUMENTS`. + +## Usage + +- `/coverage` — read cached coverage from `.coverage_baseline` (instant) +- `/coverage run` — run `script/test_coverage` and report fresh results + +## What it does + +**Cached mode (default):** Reads `.coverage_baseline` and displays the stored coverage percentage(s). This is instant and does not run any tests. + +**Run mode (`run`):** Executes `script/test_coverage` which runs: +1. Rust tests with `cargo llvm-cov` (reports line coverage %) +2. Frontend tests with `npm run test:coverage` (reports line coverage %) +3. Computes the overall average and compares to the threshold + +Reports Rust coverage, Frontend coverage, Overall coverage, and whether the run passed the threshold. + +--- + +If the arguments (`$ARGUMENTS`) equal `run`, execute `bash script/test_coverage` from the project root and show the Coverage Summary section from the output. Otherwise, read `.coverage_baseline` and display the stored coverage value(s). diff --git a/server/src/chat/commands/coverage.rs b/server/src/chat/commands/coverage.rs new file mode 100644 index 00000000..7d098b2f --- /dev/null +++ b/server/src/chat/commands/coverage.rs @@ -0,0 +1,343 @@ +//! Handler for the `coverage` command — show or refresh test coverage. +//! +//! Default (no args): reads the cached `.coverage_baseline` file and reports +//! the stored value instantly without running the test suite. +//! +//! `coverage run`: executes `script/test_coverage` to collect fresh coverage +//! data, parses the summary, and reports the result. + +use super::CommandContext; + +const BASELINE_FILE: &str = ".coverage_baseline"; +const COVERAGE_SCRIPT: &str = "script/test_coverage"; + +pub(super) fn handle_coverage(ctx: &CommandContext) -> Option { + let args = ctx.args.trim(); + + match args { + "run" => Some(run_coverage(ctx.project_root)), + "" => Some(read_cached_coverage(ctx.project_root)), + other => Some(format!( + "Usage: `coverage` (cached) or `coverage run` (fresh)\n\nUnknown argument: `{other}`" + )), + } +} + +/// Read coverage from `.coverage_baseline` and return a formatted report. +fn read_cached_coverage(project_root: &std::path::Path) -> String { + let baseline_path = project_root.join(BASELINE_FILE); + + match std::fs::read_to_string(&baseline_path) { + Ok(content) => { + let lines: Vec<&str> = content.lines().filter(|l| !l.trim().is_empty()).collect(); + if lines.is_empty() { + return "**Coverage (cached)**\n\nNo baseline data found.\n\n*Run `coverage run` to collect coverage.*".to_string(); + } + format_cached_coverage(&lines) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + "**Coverage (cached)**\n\nNo `.coverage_baseline` file found.\n\n*Run `coverage run` to collect coverage.*".to_string() + } + Err(e) => format!("**Coverage (cached)**\n\nError reading `.coverage_baseline`: {e}"), + } +} + +/// Format cached coverage lines for display. +/// +/// The baseline file stores the last overall coverage percentage written by +/// `script/test_coverage`. It always contains one line (the overall percentage) +/// but may have been seeded manually with two lines (Rust + Frontend baselines). +fn format_cached_coverage(lines: &[&str]) -> String { + let mut out = "**Coverage (cached)**\n\n".to_string(); + if lines.len() == 2 { + out.push_str(&format!("Rust: {}%\n", lines[0].trim())); + out.push_str(&format!("Frontend: {}%\n", lines[1].trim())); + } else { + out.push_str(&format!("Overall: {}%\n", lines[0].trim())); + } + out.push_str("\n*Run `coverage run` for fresh results.*"); + out +} + +/// Run `script/test_coverage`, parse its summary, and return a formatted report. +fn run_coverage(project_root: &std::path::Path) -> String { + let script_path = project_root.join(COVERAGE_SCRIPT); + + if !script_path.exists() { + return format!( + "**Coverage**\n\nCoverage script not found: `{COVERAGE_SCRIPT}`\n\nEnsure `{COVERAGE_SCRIPT}` exists in the project root." + ); + } + + let output = std::process::Command::new("bash") + .arg(&script_path) + .current_dir(project_root) + .output(); + + match output { + Err(e) => format!("**Coverage**\n\nFailed to run coverage script: {e}"), + Ok(out) => { + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + let combined = format!("{stdout}{stderr}"); + + // Check if llvm-cov is missing + if combined.contains("cargo-llvm-cov not available") && !combined.contains("TOTAL") { + return format!( + "**Coverage**\n\n`cargo-llvm-cov` is not installed.\n\nInstall it with:\n```\ncargo install cargo-llvm-cov\n```" + ); + } + + parse_coverage_output(&combined, out.status.success()) + } + } +} + +/// Parse the summary block from `script/test_coverage` output and format it. +fn parse_coverage_output(output: &str, passed: bool) -> String { + let rust_cov = extract_line_value(output, "Rust line coverage:"); + let frontend_cov = extract_line_value(output, "Frontend line coverage:"); + let overall = extract_summary_field(output, "Overall:"); + let threshold = extract_summary_field(output, "Threshold:"); + + let mut out = "**Coverage (fresh)**\n\n".to_string(); + + if let Some(r) = &rust_cov { + out.push_str(&format!("Rust: {r}\n")); + } + if let Some(f) = &frontend_cov { + out.push_str(&format!("Frontend: {f}\n")); + } + if let Some(o) = &overall { + out.push_str(&format!("Overall: {o}\n")); + } + if let Some(t) = &threshold { + out.push_str(&format!("Threshold: {t}\n")); + } + + if rust_cov.is_none() && frontend_cov.is_none() && overall.is_none() { + // Could not parse — show raw output excerpt + let excerpt: String = output.lines().rev().take(10).collect::>().into_iter().rev().collect::>().join("\n"); + return format!("**Coverage (fresh)**\n\n```\n{excerpt}\n```"); + } + + if passed { + out.push_str("\nPASS"); + } else { + out.push_str("\nFAIL: coverage below threshold"); + } + + out +} + +/// Extract a value from lines like `"Rust line coverage: 62.5%"`. +fn extract_line_value(output: &str, prefix: &str) -> Option { + output + .lines() + .find(|l| l.trim().starts_with(prefix)) + .map(|l| l.trim()[prefix.len()..].trim().to_string()) +} + +/// Extract a value from the summary block: `" Overall: 62.5%"`. +fn extract_summary_field(output: &str, label: &str) -> Option { + output + .lines() + .find(|l| l.trim().starts_with(label)) + .map(|l| l.trim()[label.len()..].trim().to_string()) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::agents::AgentPool; + use std::collections::HashSet; + use std::sync::{Arc, Mutex}; + + fn make_ctx<'a>( + agents: &'a Arc, + ambient_rooms: &'a Arc>>, + project_root: &'a std::path::Path, + args: &'a str, + ) -> super::super::CommandContext<'a> { + super::super::CommandContext { + bot_name: "Timmy", + args, + project_root, + agents, + ambient_rooms, + room_id: "!test:example.com", + } + } + + fn test_agents() -> Arc { + Arc::new(AgentPool::new_test(3000)) + } + + fn test_ambient() -> Arc>> { + Arc::new(Mutex::new(HashSet::new())) + } + + #[test] + fn coverage_command_is_registered() { + use super::super::commands; + let found = commands().iter().any(|c| c.name == "coverage"); + assert!(found, "coverage command must be in the registry"); + } + + #[test] + fn coverage_command_appears_in_help() { + let result = super::super::tests::try_cmd_addressed( + "Timmy", + "@timmy:homeserver.local", + "@timmy help", + ); + let output = result.unwrap(); + assert!( + output.contains("coverage"), + "help should list coverage command: {output}" + ); + } + + #[test] + fn coverage_no_args_reads_baseline() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join(".coverage_baseline"), "72.5\n").unwrap(); + + let agents = test_agents(); + let ambient = test_ambient(); + let ctx = make_ctx(&agents, &ambient, dir.path(), ""); + let output = handle_coverage(&ctx).unwrap(); + + assert!( + output.contains("72.5"), + "should report cached value from baseline: {output}" + ); + assert!( + output.contains("cached"), + "response should indicate cached result: {output}" + ); + } + + #[test] + fn coverage_no_args_two_line_baseline() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join(".coverage_baseline"), "60.00\n65.21\n").unwrap(); + + let agents = test_agents(); + let ambient = test_ambient(); + let ctx = make_ctx(&agents, &ambient, dir.path(), ""); + let output = handle_coverage(&ctx).unwrap(); + + assert!( + output.contains("60.00"), + "should report first baseline value: {output}" + ); + assert!( + output.contains("65.21"), + "should report second baseline value: {output}" + ); + } + + #[test] + fn coverage_missing_baseline_reports_clearly() { + let dir = tempfile::tempdir().expect("tempdir"); + + let agents = test_agents(); + let ambient = test_ambient(); + let ctx = make_ctx(&agents, &ambient, dir.path(), ""); + let output = handle_coverage(&ctx).unwrap(); + + assert!( + output.contains("No `.coverage_baseline`") || output.contains("coverage_baseline"), + "missing baseline should mention the file name: {output}" + ); + } + + #[test] + fn coverage_run_missing_script_reports_error() { + let dir = tempfile::tempdir().expect("tempdir"); + + let agents = test_agents(); + let ambient = test_ambient(); + let ctx = make_ctx(&agents, &ambient, dir.path(), "run"); + let output = handle_coverage(&ctx).unwrap(); + + assert!( + output.contains("not found") || output.contains("script"), + "missing script should produce a clear error: {output}" + ); + } + + #[test] + fn coverage_unknown_arg_returns_usage() { + let dir = tempfile::tempdir().expect("tempdir"); + let agents = test_agents(); + let ambient = test_ambient(); + let ctx = make_ctx(&agents, &ambient, dir.path(), "blah"); + let output = handle_coverage(&ctx).unwrap(); + assert!( + output.contains("Usage"), + "unknown arg should show usage: {output}" + ); + } + + #[test] + fn coverage_command_works_via_dispatch() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join(".coverage_baseline"), "55.0\n").unwrap(); + + let agents = test_agents(); + let ambient = test_ambient(); + let room_id = "!test:example.com".to_string(); + let dispatch = super::super::CommandDispatch { + bot_name: "Timmy", + bot_user_id: "@timmy:homeserver.local", + project_root: dir.path(), + agents: &agents, + ambient_rooms: &ambient, + room_id: &room_id, + }; + let result = super::super::try_handle_command(&dispatch, "@timmy coverage"); + assert!( + result.is_some(), + "coverage command must respond via dispatch (not fall through to LLM)" + ); + } + + #[test] + fn parse_coverage_output_extracts_fields() { + let sample = "\ +Rust line coverage: 62.5%\n\ +Frontend line coverage: 70.0%\n\ +=== Coverage Summary ===\n\ + Rust: 62.5%\n\ + Frontend: 70.0%\n\ + Overall: 66.25%\n\ + Threshold: 60.00%\n\ +PASS: Coverage 66.25% meets threshold 60.00%\n\ +"; + let result = parse_coverage_output(sample, true); + assert!(result.contains("62.5"), "should include Rust coverage: {result}"); + assert!(result.contains("70.0"), "should include Frontend coverage: {result}"); + assert!(result.contains("66.25"), "should include Overall coverage: {result}"); + assert!(result.contains("PASS"), "should indicate PASS: {result}"); + } + + #[test] + fn extract_line_value_finds_rust_coverage() { + let output = "Rust line coverage: 55.3%\n"; + let val = extract_line_value(output, "Rust line coverage:"); + assert_eq!(val.as_deref(), Some("55.3%")); + } + + #[test] + fn extract_summary_field_finds_overall() { + let output = " Overall: 72.1%\n"; + let val = extract_summary_field(output, "Overall:"); + assert_eq!(val.as_deref(), Some("72.1%")); + } +} diff --git a/server/src/chat/commands/mod.rs b/server/src/chat/commands/mod.rs index 9cb4cf59..28edae78 100644 --- a/server/src/chat/commands/mod.rs +++ b/server/src/chat/commands/mod.rs @@ -8,6 +8,7 @@ mod ambient; mod assign; mod cost; +mod coverage; mod git; mod help; pub(crate) mod loc; @@ -118,6 +119,11 @@ pub fn commands() -> &'static [BotCommand] { description: "Show token spend: 24h total, top stories, breakdown by agent type, and all-time total", handler: cost::handle_cost, }, + BotCommand { + name: "coverage", + description: "Show test coverage: cached baseline by default, or `coverage run` to rerun the full suite", + handler: coverage::handle_coverage, + }, BotCommand { name: "loc", description: "Show top source files by line count: `loc` (top 10), `loc `, or `loc ` for a specific file",