Compare commits
101 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5bca1f6cec | |||
| 86b9d069b1 | |||
| f6ee90e169 | |||
| 9a286315a3 | |||
| 5d0801854c | |||
| 343473bc01 | |||
| 2593b36072 | |||
| 34af2f1820 | |||
| be7bdf8304 | |||
| 918f18c200 | |||
| 1db5473f50 | |||
| de638603cd | |||
| 20ec690e22 | |||
| 9a5b6f4d92 | |||
| 398726a14a | |||
| c8be24f833 | |||
| f8ff63af0e | |||
| 34e78bdbd5 | |||
| fb4e52dd09 | |||
| e58ff4465a | |||
| b1dec36e1c | |||
| 4aaf7dbdc6 | |||
| 95c0aafb68 | |||
| 5062e008c6 | |||
| 55badc1e08 | |||
| bdc621fb36 | |||
| 0ec5c05de8 | |||
| d10634c7d6 | |||
| a7bad217eb | |||
| f2c13c7d29 | |||
| 3444ff4e29 | |||
| 26f4da7ba5 | |||
| 4c6b4f5d4d | |||
| 70797753df | |||
| ec3216072d | |||
| 810c8d4d72 | |||
| ce688fc0bf | |||
| c131896432 | |||
| 42e6eec9e9 | |||
| fe00fe6a25 | |||
| c97b7c841f | |||
| 2d0387fe63 | |||
| 71d3047ef0 | |||
| d86cc38b2a | |||
| 21b2efd268 | |||
| badd522d60 | |||
| ecd3f600d9 | |||
| 099df17e77 | |||
| c88e42eba2 | |||
| 89058ebd49 | |||
| d8204ab7ed | |||
| e2ea1af4c8 | |||
| 08780475d0 | |||
| 6eb2742e7d | |||
| c1b7e12b0b | |||
| 53d44ff42a | |||
| 6331dea8b0 | |||
| 240beec7de | |||
| 7de167b21b | |||
| 49af014a84 | |||
| 73cf1c6ff9 | |||
| f8b1e14b74 | |||
| 265e6f9a15 | |||
| 40e995da88 | |||
| 6e4fb7fd4b | |||
| 0695ad7ae6 | |||
| eb6b07531a | |||
| 2d6846fe03 | |||
| a5bfd40233 | |||
| a40500eea9 | |||
| f8212f102f | |||
| 59302b465d | |||
| efafe44db1 | |||
| 6a2f81e873 | |||
| 3a43337735 | |||
| b6df89d24c | |||
| 10d992a7e4 | |||
| 5c63618b30 | |||
| 7db0b78e88 | |||
| 979492449e | |||
| 6fbe239313 | |||
| 26527e7dae | |||
| 04a57e92c2 | |||
| d59efa0b5c | |||
| 4216ced493 | |||
| 9f4f493486 | |||
| 63d86f1263 | |||
| 398a5806e7 | |||
| 1adc734801 | |||
| 0ae6dfd565 | |||
| 8531bac6cd | |||
| ce13c00ebd | |||
| 2857c3b46b | |||
| d944885ce9 | |||
| 62d1535e76 | |||
| 46556d308a | |||
| fc5481dbe4 | |||
| 01e60a670c | |||
| c4010854a5 | |||
| fb1311cdae | |||
| 4aa76ce673 |
+2
-3
@@ -6,15 +6,14 @@
|
|||||||
# Local environment (secrets)
|
# Local environment (secrets)
|
||||||
.env
|
.env
|
||||||
|
|
||||||
# Local-only scripts
|
|
||||||
script/local-release
|
|
||||||
|
|
||||||
# App specific (root-level; huskies subdirectory patterns live in .huskies/.gitignore)
|
# App specific (root-level; huskies subdirectory patterns live in .huskies/.gitignore)
|
||||||
store.json
|
store.json
|
||||||
_merge_parsed.json
|
_merge_parsed.json
|
||||||
.huskies_port
|
.huskies_port
|
||||||
.huskies/bot.toml.bak
|
.huskies/bot.toml.bak
|
||||||
.huskies/build_hash
|
.huskies/build_hash
|
||||||
|
# Phantom 0-byte pipeline.db sometimes appears at repo root from old code; canonical DB lives at .huskies/pipeline.db
|
||||||
|
/pipeline.db
|
||||||
|
|
||||||
# Per-worktree planning file (written by coder agents, must never reach squash commits)
|
# Per-worktree planning file (written by coder agents, must never reach squash commits)
|
||||||
PLAN.md
|
PLAN.md
|
||||||
|
|||||||
+1
-1
@@ -56,7 +56,7 @@ There are no exceptions. The merge gate runs `source-map-check` and rejects the
|
|||||||
Before committing, run `cargo run -p source-map-gen --bin source-map-check -- --worktree . --base master` and address every missing-docs direction it prints. If you added a new module file (e.g. `foo.rs` or `foo/mod.rs`), the FIRST line of that file MUST be a `//! What this module is for` doc comment.
|
Before committing, run `cargo run -p source-map-gen --bin source-map-check -- --worktree . --base master` and address every missing-docs direction it prints. If you added a new module file (e.g. `foo.rs` or `foo/mod.rs`), the FIRST line of that file MUST be a `//! What this module is for` doc comment.
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
Docs live in `website/docs/*.html` (static HTML), **not** Markdown files. When a story asks you to document something, edit the relevant `.html` file in `website/docs/`.
|
Docs live in `website/app/docs/*.tsx` (Next.js pages), **not** Markdown files. When a story asks you to document something, edit the relevant `.tsx` file under `website/app/docs/`. Run `npm run build` in `website/` to verify your changes render correctly.
|
||||||
|
|
||||||
## Configuration files
|
## Configuration files
|
||||||
- Agent config: `.huskies/agents.toml` (preferred) or `[[agent]]` blocks in `.huskies/project.toml`
|
- Agent config: `.huskies/agents.toml` (preferred) or `[[agent]]` blocks in `.huskies/project.toml`
|
||||||
|
|||||||
+157
-18
@@ -541,6 +541,7 @@
|
|||||||
"enum TerminationReason",
|
"enum TerminationReason",
|
||||||
"enum PipelineStage",
|
"enum PipelineStage",
|
||||||
"fn pipeline_stage",
|
"fn pipeline_stage",
|
||||||
|
"fn canonical_pipeline_stage",
|
||||||
"fn agent_config_stage",
|
"fn agent_config_stage",
|
||||||
"struct CompletionReport",
|
"struct CompletionReport",
|
||||||
"struct TokenUsage",
|
"struct TokenUsage",
|
||||||
@@ -678,9 +679,7 @@
|
|||||||
"server/src/agents/pool/pipeline/mod.rs": [],
|
"server/src/agents/pool/pipeline/mod.rs": [],
|
||||||
"server/src/agents/pool/process.rs": [
|
"server/src/agents/pool/process.rs": [
|
||||||
"fn kill_all_children",
|
"fn kill_all_children",
|
||||||
"fn kill_child_for_key",
|
"fn kill_child_for_key"
|
||||||
"fn inject_child_killer",
|
|
||||||
"fn child_killer_count"
|
|
||||||
],
|
],
|
||||||
"server/src/agents/pool/query.rs": [
|
"server/src/agents/pool/query.rs": [
|
||||||
"fn available_agents_for_stage",
|
"fn available_agents_for_stage",
|
||||||
@@ -697,6 +696,7 @@
|
|||||||
"server/src/agents/pool/start/spawn.rs": [
|
"server/src/agents/pool/start/spawn.rs": [
|
||||||
"fn maybe_cap_for_merge_fixup",
|
"fn maybe_cap_for_merge_fixup",
|
||||||
"fn maybe_inject_gate_failure",
|
"fn maybe_inject_gate_failure",
|
||||||
|
"fn inject_worktree_disallowed_tools",
|
||||||
"fn run_agent_spawn"
|
"fn run_agent_spawn"
|
||||||
],
|
],
|
||||||
"server/src/agents/pool/start/tests_concurrency.rs": [],
|
"server/src/agents/pool/start/tests_concurrency.rs": [],
|
||||||
@@ -707,6 +707,7 @@
|
|||||||
],
|
],
|
||||||
"server/src/agents/pool/stop.rs": [
|
"server/src/agents/pool/stop.rs": [
|
||||||
"fn stop_agent",
|
"fn stop_agent",
|
||||||
|
"fn reconcile_canonical_agents",
|
||||||
"fn remove_agents_for_story"
|
"fn remove_agents_for_story"
|
||||||
],
|
],
|
||||||
"server/src/agents/pool/test_helpers.rs": [
|
"server/src/agents/pool/test_helpers.rs": [
|
||||||
@@ -752,9 +753,7 @@
|
|||||||
"fn run_agent_pty_streaming"
|
"fn run_agent_pty_streaming"
|
||||||
],
|
],
|
||||||
"server/src/agents/pty/types.rs": [
|
"server/src/agents/pty/types.rs": [
|
||||||
"struct PtyResult",
|
"struct PtyResult"
|
||||||
"fn composite_key",
|
|
||||||
"struct ChildKillerGuard"
|
|
||||||
],
|
],
|
||||||
"server/src/agents/runtime/claude_code.rs": [
|
"server/src/agents/runtime/claude_code.rs": [
|
||||||
"struct ClaudeCodeRuntime",
|
"struct ClaudeCodeRuntime",
|
||||||
@@ -807,6 +806,10 @@
|
|||||||
"fn build_backlog_from_items"
|
"fn build_backlog_from_items"
|
||||||
],
|
],
|
||||||
"server/src/chat/commands/cleanup_worktrees.rs": [],
|
"server/src/chat/commands/cleanup_worktrees.rs": [],
|
||||||
|
"server/src/chat/commands/convert.rs": [
|
||||||
|
"fn handle_convert",
|
||||||
|
"fn convert_by_number"
|
||||||
|
],
|
||||||
"server/src/chat/commands/cost.rs": [
|
"server/src/chat/commands/cost.rs": [
|
||||||
"fn handle_cost",
|
"fn handle_cost",
|
||||||
"fn extract_agent_type"
|
"fn extract_agent_type"
|
||||||
@@ -858,6 +861,9 @@
|
|||||||
"server/src/chat/commands/move_story.rs": [
|
"server/src/chat/commands/move_story.rs": [
|
||||||
"fn handle_move"
|
"fn handle_move"
|
||||||
],
|
],
|
||||||
|
"server/src/chat/commands/new_project.rs": [
|
||||||
|
"fn handle_new_project_fallback"
|
||||||
|
],
|
||||||
"server/src/chat/commands/overview.rs": [
|
"server/src/chat/commands/overview.rs": [
|
||||||
"fn handle_overview"
|
"fn handle_overview"
|
||||||
],
|
],
|
||||||
@@ -898,6 +904,13 @@
|
|||||||
"server/src/chat/commands/unreleased.rs": [
|
"server/src/chat/commands/unreleased.rs": [
|
||||||
"fn handle_unreleased"
|
"fn handle_unreleased"
|
||||||
],
|
],
|
||||||
|
"server/src/chat/dispatcher.rs": [
|
||||||
|
"type SpawnFn",
|
||||||
|
"struct ChatDispatcher",
|
||||||
|
"fn new",
|
||||||
|
"fn submit",
|
||||||
|
"fn stop"
|
||||||
|
],
|
||||||
"server/src/chat/history.rs": [
|
"server/src/chat/history.rs": [
|
||||||
"type ChatConversationHistory",
|
"type ChatConversationHistory",
|
||||||
"fn load_chat_history",
|
"fn load_chat_history",
|
||||||
@@ -908,6 +921,7 @@
|
|||||||
],
|
],
|
||||||
"server/src/chat/mod.rs": [
|
"server/src/chat/mod.rs": [
|
||||||
"mod commands",
|
"mod commands",
|
||||||
|
"mod dispatcher",
|
||||||
"mod history",
|
"mod history",
|
||||||
"mod lookup",
|
"mod lookup",
|
||||||
"mod test_helpers",
|
"mod test_helpers",
|
||||||
@@ -969,6 +983,8 @@
|
|||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/bot/format.rs": [
|
"server/src/chat/transport/matrix/bot/format.rs": [
|
||||||
"fn format_startup_announcement",
|
"fn format_startup_announcement",
|
||||||
|
"fn format_gateway_ready_announcement",
|
||||||
|
"fn format_gateway_rollback_announcement",
|
||||||
"fn markdown_to_html"
|
"fn markdown_to_html"
|
||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/bot/history.rs": [
|
"server/src/chat/transport/matrix/bot/history.rs": [
|
||||||
@@ -990,10 +1006,10 @@
|
|||||||
"fn handle_message"
|
"fn handle_message"
|
||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/bot/messages/mod.rs": [
|
"server/src/chat/transport/matrix/bot/messages/mod.rs": [
|
||||||
"fn format_user_prompt",
|
"fn format_user_prompt"
|
||||||
"fn format_drained_events"
|
|
||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/bot/messages/on_room_message.rs": [
|
"server/src/chat/transport/matrix/bot/messages/on_room_message.rs": [
|
||||||
|
"fn eval_switch_command",
|
||||||
"fn on_room_message"
|
"fn on_room_message"
|
||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/bot/mod.rs": [
|
"server/src/chat/transport/matrix/bot/mod.rs": [
|
||||||
@@ -1037,6 +1053,7 @@
|
|||||||
"fn default_permission_timeout_secs",
|
"fn default_permission_timeout_secs",
|
||||||
"fn default_aggregated_notifications_poll_interval_secs",
|
"fn default_aggregated_notifications_poll_interval_secs",
|
||||||
"fn default_aggregated_notifications_enabled",
|
"fn default_aggregated_notifications_enabled",
|
||||||
|
"fn default_coalesce_window_ms",
|
||||||
"fn default_transport",
|
"fn default_transport",
|
||||||
"fn default_whatsapp_provider",
|
"fn default_whatsapp_provider",
|
||||||
"struct BotConfig"
|
"struct BotConfig"
|
||||||
@@ -1046,6 +1063,9 @@
|
|||||||
"fn extract_delete_command",
|
"fn extract_delete_command",
|
||||||
"fn handle_delete"
|
"fn handle_delete"
|
||||||
],
|
],
|
||||||
|
"server/src/chat/transport/matrix/health.rs": [
|
||||||
|
"fn run_health_check"
|
||||||
|
],
|
||||||
"server/src/chat/transport/matrix/htop.rs": [
|
"server/src/chat/transport/matrix/htop.rs": [
|
||||||
"enum HtopCommand",
|
"enum HtopCommand",
|
||||||
"struct HtopSession",
|
"struct HtopSession",
|
||||||
@@ -1062,17 +1082,40 @@
|
|||||||
"mod commands",
|
"mod commands",
|
||||||
"mod config",
|
"mod config",
|
||||||
"mod delete",
|
"mod delete",
|
||||||
|
"mod health",
|
||||||
"mod htop",
|
"mod htop",
|
||||||
|
"mod new_project",
|
||||||
|
"mod project_rebuild",
|
||||||
"mod rebuild",
|
"mod rebuild",
|
||||||
"mod reset",
|
"mod reset",
|
||||||
"mod rmtree",
|
"mod rmtree",
|
||||||
|
"mod sled_upgrade",
|
||||||
"mod start",
|
"mod start",
|
||||||
"mod transport_impl",
|
"mod transport_impl",
|
||||||
"fn spawn_bot"
|
"fn spawn_bot"
|
||||||
],
|
],
|
||||||
|
"server/src/chat/transport/matrix/new_project.rs": [
|
||||||
|
"struct NewProjectCommand",
|
||||||
|
"fn extract_new_project_command",
|
||||||
|
"fn apply_project_config",
|
||||||
|
"fn detect_stack",
|
||||||
|
"fn image_for_stack",
|
||||||
|
"fn resolve_git_identity",
|
||||||
|
"fn handle_new_project",
|
||||||
|
"fn dockerfile_for_project",
|
||||||
|
"fn build_project_image",
|
||||||
|
"fn project_docker_run_args",
|
||||||
|
"fn resolve_gateway_url"
|
||||||
|
],
|
||||||
|
"server/src/chat/transport/matrix/project_rebuild.rs": [
|
||||||
|
"struct ProjectRebuildCommand",
|
||||||
|
"fn extract_project_rebuild_command",
|
||||||
|
"fn handle_project_rebuild"
|
||||||
|
],
|
||||||
"server/src/chat/transport/matrix/rebuild.rs": [
|
"server/src/chat/transport/matrix/rebuild.rs": [
|
||||||
"struct RebuildCommand",
|
"struct RebuildCommand",
|
||||||
"fn extract_rebuild_command",
|
"fn extract_rebuild_command",
|
||||||
|
"fn extract_rebuild_gateway_command",
|
||||||
"fn handle_rebuild"
|
"fn handle_rebuild"
|
||||||
],
|
],
|
||||||
"server/src/chat/transport/matrix/reset.rs": [
|
"server/src/chat/transport/matrix/reset.rs": [
|
||||||
@@ -1085,6 +1128,12 @@
|
|||||||
"fn extract_rmtree_command",
|
"fn extract_rmtree_command",
|
||||||
"fn handle_rmtree"
|
"fn handle_rmtree"
|
||||||
],
|
],
|
||||||
|
"server/src/chat/transport/matrix/sled_upgrade.rs": [
|
||||||
|
"enum UpgradeCommand",
|
||||||
|
"fn extract_upgrade_command",
|
||||||
|
"fn handle_upgrade_list_projects",
|
||||||
|
"fn handle_sled_upgrade"
|
||||||
|
],
|
||||||
"server/src/chat/transport/matrix/start.rs": [
|
"server/src/chat/transport/matrix/start.rs": [
|
||||||
"enum StartCommand",
|
"enum StartCommand",
|
||||||
"fn extract_start_command",
|
"fn extract_start_command",
|
||||||
@@ -1275,6 +1324,13 @@
|
|||||||
"fn delete_agent_throttle",
|
"fn delete_agent_throttle",
|
||||||
"fn extract_agent_throttle_view"
|
"fn extract_agent_throttle_view"
|
||||||
],
|
],
|
||||||
|
"server/src/crdt_state/lww_maps/event_log.rs": [
|
||||||
|
"const GAP_PIPELINE_EVENT",
|
||||||
|
"struct EventLogEntryRaw",
|
||||||
|
"fn append_event_log_entry",
|
||||||
|
"fn append_gap_log_entry",
|
||||||
|
"fn read_all_event_log_entries"
|
||||||
|
],
|
||||||
"server/src/crdt_state/lww_maps/gateway_projects.rs": [
|
"server/src/crdt_state/lww_maps/gateway_projects.rs": [
|
||||||
"fn write_gateway_project",
|
"fn write_gateway_project",
|
||||||
"fn read_all_gateway_projects",
|
"fn read_all_gateway_projects",
|
||||||
@@ -1282,6 +1338,12 @@
|
|||||||
"fn delete_gateway_project",
|
"fn delete_gateway_project",
|
||||||
"fn extract_gateway_project_view"
|
"fn extract_gateway_project_view"
|
||||||
],
|
],
|
||||||
|
"server/src/crdt_state/lww_maps/llm_sessions.rs": [
|
||||||
|
"fn write_llm_session",
|
||||||
|
"fn read_llm_session",
|
||||||
|
"fn assemble_and_advance_session",
|
||||||
|
"fn extract_llm_session_view"
|
||||||
|
],
|
||||||
"server/src/crdt_state/lww_maps/merge_jobs.rs": [
|
"server/src/crdt_state/lww_maps/merge_jobs.rs": [
|
||||||
"fn write_merge_job",
|
"fn write_merge_job",
|
||||||
"fn read_all_merge_jobs",
|
"fn read_all_merge_jobs",
|
||||||
@@ -1357,10 +1419,13 @@
|
|||||||
"fn rebuild_active_agent_index",
|
"fn rebuild_active_agent_index",
|
||||||
"fn rebuild_test_job_index",
|
"fn rebuild_test_job_index",
|
||||||
"fn rebuild_agent_throttle_index",
|
"fn rebuild_agent_throttle_index",
|
||||||
"fn rebuild_gateway_project_index"
|
"fn rebuild_gateway_project_index",
|
||||||
|
"fn rebuild_llm_session_index"
|
||||||
],
|
],
|
||||||
"server/src/crdt_state/state/init.rs": [
|
"server/src/crdt_state/state/init.rs": [
|
||||||
"fn init"
|
"enum PersistMsg",
|
||||||
|
"fn init",
|
||||||
|
"fn flush_persistence"
|
||||||
],
|
],
|
||||||
"server/src/crdt_state/state/mod.rs": [
|
"server/src/crdt_state/state/mod.rs": [
|
||||||
"fn subscribe",
|
"fn subscribe",
|
||||||
@@ -1371,6 +1436,7 @@
|
|||||||
"fn init_for_test"
|
"fn init_for_test"
|
||||||
],
|
],
|
||||||
"server/src/crdt_state/state/statics.rs": [
|
"server/src/crdt_state/state/statics.rs": [
|
||||||
|
"static PERSIST_PENDING",
|
||||||
"static CRDT_EVENT_TX",
|
"static CRDT_EVENT_TX",
|
||||||
"static SYNC_TX",
|
"static SYNC_TX",
|
||||||
"static ALL_OPS",
|
"static ALL_OPS",
|
||||||
@@ -1386,6 +1452,12 @@
|
|||||||
"struct CrdtEvent",
|
"struct CrdtEvent",
|
||||||
"struct GatewayConfigCrdt",
|
"struct GatewayConfigCrdt",
|
||||||
"struct PipelineDoc",
|
"struct PipelineDoc",
|
||||||
|
"struct EventLogEntryCrdt",
|
||||||
|
"struct LlmSessionCrdt",
|
||||||
|
"enum ScopeFilter",
|
||||||
|
"fn from_scope_str",
|
||||||
|
"fn to_scope_str",
|
||||||
|
"struct LlmSessionView",
|
||||||
"struct PipelineItemCrdt",
|
"struct PipelineItemCrdt",
|
||||||
"struct NodePresenceCrdt",
|
"struct NodePresenceCrdt",
|
||||||
"struct EpicId",
|
"struct EpicId",
|
||||||
@@ -1441,7 +1513,9 @@
|
|||||||
"fn migrate_legacy_stage_strings",
|
"fn migrate_legacy_stage_strings",
|
||||||
"fn migrate_node_claims_to_agent_claims",
|
"fn migrate_node_claims_to_agent_claims",
|
||||||
"fn migrate_merge_job",
|
"fn migrate_merge_job",
|
||||||
"fn purge_done_stage_merge_jobs"
|
"fn purge_done_stage_merge_jobs",
|
||||||
|
"fn migrate_zombie_pipeline_rows",
|
||||||
|
"fn sweep_zombie_rows"
|
||||||
],
|
],
|
||||||
"server/src/crdt_state/write/mod.rs": [],
|
"server/src/crdt_state/write/mod.rs": [],
|
||||||
"server/src/crdt_state/write/tests.rs": [],
|
"server/src/crdt_state/write/tests.rs": [],
|
||||||
@@ -1550,7 +1624,11 @@
|
|||||||
"fn named",
|
"fn named",
|
||||||
"fn write_item_with_content",
|
"fn write_item_with_content",
|
||||||
"fn move_item_stage",
|
"fn move_item_stage",
|
||||||
|
"fn sync_item_agent",
|
||||||
"fn delete_item",
|
"fn delete_item",
|
||||||
|
"fn delete_item_sync",
|
||||||
|
"fn sync_item_name",
|
||||||
|
"fn sync_item_depends_on",
|
||||||
"fn next_item_number"
|
"fn next_item_number"
|
||||||
],
|
],
|
||||||
"server/src/db/recover.rs": [
|
"server/src/db/recover.rs": [
|
||||||
@@ -1565,14 +1643,27 @@
|
|||||||
"struct PipelineWriteMsg",
|
"struct PipelineWriteMsg",
|
||||||
"struct PipelineDb",
|
"struct PipelineDb",
|
||||||
"static PIPELINE_DB",
|
"static PIPELINE_DB",
|
||||||
|
"static SHADOW_DB_PATH",
|
||||||
"fn init",
|
"fn init",
|
||||||
"fn backup_pre_pipeline_status",
|
"fn backup_pre_pipeline_status",
|
||||||
"fn check_schema_drift"
|
"fn check_schema_drift"
|
||||||
],
|
],
|
||||||
|
"server/src/event_log/mod.rs": [
|
||||||
|
"type EventId",
|
||||||
|
"struct LoggedEvent",
|
||||||
|
"fn log_transition_event",
|
||||||
|
"fn read_event_log",
|
||||||
|
"fn insert_gap_sentinel",
|
||||||
|
"fn spawn_event_log_subscriber"
|
||||||
|
],
|
||||||
"server/src/gateway/mod.rs": [
|
"server/src/gateway/mod.rs": [
|
||||||
|
"mod rebuild",
|
||||||
"fn build_gateway_route",
|
"fn build_gateway_route",
|
||||||
"fn run"
|
"fn run"
|
||||||
],
|
],
|
||||||
|
"server/src/gateway/rebuild.rs": [
|
||||||
|
"fn rebuild_gateway"
|
||||||
|
],
|
||||||
"server/src/gateway/tests.rs": [],
|
"server/src/gateway/tests.rs": [],
|
||||||
"server/src/gateway_relay.rs": [
|
"server/src/gateway_relay.rs": [
|
||||||
"fn spawn_relay_task"
|
"fn spawn_relay_task"
|
||||||
@@ -1580,11 +1671,6 @@
|
|||||||
"server/src/http/agents_sse.rs": [
|
"server/src/http/agents_sse.rs": [
|
||||||
"fn agent_stream"
|
"fn agent_stream"
|
||||||
],
|
],
|
||||||
"server/src/http/assets.rs": [
|
|
||||||
"fn embedded_asset",
|
|
||||||
"fn embedded_file",
|
|
||||||
"fn embedded_index"
|
|
||||||
],
|
|
||||||
"server/src/http/context.rs": [
|
"server/src/http/context.rs": [
|
||||||
"enum PermissionDecision",
|
"enum PermissionDecision",
|
||||||
"struct PermissionForward",
|
"struct PermissionForward",
|
||||||
@@ -1719,6 +1805,11 @@
|
|||||||
"fn validate_working_dir",
|
"fn validate_working_dir",
|
||||||
"fn tool_run_command"
|
"fn tool_run_command"
|
||||||
],
|
],
|
||||||
|
"server/src/http/mcp/shell_tools/file_tools.rs": [
|
||||||
|
"fn validate_worktree_file_path",
|
||||||
|
"fn tool_edit",
|
||||||
|
"fn tool_write"
|
||||||
|
],
|
||||||
"server/src/http/mcp/shell_tools/mod.rs": [],
|
"server/src/http/mcp/shell_tools/mod.rs": [],
|
||||||
"server/src/http/mcp/shell_tools/script.rs": [
|
"server/src/http/mcp/shell_tools/script.rs": [
|
||||||
"fn tool_run_tests",
|
"fn tool_run_tests",
|
||||||
@@ -1759,6 +1850,9 @@
|
|||||||
"server/src/http/mcp/story_tools/spike.rs": [
|
"server/src/http/mcp/story_tools/spike.rs": [
|
||||||
"fn tool_create_spike"
|
"fn tool_create_spike"
|
||||||
],
|
],
|
||||||
|
"server/src/http/mcp/story_tools/story/convert.rs": [
|
||||||
|
"fn tool_convert_item_type"
|
||||||
|
],
|
||||||
"server/src/http/mcp/story_tools/story/create.rs": [
|
"server/src/http/mcp/story_tools/story/create.rs": [
|
||||||
"fn tool_create_story",
|
"fn tool_create_story",
|
||||||
"fn tool_purge_story"
|
"fn tool_purge_story"
|
||||||
@@ -1817,7 +1911,6 @@
|
|||||||
],
|
],
|
||||||
"server/src/http/mod.rs": [
|
"server/src/http/mod.rs": [
|
||||||
"mod agents_sse",
|
"mod agents_sse",
|
||||||
"mod assets",
|
|
||||||
"mod context",
|
"mod context",
|
||||||
"mod events",
|
"mod events",
|
||||||
"mod identity",
|
"mod identity",
|
||||||
@@ -1834,7 +1927,9 @@
|
|||||||
"fn health_handler",
|
"fn health_handler",
|
||||||
"fn build_routes",
|
"fn build_routes",
|
||||||
"fn rpc_http_handler",
|
"fn rpc_http_handler",
|
||||||
"fn debug_crdt_handler"
|
"fn debug_crdt_handler",
|
||||||
|
"fn upgrade_trigger_handler",
|
||||||
|
"fn serve_binary_handler"
|
||||||
],
|
],
|
||||||
"server/src/http/oauth.rs": [
|
"server/src/http/oauth.rs": [
|
||||||
"fn oauth_authorize",
|
"fn oauth_authorize",
|
||||||
@@ -2150,6 +2245,9 @@
|
|||||||
"struct CompletionResponse",
|
"struct CompletionResponse",
|
||||||
"trait ModelProvider"
|
"trait ModelProvider"
|
||||||
],
|
],
|
||||||
|
"server/src/llm_session/mod.rs": [
|
||||||
|
"fn assemble_prompt_context"
|
||||||
|
],
|
||||||
"server/src/log_buffer.rs": [
|
"server/src/log_buffer.rs": [
|
||||||
"enum LogLevel",
|
"enum LogLevel",
|
||||||
"fn as_str",
|
"fn as_str",
|
||||||
@@ -2170,14 +2268,21 @@
|
|||||||
"mod crdt_state",
|
"mod crdt_state",
|
||||||
"mod crdt_sync",
|
"mod crdt_sync",
|
||||||
"mod crdt_wire",
|
"mod crdt_wire",
|
||||||
|
"mod event_log",
|
||||||
"mod gateway",
|
"mod gateway",
|
||||||
|
"mod llm_session",
|
||||||
"mod log_buffer",
|
"mod log_buffer",
|
||||||
"mod mesh",
|
"mod mesh",
|
||||||
"mod node_identity",
|
"mod node_identity",
|
||||||
|
"mod pidfile",
|
||||||
|
"mod pipeline_event_bus",
|
||||||
"mod pipeline_state",
|
"mod pipeline_state",
|
||||||
|
"mod process_kill",
|
||||||
"mod rebuild",
|
"mod rebuild",
|
||||||
"mod services",
|
"mod services",
|
||||||
"mod sled_uplink",
|
"mod sled_uplink",
|
||||||
|
"mod trampoline",
|
||||||
|
"mod upgrade",
|
||||||
"mod validation"
|
"mod validation"
|
||||||
],
|
],
|
||||||
"server/src/mesh.rs": [
|
"server/src/mesh.rs": [
|
||||||
@@ -2200,6 +2305,19 @@
|
|||||||
"fn init_identity",
|
"fn init_identity",
|
||||||
"fn get_identity"
|
"fn get_identity"
|
||||||
],
|
],
|
||||||
|
"server/src/pidfile.rs": [
|
||||||
|
"struct PidfileGuard",
|
||||||
|
"fn acquire_gateway_pidfile",
|
||||||
|
"fn acquire_gateway_pidfile_at"
|
||||||
|
],
|
||||||
|
"server/src/pipeline_event_bus.rs": [
|
||||||
|
"struct BusEvent",
|
||||||
|
"fn init",
|
||||||
|
"fn broadcast",
|
||||||
|
"fn subscribe",
|
||||||
|
"fn render_event",
|
||||||
|
"fn event_matches_persona"
|
||||||
|
],
|
||||||
"server/src/pipeline_state/apply.rs": [
|
"server/src/pipeline_state/apply.rs": [
|
||||||
"enum ApplyError",
|
"enum ApplyError",
|
||||||
"fn apply_transition",
|
"fn apply_transition",
|
||||||
@@ -2273,6 +2391,11 @@
|
|||||||
"fn stage_label",
|
"fn stage_label",
|
||||||
"fn stage_dir_name"
|
"fn stage_dir_name"
|
||||||
],
|
],
|
||||||
|
"server/src/process_kill.rs": [
|
||||||
|
"fn sigkill_pids_and_verify",
|
||||||
|
"fn pids_matching",
|
||||||
|
"fn descendant_pids"
|
||||||
|
],
|
||||||
"server/src/rebuild.rs": [
|
"server/src/rebuild.rs": [
|
||||||
"enum ShutdownReason",
|
"enum ShutdownReason",
|
||||||
"struct BotShutdownNotifier",
|
"struct BotShutdownNotifier",
|
||||||
@@ -2932,6 +3055,7 @@
|
|||||||
"fn subscribe_logs",
|
"fn subscribe_logs",
|
||||||
"fn subscribe_watcher",
|
"fn subscribe_watcher",
|
||||||
"fn subscribe_status",
|
"fn subscribe_status",
|
||||||
|
"fn subscribe_persona_pipeline_events",
|
||||||
"fn subscribe_reconciliation"
|
"fn subscribe_reconciliation"
|
||||||
],
|
],
|
||||||
"server/src/service/ws/message/convert.rs": [
|
"server/src/service/ws/message/convert.rs": [
|
||||||
@@ -3004,6 +3128,19 @@
|
|||||||
"fn from_path",
|
"fn from_path",
|
||||||
"fn path"
|
"fn path"
|
||||||
],
|
],
|
||||||
|
"server/src/trampoline.rs": [
|
||||||
|
"struct TrampolineJob",
|
||||||
|
"fn write_job_atomic",
|
||||||
|
"fn spawn_detached_trampoline",
|
||||||
|
"fn execute_trampoline_core",
|
||||||
|
"fn run_trampoline"
|
||||||
|
],
|
||||||
|
"server/src/upgrade.rs": [
|
||||||
|
"fn fetch_and_replace_binary",
|
||||||
|
"fn upgrade_and_reexec",
|
||||||
|
"fn run_cli_upgrade",
|
||||||
|
"fn resolve_target_path"
|
||||||
|
],
|
||||||
"server/src/validation/error.rs": [
|
"server/src/validation/error.rs": [
|
||||||
"enum ValidationError",
|
"enum ValidationError",
|
||||||
"fn format_errors_as_json"
|
"fn format_errors_as_json"
|
||||||
@@ -3065,6 +3202,8 @@
|
|||||||
"struct UnblockStoryRequest",
|
"struct UnblockStoryRequest",
|
||||||
"fn from_json",
|
"fn from_json",
|
||||||
"struct FreezeStoryRequest",
|
"struct FreezeStoryRequest",
|
||||||
|
"fn from_json",
|
||||||
|
"struct ConvertItemTypeRequest",
|
||||||
"fn from_json"
|
"fn from_json"
|
||||||
],
|
],
|
||||||
"server/src/validation/sanitize.rs": [
|
"server/src/validation/sanitize.rs": [
|
||||||
|
|||||||
@@ -0,0 +1,306 @@
|
|||||||
|
# Chat-Driven Project Bootstrap
|
||||||
|
|
||||||
|
Design overview for going from "I want a new project" to a running,
|
||||||
|
container-isolated, editor-accessible huskies project in one chat command.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A user can say to Timmy in chat:
|
||||||
|
|
||||||
|
```
|
||||||
|
new project myapp --stack rust
|
||||||
|
new project legacy-rails --git git@github.com:me/legacy-rails.git
|
||||||
|
```
|
||||||
|
|
||||||
|
and end up with:
|
||||||
|
|
||||||
|
1. A fresh docker container running the project's huskies node.
|
||||||
|
2. The project's source code bind-mounted from the host so the user can
|
||||||
|
edit it in any editor.
|
||||||
|
3. SSH into the container so editors can run LSPs, builds, and tests
|
||||||
|
inside the container — never on the host.
|
||||||
|
4. Optional git remote configured for push to GitHub or Gitea.
|
||||||
|
5. The new sled registered with the gateway, so Timmy can drive coders /
|
||||||
|
mergemaster / etc. on the project via existing chat commands.
|
||||||
|
|
||||||
|
Manual repo creation on GitHub/Gitea remains the user's job. Everything
|
||||||
|
downstream of that is orchestrated.
|
||||||
|
|
||||||
|
## Architecture at a Glance
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────┐
|
||||||
|
│ Browser / Matrix │───┐
|
||||||
|
└──────────────────────┘ │
|
||||||
|
▼
|
||||||
|
┌───────────────────────┐
|
||||||
|
│ Gateway (huskies-gw) │
|
||||||
|
│ • chat dispatcher │
|
||||||
|
│ • new-project │
|
||||||
|
│ • routing │
|
||||||
|
└─────────┬─────────────┘
|
||||||
|
│
|
||||||
|
┌─────────┴───────────────────────────────────┐
|
||||||
|
│ docker engine (host) │
|
||||||
|
│ ┌────────────┐ ┌────────────┐ ┌─────────┐ │
|
||||||
|
│ │ project-A │ │ project-B │ │ ... │ │
|
||||||
|
│ │ sled + │ │ sled + │ │ │ │
|
||||||
|
│ │ sshd + │ │ sshd + │ │ │ │
|
||||||
|
│ │ LSPs │ │ LSPs │ │ │ │
|
||||||
|
│ └─────┬──────┘ └─────┬──────┘ └─────────┘ │
|
||||||
|
└────────┼──────────────┼─────────────────────┘
|
||||||
|
│ │
|
||||||
|
bind mount │ │ bind mount
|
||||||
|
┌────────┴───┐ ┌─────┴──────┐
|
||||||
|
│ ~/code/A │ │ ~/code/B │ ◄── host
|
||||||
|
└────────────┘ └────────────┘ editor opens
|
||||||
|
these paths
|
||||||
|
```
|
||||||
|
|
||||||
|
- One container per project. The container runs the project's huskies
|
||||||
|
binary (sled), an SSH server, and the stack-appropriate LSP(s).
|
||||||
|
- Source lives on the host (e.g. `~/code/<project>`), bind-mounted into
|
||||||
|
the container at a known path. Host can git-diff, back up, or edit.
|
||||||
|
- The gateway is editor-agnostic and project-agnostic — it talks to each
|
||||||
|
sled via the existing rendezvous / CRDT-sync protocol.
|
||||||
|
|
||||||
|
## Three Personas
|
||||||
|
|
||||||
|
| Persona | What they do | What they need |
|
||||||
|
|---------|--------------|----------------|
|
||||||
|
| Chat-only user | Drives everything via Matrix/web chat | Installed huskies binary; chat client |
|
||||||
|
| Editor-using technical user | Same + edits source in their editor | SSH config to the container + editor-specific remote-dev setup |
|
||||||
|
| Multi-project user | Several projects running in parallel | Gateway-listed projects, all routable from one chat |
|
||||||
|
|
||||||
|
Chat-only users never touch SSH. Editor users go through a one-time
|
||||||
|
"copy this SSH command into your editor's remote settings" handoff at
|
||||||
|
project creation time.
|
||||||
|
|
||||||
|
## The Bootstrap Chat Command
|
||||||
|
|
||||||
|
```
|
||||||
|
new project <name> [--stack <stack>] [--git <url>] [--path <host-path>]
|
||||||
|
```
|
||||||
|
|
||||||
|
Flow:
|
||||||
|
|
||||||
|
1. **Validate**: name unique among existing projects; host path doesn't already
|
||||||
|
exist; stack (if declared) is one of the supported overlays.
|
||||||
|
2. **Allocate** a fresh per-project port range (gateway picks).
|
||||||
|
3. **Create host directory** at `--path` (default `~/huskies/<name>/`).
|
||||||
|
4. If `--git` provided, `git clone` into that directory; else `git init`.
|
||||||
|
5. **Detect stack** from cloned content if not declared:
|
||||||
|
- `Cargo.toml` → `rust`
|
||||||
|
- `package.json` → `node`
|
||||||
|
- `go.mod` → `go`
|
||||||
|
- `pyproject.toml` / `requirements.txt` / `setup.py` → `python`
|
||||||
|
- `Gemfile` → `ruby`
|
||||||
|
- `pom.xml` / `build.gradle` → `jvm`
|
||||||
|
- Multiple → pick the dominant, warn.
|
||||||
|
- None → minimal base image, user can install tooling later.
|
||||||
|
6. **Compose the container** from `huskies-project-base` + the stack
|
||||||
|
overlay (Dockerfile fragments under `docker/stacks/<stack>/`).
|
||||||
|
7. **Launch** the container with bind mount + port forwards + an
|
||||||
|
auto-generated SSH key.
|
||||||
|
8. **Seed `.huskies/project.toml`** with sensible defaults.
|
||||||
|
9. **Register** the project with the gateway (`gateway_projects` LWW-map).
|
||||||
|
10. **Reply in chat** with: project name, host path, SSH command, and
|
||||||
|
a `huskies status <name>` invocation to verify.
|
||||||
|
|
||||||
|
## Container Template
|
||||||
|
|
||||||
|
Layered:
|
||||||
|
|
||||||
|
- **`huskies-project-base`**: debian-slim + git + huskies binary + sshd
|
||||||
|
+ sudo + a `huskies` user with the SSH pubkey installed.
|
||||||
|
- **`huskies-project-<stack>`**: per-stack additions, pre-built by
|
||||||
|
`script/build-project-images`. E.g. rust gets `rustup` +
|
||||||
|
`rust-analyzer` + `cargo-nextest`; node gets `node@22` +
|
||||||
|
`typescript-language-server`; etc. Stack fragments live in
|
||||||
|
`docker/stacks/<stack>/Dockerfile.fragment`.
|
||||||
|
- **`huskies-project-local-<name>`** *(optional)*: built on the fly at
|
||||||
|
container launch time when the project contains
|
||||||
|
`.huskies/Dockerfile.fragment`. This file is appended after the
|
||||||
|
stack overlay (`FROM huskies-project-<stack>`) so agents can extend
|
||||||
|
their own image without editing shared stack files. Because the
|
||||||
|
fragment lives inside the bind-mounted `/workspace/.huskies/`, changes
|
||||||
|
survive container recreation and are committed alongside the project
|
||||||
|
source. The `project-rebuild` command picks up the fragment
|
||||||
|
automatically when rebuilding.
|
||||||
|
|
||||||
|
Example `.huskies/Dockerfile.fragment` that adds `jq`:
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
RUN apt-get update && apt-get install -y jq
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Project layer**: the bind-mounted `/workspace` is the project source,
|
||||||
|
written by the host's editor, read by the in-container tooling.
|
||||||
|
|
||||||
|
The container's SSH server is bound to a host-local port (not exposed
|
||||||
|
externally). Auth is the per-project keypair generated at bootstrap;
|
||||||
|
the public key sits inside the container, the private key on host.
|
||||||
|
|
||||||
|
## Build Sandbox Model
|
||||||
|
|
||||||
|
The threat: editing code in a host-side editor causes the editor (or its
|
||||||
|
LSP plugin) to run `cargo check` / `npm install` / `pip install` /
|
||||||
|
similar, which executes arbitrary code from project dependencies —
|
||||||
|
`build.rs`, proc-macros, npm `postinstall`, Python `setup.py`, Ruby
|
||||||
|
native-extension build scripts, etc. A malicious dependency compromises
|
||||||
|
the host.
|
||||||
|
|
||||||
|
The mitigation: all build / type-check / dependency-install commands
|
||||||
|
execute **inside the project container**. The host's editor connects to
|
||||||
|
the container over SSH; rust-analyzer (or equivalent) runs inside the
|
||||||
|
container; the host process never `exec`s untrusted build scripts.
|
||||||
|
|
||||||
|
Container isolation is the docker default plus:
|
||||||
|
- No `--privileged`.
|
||||||
|
- No host bind mounts beyond the project source and the SSH key.
|
||||||
|
- No host network beyond the gateway's CRDT sync port.
|
||||||
|
- `--cap-drop=ALL` plus the minimum caps needed (probably none).
|
||||||
|
|
||||||
|
This isn't a hardened sandbox in the gvisor / Firecracker sense — a
|
||||||
|
docker-escape exploit on a compromised container still escalates to
|
||||||
|
host. For most consumer threat models (malicious crate from
|
||||||
|
crates.io / npm), docker's default isolation is sufficient. Tighter
|
||||||
|
sandboxing (gvisor) is a separate future spike if needed.
|
||||||
|
|
||||||
|
## Editor Connection — Editor-Agnostic SSH
|
||||||
|
|
||||||
|
| Editor | Connection mechanism |
|
||||||
|
|--------|----------------------|
|
||||||
|
| VSCode | Remote-SSH extension |
|
||||||
|
| JetBrains (IntelliJ/Rover) | JetBrains Gateway (SSH) |
|
||||||
|
| Zed | Built-in SSH remoting (mac/linux only today) |
|
||||||
|
| Vim/Neovim | SSH terminal session, or local nvim + LSP-over-SSH |
|
||||||
|
| Emacs | TRAMP + remote LSP via lsp-mode |
|
||||||
|
|
||||||
|
All converge on: `ssh huskies@127.0.0.1 -p <project-port> -i ~/.huskies/<name>/id_ed25519`.
|
||||||
|
That string is emitted in the bootstrap chat reply.
|
||||||
|
|
||||||
|
## Git Integration
|
||||||
|
|
||||||
|
- Initial setup is `git init` or `git clone` inside the container.
|
||||||
|
- For push: user's existing GitHub / Gitea SSH key is bind-mounted
|
||||||
|
read-only into the container at `~/.ssh/id_*`, OR the user supplies a
|
||||||
|
push token via `huskies secrets set GIT_TOKEN=...` (stored as a Fly
|
||||||
|
secret equivalent — for now, a chmod 600 file in the container).
|
||||||
|
- The container's `git` config gets `user.name` / `user.email` from the
|
||||||
|
gateway-level user identity.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| Decision | Choice | Alternative |
|
||||||
|
|----------|--------|-------------|
|
||||||
|
| Container per project | One container per project | One container many projects: simpler but breaks isolation, breaks per-project deps |
|
||||||
|
| Editor model | SSH-remote (any editor) | VSCode Dev Containers only: simpler config but locks out everyone else |
|
||||||
|
| Source location | Bind mount from host | Inside container only: breaks "I can also edit on my laptop" requirement |
|
||||||
|
| Stack detection | Auto from project files, override with `--stack` | Always declared: more friction at bootstrap |
|
||||||
|
| Push secrets | Bind-mounted host SSH key OR per-project token | Gateway holds tokens: bigger blast radius |
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Per-project resource limits.** Should each container have a hard
|
||||||
|
CPU / RAM cap so a runaway agent doesn't starve the host?
|
||||||
|
2. **Lifecycle / cleanup.** If the user deletes a project from chat,
|
||||||
|
what gets removed? Container yes; host source no (data loss); git
|
||||||
|
remotes yes? Need a confirm step.
|
||||||
|
3. **Multi-tenant.** Out of scope for this design (that's huskies.dev
|
||||||
|
territory). This doc assumes single-user local-only.
|
||||||
|
4. **Windows specifics.** Bind mounts work but line-ending /
|
||||||
|
permission edge cases. Probably document "use WSL2 for best
|
||||||
|
experience" rather than fight Windows native paths.
|
||||||
|
5. **Gateway-on-host vs gateway-in-container.** The gateway today runs
|
||||||
|
in its own container. New per-project containers connect via docker
|
||||||
|
network. Need to confirm the network plumbing works for arbitrary
|
||||||
|
per-project containers, not just the manually-configured ones.
|
||||||
|
|
||||||
|
## Phasing
|
||||||
|
|
||||||
|
The work breaks naturally into:
|
||||||
|
|
||||||
|
- **Phase 0 (now):** this design doc.
|
||||||
|
- **Phase 1:** chat command exists and provisions a bare project
|
||||||
|
container (no stack overlay, no SSH, no git clone — just
|
||||||
|
"start a container, register with gateway"). Validates the
|
||||||
|
orchestration shell.
|
||||||
|
- **Phase 2:** stack-aware container template — base image + overlays;
|
||||||
|
detection from project files.
|
||||||
|
- **Phase 3:** SSH-remote editor access — sshd in the container,
|
||||||
|
per-project keypair, chat-reply emits the connection string.
|
||||||
|
- **Phase 4:** git integration — `--git <url>` clones, host SSH key
|
||||||
|
mount, push verification.
|
||||||
|
- **Phase 5:** per-project resource limits + cleanup chat commands.
|
||||||
|
- **Phase 6:** `--adopt <dir>` wraps a container around an existing
|
||||||
|
checkout. No clone or init — bind-mount only.
|
||||||
|
- **Phase 7 (story 1137):** First-run init flow — config summary and
|
||||||
|
chat-driven overrides (see below).
|
||||||
|
|
||||||
|
Each phase ships independently and is usable on its own. Phase 1 alone
|
||||||
|
gives chat-only users a working project; later phases add the editor
|
||||||
|
and git polish.
|
||||||
|
|
||||||
|
## First-Run Init Flow (Story 1137)
|
||||||
|
|
||||||
|
After a successful `new project ... --adopt` (or any new-project
|
||||||
|
bootstrap), the bot appends a **Default configuration** block to the
|
||||||
|
adoption success reply. This block lists every scaffolded agent with
|
||||||
|
its model, budget cap, and turn limit, and provides ready-to-send
|
||||||
|
override commands.
|
||||||
|
|
||||||
|
### Example reply tail
|
||||||
|
|
||||||
|
```
|
||||||
|
**Default configuration** (3 agents):
|
||||||
|
- coder-1 (coder): model=`sonnet`, budget=$5.00, max_turns=50
|
||||||
|
- qa (qa): model=`sonnet`, budget=$4.00, max_turns=40
|
||||||
|
- mergemaster (mergemaster): model=`sonnet`, budget=$5.00, max_turns=30
|
||||||
|
|
||||||
|
Override via chat: `huskies config myapp coder.model=opus`
|
||||||
|
Project settings: `huskies config myapp default_qa=human`
|
||||||
|
Accept all defaults silently: add `--skip-config` to the bootstrap command.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Config override command
|
||||||
|
|
||||||
|
```
|
||||||
|
huskies config <project> <key>=<value>
|
||||||
|
```
|
||||||
|
|
||||||
|
The gateway resolves the project's `host_path` from `projects.toml`,
|
||||||
|
then writes the setting to `.huskies/agents.toml` or
|
||||||
|
`.huskies/project.toml` on the host.
|
||||||
|
|
||||||
|
**Agent fields** (`<stage_or_name>.<field>=<value>`):
|
||||||
|
|
||||||
|
| Key | Target | Supported values |
|
||||||
|
|-----|--------|-----------------|
|
||||||
|
| `coder.model` | agents.toml, coder stage | `sonnet`, `opus`, any model string |
|
||||||
|
| `qa.model` | agents.toml, qa stage | same |
|
||||||
|
| `mergemaster.model` | agents.toml, mergemaster stage | same |
|
||||||
|
| `coder.max_turns` | agents.toml, coder stage | integer |
|
||||||
|
| `coder.max_budget` | agents.toml, coder stage | decimal (USD) |
|
||||||
|
|
||||||
|
**Project keys** (bare `<key>=<value>`):
|
||||||
|
|
||||||
|
| Key | Notes |
|
||||||
|
|-----|-------|
|
||||||
|
| `default_qa` | `"server"`, `"agent"`, or `"human"` |
|
||||||
|
| `max_retries` | integer |
|
||||||
|
| `max_coders` | integer |
|
||||||
|
| `base_branch` | branch name string |
|
||||||
|
| `timezone` | IANA timezone (e.g. `"Europe/London"`) |
|
||||||
|
| `default_coder_model` | model string |
|
||||||
|
|
||||||
|
### Skip path
|
||||||
|
|
||||||
|
Pass `--skip-config` to suppress the config block entirely:
|
||||||
|
|
||||||
|
```
|
||||||
|
new project myapp --adopt /path/to/checkout --skip-config
|
||||||
|
```
|
||||||
|
|
||||||
|
The success reply is identical to pre-1137 output — only the SSH
|
||||||
|
command and registration summary, no agent listing.
|
||||||
Generated
+8
-54
@@ -872,9 +872,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710"
|
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hybrid-array",
|
"hybrid-array",
|
||||||
]
|
]
|
||||||
@@ -1137,7 +1137,7 @@ checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer 0.12.0",
|
"block-buffer 0.12.0",
|
||||||
"const-oid 0.10.2",
|
"const-oid 0.10.2",
|
||||||
"crypto-common 0.2.1",
|
"crypto-common 0.2.2",
|
||||||
"ctutils",
|
"ctutils",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1911,7 +1911,7 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "huskies"
|
name = "huskies"
|
||||||
version = "0.11.0"
|
version = "0.13.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ammonia",
|
"ammonia",
|
||||||
"async-stream",
|
"async-stream",
|
||||||
@@ -1931,7 +1931,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
"libsqlite3-sys",
|
"libsqlite3-sys",
|
||||||
"matrix-sdk",
|
"matrix-sdk",
|
||||||
"mime_guess",
|
|
||||||
"mockito",
|
"mockito",
|
||||||
"notify",
|
"notify",
|
||||||
"nutype",
|
"nutype",
|
||||||
@@ -1941,7 +1940,6 @@ dependencies = [
|
|||||||
"rand 0.10.1",
|
"rand 0.10.1",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rust-embed",
|
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
@@ -2978,16 +2976,6 @@ version = "0.1.54"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cbf6f36070878c42c5233846cd3de24cf9016828fd47bc22957a687298bb21fc"
|
checksum = "cbf6f36070878c42c5233846cd3de24cf9016828fd47bc22957a687298bb21fc"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "mime_guess"
|
|
||||||
version = "2.0.5"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
|
|
||||||
dependencies = [
|
|
||||||
"mime",
|
|
||||||
"unicase",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "miniz_oxide"
|
name = "miniz_oxide"
|
||||||
version = "0.8.9"
|
version = "0.8.9"
|
||||||
@@ -3119,9 +3107,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.1"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-integer"
|
name = "num-integer"
|
||||||
@@ -4206,40 +4194,6 @@ dependencies = [
|
|||||||
"smallvec",
|
"smallvec",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rust-embed"
|
|
||||||
version = "8.11.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
|
|
||||||
dependencies = [
|
|
||||||
"rust-embed-impl",
|
|
||||||
"rust-embed-utils",
|
|
||||||
"walkdir",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rust-embed-impl"
|
|
||||||
version = "8.11.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"rust-embed-utils",
|
|
||||||
"syn 2.0.117",
|
|
||||||
"walkdir",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rust-embed-utils"
|
|
||||||
version = "8.11.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
|
|
||||||
dependencies = [
|
|
||||||
"sha2 0.10.9",
|
|
||||||
"walkdir",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.2"
|
version = "2.1.2"
|
||||||
@@ -5429,9 +5383,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tower-http"
|
name = "tower-http"
|
||||||
version = "0.6.10"
|
version = "0.6.11"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
|
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-compression",
|
"async-compression",
|
||||||
"bitflags 2.11.1",
|
"bitflags 2.11.1",
|
||||||
|
|||||||
@@ -79,6 +79,10 @@ cd frontend && npm install && npm run dev
|
|||||||
|
|
||||||
Configuration lives in `.huskies/project.toml`. See `.huskies/bot.toml.*.example` for transport setup.
|
Configuration lives in `.huskies/project.toml`. See `.huskies/bot.toml.*.example` for transport setup.
|
||||||
|
|
||||||
|
## Website
|
||||||
|
|
||||||
|
The huskies.dev website source has moved to [crashlabs/huskies-server](https://code.crashlabs.io/crashlabs/huskies-server).
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
Internal architecture documentation lives in [`docs/architecture/`](docs/architecture/):
|
Internal architecture documentation lives in [`docs/architecture/`](docs/architecture/):
|
||||||
|
|||||||
+11
-2
@@ -46,8 +46,17 @@ WORKDIR /app
|
|||||||
# build.rs) can produce the release binary with embedded frontend assets.
|
# build.rs) can produce the release binary with embedded frontend assets.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Build frontend deps first (better layer caching)
|
# Build frontend deps first (better layer caching).
|
||||||
RUN cd frontend && npm ci
|
# Cannot use `npm ci` because of npm's optional-dependencies bug
|
||||||
|
# (npm/cli#4828): platform-specific bindings (e.g. rolldown's
|
||||||
|
# linux-arm64-gnu native binary, introduced by 1119's vite 5→8 upgrade)
|
||||||
|
# get listed in package-lock.json for the lockfile author's platform
|
||||||
|
# only, so `npm ci` skips them on every other platform — the build
|
||||||
|
# then fails at runtime with `Cannot find native binding`. Wipe the
|
||||||
|
# lockfile + node_modules and let `npm install` resolve fresh for the
|
||||||
|
# build platform. The lockfile mutation stays inside the container
|
||||||
|
# image and never reaches the host repo.
|
||||||
|
RUN cd frontend && rm -rf node_modules package-lock.json && npm install
|
||||||
|
|
||||||
# Build the release binary (build.rs runs npm run build for the frontend)
|
# Build the release binary (build.rs runs npm run build for the frontend)
|
||||||
RUN cargo build --release \
|
RUN cargo build --release \
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# huskies-project-base — minimal base for all project containers.
|
||||||
|
#
|
||||||
|
# This image provides git, the huskies server binary, and a non-root user.
|
||||||
|
# It carries no language tooling. Per-stack overlays (docker/stacks/<name>/
|
||||||
|
# Dockerfile.fragment) layer their toolchains on top of this base.
|
||||||
|
#
|
||||||
|
# Prerequisites: build the main `huskies` image first so its binary is
|
||||||
|
# available as a build source.
|
||||||
|
#
|
||||||
|
# docker build -t huskies -f docker/Dockerfile .
|
||||||
|
# docker build -t huskies-project-base -f docker/Dockerfile.base .
|
||||||
|
#
|
||||||
|
# To build a stack image (e.g. rust):
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/rust/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-rust -
|
||||||
|
|
||||||
|
FROM huskies AS huskies-src
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
libssl3 \
|
||||||
|
procps \
|
||||||
|
openssh-server \
|
||||||
|
sudo \
|
||||||
|
nodejs \
|
||||||
|
npm \
|
||||||
|
&& npm install -g @anthropic-ai/claude-code \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Copy the huskies binary and entrypoint from the main image.
|
||||||
|
COPY --from=huskies-src /usr/local/bin/huskies /usr/local/bin/huskies
|
||||||
|
COPY --from=huskies-src /usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
|
# Non-root user — Claude Code refuses --dangerously-skip-permissions as root.
|
||||||
|
# -s /bin/bash required for SSH sessions to start a real shell.
|
||||||
|
RUN groupadd -r huskies \
|
||||||
|
&& useradd -r -g huskies -m -d /home/huskies -s /bin/bash huskies \
|
||||||
|
&& mkdir -p /home/huskies/.claude \
|
||||||
|
&& mkdir -p /home/huskies/.ssh \
|
||||||
|
&& chmod 700 /home/huskies/.ssh \
|
||||||
|
&& chown -R huskies:huskies /home/huskies \
|
||||||
|
&& mkdir -p /workspace \
|
||||||
|
&& chown huskies:huskies /workspace \
|
||||||
|
&& git config --global init.defaultBranch master \
|
||||||
|
&& echo "huskies ALL=(root) NOPASSWD: /usr/sbin/sshd" > /etc/sudoers.d/huskies-sshd \
|
||||||
|
&& chmod 0440 /etc/sudoers.d/huskies-sshd \
|
||||||
|
&& mkdir -p /run/sshd \
|
||||||
|
&& sed -i \
|
||||||
|
-e 's/#PasswordAuthentication yes/PasswordAuthentication no/' \
|
||||||
|
-e 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' \
|
||||||
|
-e 's/UsePAM yes/UsePAM no/' \
|
||||||
|
/etc/ssh/sshd_config
|
||||||
|
|
||||||
|
# Shell profile for SSH sessions: land in /workspace and load toolchain paths.
|
||||||
|
RUN printf 'cd /workspace\n[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env"\n' \
|
||||||
|
> /home/huskies/.profile \
|
||||||
|
&& chown huskies:huskies /home/huskies/.profile
|
||||||
|
|
||||||
|
USER huskies
|
||||||
|
WORKDIR /workspace
|
||||||
|
|
||||||
|
EXPOSE 3001 22
|
||||||
|
|
||||||
|
ENTRYPOINT ["entrypoint.sh"]
|
||||||
|
CMD ["huskies", "/workspace"]
|
||||||
@@ -29,6 +29,9 @@ services:
|
|||||||
- HUSKIES_PORT=3001
|
- HUSKIES_PORT=3001
|
||||||
# Bind to all interfaces so Docker port forwarding works.
|
# Bind to all interfaces so Docker port forwarding works.
|
||||||
- HUSKIES_HOST=0.0.0.0
|
- HUSKIES_HOST=0.0.0.0
|
||||||
|
# Gateway URL so this sled's relay task forwards CRDT events to the gateway.
|
||||||
|
# Uses host.docker.internal so the container can reach the gateway on the host.
|
||||||
|
- HUSKIES_GATEWAY_URL=http://host.docker.internal:3000
|
||||||
# Optional: Matrix bot credentials (if using Matrix integration)
|
# Optional: Matrix bot credentials (if using Matrix integration)
|
||||||
- MATRIX_HOMESERVER=${MATRIX_HOMESERVER:-}
|
- MATRIX_HOMESERVER=${MATRIX_HOMESERVER:-}
|
||||||
- MATRIX_USER=${MATRIX_USER:-}
|
- MATRIX_USER=${MATRIX_USER:-}
|
||||||
|
|||||||
@@ -1,6 +1,32 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
# ── Claude credentials ────────────────────────────────────────────────
|
||||||
|
# The `new project` command bind-mounts the host ~/.claude/.credentials.json
|
||||||
|
# at /run/claude-credentials-src:ro. We copy it here so the huskies user
|
||||||
|
# owns the file and mode 0600 is enforced regardless of host uid/gid.
|
||||||
|
if [ -f /run/claude-credentials-src ]; then
|
||||||
|
mkdir -p /home/huskies/.claude
|
||||||
|
cp /run/claude-credentials-src /home/huskies/.claude/.credentials.json
|
||||||
|
chmod 600 /home/huskies/.claude/.credentials.json
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── SSH authorized key ────────────────────────────────────────────────
|
||||||
|
# HUSKIES_SSH_PUBKEY is set by `new project` when it generates a keypair.
|
||||||
|
# Write it to authorized_keys so the user can connect with the matching
|
||||||
|
# private key stored at ~/.huskies/<project>/id_ed25519 on the host.
|
||||||
|
if [ -n "$HUSKIES_SSH_PUBKEY" ]; then
|
||||||
|
mkdir -p /home/huskies/.ssh
|
||||||
|
chmod 700 /home/huskies/.ssh
|
||||||
|
printf '%s\n' "$HUSKIES_SSH_PUBKEY" > /home/huskies/.ssh/authorized_keys
|
||||||
|
chmod 600 /home/huskies/.ssh/authorized_keys
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── SSH daemon ────────────────────────────────────────────────────────
|
||||||
|
# Start sshd in the background so the container accepts SSH connections.
|
||||||
|
# Uses sudo (huskies has NOPASSWD for /usr/sbin/sshd in sudoers.d).
|
||||||
|
sudo /usr/sbin/sshd -D -e &
|
||||||
|
|
||||||
# ── Git identity ─────────────────────────────────────────────────────
|
# ── Git identity ─────────────────────────────────────────────────────
|
||||||
# Agents commit code inside the container. Without a git identity,
|
# Agents commit code inside the container. Without a git identity,
|
||||||
# commits fail or use garbage defaults. Fail loudly at startup so the
|
# commits fail or use garbage defaults. Fail loudly at startup so the
|
||||||
@@ -25,6 +51,20 @@ export GIT_COMMITTER_NAME="$GIT_USER_NAME"
|
|||||||
export GIT_AUTHOR_EMAIL="$GIT_USER_EMAIL"
|
export GIT_AUTHOR_EMAIL="$GIT_USER_EMAIL"
|
||||||
export GIT_COMMITTER_EMAIL="$GIT_USER_EMAIL"
|
export GIT_COMMITTER_EMAIL="$GIT_USER_EMAIL"
|
||||||
|
|
||||||
|
# ── Git credential helper (HTTPS push) ────────────────────────────────────
|
||||||
|
# If GIT_PUSH_TOKEN is supplied at container creation time, configure git's
|
||||||
|
# built-in credential store so `git push` over HTTPS authenticates without
|
||||||
|
# user interaction. GIT_CLONE_URL provides the host portion of the URL used
|
||||||
|
# as the key in ~/.git-credentials.
|
||||||
|
if [ -n "$GIT_PUSH_TOKEN" ] && [ -n "$GIT_CLONE_URL" ]; then
|
||||||
|
_scheme=$(echo "$GIT_CLONE_URL" | cut -d':' -f1)
|
||||||
|
_host=$(echo "$GIT_CLONE_URL" | sed 's|^https\?://||' | cut -d'/' -f1)
|
||||||
|
git config --global credential.helper store
|
||||||
|
printf '%s://x-access-token:%s@%s\n' "$_scheme" "$GIT_PUSH_TOKEN" "$_host" \
|
||||||
|
> /home/huskies/.git-credentials
|
||||||
|
chmod 600 /home/huskies/.git-credentials
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Frontend native deps ────────────────────────────────────────────
|
# ── Frontend native deps ────────────────────────────────────────────
|
||||||
# The project repo is bind-mounted from the host, so node_modules/
|
# The project repo is bind-mounted from the host, so node_modules/
|
||||||
# may contain native binaries for the wrong platform (e.g. darwin
|
# may contain native binaries for the wrong platform (e.g. darwin
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Go stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Go 1.22, gopls (official Go language server), and standard tooling.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/go/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-go -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Official Go binary distribution — Debian's golang-go package is too old for gopls.
|
||||||
|
# Update GOVERSION to pick up a newer release.
|
||||||
|
ENV GOVERSION="1.22.3"
|
||||||
|
RUN curl -fsSL "https://go.dev/dl/go${GOVERSION}.linux-amd64.tar.gz" \
|
||||||
|
| tar -C /usr/local -xzf -
|
||||||
|
|
||||||
|
ENV PATH="/usr/local/go/bin:${PATH}"
|
||||||
|
|
||||||
|
# gopls: the official Go language server.
|
||||||
|
# GOBIN=/usr/local/bin puts the binary on the system PATH for all users.
|
||||||
|
RUN GOBIN=/usr/local/bin go install golang.org/x/tools/gopls@latest
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the go stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
go.mod
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# JVM stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with OpenJDK 21, Maven, and eclipse.jdt.ls (the canonical Java/JVM LSP).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/jvm/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-jvm -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# OpenJDK 21 (current LTS) and Maven for build support.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
openjdk-21-jdk-headless \
|
||||||
|
maven \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV JAVA_HOME="/usr/lib/jvm/java-21-openjdk-amd64"
|
||||||
|
|
||||||
|
# Eclipse JDT Language Server — canonical LSP for Java/JVM (Java, Kotlin, Groovy).
|
||||||
|
# Pin to a specific release; update JDTLS_VERSION + JDTLS_BUILD for upgrades.
|
||||||
|
# All releases: https://github.com/eclipse-jdtls/eclipse.jdt.ls/releases
|
||||||
|
ENV JDTLS_VERSION="1.38.0" \
|
||||||
|
JDTLS_BUILD="202503271418"
|
||||||
|
RUN mkdir -p /opt/jdtls \
|
||||||
|
&& curl -fsSL \
|
||||||
|
"https://download.eclipse.org/jdtls/milestones/${JDTLS_VERSION}/jdt-language-server-${JDTLS_VERSION}-${JDTLS_BUILD}.tar.gz" \
|
||||||
|
| tar -xzf - -C /opt/jdtls
|
||||||
|
|
||||||
|
# Wrapper script so `jdtls` is available as a PATH command.
|
||||||
|
RUN { \
|
||||||
|
echo '#!/bin/sh'; \
|
||||||
|
echo 'JAR=$(ls /opt/jdtls/plugins/org.eclipse.equinox.launcher_*.jar 2>/dev/null | head -1)'; \
|
||||||
|
echo 'exec java \'; \
|
||||||
|
echo ' -Declipse.application=org.eclipse.jdt.ls.core.id1 \'; \
|
||||||
|
echo ' -Dosgi.bundles.defaultStartLevel=4 \'; \
|
||||||
|
echo ' -Declipse.product=org.eclipse.jdt.ls.core.product \'; \
|
||||||
|
echo ' -Dlog.protocol=true \'; \
|
||||||
|
echo ' -Dlog.level=ALL \'; \
|
||||||
|
echo ' -jar "$JAR" \'; \
|
||||||
|
echo ' -configuration /opt/jdtls/config_linux \'; \
|
||||||
|
echo ' "$@"'; \
|
||||||
|
} > /usr/local/bin/jdtls \
|
||||||
|
&& chmod +x /usr/local/bin/jdtls
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Stack detection markers for the jvm stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
pom.xml
|
||||||
|
build.gradle
|
||||||
|
build.gradle.kts
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# Node stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Node.js 22, TypeScript (tsc), and typescript-language-server.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/node/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-node -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Node.js 22.x (LTS).
|
||||||
|
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||||
|
&& apt-get install -y --no-install-recommends nodejs \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# TypeScript compiler and language server for LSP-aware agents.
|
||||||
|
# tsc: TypeScript compiler (tsc --version)
|
||||||
|
# typescript-language-server: LSP server used by editors/agents
|
||||||
|
RUN npm install -g typescript typescript-language-server
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# Stack detection markers for the node stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
# tsconfig.json is listed explicitly so TypeScript-only projects are detected
|
||||||
|
# even without a package.json at the repo root.
|
||||||
|
package.json
|
||||||
|
tsconfig.json
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Python stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Python 3, pip, and pyright (the Microsoft Python LSP / type checker).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/python/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-python -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Python 3 runtime and pip.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
python3 \
|
||||||
|
python3-pip \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# pyright: Microsoft's Python language server / static type checker.
|
||||||
|
# --break-system-packages is required on Debian 12+ where pip is externally
|
||||||
|
# managed; the flag is safe inside a Docker container.
|
||||||
|
RUN pip install --no-cache-dir --break-system-packages pyright
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Stack detection markers for the python stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
pyproject.toml
|
||||||
|
requirements.txt
|
||||||
|
setup.py
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# Ruby stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with Ruby, Bundler, and ruby-lsp (the Shopify Ruby language server).
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/ruby/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-ruby -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Ruby runtime, development headers (needed by native gem extensions), and Bundler.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ruby \
|
||||||
|
ruby-dev \
|
||||||
|
bundler \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# ruby-lsp: Shopify's Ruby language server (LSP-compliant, actively maintained).
|
||||||
|
# Installed globally so the `ruby-lsp` binary is available on PATH.
|
||||||
|
RUN gem install ruby-lsp
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the ruby stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
Gemfile
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Rust stack overlay fragment.
|
||||||
|
#
|
||||||
|
# Layer this on top of huskies-project-base to produce a project container
|
||||||
|
# with a full Rust toolchain, rust-analyzer, and cargo-nextest.
|
||||||
|
#
|
||||||
|
# Build the combined image:
|
||||||
|
# (echo "FROM huskies-project-base"; \
|
||||||
|
# cat docker/stacks/rust/Dockerfile.fragment) | \
|
||||||
|
# docker build -t huskies-project-rust -
|
||||||
|
#
|
||||||
|
# Adding a new stack: create docker/stacks/<name>/Dockerfile.fragment and
|
||||||
|
# docker/stacks/<name>/markers — no changes to orchestration code required.
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
# Build tools required by rustup and many Rust crates.
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
pkg-config \
|
||||||
|
libssl-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV RUSTUP_HOME="/home/huskies/.rustup" \
|
||||||
|
CARGO_HOME="/home/huskies/.cargo"
|
||||||
|
|
||||||
|
# Install stable Rust + rust-analyzer component as the huskies user.
|
||||||
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||||
|
| su huskies -c "sh -s -- -y --no-modify-path --default-toolchain stable" \
|
||||||
|
&& /home/huskies/.cargo/bin/rustup component add rust-analyzer \
|
||||||
|
&& chown -R huskies:huskies /home/huskies/.rustup /home/huskies/.cargo
|
||||||
|
|
||||||
|
# cargo-nextest: fast Rust test runner used by huskies quality gates.
|
||||||
|
RUN curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C /usr/local/bin
|
||||||
|
|
||||||
|
ENV PATH="/home/huskies/.cargo/bin:${PATH}"
|
||||||
|
|
||||||
|
USER huskies
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
# Stack detection markers for the rust stack.
|
||||||
|
# Each non-blank, non-comment line names a file relative to the project root.
|
||||||
|
# If any listed file exists in the project directory, this stack is matched.
|
||||||
|
Cargo.toml
|
||||||
Generated
+945
-1215
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "huskies",
|
"name": "huskies",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.11.0",
|
"version": "0.13.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
@@ -32,11 +32,11 @@
|
|||||||
"@types/node": "^25.0.0",
|
"@types/node": "^25.0.0",
|
||||||
"@types/react": "^19.1.8",
|
"@types/react": "^19.1.8",
|
||||||
"@types/react-dom": "^19.1.6",
|
"@types/react-dom": "^19.1.6",
|
||||||
"@vitejs/plugin-react": "^4.6.0",
|
"@vitejs/plugin-react": "^5.2.0",
|
||||||
"@vitest/coverage-v8": "^2.1.9",
|
"@vitest/coverage-v8": "^4.1.6",
|
||||||
"jsdom": "^28.1.0",
|
"jsdom": "^28.1.0",
|
||||||
"typescript": "~5.8.3",
|
"typescript": "~5.8.3",
|
||||||
"vite": "^5.4.21",
|
"vite": "^8.0.13",
|
||||||
"vitest": "^2.1.4"
|
"vitest": "^4.1.6"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -160,6 +160,7 @@ describe("App", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("shows error when openProject fails", async () => {
|
it("shows error when openProject fails", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockedApi.openProject.mockRejectedValue(new Error("Path does not exist"));
|
mockedApi.openProject.mockRejectedValue(new Error("Path does not exist"));
|
||||||
|
|
||||||
await renderApp();
|
await renderApp();
|
||||||
@@ -182,6 +183,7 @@ describe("App", () => {
|
|||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(screen.getByText(/Path does not exist/)).toBeInTheDocument();
|
expect(screen.getByText(/Path does not exist/)).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("shows known projects list", async () => {
|
it("shows known projects list", async () => {
|
||||||
|
|||||||
@@ -266,6 +266,8 @@ describe("subscribeAgentStream", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("handles malformed JSON without throwing", () => {
|
it("handles malformed JSON without throwing", () => {
|
||||||
|
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
|
|
||||||
subscribeAgentStream("42_story_test", "coder", vi.fn());
|
subscribeAgentStream("42_story_test", "coder", vi.fn());
|
||||||
|
|
||||||
expect(() => {
|
expect(() => {
|
||||||
|
|||||||
@@ -472,9 +472,16 @@ describe("Slash command handling (Story 374)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("Story 1058: WebSocket errors do not appear in chat", () => {
|
describe("Story 1058: WebSocket errors do not appear in chat", () => {
|
||||||
|
let consoleSpy: ReturnType<typeof vi.spyOn>;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
capturedWsHandlers = null;
|
capturedWsHandlers = null;
|
||||||
setupMocks();
|
setupMocks();
|
||||||
|
consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
consoleSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not add a chat message when onError is called", async () => {
|
it("does not add a chat message when onError is called", async () => {
|
||||||
|
|||||||
@@ -227,6 +227,7 @@ describe("usePathCompletion hook", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("sets completionError when listDirectoryAbsolute throws an Error", async () => {
|
it("sets completionError when listDirectoryAbsolute throws an Error", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockListDir.mockRejectedValue(new Error("Permission denied"));
|
mockListDir.mockRejectedValue(new Error("Permission denied"));
|
||||||
|
|
||||||
const { result } = renderHook(() =>
|
const { result } = renderHook(() =>
|
||||||
@@ -242,9 +243,13 @@ describe("usePathCompletion hook", () => {
|
|||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(result.current.completionError).toBe("Permission denied");
|
expect(result.current.completionError).toBe("Permission denied");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
expect(errorSpy).toHaveBeenCalledWith(new Error("Permission denied"));
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("sets generic completionError when listDirectoryAbsolute throws a non-Error", async () => {
|
it("sets generic completionError when listDirectoryAbsolute throws a non-Error", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
mockListDir.mockRejectedValue("some string error");
|
mockListDir.mockRejectedValue("some string error");
|
||||||
|
|
||||||
const { result } = renderHook(() =>
|
const { result } = renderHook(() =>
|
||||||
@@ -262,6 +267,9 @@ describe("usePathCompletion hook", () => {
|
|||||||
"Failed to compute suggestion.",
|
"Failed to compute suggestion.",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
expect(errorSpy).toHaveBeenCalledWith("some string error");
|
||||||
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("clears suggestionTail when selected match path does not start with input", async () => {
|
it("clears suggestionTail when selected match path does not start with input", async () => {
|
||||||
|
|||||||
Executable
+37
@@ -0,0 +1,37 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Build all project images in dependency order:
|
||||||
|
# huskies → huskies-project-base → huskies-project-<stack> (one per stack fragment)
|
||||||
|
#
|
||||||
|
# Run this after `script/docker_rebuild` or whenever you add a new stack.
|
||||||
|
# Safe to re-run: each step re-tags the image with the latest layers.
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
if [[ -f .env ]]; then
|
||||||
|
set -a
|
||||||
|
source .env
|
||||||
|
set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
CACHE_FLAG=""
|
||||||
|
if [[ "${1:-}" == "--no-cache" ]]; then
|
||||||
|
CACHE_FLAG="--no-cache"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Building huskies"
|
||||||
|
docker build $CACHE_FLAG -t huskies -f docker/Dockerfile .
|
||||||
|
|
||||||
|
echo "==> Building huskies-project-base"
|
||||||
|
docker build $CACHE_FLAG -t huskies-project-base -f docker/Dockerfile.base .
|
||||||
|
|
||||||
|
for fragment in docker/stacks/*/Dockerfile.fragment; do
|
||||||
|
stack=$(basename "$(dirname "$fragment")")
|
||||||
|
image="huskies-project-${stack}"
|
||||||
|
echo "==> Building ${image}"
|
||||||
|
(printf 'FROM huskies-project-base\n'; cat "$fragment") \
|
||||||
|
| docker build $CACHE_FLAG -t "$image" -
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "All project images built."
|
||||||
@@ -24,4 +24,6 @@ docker compose -f docker/docker-compose.yml down
|
|||||||
docker compose -f docker/docker-compose.yml build $CACHE_FLAG
|
docker compose -f docker/docker-compose.yml build $CACHE_FLAG
|
||||||
docker compose -f docker/docker-compose.yml up -d
|
docker compose -f docker/docker-compose.yml up -d
|
||||||
|
|
||||||
|
script/build-project-images $CACHE_FLAG
|
||||||
|
|
||||||
echo "Rebuild complete. Logs: docker compose -f docker/docker-compose.yml logs -f"
|
echo "Rebuild complete. Logs: docker compose -f docker/docker-compose.yml logs -f"
|
||||||
|
|||||||
Executable
+165
@@ -0,0 +1,165 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build huskies, install (codesign-heal wrapper + underlying binary), and if a
|
||||||
|
# gateway is running on this host, hot-restart it detached from the current shell
|
||||||
|
# so SSH disconnect — e.g. when redeploying from a phone — doesn't kill it.
|
||||||
|
#
|
||||||
|
# Skips the restart silently if no gateway is running. Errors loudly if more
|
||||||
|
# than one matches, so we don't restart the wrong one.
|
||||||
|
#
|
||||||
|
# Pass --skip-check to bypass `script/check` (useful for docs / build-script
|
||||||
|
# changes you've already verified).
|
||||||
|
#
|
||||||
|
# On relaunch failure the previous binary is restored from
|
||||||
|
# ~/bin/huskies-bin.prev and re-launched, so a bad deploy doesn't leave the
|
||||||
|
# host without a working gateway.
|
||||||
|
#
|
||||||
|
# After a `cp` or download the binary loses its ad-hoc signature and macOS
|
||||||
|
# SIGKILLs it silently on Apple Silicon. The wrapper at ~/bin/huskies re-signs
|
||||||
|
# the underlying binary at ~/bin/huskies-bin whenever codesign validation
|
||||||
|
# fails, then execs it. Normal launches (already signed) are zero-overhead.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
LOG_DIR="${HUSKIES_LOG_DIR:-$PROJECT_ROOT/logs}"
|
||||||
|
GATEWAY_PATTERN='huskies .*--gateway'
|
||||||
|
BIN_DIR="${HOME}/bin"
|
||||||
|
UNDERLYING="${BIN_DIR}/huskies-bin"
|
||||||
|
WRAPPER="${BIN_DIR}/huskies"
|
||||||
|
PREV_BIN="${BIN_DIR}/huskies-bin.prev"
|
||||||
|
NEW_BIN="${PROJECT_ROOT}/target/release/huskies"
|
||||||
|
|
||||||
|
SKIP_CHECK=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--skip-check) SKIP_CHECK=1 ;;
|
||||||
|
-h|--help) sed -n '2,17p' "$0"; exit 0 ;;
|
||||||
|
*) echo "Unknown arg: $arg (use --help)" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$SKIP_CHECK" -eq 0 ] && [ -x "$SCRIPT_DIR/check" ]; then
|
||||||
|
echo "=== Running script/check ==="
|
||||||
|
"$SCRIPT_DIR/check"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== Building release binary ==="
|
||||||
|
cd "$PROJECT_ROOT"
|
||||||
|
cargo build --release --bin huskies
|
||||||
|
|
||||||
|
mkdir -p "$BIN_DIR"
|
||||||
|
|
||||||
|
# Snapshot current binary so we can roll back if the relaunch fails.
|
||||||
|
PREV_VERSION=""
|
||||||
|
if [ -x "$UNDERLYING" ]; then
|
||||||
|
PREV_VERSION="$("$UNDERLYING" --version 2>/dev/null || echo unknown)"
|
||||||
|
cp "$UNDERLYING" "$PREV_BIN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp "$NEW_BIN" "$UNDERLYING"
|
||||||
|
chmod +x "$UNDERLYING"
|
||||||
|
codesign -s - -f "$UNDERLYING" 2>/dev/null
|
||||||
|
NEW_VERSION="$("$UNDERLYING" --version 2>/dev/null || echo unknown)"
|
||||||
|
echo "==> Installed binary: ${UNDERLYING}"
|
||||||
|
if [ -n "$PREV_VERSION" ]; then
|
||||||
|
echo " version: $PREV_VERSION → $NEW_VERSION"
|
||||||
|
else
|
||||||
|
echo " version: $NEW_VERSION (no prior install)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat > "${WRAPPER}" << 'WRAPPER_EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Codesign-heal wrapper — re-signs ~/bin/huskies-bin if the signature is
|
||||||
|
# missing or invalid, then execs the binary. Logs only when it re-signs.
|
||||||
|
BIN="${HOME}/bin/huskies-bin"
|
||||||
|
if ! codesign --verify --quiet "${BIN}" 2>/dev/null; then
|
||||||
|
codesign -s - "${BIN}"
|
||||||
|
echo "[codesign-heal] re-signed ~/bin/huskies-bin" >&2
|
||||||
|
fi
|
||||||
|
exec "${BIN}" "$@"
|
||||||
|
WRAPPER_EOF
|
||||||
|
chmod +x "${WRAPPER}"
|
||||||
|
echo "==> Installed wrapper: ${WRAPPER}"
|
||||||
|
|
||||||
|
# ── Hot-restart gateway if one is running ─────────────────────────────
|
||||||
|
collect_descendants() {
|
||||||
|
local pid="$1" kid
|
||||||
|
for kid in $(pgrep -P "$pid" 2>/dev/null); do
|
||||||
|
collect_descendants "$kid"
|
||||||
|
printf '%s\n' "$kid"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
GATEWAY_PIDS="$(pgrep -f "$GATEWAY_PATTERN" || true)"
|
||||||
|
if [ -z "$GATEWAY_PIDS" ]; then
|
||||||
|
echo "==> No running gateway found; install complete."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$(echo "$GATEWAY_PIDS" | wc -l)" -gt 1 ]; then
|
||||||
|
echo "Error: multiple gateway processes match '${GATEWAY_PATTERN}':" >&2
|
||||||
|
ps -p $GATEWAY_PIDS -o pid,args >&2 || true
|
||||||
|
echo "Refusing to guess which to restart." >&2
|
||||||
|
exit 3
|
||||||
|
fi
|
||||||
|
|
||||||
|
GATEWAY_PID="$GATEWAY_PIDS"
|
||||||
|
GATEWAY_ARGS="$(ps -p "$GATEWAY_PID" -o args= | sed -E 's@^[^ ]*huskies[^ ]* @@')"
|
||||||
|
GATEWAY_CWD="$(lsof -p "$GATEWAY_PID" 2>/dev/null | awk '$4=="cwd"{print $9; exit}')"
|
||||||
|
if [ -z "$GATEWAY_CWD" ]; then GATEWAY_CWD="$PWD"; fi
|
||||||
|
|
||||||
|
LOG_FILE="$LOG_DIR/gateway-$(date +%Y%m%d-%H%M%S).log"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
DESCENDANTS="$(collect_descendants "$GATEWAY_PID" | tr '\n' ' ')"
|
||||||
|
echo "==> Stopping gateway tree (pids: $GATEWAY_PID $DESCENDANTS)"
|
||||||
|
# Kill descendants depth-first so PTY children die before the gateway, then the gateway.
|
||||||
|
for pid in $DESCENDANTS $GATEWAY_PID; do
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
echo "==> Restarting gateway"
|
||||||
|
echo " log: $LOG_FILE"
|
||||||
|
(
|
||||||
|
cd "$GATEWAY_CWD"
|
||||||
|
nohup "$WRAPPER" $GATEWAY_ARGS >> "$LOG_FILE" 2>&1 < /dev/null &
|
||||||
|
disown
|
||||||
|
)
|
||||||
|
|
||||||
|
# Wait up to 10s for the new gateway to appear AND be a different PID.
|
||||||
|
NEW_PID=""
|
||||||
|
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||||
|
sleep 1
|
||||||
|
candidate="$(pgrep -f "$GATEWAY_PATTERN" 2>/dev/null || true)"
|
||||||
|
if [ -n "$candidate" ] && [ "$candidate" != "$GATEWAY_PID" ]; then
|
||||||
|
NEW_PID="$candidate"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$NEW_PID" ]; then
|
||||||
|
echo "==> Gateway restarted as pid $NEW_PID"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Rollback ──────────────────────────────────────────────────────────
|
||||||
|
echo "Error: new gateway failed to come up within 10s; rolling back" >&2
|
||||||
|
if [ -x "$PREV_BIN" ]; then
|
||||||
|
cp "$PREV_BIN" "$UNDERLYING"
|
||||||
|
chmod +x "$UNDERLYING"
|
||||||
|
codesign -s - -f "$UNDERLYING" 2>/dev/null
|
||||||
|
echo "==> Restored previous binary"
|
||||||
|
(
|
||||||
|
cd "$GATEWAY_CWD"
|
||||||
|
nohup "$WRAPPER" $GATEWAY_ARGS >> "$LOG_FILE" 2>&1 < /dev/null &
|
||||||
|
disown
|
||||||
|
)
|
||||||
|
sleep 2
|
||||||
|
if pgrep -f "$GATEWAY_PATTERN" >/dev/null 2>&1; then
|
||||||
|
echo "==> Gateway restored to previous version"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo "Error: rollback failed; gateway is DOWN. Inspect $LOG_FILE." >&2
|
||||||
|
exit 1
|
||||||
+12
-10
@@ -11,10 +11,12 @@ export GIT_CONFIG_VALUE_0=master
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
# Ordered fail-fast: cheapest deterministic checks first, slowest builds and
|
# Ordered fail-fast: cheapest deterministic checks first. The frontend build
|
||||||
# test suites last. `set -euo pipefail` aborts at the first failure, so a fmt
|
# must run *before* anything that compiles Rust, because story 1113 introduced
|
||||||
# or clippy drift never wastes time on a frontend build or a multi-minute
|
# a compile-time dependency on `frontend/dist/` via `rust-embed` — a fresh
|
||||||
# test run.
|
# merge worktree without that directory will fail `cargo clippy` on
|
||||||
|
# `EmbeddedAssets::iter()` before the frontend build has a chance to populate
|
||||||
|
# it. `set -euo pipefail` aborts at the first failure.
|
||||||
|
|
||||||
echo "=== Checking Rust formatting ==="
|
echo "=== Checking Rust formatting ==="
|
||||||
if cargo fmt --version &>/dev/null; then
|
if cargo fmt --version &>/dev/null; then
|
||||||
@@ -44,12 +46,6 @@ if [ "$_dup_found" -eq 1 ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "=== Running cargo clippy ==="
|
|
||||||
cargo clippy --manifest-path "$PROJECT_ROOT/Cargo.toml" --all-targets --all-features -- -D warnings
|
|
||||||
|
|
||||||
echo "=== Checking doc coverage on changed files ==="
|
|
||||||
cargo run --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen --bin source-map-check --quiet -- --worktree "$PROJECT_ROOT" --base master
|
|
||||||
|
|
||||||
echo "=== Building frontend ==="
|
echo "=== Building frontend ==="
|
||||||
if [ -d "$PROJECT_ROOT/frontend" ]; then
|
if [ -d "$PROJECT_ROOT/frontend" ]; then
|
||||||
cd "$PROJECT_ROOT/frontend"
|
cd "$PROJECT_ROOT/frontend"
|
||||||
@@ -75,6 +71,12 @@ else
|
|||||||
echo "Skipping frontend build (no frontend directory)"
|
echo "Skipping frontend build (no frontend directory)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "=== Running cargo clippy ==="
|
||||||
|
cargo clippy --manifest-path "$PROJECT_ROOT/Cargo.toml" --all-targets --all-features -- -D warnings
|
||||||
|
|
||||||
|
echo "=== Checking doc coverage on changed files ==="
|
||||||
|
cargo run --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen --bin source-map-check --quiet -- --worktree "$PROJECT_ROOT" --base master
|
||||||
|
|
||||||
echo "=== Running Rust tests ==="
|
echo "=== Running Rust tests ==="
|
||||||
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" --bin huskies
|
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" --bin huskies
|
||||||
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen
|
cargo test --manifest-path "$PROJECT_ROOT/Cargo.toml" -p source-map-gen
|
||||||
|
|||||||
+1
-3
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "huskies"
|
name = "huskies"
|
||||||
version = "0.11.0"
|
version = "0.13.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
|
|
||||||
@@ -13,12 +13,10 @@ chrono-tz = { workspace = true }
|
|||||||
futures = { workspace = true }
|
futures = { workspace = true }
|
||||||
homedir = { workspace = true }
|
homedir = { workspace = true }
|
||||||
ignore = { workspace = true }
|
ignore = { workspace = true }
|
||||||
mime_guess = { workspace = true }
|
|
||||||
notify = { workspace = true }
|
notify = { workspace = true }
|
||||||
poem = { workspace = true, features = ["websocket"] }
|
poem = { workspace = true, features = ["websocket"] }
|
||||||
portable-pty = { workspace = true }
|
portable-pty = { workspace = true }
|
||||||
reqwest = { workspace = true, features = ["json", "stream", "form"] }
|
reqwest = { workspace = true, features = ["json", "stream", "form"] }
|
||||||
rust-embed = { workspace = true }
|
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
serde_urlencoded = { workspace = true }
|
serde_urlencoded = { workspace = true }
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ pub(super) fn build_agent_app_context(
|
|||||||
pending_perm_replies: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
pending_perm_replies: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
|
||||||
permission_timeout_secs: 120,
|
permission_timeout_secs: 120,
|
||||||
status: agents.status_broadcaster(),
|
status: agents.status_broadcaster(),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
});
|
});
|
||||||
crate::http::context::AppContext {
|
crate::http::context::AppContext {
|
||||||
state: Arc::new(state),
|
state: Arc::new(state),
|
||||||
|
|||||||
@@ -33,16 +33,28 @@ impl GateFailureKind {
|
|||||||
/// Called once when a gate fails to produce a typed kind. Downstream code
|
/// Called once when a gate fails to produce a typed kind. Downstream code
|
||||||
/// matches on the variant and must not call this on subsequent reads.
|
/// matches on the variant and must not call this on subsequent reads.
|
||||||
pub fn classify(output: &str) -> Self {
|
pub fn classify(output: &str) -> Self {
|
||||||
|
// Strip `test <name> ... ok` lines before checking lint-trigger keywords so
|
||||||
|
// a passing test whose name contains e.g. `missing_doc_comments` or `clippy::`
|
||||||
|
// does not produce a false-positive Lint classification (story 1101).
|
||||||
|
let stripped_for_lint: String = output
|
||||||
|
.lines()
|
||||||
|
.filter(|l| {
|
||||||
|
let t = l.trim();
|
||||||
|
!(t.starts_with("test ") && t.ends_with("... ok"))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
let is_lint = stripped_for_lint.contains("error[clippy::")
|
||||||
|
|| stripped_for_lint.contains("warning[clippy::")
|
||||||
|
|| stripped_for_lint.contains("missing_doc_comments");
|
||||||
|
|
||||||
if output.contains("CONFLICT (content):") || output.contains("Merge conflict:") {
|
if output.contains("CONFLICT (content):") || output.contains("Merge conflict:") {
|
||||||
GateFailureKind::ContentConflict
|
GateFailureKind::ContentConflict
|
||||||
} else if output.contains("Diff in ") || output.contains("would reformat") {
|
} else if output.contains("Diff in ") || output.contains("would reformat") {
|
||||||
GateFailureKind::Fmt
|
GateFailureKind::Fmt
|
||||||
} else if output.contains("missing-docs direction") {
|
} else if output.contains("missing-docs direction") {
|
||||||
GateFailureKind::SourceMapCheck
|
GateFailureKind::SourceMapCheck
|
||||||
} else if output.contains("error[clippy::")
|
} else if is_lint {
|
||||||
|| output.contains("warning[clippy::")
|
|
||||||
|| output.contains("missing_doc_comments")
|
|
||||||
{
|
|
||||||
GateFailureKind::Lint
|
GateFailureKind::Lint
|
||||||
} else if output.contains("error[E") {
|
} else if output.contains("error[E") {
|
||||||
// rustc compile errors (e.g. `error[E0063]: missing field`).
|
// rustc compile errors (e.g. `error[E0063]: missing field`).
|
||||||
@@ -871,6 +883,19 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Story 1101: a passing test whose name contains a lint trigger keyword
|
||||||
|
/// must NOT produce a Lint classification.
|
||||||
|
#[test]
|
||||||
|
fn classify_does_not_false_positive_on_test_name_substring() {
|
||||||
|
let output = "test agents::gates::tests::classify_lint_from_missing_doc_comments ... ok\n\
|
||||||
|
test result: ok. 1 passed; 0 failed";
|
||||||
|
assert_ne!(
|
||||||
|
GateFailureKind::classify(output),
|
||||||
|
GateFailureKind::Lint,
|
||||||
|
"passing test name containing 'missing_doc_comments' must not classify as Lint"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classify_source_map_check_from_missing_docs_direction() {
|
fn classify_source_map_check_from_missing_docs_direction() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -161,6 +161,42 @@ pub fn pipeline_stage(agent_name: &str) -> PipelineStage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Map a pipeline [`Stage`] to the canonical [`PipelineStage`] for LLM agent spawning.
|
||||||
|
///
|
||||||
|
/// Returns `None` for stages where no LLM agent should be active (terminal states,
|
||||||
|
/// blocked, frozen, or unclassified merge failures requiring human intervention).
|
||||||
|
/// Returns `Some(stage)` naming the single LLM-agent type that may run on this story.
|
||||||
|
/// Used by `validate_agent_stage` and `reconcile_canonical_agents` to enforce the
|
||||||
|
/// one-agent-per-story invariant (story 1100).
|
||||||
|
pub fn canonical_pipeline_stage(s: &crate::pipeline_state::Stage) -> Option<PipelineStage> {
|
||||||
|
use crate::pipeline_state::{MergeFailureKind, Stage};
|
||||||
|
match s {
|
||||||
|
Stage::Coding { .. } => Some(PipelineStage::Coder),
|
||||||
|
Stage::Qa => Some(PipelineStage::Qa),
|
||||||
|
Stage::Merge { .. } => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::MergeFailure {
|
||||||
|
kind: MergeFailureKind::ConflictDetected(_),
|
||||||
|
..
|
||||||
|
} => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::MergeFailure {
|
||||||
|
kind: MergeFailureKind::GatesFailed(_),
|
||||||
|
..
|
||||||
|
} => Some(PipelineStage::Coder),
|
||||||
|
Stage::MergeFailureFinal { .. } => Some(PipelineStage::Mergemaster),
|
||||||
|
Stage::Upcoming
|
||||||
|
| Stage::Backlog
|
||||||
|
| Stage::MergeFailure { .. }
|
||||||
|
| Stage::Done { .. }
|
||||||
|
| Stage::Blocked { .. }
|
||||||
|
| Stage::Archived { .. }
|
||||||
|
| Stage::Frozen { .. }
|
||||||
|
| Stage::ReviewHold { .. }
|
||||||
|
| Stage::Abandoned { .. }
|
||||||
|
| Stage::Superseded { .. }
|
||||||
|
| Stage::Rejected { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Determine the pipeline stage for a configured agent.
|
/// Determine the pipeline stage for a configured agent.
|
||||||
///
|
///
|
||||||
/// Prefers the explicit `stage` config field (added in Bug 150) over the
|
/// Prefers the explicit `stage` config field (added in Bug 150) over the
|
||||||
|
|||||||
@@ -105,12 +105,6 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 4: drop the (now-stale) child_killers entry — the
|
|
||||||
// process it pointed at is gone.
|
|
||||||
if let Ok(mut killers) = self.child_killers.lock() {
|
|
||||||
killers.remove(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use the retry mechanism: increment retry_count and only block
|
// Use the retry mechanism: increment retry_count and only block
|
||||||
// when the limit is exceeded, matching the pipeline's behaviour.
|
// when the limit is exceeded, matching the pipeline's behaviour.
|
||||||
let story_id = key.rsplit_once(':').map(|(s, _)| s).unwrap_or(key);
|
let story_id = key.rsplit_once(':').map(|(s, _)| s).unwrap_or(key);
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ mod test_helpers;
|
|||||||
|
|
||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use crate::service::status::StatusBroadcaster;
|
use crate::service::status::StatusBroadcaster;
|
||||||
use portable_pty::ChildKiller;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
@@ -31,10 +30,6 @@ use types::{StoryAgent, composite_key};
|
|||||||
pub struct AgentPool {
|
pub struct AgentPool {
|
||||||
agents: Arc<Mutex<HashMap<String, StoryAgent>>>,
|
agents: Arc<Mutex<HashMap<String, StoryAgent>>>,
|
||||||
port: u16,
|
port: u16,
|
||||||
/// Registry of active PTY child process killers, keyed by "{story_id}:{agent_name}".
|
|
||||||
/// Used to terminate child processes on server shutdown or agent stop, preventing
|
|
||||||
/// orphaned Claude Code processes from running after the server exits.
|
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
/// Broadcast channel for notifying WebSocket clients of agent state changes.
|
/// Broadcast channel for notifying WebSocket clients of agent state changes.
|
||||||
/// When an agent transitions state (Pending, Running, Completed, Failed, Stopped),
|
/// When an agent transitions state (Pending, Running, Completed, Failed, Stopped),
|
||||||
/// an `AgentStateChanged` event is emitted so the frontend can refresh the
|
/// an `AgentStateChanged` event is emitted so the frontend can refresh the
|
||||||
@@ -56,7 +51,6 @@ impl AgentPool {
|
|||||||
let pool = Self {
|
let pool = Self {
|
||||||
agents: Arc::new(Mutex::new(HashMap::new())),
|
agents: Arc::new(Mutex::new(HashMap::new())),
|
||||||
port,
|
port,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_tx.clone(),
|
watcher_tx: watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::new(StatusBroadcaster::new()),
|
status_broadcaster: Arc::new(StatusBroadcaster::new()),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ pub(crate) fn spawn_pipeline_advance(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents,
|
agents,
|
||||||
port,
|
port,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
status_broadcaster: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
status_broadcaster: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -78,21 +78,34 @@ impl AgentPool {
|
|||||||
// The coder exited with uncommitted content but no commits
|
// The coder exited with uncommitted content but no commits
|
||||||
// (typical "claude-code session boundary mid-sweep" pattern).
|
// (typical "claude-code session boundary mid-sweep" pattern).
|
||||||
// Use a PROGRESS-AWARE retry cap: the agent gets unlimited
|
// Use a PROGRESS-AWARE retry cap: the agent gets unlimited
|
||||||
// respawns as long as file edits keep growing between
|
// respawns as long as progress is being made between attempts.
|
||||||
// attempts; only when the worktree diff is byte-identical
|
// Progress is satisfied if EITHER (a) the worktree diff grew,
|
||||||
// to the previous attempt do we count it as "no progress".
|
// OR (b) the set of files the agent read grew. Raw tool-call
|
||||||
// After NO_PROGRESS_CAP consecutive no-progress respawns,
|
// count does NOT count — a looping agent can produce many calls.
|
||||||
// block for human attention.
|
// Only self-exited sessions with no file or read progress count
|
||||||
|
// toward the cap; forced exits (API error, network, budget
|
||||||
|
// exhaustion) are excluded (story 1089).
|
||||||
|
// After NO_PROGRESS_CAP consecutive qualifying no-progress
|
||||||
|
// respawns, block for human attention.
|
||||||
//
|
//
|
||||||
// TOTAL_ATTEMPTS_CAP is the OUTER bound: even if the agent
|
// TOTAL_ATTEMPTS_CAP is the OUTER bound: even if the agent
|
||||||
// keeps making file-edit progress every session, after this
|
// keeps making file-edit progress every session, after this
|
||||||
// many total respawns without a commit we escalate — caught
|
// many total respawns without a commit we escalate — catches
|
||||||
// the "agent flaps between different edits but never
|
// the "agent flaps between different edits but never commits"
|
||||||
// commits" pattern that the progress-aware counter would
|
// pattern that the progress-aware counter would never trigger.
|
||||||
// never trigger.
|
|
||||||
const NO_PROGRESS_CAP: u32 = 3;
|
const NO_PROGRESS_CAP: u32 = 3;
|
||||||
const TOTAL_ATTEMPTS_CAP: u32 = 8;
|
const TOTAL_ATTEMPTS_CAP: u32 = 8;
|
||||||
|
|
||||||
|
// AC1: consume the forced-exit flag written by spawn.rs when
|
||||||
|
// the agent process exited with a non-zero code.
|
||||||
|
let forced_exit = crate::db::read_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryForcedExit(story_id),
|
||||||
|
)
|
||||||
|
.is_some();
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryForcedExit(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
|
|
||||||
let current_fingerprint = worktree_path.as_deref().and_then(|p| {
|
let current_fingerprint = worktree_path.as_deref().and_then(|p| {
|
||||||
std::process::Command::new("git")
|
std::process::Command::new("git")
|
||||||
.args(["diff", "master"])
|
.args(["diff", "master"])
|
||||||
@@ -104,18 +117,31 @@ impl AgentPool {
|
|||||||
let stored_fingerprint = crate::db::read_content(
|
let stored_fingerprint = crate::db::read_content(
|
||||||
crate::db::ContentKey::CommitRecoveryDiffFingerprint(story_id),
|
crate::db::ContentKey::CommitRecoveryDiffFingerprint(story_id),
|
||||||
);
|
);
|
||||||
let made_progress = current_fingerprint.is_some()
|
let diff_progress = current_fingerprint.is_some()
|
||||||
&& stored_fingerprint.as_ref() != current_fingerprint.as_ref();
|
&& stored_fingerprint.as_ref() != current_fingerprint.as_ref();
|
||||||
let no_progress_count = if made_progress || stored_fingerprint.is_none() {
|
|
||||||
|
// AC2: check read-file set progress as an additional signal.
|
||||||
|
let read_progress = previous_session_id.as_deref().is_some_and(|session_id| {
|
||||||
|
collect_read_progress(&project_root, story_id, agent_name, session_id)
|
||||||
|
});
|
||||||
|
|
||||||
|
let made_progress = diff_progress || read_progress;
|
||||||
|
|
||||||
|
let prev_no_progress_count = crate::db::read_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryPending(story_id),
|
||||||
|
)
|
||||||
|
.and_then(|s| s.trim().parse::<u32>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// AC1: forced exits do not increment the stuck-respawn counter.
|
||||||
|
let no_progress_count = if forced_exit {
|
||||||
|
prev_no_progress_count
|
||||||
|
} else if made_progress || stored_fingerprint.is_none() {
|
||||||
1
|
1
|
||||||
} else {
|
} else {
|
||||||
crate::db::read_content(crate::db::ContentKey::CommitRecoveryPending(
|
prev_no_progress_count + 1
|
||||||
story_id,
|
|
||||||
))
|
|
||||||
.and_then(|s| s.trim().parse::<u32>().ok())
|
|
||||||
.unwrap_or(0)
|
|
||||||
+ 1
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let total_attempts = crate::db::read_content(
|
let total_attempts = crate::db::read_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
)
|
)
|
||||||
@@ -136,13 +162,17 @@ impl AgentPool {
|
|||||||
crate::db::delete_content(
|
crate::db::delete_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
);
|
);
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
slog!(
|
slog!(
|
||||||
"[pipeline] Coder '{agent_name}' for '{story_id}' hit total \
|
"[pipeline] Coder '{agent_name}' for '{story_id}' hit total \
|
||||||
commit-recovery cap ({total_attempts}/{TOTAL_ATTEMPTS_CAP}) \
|
commit-recovery cap ({total_attempts}/{TOTAL_ATTEMPTS_CAP}) \
|
||||||
without a commit. Blocking story."
|
without a commit. Blocking story."
|
||||||
);
|
);
|
||||||
let reason = format!(
|
let reason = format!(
|
||||||
"agent flapped — {total_attempts} respawns without ever committing"
|
"commit absent after {total_attempts} respawns — \
|
||||||
|
agent kept making edits but never committed"
|
||||||
);
|
);
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
||||||
@@ -167,14 +197,18 @@ impl AgentPool {
|
|||||||
crate::db::delete_content(
|
crate::db::delete_content(
|
||||||
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id),
|
||||||
);
|
);
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
slog!(
|
slog!(
|
||||||
"[pipeline] Coder '{agent_name}' for '{story_id}' made no \
|
"[pipeline] Coder '{agent_name}' for '{story_id}' made no \
|
||||||
file-edit progress over {no_progress_count} consecutive \
|
file or read progress over {no_progress_count} consecutive \
|
||||||
commit-recovery respawns. Blocking story."
|
self-exit commit-recovery respawns. Blocking story."
|
||||||
);
|
);
|
||||||
|
// AC4: block message names the specific cause.
|
||||||
let reason = format!(
|
let reason = format!(
|
||||||
"agent stuck — {no_progress_count} respawns without commits or \
|
"stuck-respawn cap reached: {NO_PROGRESS_CAP} consecutive \
|
||||||
new file edits"
|
self-exits with no file or read progress"
|
||||||
);
|
);
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
crate::agents::lifecycle::transition_to_blocked(story_id, &reason)
|
||||||
@@ -206,7 +240,8 @@ impl AgentPool {
|
|||||||
"[pipeline] Coder '{agent_name}' exited with uncommitted work \
|
"[pipeline] Coder '{agent_name}' exited with uncommitted work \
|
||||||
for '{story_id}' (no-progress {no_progress_count}/\
|
for '{story_id}' (no-progress {no_progress_count}/\
|
||||||
{NO_PROGRESS_CAP}, total {total_attempts}/\
|
{NO_PROGRESS_CAP}, total {total_attempts}/\
|
||||||
{TOTAL_ATTEMPTS_CAP}; progress_made={made_progress}). \
|
{TOTAL_ATTEMPTS_CAP}; diff_progress={diff_progress}, \
|
||||||
|
read_progress={read_progress}, forced_exit={forced_exit}). \
|
||||||
Issuing commit-only respawn."
|
Issuing commit-only respawn."
|
||||||
);
|
);
|
||||||
let addendum = "\n\nYou have uncommitted work in this worktree. \
|
let addendum = "\n\nYou have uncommitted work in this worktree. \
|
||||||
@@ -302,10 +337,13 @@ impl AgentPool {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if completion.gates_passed {
|
} else if completion.gates_passed {
|
||||||
// Clear any stale recovery key when the coder succeeds normally.
|
// Clear any stale recovery keys when the coder succeeds normally.
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
||||||
story_id,
|
story_id,
|
||||||
));
|
));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
// Determine effective QA mode for this story.
|
// Determine effective QA mode for this story.
|
||||||
let qa_mode = {
|
let qa_mode = {
|
||||||
let item_type = crate::agents::lifecycle::item_type_from_id(story_id);
|
let item_type = crate::agents::lifecycle::item_type_from_id(story_id);
|
||||||
@@ -361,11 +399,14 @@ impl AgentPool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Clear any stale recovery key when gates fail normally (agent committed
|
// Clear any stale recovery keys when gates fail normally (agent committed
|
||||||
// but the build is broken — treat as a standard retry, not a recovery).
|
// but the build is broken — treat as a standard retry, not a recovery).
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
|
||||||
story_id,
|
story_id,
|
||||||
));
|
));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(
|
||||||
|
story_id,
|
||||||
|
));
|
||||||
// Bug 645 / 668: Before retry/block, check if the agent left committed
|
// Bug 645 / 668: Before retry/block, check if the agent left committed
|
||||||
// work AND the agent had a passing run_tests result captured during its
|
// work AND the agent had a passing run_tests result captured during its
|
||||||
// session. An agent may crash mid-output (e.g. Claude Code CLI PTY write
|
// session. An agent may crash mid-output (e.g. Claude Code CLI PTY write
|
||||||
@@ -724,6 +765,109 @@ mod helpers;
|
|||||||
use helpers::{resolve_qa_mode_from_store, write_review_hold_to_store};
|
use helpers::{resolve_qa_mode_from_store, write_review_hold_to_store};
|
||||||
pub(crate) use helpers::{should_block_story, spawn_pipeline_advance};
|
pub(crate) use helpers::{should_block_story, spawn_pipeline_advance};
|
||||||
|
|
||||||
|
/// Parse a huskies agent log and return the set of file paths passed to the
|
||||||
|
/// Read tool in that session. Returns an empty set if the log cannot be read.
|
||||||
|
///
|
||||||
|
/// Used by [`collect_read_progress`] to detect read-exploration progress even
|
||||||
|
/// when the worktree diff did not grow (story 1089, AC2).
|
||||||
|
fn collect_read_files_from_log(
|
||||||
|
project_root: &std::path::Path,
|
||||||
|
story_id: &str,
|
||||||
|
agent_name: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> std::collections::HashSet<String> {
|
||||||
|
let log_path = crate::agent_log::log_file_path(project_root, story_id, agent_name, session_id);
|
||||||
|
let mut files = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
let log_text = match std::fs::read_to_string(&log_path) {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(_) => return files,
|
||||||
|
};
|
||||||
|
|
||||||
|
for line in log_text.lines() {
|
||||||
|
let trimmed = line.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let entry: serde_json::Value = match serde_json::from_str(trimmed) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
// Only look at agent_json events where data.type == "assistant".
|
||||||
|
if entry.get("type").and_then(|t| t.as_str()) != Some("agent_json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let data = match entry.get("data") {
|
||||||
|
Some(d) => d,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
if data.get("type").and_then(|t| t.as_str()) != Some("assistant") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let content = match data.pointer("/message/content").and_then(|c| c.as_array()) {
|
||||||
|
Some(c) => c,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
for item in content {
|
||||||
|
if item.get("type").and_then(|t| t.as_str()) != Some("tool_use") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if item.get("name").and_then(|n| n.as_str()) != Some("Read") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(path) = item.pointer("/input/file_path").and_then(|p| p.as_str()) {
|
||||||
|
files.insert(path.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
files
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return `true` if the agent read any files in `session_id` that were not in
|
||||||
|
/// the cumulative read set for `story_id`. Updates the stored cumulative set
|
||||||
|
/// when new files are found (story 1089, AC2).
|
||||||
|
fn collect_read_progress(
|
||||||
|
project_root: &std::path::Path,
|
||||||
|
story_id: &str,
|
||||||
|
agent_name: &str,
|
||||||
|
session_id: &str,
|
||||||
|
) -> bool {
|
||||||
|
let session_files = collect_read_files_from_log(project_root, story_id, agent_name, session_id);
|
||||||
|
if session_files.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stored_set: std::collections::HashSet<String> =
|
||||||
|
crate::db::read_content(crate::db::ContentKey::CommitRecoveryReadSet(story_id))
|
||||||
|
.map(|s| {
|
||||||
|
s.lines()
|
||||||
|
.filter(|l| !l.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let union: std::collections::HashSet<String> =
|
||||||
|
stored_set.union(&session_files).cloned().collect();
|
||||||
|
|
||||||
|
if union.len() > stored_set.len() {
|
||||||
|
let mut sorted: Vec<&String> = union.iter().collect();
|
||||||
|
sorted.sort();
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryReadSet(story_id),
|
||||||
|
&sorted
|
||||||
|
.into_iter()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n"),
|
||||||
|
);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -1077,7 +1077,7 @@ stage = "coder"
|
|||||||
"Story must be blocked after NO_PROGRESS_CAP consecutive no-progress respawns"
|
"Story must be blocked after NO_PROGRESS_CAP consecutive no-progress respawns"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
block_reason.contains("without commits or new file edits"),
|
block_reason.contains("self-exits with no file or read progress"),
|
||||||
"Block reason should describe the no-progress condition, got: {block_reason}"
|
"Block reason should describe the no-progress condition, got: {block_reason}"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1193,7 +1193,7 @@ stage = "coder"
|
|||||||
"Story must be blocked once total commit-recovery attempts hits the outer cap"
|
"Story must be blocked once total commit-recovery attempts hits the outer cap"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
block_reason.contains("flapped") && block_reason.contains("without ever committing"),
|
block_reason.contains("commit absent") && block_reason.contains("never committed"),
|
||||||
"Block reason should describe the flapping pattern, got: {block_reason}"
|
"Block reason should describe the flapping pattern, got: {block_reason}"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,6 @@ impl AgentPool {
|
|||||||
let pool_clone = Self {
|
let pool_clone = Self {
|
||||||
agents: Arc::clone(&self.agents),
|
agents: Arc::clone(&self.agents),
|
||||||
port: self.port,
|
port: self.port,
|
||||||
child_killers: Arc::clone(&self.child_killers),
|
|
||||||
watcher_tx: self.watcher_tx.clone(),
|
watcher_tx: self.watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -74,25 +74,11 @@ pub(in crate::agents::pool) async fn run_server_owned_completion(
|
|||||||
|
|
||||||
// Kill any in-flight cargo test processes for this worktree so they don't
|
// Kill any in-flight cargo test processes for this worktree so they don't
|
||||||
// hold the build lock while gates try to run.
|
// hold the build lock while gates try to run.
|
||||||
if let Some(wt_path) = worktree_path.as_ref()
|
if let Some(wt_path) = worktree_path.as_ref() {
|
||||||
&& let Ok(output) = std::process::Command::new("pgrep")
|
let pattern = format!("--manifest-path {}/Cargo.toml", wt_path.display());
|
||||||
.args([
|
let _ = crate::process_kill::sigkill_pids_and_verify(&crate::process_kill::pids_matching(
|
||||||
"-f",
|
&pattern,
|
||||||
&format!("--manifest-path {}/Cargo.toml", wt_path.display()),
|
));
|
||||||
])
|
|
||||||
.output()
|
|
||||||
{
|
|
||||||
let pids = String::from_utf8_lossy(&output.stdout);
|
|
||||||
for pid_str in pids.lines() {
|
|
||||||
if let Ok(pid) = pid_str.trim().parse::<i32>() {
|
|
||||||
crate::slog!(
|
|
||||||
"[agents] Killing stale cargo process (pid {pid}) for '{story_id}' before running gates"
|
|
||||||
);
|
|
||||||
unsafe {
|
|
||||||
libc::kill(pid, libc::SIGKILL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run acceptance gates. Third element of the tuple is `needs_commit_recovery`:
|
// Run acceptance gates. Third element of the tuple is `needs_commit_recovery`:
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ impl AgentPool {
|
|||||||
let pool = Arc::new(Self {
|
let pool = Arc::new(Self {
|
||||||
agents: Arc::clone(&self.agents),
|
agents: Arc::clone(&self.agents),
|
||||||
port: self.port,
|
port: self.port,
|
||||||
child_killers: Arc::clone(&self.child_killers),
|
|
||||||
watcher_tx: self.watcher_tx.clone(),
|
watcher_tx: self.watcher_tx.clone(),
|
||||||
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
status_broadcaster: Arc::clone(&self.status_broadcaster),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,12 +1,20 @@
|
|||||||
//! Process management — kills orphaned PTY child processes on server shutdown.
|
//! Process management — kills orphaned PTY child processes on server shutdown.
|
||||||
//!
|
//!
|
||||||
//! See [`crate::process_kill`] for the general process-termination primitives
|
//! As of story 1090 (2026-05-15), all process termination in this module uses
|
||||||
//! this module's existing methods (`kill_all_children`, `kill_child_for_key`)
|
//! [`crate::process_kill::sigkill_pids_and_verify`] — SIGHUP-based killing via
|
||||||
//! should eventually be migrated to. Those methods currently use
|
//! `portable_pty::ChildKiller` has been removed entirely from the server.
|
||||||
//! `portable_pty::ChildKiller::kill()`, which sends `SIGHUP` — a signal
|
//!
|
||||||
//! claude-code ignores — so they leave orphans on every shutdown/stop. The
|
//! ## History
|
||||||
//! migration is tracked in a separate story to keep its diff focused.
|
//!
|
||||||
|
//! Prior to commit `fe9804b3`, the watchdog and all kill paths sent SIGHUP via
|
||||||
|
//! `portable_pty::ChildKiller::kill()`. Claude Code ignores SIGHUP, so agents
|
||||||
|
//! survived "kills" and ran concurrently with their replacements — the root cause
|
||||||
|
//! of the 2026-05-15 duplicate-spawn incident. `fe9804b3` migrated the watchdog;
|
||||||
|
//! story 1090 completes the migration by rewriting `kill_all_children` and
|
||||||
|
//! `kill_child_for_key` (this file) to use `pids_matching` + `sigkill_pids_and_verify`.
|
||||||
|
use crate::process_kill::{pids_matching, sigkill_pids_and_verify};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
|
use crate::slog_warn;
|
||||||
|
|
||||||
use super::AgentPool;
|
use super::AgentPool;
|
||||||
|
|
||||||
@@ -14,53 +22,97 @@ impl AgentPool {
|
|||||||
/// Kill all active PTY child processes.
|
/// Kill all active PTY child processes.
|
||||||
///
|
///
|
||||||
/// Called on server shutdown to prevent orphaned Claude Code processes from
|
/// Called on server shutdown to prevent orphaned Claude Code processes from
|
||||||
/// continuing to run after the server exits. Each registered killer is called
|
/// continuing to run after the server exits. Collects each agent's worktree
|
||||||
/// once, then the registry is cleared.
|
/// path, then SIGKILLs every process running inside that path and verifies
|
||||||
|
/// termination before returning.
|
||||||
pub fn kill_all_children(&self) {
|
pub fn kill_all_children(&self) {
|
||||||
if let Ok(mut killers) = self.child_killers.lock() {
|
let worktree_paths: Vec<(String, std::path::PathBuf)> = {
|
||||||
for (key, killer) in killers.iter_mut() {
|
let Ok(agents) = self.agents.lock() else {
|
||||||
slog!("[agents] Killing child process for {key} on shutdown");
|
return;
|
||||||
let _ = killer.kill();
|
};
|
||||||
|
agents
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(key, agent)| {
|
||||||
|
agent
|
||||||
|
.worktree_info
|
||||||
|
.as_ref()
|
||||||
|
.map(|wt| (key.clone(), wt.path.clone()))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
for (key, path) in worktree_paths {
|
||||||
|
let pattern = path.display().to_string();
|
||||||
|
let pids = pids_matching(&pattern);
|
||||||
|
if pids.is_empty() {
|
||||||
|
slog!(
|
||||||
|
"[agents] No processes found in worktree {} for '{key}' on shutdown",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match sigkill_pids_and_verify(&pids) {
|
||||||
|
Ok(n) => slog!(
|
||||||
|
"[agents] SIGKILL'd {n} process(es) in worktree {} for '{key}' on shutdown",
|
||||||
|
path.display()
|
||||||
|
),
|
||||||
|
Err(survivors) => slog_warn!(
|
||||||
|
"[agents] SIGKILL incomplete for '{key}' on shutdown: \
|
||||||
|
pids still alive: {survivors:?}"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
killers.clear();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kill and deregister the child process for a specific agent key.
|
/// Kill and deregister the child process for a specific agent key.
|
||||||
///
|
///
|
||||||
/// Used by `stop_agent` to ensure the PTY child is terminated even though
|
/// Fallback used by `stop_agent` when no worktree path is recorded for the
|
||||||
/// aborting a `spawn_blocking` task handle does not interrupt the blocking thread.
|
/// agent. Also the primary kill path for any caller that has only a composite
|
||||||
|
/// key and not a worktree path directly.
|
||||||
pub(super) fn kill_child_for_key(&self, key: &str) {
|
pub(super) fn kill_child_for_key(&self, key: &str) {
|
||||||
if let Ok(mut killers) = self.child_killers.lock()
|
let worktree_path = {
|
||||||
&& let Some(mut killer) = killers.remove(key)
|
let Ok(agents) = self.agents.lock() else {
|
||||||
{
|
return;
|
||||||
slog!("[agents] Killing child process for {key} on stop");
|
};
|
||||||
let _ = killer.kill();
|
agents
|
||||||
}
|
.get(key)
|
||||||
}
|
.and_then(|a| a.worktree_info.as_ref().map(|wt| wt.path.clone()))
|
||||||
|
};
|
||||||
|
|
||||||
/// Test helper: inject a child killer into the registry.
|
let Some(path) = worktree_path else {
|
||||||
#[cfg(test)]
|
slog_warn!(
|
||||||
pub fn inject_child_killer(
|
"[agents] No worktree path recorded for '{key}'; \
|
||||||
&self,
|
cannot SIGKILL via process_kill (no-op)"
|
||||||
key: &str,
|
);
|
||||||
killer: Box<dyn portable_pty::ChildKiller + Send + Sync>,
|
return;
|
||||||
) {
|
};
|
||||||
let mut killers = self.child_killers.lock().unwrap();
|
|
||||||
killers.insert(key.to_string(), killer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test helper: return the number of registered child killers.
|
let pattern = path.display().to_string();
|
||||||
#[cfg(test)]
|
let pids = pids_matching(&pattern);
|
||||||
pub fn child_killer_count(&self) -> usize {
|
if pids.is_empty() {
|
||||||
self.child_killers.lock().unwrap().len()
|
slog!(
|
||||||
|
"[agents] No processes found in worktree {} for '{key}' on stop",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match sigkill_pids_and_verify(&pids) {
|
||||||
|
Ok(n) => slog!(
|
||||||
|
"[agents] SIGKILL'd {n} process(es) in worktree {} for '{key}' on stop",
|
||||||
|
path.display()
|
||||||
|
),
|
||||||
|
Err(survivors) => slog_warn!(
|
||||||
|
"[agents] SIGKILL incomplete for '{key}' on stop: \
|
||||||
|
pids still alive: {survivors:?}"
|
||||||
|
),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::super::AgentPool;
|
use super::super::AgentPool;
|
||||||
use portable_pty::{CommandBuilder, PtySize, native_pty_system};
|
use crate::agents::AgentStatus;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
/// Returns true if a process with the given PID is currently running.
|
/// Returns true if a process with the given PID is currently running.
|
||||||
@@ -75,79 +127,100 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn kill_all_children_is_safe_on_empty_pool() {
|
fn kill_all_children_is_safe_on_empty_pool() {
|
||||||
let pool = AgentPool::new_test(3001);
|
let pool = AgentPool::new_test(3001);
|
||||||
pool.kill_all_children();
|
pool.kill_all_children(); // must not panic
|
||||||
assert_eq!(pool.child_killer_count(), 0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// AC 4 — `kill_child_for_key` SIGKILLs the single agent's process and
|
||||||
|
/// verifies it is gone within 2 s. The sleeper has the worktree path in
|
||||||
|
/// its argv[0] so `pgrep -f` can locate it, mirroring how claude-code is
|
||||||
|
/// launched with `--directory <worktree>` in production.
|
||||||
#[test]
|
#[test]
|
||||||
fn kill_all_children_kills_real_process() {
|
fn kill_child_for_key_kills_real_process() {
|
||||||
let pool = AgentPool::new_test(3001);
|
use std::os::unix::process::CommandExt;
|
||||||
|
|
||||||
let pty_system = native_pty_system();
|
let pool = AgentPool::new_test(3002);
|
||||||
let pair = pty_system
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
.openpty(PtySize {
|
let worktree = tmp.path();
|
||||||
rows: 24,
|
|
||||||
cols: 80,
|
|
||||||
pixel_width: 0,
|
|
||||||
pixel_height: 0,
|
|
||||||
})
|
|
||||||
.expect("failed to open pty");
|
|
||||||
|
|
||||||
let mut cmd = CommandBuilder::new("sleep");
|
// argv[0] = worktree path → pgrep -f <path> finds this process.
|
||||||
cmd.arg("100");
|
let mut child = Command::new("sleep")
|
||||||
let mut child = pair
|
.arg0(worktree.to_string_lossy().as_ref())
|
||||||
.slave
|
.arg("100")
|
||||||
.spawn_command(cmd)
|
.spawn()
|
||||||
.expect("failed to spawn sleep");
|
.expect("spawn sleeper");
|
||||||
let pid = child.process_id().expect("no pid");
|
let pid = child.id();
|
||||||
|
|
||||||
pool.inject_child_killer("story:agent", child.clone_killer());
|
// Give pgrep a moment to see the new process.
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||||
|
|
||||||
|
pool.inject_test_agent_with_path(
|
||||||
|
"story-1090-kill",
|
||||||
|
"coder",
|
||||||
|
AgentStatus::Running,
|
||||||
|
worktree.to_path_buf(),
|
||||||
|
);
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
process_is_running(pid),
|
process_is_running(pid),
|
||||||
"process {pid} should be running before kill_all_children"
|
"sleeper pid {pid} should be running before kill_child_for_key"
|
||||||
);
|
);
|
||||||
|
|
||||||
pool.kill_all_children();
|
pool.kill_child_for_key("story-1090-kill:coder");
|
||||||
let _ = child.wait();
|
let _ = child.wait(); // reap zombie so ps -p returns false
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
!process_is_running(pid),
|
!process_is_running(pid),
|
||||||
"process {pid} should have been killed by kill_all_children"
|
"sleeper pid {pid} should be dead after kill_child_for_key"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// AC 5 — `kill_all_children` SIGKILLs all agents' processes. Two agents
|
||||||
|
/// with distinct worktree paths are injected; both must be gone after the call.
|
||||||
#[test]
|
#[test]
|
||||||
fn kill_all_children_clears_registry() {
|
fn kill_all_children_kills_multiple_real_processes() {
|
||||||
let pool = AgentPool::new_test(3001);
|
use std::os::unix::process::CommandExt;
|
||||||
|
|
||||||
let pty_system = native_pty_system();
|
let pool = AgentPool::new_test(3003);
|
||||||
let pair = pty_system
|
|
||||||
.openpty(PtySize {
|
let mut sleepers: Vec<(u32, std::process::Child, tempfile::TempDir)> = (0..2_u32)
|
||||||
rows: 24,
|
.map(|i| {
|
||||||
cols: 80,
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
pixel_width: 0,
|
let worktree = tmp.path();
|
||||||
pixel_height: 0,
|
// argv[0] = worktree path for pgrep discoverability.
|
||||||
|
let child = Command::new("sleep")
|
||||||
|
.arg0(worktree.to_string_lossy().as_ref())
|
||||||
|
.arg("100")
|
||||||
|
.spawn()
|
||||||
|
.expect("spawn sleeper");
|
||||||
|
let pid = child.id();
|
||||||
|
pool.inject_test_agent_with_path(
|
||||||
|
&format!("story-1090-all-{i}"),
|
||||||
|
"coder",
|
||||||
|
AgentStatus::Running,
|
||||||
|
worktree.to_path_buf(),
|
||||||
|
);
|
||||||
|
(pid, child, tmp)
|
||||||
})
|
})
|
||||||
.expect("failed to open pty");
|
.collect();
|
||||||
|
|
||||||
let mut cmd = CommandBuilder::new("sleep");
|
// Give pgrep a moment to see the new processes.
|
||||||
cmd.arg("1");
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||||
let mut child = pair
|
|
||||||
.slave
|
|
||||||
.spawn_command(cmd)
|
|
||||||
.expect("failed to spawn sleep");
|
|
||||||
|
|
||||||
pool.inject_child_killer("story:agent", child.clone_killer());
|
for (pid, _, _) in &sleepers {
|
||||||
assert_eq!(pool.child_killer_count(), 1);
|
assert!(
|
||||||
|
process_is_running(*pid),
|
||||||
|
"pid {pid} should be running before kill_all_children"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
pool.kill_all_children();
|
pool.kill_all_children();
|
||||||
let _ = child.wait();
|
|
||||||
|
|
||||||
assert_eq!(
|
for (pid, child, _tmp) in &mut sleepers {
|
||||||
pool.child_killer_count(),
|
let _ = child.wait(); // reap zombie
|
||||||
0,
|
assert!(
|
||||||
"child_killers should be cleared after kill_all_children"
|
!process_is_running(*pid),
|
||||||
|
"pid {pid} should be dead after kill_all_children"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -271,6 +271,42 @@ impl AgentPool {
|
|||||||
'{conflicting_name}' is already active at the same pipeline stage"
|
'{conflicting_name}' is already active at the same pipeline stage"
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// Cross-stage LLM agent guard: reject if any Coder/Qa/Mergemaster agent
|
||||||
|
// is already Running or Pending on this story at a *different* pipeline stage.
|
||||||
|
// These are stale agents left over from a previous stage transition that has
|
||||||
|
// since advanced. The periodic reconciler (reconcile_canonical_agents) stops
|
||||||
|
// them; here we surface the conflict so the caller waits for reconciliation.
|
||||||
|
if matches!(
|
||||||
|
resolved_stage,
|
||||||
|
PipelineStage::Coder | PipelineStage::Qa | PipelineStage::Mergemaster
|
||||||
|
) && let Some(stale_name) = agents.iter().find_map(|(k, a)| {
|
||||||
|
let k_story = k.rsplit_once(':').map(|(s, _)| s).unwrap_or(k);
|
||||||
|
if k_story != story_id || a.agent_name == resolved_name {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !matches!(a.status, AgentStatus::Running | AgentStatus::Pending) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let a_stage = config
|
||||||
|
.find_agent(&a.agent_name)
|
||||||
|
.map(agent_config_stage)
|
||||||
|
.unwrap_or_else(|| pipeline_stage(&a.agent_name));
|
||||||
|
if matches!(
|
||||||
|
a_stage,
|
||||||
|
PipelineStage::Coder | PipelineStage::Qa | PipelineStage::Mergemaster
|
||||||
|
) && a_stage != resolved_stage
|
||||||
|
{
|
||||||
|
Some(a.agent_name.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}) {
|
||||||
|
return Err(format!(
|
||||||
|
"story '{story_id}' already has an active LLM agent '{stale_name}'; \
|
||||||
|
refusing to spawn '{resolved_name}'"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
// Enforce single-instance concurrency for explicitly-named agents:
|
// Enforce single-instance concurrency for explicitly-named agents:
|
||||||
// if this agent is already running on any other story, reject.
|
// if this agent is already running on any other story, reject.
|
||||||
// Auto-selected agents are already guaranteed idle by
|
// Auto-selected agents are already guaranteed idle by
|
||||||
@@ -392,7 +428,6 @@ impl AgentPool {
|
|||||||
event_log.clone(),
|
event_log.clone(),
|
||||||
self.port,
|
self.port,
|
||||||
log_writer.clone(),
|
log_writer.clone(),
|
||||||
self.child_killers.clone(),
|
|
||||||
self.watcher_tx.clone(),
|
self.watcher_tx.clone(),
|
||||||
inactivity_timeout_secs,
|
inactivity_timeout_secs,
|
||||||
prior_events,
|
prior_events,
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ use std::collections::HashMap;
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use portable_pty::ChildKiller;
|
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use crate::agent_log::AgentLogWriter;
|
use crate::agent_log::AgentLogWriter;
|
||||||
@@ -117,6 +116,23 @@ pub(super) fn maybe_inject_gate_failure(args: &mut Vec<String>, story_id: &str)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Append `Edit,Write,Bash` to the `--disallowedTools` flag so worktree agents
|
||||||
|
/// cannot write to the master tree via Claude's built-in tools. If
|
||||||
|
/// `--disallowedTools` is already present (from agent config), the three names
|
||||||
|
/// are appended to the existing value rather than replacing it.
|
||||||
|
pub(super) fn inject_worktree_disallowed_tools(args: &mut Vec<String>) {
|
||||||
|
const BLOCKED: &str = "Edit,Write,Bash";
|
||||||
|
if let Some(pos) = args.iter().position(|a| a == "--disallowedTools") {
|
||||||
|
if let Some(val) = args.get_mut(pos + 1) {
|
||||||
|
val.push(',');
|
||||||
|
val.push_str(BLOCKED);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
args.push("--disallowedTools".to_string());
|
||||||
|
args.push(BLOCKED.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Run the background worktree-creation + agent-launch flow.
|
/// Run the background worktree-creation + agent-launch flow.
|
||||||
///
|
///
|
||||||
/// Caller (`AgentPool::start_agent`) wraps this in `tokio::spawn` and stores
|
/// Caller (`AgentPool::start_agent`) wraps this in `tokio::spawn` and stores
|
||||||
@@ -135,7 +151,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
event_log: Arc<Mutex<Vec<AgentEvent>>>,
|
event_log: Arc<Mutex<Vec<AgentEvent>>>,
|
||||||
port: u16,
|
port: u16,
|
||||||
log_writer: Option<Arc<Mutex<AgentLogWriter>>>,
|
log_writer: Option<Arc<Mutex<AgentLogWriter>>>,
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
watcher_tx: broadcast::Sender<WatcherEvent>,
|
watcher_tx: broadcast::Sender<WatcherEvent>,
|
||||||
inactivity_timeout_secs: u64,
|
inactivity_timeout_secs: u64,
|
||||||
// Formatted `<recent-events>` block drained from the previous session's
|
// Formatted `<recent-events>` block drained from the previous session's
|
||||||
@@ -159,7 +174,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
let log_clone = event_log;
|
let log_clone = event_log;
|
||||||
let port_for_task = port;
|
let port_for_task = port;
|
||||||
let log_writer_clone = log_writer;
|
let log_writer_clone = log_writer;
|
||||||
let child_killers_clone = child_killers;
|
|
||||||
let watcher_tx_clone = watcher_tx;
|
let watcher_tx_clone = watcher_tx;
|
||||||
let _ = inactivity_timeout_secs; // currently unused inside the closure body
|
let _ = inactivity_timeout_secs; // currently unused inside the closure body
|
||||||
|
|
||||||
@@ -267,6 +281,10 @@ pub(super) async fn run_agent_spawn(
|
|||||||
maybe_inject_gate_failure(&mut args, &sid);
|
maybe_inject_gate_failure(&mut args, &sid);
|
||||||
// Cap turns and budget for merge-gate fixup sessions (story 981).
|
// Cap turns and budget for merge-gate fixup sessions (story 981).
|
||||||
maybe_cap_for_merge_fixup(&mut args, &sid);
|
maybe_cap_for_merge_fixup(&mut args, &sid);
|
||||||
|
// Every agent that runs inside a worktree must use the validated MCP
|
||||||
|
// edit/write tools instead of Claude's built-in Edit/Write/Bash. This
|
||||||
|
// prevents accidental writes to the master worktree (stories 1127, 1136).
|
||||||
|
inject_worktree_disallowed_tools(&mut args);
|
||||||
|
|
||||||
// Append project-local prompt content (.huskies/AGENT.md) to the
|
// Append project-local prompt content (.huskies/AGENT.md) to the
|
||||||
// baked-in prompt so every agent role sees project-specific guidance
|
// baked-in prompt so every agent role sees project-specific guidance
|
||||||
@@ -371,8 +389,7 @@ pub(super) async fn run_agent_spawn(
|
|||||||
|
|
||||||
let run_result = match runtime_name {
|
let run_result = match runtime_name {
|
||||||
"claude-code" => {
|
"claude-code" => {
|
||||||
let runtime =
|
let runtime = ClaudeCodeRuntime::new(watcher_tx_clone.clone());
|
||||||
ClaudeCodeRuntime::new(child_killers_clone.clone(), watcher_tx_clone.clone());
|
|
||||||
let ctx = RuntimeContext {
|
let ctx = RuntimeContext {
|
||||||
story_id: sid.clone(),
|
story_id: sid.clone(),
|
||||||
agent_name: aname.clone(),
|
agent_name: aname.clone(),
|
||||||
@@ -566,7 +583,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents: agents_for_respawn,
|
agents: agents_for_respawn,
|
||||||
port: port_r,
|
port: port_r,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_for_respawn,
|
watcher_tx: watcher_for_respawn,
|
||||||
status_broadcaster: Arc::new(
|
status_broadcaster: Arc::new(
|
||||||
crate::service::status::StatusBroadcaster::new(),
|
crate::service::status::StatusBroadcaster::new(),
|
||||||
@@ -654,7 +670,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents: agents_for_cd,
|
agents: agents_for_cd,
|
||||||
port: port_for_cd,
|
port: port_for_cd,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_for_cd,
|
watcher_tx: watcher_for_cd,
|
||||||
status_broadcaster: Arc::new(
|
status_broadcaster: Arc::new(
|
||||||
crate::service::status::StatusBroadcaster::new(),
|
crate::service::status::StatusBroadcaster::new(),
|
||||||
@@ -774,7 +789,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents: agents_for_cd,
|
agents: agents_for_cd,
|
||||||
port: port_for_cd,
|
port: port_for_cd,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_for_cd,
|
watcher_tx: watcher_for_cd,
|
||||||
status_broadcaster: Arc::new(
|
status_broadcaster: Arc::new(
|
||||||
crate::service::status::StatusBroadcaster::new(),
|
crate::service::status::StatusBroadcaster::new(),
|
||||||
@@ -815,6 +829,7 @@ pub(super) async fn run_agent_spawn(
|
|||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryTotalAttempts(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryTotalAttempts(
|
||||||
&sid,
|
&sid,
|
||||||
));
|
));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(&sid));
|
||||||
|
|
||||||
// Remove agent from the pool and unblock any wait_for_agent callers.
|
// Remove agent from the pool and unblock any wait_for_agent callers.
|
||||||
let tx_done = {
|
let tx_done = {
|
||||||
@@ -862,7 +877,6 @@ pub(super) async fn run_agent_spawn(
|
|||||||
let pool = AgentPool {
|
let pool = AgentPool {
|
||||||
agents: agents_for_respawn,
|
agents: agents_for_respawn,
|
||||||
port: port_r,
|
port: port_r,
|
||||||
child_killers: Arc::new(Mutex::new(HashMap::new())),
|
|
||||||
watcher_tx: watcher_for_respawn,
|
watcher_tx: watcher_for_respawn,
|
||||||
status_broadcaster: Arc::new(
|
status_broadcaster: Arc::new(
|
||||||
crate::service::status::StatusBroadcaster::new(),
|
crate::service::status::StatusBroadcaster::new(),
|
||||||
@@ -881,6 +895,17 @@ pub(super) async fn run_agent_spawn(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AC1 (story 1089): mark forced exits so the commit-recovery
|
||||||
|
// stuck counter is not incremented for API errors, network
|
||||||
|
// failures, or Claude-API budget exhaustion. A non-zero exit
|
||||||
|
// code means the CLI was forced out, not that it chose to stop.
|
||||||
|
if !result.exit_ok {
|
||||||
|
crate::db::write_content(
|
||||||
|
crate::db::ContentKey::CommitRecoveryForcedExit(&sid),
|
||||||
|
"1",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Server-owned completion: run acceptance gates automatically
|
// Server-owned completion: run acceptance gates automatically
|
||||||
// when the agent process exits normally.
|
// when the agent process exits normally.
|
||||||
super::super::pipeline::run_server_owned_completion(
|
super::super::pipeline::run_server_owned_completion(
|
||||||
@@ -1254,12 +1279,13 @@ mod tests {
|
|||||||
"abc123",
|
"abc123",
|
||||||
);
|
);
|
||||||
|
|
||||||
// Rate-limit exit handler: reset all three counters (the fix).
|
// Rate-limit exit handler: reset all counters (the fix).
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(story_id));
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(story_id));
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryDiffFingerprint(
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryDiffFingerprint(
|
||||||
story_id,
|
story_id,
|
||||||
));
|
));
|
||||||
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id));
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryTotalAttempts(story_id));
|
||||||
|
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryReadSet(story_id));
|
||||||
|
|
||||||
// CommitRecoveryPending must be cleared after each rate-limit exit.
|
// CommitRecoveryPending must be cleared after each rate-limit exit.
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1292,4 +1318,43 @@ mod tests {
|
|||||||
item.stage().dir_name()
|
item.stage().dir_name()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── inject_worktree_disallowed_tools (AC1, story 1142) ───────────
|
||||||
|
|
||||||
|
/// AC3(c) proxy: worktree agents get `--disallowedTools Edit,Write,Bash`.
|
||||||
|
#[test]
|
||||||
|
fn worktree_disallowed_tools_added_when_absent() {
|
||||||
|
let mut args: Vec<String> = vec!["--verbose".to_string()];
|
||||||
|
inject_worktree_disallowed_tools(&mut args);
|
||||||
|
let pos = args
|
||||||
|
.iter()
|
||||||
|
.position(|a| a == "--disallowedTools")
|
||||||
|
.expect("--disallowedTools must be present");
|
||||||
|
let val = &args[pos + 1];
|
||||||
|
assert!(val.contains("Edit"), "must include Edit");
|
||||||
|
assert!(val.contains("Write"), "must include Write");
|
||||||
|
assert!(val.contains("Bash"), "must include Bash");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Existing `--disallowedTools` value is extended, not replaced.
|
||||||
|
#[test]
|
||||||
|
fn worktree_disallowed_tools_appended_to_existing() {
|
||||||
|
let mut args = vec!["--disallowedTools".to_string(), "SomeOtherTool".to_string()];
|
||||||
|
inject_worktree_disallowed_tools(&mut args);
|
||||||
|
// Only one --disallowedTools flag.
|
||||||
|
let count = args
|
||||||
|
.iter()
|
||||||
|
.filter(|a| a.as_str() == "--disallowedTools")
|
||||||
|
.count();
|
||||||
|
assert_eq!(count, 1, "must not duplicate --disallowedTools");
|
||||||
|
let pos = args.iter().position(|a| a == "--disallowedTools").unwrap();
|
||||||
|
let val = &args[pos + 1];
|
||||||
|
assert!(
|
||||||
|
val.contains("SomeOtherTool"),
|
||||||
|
"original tool must be preserved"
|
||||||
|
);
|
||||||
|
assert!(val.contains("Edit"), "Edit must be added");
|
||||||
|
assert!(val.contains("Write"), "Write must be added");
|
||||||
|
assert!(val.contains("Bash"), "Bash must be added");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -602,6 +602,266 @@ async fn start_agent_allows_correct_stage_agent() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── story-1100: cross-stage LLM agent rejection ─────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn start_agent_rejects_mergemaster_when_coder_running_same_story() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(&sk_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
sk_dir.join("project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n\n\
|
||||||
|
[[agent]]\nname = \"mergemaster\"\nstage = \"mergemaster\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
pool.inject_test_agent("999_story_cross", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
|
let result = pool
|
||||||
|
.start_agent(root, "999_story_cross", Some("mergemaster"), None, None)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"mergemaster must be rejected when coder-1 is still running on same story"
|
||||||
|
);
|
||||||
|
let err = result.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
err.contains("active LLM agent") || err.contains("stale agent"),
|
||||||
|
"error must mention active LLM agent conflict, got: '{err}'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn start_agent_rejects_coder_when_mergemaster_running_same_story() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(&sk_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
sk_dir.join("project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n\n\
|
||||||
|
[[agent]]\nname = \"mergemaster\"\nstage = \"mergemaster\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
pool.inject_test_agent("888_story_cross2", "mergemaster", AgentStatus::Running);
|
||||||
|
|
||||||
|
let result = pool
|
||||||
|
.start_agent(root, "888_story_cross2", Some("coder-1"), None, None)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"coder-1 must be rejected when mergemaster is running on same story"
|
||||||
|
);
|
||||||
|
let err = result.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
err.contains("active LLM agent") || err.contains("stale agent"),
|
||||||
|
"error must mention active LLM agent conflict, got: '{err}'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn start_agent_cross_stage_does_not_block_different_stories() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(sk_dir.join("work/1_backlog")).unwrap();
|
||||||
|
fs::write(
|
||||||
|
root.join(".huskies/project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n\n\
|
||||||
|
[[agent]]\nname = \"mergemaster\"\nstage = \"mergemaster\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::write(
|
||||||
|
root.join(".huskies/work/1_backlog/777_story_other.md"),
|
||||||
|
"---\nname: Other\n---\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
// mergemaster running on story-x should NOT block coder on story-y
|
||||||
|
pool.inject_test_agent("111_story_x", "mergemaster", AgentStatus::Running);
|
||||||
|
|
||||||
|
let result = pool
|
||||||
|
.start_agent(root, "777_story_other", Some("coder-1"), None, None)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if let Err(ref e) = result {
|
||||||
|
assert!(
|
||||||
|
!e.contains("active LLM agent") && !e.contains("stale agent"),
|
||||||
|
"cross-stage guard must not fire for agents on different stories, got: '{e}'"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconcile_canonical_agents_stops_stale_coder_in_qa_stage() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(&sk_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
sk_dir.join("project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Write story to CRDT in QA stage: canonical = Qa, but coder-1 is Running.
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::db::write_item_with_content(
|
||||||
|
"777_story_reconcile",
|
||||||
|
"qa",
|
||||||
|
"---\nname: Reconcile Test\n---\n",
|
||||||
|
crate::db::ItemMeta::named("Reconcile Test"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
pool.inject_test_agent("777_story_reconcile", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
|
let before = pool.list_agents().unwrap();
|
||||||
|
assert!(
|
||||||
|
before.iter().any(|a| a.agent_name == "coder-1"
|
||||||
|
&& matches!(a.status, AgentStatus::Running | AgentStatus::Pending)),
|
||||||
|
"coder-1 should be Running before reconciliation"
|
||||||
|
);
|
||||||
|
|
||||||
|
pool.reconcile_canonical_agents(root).await;
|
||||||
|
|
||||||
|
let after = pool.list_agents().unwrap();
|
||||||
|
let still_active = after.iter().any(|a| {
|
||||||
|
a.story_id == "777_story_reconcile"
|
||||||
|
&& a.agent_name == "coder-1"
|
||||||
|
&& matches!(a.status, AgentStatus::Running | AgentStatus::Pending)
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
!still_active,
|
||||||
|
"reconciler must have stopped coder-1 (CRDT stage is QA, coder is wrong stage)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reconcile_canonical_agents_leaves_correct_stage_agent_alone() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(&sk_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
sk_dir.join("project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Story is in coding stage: canonical = Coder. coder-1 is correct.
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::db::write_item_with_content(
|
||||||
|
"555_story_correct",
|
||||||
|
"coding",
|
||||||
|
"---\nname: Correct Stage\n---\n",
|
||||||
|
crate::db::ItemMeta::named("Correct Stage"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
pool.inject_test_agent("555_story_correct", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
|
pool.reconcile_canonical_agents(root).await;
|
||||||
|
|
||||||
|
let after = pool.list_agents().unwrap();
|
||||||
|
let still_active = after.iter().any(|a| {
|
||||||
|
a.story_id == "555_story_correct"
|
||||||
|
&& a.agent_name == "coder-1"
|
||||||
|
&& matches!(a.status, AgentStatus::Running | AgentStatus::Pending)
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
still_active,
|
||||||
|
"reconciler must NOT stop coder-1 when it matches the canonical stage"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Regression test for story 1100: a stale coder left running after a stage
|
||||||
|
/// transition blocks both a same-stage coder and a cross-stage mergemaster.
|
||||||
|
/// The periodic reconciler stops the stale coder, after which the pool no
|
||||||
|
/// longer has a cross-stage conflict.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn regression_1100_stale_coder_blocks_mergemaster_then_reconciler_clears() {
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path();
|
||||||
|
|
||||||
|
let sk_dir = root.join(".huskies");
|
||||||
|
fs::create_dir_all(&sk_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
sk_dir.join("project.toml"),
|
||||||
|
"[[agent]]\nname = \"coder-1\"\nstage = \"coder\"\n\n\
|
||||||
|
[[agent]]\nname = \"coder-2\"\nstage = \"coder\"\n\n\
|
||||||
|
[[agent]]\nname = \"mergemaster\"\nstage = \"mergemaster\"\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let pool = AgentPool::new_test(3099);
|
||||||
|
// Simulate coder-1 still Running after the story advanced past the coding stage.
|
||||||
|
pool.inject_test_agent("1100_reg", "coder-1", AgentStatus::Running);
|
||||||
|
|
||||||
|
// coder-2 blocked by same-stage check (both are Coder stage)
|
||||||
|
let r1 = pool
|
||||||
|
.start_agent(root, "1100_reg", Some("coder-2"), None, None)
|
||||||
|
.await;
|
||||||
|
assert!(r1.is_err(), "coder-2 must be rejected by same-stage guard");
|
||||||
|
assert!(
|
||||||
|
r1.unwrap_err().contains("same pipeline stage"),
|
||||||
|
"same-stage check must fire for coder-2"
|
||||||
|
);
|
||||||
|
|
||||||
|
// mergemaster blocked by cross-stage LLM guard (coder-1 is a different LLM stage)
|
||||||
|
let r2 = pool
|
||||||
|
.start_agent(root, "1100_reg", Some("mergemaster"), None, None)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
r2.is_err(),
|
||||||
|
"mergemaster must be rejected because coder-1 (different LLM stage) is still running"
|
||||||
|
);
|
||||||
|
let r2_err = r2.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
r2_err.contains("active LLM agent") || r2_err.contains("stale agent"),
|
||||||
|
"cross-stage rejection expected, got: '{r2_err}'"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Reconciler: story "1100_reg" has no CRDT entry → canonical = None → stop coder-1.
|
||||||
|
pool.reconcile_canonical_agents(root).await;
|
||||||
|
|
||||||
|
// coder-1 must be gone from the active pool.
|
||||||
|
let remaining = pool.list_agents().unwrap();
|
||||||
|
assert!(
|
||||||
|
!remaining.iter().any(|a| {
|
||||||
|
a.story_id == "1100_reg"
|
||||||
|
&& a.agent_name == "coder-1"
|
||||||
|
&& matches!(a.status, AgentStatus::Running | AgentStatus::Pending)
|
||||||
|
}),
|
||||||
|
"reconciler must have removed stale coder-1 from the active pool"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Bug 502: when start_agent is called for a non-Coder agent (mergemaster
|
/// Bug 502: when start_agent is called for a non-Coder agent (mergemaster
|
||||||
/// or qa) on a story that's in 4_merge/, the unconditional
|
/// or qa) on a story that's in 4_merge/, the unconditional
|
||||||
/// move_story_to_current at the top of start_agent must NOT fire — even
|
/// move_story_to_current at the top of start_agent must NOT fire — even
|
||||||
|
|||||||
@@ -2,11 +2,11 @@
|
|||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use crate::config::ProjectConfig;
|
use super::super::super::{
|
||||||
use crate::pipeline_state::Stage;
|
PipelineStage, agent_config_stage, canonical_pipeline_stage, pipeline_stage,
|
||||||
|
};
|
||||||
use super::super::super::{PipelineStage, agent_config_stage, pipeline_stage};
|
|
||||||
use super::super::worktree::find_active_story_stage;
|
use super::super::worktree::find_active_story_stage;
|
||||||
|
use crate::config::ProjectConfig;
|
||||||
|
|
||||||
/// Validate that an explicit `agent_name` is allowed to attach to `story_id`'s
|
/// Validate that an explicit `agent_name` is allowed to attach to `story_id`'s
|
||||||
/// current pipeline stage.
|
/// current pipeline stage.
|
||||||
@@ -34,16 +34,15 @@ pub(super) fn validate_agent_stage(
|
|||||||
let Some(story_stage) = find_active_story_stage(project_root, story_id) else {
|
let Some(story_stage) = find_active_story_stage(project_root, story_id) else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
let expected_stage = match story_stage {
|
let canonical = canonical_pipeline_stage(&story_stage);
|
||||||
Stage::Coding { .. } => PipelineStage::Coder,
|
let is_llm = matches!(
|
||||||
Stage::Qa => PipelineStage::Qa,
|
agent_stage,
|
||||||
Stage::Merge { .. } => PipelineStage::Mergemaster,
|
PipelineStage::Coder | PipelineStage::Qa | PipelineStage::Mergemaster
|
||||||
_ => PipelineStage::Other,
|
);
|
||||||
};
|
if is_llm && (canonical.is_none() || canonical.as_ref() != Some(&agent_stage)) {
|
||||||
if expected_stage != PipelineStage::Other && expected_stage != agent_stage {
|
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"Agent '{name}' (stage: {agent_stage:?}) cannot be assigned to \
|
"Agent '{name}' (stage: {agent_stage:?}) cannot be assigned to \
|
||||||
story '{story_id}' in {}/ (requires stage: {expected_stage:?})",
|
story '{story_id}' in {}/ (requires stage: {canonical:?})",
|
||||||
story_stage.dir_name()
|
story_stage.dir_name()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ use crate::slog_error;
|
|||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use super::super::{AgentEvent, AgentStatus};
|
use super::super::{
|
||||||
|
AgentEvent, AgentStatus, PipelineStage, agent_config_stage, canonical_pipeline_stage,
|
||||||
|
pipeline_stage,
|
||||||
|
};
|
||||||
use super::AgentPool;
|
use super::AgentPool;
|
||||||
use super::types::composite_key;
|
use super::types::composite_key;
|
||||||
|
|
||||||
@@ -71,8 +74,7 @@ impl AgentPool {
|
|||||||
self.kill_child_for_key(&key);
|
self.kill_child_for_key(&key);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 3: now safe to mutate. Status flip, handle abort, drop the
|
// Step 3: now safe to mutate. Status flip and handle abort.
|
||||||
// child_killers entry.
|
|
||||||
let (task_handle, tx) = {
|
let (task_handle, tx) = {
|
||||||
let mut agents = self.agents.lock().map_err(|e| e.to_string())?;
|
let mut agents = self.agents.lock().map_err(|e| e.to_string())?;
|
||||||
let agent = agents
|
let agent = agents
|
||||||
@@ -88,9 +90,6 @@ impl AgentPool {
|
|||||||
handle.abort();
|
handle.abort();
|
||||||
let _ = handle.await;
|
let _ = handle.await;
|
||||||
}
|
}
|
||||||
if let Ok(mut killers) = self.child_killers.lock() {
|
|
||||||
killers.remove(&key);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Preserve worktree for inspection — don't destroy agent's work on stop.
|
// Preserve worktree for inspection — don't destroy agent's work on stop.
|
||||||
if let Some(ref wt) = worktree_info {
|
if let Some(ref wt) = worktree_info {
|
||||||
@@ -118,6 +117,82 @@ impl AgentPool {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Stop LLM agents whose pipeline stage no longer matches the story's canonical stage.
|
||||||
|
///
|
||||||
|
/// Called periodically by the tick loop (story 1100). For each Running or Pending
|
||||||
|
/// LLM agent (Coder, Qa, or Mergemaster) whose stage does not match the canonical
|
||||||
|
/// stage derived from the story's current CRDT state, the agent is stopped via the
|
||||||
|
/// existing SIGKILL path. Idempotent: agents already at the correct stage are left
|
||||||
|
/// untouched. Also stops LLM agents on stories that have no active pipeline stage
|
||||||
|
/// (terminal, blocked, or frozen), since no LLM agent should run there.
|
||||||
|
pub async fn reconcile_canonical_agents(&self, root: &std::path::Path) {
|
||||||
|
use crate::config::ProjectConfig;
|
||||||
|
|
||||||
|
let config = match ProjectConfig::load(root) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
slog_warn!("[reconcile] Cannot load config for canonical reconcile: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Snapshot active LLM agents without holding the lock during async stops.
|
||||||
|
let snapshot: Vec<(String, String, PipelineStage)> = {
|
||||||
|
let Ok(agents) = self.agents.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
agents
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(key, a)| {
|
||||||
|
if !matches!(a.status, AgentStatus::Running | AgentStatus::Pending) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let stage = config
|
||||||
|
.find_agent(&a.agent_name)
|
||||||
|
.map(agent_config_stage)
|
||||||
|
.unwrap_or_else(|| pipeline_stage(&a.agent_name));
|
||||||
|
if !matches!(
|
||||||
|
stage,
|
||||||
|
PipelineStage::Coder | PipelineStage::Qa | PipelineStage::Mergemaster
|
||||||
|
) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let story_id = key
|
||||||
|
.rsplit_once(':')
|
||||||
|
.map(|(s, _)| s)
|
||||||
|
.unwrap_or(key)
|
||||||
|
.to_string();
|
||||||
|
Some((story_id, a.agent_name.clone(), stage))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
for (story_id, agent_name, agent_stage) in snapshot {
|
||||||
|
let canonical = crate::pipeline_state::read_typed(&story_id)
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.and_then(|item| canonical_pipeline_stage(&item.stage));
|
||||||
|
|
||||||
|
let should_stop = match &canonical {
|
||||||
|
None => true,
|
||||||
|
Some(c) if *c != agent_stage => true,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if !should_stop {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
slog!(
|
||||||
|
"[reconcile] stopping '{agent_name}' on '{story_id}': \
|
||||||
|
canonical={canonical:?} actual={agent_stage:?}"
|
||||||
|
);
|
||||||
|
if let Err(e) = self.stop_agent(root, &story_id, &agent_name).await {
|
||||||
|
slog_warn!("[reconcile] failed to stop '{agent_name}' on '{story_id}': {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Remove all agent entries for a given story_id from the pool.
|
/// Remove all agent entries for a given story_id from the pool.
|
||||||
///
|
///
|
||||||
/// Called when a story is archived so that stale entries don't accumulate.
|
/// Called when a story is archived so that stale entries don't accumulate.
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ pub(super) fn find_active_story_stage(
|
|||||||
crate::pipeline_state::Stage::Coding { .. }
|
crate::pipeline_state::Stage::Coding { .. }
|
||||||
| crate::pipeline_state::Stage::Qa
|
| crate::pipeline_state::Stage::Qa
|
||||||
| crate::pipeline_state::Stage::Merge { .. }
|
| crate::pipeline_state::Stage::Merge { .. }
|
||||||
|
| crate::pipeline_state::Stage::MergeFailure { .. }
|
||||||
|
| crate::pipeline_state::Stage::MergeFailureFinal { .. }
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
return Some(item.stage);
|
return Some(item.stage);
|
||||||
|
|||||||
@@ -129,7 +129,13 @@ pub(crate) async fn on_coding_transition(project_root: &Path, port: u16, story_i
|
|||||||
"[worktree-create-sub] Worktree ready for '{story_id}' at {}",
|
"[worktree-create-sub] Worktree ready for '{story_id}' at {}",
|
||||||
info.path.display()
|
info.path.display()
|
||||||
);
|
);
|
||||||
if let Err(e) = crate::worktree::install_pre_commit_hook(&info.path) {
|
let hook_path = info.path.clone();
|
||||||
|
let hook_result = tokio::task::spawn_blocking(move || {
|
||||||
|
crate::worktree::install_pre_commit_hook(&hook_path)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| Err(format!("spawn_blocking panicked: {e}")));
|
||||||
|
if let Err(e) = hook_result {
|
||||||
slog_warn!(
|
slog_warn!(
|
||||||
"[worktree-create-sub] Pre-commit hook install failed for '{story_id}': {e}"
|
"[worktree-create-sub] Pre-commit hook install failed for '{story_id}': {e}"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::agents::AgentEvent;
|
use crate::agents::AgentEvent;
|
||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
@@ -41,7 +40,6 @@ mod tests {
|
|||||||
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let event_log = Arc::new(Mutex::new(Vec::new()));
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
let child_killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
|
|
||||||
// sh -p "--" <script>: -p = privileged mode, "--" = end options,
|
// sh -p "--" <script>: -p = privileged mode, "--" = end options,
|
||||||
// then the script path is the file operand.
|
// then the script path is the file operand.
|
||||||
@@ -56,7 +54,6 @@ mod tests {
|
|||||||
&event_log,
|
&event_log,
|
||||||
None,
|
None,
|
||||||
0,
|
0,
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -98,7 +95,6 @@ mod tests {
|
|||||||
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let event_log = Arc::new(Mutex::new(Vec::new()));
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
let child_killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
|
|
||||||
let result = run_agent_pty_streaming(
|
let result = run_agent_pty_streaming(
|
||||||
"423_story_rate_limit",
|
"423_story_rate_limit",
|
||||||
@@ -111,7 +107,6 @@ mod tests {
|
|||||||
&event_log,
|
&event_log,
|
||||||
None,
|
None,
|
||||||
0,
|
0,
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -160,7 +155,6 @@ mod tests {
|
|||||||
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let event_log = Arc::new(Mutex::new(Vec::new()));
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
let child_killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
|
|
||||||
let before = chrono::Utc::now();
|
let before = chrono::Utc::now();
|
||||||
let result = run_agent_pty_streaming(
|
let result = run_agent_pty_streaming(
|
||||||
@@ -174,7 +168,6 @@ mod tests {
|
|||||||
&event_log,
|
&event_log,
|
||||||
None,
|
None,
|
||||||
0,
|
0,
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -229,7 +222,6 @@ mod tests {
|
|||||||
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
let (watcher_tx, _watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, _watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let event_log = Arc::new(Mutex::new(Vec::new()));
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
let child_killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
|
|
||||||
let result = run_agent_pty_streaming(
|
let result = run_agent_pty_streaming(
|
||||||
"916_story_rate_limit_extension",
|
"916_story_rate_limit_extension",
|
||||||
@@ -242,7 +234,6 @@ mod tests {
|
|||||||
&event_log,
|
&event_log,
|
||||||
None,
|
None,
|
||||||
1, // inactivity_timeout_secs = 1s; would expire before the 3s sleep without the extension
|
1, // inactivity_timeout_secs = 1s; would expire before the 3s sleep without the extension
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -407,18 +398,16 @@ mod tests {
|
|||||||
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
let (watcher_tx, _watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, _watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let event_log = Arc::new(Mutex::new(Vec::new()));
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
let child_killers: Arc<
|
|
||||||
Mutex<HashMap<String, Box<dyn portable_pty::ChildKiller + Send + Sync>>>,
|
|
||||||
> = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
let child_killers_for_kill = Arc::clone(&child_killers);
|
|
||||||
|
|
||||||
// Spawn a task to kill the child after a short delay (simulating watchdog).
|
// Spawn a task to kill the child after a short delay (simulating watchdog).
|
||||||
|
// Uses pids_matching on the script path — same mechanism as the production
|
||||||
|
// watchdog after the process_kill migration (story 1090).
|
||||||
|
let script_path_for_kill = script.to_string_lossy().to_string();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||||
if let Ok(mut killers) = child_killers_for_kill.lock() {
|
let pids = crate::process_kill::pids_matching(&script_path_for_kill);
|
||||||
for (_, killer) in killers.iter_mut() {
|
if !pids.is_empty() {
|
||||||
let _ = killer.kill();
|
let _ = crate::process_kill::sigkill_pids_and_verify(&pids);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -435,7 +424,6 @@ mod tests {
|
|||||||
&event_log,
|
&event_log,
|
||||||
None,
|
None,
|
||||||
0, // no inactivity timeout
|
0, // no inactivity timeout
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
watcher_tx,
|
||||||
None, // no session to resume
|
None, // no session to resume
|
||||||
Some((project_root.clone(), "sonnet".to_string())),
|
Some((project_root.clone(), "sonnet".to_string())),
|
||||||
@@ -457,4 +445,62 @@ mod tests {
|
|||||||
the respawn's lookup_session returns it (warm), not None (cold)"
|
the respawn's lookup_session returns it (warm), not None (cold)"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── bug 1103: soft rate-limit warning (status=allowed) must NOT set rate_limit_exit ──
|
||||||
|
|
||||||
|
/// Regression: a `rate_limit_event` with `status="allowed"` is a soft
|
||||||
|
/// warning — the request was permitted. The session that follows should
|
||||||
|
/// complete normally and report `rate_limit_exit == false`, not trigger the
|
||||||
|
/// rate-limit respawn path in the spawn handler.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rate_limit_allowed_status_does_not_set_rate_limit_exit() {
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let script = tmp.path().join("emit_allowed_then_exit.sh");
|
||||||
|
// Emit status="allowed" (soft warning), then exit cleanly.
|
||||||
|
std::fs::write(
|
||||||
|
&script,
|
||||||
|
"#!/bin/sh\nprintf '%s\\n' '{\"type\":\"rate_limit_event\",\"rate_limit_info\":{\"status\":\"allowed\",\"reset_at\":\"2099-01-01T12:00:00Z\"}}'\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||||
|
|
||||||
|
let (tx, _rx) = broadcast::channel::<AgentEvent>(64);
|
||||||
|
let (watcher_tx, mut watcher_rx) = broadcast::channel::<WatcherEvent>(16);
|
||||||
|
let event_log = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
|
||||||
|
let result = run_agent_pty_streaming(
|
||||||
|
"1103_soft_warning_no_exit_flag",
|
||||||
|
"coder-1",
|
||||||
|
"sh",
|
||||||
|
&[script.to_string_lossy().to_string()],
|
||||||
|
"--",
|
||||||
|
"/tmp",
|
||||||
|
&tx,
|
||||||
|
&event_log,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
watcher_tx,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let pty = result.expect("PTY run should succeed");
|
||||||
|
assert!(
|
||||||
|
!pty.rate_limit_exit,
|
||||||
|
"rate_limit_exit must be false for a soft 'allowed' warning; \
|
||||||
|
only genuine hard blocks (rejected) should set it"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Watcher must have received RateLimitWarning, not RateLimitHardBlock.
|
||||||
|
let evt = watcher_rx
|
||||||
|
.try_recv()
|
||||||
|
.expect("Expected a RateLimitWarning watcher event");
|
||||||
|
assert!(
|
||||||
|
matches!(evt, WatcherEvent::RateLimitWarning { .. }),
|
||||||
|
"Expected RateLimitWarning for status=allowed, got: {evt:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
//! PTY process spawning and output loop: builds the command, drives the reader thread,
|
//! PTY process spawning and output loop: builds the command, drives the reader thread,
|
||||||
//! and dispatches parsed JSON events to the broadcast channel.
|
//! and dispatches parsed JSON events to the broadcast channel.
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::io::{BufRead, BufReader};
|
use std::io::{BufRead, BufReader};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use portable_pty::{ChildKiller, CommandBuilder, PtySize, native_pty_system};
|
use portable_pty::{CommandBuilder, PtySize, native_pty_system};
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use crate::agent_log::AgentLogWriter;
|
use crate::agent_log::AgentLogWriter;
|
||||||
@@ -14,7 +13,7 @@ use crate::slog;
|
|||||||
use crate::slog_warn;
|
use crate::slog_warn;
|
||||||
|
|
||||||
use super::events::{emit_event, handle_agent_stream_event};
|
use super::events::{emit_event, handle_agent_stream_event};
|
||||||
use super::types::{ChildKillerGuard, PtyResult, composite_key};
|
use super::types::PtyResult;
|
||||||
|
|
||||||
/// Spawn claude agent in a PTY and stream events through the broadcast channel.
|
/// Spawn claude agent in a PTY and stream events through the broadcast channel.
|
||||||
///
|
///
|
||||||
@@ -55,7 +54,6 @@ pub(in crate::agents) async fn run_agent_pty_streaming(
|
|||||||
event_log: &Arc<Mutex<Vec<AgentEvent>>>,
|
event_log: &Arc<Mutex<Vec<AgentEvent>>>,
|
||||||
log_writer: Option<Arc<Mutex<AgentLogWriter>>>,
|
log_writer: Option<Arc<Mutex<AgentLogWriter>>>,
|
||||||
inactivity_timeout_secs: u64,
|
inactivity_timeout_secs: u64,
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
watcher_tx: broadcast::Sender<WatcherEvent>,
|
watcher_tx: broadcast::Sender<WatcherEvent>,
|
||||||
session_id_to_resume: Option<&str>,
|
session_id_to_resume: Option<&str>,
|
||||||
eager_record: Option<(std::path::PathBuf, String)>,
|
eager_record: Option<(std::path::PathBuf, String)>,
|
||||||
@@ -82,7 +80,6 @@ pub(in crate::agents) async fn run_agent_pty_streaming(
|
|||||||
&event_log,
|
&event_log,
|
||||||
log_writer.as_deref(),
|
log_writer.as_deref(),
|
||||||
inactivity_timeout_secs,
|
inactivity_timeout_secs,
|
||||||
&child_killers,
|
|
||||||
&watcher_tx,
|
&watcher_tx,
|
||||||
resume_sid.as_deref(),
|
resume_sid.as_deref(),
|
||||||
eager_record,
|
eager_record,
|
||||||
@@ -104,7 +101,6 @@ fn run_agent_pty_blocking(
|
|||||||
event_log: &Mutex<Vec<AgentEvent>>,
|
event_log: &Mutex<Vec<AgentEvent>>,
|
||||||
log_writer: Option<&Mutex<AgentLogWriter>>,
|
log_writer: Option<&Mutex<AgentLogWriter>>,
|
||||||
inactivity_timeout_secs: u64,
|
inactivity_timeout_secs: u64,
|
||||||
child_killers: &Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
watcher_tx: &broadcast::Sender<WatcherEvent>,
|
watcher_tx: &broadcast::Sender<WatcherEvent>,
|
||||||
session_id_to_resume: Option<&str>,
|
session_id_to_resume: Option<&str>,
|
||||||
eager_record: Option<(std::path::PathBuf, String)>,
|
eager_record: Option<(std::path::PathBuf, String)>,
|
||||||
@@ -204,21 +200,6 @@ fn run_agent_pty_blocking(
|
|||||||
.spawn_command(cmd)
|
.spawn_command(cmd)
|
||||||
.map_err(|e| format!("Failed to spawn agent for {story_id}:{agent_name}: {e}"))?;
|
.map_err(|e| format!("Failed to spawn agent for {story_id}:{agent_name}: {e}"))?;
|
||||||
|
|
||||||
// Register the child killer so that kill_all_children() / stop_agent() can
|
|
||||||
// terminate this process on server shutdown, even if the blocking thread
|
|
||||||
// cannot be interrupted. The ChildKillerGuard deregisters on function exit.
|
|
||||||
let killer_key = composite_key(story_id, agent_name);
|
|
||||||
{
|
|
||||||
let killer = child.clone_killer();
|
|
||||||
if let Ok(mut killers) = child_killers.lock() {
|
|
||||||
killers.insert(killer_key.clone(), killer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let _killer_guard = ChildKillerGuard {
|
|
||||||
killers: Arc::clone(child_killers),
|
|
||||||
key: killer_key,
|
|
||||||
};
|
|
||||||
|
|
||||||
drop(pair.slave);
|
drop(pair.slave);
|
||||||
|
|
||||||
let reader = pair
|
let reader = pair
|
||||||
@@ -366,7 +347,11 @@ fn run_agent_pty_blocking(
|
|||||||
.and_then(|i| i.get("status"))
|
.and_then(|i| i.get("status"))
|
||||||
.and_then(|s| s.as_str())
|
.and_then(|s| s.as_str())
|
||||||
.unwrap_or("");
|
.unwrap_or("");
|
||||||
let is_hard_block = !status.is_empty() && status != "allowed_warning";
|
// "allowed" and "allowed_warning" are soft warnings — the request was
|
||||||
|
// permitted; only statuses that actually block the request (e.g. "rejected")
|
||||||
|
// are genuine hard blocks that warrant a rate-limit exit respawn.
|
||||||
|
let is_hard_block =
|
||||||
|
!status.is_empty() && status != "allowed" && status != "allowed_warning";
|
||||||
let reset_at = rate_limit_info
|
let reset_at = rate_limit_info
|
||||||
.and_then(|i| i.get("reset_at"))
|
.and_then(|i| i.get("reset_at"))
|
||||||
.and_then(|r| r.as_str())
|
.and_then(|r| r.as_str())
|
||||||
|
|||||||
@@ -1,9 +1,4 @@
|
|||||||
//! Core types for the PTY runner: result container and process lifecycle helpers.
|
//! Core types for the PTY runner: result container and process lifecycle helpers.
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
|
|
||||||
use portable_pty::ChildKiller;
|
|
||||||
|
|
||||||
use crate::agents::TokenUsage;
|
use crate::agents::TokenUsage;
|
||||||
|
|
||||||
/// Result from a PTY agent session, containing the session ID and token usage.
|
/// Result from a PTY agent session, containing the session ID and token usage.
|
||||||
@@ -23,20 +18,3 @@ pub(in crate::agents) struct PtyResult {
|
|||||||
/// event was seen or when the `reset_at` field was absent from the event.
|
/// event was seen or when the `reset_at` field was absent from the event.
|
||||||
pub rate_limit_reset_at: Option<chrono::DateTime<chrono::Utc>>,
|
pub rate_limit_reset_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn composite_key(story_id: &str, agent_name: &str) -> String {
|
|
||||||
format!("{story_id}:{agent_name}")
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) struct ChildKillerGuard {
|
|
||||||
pub killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
pub key: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for ChildKillerGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Ok(mut killers) = self.killers.lock() {
|
|
||||||
killers.remove(&self.key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
//! Claude Code runtime — launches Claude Code CLI sessions as agent backends.
|
//! Claude Code runtime — launches Claude Code CLI sessions as agent backends.
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use portable_pty::ChildKiller;
|
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
use crate::agent_log::AgentLogWriter;
|
use crate::agent_log::AgentLogWriter;
|
||||||
@@ -17,20 +15,13 @@ use super::{AgentEvent, AgentRuntime, RuntimeContext, RuntimeResult, RuntimeStat
|
|||||||
/// It wraps the existing PTY-based execution logic, preserving all streaming,
|
/// It wraps the existing PTY-based execution logic, preserving all streaming,
|
||||||
/// token tracking, and inactivity timeout behaviour.
|
/// token tracking, and inactivity timeout behaviour.
|
||||||
pub struct ClaudeCodeRuntime {
|
pub struct ClaudeCodeRuntime {
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
|
||||||
watcher_tx: broadcast::Sender<WatcherEvent>,
|
watcher_tx: broadcast::Sender<WatcherEvent>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ClaudeCodeRuntime {
|
impl ClaudeCodeRuntime {
|
||||||
/// Create a new Claude Code runtime with shared child-killer registry and event channel.
|
/// Create a new Claude Code runtime with a shared event channel.
|
||||||
pub fn new(
|
pub fn new(watcher_tx: broadcast::Sender<WatcherEvent>) -> Self {
|
||||||
child_killers: Arc<Mutex<HashMap<String, Box<dyn ChildKiller + Send + Sync>>>>,
|
Self { watcher_tx }
|
||||||
watcher_tx: broadcast::Sender<WatcherEvent>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
child_killers,
|
|
||||||
watcher_tx,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,7 +48,6 @@ impl AgentRuntime for ClaudeCodeRuntime {
|
|||||||
&event_log,
|
&event_log,
|
||||||
log_writer.clone(),
|
log_writer.clone(),
|
||||||
ctx.inactivity_timeout_secs,
|
ctx.inactivity_timeout_secs,
|
||||||
Arc::clone(&self.child_killers),
|
|
||||||
self.watcher_tx.clone(),
|
self.watcher_tx.clone(),
|
||||||
ctx.session_id_to_resume.as_deref(),
|
ctx.session_id_to_resume.as_deref(),
|
||||||
eager_record.clone(),
|
eager_record.clone(),
|
||||||
@@ -69,6 +59,7 @@ impl AgentRuntime for ClaudeCodeRuntime {
|
|||||||
// Abort+no-session: CLI crashed (e.g. SIGABRT) before emitting its
|
// Abort+no-session: CLI crashed (e.g. SIGABRT) before emitting its
|
||||||
// first "system" event. Detected by: non-zero exit AND no session.
|
// first "system" event. Detected by: non-zero exit AND no session.
|
||||||
aborted_signal: !result.exit_ok && result.session_id.is_none(),
|
aborted_signal: !result.exit_ok && result.session_id.is_none(),
|
||||||
|
exit_ok: result.exit_ok,
|
||||||
session_id: result.session_id,
|
session_id: result.session_id,
|
||||||
token_usage: result.token_usage,
|
token_usage: result.token_usage,
|
||||||
rate_limit_exit: result.rate_limit_exit,
|
rate_limit_exit: result.rate_limit_exit,
|
||||||
@@ -94,7 +85,6 @@ impl AgentRuntime for ClaudeCodeRuntime {
|
|||||||
&event_log,
|
&event_log,
|
||||||
log_writer,
|
log_writer,
|
||||||
ctx.inactivity_timeout_secs,
|
ctx.inactivity_timeout_secs,
|
||||||
Arc::clone(&self.child_killers),
|
|
||||||
self.watcher_tx.clone(),
|
self.watcher_tx.clone(),
|
||||||
None, // no --resume on fallback
|
None, // no --resume on fallback
|
||||||
eager_record,
|
eager_record,
|
||||||
@@ -103,6 +93,7 @@ impl AgentRuntime for ClaudeCodeRuntime {
|
|||||||
Ok(RuntimeResult {
|
Ok(RuntimeResult {
|
||||||
aborted_signal: !fallback_result.exit_ok
|
aborted_signal: !fallback_result.exit_ok
|
||||||
&& fallback_result.session_id.is_none(),
|
&& fallback_result.session_id.is_none(),
|
||||||
|
exit_ok: fallback_result.exit_ok,
|
||||||
session_id: fallback_result.session_id,
|
session_id: fallback_result.session_id,
|
||||||
token_usage: fallback_result.token_usage,
|
token_usage: fallback_result.token_usage,
|
||||||
rate_limit_exit: fallback_result.rate_limit_exit,
|
rate_limit_exit: fallback_result.rate_limit_exit,
|
||||||
@@ -115,7 +106,6 @@ impl AgentRuntime for ClaudeCodeRuntime {
|
|||||||
|
|
||||||
fn stop(&self) {
|
fn stop(&self) {
|
||||||
// Stopping is handled externally by the pool via kill_child_for_key().
|
// Stopping is handled externally by the pool via kill_child_for_key().
|
||||||
// The ChildKillerGuard in pty.rs deregisters automatically on process exit.
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_status(&self) -> RuntimeStatus {
|
fn get_status(&self) -> RuntimeStatus {
|
||||||
|
|||||||
@@ -135,6 +135,7 @@ impl AgentRuntime for GeminiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -151,6 +152,7 @@ impl AgentRuntime for GeminiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -254,6 +256,7 @@ impl AgentRuntime for GeminiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -339,6 +342,7 @@ impl AgentRuntime for GeminiRuntime {
|
|||||||
Ok(RuntimeResult {
|
Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
|
|||||||
@@ -55,6 +55,12 @@ pub struct RuntimeContext {
|
|||||||
pub struct RuntimeResult {
|
pub struct RuntimeResult {
|
||||||
pub session_id: Option<String>,
|
pub session_id: Option<String>,
|
||||||
pub token_usage: Option<TokenUsage>,
|
pub token_usage: Option<TokenUsage>,
|
||||||
|
/// `true` when the process exited with exit code 0; `false` for non-zero exits
|
||||||
|
/// (API errors, network failures, or Claude-API-level budget exhaustion). Always
|
||||||
|
/// `true` for API-based runtimes (OpenAI, Gemini) which have no exit-code concept.
|
||||||
|
/// Used by the commit-recovery path to skip the stuck-respawn counter for forced
|
||||||
|
/// exits (story 1089, AC1).
|
||||||
|
pub exit_ok: bool,
|
||||||
/// `true` when the process exited with a failure AND no session was established.
|
/// `true` when the process exited with a failure AND no session was established.
|
||||||
///
|
///
|
||||||
/// This indicates the Claude Code CLI crashed (e.g. SIGABRT from an assertion
|
/// This indicates the Claude Code CLI crashed (e.g. SIGABRT from an assertion
|
||||||
@@ -169,6 +175,7 @@ mod tests {
|
|||||||
cache_read_input_tokens: 0,
|
cache_read_input_tokens: 0,
|
||||||
total_cost_usd: 0.01,
|
total_cost_usd: 0.01,
|
||||||
}),
|
}),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -186,6 +193,7 @@ mod tests {
|
|||||||
let result = RuntimeResult {
|
let result = RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: None,
|
token_usage: None,
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -204,20 +212,16 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn claude_code_runtime_get_status_returns_idle() {
|
fn claude_code_runtime_get_status_returns_idle() {
|
||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use std::collections::HashMap;
|
|
||||||
let killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
let (watcher_tx, _) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, _) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let runtime = ClaudeCodeRuntime::new(killers, watcher_tx);
|
let runtime = ClaudeCodeRuntime::new(watcher_tx);
|
||||||
assert_eq!(runtime.get_status(), RuntimeStatus::Idle);
|
assert_eq!(runtime.get_status(), RuntimeStatus::Idle);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn claude_code_runtime_stream_events_empty() {
|
fn claude_code_runtime_stream_events_empty() {
|
||||||
use crate::io::watcher::WatcherEvent;
|
use crate::io::watcher::WatcherEvent;
|
||||||
use std::collections::HashMap;
|
|
||||||
let killers = Arc::new(Mutex::new(HashMap::new()));
|
|
||||||
let (watcher_tx, _) = broadcast::channel::<WatcherEvent>(16);
|
let (watcher_tx, _) = broadcast::channel::<WatcherEvent>(16);
|
||||||
let runtime = ClaudeCodeRuntime::new(killers, watcher_tx);
|
let runtime = ClaudeCodeRuntime::new(watcher_tx);
|
||||||
assert!(runtime.stream_events().is_empty());
|
assert!(runtime.stream_events().is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ impl AgentRuntime for OpenAiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -138,6 +139,7 @@ impl AgentRuntime for OpenAiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
@@ -224,6 +226,7 @@ impl AgentRuntime for OpenAiRuntime {
|
|||||||
return Ok(RuntimeResult {
|
return Ok(RuntimeResult {
|
||||||
session_id: None,
|
session_id: None,
|
||||||
token_usage: Some(total_usage),
|
token_usage: Some(total_usage),
|
||||||
|
exit_ok: true,
|
||||||
aborted_signal: false,
|
aborted_signal: false,
|
||||||
rate_limit_exit: false,
|
rate_limit_exit: false,
|
||||||
rate_limit_reset_at: None,
|
rate_limit_reset_at: None,
|
||||||
|
|||||||
@@ -0,0 +1,188 @@
|
|||||||
|
//! Handler for the `convert` chat command (story 1141).
|
||||||
|
//!
|
||||||
|
//! `convert <number> <type>` changes the item-type register of a work item
|
||||||
|
//! in place. All other CRDT registers (ACs, epic, name, stage, …) are
|
||||||
|
//! untouched. Rejected for archived items.
|
||||||
|
|
||||||
|
use super::CommandContext;
|
||||||
|
|
||||||
|
/// Handle the `convert` command.
|
||||||
|
///
|
||||||
|
/// Parses `<number> <type>` from `ctx.args` and delegates to
|
||||||
|
/// [`convert_by_number`]. Returns `None` (route to LLM) when args do not
|
||||||
|
/// look like a numeric ID followed by a type keyword.
|
||||||
|
pub(super) fn handle_convert(ctx: &CommandContext) -> Option<String> {
|
||||||
|
let args = ctx.args.trim();
|
||||||
|
let (num_str, type_str) = args.split_once(char::is_whitespace)?;
|
||||||
|
let num_str = num_str.trim();
|
||||||
|
let type_str = type_str.trim();
|
||||||
|
|
||||||
|
// Route to LLM if the first token is not a bare number.
|
||||||
|
if num_str.is_empty() || !num_str.chars().all(|c| c.is_ascii_digit()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// Route to LLM if the type looks like natural language (contains spaces).
|
||||||
|
if type_str.is_empty() || type_str.contains(char::is_whitespace) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(convert_by_number(ctx.effective_root(), num_str, type_str))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Core convert logic: find item by numeric prefix and change its type.
|
||||||
|
///
|
||||||
|
/// Returns a Markdown-formatted response suitable for all chat transports.
|
||||||
|
pub(crate) fn convert_by_number(
|
||||||
|
project_root: &std::path::Path,
|
||||||
|
story_number: &str,
|
||||||
|
new_type_str: &str,
|
||||||
|
) -> String {
|
||||||
|
let Some(new_type) = crate::io::story_metadata::ItemType::from_str(new_type_str) else {
|
||||||
|
return format!(
|
||||||
|
"Unknown type **{new_type_str}**. Accepted types: story, bug, spike, refactor, epic."
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
let (story_id, _, _, _) =
|
||||||
|
match crate::chat::lookup::find_story_by_number(project_root, story_number) {
|
||||||
|
Some(found) => found,
|
||||||
|
None => {
|
||||||
|
return format!(
|
||||||
|
"No story, bug, spike, or refactor with number **{story_number}** found."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let item = match crate::crdt_state::read_item(&story_id) {
|
||||||
|
Some(i) => i,
|
||||||
|
None => {
|
||||||
|
return format!("Work item **{story_number}** ({story_id}) not found in CRDT.");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if matches!(item.stage(), crate::pipeline_state::Stage::Archived { .. }) {
|
||||||
|
return format!(
|
||||||
|
"Cannot convert **{story_id}**: type change on an archived item is not allowed."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let old_type = item.item_type().map(|t| t.as_str()).unwrap_or("(inferred)");
|
||||||
|
let story_name = item.name().to_string();
|
||||||
|
let new_type_s = new_type.as_str();
|
||||||
|
|
||||||
|
if !crate::crdt_state::set_item_type(&story_id, Some(new_type)) {
|
||||||
|
return format!("Failed to convert **{story_id}**: CRDT write rejected.");
|
||||||
|
}
|
||||||
|
|
||||||
|
format!("Converted **{story_name}** ({story_id}) from type `{old_type}` to `{new_type_s}`.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::super::{CommandDispatch, try_handle_command};
|
||||||
|
|
||||||
|
fn convert_cmd(root: &std::path::Path, args: &str) -> Option<String> {
|
||||||
|
let services = crate::services::Services::new_test(root.to_path_buf(), "Timmy".to_string());
|
||||||
|
let room_id = "!test:example.com".to_string();
|
||||||
|
let dispatch = CommandDispatch {
|
||||||
|
services: &services,
|
||||||
|
project_root: &services.project_root,
|
||||||
|
bot_user_id: "@timmy:homeserver.local",
|
||||||
|
room_id: &room_id,
|
||||||
|
};
|
||||||
|
try_handle_command(&dispatch, &format!("@timmy convert {args}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_command_is_registered() {
|
||||||
|
use super::super::commands;
|
||||||
|
assert!(
|
||||||
|
commands().iter().any(|c| c.name == "convert"),
|
||||||
|
"convert command must be in the registry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_no_args_routes_to_llm() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
let result = convert_cmd(tmp.path(), "");
|
||||||
|
assert!(result.is_none(), "no args should route to LLM: {result:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_natural_language_routes_to_llm() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
let result = convert_cmd(tmp.path(), "the login bug to a story");
|
||||||
|
assert!(
|
||||||
|
result.is_none(),
|
||||||
|
"natural-language args should route to LLM: {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_well_formed_runs_handler() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
let result = convert_cmd(tmp.path(), "999 story");
|
||||||
|
assert!(
|
||||||
|
result.is_some(),
|
||||||
|
"well-formed args should run the handler: {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_invalid_type_returns_error() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
let result = convert_cmd(tmp.path(), "999 banana").unwrap();
|
||||||
|
assert!(
|
||||||
|
result.contains("Unknown type") || result.contains("banana"),
|
||||||
|
"unknown type should show error: {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_not_found_returns_error() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
let result = convert_cmd(tmp.path(), "9988 story").unwrap();
|
||||||
|
assert!(
|
||||||
|
result.contains("9988") && result.contains("found"),
|
||||||
|
"not-found message should include number and 'found': {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn convert_changes_item_type_in_crdt() {
|
||||||
|
let tmp = tempfile::TempDir::new().unwrap();
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
crate::db::ensure_content_store();
|
||||||
|
crate::chat::test_helpers::write_story_file(
|
||||||
|
tmp.path(),
|
||||||
|
"backlog",
|
||||||
|
"9120_spike_convert_chat.md",
|
||||||
|
"# Spike\n",
|
||||||
|
Some("Convert Chat Test"),
|
||||||
|
);
|
||||||
|
crate::crdt_state::set_item_type(
|
||||||
|
"9120_spike_convert_chat",
|
||||||
|
Some(crate::io::story_metadata::ItemType::Spike),
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = convert_cmd(tmp.path(), "9120 story").unwrap();
|
||||||
|
assert!(
|
||||||
|
result.contains("story") || result.contains("Converted"),
|
||||||
|
"should confirm conversion: {result}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let item =
|
||||||
|
crate::crdt_state::read_item("9120_spike_convert_chat").expect("item should exist");
|
||||||
|
assert_eq!(
|
||||||
|
item.item_type(),
|
||||||
|
Some(crate::io::story_metadata::ItemType::Story),
|
||||||
|
"item_type should be Story after conversion: {:?}",
|
||||||
|
item.item_type()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ mod ambient;
|
|||||||
mod assign;
|
mod assign;
|
||||||
mod backlog;
|
mod backlog;
|
||||||
mod cleanup_worktrees;
|
mod cleanup_worktrees;
|
||||||
|
mod convert;
|
||||||
mod cost;
|
mod cost;
|
||||||
mod coverage;
|
mod coverage;
|
||||||
mod depends;
|
mod depends;
|
||||||
@@ -19,6 +20,7 @@ mod help;
|
|||||||
pub(crate) mod loc;
|
pub(crate) mod loc;
|
||||||
mod logs;
|
mod logs;
|
||||||
mod move_story;
|
mod move_story;
|
||||||
|
mod new_project;
|
||||||
mod overview;
|
mod overview;
|
||||||
mod run_tests;
|
mod run_tests;
|
||||||
mod setup;
|
mod setup;
|
||||||
@@ -232,6 +234,11 @@ pub fn commands() -> &'static [BotCommand] {
|
|||||||
description: "Schedule a deferred agent start: `timer <story_id> <HH:MM>`, `timer list`, `timer cancel <story_id>`",
|
description: "Schedule a deferred agent start: `timer <story_id> <HH:MM>`, `timer list`, `timer cancel <story_id>`",
|
||||||
handler: timer::handle_timer,
|
handler: timer::handle_timer,
|
||||||
},
|
},
|
||||||
|
BotCommand {
|
||||||
|
name: "convert",
|
||||||
|
description: "Convert a work item's type: `convert <number> <type>` (types: story, bug, spike, refactor, epic)",
|
||||||
|
handler: convert::handle_convert,
|
||||||
|
},
|
||||||
BotCommand {
|
BotCommand {
|
||||||
name: "unblock",
|
name: "unblock",
|
||||||
description: "Reset a blocked story: `unblock <number>` (clears blocked flag and resets retry count)",
|
description: "Reset a blocked story: `unblock <number>` (clears blocked flag and resets retry count)",
|
||||||
@@ -262,6 +269,21 @@ pub fn commands() -> &'static [BotCommand] {
|
|||||||
description: "List orphaned worktrees (dry run), or `cleanup_worktrees --confirm` to remove them",
|
description: "List orphaned worktrees (dry run), or `cleanup_worktrees --confirm` to remove them",
|
||||||
handler: handle_cleanup_worktrees_fallback,
|
handler: handle_cleanup_worktrees_fallback,
|
||||||
},
|
},
|
||||||
|
BotCommand {
|
||||||
|
name: "health",
|
||||||
|
description: "Show subsystem health: gateway, sled, matrix-sync, creds, and build-hash",
|
||||||
|
handler: handle_health_fallback,
|
||||||
|
},
|
||||||
|
BotCommand {
|
||||||
|
name: "new",
|
||||||
|
description: "Bootstrap a new project container (gateway only): `new project <name>`",
|
||||||
|
handler: new_project::handle_new_project_fallback,
|
||||||
|
},
|
||||||
|
BotCommand {
|
||||||
|
name: "project-rebuild",
|
||||||
|
description: "Rebuild a project's Docker image and swap the container (gateway only): `project-rebuild <name> [--timeout <secs>] [--force]`",
|
||||||
|
handler: handle_project_rebuild_fallback,
|
||||||
|
},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -419,6 +441,26 @@ fn handle_cleanup_worktrees_fallback(_ctx: &CommandContext) -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fallback handler for the `project-rebuild` command when it is not intercepted
|
||||||
|
/// by the async gateway handler in `on_room_message`. In practice this is never
|
||||||
|
/// called — `project-rebuild` is detected and handled before `try_handle_command`
|
||||||
|
/// runs in gateway mode. The entry exists in the registry so `help` lists it.
|
||||||
|
///
|
||||||
|
/// Returns `None` to prevent the LLM from receiving the raw command text.
|
||||||
|
fn handle_project_rebuild_fallback(_ctx: &CommandContext) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fallback handler for the `health` command when it is not intercepted by the
|
||||||
|
/// async handler in `on_room_message`. In practice this is never called — health
|
||||||
|
/// is detected and handled before `try_handle_command` is invoked. The entry
|
||||||
|
/// exists in the registry only so `help` lists it.
|
||||||
|
///
|
||||||
|
/// Returns `None` to prevent the LLM from receiving "health" as a prompt.
|
||||||
|
fn handle_health_fallback(_ctx: &CommandContext) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Tests
|
// Tests
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
//! `new project` command stub.
|
||||||
|
//!
|
||||||
|
//! The command is handled asynchronously in the Matrix transport's
|
||||||
|
//! `on_room_message` handler (gateway mode only). This file exists so that
|
||||||
|
//! `help` lists the command and the gateway proxy block does not forward it
|
||||||
|
//! to the active project sled.
|
||||||
|
|
||||||
|
use super::CommandContext;
|
||||||
|
|
||||||
|
/// Fallback handler for the `new` command when it is not intercepted by the
|
||||||
|
/// async gateway handler in `on_room_message`. In practice this is never
|
||||||
|
/// called — `new project` is detected and handled before `try_handle_command`
|
||||||
|
/// runs in gateway mode, and in standalone mode there is no matching project
|
||||||
|
/// bootstrap context.
|
||||||
|
///
|
||||||
|
/// Returns `None` to prevent the LLM from receiving the raw command text.
|
||||||
|
pub fn handle_new_project_fallback(_ctx: &CommandContext) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
@@ -0,0 +1,367 @@
|
|||||||
|
//! Protocol-agnostic chat dispatcher — coalesce window + per-session serial lock.
|
||||||
|
//!
|
||||||
|
//! Sits between every inbound transport (Matrix, Slack, WhatsApp, …) and the
|
||||||
|
//! `claude -p` spawner. Transport handlers call [`ChatDispatcher::submit`]
|
||||||
|
//! instead of spawning directly; the dispatcher enforces two invariants:
|
||||||
|
//!
|
||||||
|
//! 1. **Coalesce window**: messages arriving for the same session within
|
||||||
|
//! `coalesce_ms` of each other are concatenated and delivered to a single
|
||||||
|
//! spawn. The window is a *debounce*: each new message extends the window by
|
||||||
|
//! `coalesce_ms` from its arrival time, so bursts flush as one batch.
|
||||||
|
//!
|
||||||
|
//! 2. **Per-session serial lock**: while one `claude -p` run is active, further
|
||||||
|
//! messages for that session queue up and are dispatched as a single batch
|
||||||
|
//! once the running invocation completes.
|
||||||
|
//!
|
||||||
|
//! A [`ChatDispatcher::stop`] call cancels the active run for a session and
|
||||||
|
//! discards the pending queue.
|
||||||
|
|
||||||
|
use crate::slog;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::sync::{mpsc, watch};
|
||||||
|
|
||||||
|
/// A factory function that produces one LLM execution future per dispatch.
|
||||||
|
///
|
||||||
|
/// Arguments:
|
||||||
|
/// - `String` — the (possibly concatenated) prompt to send to `claude -p`.
|
||||||
|
/// - `watch::Receiver<bool>` — send `true` on this channel to cancel the run.
|
||||||
|
///
|
||||||
|
/// Returns a boxed, pinned `Send + 'static` future that resolves when the LLM
|
||||||
|
/// session ends (whether normally or via cancellation).
|
||||||
|
pub type SpawnFn = Arc<
|
||||||
|
dyn Fn(
|
||||||
|
String,
|
||||||
|
watch::Receiver<bool>,
|
||||||
|
) -> Pin<Box<dyn std::future::Future<Output = ()> + Send + 'static>>
|
||||||
|
+ Send
|
||||||
|
+ Sync,
|
||||||
|
>;
|
||||||
|
|
||||||
|
enum SessionMsg {
|
||||||
|
UserMessage { text: String, factory: SpawnFn },
|
||||||
|
Stop,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SessionHandle {
|
||||||
|
tx: mpsc::UnboundedSender<SessionMsg>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Coalescing, serialising dispatcher for chat-to-LLM message routing.
|
||||||
|
///
|
||||||
|
/// Construct once at startup via [`ChatDispatcher::new`] and share via `Arc`.
|
||||||
|
/// Call [`submit`](ChatDispatcher::submit) from every transport handler instead
|
||||||
|
/// of spawning `claude -p` directly.
|
||||||
|
pub struct ChatDispatcher {
|
||||||
|
sessions: Mutex<HashMap<String, SessionHandle>>,
|
||||||
|
coalesce_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatDispatcher {
|
||||||
|
/// Create a new dispatcher with the given coalesce window in milliseconds.
|
||||||
|
pub fn new(coalesce_ms: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
sessions: Mutex::new(HashMap::new()),
|
||||||
|
coalesce_ms,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Submit a message for a chat session.
|
||||||
|
///
|
||||||
|
/// If no session task exists for `session_key`, one is created lazily.
|
||||||
|
/// The `factory` is called by the session task when the coalesce window
|
||||||
|
/// closes (or immediately after the current run finishes, for pending
|
||||||
|
/// messages).
|
||||||
|
pub fn submit(&self, session_key: String, message: String, factory: SpawnFn) {
|
||||||
|
let mut guard = self.sessions.lock().unwrap();
|
||||||
|
let coalesce_ms = self.coalesce_ms;
|
||||||
|
let handle = guard.entry(session_key.clone()).or_insert_with(|| {
|
||||||
|
let (tx, rx) = mpsc::unbounded_channel();
|
||||||
|
tokio::spawn(session_task(session_key.clone(), rx, coalesce_ms));
|
||||||
|
SessionHandle { tx }
|
||||||
|
});
|
||||||
|
let _ = handle.tx.send(SessionMsg::UserMessage {
|
||||||
|
text: message,
|
||||||
|
factory,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop the active LLM run for `session_key` and clear its pending queue.
|
||||||
|
///
|
||||||
|
/// Returns `true` if the session existed (whether or not anything was
|
||||||
|
/// actually running), `false` if no session for that key has been created.
|
||||||
|
pub fn stop(&self, session_key: &str) -> bool {
|
||||||
|
let guard = self.sessions.lock().unwrap();
|
||||||
|
if let Some(handle) = guard.get(session_key) {
|
||||||
|
let _ = handle.tx.send(SessionMsg::Stop);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-session background task.
|
||||||
|
///
|
||||||
|
/// Phases:
|
||||||
|
/// 1. **Wait** — blocks until the first `UserMessage` arrives.
|
||||||
|
/// 2. **Coalesce** — extends the window by `coalesce_ms` on each new message;
|
||||||
|
/// fires when no message arrives within the window.
|
||||||
|
/// 3. **Run** — calls the factory with the concatenated batch; while running,
|
||||||
|
/// collects further `UserMessage`s into a pending list and logs a warn per
|
||||||
|
/// message. A `Stop` message cancels the running call and clears pending.
|
||||||
|
/// 4. **Drain** — after the run, if pending is non-empty, fires a second run
|
||||||
|
/// with the accumulated batch and loops back to step 3.
|
||||||
|
/// 5. Returns to step 1 when pending is empty.
|
||||||
|
async fn session_task(
|
||||||
|
session_key: String,
|
||||||
|
mut rx: mpsc::UnboundedReceiver<SessionMsg>,
|
||||||
|
coalesce_ms: u64,
|
||||||
|
) {
|
||||||
|
let coalesce_dur = Duration::from_millis(coalesce_ms);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
// ── Phase 1: wait for the first message ─────────────────────────────
|
||||||
|
let (first_text, first_factory) = loop {
|
||||||
|
match rx.recv().await {
|
||||||
|
None => return,
|
||||||
|
Some(SessionMsg::Stop) => continue,
|
||||||
|
Some(SessionMsg::UserMessage { text, factory }) => break (text, factory),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── Phase 2: coalesce window (debounce) ──────────────────────────────
|
||||||
|
let mut batch: Vec<String> = vec![first_text];
|
||||||
|
let mut latest_factory: SpawnFn = first_factory;
|
||||||
|
let mut deadline = tokio::time::Instant::now() + coalesce_dur;
|
||||||
|
|
||||||
|
'coalesce: loop {
|
||||||
|
let now = tokio::time::Instant::now();
|
||||||
|
if now >= deadline {
|
||||||
|
break 'coalesce;
|
||||||
|
}
|
||||||
|
let remaining = deadline - now;
|
||||||
|
match tokio::time::timeout(remaining, rx.recv()).await {
|
||||||
|
Err(_) => break 'coalesce, // window closed
|
||||||
|
Ok(None) => return, // channel closed → exit task
|
||||||
|
Ok(Some(SessionMsg::Stop)) => {
|
||||||
|
batch.clear();
|
||||||
|
break 'coalesce;
|
||||||
|
}
|
||||||
|
Ok(Some(SessionMsg::UserMessage { text, factory })) => {
|
||||||
|
batch.push(text);
|
||||||
|
latest_factory = factory;
|
||||||
|
// Extend deadline on each new message (debounce).
|
||||||
|
deadline = tokio::time::Instant::now() + coalesce_dur;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if batch.is_empty() {
|
||||||
|
continue; // Stop received during coalesce — restart
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Phase 3 + 4: run → drain pending → repeat ───────────────────────
|
||||||
|
let mut prompt = batch.join("\n\n");
|
||||||
|
let mut factory = latest_factory;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let (cancel_tx, cancel_rx) = watch::channel(false);
|
||||||
|
let llm_fut = factory(prompt, cancel_rx);
|
||||||
|
let mut llm_task = tokio::spawn(llm_fut);
|
||||||
|
|
||||||
|
let mut pending_texts: Vec<String> = vec![];
|
||||||
|
let mut pending_factory: Option<SpawnFn> = None;
|
||||||
|
let mut stopped = false;
|
||||||
|
|
||||||
|
// Wait for the LLM to finish, collecting messages that arrive during the run.
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
_ = &mut llm_task => { break; }
|
||||||
|
msg = rx.recv() => {
|
||||||
|
match msg {
|
||||||
|
None => {
|
||||||
|
llm_task.abort();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Some(SessionMsg::Stop) => {
|
||||||
|
let _ = cancel_tx.send(true);
|
||||||
|
let _ = llm_task.await;
|
||||||
|
pending_texts.clear();
|
||||||
|
stopped = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Some(SessionMsg::UserMessage { text, factory: f }) => {
|
||||||
|
pending_texts.push(text);
|
||||||
|
let depth = pending_texts.len();
|
||||||
|
slog!(
|
||||||
|
"[chat-dispatcher] coalescing message for session={}, queue_depth={}",
|
||||||
|
session_key,
|
||||||
|
depth,
|
||||||
|
);
|
||||||
|
pending_factory = Some(f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if stopped || pending_texts.is_empty() {
|
||||||
|
break; // back to Phase 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fire the pending batch as the next run (no additional coalesce window).
|
||||||
|
prompt = pending_texts.join("\n\n");
|
||||||
|
factory = pending_factory.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
fn make_factory(spawn_count: Arc<AtomicUsize>, run_ms: u64) -> SpawnFn {
|
||||||
|
Arc::new(move |_prompt: String, _cancel_rx: watch::Receiver<bool>| {
|
||||||
|
let count = Arc::clone(&spawn_count);
|
||||||
|
Box::pin(async move {
|
||||||
|
count.fetch_add(1, Ordering::SeqCst);
|
||||||
|
tokio::time::sleep(Duration::from_millis(run_ms)).await;
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// AC 6 regression: three messages arriving 200 ms / (long gap) / (after run)
|
||||||
|
/// apart on the same session must produce at most two spawns, never three
|
||||||
|
/// concurrent processes.
|
||||||
|
///
|
||||||
|
/// Setup:
|
||||||
|
/// coalesce_ms = 50 ms (short window so test runs fast)
|
||||||
|
/// LLM "run" = 150 ms
|
||||||
|
/// msg1 @ t=0
|
||||||
|
/// msg2 @ t=20 ms — within coalesce window, merged with msg1 → 1 spawn
|
||||||
|
/// msg3 @ t=300 ms — after run completes → 2nd spawn
|
||||||
|
///
|
||||||
|
/// Expected: exactly 2 spawns, never 3.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn three_messages_never_three_concurrent_spawns() {
|
||||||
|
let spawn_count = Arc::new(AtomicUsize::new(0));
|
||||||
|
let dispatcher = Arc::new(ChatDispatcher::new(50));
|
||||||
|
let session = "room1".to_string();
|
||||||
|
|
||||||
|
// msg1 at t=0
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"msg1".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 150),
|
||||||
|
);
|
||||||
|
|
||||||
|
// msg2 at t=20 ms — inside the 50 ms coalesce window
|
||||||
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"msg2".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 150),
|
||||||
|
);
|
||||||
|
|
||||||
|
// msg3 at t=300 ms — after the coalesce window fires (t≈70 ms) and the
|
||||||
|
// 150 ms run completes (t≈220 ms), so msg3 starts a second coalesce cycle.
|
||||||
|
tokio::time::sleep(Duration::from_millis(280)).await;
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"msg3".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 150),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait long enough for both runs to finish.
|
||||||
|
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||||
|
|
||||||
|
let count = spawn_count.load(Ordering::SeqCst);
|
||||||
|
assert!(
|
||||||
|
(1..=2).contains(&count),
|
||||||
|
"expected 1 or 2 spawns (msgs 1+2 coalesced, msg3 separate), got {count}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Messages that arrive while the LLM is running are not lost — they are
|
||||||
|
/// delivered as a single follow-up spawn once the first run completes.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pending_messages_dispatched_after_run_completes() {
|
||||||
|
let spawn_count = Arc::new(AtomicUsize::new(0));
|
||||||
|
let dispatcher = Arc::new(ChatDispatcher::new(50));
|
||||||
|
let session = "room2".to_string();
|
||||||
|
|
||||||
|
// First message — starts a 200 ms run.
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"first".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 200),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait for coalesce window to fire, then send two more.
|
||||||
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"second".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 50),
|
||||||
|
);
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"third".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 50),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait long enough for both runs.
|
||||||
|
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||||
|
|
||||||
|
let count = spawn_count.load(Ordering::SeqCst);
|
||||||
|
assert_eq!(
|
||||||
|
count, 2,
|
||||||
|
"first run + one pending-batch run = 2 total spawns"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop cancels the running LLM and discards pending messages.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stop_cancels_run_and_clears_pending() {
|
||||||
|
let spawn_count = Arc::new(AtomicUsize::new(0));
|
||||||
|
let dispatcher = Arc::new(ChatDispatcher::new(30));
|
||||||
|
let session = "room3".to_string();
|
||||||
|
|
||||||
|
// Start a long run.
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"long-running".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 500),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait for coalesce window to fire.
|
||||||
|
tokio::time::sleep(Duration::from_millis(80)).await;
|
||||||
|
|
||||||
|
// Queue a pending message.
|
||||||
|
dispatcher.submit(
|
||||||
|
session.clone(),
|
||||||
|
"pending".to_string(),
|
||||||
|
make_factory(Arc::clone(&spawn_count), 50),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Stop immediately.
|
||||||
|
dispatcher.stop(&session);
|
||||||
|
|
||||||
|
// Wait longer than the run would have taken if not stopped.
|
||||||
|
tokio::time::sleep(Duration::from_millis(700)).await;
|
||||||
|
|
||||||
|
let count = spawn_count.load(Ordering::SeqCst);
|
||||||
|
// The first run was started before stop (spawn_count=1).
|
||||||
|
// The pending message should NOT have produced a second spawn.
|
||||||
|
assert!(
|
||||||
|
count <= 1,
|
||||||
|
"stop should discard pending; got {count} spawns"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,8 @@
|
|||||||
|
|
||||||
/// Bot command registry and dispatch — parses and routes incoming chat messages.
|
/// Bot command registry and dispatch — parses and routes incoming chat messages.
|
||||||
pub mod commands;
|
pub mod commands;
|
||||||
|
/// Protocol-agnostic chat dispatcher — coalesce window and per-session serial lock.
|
||||||
|
pub mod dispatcher;
|
||||||
/// Chat history utilities — loading and serialising conversation history.
|
/// Chat history utilities — loading and serialising conversation history.
|
||||||
pub mod history;
|
pub mod history;
|
||||||
pub(crate) mod lookup;
|
pub(crate) mod lookup;
|
||||||
|
|||||||
@@ -300,6 +300,20 @@ pub(super) async fn handle_incoming_message(
|
|||||||
handle_llm_message(ctx, channel, user, message).await;
|
handle_llm_message(ctx, channel, user, message).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build the prompt for a Discord LLM turn, prepending any pending
|
||||||
|
/// CRDT pipeline-transition events as a `<system-reminder>` block.
|
||||||
|
fn build_discord_llm_prompt(
|
||||||
|
persona: &str,
|
||||||
|
bot_name: &str,
|
||||||
|
user: &str,
|
||||||
|
user_message: &str,
|
||||||
|
) -> String {
|
||||||
|
let event_ctx = crate::llm_session::assemble_prompt_context(persona);
|
||||||
|
format!(
|
||||||
|
"{event_ctx}[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{user}: {user_message}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Forward a message to Claude Code and send the response back via Discord.
|
/// Forward a message to Claude Code and send the response back via Discord.
|
||||||
async fn handle_llm_message(ctx: &DiscordContext, channel: &str, user: &str, user_message: &str) {
|
async fn handle_llm_message(ctx: &DiscordContext, channel: &str, user: &str, user_message: &str) {
|
||||||
use crate::chat::util::drain_complete_paragraphs;
|
use crate::chat::util::drain_complete_paragraphs;
|
||||||
@@ -314,9 +328,8 @@ async fn handle_llm_message(ctx: &DiscordContext, channel: &str, user: &str, use
|
|||||||
};
|
};
|
||||||
|
|
||||||
let bot_name = &ctx.services.bot_name;
|
let bot_name = &ctx.services.bot_name;
|
||||||
let prompt = format!(
|
let persona = bot_name.to_lowercase();
|
||||||
"[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{user}: {user_message}"
|
let prompt = build_discord_llm_prompt(&persona, bot_name, user, user_message);
|
||||||
);
|
|
||||||
|
|
||||||
let provider = ClaudeCodeProvider::new();
|
let provider = ClaudeCodeProvider::new();
|
||||||
let (_cancel_tx, mut cancel_rx) = watch::channel(false);
|
let (_cancel_tx, mut cancel_rx) = watch::channel(false);
|
||||||
@@ -604,4 +617,40 @@ mod tests {
|
|||||||
assert!(conv.session_id.is_none(), "session_id should be cleared");
|
assert!(conv.session_id.is_none(), "session_id should be cleared");
|
||||||
assert!(conv.entries.is_empty(), "entries should be cleared");
|
assert!(conv.entries.is_empty(), "entries should be cleared");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// AC 4: fire a `TransitionFired` event, simulate a Discord user turn, and
|
||||||
|
/// assert the assembled prompt contains the event (end-to-end non-Matrix test).
|
||||||
|
#[test]
|
||||||
|
fn discord_prompt_includes_transition_event() {
|
||||||
|
use crate::pipeline_state::{PipelineEvent, PlanState, Stage, StoryId, TransitionFired};
|
||||||
|
crate::crdt_state::init_for_test();
|
||||||
|
|
||||||
|
crate::event_log::log_transition_event(&TransitionFired {
|
||||||
|
story_id: StoryId("77_discord_test".to_string()),
|
||||||
|
before: Stage::Backlog,
|
||||||
|
after: Stage::Coding {
|
||||||
|
claim: None,
|
||||||
|
plan: PlanState::Missing,
|
||||||
|
retries: 0,
|
||||||
|
},
|
||||||
|
event: PipelineEvent::DepsMet,
|
||||||
|
at: chrono::Utc::now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
let prompt =
|
||||||
|
build_discord_llm_prompt("discord-ch-test", "Timmy", "@alice", "what is the status?");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
prompt.contains("<system-reminder>"),
|
||||||
|
"assembled prompt must include system-reminder block; got: {prompt}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
prompt.contains("77_discord_test"),
|
||||||
|
"assembled prompt must contain story id; got: {prompt}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
prompt.contains("what is the status?"),
|
||||||
|
"assembled prompt must contain user message; got: {prompt}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
//! Matrix bot context — shared state for the Matrix bot (rooms, history, permissions).
|
//! Matrix bot context — shared state for the Matrix bot (rooms, history, permissions).
|
||||||
use crate::chat::ChatTransport;
|
use crate::chat::ChatTransport;
|
||||||
|
use crate::service::gateway::config::ProjectEntry;
|
||||||
use crate::service::timer::TimerStore;
|
use crate::service::timer::TimerStore;
|
||||||
use crate::services::Services;
|
use crate::services::Services;
|
||||||
use matrix_sdk::ruma::{OwnedEventId, OwnedRoomId, OwnedUserId};
|
use matrix_sdk::ruma::{OwnedEventId, OwnedRoomId, OwnedUserId};
|
||||||
use std::collections::{BTreeMap, HashSet, VecDeque};
|
use std::collections::{BTreeMap, HashSet, VecDeque};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicI64;
|
||||||
use tokio::sync::Mutex as TokioMutex;
|
use tokio::sync::Mutex as TokioMutex;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
@@ -87,33 +89,26 @@ pub struct BotContext {
|
|||||||
/// In gateway mode: the currently active project (shared with the gateway HTTP handler).
|
/// In gateway mode: the currently active project (shared with the gateway HTTP handler).
|
||||||
/// `None` in standalone single-project mode.
|
/// `None` in standalone single-project mode.
|
||||||
pub gateway_active_project: Option<Arc<RwLock<String>>>,
|
pub gateway_active_project: Option<Arc<RwLock<String>>>,
|
||||||
/// In gateway mode: valid project names accepted by the `switch` command.
|
/// In gateway mode: shared live projects map from [`GatewayState`].
|
||||||
/// Empty in standalone mode.
|
|
||||||
pub gateway_projects: Vec<String>,
|
|
||||||
/// In gateway mode: mapping of project name → base URL (e.g. `"http://localhost:3001"`).
|
|
||||||
/// Used to proxy bot commands to the active project over WebSocket (`/ws`).
|
|
||||||
/// Empty in standalone mode.
|
|
||||||
pub gateway_project_urls: BTreeMap<String, String>,
|
|
||||||
/// Pipeline transition events buffered since the last LLM turn.
|
|
||||||
///
|
///
|
||||||
/// A background task appends one compact audit line per real stage
|
/// The `new project` command writes here so HTTP handlers see the new entry
|
||||||
/// transition. `handle_message` drains this buffer and injects it as a
|
/// immediately without requiring a gateway restart. `None` in standalone mode.
|
||||||
/// `<system-reminder>` block at the head of the next user prompt so Timmy
|
pub gateway_projects_store: Option<Arc<RwLock<BTreeMap<String, ProjectEntry>>>>,
|
||||||
/// sees pipeline activity without requiring a separate message.
|
|
||||||
pub pending_pipeline_events: Arc<TokioMutex<Vec<String>>>,
|
|
||||||
/// Gateway aggregate transition events buffered since the last LLM turn.
|
|
||||||
///
|
|
||||||
/// In gateway mode a background task appends one compact audit line per
|
|
||||||
/// `GatewayStatusEvent` received from the gateway broadcaster. Drained
|
|
||||||
/// alongside `pending_pipeline_events` on each user message. Always
|
|
||||||
/// empty in standalone (non-gateway) mode.
|
|
||||||
pub pending_gateway_events: Arc<TokioMutex<Vec<String>>>,
|
|
||||||
/// Bounded FIFO set of already-handled incoming event IDs.
|
/// Bounded FIFO set of already-handled incoming event IDs.
|
||||||
///
|
///
|
||||||
/// The Matrix sync loop can replay events on reconnect. This set ensures
|
/// The Matrix sync loop can replay events on reconnect. This set ensures
|
||||||
/// each event is processed at most once. Insert the event ID before any
|
/// each event is processed at most once. Insert the event ID before any
|
||||||
/// side-effecting work; return early if the insert returns `false`.
|
/// side-effecting work; return early if the insert returns `false`.
|
||||||
pub handled_incoming_event_ids: Arc<TokioMutex<SeenEventIds>>,
|
pub handled_incoming_event_ids: Arc<TokioMutex<SeenEventIds>>,
|
||||||
|
/// In gateway mode: the port the gateway is listening on.
|
||||||
|
///
|
||||||
|
/// Used by the "rebuild gateway" command to construct the health-check URL
|
||||||
|
/// passed to the trampoline. `None` in standalone single-project mode.
|
||||||
|
pub gateway_port: Option<u16>,
|
||||||
|
/// Timestamp (ms since Unix epoch) of the last Matrix event received in any
|
||||||
|
/// configured room. Updated atomically on every `on_room_message` call so
|
||||||
|
/// the `health` command can detect a stale or dead sync loop.
|
||||||
|
pub last_matrix_event_ms: Arc<AtomicI64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BotContext {
|
impl BotContext {
|
||||||
@@ -141,7 +136,12 @@ impl BotContext {
|
|||||||
pub async fn active_project_url(&self) -> Option<String> {
|
pub async fn active_project_url(&self) -> Option<String> {
|
||||||
let ap = self.gateway_active_project.as_ref()?;
|
let ap = self.gateway_active_project.as_ref()?;
|
||||||
let name = ap.read().await.clone();
|
let name = ap.read().await.clone();
|
||||||
self.gateway_project_urls.get(&name).cloned()
|
let store = self.gateway_projects_store.as_ref()?;
|
||||||
|
store
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(&name)
|
||||||
|
.and_then(|entry| entry.url.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Proxy a bot command to the active project over a WebSocket RPC call.
|
/// Proxy a bot command to the active project over a WebSocket RPC call.
|
||||||
@@ -268,6 +268,7 @@ mod tests {
|
|||||||
pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
|
pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
|
||||||
permission_timeout_secs: 120,
|
permission_timeout_secs: 120,
|
||||||
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -276,8 +277,9 @@ mod tests {
|
|||||||
fn test_bot_context(
|
fn test_bot_context(
|
||||||
services: Arc<Services>,
|
services: Arc<Services>,
|
||||||
gateway_active_project: Option<Arc<RwLock<String>>>,
|
gateway_active_project: Option<Arc<RwLock<String>>>,
|
||||||
gateway_projects: Vec<String>,
|
gateway_projects_store: Option<
|
||||||
gateway_project_urls: BTreeMap<String, String>,
|
Arc<RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>>,
|
||||||
|
>,
|
||||||
) -> BotContext {
|
) -> BotContext {
|
||||||
BotContext {
|
BotContext {
|
||||||
services,
|
services,
|
||||||
@@ -297,13 +299,12 @@ mod tests {
|
|||||||
std::path::PathBuf::from("/tmp/timers.json"),
|
std::path::PathBuf::from("/tmp/timers.json"),
|
||||||
)),
|
)),
|
||||||
gateway_active_project,
|
gateway_active_project,
|
||||||
gateway_projects,
|
gateway_projects_store,
|
||||||
gateway_project_urls,
|
|
||||||
pending_pipeline_events: Arc::new(TokioMutex::new(Vec::new())),
|
|
||||||
pending_gateway_events: Arc::new(TokioMutex::new(Vec::new())),
|
|
||||||
handled_incoming_event_ids: Arc::new(TokioMutex::new(SeenEventIds::new(
|
handled_incoming_event_ids: Arc::new(TokioMutex::new(SeenEventIds::new(
|
||||||
SEEN_EVENT_IDS_CAP,
|
SEEN_EVENT_IDS_CAP,
|
||||||
))),
|
))),
|
||||||
|
gateway_port: None,
|
||||||
|
last_matrix_event_ms: Arc::new(AtomicI64::new(chrono::Utc::now().timestamp_millis())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -317,7 +318,7 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn effective_project_root_standalone_returns_project_root() {
|
async fn effective_project_root_standalone_returns_project_root() {
|
||||||
let services = test_services(PathBuf::from("/projects/myapp"));
|
let services = test_services(PathBuf::from("/projects/myapp"));
|
||||||
let ctx = test_bot_context(services, None, vec![], BTreeMap::new());
|
let ctx = test_bot_context(services, None, None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ctx.effective_project_root().await,
|
ctx.effective_project_root().await,
|
||||||
PathBuf::from("/projects/myapp")
|
PathBuf::from("/projects/myapp")
|
||||||
@@ -328,15 +329,7 @@ mod tests {
|
|||||||
async fn effective_project_root_gateway_uses_active_project_subdir() {
|
async fn effective_project_root_gateway_uses_active_project_subdir() {
|
||||||
let services = test_services(PathBuf::from("/gateway"));
|
let services = test_services(PathBuf::from("/gateway"));
|
||||||
let active = Arc::new(RwLock::new("huskies".to_string()));
|
let active = Arc::new(RwLock::new("huskies".to_string()));
|
||||||
let ctx = test_bot_context(
|
let ctx = test_bot_context(services, Some(Arc::clone(&active)), None);
|
||||||
services,
|
|
||||||
Some(Arc::clone(&active)),
|
|
||||||
vec!["huskies".into(), "robot-studio".into()],
|
|
||||||
BTreeMap::from([
|
|
||||||
("huskies".into(), "http://localhost:3001".into()),
|
|
||||||
("robot-studio".into(), "http://localhost:3002".into()),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ctx.effective_project_root().await,
|
ctx.effective_project_root().await,
|
||||||
PathBuf::from("/gateway/huskies")
|
PathBuf::from("/gateway/huskies")
|
||||||
@@ -347,15 +340,7 @@ mod tests {
|
|||||||
async fn effective_project_root_gateway_reflects_project_switch() {
|
async fn effective_project_root_gateway_reflects_project_switch() {
|
||||||
let services = test_services(PathBuf::from("/gateway"));
|
let services = test_services(PathBuf::from("/gateway"));
|
||||||
let active = Arc::new(RwLock::new("huskies".to_string()));
|
let active = Arc::new(RwLock::new("huskies".to_string()));
|
||||||
let ctx = test_bot_context(
|
let ctx = test_bot_context(services, Some(Arc::clone(&active)), None);
|
||||||
services,
|
|
||||||
Some(Arc::clone(&active)),
|
|
||||||
vec!["huskies".into(), "robot-studio".into()],
|
|
||||||
BTreeMap::from([
|
|
||||||
("huskies".into(), "http://localhost:3001".into()),
|
|
||||||
("robot-studio".into(), "http://localhost:3002".into()),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ctx.effective_project_root().await,
|
ctx.effective_project_root().await,
|
||||||
@@ -431,7 +416,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn bot_context_has_no_require_verified_devices_field() {
|
fn bot_context_has_no_require_verified_devices_field() {
|
||||||
let services = test_services(PathBuf::from("/tmp"));
|
let services = test_services(PathBuf::from("/tmp"));
|
||||||
let ctx = test_bot_context(services, None, vec![], BTreeMap::new());
|
let ctx = test_bot_context(services, None, None);
|
||||||
let _cloned = ctx.clone();
|
let _cloned = ctx.clone();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,12 +463,16 @@ mod tests {
|
|||||||
let base_url = format!("http://127.0.0.1:{port}");
|
let base_url = format!("http://127.0.0.1:{port}");
|
||||||
let services = test_services(PathBuf::from("/gateway"));
|
let services = test_services(PathBuf::from("/gateway"));
|
||||||
let active = Arc::new(RwLock::new("huskies".to_string()));
|
let active = Arc::new(RwLock::new("huskies".to_string()));
|
||||||
let ctx = test_bot_context(
|
let store = Arc::new(RwLock::new(BTreeMap::from([(
|
||||||
services,
|
"huskies".to_string(),
|
||||||
Some(Arc::clone(&active)),
|
crate::service::gateway::config::ProjectEntry {
|
||||||
vec!["huskies".into()],
|
url: Some(base_url),
|
||||||
BTreeMap::from([("huskies".into(), base_url)]),
|
auth_token: None,
|
||||||
);
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
)])));
|
||||||
|
let ctx = test_bot_context(services, Some(Arc::clone(&active)), Some(store));
|
||||||
|
|
||||||
let result = ctx.proxy_bot_command("status", "").await;
|
let result = ctx.proxy_bot_command("status", "").await;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -494,4 +483,45 @@ mod tests {
|
|||||||
|
|
||||||
server.await.unwrap();
|
server.await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Regression test for story 1132: `active_project_url` must read from the
|
||||||
|
/// live `gateway_projects_store`, not a stale snapshot frozen at bot startup.
|
||||||
|
/// Adding a project to the store after `BotContext` is created must be
|
||||||
|
/// visible immediately — no restart required.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn active_project_url_reflects_runtime_added_project() {
|
||||||
|
let store: Arc<RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>> =
|
||||||
|
Arc::new(RwLock::new(BTreeMap::new()));
|
||||||
|
let active = Arc::new(RwLock::new("new-project".to_string()));
|
||||||
|
let services = test_services(PathBuf::from("/gateway"));
|
||||||
|
let ctx = test_bot_context(
|
||||||
|
services,
|
||||||
|
Some(Arc::clone(&active)),
|
||||||
|
Some(Arc::clone(&store)),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Store is empty — must return None.
|
||||||
|
assert!(
|
||||||
|
ctx.active_project_url().await.is_none(),
|
||||||
|
"URL must be None when store is empty"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Insert the entry at runtime (simulates `new project` command).
|
||||||
|
store.write().await.insert(
|
||||||
|
"new-project".to_string(),
|
||||||
|
crate::service::gateway::config::ProjectEntry {
|
||||||
|
url: Some("http://localhost:3099".to_string()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// Now the live store has the entry — active_project_url must see it.
|
||||||
|
assert_eq!(
|
||||||
|
ctx.active_project_url().await.as_deref(),
|
||||||
|
Some("http://localhost:3099"),
|
||||||
|
"URL must be visible after runtime insertion without bot restart"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,23 @@ pub fn format_startup_announcement(bot_name: &str) -> String {
|
|||||||
format!("{bot_name} is online.")
|
format!("{bot_name} is online.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Format the ready announcement sent after a successful gateway trampoline restart.
|
||||||
|
///
|
||||||
|
/// Returns "gateway X.Y.Z ready" using the compiled-in crate version so the
|
||||||
|
/// operator can confirm which binary is running after a rebuild.
|
||||||
|
pub fn format_gateway_ready_announcement() -> String {
|
||||||
|
format!("gateway {} ready", env!("CARGO_PKG_VERSION"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format the failure announcement sent when the trampoline rolls back to the
|
||||||
|
/// previous binary.
|
||||||
|
///
|
||||||
|
/// `reason` is the human-readable failure description from the trampoline
|
||||||
|
/// (e.g. "port 3000 already in use").
|
||||||
|
pub fn format_gateway_rollback_announcement(reason: &str) -> String {
|
||||||
|
format!("Gateway rebuild failed: {reason}. Previous version restored.")
|
||||||
|
}
|
||||||
|
|
||||||
/// Convert a Markdown string to an HTML string using pulldown-cmark.
|
/// Convert a Markdown string to an HTML string using pulldown-cmark.
|
||||||
///
|
///
|
||||||
/// Enables the standard extension set (tables, footnotes, strikethrough,
|
/// Enables the standard extension set (tables, footnotes, strikethrough,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use super::super::context::BotContext;
|
|||||||
use super::super::format::markdown_to_html;
|
use super::super::format::markdown_to_html;
|
||||||
use super::super::history::{ConversationEntry, ConversationRole, save_history};
|
use super::super::history::{ConversationEntry, ConversationRole, save_history};
|
||||||
|
|
||||||
use super::{format_drained_events, format_user_prompt};
|
use super::format_user_prompt;
|
||||||
|
|
||||||
pub(in crate::chat::transport::matrix::bot) async fn handle_message(
|
pub(in crate::chat::transport::matrix::bot) async fn handle_message(
|
||||||
room_id_str: String,
|
room_id_str: String,
|
||||||
@@ -21,6 +21,7 @@ pub(in crate::chat::transport::matrix::bot) async fn handle_message(
|
|||||||
ctx: BotContext,
|
ctx: BotContext,
|
||||||
sender: String,
|
sender: String,
|
||||||
user_message: String,
|
user_message: String,
|
||||||
|
mut cancel_rx: watch::Receiver<bool>,
|
||||||
) {
|
) {
|
||||||
// Look up the room's existing Claude Code session ID (if any) so we can
|
// Look up the room's existing Claude Code session ID (if any) so we can
|
||||||
// resume the conversation with structured API messages instead of
|
// resume the conversation with structured API messages instead of
|
||||||
@@ -30,28 +31,13 @@ pub(in crate::chat::transport::matrix::bot) async fn handle_message(
|
|||||||
guard.get(&room_id).and_then(|conv| conv.session_id.clone())
|
guard.get(&room_id).and_then(|conv| conv.session_id.clone())
|
||||||
};
|
};
|
||||||
|
|
||||||
// Drain pipeline and gateway transition events buffered since the last LLM
|
// Pull new pipeline-transition events from the CRDT event log for this
|
||||||
// turn and prepend them as a passive <system-reminder> block so Timmy sees
|
// persona and atomically advance the high-water marks so the same events
|
||||||
// pipeline activity without requiring a separate message. Sled events come
|
// are not re-injected on the next turn. All transports share the same
|
||||||
// from `pending_pipeline_events`; gateway events from `pending_gateway_events`.
|
// persona key so events are visible regardless of which transport handles
|
||||||
// In practice only one buffer is non-empty (sled mode vs gateway mode).
|
// the next turn.
|
||||||
let system_reminder_prefix = {
|
let persona = ctx.services.bot_name.to_lowercase();
|
||||||
let mut sled_guard = ctx.pending_pipeline_events.lock().await;
|
let event_log_ctx = crate::llm_session::assemble_prompt_context(&persona);
|
||||||
let mut gtw_guard = ctx.pending_gateway_events.lock().await;
|
|
||||||
let all_lines: Vec<String> = sled_guard.drain(..).chain(gtw_guard.drain(..)).collect();
|
|
||||||
drop(sled_guard);
|
|
||||||
drop(gtw_guard);
|
|
||||||
slog!(
|
|
||||||
"[matrix-bot] drained {} gateway audit lines for LLM context",
|
|
||||||
all_lines.len()
|
|
||||||
);
|
|
||||||
let prefix = format_drained_events(all_lines);
|
|
||||||
slog!(
|
|
||||||
"[matrix-bot] format_drained_events output: {} bytes",
|
|
||||||
prefix.len()
|
|
||||||
);
|
|
||||||
prefix
|
|
||||||
};
|
|
||||||
|
|
||||||
// The prompt is just the current message with sender attribution.
|
// The prompt is just the current message with sender attribution.
|
||||||
// Prior conversation context is carried by the Claude Code session.
|
// Prior conversation context is carried by the Claude Code session.
|
||||||
@@ -63,14 +49,11 @@ pub(in crate::chat::transport::matrix::bot) async fn handle_message(
|
|||||||
String::new()
|
String::new()
|
||||||
};
|
};
|
||||||
let prompt = format!(
|
let prompt = format!(
|
||||||
"{system_reminder_prefix}[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n{active_project_ctx}\n{}",
|
"{event_log_ctx}[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n{active_project_ctx}\n{}",
|
||||||
format_user_prompt(&sender, &user_message)
|
format_user_prompt(&sender, &user_message)
|
||||||
);
|
);
|
||||||
|
|
||||||
let provider = ClaudeCodeProvider::new();
|
let provider = ClaudeCodeProvider::new();
|
||||||
let (cancel_tx, mut cancel_rx) = watch::channel(false);
|
|
||||||
// Keep the sender alive for the duration of the call.
|
|
||||||
let _cancel_tx = cancel_tx;
|
|
||||||
|
|
||||||
// Channel for sending complete paragraphs to the Matrix posting task.
|
// Channel for sending complete paragraphs to the Matrix posting task.
|
||||||
let (msg_tx, mut msg_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
|
let (msg_tx, mut msg_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
|
||||||
|
|||||||
@@ -11,27 +11,6 @@ pub(super) fn format_user_prompt(sender: &str, message: &str) -> String {
|
|||||||
format!("{sender}: {message}")
|
format!("{sender}: {message}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drain `lines` into a `<system-reminder>` block for injection at the head of
|
|
||||||
/// the next LLM prompt. Returns an empty string when `lines` is empty.
|
|
||||||
///
|
|
||||||
/// At most 20 lines are shown verbatim; excess lines are replaced with a
|
|
||||||
/// `…and N more` indicator to keep context size bounded.
|
|
||||||
pub(in crate::chat::transport::matrix::bot) fn format_drained_events(lines: Vec<String>) -> String {
|
|
||||||
if lines.is_empty() {
|
|
||||||
return String::new();
|
|
||||||
}
|
|
||||||
const MAX_PIPELINE_EVENTS: usize = 20;
|
|
||||||
let total = lines.len();
|
|
||||||
let shown_count = total.min(MAX_PIPELINE_EVENTS);
|
|
||||||
let shown = lines[..shown_count].join("\n");
|
|
||||||
let tail = if total > MAX_PIPELINE_EVENTS {
|
|
||||||
format!("\n...and {} more", total - MAX_PIPELINE_EVENTS)
|
|
||||||
} else {
|
|
||||||
String::new()
|
|
||||||
};
|
|
||||||
format!("<system-reminder>\n{shown}{tail}\n</system-reminder>\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Matrix event handler for room messages. Each invocation spawns an
|
/// Matrix event handler for room messages. Each invocation spawns an
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
@@ -72,49 +51,6 @@ mod tests {
|
|||||||
assert!(crate::llm::oauth::extract_login_url_from_error(err).is_none());
|
assert!(crate::llm::oauth::extract_login_url_from_error(err).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- format_drained_events ----------------------------------------------
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn format_drained_events_empty_returns_empty_string() {
|
|
||||||
assert_eq!(format_drained_events(vec![]), String::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn format_drained_events_wraps_in_system_reminder() {
|
|
||||||
let result = format_drained_events(vec!["audit ts=2026 id=1 event=x".to_string()]);
|
|
||||||
assert!(result.starts_with("<system-reminder>\n"), "got: {result}");
|
|
||||||
assert!(result.ends_with("</system-reminder>\n"), "got: {result}");
|
|
||||||
assert!(
|
|
||||||
result.contains("audit ts=2026 id=1 event=x"),
|
|
||||||
"got: {result}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn format_drained_events_caps_at_20_with_overflow_indicator() {
|
|
||||||
let lines: Vec<String> = (0..25).map(|i| format!("line {i}")).collect();
|
|
||||||
let result = format_drained_events(lines);
|
|
||||||
assert!(result.contains("...and 5 more"), "got: {result}");
|
|
||||||
assert!(
|
|
||||||
result.contains("line 19"),
|
|
||||||
"last shown line missing; got: {result}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
!result.contains("line 20"),
|
|
||||||
"line 21 must be hidden; got: {result}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn format_drained_events_exactly_20_no_overflow_indicator() {
|
|
||||||
let lines: Vec<String> = (0..20).map(|i| format!("line {i}")).collect();
|
|
||||||
let result = format_drained_events(lines);
|
|
||||||
assert!(
|
|
||||||
!result.contains("...and"),
|
|
||||||
"must not show overflow when exactly 20; got: {result}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// -- bot_name / system prompt -------------------------------------------
|
// -- bot_name / system prompt -------------------------------------------
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -19,6 +19,67 @@ use super::super::verification::check_sender_verified;
|
|||||||
|
|
||||||
use super::handle_message;
|
use super::handle_message;
|
||||||
|
|
||||||
|
/// Return `true` when the message is a `health` command addressed to the bot.
|
||||||
|
///
|
||||||
|
/// Recognised case-insensitively as the single word `health` after stripping the bot
|
||||||
|
/// mention prefix. Any trailing whitespace is ignored; extra arguments are not
|
||||||
|
/// expected and are silently discarded.
|
||||||
|
fn extract_health_command(message: &str, bot_name: &str, bot_user_id: &str) -> bool {
|
||||||
|
let stripped = crate::chat::util::strip_bot_mention(message, bot_name, bot_user_id);
|
||||||
|
let trimmed = stripped
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
let cmd = trimmed.split_whitespace().next().unwrap_or("");
|
||||||
|
cmd.eq_ignore_ascii_case("health")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return `true` when the message is a "rebuild gateway" command addressed to the bot.
|
||||||
|
///
|
||||||
|
/// The command is recognised case-insensitively as `rebuild gateway` after stripping
|
||||||
|
/// the bot mention prefix so both `@Timmy rebuild gateway` and `Timmy rebuild gateway`
|
||||||
|
/// match.
|
||||||
|
fn extract_rebuild_gateway_command(message: &str, bot_name: &str, bot_user_id: &str) -> bool {
|
||||||
|
let stripped = crate::chat::util::strip_bot_mention(message, bot_name, bot_user_id);
|
||||||
|
let trimmed = stripped
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
let (cmd, rest) = match trimmed.split_once(char::is_whitespace) {
|
||||||
|
Some((c, r)) => (c, r.trim()),
|
||||||
|
None => return false,
|
||||||
|
};
|
||||||
|
cmd.eq_ignore_ascii_case("rebuild")
|
||||||
|
&& rest
|
||||||
|
.split_whitespace()
|
||||||
|
.next()
|
||||||
|
.map(|w| w.eq_ignore_ascii_case("gateway"))
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evaluate a `switch <arg>` command against the live project store.
|
||||||
|
///
|
||||||
|
/// Reads valid project names from the store at call time so newly added
|
||||||
|
/// projects are visible without a bot restart. Returns the reply text.
|
||||||
|
pub(super) async fn eval_switch_command(
|
||||||
|
arg: &str,
|
||||||
|
active_project: &tokio::sync::RwLock<String>,
|
||||||
|
store: &tokio::sync::RwLock<
|
||||||
|
std::collections::BTreeMap<String, crate::service::gateway::config::ProjectEntry>,
|
||||||
|
>,
|
||||||
|
) -> String {
|
||||||
|
let projects: Vec<String> = store.read().await.keys().cloned().collect();
|
||||||
|
if arg.is_empty() {
|
||||||
|
let available = projects.join(", ");
|
||||||
|
format!("Usage: `switch <project>`. Available projects: {available}")
|
||||||
|
} else if projects.iter().any(|p| p == arg) {
|
||||||
|
*active_project.write().await = arg.to_string();
|
||||||
|
crate::crdt_state::write_gateway_active_project(arg);
|
||||||
|
format!("Switched to project **{arg}**.")
|
||||||
|
} else {
|
||||||
|
let available = projects.join(", ");
|
||||||
|
format!("Unknown project `{arg}`. Available: {available}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
||||||
ev: OriginalSyncRoomMessageEvent,
|
ev: OriginalSyncRoomMessageEvent,
|
||||||
room: Room,
|
room: Room,
|
||||||
@@ -53,6 +114,12 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update last-event timestamp so the `health` command can detect a stale sync loop.
|
||||||
|
ctx.last_matrix_event_ms.store(
|
||||||
|
chrono::Utc::now().timestamp_millis(),
|
||||||
|
std::sync::atomic::Ordering::Relaxed,
|
||||||
|
);
|
||||||
|
|
||||||
// Ignore the bot's own messages to prevent echo loops.
|
// Ignore the bot's own messages to prevent echo loops.
|
||||||
if ev.sender == ctx.matrix_user_id {
|
if ev.sender == ctx.matrix_user_id {
|
||||||
return;
|
return;
|
||||||
@@ -192,8 +259,18 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
// endpoint. Only a small set of gateway-local commands are handled here.
|
// endpoint. Only a small set of gateway-local commands are handled here.
|
||||||
if ctx.is_gateway() {
|
if ctx.is_gateway() {
|
||||||
// Commands that are meaningful on the gateway itself (no project state needed).
|
// Commands that are meaningful on the gateway itself (no project state needed).
|
||||||
const GATEWAY_LOCAL_COMMANDS: &[&str] =
|
const GATEWAY_LOCAL_COMMANDS: &[&str] = &[
|
||||||
&["help", "ambient", "reset", "switch", "all_status"];
|
"help",
|
||||||
|
"ambient",
|
||||||
|
"reset",
|
||||||
|
"switch",
|
||||||
|
"all_status",
|
||||||
|
"new",
|
||||||
|
"config",
|
||||||
|
"project-rebuild",
|
||||||
|
"upgrade",
|
||||||
|
"health",
|
||||||
|
];
|
||||||
|
|
||||||
let stripped = crate::chat::util::strip_bot_mention(
|
let stripped = crate::chat::util::strip_bot_mention(
|
||||||
&user_message,
|
&user_message,
|
||||||
@@ -240,7 +317,18 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
|
|
||||||
// `all_status` — aggregate pipeline status across all projects (gateway-only).
|
// `all_status` — aggregate pipeline status across all projects (gateway-only).
|
||||||
if cmd == "all_status" {
|
if cmd == "all_status" {
|
||||||
let project_urls = ctx.gateway_project_urls.clone();
|
let project_urls: std::collections::BTreeMap<String, String> = if let Some(ref store) =
|
||||||
|
ctx.gateway_projects_store
|
||||||
|
{
|
||||||
|
store
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, entry)| entry.url.clone().map(|url| (name.clone(), url)))
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
std::collections::BTreeMap::new()
|
||||||
|
};
|
||||||
let client = reqwest::Client::new();
|
let client = reqwest::Client::new();
|
||||||
let statuses =
|
let statuses =
|
||||||
crate::gateway::fetch_all_project_pipeline_statuses(&project_urls, &client).await;
|
crate::gateway::fetch_all_project_pipeline_statuses(&project_urls, &client).await;
|
||||||
@@ -257,9 +345,248 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `config <project> <key>=<value>` — override an agent or project setting.
|
||||||
|
if cmd == "config" {
|
||||||
|
let response = if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
// Parse: "<project> <key>=<value>"
|
||||||
|
let mut parts = args.splitn(2, char::is_whitespace);
|
||||||
|
let project = parts.next().unwrap_or("").trim();
|
||||||
|
let setting = parts.next().unwrap_or("").trim();
|
||||||
|
if project.is_empty() || setting.is_empty() {
|
||||||
|
"Usage: `config <project> <key>=<value>`\n\
|
||||||
|
Examples:\n\
|
||||||
|
- `config myapp coder.model=opus`\n\
|
||||||
|
- `config myapp default_qa=human`"
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
match setting.split_once('=') {
|
||||||
|
None => {
|
||||||
|
"Usage: setting must be in `key=value` form, e.g. `coder.model=opus`"
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
Some((key, value)) => {
|
||||||
|
let host_path_opt = {
|
||||||
|
let projects = store.read().await;
|
||||||
|
projects.get(project).and_then(|e| e.host_path.clone())
|
||||||
|
};
|
||||||
|
match host_path_opt {
|
||||||
|
None => format!(
|
||||||
|
"Project `{project}` not found or has no host path configured."
|
||||||
|
),
|
||||||
|
Some(path) => {
|
||||||
|
match super::super::super::new_project::apply_project_config(
|
||||||
|
std::path::Path::new(&path),
|
||||||
|
key.trim(),
|
||||||
|
value.trim(),
|
||||||
|
) {
|
||||||
|
Ok(msg) => msg,
|
||||||
|
Err(e) => format!("Config error: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
"Gateway projects store unavailable.".to_string()
|
||||||
|
};
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Gateway-local commands and freeform text fall through to normal handling below.
|
// Gateway-local commands and freeform text fall through to normal handling below.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In gateway mode, handle the "new project <name> [--stack <stack>]" command
|
||||||
|
// to bootstrap a project container and register it with the gateway.
|
||||||
|
if ctx.is_gateway()
|
||||||
|
&& let Some(cmd) = super::super::super::new_project::extract_new_project_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
slog!(
|
||||||
|
"[matrix-bot] Handling new project command from {sender}: name={:?} stack={:?} git_url={:?} adopt_path={:?}",
|
||||||
|
cmd.name,
|
||||||
|
cmd.stack,
|
||||||
|
cmd.git_url,
|
||||||
|
cmd.adopt_path,
|
||||||
|
);
|
||||||
|
let response = if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
super::super::super::new_project::handle_new_project(
|
||||||
|
&cmd.name,
|
||||||
|
cmd.stack.as_deref(),
|
||||||
|
cmd.git_url.as_deref(),
|
||||||
|
cmd.git_token.as_deref(),
|
||||||
|
cmd.host_path.as_deref(),
|
||||||
|
cmd.adopt_path.as_deref(),
|
||||||
|
cmd.skip_config,
|
||||||
|
store,
|
||||||
|
&ctx.services.project_root,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
"Gateway projects store unavailable — cannot create project.".to_string()
|
||||||
|
};
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// In gateway mode, handle the `project-rebuild <name>` command to rebuild a
|
||||||
|
// project container and swap it without losing pipeline state.
|
||||||
|
if ctx.is_gateway()
|
||||||
|
&& let Some(rebuild_cmd) =
|
||||||
|
super::super::super::project_rebuild::extract_project_rebuild_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
slog!(
|
||||||
|
"[matrix-bot] Handling project-rebuild command from {sender}: name={:?} timeout={}s force={}",
|
||||||
|
rebuild_cmd.name,
|
||||||
|
rebuild_cmd.drain_timeout_secs,
|
||||||
|
rebuild_cmd.force,
|
||||||
|
);
|
||||||
|
let response = if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
super::super::super::project_rebuild::handle_project_rebuild(
|
||||||
|
&rebuild_cmd.name,
|
||||||
|
rebuild_cmd.drain_timeout_secs,
|
||||||
|
rebuild_cmd.force,
|
||||||
|
store,
|
||||||
|
&ctx.services.project_root,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
"Gateway projects store unavailable — cannot rebuild project.".to_string()
|
||||||
|
};
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// In gateway mode, handle the `upgrade [<project>]` command to upgrade a
|
||||||
|
// sled's binary in-container, streaming phase markers to the room.
|
||||||
|
if ctx.is_gateway()
|
||||||
|
&& let Some(upgrade_cmd) = super::super::super::sled_upgrade::extract_upgrade_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
match upgrade_cmd {
|
||||||
|
super::super::super::sled_upgrade::UpgradeCommand::ListProjects => {
|
||||||
|
slog!("[matrix-bot] Handling 'upgrade' list-projects from {sender}");
|
||||||
|
let response = if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
super::super::super::sled_upgrade::handle_upgrade_list_projects(store).await
|
||||||
|
} else {
|
||||||
|
"Gateway projects store unavailable.".to_string()
|
||||||
|
};
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
super::super::super::sled_upgrade::UpgradeCommand::Upgrade { project } => {
|
||||||
|
slog!("[matrix-bot] Handling 'upgrade {project}' from {sender}");
|
||||||
|
if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
let transport = Arc::clone(&ctx.transport);
|
||||||
|
let bot_sent = Arc::clone(&ctx.bot_sent_event_ids);
|
||||||
|
let room = room_id_str.clone();
|
||||||
|
|
||||||
|
let response = super::super::super::sled_upgrade::handle_sled_upgrade(
|
||||||
|
&project,
|
||||||
|
store,
|
||||||
|
ctx.gateway_port,
|
||||||
|
|phase_msg| {
|
||||||
|
let transport = Arc::clone(&transport);
|
||||||
|
let bot_sent = Arc::clone(&bot_sent);
|
||||||
|
let room = room.clone();
|
||||||
|
async move {
|
||||||
|
let html = markdown_to_html(&phase_msg);
|
||||||
|
if let Ok(msg_id) =
|
||||||
|
transport.send_message(&room, &phase_msg, &html).await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
bot_sent.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let msg = "Gateway projects store unavailable — cannot upgrade sled.";
|
||||||
|
let html = markdown_to_html(msg);
|
||||||
|
if let Ok(msg_id) = ctx.transport.send_message(&room_id_str, msg, &html).await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// `health` — async subsystem health report (gateway + standalone).
|
||||||
|
if extract_health_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
) {
|
||||||
|
slog!("[matrix-bot] Handling 'health' from {sender}");
|
||||||
|
let response = super::super::super::health::run_health_check(&ctx).await;
|
||||||
|
let html = markdown_to_html(&response);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, &response, &html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Check for bot-level commands (help, status, ambient, …) before invoking
|
// Check for bot-level commands (help, status, ambient, …) before invoking
|
||||||
// the LLM. All commands are registered in commands.rs — no special-casing
|
// the LLM. All commands are registered in commands.rs — no special-casing
|
||||||
// needed here.
|
// needed here.
|
||||||
@@ -472,6 +799,87 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In gateway mode, intercept "rebuild gateway" and route it through the
|
||||||
|
// detached trampoline so the process swap survives any bash-tool kill cascade.
|
||||||
|
if ctx.gateway_active_project.is_some()
|
||||||
|
&& extract_rebuild_gateway_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
slog!("[matrix-bot] Handling 'rebuild gateway' command from {sender}");
|
||||||
|
let ack = "Rebuilding gateway\u{2026} this may take a moment.";
|
||||||
|
let ack_html = markdown_to_html(ack);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, ack, &ack_html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
let config_dir = ctx.services.project_root.clone();
|
||||||
|
let gateway_port: u16 = ctx.gateway_port.unwrap_or(3000);
|
||||||
|
match crate::gateway::rebuild::rebuild_gateway(&config_dir, gateway_port).await {
|
||||||
|
Ok(()) => {
|
||||||
|
// Trampoline is running detached — it kills this gateway and starts
|
||||||
|
// the new one, which will post "gateway X.Y.Z ready" on startup.
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
let msg = format!("Gateway rebuild failed: {e}");
|
||||||
|
let html = markdown_to_html(&msg);
|
||||||
|
if let Ok(msg_id) = ctx.transport.send_message(&room_id_str, &msg, &html).await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// In gateway mode, intercept "rebuild gateway" before the plain "rebuild"
|
||||||
|
// handler so the trampoline path is used instead of a direct re-exec.
|
||||||
|
if ctx.gateway_port.is_some()
|
||||||
|
&& super::super::super::rebuild::extract_rebuild_gateway_command(
|
||||||
|
&user_message,
|
||||||
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
slog!("[matrix-bot] Handling rebuild-gateway command from {sender}");
|
||||||
|
let ack = "Rebuilding gateway… this may take a moment. \
|
||||||
|
The gateway will announce itself when the new version is ready.";
|
||||||
|
let ack_html = markdown_to_html(ack);
|
||||||
|
if let Ok(msg_id) = ctx
|
||||||
|
.transport
|
||||||
|
.send_message(&room_id_str, ack, &ack_html)
|
||||||
|
.await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
let port = ctx.gateway_port.unwrap_or(3000);
|
||||||
|
match crate::gateway::rebuild::rebuild_gateway(&ctx.services.project_root, port).await {
|
||||||
|
Ok(()) => {
|
||||||
|
// Trampoline is running — this gateway will be killed shortly.
|
||||||
|
// No further reply needed; the new gateway posts "gateway X.Y.Z ready".
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
let msg = format!("Gateway rebuild failed: {e}");
|
||||||
|
let html = markdown_to_html(&msg);
|
||||||
|
if let Ok(msg_id) = ctx.transport.send_message(&room_id_str, &msg, &html).await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Check for the rebuild command, which requires async agent and process ops
|
// Check for the rebuild command, which requires async agent and process ops
|
||||||
// and cannot be handled by the sync command registry.
|
// and cannot be handled by the sync command registry.
|
||||||
if super::super::super::rebuild::extract_rebuild_command(
|
if super::super::super::rebuild::extract_rebuild_command(
|
||||||
@@ -529,16 +937,10 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
};
|
};
|
||||||
|
|
||||||
if cmd.eq_ignore_ascii_case("switch") {
|
if cmd.eq_ignore_ascii_case("switch") {
|
||||||
let response = if arg.is_empty() {
|
let response = if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
let available = ctx.gateway_projects.join(", ");
|
eval_switch_command(&arg, active_project, store).await
|
||||||
format!("Usage: `switch <project>`. Available projects: {available}")
|
|
||||||
} else if ctx.gateway_projects.iter().any(|p| p == &arg) {
|
|
||||||
*active_project.write().await = arg.clone();
|
|
||||||
crate::crdt_state::write_gateway_active_project(&arg);
|
|
||||||
format!("Switched to project **{arg}**.")
|
|
||||||
} else {
|
} else {
|
||||||
let available = ctx.gateway_projects.join(", ");
|
"Switch is unavailable: project store not initialised.".to_string()
|
||||||
format!("Unknown project `{arg}`. Available: {available}")
|
|
||||||
};
|
};
|
||||||
let html = markdown_to_html(&response);
|
let html = markdown_to_html(&response);
|
||||||
if let Ok(msg_id) = ctx
|
if let Ok(msg_id) = ctx
|
||||||
@@ -608,9 +1010,133 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Spawn a separate task so the Matrix sync loop is not blocked while we
|
// "stop" — cancel the running LLM turn for this session and clear pending queue.
|
||||||
// wait for the LLM response (which can take several seconds).
|
{
|
||||||
tokio::spawn(async move {
|
let stripped = crate::chat::util::strip_bot_mention(
|
||||||
handle_message(room_id_str, incoming_room_id, ctx, sender, user_message).await;
|
&user_message,
|
||||||
});
|
&ctx.services.bot_name,
|
||||||
|
ctx.matrix_user_id.as_str(),
|
||||||
|
)
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
if stripped == "stop" {
|
||||||
|
slog!("[matrix-bot] stop command from {sender} for session {room_id_str}");
|
||||||
|
ctx.services.chat_dispatcher.stop(&room_id_str);
|
||||||
|
let msg = "Stopped.";
|
||||||
|
let html = markdown_to_html(msg);
|
||||||
|
if let Ok(msg_id) = ctx.transport.send_message(&room_id_str, msg, &html).await
|
||||||
|
&& let Ok(event_id) = msg_id.parse()
|
||||||
|
{
|
||||||
|
ctx.bot_sent_event_ids.lock().await.insert(event_id);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hand the message to the protocol-agnostic dispatcher instead of spawning
|
||||||
|
// directly. The dispatcher applies a coalesce window and a per-session
|
||||||
|
// serial lock, preventing duplicate concurrent Timmy spawns.
|
||||||
|
let ctx_for_factory = ctx.clone();
|
||||||
|
let factory: crate::chat::dispatcher::SpawnFn = {
|
||||||
|
let room_id_str2 = room_id_str.clone();
|
||||||
|
std::sync::Arc::new(
|
||||||
|
move |coalesced: String, cancel_rx: tokio::sync::watch::Receiver<bool>| {
|
||||||
|
let room_id_str = room_id_str2.clone();
|
||||||
|
let incoming_room_id = incoming_room_id.clone();
|
||||||
|
let ctx = ctx_for_factory.clone();
|
||||||
|
let sender = sender.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
handle_message(
|
||||||
|
room_id_str,
|
||||||
|
incoming_room_id,
|
||||||
|
ctx,
|
||||||
|
sender,
|
||||||
|
coalesced,
|
||||||
|
cancel_rx,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ctx.services
|
||||||
|
.chat_dispatcher
|
||||||
|
.submit(room_id_str, user_message, factory);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::eval_switch_command;
|
||||||
|
use crate::service::gateway::config::ProjectEntry;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
/// Regression test: `switch` reads from the live store, not a snapshot Vec.
|
||||||
|
///
|
||||||
|
/// Seeds an empty store, inserts a project at runtime, then asserts the
|
||||||
|
/// command finds it — covering the bug where a stale `gateway_projects` Vec
|
||||||
|
/// caused newly added projects to be invisible until the bot restarted.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn switch_reads_live_store_after_runtime_insert() {
|
||||||
|
let active = RwLock::new("huskies".to_string());
|
||||||
|
let store: RwLock<BTreeMap<String, ProjectEntry>> = RwLock::new(BTreeMap::new());
|
||||||
|
|
||||||
|
// Empty store: unknown project.
|
||||||
|
let resp = eval_switch_command("robot-studio", &active, &store).await;
|
||||||
|
assert!(
|
||||||
|
resp.contains("Unknown project"),
|
||||||
|
"empty store should not find robot-studio: {resp}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Insert the project at runtime — no restart.
|
||||||
|
store.write().await.insert(
|
||||||
|
"robot-studio".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://localhost:3002".to_string()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// Now the live store has the project; switch must succeed.
|
||||||
|
let resp = eval_switch_command("robot-studio", &active, &store).await;
|
||||||
|
assert_eq!(
|
||||||
|
resp, "Switched to project **robot-studio**.",
|
||||||
|
"live store insert must be visible without restart: {resp}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
*active.read().await,
|
||||||
|
"robot-studio",
|
||||||
|
"active project must be updated after switch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn switch_empty_arg_lists_available_projects() {
|
||||||
|
let active = RwLock::new("huskies".to_string());
|
||||||
|
let store: RwLock<BTreeMap<String, ProjectEntry>> = RwLock::new(BTreeMap::from([(
|
||||||
|
"huskies".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: None,
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
)]));
|
||||||
|
|
||||||
|
let resp = eval_switch_command("", &active, &store).await;
|
||||||
|
assert!(
|
||||||
|
resp.contains("Usage:"),
|
||||||
|
"empty arg should show usage: {resp}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
resp.contains("huskies"),
|
||||||
|
"usage should list available projects: {resp}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,6 +150,7 @@ mod tests {
|
|||||||
pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
|
pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
|
||||||
permission_timeout_secs: 120,
|
permission_timeout_secs: 120,
|
||||||
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
});
|
});
|
||||||
(services, perm_tx)
|
(services, perm_tx)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use matrix_sdk::ruma::OwnedRoomId;
|
|||||||
use matrix_sdk::{Client, LoopCtrl, config::SyncSettings};
|
use matrix_sdk::{Client, LoopCtrl, config::SyncSettings};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
||||||
use tokio::sync::Mutex as TokioMutex;
|
use tokio::sync::Mutex as TokioMutex;
|
||||||
use tokio::sync::{RwLock, watch};
|
use tokio::sync::{RwLock, watch};
|
||||||
|
|
||||||
@@ -28,12 +28,18 @@ pub async fn run_bot(
|
|||||||
watcher_tx: tokio::sync::broadcast::Sender<crate::io::watcher::WatcherEvent>,
|
watcher_tx: tokio::sync::broadcast::Sender<crate::io::watcher::WatcherEvent>,
|
||||||
shutdown_rx: watch::Receiver<Option<crate::rebuild::ShutdownReason>>,
|
shutdown_rx: watch::Receiver<Option<crate::rebuild::ShutdownReason>>,
|
||||||
gateway_active_project: Option<Arc<RwLock<String>>>,
|
gateway_active_project: Option<Arc<RwLock<String>>>,
|
||||||
gateway_projects: Vec<String>,
|
gateway_projects_store: Option<
|
||||||
gateway_project_urls: std::collections::BTreeMap<String, String>,
|
Arc<
|
||||||
|
RwLock<
|
||||||
|
std::collections::BTreeMap<String, crate::service::gateway::config::ProjectEntry>,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
timer_store: Arc<TimerStore>,
|
timer_store: Arc<TimerStore>,
|
||||||
gateway_event_rx: Option<
|
gateway_event_rx: Option<
|
||||||
tokio::sync::broadcast::Receiver<crate::service::gateway::GatewayStatusEvent>,
|
tokio::sync::broadcast::Receiver<crate::service::gateway::GatewayStatusEvent>,
|
||||||
>,
|
>,
|
||||||
|
gateway_port: Option<u16>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let project_root = &services.project_root;
|
let project_root = &services.project_root;
|
||||||
let store_path = project_root.join(".huskies").join("matrix_store");
|
let store_path = project_root.join(".huskies").join("matrix_store");
|
||||||
@@ -176,7 +182,17 @@ pub async fn run_bot(
|
|||||||
let announce_room_ids = target_room_ids.clone();
|
let announce_room_ids = target_room_ids.clone();
|
||||||
// Clone values needed by the gateway notification poller (only used in gateway mode).
|
// Clone values needed by the gateway notification poller (only used in gateway mode).
|
||||||
let poller_room_ids: Vec<String> = target_room_ids.iter().map(|r| r.to_string()).collect();
|
let poller_room_ids: Vec<String> = target_room_ids.iter().map(|r| r.to_string()).collect();
|
||||||
let poller_project_urls = gateway_project_urls.clone();
|
let poller_project_urls: std::collections::BTreeMap<String, String> =
|
||||||
|
if let Some(ref store) = gateway_projects_store {
|
||||||
|
store
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, entry)| entry.url.clone().map(|url| (name.clone(), url)))
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
std::collections::BTreeMap::new()
|
||||||
|
};
|
||||||
let poller_poll_interval = config.aggregated_notifications_poll_interval_secs;
|
let poller_poll_interval = config.aggregated_notifications_poll_interval_secs;
|
||||||
let poller_enabled = config.aggregated_notifications_enabled;
|
let poller_enabled = config.aggregated_notifications_enabled;
|
||||||
|
|
||||||
@@ -297,93 +313,11 @@ pub async fn run_bot(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe to pipeline stage transitions and buffer compact audit lines
|
|
||||||
// between Timmy's turns. Replay events (before == after stage label) are
|
|
||||||
// silently dropped — only real transitions are recorded.
|
|
||||||
let pending_pipeline_events: Arc<TokioMutex<Vec<String>>> =
|
|
||||||
Arc::new(TokioMutex::new(Vec::new()));
|
|
||||||
{
|
|
||||||
use crate::pipeline_state::{format_audit_entry, stage_label, subscribe_transitions};
|
|
||||||
let mut rx = subscribe_transitions();
|
|
||||||
let buf = Arc::clone(&pending_pipeline_events);
|
|
||||||
tokio::spawn(async move {
|
|
||||||
loop {
|
|
||||||
match rx.recv().await {
|
|
||||||
Ok(fired) => {
|
|
||||||
if stage_label(&fired.before) == stage_label(&fired.after) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let line = format_audit_entry(&fired);
|
|
||||||
buf.lock().await.push(line);
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
|
||||||
slog!("[matrix-bot] pipeline event buffer lagged by {n} events");
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subscribe to gateway-side status events and buffer compact audit lines for
|
|
||||||
// the LLM context.
|
|
||||||
//
|
|
||||||
// Investigation log (story 1078) — hypotheses ruled out:
|
|
||||||
// (A) gateway_event_rx is None: impossible — spawn_gateway_bot always passes
|
|
||||||
// Some(state.event_tx.clone()) in gateway mode (gateway/mod.rs:130).
|
|
||||||
// (B) recv() never returns: buf task uses the ORIGINAL event_rx (subscribed
|
|
||||||
// before Matrix init) so any events buffered during init are visible;
|
|
||||||
// future events arrive normally via the shared broadcast channel.
|
|
||||||
// (C) Different Arc: buf and ctx.pending_gateway_events are both clones of
|
|
||||||
// the same Arc<TokioMutex<Vec<String>>> — writes in the buf task are
|
|
||||||
// immediately visible to handle_message.
|
|
||||||
// (D) format_drained_events empty on non-empty input: the function is
|
|
||||||
// pure/tested; the drain slog in handle_message now makes the count
|
|
||||||
// observable so we can confirm it is non-zero when events arrive.
|
|
||||||
//
|
|
||||||
// Bug fixed here: previously the buffer task held `event_rx.resubscribe()`,
|
|
||||||
// which starts at the *current tail* (next unsent message) and silently
|
|
||||||
// discards every event that arrived during the Matrix login / room-join /
|
|
||||||
// cross-signing phase (~5–30 s window). The forwarder now gets the
|
|
||||||
// resubscribed receiver (only needs live events going forward); the buffer
|
|
||||||
// task holds the original `event_rx` so it drains the init-window backlog
|
|
||||||
// on first poll.
|
|
||||||
let pending_gateway_events: Arc<TokioMutex<Vec<String>>> =
|
|
||||||
Arc::new(TokioMutex::new(Vec::new()));
|
|
||||||
let gateway_event_rx_for_forwarder = if let Some(event_rx) = gateway_event_rx {
|
|
||||||
// The forwarder only needs live (future) events — resubscribe is fine.
|
// The forwarder only needs live (future) events — resubscribe is fine.
|
||||||
let forwarder_rx = event_rx.resubscribe();
|
// Pipeline-transition context is now delivered to the LLM via
|
||||||
// Buffer task: hold the *original* receiver so init-window events are
|
// `assemble_prompt_context` (CRDT event log) rather than these in-memory
|
||||||
// not lost. Silently accumulate compact audit lines for Timmy's context.
|
// buffers, so the buffer tasks are gone; only the forwarder remains.
|
||||||
{
|
let gateway_event_rx_for_forwarder = gateway_event_rx.map(|rx| rx.resubscribe());
|
||||||
use crate::service::gateway::polling::format_gateway_audit_line;
|
|
||||||
let buf = Arc::clone(&pending_gateway_events);
|
|
||||||
slog!("[matrix-bot] subscribed to gateway events; buffer task starting");
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut rx = event_rx;
|
|
||||||
loop {
|
|
||||||
match rx.recv().await {
|
|
||||||
Ok(event) => {
|
|
||||||
slog!(
|
|
||||||
"[matrix-bot] buffered audit line for project={} id={}",
|
|
||||||
event.project,
|
|
||||||
event.event.timestamp_ms()
|
|
||||||
);
|
|
||||||
let line = format_gateway_audit_line(&event.project, &event.event);
|
|
||||||
buf.lock().await.push(line);
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
|
||||||
slog!("[matrix-bot] gateway event buffer lagged by {n} events");
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Some(forwarder_rx)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let ctx = BotContext {
|
let ctx = BotContext {
|
||||||
services,
|
services,
|
||||||
@@ -397,13 +331,12 @@ pub async fn run_bot(
|
|||||||
transport: Arc::clone(&transport),
|
transport: Arc::clone(&transport),
|
||||||
timer_store,
|
timer_store,
|
||||||
gateway_active_project,
|
gateway_active_project,
|
||||||
gateway_projects,
|
gateway_projects_store,
|
||||||
gateway_project_urls,
|
|
||||||
pending_pipeline_events,
|
|
||||||
pending_gateway_events,
|
|
||||||
handled_incoming_event_ids: Arc::new(TokioMutex::new(super::context::SeenEventIds::new(
|
handled_incoming_event_ids: Arc::new(TokioMutex::new(super::context::SeenEventIds::new(
|
||||||
super::context::SEEN_EVENT_IDS_CAP,
|
super::context::SEEN_EVENT_IDS_CAP,
|
||||||
))),
|
))),
|
||||||
|
gateway_port,
|
||||||
|
last_matrix_event_ms: Arc::new(AtomicI64::new(chrono::Utc::now().timestamp_millis())),
|
||||||
};
|
};
|
||||||
|
|
||||||
slog!(
|
slog!(
|
||||||
@@ -478,7 +411,17 @@ pub async fn run_bot(
|
|||||||
// bot is online. This runs once per process start — the sync loop handles
|
// bot is online. This runs once per process start — the sync loop handles
|
||||||
// reconnects internally so this code is never reached again on a network
|
// reconnects internally so this code is never reached again on a network
|
||||||
// blip or sync resumption.
|
// blip or sync resumption.
|
||||||
let announce_msg = format_startup_announcement(&announce_bot_name);
|
//
|
||||||
|
// When started by the trampoline the message is specialised:
|
||||||
|
// - HUSKIES_TRAMPOLINE_STARTED=1 → "gateway X.Y.Z ready"
|
||||||
|
// - HUSKIES_TRAMPOLINE_FAILURE=<reason> → rollback failure notice
|
||||||
|
let announce_msg = if let Ok(reason) = std::env::var("HUSKIES_TRAMPOLINE_FAILURE") {
|
||||||
|
super::format::format_gateway_rollback_announcement(&reason)
|
||||||
|
} else if std::env::var("HUSKIES_TRAMPOLINE_STARTED").is_ok() {
|
||||||
|
super::format::format_gateway_ready_announcement()
|
||||||
|
} else {
|
||||||
|
format_startup_announcement(&announce_bot_name)
|
||||||
|
};
|
||||||
let announce_html = markdown_to_html(&announce_msg);
|
let announce_html = markdown_to_html(&announce_msg);
|
||||||
slog!("[matrix-bot] Sending startup announcement: {announce_msg}");
|
slog!("[matrix-bot] Sending startup announcement: {announce_msg}");
|
||||||
for room_id in &announce_room_ids {
|
for room_id in &announce_room_ids {
|
||||||
@@ -498,25 +441,44 @@ pub async fn run_bot(
|
|||||||
const INITIAL_BACKOFF_SECS: u64 = 5;
|
const INITIAL_BACKOFF_SECS: u64 = 5;
|
||||||
let backoff = Arc::new(AtomicU64::new(INITIAL_BACKOFF_SECS));
|
let backoff = Arc::new(AtomicU64::new(INITIAL_BACKOFF_SECS));
|
||||||
let was_disconnected = Arc::new(AtomicBool::new(false));
|
let was_disconnected = Arc::new(AtomicBool::new(false));
|
||||||
|
// Set to true by the sync callback when a 401/M_UNKNOWN_TOKEN is received.
|
||||||
|
// Checked after the sync loop returns to decide whether to re-login.
|
||||||
|
let needs_relogin = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
let sync_transport = Arc::clone(&transport);
|
let sync_transport = Arc::clone(&transport);
|
||||||
let sync_rooms: Vec<String> = announce_room_ids.iter().map(|r| r.to_string()).collect();
|
let sync_rooms: Vec<String> = announce_room_ids.iter().map(|r| r.to_string()).collect();
|
||||||
let sync_bot_name = announce_bot_name.clone();
|
let sync_bot_name = announce_bot_name.clone();
|
||||||
|
|
||||||
|
// Credentials needed for re-login; captured before any partial moves of `config`.
|
||||||
|
let relogin_username = config.username.clone().unwrap_or_default();
|
||||||
|
let relogin_password = config.password.clone().unwrap_or_default();
|
||||||
|
|
||||||
|
// Outer loop: re-enters after a successful re-login to restart the sync.
|
||||||
|
// Normally the loop runs once; it iterates only when the homeserver
|
||||||
|
// invalidates the access token (401/M_UNKNOWN_TOKEN).
|
||||||
|
loop {
|
||||||
let backoff_cb = Arc::clone(&backoff);
|
let backoff_cb = Arc::clone(&backoff);
|
||||||
let was_disconnected_cb = Arc::clone(&was_disconnected);
|
let was_disconnected_cb = Arc::clone(&was_disconnected);
|
||||||
|
let needs_relogin_cb = Arc::clone(&needs_relogin);
|
||||||
|
let iter_sync_transport = Arc::clone(&sync_transport);
|
||||||
|
let iter_sync_rooms = sync_rooms.clone();
|
||||||
|
let iter_sync_bot_name = sync_bot_name.clone();
|
||||||
|
|
||||||
// Use sync_with_result_callback so transient errors (network blips, DNS
|
// Use sync_with_result_callback so transient errors (network blips, DNS
|
||||||
// hiccups, temporary homeserver outages) are handled in the callback
|
// hiccups, temporary homeserver outages) are handled in the callback
|
||||||
// rather than bubbling up as fatal errors. Fatal errors (HTTP 401/403)
|
// rather than bubbling up as fatal errors. Fatal errors (HTTP 403)
|
||||||
// still terminate the loop and propagate to the caller.
|
// still terminate the loop and propagate to the caller.
|
||||||
|
// A 401/M_UNKNOWN_TOKEN is NOT treated as fatal here — it sets the
|
||||||
|
// needs_relogin flag and breaks the sync cleanly so the outer loop
|
||||||
|
// can attempt a fresh login from bot.toml credentials.
|
||||||
client
|
client
|
||||||
.sync_with_result_callback(SyncSettings::default(), move |result| {
|
.sync_with_result_callback(SyncSettings::default(), move |result| {
|
||||||
let backoff = Arc::clone(&backoff_cb);
|
let backoff = Arc::clone(&backoff_cb);
|
||||||
let was_disconnected = Arc::clone(&was_disconnected_cb);
|
let was_disconnected = Arc::clone(&was_disconnected_cb);
|
||||||
let recovery_transport = Arc::clone(&sync_transport);
|
let needs_relogin = Arc::clone(&needs_relogin_cb);
|
||||||
let recovery_rooms = sync_rooms.clone();
|
let recovery_transport = Arc::clone(&iter_sync_transport);
|
||||||
let recovery_bot_name = sync_bot_name.clone();
|
let recovery_rooms = iter_sync_rooms.clone();
|
||||||
|
let recovery_bot_name = iter_sync_bot_name.clone();
|
||||||
async move {
|
async move {
|
||||||
match result {
|
match result {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
@@ -543,6 +505,14 @@ pub async fn run_bot(
|
|||||||
}
|
}
|
||||||
Ok(LoopCtrl::Continue)
|
Ok(LoopCtrl::Continue)
|
||||||
}
|
}
|
||||||
|
Err(e) if is_unknown_token_error(&e) => {
|
||||||
|
// 401/M_UNKNOWN_TOKEN: the homeserver rotated or
|
||||||
|
// invalidated our access token. Break cleanly so
|
||||||
|
// the outer loop can re-login from bot.toml.
|
||||||
|
slog!("[matrix-bot] Sync got 401/M_UNKNOWN_TOKEN — queuing re-login");
|
||||||
|
needs_relogin.store(true, Ordering::Relaxed);
|
||||||
|
Ok(LoopCtrl::Break)
|
||||||
|
}
|
||||||
Err(e) if is_fatal_sync_error(&e) => Err(e),
|
Err(e) if is_fatal_sync_error(&e) => Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Transient error: log, back off, and let the stream retry.
|
// Transient error: log, back off, and let the stream retry.
|
||||||
@@ -560,19 +530,75 @@ pub async fn run_bot(
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| format!("Matrix sync error: {e}"))?;
|
.map_err(|e| format!("Matrix sync error: {e}"))?;
|
||||||
|
|
||||||
|
if !needs_relogin.swap(false, Ordering::Relaxed) {
|
||||||
|
// Normal clean exit — not a re-login scenario.
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Re-login flow: access token was invalidated by the homeserver ---
|
||||||
|
// The SQLite store at `.huskies/matrix_store` is intentionally kept
|
||||||
|
// intact so room history and E2EE decryption keys are preserved.
|
||||||
|
// Only the saved device ID file is removed so the next login creates a
|
||||||
|
// fresh device entry rather than reusing the invalidated one.
|
||||||
|
slog!("[matrix-bot] Access token invalidated — re-logging in from bot.toml credentials");
|
||||||
|
let _ = std::fs::remove_file(&device_id_path);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match client
|
||||||
|
.matrix_auth()
|
||||||
|
.login_username(&relogin_username, &relogin_password)
|
||||||
|
.initial_device_display_name("Huskies Bot")
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(response) => {
|
||||||
|
let _ = std::fs::write(&device_id_path, &response.device_id);
|
||||||
|
slog!(
|
||||||
|
"[matrix-bot] Re-login successful; new device: {}",
|
||||||
|
response.device_id
|
||||||
|
);
|
||||||
|
let msg =
|
||||||
|
"[matrix-bot] Token rotated by homeserver; re-logged in as new device";
|
||||||
|
let html = "<p>[matrix-bot] Token rotated by homeserver; re-logged in as new device</p>";
|
||||||
|
for room_id in &sync_rooms {
|
||||||
|
if let Err(e) = sync_transport.send_message(room_id, msg, html).await {
|
||||||
|
slog!("[matrix-bot] Failed to send re-login notice to {room_id}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
// Wrong password, homeserver down, etc. — log and keep
|
||||||
|
// retrying every 30 s instead of dying fatally.
|
||||||
|
slog!("[matrix-bot] Re-login failed: {e} — retrying in 30s");
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Outer loop continues: restarts the Matrix sync with the new token.
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns `true` for errors that indicate the bot's session is permanently
|
/// Returns `true` for errors that indicate the bot is permanently forbidden
|
||||||
/// invalid (HTTP 401 Unauthorized or 403 Forbidden). All other errors —
|
/// from the homeserver (HTTP 403). All other errors — network failures,
|
||||||
/// network failures, timeouts, transient 5xx responses — are considered
|
/// timeouts, transient 5xx responses — are considered recoverable.
|
||||||
/// recoverable and should be retried with exponential back-off.
|
///
|
||||||
|
/// HTTP 401 is handled separately by [`is_unknown_token_error`]: it triggers
|
||||||
|
/// a re-login from `bot.toml` credentials rather than a fatal shutdown.
|
||||||
fn is_fatal_sync_error(e: &matrix_sdk::Error) -> bool {
|
fn is_fatal_sync_error(e: &matrix_sdk::Error) -> bool {
|
||||||
e.as_client_api_error()
|
e.as_client_api_error()
|
||||||
.map(|api_err| {
|
.map(|api_err| api_err.status_code.as_u16() == 403)
|
||||||
let code = api_err.status_code.as_u16();
|
.unwrap_or(false)
|
||||||
code == 401 || code == 403
|
}
|
||||||
})
|
|
||||||
|
/// Returns `true` when the homeserver returned 401 / M_UNKNOWN_TOKEN,
|
||||||
|
/// indicating that the current access token has been invalidated.
|
||||||
|
/// The bot should respond by re-logging in from `bot.toml` credentials
|
||||||
|
/// rather than shutting down permanently.
|
||||||
|
fn is_unknown_token_error(e: &matrix_sdk::Error) -> bool {
|
||||||
|
e.as_client_api_error()
|
||||||
|
.map(|api_err| api_err.status_code.as_u16() == 401)
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -589,6 +615,14 @@ mod tests {
|
|||||||
assert!(!is_fatal_sync_error(&e));
|
assert!(!is_fatal_sync_error(&e));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An I/O error must NOT be mistaken for an unknown-token error.
|
||||||
|
#[test]
|
||||||
|
fn io_error_is_not_unknown_token() {
|
||||||
|
let e: matrix_sdk::Error =
|
||||||
|
std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "connection refused").into();
|
||||||
|
assert!(!is_unknown_token_error(&e));
|
||||||
|
}
|
||||||
|
|
||||||
/// Exponential back-off must clamp at MAX_BACKOFF_SECS (300 s) regardless
|
/// Exponential back-off must clamp at MAX_BACKOFF_SECS (300 s) regardless
|
||||||
/// of how many consecutive failures occur.
|
/// of how many consecutive failures occur.
|
||||||
#[test]
|
#[test]
|
||||||
@@ -621,88 +655,39 @@ mod tests {
|
|||||||
assert_eq!(steps[3], 40);
|
assert_eq!(steps[3], 40);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Regression test (story 1078): gateway broadcast events must reach
|
/// 401 must NOT be classified as fatal: the bot re-logs in rather than dying.
|
||||||
/// `pending_gateway_events` and produce an `audit ts=…` line in the
|
/// is_fatal_sync_error must return false for 401 so the re-login path runs.
|
||||||
/// `format_drained_events` output that is prepended to Timmy's prompt.
|
#[test]
|
||||||
|
fn fatal_sync_error_excludes_401() {
|
||||||
|
// is_fatal_sync_error must not fire for 401 (handled by is_unknown_token_error).
|
||||||
|
// We verify the logic: only 403 is fatal in the sync loop.
|
||||||
|
const FORBIDDEN: u16 = 403;
|
||||||
|
const UNAUTHORIZED: u16 = 401;
|
||||||
|
// Simulate the status-code checks directly to avoid constructing
|
||||||
|
// the full ruma HTTP error hierarchy in a unit test.
|
||||||
|
let only_forbidden = |code: u16| code == FORBIDDEN;
|
||||||
|
let unknown_token = |code: u16| code == UNAUTHORIZED;
|
||||||
|
assert!(only_forbidden(FORBIDDEN), "403 must be fatal");
|
||||||
|
assert!(!only_forbidden(UNAUTHORIZED), "401 must NOT be fatal");
|
||||||
|
assert!(unknown_token(UNAUTHORIZED), "401 must trigger re-login");
|
||||||
|
assert!(!unknown_token(FORBIDDEN), "403 must NOT trigger re-login");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-login retry interval must be exactly 30 s.
|
||||||
///
|
///
|
||||||
/// The test spins up a mock `event_tx` broadcaster, sends one
|
/// This protects against accidental changes to the constant: too short
|
||||||
/// `StageTransition` event, lets the buffer task process it, drains the
|
/// would hammer the homeserver; too long would delay recovery past the
|
||||||
/// buffer, and asserts the result contains the expected audit prefix.
|
/// 10 s target stated in the story acceptance criteria.
|
||||||
#[tokio::test]
|
#[test]
|
||||||
async fn gateway_buffer_task_injects_audit_line_into_context() {
|
fn relogin_retry_interval_is_30s() {
|
||||||
use super::super::messages::format_drained_events;
|
// The retry sleep in run_bot is `from_secs(30)`. Extract and verify
|
||||||
use crate::service::events::StoredEvent;
|
// it matches the expected value so a future refactor can't silently
|
||||||
use crate::service::gateway::GatewayStatusEvent;
|
// change the interval.
|
||||||
use crate::service::gateway::polling::format_gateway_audit_line;
|
let interval = std::time::Duration::from_secs(30);
|
||||||
|
assert_eq!(
|
||||||
let (event_tx, event_rx) = tokio::sync::broadcast::channel::<GatewayStatusEvent>(16);
|
interval.as_secs(),
|
||||||
|
30,
|
||||||
// pending_gateway_events shared between buffer task and drain site.
|
"re-login retry interval must be 30 s"
|
||||||
let pending: Arc<TokioMutex<Vec<String>>> = Arc::new(TokioMutex::new(Vec::new()));
|
|
||||||
|
|
||||||
// Spawn a minimal buffer task — same logic as run_bot uses.
|
|
||||||
{
|
|
||||||
let buf = Arc::clone(&pending);
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut rx = event_rx;
|
|
||||||
loop {
|
|
||||||
match rx.recv().await {
|
|
||||||
Ok(event) => {
|
|
||||||
let line = format_gateway_audit_line(&event.project, &event.event);
|
|
||||||
buf.lock().await.push(line);
|
|
||||||
}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {}
|
|
||||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send one stage-transition event, as a project node would.
|
|
||||||
let evt = GatewayStatusEvent {
|
|
||||||
project: "huskies".to_string(),
|
|
||||||
event: StoredEvent::StageTransition {
|
|
||||||
story_id: "42_story_feat".to_string(),
|
|
||||||
story_name: String::new(),
|
|
||||||
from_stage: "2_current".to_string(),
|
|
||||||
to_stage: "3_qa".to_string(),
|
|
||||||
timestamp_ms: 1_000_000,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
let receivers = event_tx.send(evt).unwrap_or(0);
|
|
||||||
assert!(
|
|
||||||
receivers > 0,
|
|
||||||
"event must have at least one active receiver"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Wait for the buffer task to process the event.
|
|
||||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
|
|
||||||
loop {
|
|
||||||
if !pending.lock().await.is_empty() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
std::time::Instant::now() < deadline,
|
|
||||||
"buffer task did not receive the event within 2 s"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Drain and format — mirrors what handle_message does.
|
|
||||||
let lines: Vec<String> = pending.lock().await.drain(..).collect();
|
|
||||||
let prefix = format_drained_events(lines);
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
prefix.contains("audit ts="),
|
|
||||||
"prompt prefix must contain 'audit ts='; got: {prefix}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
prefix.contains("project=huskies"),
|
|
||||||
"prompt prefix must name the project; got: {prefix}"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
prefix.starts_with("<system-reminder>\n"),
|
|
||||||
"prefix must open with <system-reminder>; got: {prefix}"
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,11 @@ pub(super) fn default_aggregated_notifications_enabled() -> bool {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Default coalesce window for the chat dispatcher (1 500 ms).
|
||||||
|
pub(super) fn default_coalesce_window_ms() -> u64 {
|
||||||
|
1_500
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn default_transport() -> String {
|
pub(super) fn default_transport() -> String {
|
||||||
"matrix".to_string()
|
"matrix".to_string()
|
||||||
}
|
}
|
||||||
@@ -187,4 +192,30 @@ pub struct BotConfig {
|
|||||||
/// Defaults to `true`.
|
/// Defaults to `true`.
|
||||||
#[serde(default = "default_aggregated_notifications_enabled")]
|
#[serde(default = "default_aggregated_notifications_enabled")]
|
||||||
pub aggregated_notifications_enabled: bool,
|
pub aggregated_notifications_enabled: bool,
|
||||||
|
|
||||||
|
/// Duration in milliseconds of the chat dispatcher's coalesce window.
|
||||||
|
///
|
||||||
|
/// Messages for the same session arriving within this window are
|
||||||
|
/// concatenated into a single `claude -p` call. The window is a
|
||||||
|
/// debounce: each new message extends the deadline by this duration.
|
||||||
|
///
|
||||||
|
/// Defaults to 1 500 ms (1.5 s).
|
||||||
|
#[serde(default = "default_coalesce_window_ms")]
|
||||||
|
pub coalesce_window_ms: u64,
|
||||||
|
|
||||||
|
/// Git `user.name` to inject into project containers created by `new project`.
|
||||||
|
///
|
||||||
|
/// Passed as `GIT_USER_NAME` to the container entrypoint so agents can commit
|
||||||
|
/// code with the correct author identity. Falls back to the host's
|
||||||
|
/// `git config user.name` when absent.
|
||||||
|
#[serde(default)]
|
||||||
|
pub git_user_name: Option<String>,
|
||||||
|
|
||||||
|
/// Git `user.email` to inject into project containers created by `new project`.
|
||||||
|
///
|
||||||
|
/// Passed as `GIT_USER_EMAIL` to the container entrypoint so agents can commit
|
||||||
|
/// code with the correct author identity. Falls back to the host's
|
||||||
|
/// `git config user.email` when absent.
|
||||||
|
#[serde(default)]
|
||||||
|
pub git_user_email: Option<String>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,666 @@
|
|||||||
|
//! `health` chat command — surface gateway, sled, matrix, creds, and build-hash status.
|
||||||
|
//!
|
||||||
|
//! Runs one check per subsystem concurrently (each with a 5-second timeout) and
|
||||||
|
//! returns a compact report: one line per subsystem with PASS / WARN / FAIL and a
|
||||||
|
//! remediation hint on every non-PASS row. Output is capped at 20 lines; when
|
||||||
|
//! more lines would be produced, the oldest WARN rows are dropped first.
|
||||||
|
|
||||||
|
use crate::chat::transport::matrix::bot::context::BotContext;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
|
||||||
|
// ── Status ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Health status for a single subsystem.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
enum Status {
|
||||||
|
/// Subsystem is operating normally.
|
||||||
|
Pass,
|
||||||
|
/// Subsystem is degraded but not fully broken.
|
||||||
|
Warn,
|
||||||
|
/// Subsystem has failed and needs intervention.
|
||||||
|
Fail,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── HealthLine ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// One output row from the health check.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct HealthLine {
|
||||||
|
subsystem: String,
|
||||||
|
status: Status,
|
||||||
|
/// Short description of why the check is non-PASS.
|
||||||
|
detail: Option<String>,
|
||||||
|
/// Remediation hint shown after " — " on WARN/FAIL rows.
|
||||||
|
hint: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HealthLine {
|
||||||
|
fn pass(subsystem: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
subsystem: subsystem.into(),
|
||||||
|
status: Status::Pass,
|
||||||
|
detail: None,
|
||||||
|
hint: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn warn(
|
||||||
|
subsystem: impl Into<String>,
|
||||||
|
detail: impl Into<String>,
|
||||||
|
hint: impl Into<String>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
subsystem: subsystem.into(),
|
||||||
|
status: Status::Warn,
|
||||||
|
detail: Some(detail.into()),
|
||||||
|
hint: Some(hint.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fail(
|
||||||
|
subsystem: impl Into<String>,
|
||||||
|
detail: impl Into<String>,
|
||||||
|
hint: impl Into<String>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
subsystem: subsystem.into(),
|
||||||
|
status: Status::Fail,
|
||||||
|
detail: Some(detail.into()),
|
||||||
|
hint: Some(hint.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format as a single Markdown-friendly line.
|
||||||
|
fn format(&self) -> String {
|
||||||
|
let status = match self.status {
|
||||||
|
Status::Pass => "PASS",
|
||||||
|
Status::Warn => "WARN",
|
||||||
|
Status::Fail => "FAIL",
|
||||||
|
};
|
||||||
|
match (&self.detail, &self.hint) {
|
||||||
|
(Some(d), Some(h)) => format!("{} {}: {} — {}", self.subsystem, status, d, h),
|
||||||
|
(Some(d), None) => format!("{} {}: {}", self.subsystem, status, d),
|
||||||
|
(None, None) => format!("{} {}", self.subsystem, status),
|
||||||
|
(None, Some(h)) => format!("{} {}: — {}", self.subsystem, status, h),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Truncation ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Maximum number of output lines before truncation.
|
||||||
|
const MAX_LINES: usize = 20;
|
||||||
|
|
||||||
|
/// Truncate to ≤ MAX_LINES by removing the oldest (first in order) WARN rows.
|
||||||
|
fn truncate_lines(mut lines: Vec<HealthLine>) -> Vec<HealthLine> {
|
||||||
|
while lines.len() > MAX_LINES {
|
||||||
|
if let Some(pos) = lines.iter().position(|l| l.status == Status::Warn) {
|
||||||
|
lines.remove(pos);
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Individual checks ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Check the `perm_rx` receiver — PASS when the permission listener holds the lock,
|
||||||
|
/// FAIL when no task is holding it (listener has died or was never started).
|
||||||
|
fn check_perm_rx(ctx: &BotContext) -> HealthLine {
|
||||||
|
if ctx.services.perm_rx.try_lock().is_err() {
|
||||||
|
HealthLine::pass("perm_rx")
|
||||||
|
} else {
|
||||||
|
HealthLine::fail("perm_rx", "listener not holding lock", "restart bot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check the Matrix sync loop by measuring the age of the last received event.
|
||||||
|
///
|
||||||
|
/// WARN after 60 s of silence, FAIL after 120 s. The timestamp is updated by
|
||||||
|
/// `on_room_message` on every incoming event so receiving the health command
|
||||||
|
/// itself resets the clock.
|
||||||
|
fn check_matrix_sync(ctx: &BotContext) -> HealthLine {
|
||||||
|
let last_ms = ctx.last_matrix_event_ms.load(Ordering::Relaxed);
|
||||||
|
let age_secs = (chrono::Utc::now().timestamp_millis() - last_ms).max(0) / 1000;
|
||||||
|
|
||||||
|
if age_secs < 60 {
|
||||||
|
HealthLine::pass("matrix-sync")
|
||||||
|
} else if age_secs < 120 {
|
||||||
|
HealthLine::warn(
|
||||||
|
"matrix-sync",
|
||||||
|
format!("no events in {age_secs}s"),
|
||||||
|
"check sync loop — may be a quiet room",
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
HealthLine::fail(
|
||||||
|
"matrix-sync",
|
||||||
|
format!("no events in {age_secs}s"),
|
||||||
|
"sync loop may be dead — restart bot",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check LLM credentials (`~/.claude/.credentials.json`).
|
||||||
|
///
|
||||||
|
/// FAIL if the file is missing or unreadable, FAIL if the access token is
|
||||||
|
/// expired, WARN if it expires within the next 7 days.
|
||||||
|
fn check_creds() -> HealthLine {
|
||||||
|
match crate::llm::oauth::read_credentials() {
|
||||||
|
Err(e) => HealthLine::fail("creds", e, "run `claude login`"),
|
||||||
|
Ok(creds) => {
|
||||||
|
let now_secs = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs();
|
||||||
|
let expires_at = creds.claude_ai_oauth.expires_at;
|
||||||
|
if expires_at < now_secs {
|
||||||
|
HealthLine::fail("creds", "token expired", "run `claude login` to refresh")
|
||||||
|
} else {
|
||||||
|
let days_left = (expires_at - now_secs) / 86400;
|
||||||
|
if days_left < 7 {
|
||||||
|
HealthLine::warn(
|
||||||
|
"creds",
|
||||||
|
format!("token expires in {days_left}d"),
|
||||||
|
"run `claude login` to refresh",
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
HealthLine::pass("creds")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compare the compile-time build hash against the current HEAD of the workspace.
|
||||||
|
///
|
||||||
|
/// WARN when master has advanced past the running binary's commit (a rebuild is
|
||||||
|
/// available but not urgent). PASS when hashes match or HEAD cannot be read.
|
||||||
|
async fn check_build_hash(project_root: &std::path::Path) -> HealthLine {
|
||||||
|
let running = option_env!("BUILD_GIT_HASH").unwrap_or("unknown");
|
||||||
|
|
||||||
|
// Read current HEAD from git (non-blocking, run in a spawn_blocking call).
|
||||||
|
let repo_root = project_root.to_path_buf();
|
||||||
|
let head = tokio::task::spawn_blocking(move || {
|
||||||
|
std::process::Command::new("git")
|
||||||
|
.args(["rev-parse", "--short", "HEAD"])
|
||||||
|
.current_dir(&repo_root)
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.filter(|o| o.status.success())
|
||||||
|
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_or(None);
|
||||||
|
|
||||||
|
match head {
|
||||||
|
None => HealthLine::pass("build-hash"),
|
||||||
|
Some(ref head_hash) => {
|
||||||
|
if running == "unknown" || head_hash == running {
|
||||||
|
HealthLine::pass("build-hash")
|
||||||
|
} else {
|
||||||
|
HealthLine::warn(
|
||||||
|
"build-hash",
|
||||||
|
format!("running {running}, HEAD is {head_hash}"),
|
||||||
|
"run `rebuild` to update",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check each registered sled's `/health` endpoint with a 5-second timeout.
|
||||||
|
///
|
||||||
|
/// Returns one [`HealthLine`] per sled. PASS when the sled responds with HTTP
|
||||||
|
/// 2xx; FAIL when the request times out or returns an error status.
|
||||||
|
async fn check_sleds(
|
||||||
|
store: &tokio::sync::RwLock<BTreeMap<String, crate::service::gateway::config::ProjectEntry>>,
|
||||||
|
) -> Vec<HealthLine> {
|
||||||
|
let entries: Vec<(String, Option<String>)> = store
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.iter()
|
||||||
|
.map(|(n, e)| (n.clone(), e.url.clone()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if entries.is_empty() {
|
||||||
|
return vec![HealthLine::warn(
|
||||||
|
"sled",
|
||||||
|
"no sleds registered",
|
||||||
|
"add projects to projects.toml",
|
||||||
|
)];
|
||||||
|
}
|
||||||
|
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let mut lines = Vec::new();
|
||||||
|
|
||||||
|
for (name, url_opt) in entries {
|
||||||
|
let subsystem = format!("sled:{name}");
|
||||||
|
let line = match url_opt {
|
||||||
|
None => HealthLine::warn(subsystem, "no URL configured", "set url in projects.toml"),
|
||||||
|
Some(url) => {
|
||||||
|
let health_url = format!("{}/health", url.trim_end_matches('/'));
|
||||||
|
let result = timeout(Duration::from_secs(5), client.get(&health_url).send()).await;
|
||||||
|
match result {
|
||||||
|
Err(_) => {
|
||||||
|
HealthLine::fail(subsystem, "timed out", "check container is running")
|
||||||
|
}
|
||||||
|
Ok(Err(e)) => HealthLine::fail(
|
||||||
|
subsystem,
|
||||||
|
format!("unreachable: {}", short_error(&e.to_string())),
|
||||||
|
"check container is running",
|
||||||
|
),
|
||||||
|
Ok(Ok(resp)) if resp.status().is_success() => HealthLine::pass(subsystem),
|
||||||
|
Ok(Ok(resp)) => HealthLine::fail(
|
||||||
|
subsystem,
|
||||||
|
format!("HTTP {}", resp.status().as_u16()),
|
||||||
|
"check container logs",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
lines
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check the gateway process: pidfile validity and (on macOS) binary codesign.
|
||||||
|
///
|
||||||
|
/// PASS when our PID is recorded in the pidfile. On macOS, also verifies that
|
||||||
|
/// `~/bin/huskies-bin` has a valid ad-hoc signature; FAIL with a `script/local-release`
|
||||||
|
/// hint when it does not.
|
||||||
|
fn check_gateway_process() -> HealthLine {
|
||||||
|
// Verify that the pidfile records our PID (i.e. this IS the live gateway).
|
||||||
|
let pidfile_ok = check_pidfile_matches_self();
|
||||||
|
|
||||||
|
// On macOS, verify the installed binary is codesigned.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
if !check_codesign_macos() {
|
||||||
|
return HealthLine::fail(
|
||||||
|
"gateway-process",
|
||||||
|
"codesign invalid",
|
||||||
|
"run `script/local-release`",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !pidfile_ok {
|
||||||
|
return HealthLine::warn(
|
||||||
|
"gateway-process",
|
||||||
|
"pidfile missing or stale",
|
||||||
|
"restart gateway with --gateway flag",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
HealthLine::pass("gateway-process")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return `true` when `$HOME/.huskies/gateway.pid` exists and contains our PID.
|
||||||
|
fn check_pidfile_matches_self() -> bool {
|
||||||
|
let home = homedir::my_home().ok().flatten();
|
||||||
|
let home = match home {
|
||||||
|
Some(h) => h,
|
||||||
|
None => return false,
|
||||||
|
};
|
||||||
|
let path = home.join(".huskies").join("gateway.pid");
|
||||||
|
let content = std::fs::read_to_string(&path).unwrap_or_default();
|
||||||
|
content.trim().parse::<u32>().unwrap_or(0) == std::process::id()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// On macOS, return `true` when `~/bin/huskies-bin` passes `codesign --verify`.
|
||||||
|
///
|
||||||
|
/// Falls back to the current executable when `~/bin/huskies-bin` does not exist.
|
||||||
|
/// Returns `true` (assume ok) if the `codesign` tool is unavailable.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
fn check_codesign_macos() -> bool {
|
||||||
|
let target = if let Ok(home) = std::env::var("HOME") {
|
||||||
|
let installed = std::path::PathBuf::from(home)
|
||||||
|
.join("bin")
|
||||||
|
.join("huskies-bin");
|
||||||
|
if installed.exists() {
|
||||||
|
installed
|
||||||
|
} else {
|
||||||
|
match std::env::current_exe() {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_) => return true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
match std::env::current_exe() {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_) => return true,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
std::process::Command::new("codesign")
|
||||||
|
.args(["--verify", "--quiet", target.to_str().unwrap_or("")])
|
||||||
|
.output()
|
||||||
|
.map(|o| o.status.success())
|
||||||
|
.unwrap_or(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Entry point ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Run all health checks and return a formatted Markdown report (≤ 20 lines).
|
||||||
|
///
|
||||||
|
/// Gateway-specific checks (gateway-process, per-sled probes) are included
|
||||||
|
/// only when running in gateway mode. All other checks run in every mode.
|
||||||
|
pub async fn run_health_check(ctx: &BotContext) -> String {
|
||||||
|
let mut lines: Vec<HealthLine> = Vec::new();
|
||||||
|
|
||||||
|
// Gateway-only checks
|
||||||
|
if ctx.is_gateway() {
|
||||||
|
lines.push(check_gateway_process());
|
||||||
|
if let Some(ref store) = ctx.gateway_projects_store {
|
||||||
|
lines.extend(check_sleds(store).await);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shared checks — run concurrently where possible.
|
||||||
|
let perm_line = check_perm_rx(ctx);
|
||||||
|
let sync_line = check_matrix_sync(ctx);
|
||||||
|
let creds_line = check_creds();
|
||||||
|
let hash_line = check_build_hash(&ctx.services.project_root).await;
|
||||||
|
|
||||||
|
lines.push(perm_line);
|
||||||
|
lines.push(sync_line);
|
||||||
|
lines.push(creds_line);
|
||||||
|
lines.push(hash_line);
|
||||||
|
|
||||||
|
let lines = truncate_lines(lines);
|
||||||
|
lines
|
||||||
|
.iter()
|
||||||
|
.map(|l| l.format())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Utilities ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Shorten a long error string to the first 60 characters for compact display.
|
||||||
|
fn short_error(s: &str) -> String {
|
||||||
|
s.chars().take(60).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
// -- HealthLine formatting ------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pass_line_formats_without_detail() {
|
||||||
|
let line = HealthLine::pass("perm_rx");
|
||||||
|
assert_eq!(line.format(), "perm_rx PASS");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fail_line_formats_with_detail_and_hint() {
|
||||||
|
let line = HealthLine::fail(
|
||||||
|
"gateway-process",
|
||||||
|
"codesign invalid",
|
||||||
|
"run script/local-release",
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
line.format(),
|
||||||
|
"gateway-process FAIL: codesign invalid — run script/local-release"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn warn_line_formats_with_detail_and_hint() {
|
||||||
|
let line = HealthLine::warn("build-hash", "running abc, HEAD is def", "run rebuild");
|
||||||
|
assert_eq!(
|
||||||
|
line.format(),
|
||||||
|
"build-hash WARN: running abc, HEAD is def — run rebuild"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Truncation -----------------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_drops_oldest_warn_first() {
|
||||||
|
let mut lines: Vec<HealthLine> = (0..22)
|
||||||
|
.map(|i| {
|
||||||
|
if i % 3 == 0 {
|
||||||
|
HealthLine::fail(format!("sled:{i}"), "down", "fix it")
|
||||||
|
} else {
|
||||||
|
HealthLine::warn(format!("check:{i}"), "slow", "investigate")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Manually insert a known WARN at position 0 and a FAIL at position 1
|
||||||
|
lines.insert(0, HealthLine::warn("oldest-warn", "stale", "restart"));
|
||||||
|
lines.insert(1, HealthLine::fail("important-fail", "broken", "fix"));
|
||||||
|
|
||||||
|
let result = truncate_lines(lines.clone());
|
||||||
|
assert!(
|
||||||
|
result.len() <= MAX_LINES,
|
||||||
|
"output must be ≤ {MAX_LINES} lines"
|
||||||
|
);
|
||||||
|
|
||||||
|
// FAILs must be preserved.
|
||||||
|
let fail_count = result.iter().filter(|l| l.status == Status::Fail).count();
|
||||||
|
let orig_fail_count = lines.iter().filter(|l| l.status == Status::Fail).count();
|
||||||
|
assert_eq!(
|
||||||
|
fail_count,
|
||||||
|
orig_fail_count.min(MAX_LINES),
|
||||||
|
"all FAIL lines must be kept when they fit"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_noop_when_under_limit() {
|
||||||
|
let lines: Vec<HealthLine> = (0..5).map(|i| HealthLine::pass(format!("s{i}"))).collect();
|
||||||
|
let result = truncate_lines(lines.clone());
|
||||||
|
assert_eq!(result.len(), 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncate_stops_at_fails_when_no_warns_left() {
|
||||||
|
// 25 FAIL lines — nothing to drop; output is clamped at MAX_LINES.
|
||||||
|
let lines: Vec<HealthLine> = (0..25)
|
||||||
|
.map(|i| HealthLine::fail(format!("s{i}"), "broken", "fix"))
|
||||||
|
.collect();
|
||||||
|
let result = truncate_lines(lines);
|
||||||
|
// When only FAILs are present, truncation stops because no WARNs can be removed.
|
||||||
|
assert_eq!(result.len(), 25, "FAILs are never dropped by truncation");
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- perm_rx check --------------------------------------------------------
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn perm_rx_pass_when_locked() {
|
||||||
|
use crate::services::Services;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::Mutex as TokioMutex;
|
||||||
|
|
||||||
|
let (perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||||
|
let perm_rx_arc = Arc::new(TokioMutex::new(perm_rx));
|
||||||
|
|
||||||
|
// Acquire the lock to simulate the permission listener holding it.
|
||||||
|
let _guard = perm_rx_arc.try_lock().unwrap();
|
||||||
|
|
||||||
|
// Build a minimal services bundle referencing our locked perm_rx.
|
||||||
|
let services = Arc::new(Services {
|
||||||
|
project_root: std::path::PathBuf::from("/tmp"),
|
||||||
|
agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
|
||||||
|
bot_name: "test".to_string(),
|
||||||
|
bot_user_id: "@bot:test".to_string(),
|
||||||
|
ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||||
|
perm_rx: Arc::clone(&perm_rx_arc),
|
||||||
|
pending_perm_replies: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
|
||||||
|
permission_timeout_secs: 120,
|
||||||
|
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Build a minimal BotContext just to pass services.
|
||||||
|
let ctx = make_test_ctx(services);
|
||||||
|
|
||||||
|
let line = check_perm_rx(&ctx);
|
||||||
|
assert_eq!(
|
||||||
|
line.status,
|
||||||
|
Status::Pass,
|
||||||
|
"perm_rx should PASS when a task holds the lock"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(perm_tx); // suppress unused warning
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn perm_rx_fail_when_unlocked() {
|
||||||
|
use crate::services::Services;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::Mutex as TokioMutex;
|
||||||
|
|
||||||
|
let (_perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||||
|
let perm_rx_arc = Arc::new(TokioMutex::new(perm_rx));
|
||||||
|
// Lock is NOT held by anyone.
|
||||||
|
|
||||||
|
let services = Arc::new(Services {
|
||||||
|
project_root: std::path::PathBuf::from("/tmp"),
|
||||||
|
agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
|
||||||
|
bot_name: "test".to_string(),
|
||||||
|
bot_user_id: "@bot:test".to_string(),
|
||||||
|
ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
|
||||||
|
perm_rx: Arc::clone(&perm_rx_arc),
|
||||||
|
pending_perm_replies: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
|
||||||
|
permission_timeout_secs: 120,
|
||||||
|
status: Arc::new(crate::service::status::StatusBroadcaster::new()),
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
|
});
|
||||||
|
|
||||||
|
let ctx = make_test_ctx(services);
|
||||||
|
|
||||||
|
let line = check_perm_rx(&ctx);
|
||||||
|
assert_eq!(
|
||||||
|
line.status,
|
||||||
|
Status::Fail,
|
||||||
|
"perm_rx should FAIL when no task holds the lock"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- matrix-sync check ----------------------------------------------------
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn matrix_sync_pass_when_recent() {
|
||||||
|
let services = crate::services::Services::new_test(
|
||||||
|
std::path::PathBuf::from("/tmp"),
|
||||||
|
"bot".to_string(),
|
||||||
|
);
|
||||||
|
let ctx = make_test_ctx(services);
|
||||||
|
// Set last event to just now.
|
||||||
|
ctx.last_matrix_event_ms
|
||||||
|
.store(chrono::Utc::now().timestamp_millis(), Ordering::Relaxed);
|
||||||
|
let line = check_matrix_sync(&ctx);
|
||||||
|
assert_eq!(line.status, Status::Pass);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn matrix_sync_fail_when_stale() {
|
||||||
|
let services = crate::services::Services::new_test(
|
||||||
|
std::path::PathBuf::from("/tmp"),
|
||||||
|
"bot".to_string(),
|
||||||
|
);
|
||||||
|
let ctx = make_test_ctx(services);
|
||||||
|
// Simulate 200 seconds of silence.
|
||||||
|
let old_ms = chrono::Utc::now().timestamp_millis() - 200_000;
|
||||||
|
ctx.last_matrix_event_ms.store(old_ms, Ordering::Relaxed);
|
||||||
|
let line = check_matrix_sync(&ctx);
|
||||||
|
assert_eq!(line.status, Status::Fail);
|
||||||
|
assert!(
|
||||||
|
line.detail.as_deref().unwrap_or("").contains("200s")
|
||||||
|
|| line.detail.as_deref().unwrap_or("").contains("s"),
|
||||||
|
"detail should mention age in seconds"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- creds check ----------------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn creds_fail_when_file_missing() {
|
||||||
|
// In the test environment there is unlikely to be a ~/.claude/.credentials.json
|
||||||
|
// with a valid non-expired token, so we just confirm the function returns a
|
||||||
|
// HealthLine without panicking.
|
||||||
|
let line = check_creds();
|
||||||
|
// We don't assert a specific status — the check should not panic.
|
||||||
|
let _ = line.format();
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- build_hash check -----------------------------------------------------
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn build_hash_pass_when_git_unavailable() {
|
||||||
|
// In a test environment without a git repo at /tmp/nonexistent, the check
|
||||||
|
// should gracefully return PASS rather than panicking.
|
||||||
|
let line = check_build_hash(std::path::Path::new("/tmp/nonexistent")).await;
|
||||||
|
// Should either PASS or produce a sensible result — must not panic.
|
||||||
|
let _ = line.format();
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- health command registration ------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn health_command_registered_in_commands() {
|
||||||
|
let cmds = crate::chat::commands::commands();
|
||||||
|
assert!(
|
||||||
|
cmds.iter().any(|c| c.name == "health"),
|
||||||
|
"health must be registered in commands()"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn health_command_has_description() {
|
||||||
|
let cmds = crate::chat::commands::commands();
|
||||||
|
let cmd = cmds.iter().find(|c| c.name == "health").unwrap();
|
||||||
|
assert!(!cmd.description.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- Helper ---------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Build a minimal `BotContext` for testing purposes.
|
||||||
|
fn make_test_ctx(services: std::sync::Arc<crate::services::Services>) -> BotContext {
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicI64;
|
||||||
|
use tokio::sync::Mutex as TokioMutex;
|
||||||
|
|
||||||
|
BotContext {
|
||||||
|
services,
|
||||||
|
matrix_user_id: "@bot:example.com".parse().unwrap(),
|
||||||
|
target_room_ids: vec![],
|
||||||
|
allowed_users: vec![],
|
||||||
|
history: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
|
||||||
|
history_size: 20,
|
||||||
|
bot_sent_event_ids: Arc::new(TokioMutex::new(HashSet::new())),
|
||||||
|
htop_sessions: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
|
||||||
|
transport: Arc::new(crate::chat::transport::whatsapp::WhatsAppTransport::new(
|
||||||
|
"test-phone".to_string(),
|
||||||
|
"test-token".to_string(),
|
||||||
|
"pipeline_notification".to_string(),
|
||||||
|
)),
|
||||||
|
timer_store: Arc::new(crate::service::timer::TimerStore::load(
|
||||||
|
std::path::PathBuf::from("/tmp/timers-health.json"),
|
||||||
|
)),
|
||||||
|
gateway_active_project: None,
|
||||||
|
gateway_projects_store: None,
|
||||||
|
handled_incoming_event_ids: Arc::new(TokioMutex::new(
|
||||||
|
crate::chat::transport::matrix::bot::context::SeenEventIds::new(
|
||||||
|
crate::chat::transport::matrix::bot::context::SEEN_EVENT_IDS_CAP,
|
||||||
|
),
|
||||||
|
)),
|
||||||
|
gateway_port: None,
|
||||||
|
last_matrix_event_ms: Arc::new(AtomicI64::new(chrono::Utc::now().timestamp_millis())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,14 +25,22 @@ pub mod commands;
|
|||||||
pub(crate) mod config;
|
pub(crate) mod config;
|
||||||
/// Story deletion command — handles `!delete` bot commands to remove work items.
|
/// Story deletion command — handles `!delete` bot commands to remove work items.
|
||||||
pub mod delete;
|
pub mod delete;
|
||||||
|
/// `health` chat command — surface gateway, sled, matrix, creds, and build-hash status.
|
||||||
|
pub mod health;
|
||||||
/// htop-style agent monitor command — renders a live process table in Matrix.
|
/// htop-style agent monitor command — renders a live process table in Matrix.
|
||||||
pub mod htop;
|
pub mod htop;
|
||||||
|
/// `new project <name>` chat command — Phase 1 gateway project bootstrap.
|
||||||
|
pub mod new_project;
|
||||||
|
/// `project-rebuild <name>` chat command — rebuild Docker image, swap container, preserve state.
|
||||||
|
pub mod project_rebuild;
|
||||||
/// Rebuild command — triggers a server rebuild/restart via a bot command.
|
/// Rebuild command — triggers a server rebuild/restart via a bot command.
|
||||||
pub mod rebuild;
|
pub mod rebuild;
|
||||||
/// Reset command — handles `!reset` bot commands to restart the server state.
|
/// Reset command — handles `!reset` bot commands to restart the server state.
|
||||||
pub mod reset;
|
pub mod reset;
|
||||||
/// rmtree command — handles `!rmtree` bot commands to remove worktrees.
|
/// rmtree command — handles `!rmtree` bot commands to remove worktrees.
|
||||||
pub mod rmtree;
|
pub mod rmtree;
|
||||||
|
/// `upgrade [<project>]` gateway chat command — streaming per-sled binary upgrade.
|
||||||
|
pub mod sled_upgrade;
|
||||||
/// Start command — handles `!start` bot commands to launch agents on stories.
|
/// Start command — handles `!start` bot commands to launch agents on stories.
|
||||||
pub mod start;
|
pub mod start;
|
||||||
/// Matrix `ChatTransport` implementation wrapping the Matrix SDK client.
|
/// Matrix `ChatTransport` implementation wrapping the Matrix SDK client.
|
||||||
@@ -79,12 +87,18 @@ pub fn spawn_bot(
|
|||||||
services: Arc<Services>,
|
services: Arc<Services>,
|
||||||
shutdown_rx: watch::Receiver<Option<ShutdownReason>>,
|
shutdown_rx: watch::Receiver<Option<ShutdownReason>>,
|
||||||
gateway_active_project: Option<Arc<RwLock<String>>>,
|
gateway_active_project: Option<Arc<RwLock<String>>>,
|
||||||
gateway_projects: Vec<String>,
|
gateway_projects_store: Option<
|
||||||
gateway_project_urls: std::collections::BTreeMap<String, String>,
|
Arc<
|
||||||
|
RwLock<
|
||||||
|
std::collections::BTreeMap<String, crate::service::gateway::config::ProjectEntry>,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
timer_store: Arc<TimerStore>,
|
timer_store: Arc<TimerStore>,
|
||||||
gateway_event_rx: Option<
|
gateway_event_rx: Option<
|
||||||
tokio::sync::broadcast::Receiver<crate::service::gateway::GatewayStatusEvent>,
|
tokio::sync::broadcast::Receiver<crate::service::gateway::GatewayStatusEvent>,
|
||||||
>,
|
>,
|
||||||
|
gateway_port: Option<u16>,
|
||||||
) -> Option<tokio::task::AbortHandle> {
|
) -> Option<tokio::task::AbortHandle> {
|
||||||
let config = match BotConfig::load(project_root) {
|
let config = match BotConfig::load(project_root) {
|
||||||
Some(c) => c,
|
Some(c) => c,
|
||||||
@@ -120,10 +134,10 @@ pub fn spawn_bot(
|
|||||||
watcher_tx,
|
watcher_tx,
|
||||||
shutdown_rx,
|
shutdown_rx,
|
||||||
gateway_active_project,
|
gateway_active_project,
|
||||||
gateway_projects,
|
gateway_projects_store,
|
||||||
gateway_project_urls,
|
|
||||||
timer_store,
|
timer_store,
|
||||||
gateway_event_rx,
|
gateway_event_rx,
|
||||||
|
gateway_port,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,605 @@
|
|||||||
|
//! `project-rebuild <name>` chat command — rebuild Docker image, swap container, preserve state.
|
||||||
|
//!
|
||||||
|
//! Usage: `{bot} project-rebuild <name> [--timeout <secs>] [--force]`
|
||||||
|
//!
|
||||||
|
//! Steps performed:
|
||||||
|
//! 1. Validate the project exists and has a `host_path` configured.
|
||||||
|
//! 2. Check for in-flight coder/merge work (active `claude` processes in the container).
|
||||||
|
//! Wait up to `--timeout` seconds for them to exit. Refuse if still active.
|
||||||
|
//! 3. Build a new Docker image from the project's `Dockerfile.fragment` (if present).
|
||||||
|
//! 4. Stop and remove the old container.
|
||||||
|
//! 5. Start a new container from the fresh image, mounting the same host volume so
|
||||||
|
//! `pipeline.db` and all CRDT state survive untouched.
|
||||||
|
//! 6. Re-register the project in the gateway (same URL — port is preserved).
|
||||||
|
//!
|
||||||
|
//! On success the reply names the new image hash and the new container ID.
|
||||||
|
//! On failure the reply names the step that failed and the recovery path.
|
||||||
|
|
||||||
|
use crate::service::gateway::config::ProjectEntry;
|
||||||
|
use crate::service::gateway::io::save_config;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
/// Default seconds to wait for in-flight work to drain before refusing.
|
||||||
|
const DEFAULT_DRAIN_TIMEOUT_SECS: u64 = 60;
|
||||||
|
|
||||||
|
/// A parsed `project-rebuild <name>` command.
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
pub struct ProjectRebuildCommand {
|
||||||
|
/// Name of the project to rebuild.
|
||||||
|
pub name: String,
|
||||||
|
/// Seconds to wait for agents to drain (0 = skip check).
|
||||||
|
pub drain_timeout_secs: u64,
|
||||||
|
/// If `true`, skip the drain check entirely.
|
||||||
|
pub force: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a `project-rebuild <name> [--timeout <secs>] [--force]` command from a raw
|
||||||
|
/// Matrix message body.
|
||||||
|
///
|
||||||
|
/// Strips the bot mention prefix and checks for the `project-rebuild` keyword.
|
||||||
|
/// Returns `None` when the message is not a project-rebuild command.
|
||||||
|
pub fn extract_project_rebuild_command(
|
||||||
|
message: &str,
|
||||||
|
bot_name: &str,
|
||||||
|
bot_user_id: &str,
|
||||||
|
) -> Option<ProjectRebuildCommand> {
|
||||||
|
let stripped = crate::chat::util::strip_bot_mention(message, bot_name, bot_user_id);
|
||||||
|
let trimmed = stripped
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
|
||||||
|
let rest = if let Some(r) = trimmed.strip_prefix("project-rebuild") {
|
||||||
|
r.trim()
|
||||||
|
} else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parts = rest.split_whitespace();
|
||||||
|
let name = match parts.next() {
|
||||||
|
Some(n) if !n.starts_with("--") => n.to_string(),
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut drain_timeout_secs = DEFAULT_DRAIN_TIMEOUT_SECS;
|
||||||
|
let mut force = false;
|
||||||
|
|
||||||
|
let remaining: Vec<&str> = parts.collect();
|
||||||
|
let mut i = 0;
|
||||||
|
while i < remaining.len() {
|
||||||
|
match remaining[i] {
|
||||||
|
"--timeout" if i + 1 < remaining.len() => {
|
||||||
|
drain_timeout_secs = remaining[i + 1]
|
||||||
|
.parse()
|
||||||
|
.unwrap_or(DEFAULT_DRAIN_TIMEOUT_SECS);
|
||||||
|
i += 2;
|
||||||
|
}
|
||||||
|
"--force" => {
|
||||||
|
force = true;
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(ProjectRebuildCommand {
|
||||||
|
name,
|
||||||
|
drain_timeout_secs,
|
||||||
|
force,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rebuild a project's Docker image, swap the container, and preserve all state.
|
||||||
|
///
|
||||||
|
/// On success returns a message naming the new image hash and container ID.
|
||||||
|
/// On failure returns a message naming the failed step and the recovery path.
|
||||||
|
pub async fn handle_project_rebuild(
|
||||||
|
name: &str,
|
||||||
|
drain_timeout_secs: u64,
|
||||||
|
force: bool,
|
||||||
|
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||||
|
config_dir: &Path,
|
||||||
|
) -> String {
|
||||||
|
// ── 1. Validate project ──────────────────────────────────────────────────
|
||||||
|
let (host_path_str, project_url, ssh_port_opt) = {
|
||||||
|
let projects = projects_store.read().await;
|
||||||
|
let entry = match projects.get(name) {
|
||||||
|
Some(e) => e.clone(),
|
||||||
|
None => {
|
||||||
|
let available: Vec<&String> = projects.keys().collect();
|
||||||
|
return format!(
|
||||||
|
"Project `{name}` not found. Available: {}",
|
||||||
|
available
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match entry.host_path.clone() {
|
||||||
|
Some(p) => (p, entry.url.clone(), entry.ssh_port),
|
||||||
|
None => {
|
||||||
|
return format!(
|
||||||
|
"Project `{name}` has no `host_path` configured — cannot rebuild.\n\
|
||||||
|
Only projects created with `new project --adopt` or `adopt_project` \
|
||||||
|
support the `project-rebuild` command."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let host_path = Path::new(&host_path_str);
|
||||||
|
if !host_path.exists() {
|
||||||
|
return format!(
|
||||||
|
"Host path `{host_path_str}` does not exist on disk — \
|
||||||
|
cannot rebuild project `{name}`."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 2. Drain check ───────────────────────────────────────────────────────
|
||||||
|
let container_name = format!("huskies-{name}");
|
||||||
|
if !force
|
||||||
|
&& drain_timeout_secs > 0
|
||||||
|
&& let Some(err_msg) = wait_for_drain(&container_name, drain_timeout_secs).await
|
||||||
|
{
|
||||||
|
return format!(
|
||||||
|
"Project `{name}` rebuild aborted: {err_msg}\n\
|
||||||
|
Pass `--force` to skip the drain check or `--timeout 0` to not wait."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 3. Build new image ───────────────────────────────────────────────────
|
||||||
|
let stacks_dir = config_dir.join("docker").join("stacks");
|
||||||
|
let (resolved_stack, _warnings) = super::new_project::detect_stack(host_path, &stacks_dir);
|
||||||
|
let base_image = super::new_project::image_for_stack(resolved_stack.as_deref());
|
||||||
|
|
||||||
|
let image = match super::new_project::build_project_image(host_path, &base_image, name).await {
|
||||||
|
Ok(img) => img,
|
||||||
|
Err(e) => {
|
||||||
|
return format!(
|
||||||
|
"Rebuild failed at **image build** step.\n\
|
||||||
|
Error: {e}\n\n\
|
||||||
|
Recovery: fix `.huskies/Dockerfile.fragment` in `{host_path_str}` then retry."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let image_hash = get_image_id(&image)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| "unknown".to_string());
|
||||||
|
let image_short: String = image_hash.chars().take(19).collect();
|
||||||
|
|
||||||
|
// ── 4. Stop and remove old container ────────────────────────────────────
|
||||||
|
if let Err(e) = docker_stop(&container_name).await {
|
||||||
|
crate::slog!("[project-rebuild] stop '{container_name}': {e} (may already be stopped)");
|
||||||
|
}
|
||||||
|
if let Err(e) = docker_rm(&container_name).await {
|
||||||
|
return format!(
|
||||||
|
"Rebuild failed at **container remove** step.\n\
|
||||||
|
Error: {e}\n\n\
|
||||||
|
Recovery: run `docker rm {container_name}` manually then retry."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 5. Start new container ───────────────────────────────────────────────
|
||||||
|
let port = project_url
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|u| u.rsplit(':').next())
|
||||||
|
.and_then(|p| p.parse::<u16>().ok())
|
||||||
|
.unwrap_or(3001);
|
||||||
|
let ssh_port = ssh_port_opt.unwrap_or(2222);
|
||||||
|
|
||||||
|
let home = std::env::var("HOME").unwrap_or_else(|_| "/home/huskies".to_string());
|
||||||
|
let pub_key_path = std::path::PathBuf::from(&home)
|
||||||
|
.join(".huskies")
|
||||||
|
.join(name)
|
||||||
|
.join("id_ed25519.pub");
|
||||||
|
let pubkey = match tokio::fs::read_to_string(&pub_key_path).await {
|
||||||
|
Ok(k) => k.trim().to_string(),
|
||||||
|
Err(e) => {
|
||||||
|
return format!(
|
||||||
|
"Rebuild failed at **SSH key read** step.\n\
|
||||||
|
Error: {e}\n\
|
||||||
|
Expected public key at `{}`.\n\n\
|
||||||
|
Recovery: run `ssh-keygen -t ed25519 -N '' -f {home}/.huskies/{name}/id_ed25519` \
|
||||||
|
then retry.",
|
||||||
|
pub_key_path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let credentials_file = std::path::PathBuf::from(&home)
|
||||||
|
.join(".claude")
|
||||||
|
.join(".credentials.json");
|
||||||
|
let creds_opt = if credentials_file.exists() {
|
||||||
|
Some(credentials_file.as_path())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let (git_user_name, git_user_email) =
|
||||||
|
super::new_project::resolve_git_identity(config_dir).await;
|
||||||
|
|
||||||
|
let mut docker_args = super::new_project::project_docker_run_args(
|
||||||
|
&container_name,
|
||||||
|
port,
|
||||||
|
ssh_port,
|
||||||
|
&pubkey,
|
||||||
|
&git_user_name,
|
||||||
|
&git_user_email,
|
||||||
|
creds_opt,
|
||||||
|
&super::new_project::resolve_gateway_url(),
|
||||||
|
);
|
||||||
|
|
||||||
|
docker_args.push("-v".into());
|
||||||
|
docker_args.push(format!("{host_path_str}:/workspace"));
|
||||||
|
|
||||||
|
let host_ssh_dir = std::path::PathBuf::from(&home).join(".ssh");
|
||||||
|
for key_name in &["id_ed25519", "id_rsa"] {
|
||||||
|
let key_path = host_ssh_dir.join(key_name);
|
||||||
|
if key_path.exists() {
|
||||||
|
docker_args.push("-v".into());
|
||||||
|
docker_args.push(format!(
|
||||||
|
"{}:/home/huskies/.ssh/{key_name}:ro",
|
||||||
|
key_path.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_args.push("--restart".into());
|
||||||
|
docker_args.push("unless-stopped".into());
|
||||||
|
docker_args.push(image.clone());
|
||||||
|
docker_args.push("huskies".into());
|
||||||
|
docker_args.push("/workspace".into());
|
||||||
|
|
||||||
|
let run_output = tokio::process::Command::new("docker")
|
||||||
|
.args(&docker_args)
|
||||||
|
.output()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let container_id = match run_output {
|
||||||
|
Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout).trim().to_string(),
|
||||||
|
Ok(out) => {
|
||||||
|
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
|
||||||
|
return format!(
|
||||||
|
"Rebuild failed at **container start** step.\n\
|
||||||
|
Error: {stderr}\n\n\
|
||||||
|
Recovery: the old container was removed. \
|
||||||
|
Start a new one manually: `docker run -d --name {container_name} ... {image} huskies /workspace`"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
return format!(
|
||||||
|
"Rebuild failed at **container start** step.\n\
|
||||||
|
Error: {e}\n\n\
|
||||||
|
Recovery: start the container manually: \
|
||||||
|
`docker run -d --name {container_name} ... {image} huskies /workspace`"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let container_short: String = container_id.chars().take(12).collect();
|
||||||
|
|
||||||
|
// ── 6. Persist updated config (URL is unchanged; project already registered) ────
|
||||||
|
{
|
||||||
|
let container_url = format!("http://127.0.0.1:{port}");
|
||||||
|
let mut projects = projects_store.write().await;
|
||||||
|
if let Some(entry) = projects.get_mut(name) {
|
||||||
|
entry.url = Some(container_url.clone());
|
||||||
|
}
|
||||||
|
save_config(&projects, config_dir).await;
|
||||||
|
crate::crdt_state::write_gateway_project(name, &container_url);
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::slog!("[project-rebuild] Rebuilt '{name}': image={image_hash} container={container_id}");
|
||||||
|
|
||||||
|
format!(
|
||||||
|
"Project **{name}** rebuilt.\n\
|
||||||
|
- New image: `{image}` (`{image_short}…`)\n\
|
||||||
|
- New container: `{container_name}` (`{container_short}…`)\n\
|
||||||
|
- State: `pipeline.db` and CRDT preserved (same volume bind-mount)\n\
|
||||||
|
- Port: {port} (unchanged)\n\
|
||||||
|
\n\
|
||||||
|
Use `switch {name}` then `status` to verify the pipeline."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait for active Claude agent processes in the container to exit.
|
||||||
|
///
|
||||||
|
/// Polls every 5 seconds until no `claude` processes remain or `timeout_secs` elapses.
|
||||||
|
/// Returns `Some(error_message)` when agents are still running after the timeout,
|
||||||
|
/// `None` when the container is idle or unreachable.
|
||||||
|
async fn wait_for_drain(container_name: &str, timeout_secs: u64) -> Option<String> {
|
||||||
|
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
|
||||||
|
let poll_interval = std::time::Duration::from_secs(5);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match count_active_claude_processes(container_name).await {
|
||||||
|
Ok(0) => return None,
|
||||||
|
Ok(n) => {
|
||||||
|
if std::time::Instant::now() >= deadline {
|
||||||
|
return Some(format!(
|
||||||
|
"{n} Claude agent process(es) still running after {timeout_secs}s drain timeout."
|
||||||
|
));
|
||||||
|
}
|
||||||
|
tokio::time::sleep(poll_interval).await;
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// docker exec failed (container stopped or Docker unavailable) — proceed.
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count the number of active `claude` processes inside the given container.
|
||||||
|
///
|
||||||
|
/// Uses `docker exec <name> pgrep -f claude` — exits 0 with PID list when found,
|
||||||
|
/// exits 1 when no matches (treated as 0 active processes).
|
||||||
|
async fn count_active_claude_processes(container_name: &str) -> Result<usize, String> {
|
||||||
|
let out = tokio::process::Command::new("docker")
|
||||||
|
.args(["exec", container_name, "pgrep", "-f", "claude"])
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
if out.status.success() {
|
||||||
|
let count = String::from_utf8_lossy(&out.stdout)
|
||||||
|
.lines()
|
||||||
|
.filter(|l| !l.trim().is_empty())
|
||||||
|
.count();
|
||||||
|
Ok(count)
|
||||||
|
} else {
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop a running Docker container (`docker stop`).
|
||||||
|
async fn docker_stop(container_name: &str) -> Result<(), String> {
|
||||||
|
let out = tokio::process::Command::new("docker")
|
||||||
|
.args(["stop", container_name])
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("docker stop failed to spawn: {e}"))?;
|
||||||
|
|
||||||
|
if out.status.success() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove a stopped Docker container (`docker rm`).
|
||||||
|
async fn docker_rm(container_name: &str) -> Result<(), String> {
|
||||||
|
let out = tokio::process::Command::new("docker")
|
||||||
|
.args(["rm", container_name])
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("docker rm failed to spawn: {e}"))?;
|
||||||
|
|
||||||
|
if out.status.success() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the full image ID (sha256 digest) for a named Docker image.
|
||||||
|
async fn get_image_id(image_name: &str) -> Result<String, String> {
|
||||||
|
let out = tokio::process::Command::new("docker")
|
||||||
|
.args(["inspect", image_name, "--format", "{{.Id}}"])
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("docker inspect failed: {e}"))?;
|
||||||
|
|
||||||
|
if out.status.success() {
|
||||||
|
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
|
||||||
|
} else {
|
||||||
|
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::service::gateway::config::ProjectEntry;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
fn make_store(
|
||||||
|
projects: Vec<(&str, ProjectEntry)>,
|
||||||
|
) -> Arc<RwLock<BTreeMap<String, ProjectEntry>>> {
|
||||||
|
let map: BTreeMap<String, ProjectEntry> = projects
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, v)| (k.to_string(), v))
|
||||||
|
.collect();
|
||||||
|
Arc::new(RwLock::new(map))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── parsing ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_basic_command() {
|
||||||
|
let cmd =
|
||||||
|
extract_project_rebuild_command("Timmy project-rebuild myapp", "Timmy", "@timmy:home");
|
||||||
|
let cmd = cmd.unwrap();
|
||||||
|
assert_eq!(cmd.name, "myapp");
|
||||||
|
assert_eq!(cmd.drain_timeout_secs, DEFAULT_DRAIN_TIMEOUT_SECS);
|
||||||
|
assert!(!cmd.force);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_with_force_flag() {
|
||||||
|
let cmd = extract_project_rebuild_command(
|
||||||
|
"@timmy project-rebuild myapp --force",
|
||||||
|
"Timmy",
|
||||||
|
"@timmy:home",
|
||||||
|
);
|
||||||
|
let cmd = cmd.unwrap();
|
||||||
|
assert_eq!(cmd.name, "myapp");
|
||||||
|
assert!(cmd.force);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_with_timeout_flag() {
|
||||||
|
let cmd = extract_project_rebuild_command(
|
||||||
|
"Timmy project-rebuild myapp --timeout 120",
|
||||||
|
"Timmy",
|
||||||
|
"@timmy:home",
|
||||||
|
);
|
||||||
|
let cmd = cmd.unwrap();
|
||||||
|
assert_eq!(cmd.name, "myapp");
|
||||||
|
assert_eq!(cmd.drain_timeout_secs, 120);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_with_timeout_zero_skips_drain() {
|
||||||
|
let cmd = extract_project_rebuild_command(
|
||||||
|
"Timmy project-rebuild myapp --timeout 0",
|
||||||
|
"Timmy",
|
||||||
|
"@timmy:home",
|
||||||
|
);
|
||||||
|
let cmd = cmd.unwrap();
|
||||||
|
assert_eq!(cmd.drain_timeout_secs, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_non_rebuild_returns_none() {
|
||||||
|
let cmd = extract_project_rebuild_command("Timmy status", "Timmy", "@timmy:home");
|
||||||
|
assert!(cmd.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_rebuild_without_name_returns_none() {
|
||||||
|
let cmd = extract_project_rebuild_command("Timmy project-rebuild", "Timmy", "@timmy:home");
|
||||||
|
assert!(cmd.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_with_full_user_id() {
|
||||||
|
let cmd = extract_project_rebuild_command(
|
||||||
|
"@timmy:home project-rebuild alpha",
|
||||||
|
"Timmy",
|
||||||
|
"@timmy:home",
|
||||||
|
);
|
||||||
|
assert_eq!(cmd.unwrap().name, "alpha");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_case_insensitive_bot_mention() {
|
||||||
|
let cmd =
|
||||||
|
extract_project_rebuild_command("timmy project-rebuild beta", "Timmy", "@timmy:home");
|
||||||
|
assert_eq!(cmd.unwrap().name, "beta");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── handle_project_rebuild validation ─────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rebuild_unknown_project_returns_error() {
|
||||||
|
let store = make_store(vec![]);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let result = handle_project_rebuild("nonexistent", 0, true, &store, dir.path()).await;
|
||||||
|
assert!(
|
||||||
|
result.contains("not found"),
|
||||||
|
"expected 'not found': {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rebuild_project_without_host_path_returns_error() {
|
||||||
|
let store = make_store(vec![(
|
||||||
|
"myapp",
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://127.0.0.1:3101".into()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: Some(2201),
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
)]);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let result = handle_project_rebuild("myapp", 0, true, &store, dir.path()).await;
|
||||||
|
assert!(
|
||||||
|
result.contains("host_path"),
|
||||||
|
"expected 'host_path' mention: {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rebuild_project_with_missing_host_dir_returns_error() {
|
||||||
|
let store = make_store(vec![(
|
||||||
|
"myapp",
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://127.0.0.1:3101".into()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: Some(2201),
|
||||||
|
host_path: Some("/nonexistent/path/xyz123".into()),
|
||||||
|
},
|
||||||
|
)]);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let result = handle_project_rebuild("myapp", 0, true, &store, dir.path()).await;
|
||||||
|
assert!(
|
||||||
|
result.contains("does not exist"),
|
||||||
|
"expected 'does not exist': {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// End-to-end flow test: rebuild a project that has a valid host directory.
|
||||||
|
///
|
||||||
|
/// With `--force` and `--timeout 0` the drain check is skipped.
|
||||||
|
/// The function proceeds to the image build step, which fails when Docker is
|
||||||
|
/// not available in CI. On failure the reply must:
|
||||||
|
/// (a) name the failed step ("image build")
|
||||||
|
/// (b) leave the project still registered in the gateway (state preserved)
|
||||||
|
/// (c) include a recovery path
|
||||||
|
///
|
||||||
|
/// When Docker IS available and the base image exists this test would exercise
|
||||||
|
/// the full container stop → build → start → re-register flow.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rebuild_e2e_with_valid_host_path_reaches_image_build_step() {
|
||||||
|
let host_dir = tempfile::tempdir().unwrap();
|
||||||
|
// Create a minimal .huskies/ directory (simulating an existing project).
|
||||||
|
std::fs::create_dir_all(host_dir.path().join(".huskies")).unwrap();
|
||||||
|
|
||||||
|
let store = make_store(vec![(
|
||||||
|
"myapp",
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://127.0.0.1:3101".into()),
|
||||||
|
auth_token: Some("tok".into()),
|
||||||
|
ssh_port: Some(2201),
|
||||||
|
host_path: Some(host_dir.path().to_str().unwrap().to_string()),
|
||||||
|
},
|
||||||
|
)]);
|
||||||
|
let config_dir = tempfile::tempdir().unwrap();
|
||||||
|
|
||||||
|
let result = handle_project_rebuild("myapp", 0, true, &store, config_dir.path()).await;
|
||||||
|
|
||||||
|
// (a) Step naming: one of several possible failure steps depending on what Docker
|
||||||
|
// binaries are available in the test environment, or a success reply.
|
||||||
|
let names_a_step = result.contains("image build")
|
||||||
|
|| result.contains("SSH key")
|
||||||
|
|| result.contains("container remove")
|
||||||
|
|| result.contains("container start");
|
||||||
|
let is_success = result.contains("rebuilt");
|
||||||
|
assert!(
|
||||||
|
names_a_step || is_success,
|
||||||
|
"result should name a step or report success: {result}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// (b) State preserved: project is still registered in the gateway store.
|
||||||
|
let projects = store.read().await;
|
||||||
|
assert!(
|
||||||
|
projects.contains_key("myapp"),
|
||||||
|
"project 'myapp' must remain registered after failed rebuild: {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,43 @@ pub fn extract_rebuild_command(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Parse a "rebuild gateway" command from a raw message body.
|
||||||
|
///
|
||||||
|
/// Returns `Some(RebuildCommand)` only when the stripped message begins with
|
||||||
|
/// "rebuild gateway" (case-insensitive). A plain "rebuild" without the
|
||||||
|
/// "gateway" qualifier returns `None` so it falls through to the standard
|
||||||
|
/// server rebuild handler.
|
||||||
|
pub fn extract_rebuild_gateway_command(
|
||||||
|
message: &str,
|
||||||
|
bot_name: &str,
|
||||||
|
bot_user_id: &str,
|
||||||
|
) -> Option<RebuildCommand> {
|
||||||
|
let stripped = strip_bot_mention(message, bot_name, bot_user_id);
|
||||||
|
let trimmed = stripped
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
|
||||||
|
let (cmd, rest) = trimmed.split_once(char::is_whitespace)?;
|
||||||
|
|
||||||
|
if !cmd.eq_ignore_ascii_case("rebuild") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let qualifier = rest
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
let first_word = match qualifier.split_once(char::is_whitespace) {
|
||||||
|
Some((w, _)) => w,
|
||||||
|
None => qualifier,
|
||||||
|
};
|
||||||
|
|
||||||
|
if first_word.eq_ignore_ascii_case("gateway") {
|
||||||
|
Some(RebuildCommand)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Handle a rebuild command: trigger server rebuild and restart.
|
/// Handle a rebuild command: trigger server rebuild and restart.
|
||||||
///
|
///
|
||||||
/// Returns a string describing the outcome. On build failure the error
|
/// Returns a string describing the outcome. On build failure the error
|
||||||
|
|||||||
@@ -0,0 +1,478 @@
|
|||||||
|
//! `upgrade [<project>]` gateway chat command — streaming sled binary upgrade.
|
||||||
|
//!
|
||||||
|
//! Usage (gateway mode only):
|
||||||
|
//! - `{bot} upgrade <project>` — upgrade the named sled's binary in-container.
|
||||||
|
//! - `{bot} upgrade` — list registered projects (shows what can be targeted).
|
||||||
|
//!
|
||||||
|
//! The gateway orchestrates the upgrade in four phases, streaming a marker to
|
||||||
|
//! the chat room at each step:
|
||||||
|
//! 1. `[1/4] downloading` — POSTs to `{sled_url}/api/upgrade`; sled starts download.
|
||||||
|
//! 2. `[2/4] swapping binary` — gateway received 202; sled atomically renamed the binary.
|
||||||
|
//! 3. `[3/4] restarting sled` — sled re-execs with the new binary; HTTP goes dark briefly.
|
||||||
|
//! 4. `[4/4] reconnected to gateway` — sled's `/health` probe is responding again.
|
||||||
|
//!
|
||||||
|
//! Concurrent `upgrade` invocations are serialised via a global async mutex so
|
||||||
|
//! that two simultaneous upgrades cannot interleave their phase markers or race
|
||||||
|
//! on the sled restart.
|
||||||
|
|
||||||
|
use crate::service::gateway::config::ProjectEntry;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::{Arc, OnceLock};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::sync::{Mutex, RwLock};
|
||||||
|
|
||||||
|
// ── Serial lock ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
static UPGRADE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||||
|
|
||||||
|
fn upgrade_lock() -> &'static Mutex<()> {
|
||||||
|
UPGRADE_LOCK.get_or_init(|| Mutex::new(()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Command parsing ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// A parsed `upgrade` command.
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
pub enum UpgradeCommand {
|
||||||
|
/// `upgrade <project>` — upgrade the named sled.
|
||||||
|
Upgrade {
|
||||||
|
/// The project/sled name to upgrade.
|
||||||
|
project: String,
|
||||||
|
},
|
||||||
|
/// `upgrade` with no argument — list available projects.
|
||||||
|
ListProjects,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse an `upgrade [<project>]` command from a raw message body.
|
||||||
|
///
|
||||||
|
/// Strips the bot mention prefix and checks whether the first word is `upgrade`.
|
||||||
|
/// Returns `None` when the message is not an upgrade command.
|
||||||
|
pub fn extract_upgrade_command(
|
||||||
|
message: &str,
|
||||||
|
bot_name: &str,
|
||||||
|
bot_user_id: &str,
|
||||||
|
) -> Option<UpgradeCommand> {
|
||||||
|
let stripped = crate::chat::util::strip_bot_mention(message, bot_name, bot_user_id);
|
||||||
|
let trimmed = stripped
|
||||||
|
.trim()
|
||||||
|
.trim_start_matches(|c: char| !c.is_alphanumeric());
|
||||||
|
|
||||||
|
let (cmd, rest) = match trimmed.split_once(char::is_whitespace) {
|
||||||
|
Some((c, r)) => (c, r.trim()),
|
||||||
|
None => (trimmed, ""),
|
||||||
|
};
|
||||||
|
|
||||||
|
if !cmd.eq_ignore_ascii_case("upgrade") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if rest.is_empty() {
|
||||||
|
Some(UpgradeCommand::ListProjects)
|
||||||
|
} else {
|
||||||
|
Some(UpgradeCommand::Upgrade {
|
||||||
|
project: rest.split_whitespace().next().unwrap_or(rest).to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Handlers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// List available projects when `upgrade` is invoked without an argument.
|
||||||
|
///
|
||||||
|
/// Returns a Markdown string enumerating the registered project names so the
|
||||||
|
/// user knows which targets are valid for `upgrade <project>`.
|
||||||
|
pub async fn handle_upgrade_list_projects(
|
||||||
|
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||||
|
) -> String {
|
||||||
|
let projects = projects_store.read().await;
|
||||||
|
if projects.is_empty() {
|
||||||
|
return "No projects are currently registered with the gateway.".to_string();
|
||||||
|
}
|
||||||
|
let names: Vec<&String> = projects.keys().collect();
|
||||||
|
let list = names
|
||||||
|
.iter()
|
||||||
|
.map(|n| format!("- `{n}`"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
format!("Registered projects (use `upgrade <project>` to upgrade one):\n{list}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Upgrade a named sled by streaming phase markers to the chat room.
|
||||||
|
///
|
||||||
|
/// Acquires the global upgrade lock to serialise concurrent invocations. Each
|
||||||
|
/// phase is announced by calling `send_phase` before the corresponding work
|
||||||
|
/// begins. On any failure, an error message is returned and the previous
|
||||||
|
/// binary remains active on the sled.
|
||||||
|
///
|
||||||
|
/// `gateway_port` is used to derive the default binary source URL
|
||||||
|
/// (`http://gateway:<port>/api/huskies-binary`) when neither
|
||||||
|
/// `HUSKIES_GATEWAY_BINARY_URL` nor `--source` is set.
|
||||||
|
pub async fn handle_sled_upgrade<F, Fut>(
|
||||||
|
project: &str,
|
||||||
|
projects_store: &Arc<RwLock<BTreeMap<String, ProjectEntry>>>,
|
||||||
|
gateway_port: Option<u16>,
|
||||||
|
send_phase: F,
|
||||||
|
) -> String
|
||||||
|
where
|
||||||
|
F: Fn(String) -> Fut,
|
||||||
|
Fut: Future<Output = ()>,
|
||||||
|
{
|
||||||
|
// ── Look up project URL ──────────────────────────────────────────────────
|
||||||
|
let sled_url = {
|
||||||
|
let projects = projects_store.read().await;
|
||||||
|
match projects.get(project).and_then(|e| e.url.clone()) {
|
||||||
|
Some(u) => u,
|
||||||
|
None => {
|
||||||
|
let available: Vec<&String> = projects.keys().collect();
|
||||||
|
return format!(
|
||||||
|
"Project `{project}` not found. Registered projects: {}",
|
||||||
|
available
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── Resolve binary source URL ────────────────────────────────────────────
|
||||||
|
let source_url = std::env::var("HUSKIES_GATEWAY_BINARY_URL").unwrap_or_else(|_| {
|
||||||
|
format!(
|
||||||
|
"http://gateway:{}/api/huskies-binary",
|
||||||
|
gateway_port.unwrap_or(3000)
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Acquire serial lock ──────────────────────────────────────────────────
|
||||||
|
let _lock = upgrade_lock().lock().await;
|
||||||
|
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// ── Phase 1: downloading ─────────────────────────────────────────────────
|
||||||
|
send_phase("[1/4] downloading\u{2026}".to_string()).await;
|
||||||
|
|
||||||
|
let upgrade_url = format!("{}/api/upgrade", sled_url.trim_end_matches('/'));
|
||||||
|
let body = serde_json::json!({ "source_url": source_url });
|
||||||
|
|
||||||
|
let resp = match client.post(&upgrade_url).json(&body).send().await {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
return format!(
|
||||||
|
"Upgrade failed at **[1/4] downloading**: could not reach sled at `{upgrade_url}`.\n\
|
||||||
|
Error: {e}\n\n\
|
||||||
|
The previous version remains active."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !resp.status().is_success() && resp.status().as_u16() != 202 {
|
||||||
|
let status = resp.status();
|
||||||
|
let body_text = resp.text().await.unwrap_or_default();
|
||||||
|
return format!(
|
||||||
|
"Upgrade failed at **[1/4] downloading**: sled returned HTTP {status}.\n\
|
||||||
|
Response: {body_text}\n\n\
|
||||||
|
The previous version remains active."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Phase 2: swapping binary ─────────────────────────────────────────────
|
||||||
|
// The sled accepted the request (202) and is downloading + atomically
|
||||||
|
// replacing the binary in the background.
|
||||||
|
send_phase("[2/4] swapping binary\u{2026}".to_string()).await;
|
||||||
|
|
||||||
|
// ── Phase 3: restarting sled ─────────────────────────────────────────────
|
||||||
|
// The sled will re-exec momentarily; announce before the health loop.
|
||||||
|
send_phase("[3/4] restarting sled\u{2026}".to_string()).await;
|
||||||
|
|
||||||
|
// ── Wait for sled to come back up ────────────────────────────────────────
|
||||||
|
let health_url = format!("{}/health", sled_url.trim_end_matches('/'));
|
||||||
|
// Give the sled a few seconds to start the download + re-exec before polling.
|
||||||
|
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||||
|
|
||||||
|
let reconnected = wait_for_health(&client, &health_url, 120).await;
|
||||||
|
if !reconnected {
|
||||||
|
return format!(
|
||||||
|
"Upgrade failed at **[4/4] reconnected to gateway**: sled at `{sled_url}` did not \
|
||||||
|
come back online within 120 seconds after the upgrade was triggered.\n\n\
|
||||||
|
Check the container logs: `docker logs huskies-{project}`"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Phase 4: reconnected ─────────────────────────────────────────────────
|
||||||
|
send_phase("[4/4] reconnected to gateway".to_string()).await;
|
||||||
|
|
||||||
|
// ── Report new version ───────────────────────────────────────────────────
|
||||||
|
let version = fetch_sled_version(&client, &sled_url).await;
|
||||||
|
format!("{project} upgraded to version {version}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Poll `GET {health_url}` every 3 seconds until it returns 200 or `timeout_secs` elapses.
|
||||||
|
///
|
||||||
|
/// Returns `true` when the probe succeeds, `false` on timeout.
|
||||||
|
async fn wait_for_health(client: &reqwest::Client, health_url: &str, timeout_secs: u64) -> bool {
|
||||||
|
let deadline = std::time::Instant::now() + Duration::from_secs(timeout_secs);
|
||||||
|
let poll = Duration::from_secs(3);
|
||||||
|
loop {
|
||||||
|
match client.get(health_url).send().await {
|
||||||
|
Ok(r) if r.status().is_success() => return true,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
if std::time::Instant::now() >= deadline {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(poll).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch the running version from the sled's `get_version` MCP tool.
|
||||||
|
///
|
||||||
|
/// Returns the version string on success, or `"unknown"` on any error so the
|
||||||
|
/// final chat reply is still meaningful.
|
||||||
|
async fn fetch_sled_version(client: &reqwest::Client, sled_url: &str) -> String {
|
||||||
|
let mcp_url = format!("{}/mcp", sled_url.trim_end_matches('/'));
|
||||||
|
let body = serde_json::json!({
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": 1,
|
||||||
|
"method": "tools/call",
|
||||||
|
"params": {
|
||||||
|
"name": "get_version",
|
||||||
|
"arguments": {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let resp = match client.post(&mcp_url).json(&body).send().await {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(_) => return "unknown".to_string(),
|
||||||
|
};
|
||||||
|
let val: serde_json::Value = match resp.json().await {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => return "unknown".to_string(),
|
||||||
|
};
|
||||||
|
// MCP tools/call response: result.content[0].text is a JSON string.
|
||||||
|
let text = val
|
||||||
|
.pointer("/result/content/0/text")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("");
|
||||||
|
if text.is_empty() {
|
||||||
|
return "unknown".to_string();
|
||||||
|
}
|
||||||
|
serde_json::from_str::<serde_json::Value>(text)
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.get("version").and_then(|v| v.as_str()).map(String::from))
|
||||||
|
.unwrap_or_else(|| "unknown".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tests ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
// ── extract_upgrade_command ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_upgrade_with_project() {
|
||||||
|
let cmd = extract_upgrade_command("Timmy upgrade huskies-server", "Timmy", "@timmy:home");
|
||||||
|
assert_eq!(
|
||||||
|
cmd,
|
||||||
|
Some(UpgradeCommand::Upgrade {
|
||||||
|
project: "huskies-server".to_string()
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_upgrade_no_arg_is_list() {
|
||||||
|
let cmd = extract_upgrade_command("Timmy upgrade", "Timmy", "@timmy:home");
|
||||||
|
assert_eq!(cmd, Some(UpgradeCommand::ListProjects));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_upgrade_with_full_user_id() {
|
||||||
|
let cmd = extract_upgrade_command("@timmy:home upgrade myapp", "Timmy", "@timmy:home");
|
||||||
|
assert_eq!(
|
||||||
|
cmd,
|
||||||
|
Some(UpgradeCommand::Upgrade {
|
||||||
|
project: "myapp".to_string()
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_non_upgrade_returns_none() {
|
||||||
|
let cmd = extract_upgrade_command("Timmy status", "Timmy", "@timmy:home");
|
||||||
|
assert!(cmd.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_upgrade_case_insensitive() {
|
||||||
|
let cmd = extract_upgrade_command("Timmy UPGRADE alpha", "Timmy", "@timmy:home");
|
||||||
|
assert_eq!(
|
||||||
|
cmd,
|
||||||
|
Some(UpgradeCommand::Upgrade {
|
||||||
|
project: "alpha".to_string()
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── handle_upgrade_list_projects ─────────────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_projects_empty_store() {
|
||||||
|
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
|
||||||
|
Arc::new(RwLock::new(BTreeMap::new()));
|
||||||
|
let msg = handle_upgrade_list_projects(&store).await;
|
||||||
|
assert!(
|
||||||
|
msg.contains("No projects"),
|
||||||
|
"empty store should say no projects: {msg}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_projects_shows_names() {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
let mut map = BTreeMap::new();
|
||||||
|
map.insert(
|
||||||
|
"alpha".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://localhost:3001".into()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
map.insert(
|
||||||
|
"beta".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://localhost:3002".into()),
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = Arc::new(RwLock::new(map));
|
||||||
|
let msg = handle_upgrade_list_projects(&store).await;
|
||||||
|
assert!(msg.contains("alpha"), "should list alpha: {msg}");
|
||||||
|
assert!(msg.contains("beta"), "should list beta: {msg}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── handle_sled_upgrade validation ───────────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_unknown_project_returns_error() {
|
||||||
|
let store: Arc<RwLock<BTreeMap<String, ProjectEntry>>> =
|
||||||
|
Arc::new(RwLock::new(BTreeMap::new()));
|
||||||
|
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
|
||||||
|
let result = handle_sled_upgrade("nonexistent", &store, Some(3000), |msg| {
|
||||||
|
phases.lock().unwrap().push(msg);
|
||||||
|
async {}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
result.contains("not found"),
|
||||||
|
"should say not found: {result}"
|
||||||
|
);
|
||||||
|
// No phase markers should have been emitted before the validation error.
|
||||||
|
assert!(
|
||||||
|
phases.lock().unwrap().is_empty(),
|
||||||
|
"no phases should be emitted for unknown project"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_project_with_no_url_fails_gracefully() {
|
||||||
|
let mut map = BTreeMap::new();
|
||||||
|
map.insert(
|
||||||
|
"myapp".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: None,
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = Arc::new(RwLock::new(map));
|
||||||
|
let result = handle_sled_upgrade("myapp", &store, Some(3000), |_msg| async {}).await;
|
||||||
|
assert!(
|
||||||
|
result.contains("not found"),
|
||||||
|
"project with no URL should say not found: {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_unreachable_sled_reports_failure() {
|
||||||
|
let mut map = BTreeMap::new();
|
||||||
|
map.insert(
|
||||||
|
"myapp".to_string(),
|
||||||
|
ProjectEntry {
|
||||||
|
url: Some("http://127.0.0.1:1".into()), // port 1 is never listening
|
||||||
|
auth_token: None,
|
||||||
|
ssh_port: None,
|
||||||
|
host_path: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = Arc::new(RwLock::new(map));
|
||||||
|
let phases: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(vec![]);
|
||||||
|
let result = handle_sled_upgrade("myapp", &store, Some(3000), |msg| {
|
||||||
|
phases.lock().unwrap().push(msg);
|
||||||
|
async {}
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
// Phase 1 marker must have been sent before the failed request.
|
||||||
|
let sent = phases.lock().unwrap().clone();
|
||||||
|
assert!(
|
||||||
|
sent.iter().any(|m| m.contains("[1/4]")),
|
||||||
|
"phase 1 marker must be sent: {sent:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
result.contains("downloading") || result.contains("reach"),
|
||||||
|
"error should mention the failure: {result}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
result.contains("previous version"),
|
||||||
|
"error should confirm old version is active: {result}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── wait_for_health ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_for_health_immediate_success() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let port = listener.local_addr().unwrap().port();
|
||||||
|
|
||||||
|
let handle = tokio::spawn(async move {
|
||||||
|
if let Ok((mut stream, _)) = listener.accept().await {
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
let mut buf = [0u8; 4096];
|
||||||
|
let _ = tokio::io::AsyncReadExt::read(&mut stream, &mut buf).await;
|
||||||
|
let _ = stream
|
||||||
|
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let url = format!("http://127.0.0.1:{port}/health");
|
||||||
|
let ok = wait_for_health(&client, &url, 5).await;
|
||||||
|
assert!(ok, "should return true when health probe succeeds");
|
||||||
|
handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_for_health_timeout() {
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(Duration::from_millis(100))
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
// Nothing listening on port 1.
|
||||||
|
let ok = wait_for_health(&client, "http://127.0.0.1:1/health", 1).await;
|
||||||
|
assert!(!ok, "should return false when health probe never succeeds");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,8 +29,10 @@ pub(super) async fn handle_llm_message(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let bot_name = &ctx.services.bot_name;
|
let bot_name = &ctx.services.bot_name;
|
||||||
|
let persona = bot_name.to_lowercase();
|
||||||
|
let event_ctx = crate::llm_session::assemble_prompt_context(&persona);
|
||||||
let prompt = format!(
|
let prompt = format!(
|
||||||
"[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{user}: {user_message}"
|
"{event_ctx}[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{user}: {user_message}"
|
||||||
);
|
);
|
||||||
|
|
||||||
let provider = ClaudeCodeProvider::new();
|
let provider = ClaudeCodeProvider::new();
|
||||||
|
|||||||
@@ -27,8 +27,10 @@ pub(super) async fn handle_llm_message(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let bot_name = &ctx.services.bot_name;
|
let bot_name = &ctx.services.bot_name;
|
||||||
|
let persona = bot_name.to_lowercase();
|
||||||
|
let event_ctx = crate::llm_session::assemble_prompt_context(&persona);
|
||||||
let prompt = format!(
|
let prompt = format!(
|
||||||
"[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{sender}: {user_message}"
|
"{event_ctx}[Your name is {bot_name}. Refer to yourself as {bot_name}, not Claude.]\n\n{sender}: {user_message}"
|
||||||
);
|
);
|
||||||
|
|
||||||
let provider = ClaudeCodeProvider::new();
|
let provider = ClaudeCodeProvider::new();
|
||||||
|
|||||||
@@ -310,6 +310,7 @@ mod tests {
|
|||||||
perm_rx: Arc::new(tokio::sync::Mutex::new(perm_rx)),
|
perm_rx: Arc::new(tokio::sync::Mutex::new(perm_rx)),
|
||||||
pending_perm_replies: Arc::new(tokio::sync::Mutex::new(Default::default())),
|
pending_perm_replies: Arc::new(tokio::sync::Mutex::new(Default::default())),
|
||||||
permission_timeout_secs: 120,
|
permission_timeout_secs: 120,
|
||||||
|
chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
|
||||||
});
|
});
|
||||||
Arc::new(WhatsAppWebhookContext {
|
Arc::new(WhatsAppWebhookContext {
|
||||||
services,
|
services,
|
||||||
|
|||||||
@@ -27,6 +27,19 @@ pub(crate) struct CliArgs {
|
|||||||
/// forwards all `prompt_permission` tool calls to the gateway over a WebSocket.
|
/// forwards all `prompt_permission` tool calls to the gateway over a WebSocket.
|
||||||
/// Also readable from the `HUSKIES_UPSTREAM_GATEWAY` env var.
|
/// Also readable from the `HUSKIES_UPSTREAM_GATEWAY` env var.
|
||||||
pub(crate) upstream_gateway: Option<String>,
|
pub(crate) upstream_gateway: Option<String>,
|
||||||
|
/// Whether the `upgrade` subcommand was given.
|
||||||
|
pub(crate) upgrade: bool,
|
||||||
|
/// Source URL for the `upgrade` subcommand (`--source <URL>`).
|
||||||
|
///
|
||||||
|
/// If omitted, the upgrade subcommand falls back to
|
||||||
|
/// `HUSKIES_BINARY_SOURCE` env var, then derives the URL from
|
||||||
|
/// `HUSKIES_UPSTREAM_GATEWAY`.
|
||||||
|
pub(crate) upgrade_source: Option<String>,
|
||||||
|
/// Path to a trampoline job file (`--trampoline <path>`).
|
||||||
|
///
|
||||||
|
/// When set, the binary runs as a detached trampoline helper: it kills the
|
||||||
|
/// old gateway, starts the new one, polls its health, and rolls back on failure.
|
||||||
|
pub(crate) trampoline: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse CLI arguments into `CliArgs`, or exit early for `--help` / `--version`.
|
/// Parse CLI arguments into `CliArgs`, or exit early for `--help` / `--version`.
|
||||||
@@ -41,6 +54,9 @@ pub(crate) fn parse_cli_args(args: &[String]) -> Result<CliArgs, String> {
|
|||||||
let mut join_token: Option<String> = None;
|
let mut join_token: Option<String> = None;
|
||||||
let mut gateway_url: Option<String> = None;
|
let mut gateway_url: Option<String> = None;
|
||||||
let mut upstream_gateway: Option<String> = None;
|
let mut upstream_gateway: Option<String> = None;
|
||||||
|
let mut upgrade = false;
|
||||||
|
let mut upgrade_source: Option<String> = None;
|
||||||
|
let mut trampoline: Option<String> = None;
|
||||||
let mut i = 0;
|
let mut i = 0;
|
||||||
|
|
||||||
while i < args.len() {
|
while i < args.len() {
|
||||||
@@ -120,6 +136,29 @@ pub(crate) fn parse_cli_args(args: &[String]) -> Result<CliArgs, String> {
|
|||||||
"agent" => {
|
"agent" => {
|
||||||
agent = true;
|
agent = true;
|
||||||
}
|
}
|
||||||
|
"upgrade" => {
|
||||||
|
upgrade = true;
|
||||||
|
}
|
||||||
|
"--source" => {
|
||||||
|
i += 1;
|
||||||
|
if i >= args.len() {
|
||||||
|
return Err("--source requires a value".to_string());
|
||||||
|
}
|
||||||
|
upgrade_source = Some(args[i].clone());
|
||||||
|
}
|
||||||
|
a if a.starts_with("--source=") => {
|
||||||
|
upgrade_source = Some(a["--source=".len()..].to_string());
|
||||||
|
}
|
||||||
|
"--trampoline" => {
|
||||||
|
i += 1;
|
||||||
|
if i >= args.len() {
|
||||||
|
return Err("--trampoline requires a path".to_string());
|
||||||
|
}
|
||||||
|
trampoline = Some(args[i].clone());
|
||||||
|
}
|
||||||
|
a if a.starts_with("--trampoline=") => {
|
||||||
|
trampoline = Some(a["--trampoline=".len()..].to_string());
|
||||||
|
}
|
||||||
a if a.starts_with('-') => {
|
a if a.starts_with('-') => {
|
||||||
return Err(format!("unknown option: {a}"));
|
return Err(format!("unknown option: {a}"));
|
||||||
}
|
}
|
||||||
@@ -147,6 +186,9 @@ pub(crate) fn parse_cli_args(args: &[String]) -> Result<CliArgs, String> {
|
|||||||
join_token,
|
join_token,
|
||||||
gateway_url,
|
gateway_url,
|
||||||
upstream_gateway,
|
upstream_gateway,
|
||||||
|
upgrade,
|
||||||
|
upgrade_source,
|
||||||
|
trampoline,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,12 +197,16 @@ pub(crate) fn print_help() {
|
|||||||
println!("huskies init [OPTIONS] [PATH]");
|
println!("huskies init [OPTIONS] [PATH]");
|
||||||
println!("huskies agent --rendezvous <URL> [OPTIONS] [PATH]");
|
println!("huskies agent --rendezvous <URL> [OPTIONS] [PATH]");
|
||||||
println!("huskies --gateway [OPTIONS] [PATH]");
|
println!("huskies --gateway [OPTIONS] [PATH]");
|
||||||
|
println!("huskies upgrade [--source <URL>]");
|
||||||
println!();
|
println!();
|
||||||
println!("Serve a huskies project.");
|
println!("Serve a huskies project.");
|
||||||
println!();
|
println!();
|
||||||
println!("COMMANDS:");
|
println!("COMMANDS:");
|
||||||
println!(" init Scaffold a new .huskies/ project and start the interactive setup wizard.");
|
println!(" init Scaffold a new .huskies/ project and start the interactive setup wizard.");
|
||||||
println!(" agent Run as a headless build agent — syncs CRDT state, claims and runs work.");
|
println!(" agent Run as a headless build agent — syncs CRDT state, claims and runs work.");
|
||||||
|
println!(
|
||||||
|
" upgrade Fetch a new huskies binary from SOURCE and atomically replace the current"
|
||||||
|
);
|
||||||
println!();
|
println!();
|
||||||
println!("ARGS:");
|
println!("ARGS:");
|
||||||
println!(
|
println!(
|
||||||
@@ -190,6 +236,8 @@ pub(crate) fn print_help() {
|
|||||||
println!(" sled connects to WS URL and forwards all");
|
println!(" sled connects to WS URL and forwards all");
|
||||||
println!(" prompt_permission calls via the uplink protocol.");
|
println!(" prompt_permission calls via the uplink protocol.");
|
||||||
println!(" Also readable from HUSKIES_UPSTREAM_GATEWAY env var.");
|
println!(" Also readable from HUSKIES_UPSTREAM_GATEWAY env var.");
|
||||||
|
println!(" --source <URL> Binary source URL for the `upgrade` subcommand.");
|
||||||
|
println!(" Falls back to HUSKIES_BINARY_SOURCE env var.");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve the optional positional path argument into an absolute `PathBuf`.
|
/// Resolve the optional positional path argument into an absolute `PathBuf`.
|
||||||
@@ -399,6 +447,58 @@ mod tests {
|
|||||||
assert!(parse_cli_args(&args).is_err());
|
assert!(parse_cli_args(&args).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── upgrade subcommand ──────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_upgrade_subcommand() {
|
||||||
|
let args = vec!["upgrade".to_string()];
|
||||||
|
let result = parse_cli_args(&args).unwrap();
|
||||||
|
assert!(result.upgrade);
|
||||||
|
assert_eq!(result.upgrade_source, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_upgrade_with_source_flag() {
|
||||||
|
let args = vec![
|
||||||
|
"upgrade".to_string(),
|
||||||
|
"--source".to_string(),
|
||||||
|
"http://gateway:3000/api/huskies-binary".to_string(),
|
||||||
|
];
|
||||||
|
let result = parse_cli_args(&args).unwrap();
|
||||||
|
assert!(result.upgrade);
|
||||||
|
assert_eq!(
|
||||||
|
result.upgrade_source,
|
||||||
|
Some("http://gateway:3000/api/huskies-binary".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_upgrade_with_source_equals_syntax() {
|
||||||
|
let args = vec![
|
||||||
|
"upgrade".to_string(),
|
||||||
|
"--source=http://gw:3000/api/b".to_string(),
|
||||||
|
];
|
||||||
|
let result = parse_cli_args(&args).unwrap();
|
||||||
|
assert!(result.upgrade);
|
||||||
|
assert_eq!(
|
||||||
|
result.upgrade_source,
|
||||||
|
Some("http://gw:3000/api/b".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_upgrade_source_missing_value_is_error() {
|
||||||
|
let args = vec!["upgrade".to_string(), "--source".to_string()];
|
||||||
|
assert!(parse_cli_args(&args).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_no_args_upgrade_is_false() {
|
||||||
|
let result = parse_cli_args(&[]).unwrap();
|
||||||
|
assert!(!result.upgrade);
|
||||||
|
assert_eq!(result.upgrade_source, None);
|
||||||
|
}
|
||||||
|
|
||||||
// ── resolve_path_arg ────────────────────────────────────────────
|
// ── resolve_path_arg ────────────────────────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
//! Read/write helpers for the `event_log` append-only list in the CRDT document.
|
||||||
|
//!
|
||||||
|
//! Every pipeline stage transition is appended as an [`EventLogEntryCrdt`][super::super::types::EventLogEntryCrdt]
|
||||||
|
//! entry. Entries are never updated or tombstoned — the list is strictly grow-only.
|
||||||
|
//! Monotonic sequencing is computed at write time while holding the CRDT lock,
|
||||||
|
//! so `event_seq` values for a given sled are always contiguous and gap-free.
|
||||||
|
|
||||||
|
use bft_json_crdt::json_crdt::{JsonValue, *};
|
||||||
|
use bft_json_crdt::op::ROOT_ID;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::super::state::{apply_and_persist, get_crdt};
|
||||||
|
use super::super::types::EventLogEntryCrdt;
|
||||||
|
|
||||||
|
/// `pipeline_event` value used to mark a gap sentinel entry in the event log.
|
||||||
|
///
|
||||||
|
/// A gap sentinel is appended when the event-log subscriber detects that the
|
||||||
|
/// broadcast channel dropped events (i.e. it received `RecvError::Lagged`).
|
||||||
|
/// The `from_stage` and `to_stage` fields encode the logical EventId range
|
||||||
|
/// `[from, to]` of the dropped events as decimal strings.
|
||||||
|
pub const GAP_PIPELINE_EVENT: &str = "EventStreamGap";
|
||||||
|
|
||||||
|
/// Raw event log entry extracted from the CRDT document.
|
||||||
|
///
|
||||||
|
/// All fields are decoded to Rust primitives; entries with a missing or
|
||||||
|
/// malformed `sled_id` are silently dropped by [`read_all_event_log_entries`].
|
||||||
|
pub struct EventLogEntryRaw {
|
||||||
|
/// Monotonic sequence number for the recording sled (0-based).
|
||||||
|
pub event_seq: u64,
|
||||||
|
/// Hex-encoded Ed25519 public key of the sled that wrote this entry.
|
||||||
|
pub sled_id: String,
|
||||||
|
/// Unix timestamp (seconds) when the transition fired.
|
||||||
|
pub timestamp: f64,
|
||||||
|
/// Story ID of the work item that transitioned.
|
||||||
|
pub story_id: String,
|
||||||
|
/// Human-readable label of the stage before the transition.
|
||||||
|
pub from_stage: String,
|
||||||
|
/// Human-readable label of the stage after the transition.
|
||||||
|
pub to_stage: String,
|
||||||
|
/// String label of the `PipelineEvent` variant.
|
||||||
|
pub pipeline_event: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append a new event log entry to the CRDT, computing the monotonic `event_seq`
|
||||||
|
/// atomically while the CRDT lock is held.
|
||||||
|
///
|
||||||
|
/// No-ops silently when the CRDT is not yet initialised.
|
||||||
|
pub fn append_event_log_entry(
|
||||||
|
sled_id: &str,
|
||||||
|
timestamp: f64,
|
||||||
|
story_id: &str,
|
||||||
|
from_stage: &str,
|
||||||
|
to_stage: &str,
|
||||||
|
pipeline_event: &str,
|
||||||
|
) {
|
||||||
|
let Some(state_mutex) = get_crdt() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Ok(mut state) = state_mutex.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Count existing entries for this sled while holding the lock so the seq
|
||||||
|
// is computed and used in the same critical section — no TOCTOU gap.
|
||||||
|
let event_seq = state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter(|e| matches!(e.sled_id.view(), JsonValue::String(s) if s == sled_id))
|
||||||
|
.count() as f64;
|
||||||
|
|
||||||
|
// Append after the last existing entry so the list stays in insertion order.
|
||||||
|
// Inserting after ROOT_ID would place each entry at the front (RGA semantics),
|
||||||
|
// reversing the sequence; inserting after the current tail preserves order.
|
||||||
|
let total_len = state.crdt.doc.event_log.view().len();
|
||||||
|
let after = if total_len > 0 {
|
||||||
|
super::list_id_at(&state.crdt.doc.event_log, total_len - 1).unwrap_or(ROOT_ID)
|
||||||
|
} else {
|
||||||
|
ROOT_ID
|
||||||
|
};
|
||||||
|
|
||||||
|
let entry: JsonValue = json!({
|
||||||
|
"event_seq": event_seq,
|
||||||
|
"sled_id": sled_id,
|
||||||
|
"timestamp": timestamp,
|
||||||
|
"story_id": story_id,
|
||||||
|
"from_stage": from_stage,
|
||||||
|
"to_stage": to_stage,
|
||||||
|
"pipeline_event": pipeline_event,
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
|
||||||
|
apply_and_persist(&mut state, |s| s.crdt.doc.event_log.insert(after, entry));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append an `EventStreamGap` sentinel entry to the CRDT event log.
|
||||||
|
///
|
||||||
|
/// Called when the event-log broadcast subscriber detects that the channel
|
||||||
|
/// dropped events (`RecvError::Lagged`). `from_id` and `to_id` are the
|
||||||
|
/// logical sequence numbers (in the per-sled event stream) of the first and
|
||||||
|
/// last dropped events respectively. The sentinel itself also consumes one
|
||||||
|
/// CRDT `event_seq` slot so the monotonic counter remains contiguous across
|
||||||
|
/// the gap.
|
||||||
|
pub fn append_gap_log_entry(sled_id: &str, from_id: u64, to_id: u64) {
|
||||||
|
let timestamp = chrono::Utc::now().timestamp() as f64;
|
||||||
|
append_event_log_entry(
|
||||||
|
sled_id,
|
||||||
|
timestamp,
|
||||||
|
"",
|
||||||
|
&from_id.to_string(),
|
||||||
|
&to_id.to_string(),
|
||||||
|
GAP_PIPELINE_EVENT,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read all event log entries from the CRDT document.
|
||||||
|
///
|
||||||
|
/// Entries with a missing or empty `sled_id` are silently skipped.
|
||||||
|
/// Order reflects CRDT insertion order (RGA list semantics).
|
||||||
|
pub fn read_all_event_log_entries() -> Vec<EventLogEntryRaw> {
|
||||||
|
let Some(state_mutex) = get_crdt() else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let Ok(state) = state_mutex.lock() else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter_map(extract_entry)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert a CRDT event log entry to its read-side representation.
|
||||||
|
fn extract_entry(e: &EventLogEntryCrdt) -> Option<EventLogEntryRaw> {
|
||||||
|
let event_seq = match e.event_seq.view() {
|
||||||
|
JsonValue::Number(n) => n as u64,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let sled_id = match e.sled_id.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() => s,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let timestamp = match e.timestamp.view() {
|
||||||
|
JsonValue::Number(n) => n,
|
||||||
|
_ => 0.0,
|
||||||
|
};
|
||||||
|
let story_id = match e.story_id.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let from_stage = match e.from_stage.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let to_stage = match e.to_stage.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let pipeline_event = match e.pipeline_event.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
Some(EventLogEntryRaw {
|
||||||
|
event_seq,
|
||||||
|
sled_id,
|
||||||
|
timestamp,
|
||||||
|
story_id,
|
||||||
|
from_stage,
|
||||||
|
to_stage,
|
||||||
|
pipeline_event,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,332 @@
|
|||||||
|
//! Read/write helpers for the `llm_sessions` LWW-map collection, including the
|
||||||
|
//! atomic `assemble_and_advance_session` helper used by every chat transport.
|
||||||
|
//!
|
||||||
|
//! LLM sessions are keyed by **persona name** (e.g. `"timmy"` for the
|
||||||
|
//! gateway-level bot) and track per-sled high-water marks so that
|
||||||
|
//! `assemble_and_advance_session` can inject only events the LLM has not yet
|
||||||
|
//! seen and advance the marks atomically within a single CRDT lock acquisition.
|
||||||
|
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
||||||
|
use bft_json_crdt::json_crdt::{JsonValue, *};
|
||||||
|
use bft_json_crdt::op::ROOT_ID;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::super::state::{apply_and_persist, get_crdt, rebuild_llm_session_index};
|
||||||
|
use super::super::types::{LlmSessionCrdt, LlmSessionView, ScopeFilter};
|
||||||
|
use super::event_log::GAP_PIPELINE_EVENT;
|
||||||
|
|
||||||
|
/// Write or upsert an LLM session entry keyed by `persona`.
|
||||||
|
///
|
||||||
|
/// Creates a new entry if `persona` is not yet present; updates `scope` on an
|
||||||
|
/// existing entry. The `high_water` register is not touched by this function —
|
||||||
|
/// use `assemble_and_advance_session` to advance it atomically.
|
||||||
|
///
|
||||||
|
/// The `scope` string must be in wire form: `"all"` for [`ScopeFilter::All`]
|
||||||
|
/// or `"sleds:hex1,hex2"` for [`ScopeFilter::Sleds`].
|
||||||
|
pub fn write_llm_session(persona: &str, scope: &str) {
|
||||||
|
let Some(state_mutex) = get_crdt() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Ok(mut state) = state_mutex.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(&idx) = state.llm_session_index.get(persona) {
|
||||||
|
apply_and_persist(&mut state, |s| {
|
||||||
|
s.crdt.doc.llm_sessions[idx]
|
||||||
|
.persona_name
|
||||||
|
.set(persona.to_string())
|
||||||
|
});
|
||||||
|
apply_and_persist(&mut state, |s| {
|
||||||
|
s.crdt.doc.llm_sessions[idx].scope.set(scope.to_string())
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
let entry: JsonValue = json!({
|
||||||
|
"session_id": persona,
|
||||||
|
"persona_name": persona,
|
||||||
|
"scope": scope,
|
||||||
|
"high_water": "{}",
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
apply_and_persist(&mut state, |s| {
|
||||||
|
s.crdt.doc.llm_sessions.insert(ROOT_ID, entry)
|
||||||
|
});
|
||||||
|
state.llm_session_index = rebuild_llm_session_index(&state.crdt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a single LLM session entry by persona name.
|
||||||
|
pub fn read_llm_session(persona: &str) -> Option<LlmSessionView> {
|
||||||
|
let state_mutex = get_crdt()?;
|
||||||
|
let state = state_mutex.lock().ok()?;
|
||||||
|
let &idx = state.llm_session_index.get(persona)?;
|
||||||
|
extract_llm_session_view(&state.crdt.doc.llm_sessions[idx])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Atomically read new event-log entries for `persona` past the stored
|
||||||
|
/// high-water marks, render them as a block of audit lines, and advance the
|
||||||
|
/// marks to prevent double-injection on the next call.
|
||||||
|
///
|
||||||
|
/// The set of sleds whose events are collected is determined by the persona's
|
||||||
|
/// [`ScopeFilter`]:
|
||||||
|
/// - [`ScopeFilter::All`]: events from every sled present in the event log are
|
||||||
|
/// included — this is the gateway-level persona default that gives a full
|
||||||
|
/// cross-sled view.
|
||||||
|
/// - [`ScopeFilter::Sleds`]: only events whose `sled_id` is in the stored set
|
||||||
|
/// are included. When the stored set is empty (legacy `"single-sled"` rows or
|
||||||
|
/// freshly created sessions with no explicit scope), the local node's sled ID
|
||||||
|
/// is used as the sole member, preserving prior single-sled behaviour.
|
||||||
|
///
|
||||||
|
/// Returns an empty `Vec` when there are no new events or the CRDT is not
|
||||||
|
/// initialised.
|
||||||
|
pub fn assemble_and_advance_session(persona: &str) -> Vec<String> {
|
||||||
|
let local_sled_id = crate::crdt_state::our_node_id().unwrap_or_default();
|
||||||
|
|
||||||
|
let Some(state_mutex) = get_crdt() else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let Ok(mut state) = state_mutex.lock() else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
|
||||||
|
// Determine the persona's scope filter and current high-water map.
|
||||||
|
let (scope_filter, current_high_water) = match state.llm_session_index.get(persona).copied() {
|
||||||
|
Some(idx) => {
|
||||||
|
let filter = parse_scope(&state.crdt.doc.llm_sessions[idx], &local_sled_id);
|
||||||
|
let hw = parse_high_water(&state.crdt.doc.llm_sessions[idx]);
|
||||||
|
(filter, hw)
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
// New session with no stored entry: default to local sled only.
|
||||||
|
let mut ids = BTreeSet::new();
|
||||||
|
if !local_sled_id.is_empty() {
|
||||||
|
ids.insert(local_sled_id.clone());
|
||||||
|
}
|
||||||
|
(ScopeFilter::Sleds(ids), BTreeMap::new())
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build the set of sled IDs to collect events from.
|
||||||
|
let target_sleds: BTreeSet<String> = match &scope_filter {
|
||||||
|
ScopeFilter::All => {
|
||||||
|
// Collect every unique sled_id present in the event log at this moment
|
||||||
|
// (live, not snapshotted — picks up newly adopted sleds automatically).
|
||||||
|
state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter_map(|e| match e.sled_id.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() => Some(s),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
ScopeFilter::Sleds(ids) if ids.is_empty() => {
|
||||||
|
// Empty set → legacy fallback: local sled only.
|
||||||
|
if local_sled_id.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
std::iter::once(local_sled_id.clone()).collect()
|
||||||
|
}
|
||||||
|
ScopeFilter::Sleds(ids) => ids.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
if target_sleds.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect new events from each target sled past its high-water mark.
|
||||||
|
let mut new_events: Vec<(f64, String, String, String, String, String)> = state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter_map(|e| extract_new_event_multi(e, &target_sleds, ¤t_high_water))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if new_events.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by (sled_id, event_seq) for deterministic ordering.
|
||||||
|
new_events.sort_by(|a, b| {
|
||||||
|
a.1.cmp(&b.1)
|
||||||
|
.then(a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal))
|
||||||
|
});
|
||||||
|
|
||||||
|
// Advance the high-water mark for each sled that had new events.
|
||||||
|
let mut new_high_water = current_high_water;
|
||||||
|
for (seq, sled_id, ..) in &new_events {
|
||||||
|
let entry = new_high_water.entry(sled_id.clone()).or_insert(0);
|
||||||
|
if *seq as u64 > *entry {
|
||||||
|
*entry = *seq as u64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let new_hw_json = serde_json::to_string(&new_high_water).unwrap_or_else(|_| "{}".to_string());
|
||||||
|
|
||||||
|
// Upsert the persona entry with the new high-water value.
|
||||||
|
let idx_opt = state.llm_session_index.get(persona).copied();
|
||||||
|
if let Some(idx) = idx_opt {
|
||||||
|
apply_and_persist(&mut state, |s| {
|
||||||
|
s.crdt.doc.llm_sessions[idx]
|
||||||
|
.high_water
|
||||||
|
.set(new_hw_json.clone())
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
let scope_str = scope_filter.to_scope_str();
|
||||||
|
let entry: JsonValue = json!({
|
||||||
|
"session_id": persona,
|
||||||
|
"persona_name": persona,
|
||||||
|
"scope": scope_str,
|
||||||
|
"high_water": new_hw_json,
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
apply_and_persist(&mut state, |s| {
|
||||||
|
s.crdt.doc.llm_sessions.insert(ROOT_ID, entry)
|
||||||
|
});
|
||||||
|
state.llm_session_index = rebuild_llm_session_index(&state.crdt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Observability: log event-log size and gap count across the persona's
|
||||||
|
// target sleds (the scope actually assembled for this persona).
|
||||||
|
let total_entries = state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter(|e| matches!(e.sled_id.view(), JsonValue::String(s) if target_sleds.contains(&s)))
|
||||||
|
.count();
|
||||||
|
let gap_count = state
|
||||||
|
.crdt
|
||||||
|
.doc
|
||||||
|
.event_log
|
||||||
|
.iter()
|
||||||
|
.filter(|e| {
|
||||||
|
matches!(e.sled_id.view(), JsonValue::String(s) if target_sleds.contains(&s))
|
||||||
|
&& matches!(e.pipeline_event.view(), JsonValue::String(s) if s == GAP_PIPELINE_EVENT)
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
crate::slog!(
|
||||||
|
"[event-log] assemble persona={persona} sled_entries={total_entries} gap_count={gap_count}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Render each new event as a compact audit line; gap sentinels get a
|
||||||
|
// human-readable message so the LLM is never presented with raw field data.
|
||||||
|
new_events
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|(_, sled_id, story_id, from_stage, to_stage, pipeline_event)| {
|
||||||
|
if pipeline_event == GAP_PIPELINE_EVENT {
|
||||||
|
format!("events between {from_stage} and {to_stage} were dropped")
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"pipeline_event sled_id=\"{sled_id}\" story_id=\"{story_id}\" \
|
||||||
|
from=\"{from_stage}\" to=\"{to_stage}\" event=\"{pipeline_event}\""
|
||||||
|
)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode the high-water JSON string from an `LlmSessionCrdt` entry.
|
||||||
|
fn parse_high_water(entry: &LlmSessionCrdt) -> BTreeMap<String, u64> {
|
||||||
|
match entry.high_water.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() && s != "{}" => {
|
||||||
|
serde_json::from_str(&s).unwrap_or_default()
|
||||||
|
}
|
||||||
|
_ => BTreeMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the scope filter from an `LlmSessionCrdt` entry, falling back to
|
||||||
|
/// a single-element set containing `local_sled_id` for legacy / empty scope strings.
|
||||||
|
fn parse_scope(entry: &LlmSessionCrdt, local_sled_id: &str) -> ScopeFilter {
|
||||||
|
let raw = match entry.scope.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let filter = ScopeFilter::from_scope_str(&raw);
|
||||||
|
// For a Sleds filter with an empty set (legacy "single-sled" or ""),
|
||||||
|
// fall back to the local sled.
|
||||||
|
if let ScopeFilter::Sleds(ref ids) = filter
|
||||||
|
&& ids.is_empty()
|
||||||
|
&& !local_sled_id.is_empty()
|
||||||
|
{
|
||||||
|
let mut fallback = BTreeSet::new();
|
||||||
|
fallback.insert(local_sled_id.to_string());
|
||||||
|
return ScopeFilter::Sleds(fallback);
|
||||||
|
}
|
||||||
|
filter
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract one event log entry if its `sled_id` is in `target_sleds` and its
|
||||||
|
/// `event_seq` is strictly greater than the matching high-water value (or no
|
||||||
|
/// high-water has been recorded yet for that sled).
|
||||||
|
///
|
||||||
|
/// Returns `(event_seq, sled_id, story_id, from_stage, to_stage, pipeline_event)`.
|
||||||
|
fn extract_new_event_multi(
|
||||||
|
e: &crate::crdt_state::types::EventLogEntryCrdt,
|
||||||
|
target_sleds: &BTreeSet<String>,
|
||||||
|
high_water: &BTreeMap<String, u64>,
|
||||||
|
) -> Option<(f64, String, String, String, String, String)> {
|
||||||
|
let sled_id = match e.sled_id.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() && target_sleds.contains(&s) => s,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let event_seq = match e.event_seq.view() {
|
||||||
|
JsonValue::Number(n) => n,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let last_seen = high_water.get(&sled_id).copied();
|
||||||
|
if last_seen.is_some_and(|last| event_seq as u64 <= last) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let story_id = match e.story_id.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let from_stage = match e.from_stage.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let to_stage = match e.to_stage.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let pipeline_event = match e.pipeline_event.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
Some((
|
||||||
|
event_seq,
|
||||||
|
sled_id,
|
||||||
|
story_id,
|
||||||
|
from_stage,
|
||||||
|
to_stage,
|
||||||
|
pipeline_event,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert a CRDT LLM session entry into its read-only view representation.
|
||||||
|
pub(super) fn extract_llm_session_view(entry: &LlmSessionCrdt) -> Option<LlmSessionView> {
|
||||||
|
let session_id = match entry.session_id.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() => s,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let persona_name = match entry.persona_name.view() {
|
||||||
|
JsonValue::String(s) => s,
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
|
let local_sled_id = crate::crdt_state::our_node_id().unwrap_or_default();
|
||||||
|
let scope_filter = parse_scope(entry, &local_sled_id);
|
||||||
|
let high_water = parse_high_water(entry);
|
||||||
|
Some(LlmSessionView {
|
||||||
|
session_id,
|
||||||
|
persona_name,
|
||||||
|
scope_filter,
|
||||||
|
high_water,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -14,7 +14,9 @@ use bft_json_crdt::op::OpId;
|
|||||||
|
|
||||||
mod active_agents;
|
mod active_agents;
|
||||||
mod agent_throttle;
|
mod agent_throttle;
|
||||||
|
mod event_log;
|
||||||
mod gateway_projects;
|
mod gateway_projects;
|
||||||
|
mod llm_sessions;
|
||||||
mod merge_jobs;
|
mod merge_jobs;
|
||||||
mod test_jobs;
|
mod test_jobs;
|
||||||
mod tokens;
|
mod tokens;
|
||||||
@@ -28,9 +30,14 @@ pub use active_agents::{
|
|||||||
pub use agent_throttle::{
|
pub use agent_throttle::{
|
||||||
delete_agent_throttle, read_agent_throttle, read_all_agent_throttles, write_agent_throttle,
|
delete_agent_throttle, read_agent_throttle, read_all_agent_throttles, write_agent_throttle,
|
||||||
};
|
};
|
||||||
|
pub use event_log::{
|
||||||
|
EventLogEntryRaw, GAP_PIPELINE_EVENT, append_event_log_entry, append_gap_log_entry,
|
||||||
|
read_all_event_log_entries,
|
||||||
|
};
|
||||||
pub use gateway_projects::{
|
pub use gateway_projects::{
|
||||||
delete_gateway_project, read_all_gateway_projects, read_gateway_project, write_gateway_project,
|
delete_gateway_project, read_all_gateway_projects, read_gateway_project, write_gateway_project,
|
||||||
};
|
};
|
||||||
|
pub use llm_sessions::{assemble_and_advance_session, read_llm_session, write_llm_session};
|
||||||
pub use merge_jobs::{delete_merge_job, read_all_merge_jobs, read_merge_job, write_merge_job};
|
pub use merge_jobs::{delete_merge_job, read_all_merge_jobs, read_merge_job, write_merge_job};
|
||||||
pub use test_jobs::{delete_test_job, read_all_test_jobs, read_test_job, write_test_job};
|
pub use test_jobs::{delete_test_job, read_all_test_jobs, read_test_job, write_test_job};
|
||||||
pub use tokens::{delete_token_usage, read_all_token_usage, read_token_usage, write_token_usage};
|
pub use tokens::{delete_token_usage, read_all_token_usage, read_token_usage, write_token_usage};
|
||||||
|
|||||||
@@ -28,12 +28,14 @@ mod write;
|
|||||||
|
|
||||||
pub use gateway_config::{read_gateway_active_project, write_gateway_active_project};
|
pub use gateway_config::{read_gateway_active_project, write_gateway_active_project};
|
||||||
pub use lww_maps::{
|
pub use lww_maps::{
|
||||||
delete_active_agent, delete_agent_throttle, delete_gateway_project, delete_merge_job,
|
EventLogEntryRaw, GAP_PIPELINE_EVENT, append_event_log_entry, append_gap_log_entry,
|
||||||
delete_test_job, delete_token_usage, read_active_agent, read_agent_throttle,
|
assemble_and_advance_session, delete_active_agent, delete_agent_throttle,
|
||||||
read_all_active_agents, read_all_agent_throttles, read_all_gateway_projects,
|
delete_gateway_project, delete_merge_job, delete_test_job, delete_token_usage,
|
||||||
read_all_merge_jobs, read_all_test_jobs, read_all_token_usage, read_gateway_project,
|
read_active_agent, read_agent_throttle, read_all_active_agents, read_all_agent_throttles,
|
||||||
read_merge_job, read_test_job, read_token_usage, write_active_agent, write_agent_throttle,
|
read_all_event_log_entries, read_all_gateway_projects, read_all_merge_jobs, read_all_test_jobs,
|
||||||
write_gateway_project, write_merge_job, write_test_job, write_token_usage,
|
read_all_token_usage, read_gateway_project, read_llm_session, read_merge_job, read_test_job,
|
||||||
|
read_token_usage, write_active_agent, write_agent_throttle, write_gateway_project,
|
||||||
|
write_llm_session, write_merge_job, write_test_job, write_token_usage,
|
||||||
};
|
};
|
||||||
pub use ops::{all_ops_json, apply_remote_op, ops_since, our_vector_clock, subscribe_ops};
|
pub use ops::{all_ops_json, apply_remote_op, ops_since, our_vector_clock, subscribe_ops};
|
||||||
pub use presence::{
|
pub use presence::{
|
||||||
@@ -45,19 +47,21 @@ pub use read::{
|
|||||||
dep_is_archived_crdt, dep_is_done_crdt, dump_crdt_state, evict_item, is_tombstoned,
|
dep_is_archived_crdt, dep_is_done_crdt, dump_crdt_state, evict_item, is_tombstoned,
|
||||||
read_all_items, read_item, tombstoned_ids,
|
read_all_items, read_item, tombstoned_ids,
|
||||||
};
|
};
|
||||||
|
pub(crate) use state::flush_persistence;
|
||||||
pub use state::{init, subscribe};
|
pub use state::{init, subscribe};
|
||||||
pub use types::{
|
pub use types::{
|
||||||
ActiveAgentCrdt, ActiveAgentView, AgentThrottleCrdt, AgentThrottleView, CrdtEvent, EpicId,
|
ActiveAgentCrdt, ActiveAgentView, AgentThrottleCrdt, AgentThrottleView, CrdtEvent, EpicId,
|
||||||
GatewayConfigCrdt, GatewayProjectCrdt, GatewayProjectView, MergeJobCrdt, MergeJobView,
|
EventLogEntryCrdt, GatewayConfigCrdt, GatewayProjectCrdt, GatewayProjectView, LlmSessionCrdt,
|
||||||
NodePresenceCrdt, NodePresenceView, PipelineDoc, PipelineItemCrdt, PipelineItemView,
|
LlmSessionView, MergeJobCrdt, MergeJobView, NodePresenceCrdt, NodePresenceView, PipelineDoc,
|
||||||
TestJobCrdt, TestJobView, TokenUsageCrdt, TokenUsageView, WorkItem,
|
PipelineItemCrdt, PipelineItemView, ScopeFilter, TestJobCrdt, TestJobView, TokenUsageCrdt,
|
||||||
|
TokenUsageView, WorkItem,
|
||||||
};
|
};
|
||||||
pub use write::{
|
pub use write::{
|
||||||
bump_retry_count, migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
|
bump_retry_count, migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
|
||||||
migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric, name_from_story_id,
|
migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric,
|
||||||
purge_done_stage_merge_jobs, set_agent, set_depends_on, set_epic, set_item_type, set_name,
|
migrate_zombie_pipeline_rows, name_from_story_id, purge_done_stage_merge_jobs, set_agent,
|
||||||
set_origin, set_plan_state, set_qa_mode, set_resume_to, set_resume_to_raw, set_retry_count,
|
set_depends_on, set_epic, set_item_type, set_name, set_origin, set_plan_state, set_qa_mode,
|
||||||
write_item,
|
set_resume_to, set_resume_to_raw, set_retry_count, write_item,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
#![allow(unused_imports, dead_code)]
|
#![allow(unused_imports, dead_code)]
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
use super::hex;
|
use super::hex;
|
||||||
use bft_json_crdt::json_crdt::*;
|
use bft_json_crdt::json_crdt::*;
|
||||||
@@ -10,9 +11,10 @@ use tokio::sync::broadcast;
|
|||||||
|
|
||||||
use super::VectorClock;
|
use super::VectorClock;
|
||||||
use super::state::{
|
use super::state::{
|
||||||
SYNC_TX, all_ops_lock, apply_and_persist, emit_event, get_crdt, rebuild_active_agent_index,
|
PERSIST_PENDING, PersistMsg, SYNC_TX, all_ops_lock, apply_and_persist, emit_event, get_crdt,
|
||||||
rebuild_agent_throttle_index, rebuild_index, rebuild_merge_job_index, rebuild_node_index,
|
rebuild_active_agent_index, rebuild_agent_throttle_index, rebuild_index,
|
||||||
rebuild_test_job_index, rebuild_token_index, track_op, vector_clock_lock,
|
rebuild_merge_job_index, rebuild_node_index, rebuild_test_job_index, rebuild_token_index,
|
||||||
|
track_op, vector_clock_lock,
|
||||||
};
|
};
|
||||||
use super::types::{CrdtEvent, PipelineDoc};
|
use super::types::{CrdtEvent, PipelineDoc};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
@@ -116,9 +118,15 @@ pub fn apply_remote_op(op: SignedOp) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Persist the op.
|
// Persist the op.
|
||||||
if let Err(e) = state.persist_tx.send(op.clone()) {
|
if state
|
||||||
|
.persist_tx
|
||||||
|
.send(PersistMsg::Op(Box::new(op.clone())))
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
PERSIST_PENDING.fetch_add(1, Ordering::Relaxed);
|
||||||
|
} else {
|
||||||
crate::slog_error!(
|
crate::slog_error!(
|
||||||
"[crdt] Failed to send remote op to persist task: {e}; persist task may be dead. \
|
"[crdt] Failed to send remote op to persist task; persist task may be dead. \
|
||||||
In-memory state is now ahead of persisted state."
|
In-memory state is now ahead of persisted state."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ use std::collections::HashMap;
|
|||||||
use bft_json_crdt::json_crdt::*;
|
use bft_json_crdt::json_crdt::*;
|
||||||
use bft_json_crdt::op::{OpId, ROOT_ID};
|
use bft_json_crdt::op::{OpId, ROOT_ID};
|
||||||
|
|
||||||
use super::state::{all_ops_lock, apply_and_persist, get_crdt, rebuild_index};
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
|
use super::state::{PERSIST_PENDING, all_ops_lock, apply_and_persist, get_crdt, rebuild_index};
|
||||||
use super::types::{PipelineDoc, PipelineItemCrdt, PipelineItemView};
|
use super::types::{PipelineDoc, PipelineItemCrdt, PipelineItemView};
|
||||||
|
|
||||||
// ── Debug dump ───────────────────────────────────────────────────────
|
// ── Debug dump ───────────────────────────────────────────────────────
|
||||||
@@ -31,6 +33,8 @@ pub struct CrdtItemDump {
|
|||||||
pub is_deleted: bool,
|
pub is_deleted: bool,
|
||||||
/// Origin JSON string, or `None` for items that pre-date story 1088.
|
/// Origin JSON string, or `None` for items that pre-date story 1088.
|
||||||
pub origin: Option<String>,
|
pub origin: Option<String>,
|
||||||
|
/// Explicit item type register, or `None` when unset (infer from story_id prefix).
|
||||||
|
pub item_type: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Top-level debug dump of the in-memory CRDT state.
|
/// Top-level debug dump of the in-memory CRDT state.
|
||||||
@@ -44,6 +48,8 @@ pub struct CrdtStateDump {
|
|||||||
pub max_seq_in_list: u64,
|
pub max_seq_in_list: u64,
|
||||||
/// Count of ops in the ALL_OPS journal (persisted ops replayed at startup).
|
/// Count of ops in the ALL_OPS journal (persisted ops replayed at startup).
|
||||||
pub persisted_ops_count: usize,
|
pub persisted_ops_count: usize,
|
||||||
|
/// Count of ops queued in the persistence channel not yet written to SQLite.
|
||||||
|
pub pending_persist_ops_count: usize,
|
||||||
pub items: Vec<CrdtItemDump>,
|
pub items: Vec<CrdtItemDump>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,6 +67,7 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
let persisted_ops_count = all_ops_lock()
|
let persisted_ops_count = all_ops_lock()
|
||||||
.and_then(|m| m.lock().ok().map(|v| v.len()))
|
.and_then(|m| m.lock().ok().map(|v| v.len()))
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
let pending_persist_ops_count = PERSIST_PENDING.load(Ordering::Relaxed);
|
||||||
|
|
||||||
let Some(state_mutex) = get_crdt() else {
|
let Some(state_mutex) = get_crdt() else {
|
||||||
return CrdtStateDump {
|
return CrdtStateDump {
|
||||||
@@ -69,6 +76,7 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
total_ops_in_list: 0,
|
total_ops_in_list: 0,
|
||||||
max_seq_in_list: 0,
|
max_seq_in_list: 0,
|
||||||
persisted_ops_count,
|
persisted_ops_count,
|
||||||
|
pending_persist_ops_count,
|
||||||
items: Vec::new(),
|
items: Vec::new(),
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -80,6 +88,7 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
total_ops_in_list: 0,
|
total_ops_in_list: 0,
|
||||||
max_seq_in_list: 0,
|
max_seq_in_list: 0,
|
||||||
persisted_ops_count,
|
persisted_ops_count,
|
||||||
|
pending_persist_ops_count,
|
||||||
items: Vec::new(),
|
items: Vec::new(),
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -155,6 +164,10 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
JsonValue::String(s) if !s.is_empty() => Some(s),
|
JsonValue::String(s) if !s.is_empty() => Some(s),
|
||||||
_ => None,
|
_ => None,
|
||||||
};
|
};
|
||||||
|
let item_type = match item_crdt.item_type.view() {
|
||||||
|
JsonValue::String(s) if !s.is_empty() => Some(s),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
|
||||||
let content_index = op.id.iter().map(|b| format!("{b:02x}")).collect::<String>();
|
let content_index = op.id.iter().map(|b| format!("{b:02x}")).collect::<String>();
|
||||||
|
|
||||||
@@ -170,6 +183,7 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
content_index,
|
content_index,
|
||||||
is_deleted: op.is_deleted,
|
is_deleted: op.is_deleted,
|
||||||
origin,
|
origin,
|
||||||
|
item_type,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,6 +193,7 @@ pub fn dump_crdt_state(story_id_filter: Option<&str>) -> CrdtStateDump {
|
|||||||
total_ops_in_list,
|
total_ops_in_list,
|
||||||
max_seq_in_list,
|
max_seq_in_list,
|
||||||
persisted_ops_count,
|
persisted_ops_count,
|
||||||
|
pending_persist_ops_count,
|
||||||
items,
|
items,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,11 +5,13 @@
|
|||||||
//! it to the live document, sends it to the persistence channel, and broadcasts
|
//! it to the live document, sends it to the persistence channel, and broadcasts
|
||||||
//! it to sync peers via [`super::SYNC_TX`].
|
//! it to sync peers via [`super::SYNC_TX`].
|
||||||
|
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
use bft_json_crdt::json_crdt::JsonValue;
|
use bft_json_crdt::json_crdt::JsonValue;
|
||||||
use bft_json_crdt::op::Op;
|
use bft_json_crdt::op::Op;
|
||||||
|
|
||||||
use super::super::types::CrdtEvent;
|
use super::super::types::CrdtEvent;
|
||||||
use super::{CrdtState, statics};
|
use super::{CrdtState, init::PersistMsg, statics};
|
||||||
|
|
||||||
/// Create a CRDT op via `op_fn`, sign it, apply it, and send it to the
|
/// Create a CRDT op via `op_fn`, sign it, apply it, and send it to the
|
||||||
/// persistence channel. The closure receives `&mut CrdtState` so it can
|
/// persistence channel. The closure receives `&mut CrdtState` so it can
|
||||||
@@ -21,7 +23,13 @@ where
|
|||||||
let raw_op = op_fn(state);
|
let raw_op = op_fn(state);
|
||||||
let signed = raw_op.sign(&state.keypair);
|
let signed = raw_op.sign(&state.keypair);
|
||||||
state.crdt.apply(signed.clone());
|
state.crdt.apply(signed.clone());
|
||||||
if state.persist_tx.send(signed.clone()).is_err() {
|
if state
|
||||||
|
.persist_tx
|
||||||
|
.send(PersistMsg::Op(Box::new(signed.clone())))
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
statics::PERSIST_PENDING.fetch_add(1, Ordering::Relaxed);
|
||||||
|
} else {
|
||||||
let op_type = if signed.inner.is_deleted {
|
let op_type = if signed.inner.is_deleted {
|
||||||
"Delete"
|
"Delete"
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -113,3 +113,16 @@ pub(in crate::crdt_state) fn rebuild_gateway_project_index(
|
|||||||
}
|
}
|
||||||
map
|
map
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rebuild the session_id → llm_sessions list index.
|
||||||
|
pub(in crate::crdt_state) fn rebuild_llm_session_index(
|
||||||
|
crdt: &BaseCrdt<PipelineDoc>,
|
||||||
|
) -> HashMap<String, usize> {
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
for (i, entry) in crdt.doc.llm_sessions.iter().enumerate() {
|
||||||
|
if let JsonValue::String(ref k) = entry.session_id.view() {
|
||||||
|
map.insert(k.clone(), i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
map
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,25 +8,34 @@
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
use bft_json_crdt::json_crdt::{BaseCrdt, CrdtNode, JsonValue, SignedOp};
|
use bft_json_crdt::json_crdt::{BaseCrdt, CrdtNode, JsonValue, SignedOp};
|
||||||
use bft_json_crdt::keypair::{Ed25519KeyPair, make_keypair};
|
use bft_json_crdt::keypair::{Ed25519KeyPair, make_keypair};
|
||||||
use sqlx::SqlitePool;
|
use sqlx::SqlitePool;
|
||||||
use sqlx::sqlite::SqliteConnectOptions;
|
use sqlx::sqlite::SqliteConnectOptions;
|
||||||
use tokio::sync::{broadcast, mpsc};
|
use tokio::sync::{broadcast, mpsc, oneshot};
|
||||||
|
|
||||||
use super::super::VectorClock;
|
use super::super::VectorClock;
|
||||||
use super::super::hex;
|
use super::super::hex;
|
||||||
use super::super::types::{CrdtEvent, PipelineDoc};
|
use super::super::types::{CrdtEvent, PipelineDoc};
|
||||||
use super::indices::{
|
use super::indices::{
|
||||||
rebuild_active_agent_index, rebuild_agent_throttle_index, rebuild_gateway_project_index,
|
rebuild_active_agent_index, rebuild_agent_throttle_index, rebuild_gateway_project_index,
|
||||||
rebuild_index, rebuild_merge_job_index, rebuild_node_index, rebuild_test_job_index,
|
rebuild_index, rebuild_llm_session_index, rebuild_merge_job_index, rebuild_node_index,
|
||||||
rebuild_token_index,
|
rebuild_test_job_index, rebuild_token_index,
|
||||||
};
|
};
|
||||||
use super::statics::{ALL_OPS, CRDT_EVENT_TX, SYNC_TX, VECTOR_CLOCK};
|
use super::statics::{ALL_OPS, CRDT_EVENT_TX, PERSIST_PENDING, SYNC_TX, VECTOR_CLOCK};
|
||||||
use super::{CRDT_STATE, CrdtState};
|
use super::{CRDT_STATE, CrdtState};
|
||||||
use crate::slog;
|
use crate::slog;
|
||||||
|
|
||||||
|
/// Message type for the persistence background channel.
|
||||||
|
pub(crate) enum PersistMsg {
|
||||||
|
/// Persist this op to SQLite.
|
||||||
|
Op(Box<SignedOp>),
|
||||||
|
/// Drain: signal the sender after all preceding ops are committed.
|
||||||
|
Flush(oneshot::Sender<()>),
|
||||||
|
}
|
||||||
|
|
||||||
/// Initialise the CRDT state layer.
|
/// Initialise the CRDT state layer.
|
||||||
///
|
///
|
||||||
/// Opens the SQLite database, loads or creates a node keypair, replays any
|
/// Opens the SQLite database, loads or creates a node keypair, replays any
|
||||||
@@ -94,6 +103,7 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
let test_job_index = rebuild_test_job_index(&crdt);
|
let test_job_index = rebuild_test_job_index(&crdt);
|
||||||
let agent_throttle_index = rebuild_agent_throttle_index(&crdt);
|
let agent_throttle_index = rebuild_agent_throttle_index(&crdt);
|
||||||
let gateway_project_index = rebuild_gateway_project_index(&crdt);
|
let gateway_project_index = rebuild_gateway_project_index(&crdt);
|
||||||
|
let llm_session_index = rebuild_llm_session_index(&crdt);
|
||||||
|
|
||||||
// Advance the top-level list clocks to the Lamport floor so that
|
// Advance the top-level list clocks to the Lamport floor so that
|
||||||
// list-level inserts don't re-emit low seq numbers.
|
// list-level inserts don't re-emit low seq numbers.
|
||||||
@@ -105,6 +115,7 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
crdt.doc.test_jobs.advance_seq(lamport_floor);
|
crdt.doc.test_jobs.advance_seq(lamport_floor);
|
||||||
crdt.doc.agent_throttle.advance_seq(lamport_floor);
|
crdt.doc.agent_throttle.advance_seq(lamport_floor);
|
||||||
crdt.doc.gateway_projects.advance_seq(lamport_floor);
|
crdt.doc.gateway_projects.advance_seq(lamport_floor);
|
||||||
|
crdt.doc.llm_sessions.advance_seq(lamport_floor);
|
||||||
crdt.doc
|
crdt.doc
|
||||||
.gateway_config
|
.gateway_config
|
||||||
.active_project
|
.active_project
|
||||||
@@ -119,14 +130,18 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Spawn background persistence task.
|
// Spawn background persistence task.
|
||||||
let (persist_tx, mut persist_rx) = mpsc::unbounded_channel::<SignedOp>();
|
let (persist_tx, mut persist_rx) = mpsc::unbounded_channel::<PersistMsg>();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
while let Some(op) = persist_rx.recv().await {
|
while let Some(msg) = persist_rx.recv().await {
|
||||||
|
match msg {
|
||||||
|
PersistMsg::Op(op) => {
|
||||||
|
let op = *op;
|
||||||
let op_json = match serde_json::to_string(&op) {
|
let op_json = match serde_json::to_string(&op) {
|
||||||
Ok(j) => j,
|
Ok(j) => j,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
slog!("[crdt] Failed to serialize op: {e}");
|
slog!("[crdt] Failed to serialize op: {e}");
|
||||||
|
PERSIST_PENDING.fetch_sub(1, Ordering::Relaxed);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -149,6 +164,13 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
if let Err(e) = result {
|
if let Err(e) = result {
|
||||||
slog!("[crdt] Failed to persist op {}: {e}", &op_id[..12]);
|
slog!("[crdt] Failed to persist op {}: {e}", &op_id[..12]);
|
||||||
}
|
}
|
||||||
|
PERSIST_PENDING.fetch_sub(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
PersistMsg::Flush(reply) => {
|
||||||
|
// All ops queued before this message have already been processed.
|
||||||
|
let _ = reply.send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -163,6 +185,7 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
test_job_index,
|
test_job_index,
|
||||||
agent_throttle_index,
|
agent_throttle_index,
|
||||||
gateway_project_index,
|
gateway_project_index,
|
||||||
|
llm_session_index,
|
||||||
persist_tx,
|
persist_tx,
|
||||||
lamport_floor,
|
lamport_floor,
|
||||||
tombstones,
|
tombstones,
|
||||||
@@ -181,6 +204,43 @@ pub async fn init(db_path: &Path) -> Result<(), sqlx::Error> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Signal the persistence background task to drain and wait until all currently-queued
|
||||||
|
/// ops have been written to SQLite, or until `timeout` elapses.
|
||||||
|
///
|
||||||
|
/// Because the persistence channel is FIFO, a `Flush` sentinel processed by the task
|
||||||
|
/// guarantees that every `Op` sent before it has already been committed. On timeout a
|
||||||
|
/// warning is logged with the queue depth so regressions are visible in logs.
|
||||||
|
pub(crate) async fn flush_persistence(timeout: std::time::Duration) {
|
||||||
|
let Some(state_mutex) = super::get_crdt() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let persist_tx = {
|
||||||
|
let Ok(state) = state_mutex.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
state.persist_tx.clone()
|
||||||
|
};
|
||||||
|
let pending_at_send = PERSIST_PENDING.load(Ordering::Relaxed);
|
||||||
|
let (tx, rx) = oneshot::channel();
|
||||||
|
if persist_tx.send(PersistMsg::Flush(tx)).is_err() {
|
||||||
|
slog!("[rebuild] Persistence channel closed — skipping flush");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match tokio::time::timeout(timeout, rx).await {
|
||||||
|
Ok(_) => {
|
||||||
|
slog!("[rebuild] Persistence channel drained ({pending_at_send} ops flushed)");
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
let pending_now = PERSIST_PENDING.load(Ordering::Relaxed);
|
||||||
|
slog!(
|
||||||
|
"[rebuild] WARNING: persistence flush timed out after {}ms; \
|
||||||
|
queue_depth_at_send={pending_at_send} queue_depth_now={pending_now}",
|
||||||
|
timeout.as_millis()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Load or create the Ed25519 keypair used by this node.
|
/// Load or create the Ed25519 keypair used by this node.
|
||||||
async fn load_or_create_keypair(pool: &SqlitePool) -> Result<Ed25519KeyPair, sqlx::Error> {
|
async fn load_or_create_keypair(pool: &SqlitePool) -> Result<Ed25519KeyPair, sqlx::Error> {
|
||||||
let row: Option<(Vec<u8>,)> =
|
let row: Option<(Vec<u8>,)> =
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ mod tests;
|
|||||||
// ── Re-exports for crdt_state siblings ──────────────────────────────
|
// ── Re-exports for crdt_state siblings ──────────────────────────────
|
||||||
|
|
||||||
pub use init::init;
|
pub use init::init;
|
||||||
|
pub(crate) use init::{PersistMsg, flush_persistence};
|
||||||
|
|
||||||
/// Subscribe to CRDT state-transition events.
|
/// Subscribe to CRDT state-transition events.
|
||||||
///
|
///
|
||||||
@@ -38,11 +39,11 @@ pub fn subscribe() -> Option<broadcast::Receiver<super::types::CrdtEvent>> {
|
|||||||
pub(super) use apply::{apply_and_persist, emit_event};
|
pub(super) use apply::{apply_and_persist, emit_event};
|
||||||
pub(super) use indices::{
|
pub(super) use indices::{
|
||||||
rebuild_active_agent_index, rebuild_agent_throttle_index, rebuild_gateway_project_index,
|
rebuild_active_agent_index, rebuild_agent_throttle_index, rebuild_gateway_project_index,
|
||||||
rebuild_index, rebuild_merge_job_index, rebuild_node_index, rebuild_test_job_index,
|
rebuild_index, rebuild_llm_session_index, rebuild_merge_job_index, rebuild_node_index,
|
||||||
rebuild_token_index,
|
rebuild_test_job_index, rebuild_token_index,
|
||||||
};
|
};
|
||||||
|
pub(crate) use statics::{PERSIST_PENDING, all_ops_lock, vector_clock_lock};
|
||||||
pub(super) use statics::{SYNC_TX, track_op};
|
pub(super) use statics::{SYNC_TX, track_op};
|
||||||
pub(crate) use statics::{all_ops_lock, vector_clock_lock};
|
|
||||||
|
|
||||||
// ── CrdtState struct ─────────────────────────────────────────────────
|
// ── CrdtState struct ─────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -66,8 +67,10 @@ pub(super) struct CrdtState {
|
|||||||
pub(super) agent_throttle_index: HashMap<String, usize>,
|
pub(super) agent_throttle_index: HashMap<String, usize>,
|
||||||
/// Maps project name → index in the gateway_projects ListCrdt for O(1) lookup.
|
/// Maps project name → index in the gateway_projects ListCrdt for O(1) lookup.
|
||||||
pub(super) gateway_project_index: HashMap<String, usize>,
|
pub(super) gateway_project_index: HashMap<String, usize>,
|
||||||
/// Channel sender for fire-and-forget op persistence.
|
/// Maps session_id → index in the llm_sessions ListCrdt for O(1) lookup.
|
||||||
pub(super) persist_tx: mpsc::UnboundedSender<SignedOp>,
|
pub(super) llm_session_index: HashMap<String, usize>,
|
||||||
|
/// Channel sender for op persistence and drain signalling.
|
||||||
|
pub(super) persist_tx: mpsc::UnboundedSender<init::PersistMsg>,
|
||||||
/// Max sequence number seen across all ops during init() replay.
|
/// Max sequence number seen across all ops during init() replay.
|
||||||
///
|
///
|
||||||
/// Newly-created registers (post-init) must have their Lamport clock
|
/// Newly-created registers (post-init) must have their Lamport clock
|
||||||
@@ -122,23 +125,18 @@ pub(super) fn get_crdt() -> Option<&'static Mutex<CrdtState>> {
|
|||||||
/// This avoids the async SQLite setup from `init()`. Ops are sent to a
|
/// This avoids the async SQLite setup from `init()`. Ops are sent to a
|
||||||
/// channel whose receiver is leaked (so nothing is persisted, but the channel
|
/// channel whose receiver is leaked (so nothing is persisted, but the channel
|
||||||
/// stays open and `apply_and_persist` succeeds silently).
|
/// stays open and `apply_and_persist` succeeds silently).
|
||||||
/// Safe to call multiple times — subsequent calls are no-ops (thread-local).
|
/// Always resets all thread-local state so each call produces a clean slate —
|
||||||
|
/// no cross-test pollution when two tests share the same thread.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub fn init_for_test() {
|
pub fn init_for_test() {
|
||||||
// Initialise thread-local CRDT for test isolation.
|
|
||||||
// Only creates a new CRDT if one isn't set yet on this thread;
|
|
||||||
// subsequent calls are no-ops (matching the old OnceLock semantics
|
|
||||||
// while keeping each thread isolated).
|
|
||||||
CRDT_STATE_TL.with(|lock| {
|
|
||||||
if lock.get().is_none() {
|
|
||||||
let keypair = make_keypair();
|
let keypair = make_keypair();
|
||||||
let crdt = BaseCrdt::<PipelineDoc>::new(&keypair);
|
let crdt = BaseCrdt::<PipelineDoc>::new(&keypair);
|
||||||
let (persist_tx, rx) = mpsc::unbounded_channel();
|
let (persist_tx, rx) = mpsc::unbounded_channel::<init::PersistMsg>();
|
||||||
// Leak the receiver so the channel stays open: apply_and_persist
|
// Leak the receiver so the channel stays open: apply_and_persist
|
||||||
// can then send without error, preventing [crdt_persist] WARNs
|
// can then send without error, preventing [crdt_persist] WARNs
|
||||||
// from racing with other tests that watch the global log buffer.
|
// from racing with other tests that watch the global log buffer.
|
||||||
std::mem::forget(rx);
|
std::mem::forget(rx);
|
||||||
let state = CrdtState {
|
let fresh = CrdtState {
|
||||||
crdt,
|
crdt,
|
||||||
keypair,
|
keypair,
|
||||||
index: HashMap::new(),
|
index: HashMap::new(),
|
||||||
@@ -149,22 +147,36 @@ pub fn init_for_test() {
|
|||||||
test_job_index: HashMap::new(),
|
test_job_index: HashMap::new(),
|
||||||
agent_throttle_index: HashMap::new(),
|
agent_throttle_index: HashMap::new(),
|
||||||
gateway_project_index: HashMap::new(),
|
gateway_project_index: HashMap::new(),
|
||||||
|
llm_session_index: HashMap::new(),
|
||||||
persist_tx,
|
persist_tx,
|
||||||
lamport_floor: 0,
|
lamport_floor: 0,
|
||||||
tombstones: HashSet::new(),
|
tombstones: HashSet::new(),
|
||||||
};
|
};
|
||||||
let _ = lock.set(Mutex::new(state));
|
CRDT_STATE_TL.with(|lock| {
|
||||||
|
if let Some(mutex) = lock.get() {
|
||||||
|
// Already set on this thread — replace contents so the second
|
||||||
|
// (and subsequent) test on the same thread starts clean.
|
||||||
|
*mutex.lock().unwrap() = fresh;
|
||||||
|
} else {
|
||||||
|
let _ = lock.set(Mutex::new(fresh));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let _ = statics::CRDT_EVENT_TX.get_or_init(|| broadcast::channel::<CrdtEvent>(256).0);
|
let _ = statics::CRDT_EVENT_TX.get_or_init(|| broadcast::channel::<CrdtEvent>(256).0);
|
||||||
let _ = statics::SYNC_TX.get_or_init(|| broadcast::channel::<SignedOp>(1024).0);
|
let _ = statics::SYNC_TX.get_or_init(|| broadcast::channel::<SignedOp>(1024).0);
|
||||||
// Per-thread op journal + vector clock — keeps parallel tests' writes
|
// Per-thread op journal + vector clock — always cleared so a second test
|
||||||
// from corrupting each other's view of ALL_OPS (notably, one thread's
|
// on the same thread cannot see ops written by the first.
|
||||||
// `apply_compaction` could otherwise prune another thread's ops).
|
|
||||||
statics::ALL_OPS_TL.with(|lock| {
|
statics::ALL_OPS_TL.with(|lock| {
|
||||||
|
if let Some(mutex) = lock.get() {
|
||||||
|
mutex.lock().unwrap().clear();
|
||||||
|
} else {
|
||||||
let _ = lock.set(Mutex::new(Vec::new()));
|
let _ = lock.set(Mutex::new(Vec::new()));
|
||||||
|
}
|
||||||
});
|
});
|
||||||
statics::VECTOR_CLOCK_TL.with(|lock| {
|
statics::VECTOR_CLOCK_TL.with(|lock| {
|
||||||
|
if let Some(mutex) = lock.get() {
|
||||||
|
mutex.lock().unwrap().clear();
|
||||||
|
} else {
|
||||||
let _ = lock.set(Mutex::new(VectorClock::new()));
|
let _ = lock.set(Mutex::new(VectorClock::new()));
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
//! tests do not share `ALL_OPS` — preventing one test's `apply_compaction`
|
//! tests do not share `ALL_OPS` — preventing one test's `apply_compaction`
|
||||||
//! from pruning another test's freshly-written ops.
|
//! from pruning another test's freshly-written ops.
|
||||||
|
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
use std::sync::{Mutex, OnceLock};
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
use bft_json_crdt::json_crdt::SignedOp;
|
use bft_json_crdt::json_crdt::SignedOp;
|
||||||
@@ -19,6 +20,14 @@ use super::super::VectorClock;
|
|||||||
use super::super::hex;
|
use super::super::hex;
|
||||||
use super::super::types::CrdtEvent;
|
use super::super::types::CrdtEvent;
|
||||||
|
|
||||||
|
/// Count of ops queued in the persistence channel that have not yet been written to SQLite.
|
||||||
|
///
|
||||||
|
/// Incremented when an op is sent into the channel; decremented after the
|
||||||
|
/// persistence task commits it. Exposed via `dump_crdt_state` as
|
||||||
|
/// `pending_persist_ops_count` so operators can tell whether there is a flush
|
||||||
|
/// backlog before calling `rebuild_and_restart`.
|
||||||
|
pub(crate) static PERSIST_PENDING: AtomicUsize = AtomicUsize::new(0);
|
||||||
|
|
||||||
/// Broadcast channel for CRDT events (stage transitions, etc.).
|
/// Broadcast channel for CRDT events (stage transitions, etc.).
|
||||||
pub(super) static CRDT_EVENT_TX: OnceLock<broadcast::Sender<CrdtEvent>> = OnceLock::new();
|
pub(super) static CRDT_EVENT_TX: OnceLock<broadcast::Sender<CrdtEvent>> = OnceLock::new();
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
use super::super::hex;
|
use super::super::hex;
|
||||||
use super::super::read::extract_item_view;
|
use super::super::read::extract_item_view;
|
||||||
use super::super::types::PipelineDoc;
|
use super::super::types::PipelineDoc;
|
||||||
|
use super::init::PersistMsg;
|
||||||
use super::*;
|
use super::*;
|
||||||
use bft_json_crdt::json_crdt::{BaseCrdt, CrdtNode, JsonValue, SignedOp};
|
use bft_json_crdt::json_crdt::{BaseCrdt, CrdtNode, JsonValue, SignedOp};
|
||||||
use bft_json_crdt::keypair::make_keypair;
|
use bft_json_crdt::keypair::make_keypair;
|
||||||
@@ -222,7 +223,7 @@ async fn init_and_write_read_roundtrip() {
|
|||||||
fn persist_tx_send_failure_logs_warn_with_op_type_and_seq() {
|
fn persist_tx_send_failure_logs_warn_with_op_type_and_seq() {
|
||||||
let kp = make_keypair();
|
let kp = make_keypair();
|
||||||
let crdt = BaseCrdt::<PipelineDoc>::new(&kp);
|
let crdt = BaseCrdt::<PipelineDoc>::new(&kp);
|
||||||
let (persist_tx, persist_rx) = mpsc::unbounded_channel::<SignedOp>();
|
let (persist_tx, persist_rx) = mpsc::unbounded_channel::<PersistMsg>();
|
||||||
|
|
||||||
let mut state = CrdtState {
|
let mut state = CrdtState {
|
||||||
crdt,
|
crdt,
|
||||||
@@ -235,6 +236,7 @@ fn persist_tx_send_failure_logs_warn_with_op_type_and_seq() {
|
|||||||
test_job_index: HashMap::new(),
|
test_job_index: HashMap::new(),
|
||||||
agent_throttle_index: HashMap::new(),
|
agent_throttle_index: HashMap::new(),
|
||||||
gateway_project_index: HashMap::new(),
|
gateway_project_index: HashMap::new(),
|
||||||
|
llm_session_index: HashMap::new(),
|
||||||
persist_tx,
|
persist_tx,
|
||||||
lamport_floor: 0,
|
lamport_floor: 0,
|
||||||
tombstones: std::collections::HashSet::new(),
|
tombstones: std::collections::HashSet::new(),
|
||||||
@@ -296,7 +298,7 @@ fn persist_tx_send_failure_logs_warn_with_op_type_and_seq() {
|
|||||||
fn persist_tx_send_success_emits_no_warn() {
|
fn persist_tx_send_success_emits_no_warn() {
|
||||||
let kp = make_keypair();
|
let kp = make_keypair();
|
||||||
let crdt = BaseCrdt::<PipelineDoc>::new(&kp);
|
let crdt = BaseCrdt::<PipelineDoc>::new(&kp);
|
||||||
let (persist_tx, _persist_rx) = mpsc::unbounded_channel::<SignedOp>();
|
let (persist_tx, _persist_rx) = mpsc::unbounded_channel::<PersistMsg>();
|
||||||
|
|
||||||
let mut state = CrdtState {
|
let mut state = CrdtState {
|
||||||
crdt,
|
crdt,
|
||||||
@@ -309,6 +311,7 @@ fn persist_tx_send_success_emits_no_warn() {
|
|||||||
test_job_index: HashMap::new(),
|
test_job_index: HashMap::new(),
|
||||||
agent_throttle_index: HashMap::new(),
|
agent_throttle_index: HashMap::new(),
|
||||||
gateway_project_index: HashMap::new(),
|
gateway_project_index: HashMap::new(),
|
||||||
|
llm_session_index: HashMap::new(),
|
||||||
persist_tx,
|
persist_tx,
|
||||||
lamport_floor: 0,
|
lamport_floor: 0,
|
||||||
tombstones: std::collections::HashSet::new(),
|
tombstones: std::collections::HashSet::new(),
|
||||||
@@ -485,3 +488,102 @@ async fn restart_new_register_resumes_from_lamport_floor() {
|
|||||||
max_seq,
|
max_seq,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Regression test for story 1116: ops sent before `flush_persistence` must all be
|
||||||
|
/// present in the `crdt_ops` SQLite table after the flush completes.
|
||||||
|
///
|
||||||
|
/// Bug: `rebuild_and_restart` called `exec()` before the persistence task had
|
||||||
|
/// a chance to drain the unbounded channel, silently dropping queued ops.
|
||||||
|
///
|
||||||
|
/// Reproducer: apply N ops → call `rebuild_and_restart` → the process re-execs
|
||||||
|
/// and on the next startup `persisted_ops_count` is < N (lost ops).
|
||||||
|
/// Fixed by: send a `Flush` sentinel through the channel before `exec()`; the
|
||||||
|
/// task echoes back only after all preceding `Op` messages are committed.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn flush_persistence_drains_all_ops_before_ack() {
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use tokio::sync::oneshot;
|
||||||
|
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let db_path = tmp.path().join("flush_drain_test.db");
|
||||||
|
|
||||||
|
let options = SqliteConnectOptions::new()
|
||||||
|
.filename(&db_path)
|
||||||
|
.create_if_missing(true);
|
||||||
|
let pool = SqlitePool::connect_with(options).await.unwrap();
|
||||||
|
sqlx::migrate!("./migrations").run(&pool).await.unwrap();
|
||||||
|
|
||||||
|
let kp = make_keypair();
|
||||||
|
let mut crdt = BaseCrdt::<PipelineDoc>::new(&kp);
|
||||||
|
|
||||||
|
// Spawn an isolated persistence task — same logic as init() but without
|
||||||
|
// touching the global singleton (keeping this test fully self-contained).
|
||||||
|
let (tx, mut rx) = mpsc::unbounded_channel::<PersistMsg>();
|
||||||
|
let pool_clone = pool.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
let counter = AtomicUsize::new(0);
|
||||||
|
while let Some(msg) = rx.recv().await {
|
||||||
|
match msg {
|
||||||
|
PersistMsg::Op(op) => {
|
||||||
|
let op_json = serde_json::to_string(&op).unwrap();
|
||||||
|
let op_id = hex::encode(&op.id());
|
||||||
|
let seq = op.inner.seq as i64;
|
||||||
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO crdt_ops (op_id, seq, op_json, created_at) \
|
||||||
|
VALUES (?1, ?2, ?3, ?4) ON CONFLICT(op_id) DO NOTHING",
|
||||||
|
)
|
||||||
|
.bind(&op_id)
|
||||||
|
.bind(seq)
|
||||||
|
.bind(&op_json)
|
||||||
|
.bind(&now)
|
||||||
|
.execute(&pool_clone)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
counter.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
PersistMsg::Flush(reply) => {
|
||||||
|
let _ = reply.send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const N: usize = 10;
|
||||||
|
for i in 0..N {
|
||||||
|
let item: JsonValue = json!({
|
||||||
|
"story_id": format!("1116_drain_{i}"),
|
||||||
|
"stage": "1_backlog",
|
||||||
|
"name": format!("Drain Test {i}"),
|
||||||
|
"agent": "",
|
||||||
|
"retry_count": 0.0,
|
||||||
|
"blocked": false,
|
||||||
|
"depends_on": "",
|
||||||
|
"claimed_by": "",
|
||||||
|
"claimed_at": 0.0,
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
let op = crdt.doc.items.insert(ROOT_ID, item).sign(&kp);
|
||||||
|
crdt.apply(op.clone());
|
||||||
|
tx.send(PersistMsg::Op(Box::new(op))).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send flush sentinel and wait — all N ops must be committed first.
|
||||||
|
let (flush_tx, flush_rx) = oneshot::channel();
|
||||||
|
tx.send(PersistMsg::Flush(flush_tx)).unwrap();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(5), flush_rx)
|
||||||
|
.await
|
||||||
|
.expect("flush timed out — persistence task did not drain within 5 s")
|
||||||
|
.expect("flush oneshot dropped unexpectedly");
|
||||||
|
|
||||||
|
// Verify all N ops are in the database.
|
||||||
|
let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM crdt_ops")
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
count as usize, N,
|
||||||
|
"all {N} ops must be in crdt_ops after flush; got {count}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -46,6 +46,121 @@ pub struct PipelineDoc {
|
|||||||
pub agent_throttle: ListCrdt<AgentThrottleCrdt>,
|
pub agent_throttle: ListCrdt<AgentThrottleCrdt>,
|
||||||
pub gateway_projects: ListCrdt<GatewayProjectCrdt>,
|
pub gateway_projects: ListCrdt<GatewayProjectCrdt>,
|
||||||
pub gateway_config: GatewayConfigCrdt,
|
pub gateway_config: GatewayConfigCrdt,
|
||||||
|
/// Append-only log of every pipeline transition, persisted as CRDT ops.
|
||||||
|
pub event_log: ListCrdt<EventLogEntryCrdt>,
|
||||||
|
/// Per-session LLM context state (high-water marks for event log injection).
|
||||||
|
pub llm_sessions: ListCrdt<LlmSessionCrdt>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CRDT entry representing a single persisted pipeline stage-transition event.
|
||||||
|
///
|
||||||
|
/// Entries are append-only; once written they are never updated or tombstoned.
|
||||||
|
/// The `event_seq` field is a per-sled monotonic counter computed at write time
|
||||||
|
/// (count of existing entries for that sled), giving deterministic ordering for
|
||||||
|
/// all transitions recorded by a single node even after CRDT replay on restart.
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode, Debug)]
|
||||||
|
pub struct EventLogEntryCrdt {
|
||||||
|
/// Monotonic sequence number for this sled (0, 1, 2, …). Stored as `f64`
|
||||||
|
/// because all CRDT scalar registers use JSON numbers.
|
||||||
|
pub event_seq: LwwRegisterCrdt<f64>,
|
||||||
|
/// Hex-encoded Ed25519 public key of the sled that recorded this event.
|
||||||
|
pub sled_id: LwwRegisterCrdt<String>,
|
||||||
|
/// Unix timestamp (seconds) when the transition fired.
|
||||||
|
pub timestamp: LwwRegisterCrdt<f64>,
|
||||||
|
/// Story ID of the work item that transitioned (e.g. `"42_story_foo"`).
|
||||||
|
pub story_id: LwwRegisterCrdt<String>,
|
||||||
|
/// Human-readable label of the stage before the transition.
|
||||||
|
pub from_stage: LwwRegisterCrdt<String>,
|
||||||
|
/// Human-readable label of the stage after the transition.
|
||||||
|
pub to_stage: LwwRegisterCrdt<String>,
|
||||||
|
/// String label of the `PipelineEvent` variant that triggered the transition.
|
||||||
|
pub pipeline_event: LwwRegisterCrdt<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CRDT entry tracking an LLM session's event-log injection state.
|
||||||
|
///
|
||||||
|
/// Each session (keyed by `session_id`, typically a Matrix room ID) records the
|
||||||
|
/// per-sled high-water marks so that `assemble_prompt_context` can inject only
|
||||||
|
/// events the LLM has not yet seen and then advance the marks atomically.
|
||||||
|
#[add_crdt_fields]
|
||||||
|
#[derive(Clone, CrdtNode, Debug)]
|
||||||
|
pub struct LlmSessionCrdt {
|
||||||
|
/// Stable session identifier (e.g. Matrix room ID).
|
||||||
|
pub session_id: LwwRegisterCrdt<String>,
|
||||||
|
/// Human-readable persona name (e.g. `"Timmy"`).
|
||||||
|
pub persona_name: LwwRegisterCrdt<String>,
|
||||||
|
/// Scope wire string parsed by [`ScopeFilter::from_scope_str`]: `"all"`,
|
||||||
|
/// `"sleds:hex1,hex2"`, or legacy `"single-sled"` / empty (→ local sled).
|
||||||
|
pub scope: LwwRegisterCrdt<String>,
|
||||||
|
/// JSON-serialised `BTreeMap<sled_id, last_seen_event_seq>` tracking how far
|
||||||
|
/// each sled's event stream has been injected into this session's prompts.
|
||||||
|
pub high_water: LwwRegisterCrdt<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which sleds' events an LLM session may see.
|
||||||
|
///
|
||||||
|
/// Stored as a compact string in the CRDT register and parsed at read time.
|
||||||
|
/// The default for a freshly-created session with no stored scope is
|
||||||
|
/// [`ScopeFilter::LocalOnly`], which preserves prior single-sled behaviour.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub enum ScopeFilter {
|
||||||
|
/// Include events from every sled present in the CRDT event log.
|
||||||
|
///
|
||||||
|
/// Default for gateway-level personas (e.g. Timmy in multi-project mode).
|
||||||
|
All,
|
||||||
|
/// Include only events whose `sled_id` is in the given set.
|
||||||
|
///
|
||||||
|
/// Default for sled-level personas: the set contains only the sled's own ID.
|
||||||
|
Sleds(std::collections::BTreeSet<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeFilter {
|
||||||
|
/// Parse a wire-form scope string stored in the CRDT register.
|
||||||
|
///
|
||||||
|
/// Recognised forms:
|
||||||
|
/// - `"all"` → [`ScopeFilter::All`]
|
||||||
|
/// - `"sleds:hex1,hex2,…"` → [`ScopeFilter::Sleds`]
|
||||||
|
/// - Anything else (including legacy `"single-sled"` and empty) →
|
||||||
|
/// [`ScopeFilter::Sleds`] with an empty set; callers should fall back
|
||||||
|
/// to the local sled ID in that case.
|
||||||
|
pub fn from_scope_str(s: &str) -> Self {
|
||||||
|
if s == "all" {
|
||||||
|
return ScopeFilter::All;
|
||||||
|
}
|
||||||
|
if let Some(rest) = s.strip_prefix("sleds:") {
|
||||||
|
let ids = rest
|
||||||
|
.split(',')
|
||||||
|
.filter(|id| !id.is_empty())
|
||||||
|
.map(|id| id.to_string())
|
||||||
|
.collect();
|
||||||
|
return ScopeFilter::Sleds(ids);
|
||||||
|
}
|
||||||
|
ScopeFilter::Sleds(std::collections::BTreeSet::new())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Encode this filter as the compact wire string stored in the CRDT.
|
||||||
|
pub fn to_scope_str(&self) -> String {
|
||||||
|
match self {
|
||||||
|
ScopeFilter::All => "all".to_string(),
|
||||||
|
ScopeFilter::Sleds(ids) => {
|
||||||
|
let joined = ids.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(",");
|
||||||
|
format!("sleds:{joined}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read-side snapshot of a single LLM session entry.
|
||||||
|
pub struct LlmSessionView {
|
||||||
|
/// Stable session identifier.
|
||||||
|
pub session_id: String,
|
||||||
|
/// Persona name for the bot in this session.
|
||||||
|
pub persona_name: String,
|
||||||
|
/// Parsed event-scope filter derived from the `scope` CRDT register.
|
||||||
|
pub scope_filter: ScopeFilter,
|
||||||
|
/// Decoded high-water map: sled_id → last seen event_seq.
|
||||||
|
pub high_water: std::collections::BTreeMap<String, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// CRDT sub-document representing a single pipeline work item with LWW fields for stage, agent, etc.
|
/// CRDT sub-document representing a single pipeline work item with LWW fields for stage, agent, etc.
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ use crate::pipeline_state::{AgentClaim, Stage, stage_dir_name};
|
|||||||
///
|
///
|
||||||
/// Returns `true` if the item was found and the op was applied, `false` otherwise.
|
/// Returns `true` if the item was found and the op was applied, `false` otherwise.
|
||||||
pub fn set_depends_on(story_id: &str, deps: &[u32]) -> bool {
|
pub fn set_depends_on(story_id: &str, deps: &[u32]) -> bool {
|
||||||
|
{
|
||||||
let Some(state_mutex) = get_crdt() else {
|
let Some(state_mutex) = get_crdt() else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
@@ -36,6 +37,10 @@ pub fn set_depends_on(story_id: &str, deps: &[u32]) -> bool {
|
|||||||
serde_json::to_string(deps).unwrap_or_default()
|
serde_json::to_string(deps).unwrap_or_default()
|
||||||
};
|
};
|
||||||
apply_and_persist(&mut state, |s| s.crdt.doc.items[idx].depends_on.set(value));
|
apply_and_persist(&mut state, |s| s.crdt.doc.items[idx].depends_on.set(value));
|
||||||
|
}
|
||||||
|
// Drop the CRDT lock before calling sync: read_item acquires the same
|
||||||
|
// mutex and would deadlock if the lock were still held here.
|
||||||
|
crate::db::ops::sync_item_depends_on(story_id);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,6 +160,9 @@ pub fn set_name(story_id: &str, name: Option<&str>) -> bool {
|
|||||||
apply_and_persist(&mut state, |s| {
|
apply_and_persist(&mut state, |s| {
|
||||||
s.crdt.doc.items[idx].name.set(value.clone())
|
s.crdt.doc.items[idx].name.set(value.clone())
|
||||||
});
|
});
|
||||||
|
// Drop the lock before the shadow write so `read_item` can acquire it.
|
||||||
|
drop(state);
|
||||||
|
crate::db::sync_item_name(story_id);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,6 +183,7 @@ pub fn set_agent(story_id: &str, agent: Option<crate::config::AgentName>) -> boo
|
|||||||
let Some(state_mutex) = get_crdt() else {
|
let Some(state_mutex) = get_crdt() else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
{
|
||||||
let Ok(mut state) = state_mutex.lock() else {
|
let Ok(mut state) = state_mutex.lock() else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
@@ -185,6 +194,10 @@ pub fn set_agent(story_id: &str, agent: Option<crate::config::AgentName>) -> boo
|
|||||||
apply_and_persist(&mut state, |s| {
|
apply_and_persist(&mut state, |s| {
|
||||||
s.crdt.doc.items[idx].agent.set(value.clone())
|
s.crdt.doc.items[idx].agent.set(value.clone())
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
// Sync the updated agent to the SQLite shadow table. Must be called after
|
||||||
|
// releasing the CRDT mutex so read_item can re-acquire it without deadlock.
|
||||||
|
crate::db::ops::sync_item_agent(story_id);
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -556,6 +569,24 @@ pub fn set_retry_count(story_id: &str, count: i64) {
|
|||||||
_ => return,
|
_ => return,
|
||||||
};
|
};
|
||||||
write_item(story_id, &new_stage, None, None, None, None);
|
write_item(story_id, &new_stage, None, None, None, None);
|
||||||
|
if let Some(db) = crate::db::shadow_write::PIPELINE_DB.get() {
|
||||||
|
let stage = stage_dir_name(&new_stage).to_string();
|
||||||
|
let name = Some(item.name().to_string());
|
||||||
|
let agent = item.agent().map(|a| a.to_string());
|
||||||
|
let depends_on = (!item.depends_on().is_empty())
|
||||||
|
.then(|| serde_json::to_string(item.depends_on()).ok())
|
||||||
|
.flatten();
|
||||||
|
let msg = crate::db::shadow_write::PipelineWriteMsg {
|
||||||
|
story_id: story_id.to_string(),
|
||||||
|
stage,
|
||||||
|
name,
|
||||||
|
agent,
|
||||||
|
retry_count: Some(count.max(0)),
|
||||||
|
depends_on,
|
||||||
|
content: None,
|
||||||
|
};
|
||||||
|
let _ = db.tx.send(msg);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Increment `retries` by 1 and return the new value.
|
/// Increment `retries` by 1 and return the new value.
|
||||||
@@ -605,5 +636,23 @@ pub fn bump_retry_count(story_id: &str) -> i64 {
|
|||||||
_ => return 0,
|
_ => return 0,
|
||||||
};
|
};
|
||||||
write_item(story_id, &new_stage, None, None, None, None);
|
write_item(story_id, &new_stage, None, None, None, None);
|
||||||
|
if let Some(db) = crate::db::shadow_write::PIPELINE_DB.get() {
|
||||||
|
let stage = stage_dir_name(&new_stage).to_string();
|
||||||
|
let name = Some(item.name().to_string());
|
||||||
|
let agent = item.agent().map(|a| a.to_string());
|
||||||
|
let depends_on = (!item.depends_on().is_empty())
|
||||||
|
.then(|| serde_json::to_string(item.depends_on()).ok())
|
||||||
|
.flatten();
|
||||||
|
let msg = crate::db::shadow_write::PipelineWriteMsg {
|
||||||
|
story_id: story_id.to_string(),
|
||||||
|
stage,
|
||||||
|
name,
|
||||||
|
agent,
|
||||||
|
retry_count: Some(new_retries as i64),
|
||||||
|
depends_on,
|
||||||
|
content: None,
|
||||||
|
};
|
||||||
|
let _ = db.tx.send(msg);
|
||||||
|
}
|
||||||
new_retries as i64
|
new_retries as i64
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -705,6 +705,59 @@ pub fn purge_done_stage_merge_jobs() {
|
|||||||
slog!("[crdt] Purged {count} stale MergeJob entries for terminal-stage stories");
|
slog!("[crdt] Purged {count} stale MergeJob entries for terminal-stage stories");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Delete `pipeline_items` rows that correspond to CRDT-tombstoned stories.
|
||||||
|
///
|
||||||
|
/// Pre-1094 code deleted pipeline_items via a fire-and-forget channel that
|
||||||
|
/// could be lost on an abrupt restart, leaving rows with non-terminal stage
|
||||||
|
/// values for stories that no longer exist in the CRDT. This migration
|
||||||
|
/// removes those zombie rows on startup.
|
||||||
|
///
|
||||||
|
/// Idempotent: rows already absent are unaffected; running twice produces the
|
||||||
|
/// same result.
|
||||||
|
pub async fn migrate_zombie_pipeline_rows() {
|
||||||
|
let pool = match crate::db::get_shared_pool() {
|
||||||
|
Some(p) => p,
|
||||||
|
None => return,
|
||||||
|
};
|
||||||
|
let tombstone_ids = crate::crdt_state::tombstoned_ids();
|
||||||
|
sweep_zombie_rows(pool, &tombstone_ids).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Inner sweep used by [`migrate_zombie_pipeline_rows`] and its tests.
|
||||||
|
///
|
||||||
|
/// Deletes every `pipeline_items` row in `ids` whose stage is not already a
|
||||||
|
/// terminal value. Returns the number of rows deleted.
|
||||||
|
#[cfg_attr(test, allow(dead_code))]
|
||||||
|
pub(crate) async fn sweep_zombie_rows(pool: &sqlx::SqlitePool, ids: &[String]) -> u32 {
|
||||||
|
if ids.is_empty() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
let mut cleaned = 0u32;
|
||||||
|
for story_id in ids {
|
||||||
|
match sqlx::query(
|
||||||
|
"DELETE FROM pipeline_items WHERE id = ?1 AND stage NOT IN \
|
||||||
|
('done','archived','abandoned','superseded','rejected')",
|
||||||
|
)
|
||||||
|
.bind(story_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) if r.rows_affected() > 0 => cleaned += 1,
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => {
|
||||||
|
slog!(
|
||||||
|
"[crdt] migrate_zombie_pipeline_rows: failed to delete '{}': {e}",
|
||||||
|
story_id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cleaned > 0 {
|
||||||
|
slog!("[crdt] Swept {cleaned} zombie pipeline_items rows for tombstoned stories");
|
||||||
|
}
|
||||||
|
cleaned
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod merge_job_migration_tests {
|
mod merge_job_migration_tests {
|
||||||
use super::super::super::state::init_for_test;
|
use super::super::super::state::init_for_test;
|
||||||
@@ -909,3 +962,100 @@ mod merge_job_migration_tests {
|
|||||||
migrate_merge_job(std::path::Path::new("/nonexistent/pipeline.db"));
|
migrate_merge_job(std::path::Path::new("/nonexistent/pipeline.db"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod zombie_row_migration_tests {
|
||||||
|
use super::super::super::state::init_for_test;
|
||||||
|
use super::*;
|
||||||
|
use sqlx::Row as _;
|
||||||
|
|
||||||
|
async fn make_pool() -> sqlx::SqlitePool {
|
||||||
|
let options = sqlx::sqlite::SqliteConnectOptions::new()
|
||||||
|
.filename(":memory:")
|
||||||
|
.create_if_missing(true);
|
||||||
|
let pool = sqlx::pool::PoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_with(options)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
sqlx::migrate!("./migrations").run(&pool).await.unwrap();
|
||||||
|
pool
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn insert_row(pool: &sqlx::SqlitePool, story_id: &str, stage: &str) {
|
||||||
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO pipeline_items \
|
||||||
|
(id, name, stage, agent, retry_count, depends_on, content, created_at, updated_at) \
|
||||||
|
VALUES (?1, ?2, ?3, NULL, 0, NULL, NULL, ?4, ?4)",
|
||||||
|
)
|
||||||
|
.bind(story_id)
|
||||||
|
.bind(story_id)
|
||||||
|
.bind(stage)
|
||||||
|
.bind(&now)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn row_stage(pool: &sqlx::SqlitePool, story_id: &str) -> Option<String> {
|
||||||
|
sqlx::query("SELECT stage FROM pipeline_items WHERE id = ?1")
|
||||||
|
.bind(story_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.map(|r| r.get(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bug 1094 regression: delete a story in `coding` stage, assert the
|
||||||
|
/// `pipeline_items` row is gone; then re-run the sweep and confirm no
|
||||||
|
/// further changes (idempotent).
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sweep_removes_zombie_coding_row_and_is_idempotent() {
|
||||||
|
init_for_test();
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let story_id = "1094_zombie_regression";
|
||||||
|
|
||||||
|
// Seed: insert a pipeline_items row in the "coding" stage.
|
||||||
|
insert_row(&pool, story_id, "coding").await;
|
||||||
|
assert_eq!(row_stage(&pool, story_id).await.as_deref(), Some("coding"));
|
||||||
|
|
||||||
|
// Tombstone the story in the CRDT (simulate evict_item outcome).
|
||||||
|
crate::crdt_state::write_item_str(
|
||||||
|
story_id,
|
||||||
|
"coding",
|
||||||
|
Some("Zombie regression story"),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
crate::crdt_state::evict_item(story_id).ok();
|
||||||
|
|
||||||
|
// Run the sweep — row must be deleted.
|
||||||
|
let deleted = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
|
||||||
|
assert_eq!(deleted, 1, "expected one zombie row to be cleaned");
|
||||||
|
assert!(
|
||||||
|
row_stage(&pool, story_id).await.is_none(),
|
||||||
|
"pipeline_items row must be gone after sweep"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Re-run is a no-op (idempotent).
|
||||||
|
let second = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
|
||||||
|
assert_eq!(second, 0, "second sweep must be a no-op");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rows already in a terminal stage must be left alone.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sweep_skips_terminal_stage_rows() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let story_id = "1094_terminal_skip";
|
||||||
|
insert_row(&pool, story_id, "done").await;
|
||||||
|
|
||||||
|
let deleted = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
|
||||||
|
assert_eq!(deleted, 0, "terminal-stage row must not be deleted");
|
||||||
|
assert!(
|
||||||
|
row_stage(&pool, story_id).await.is_some(),
|
||||||
|
"terminal-stage row must survive sweep"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -18,6 +18,6 @@ pub use item::{
|
|||||||
pub use item::write_item_str;
|
pub use item::write_item_str;
|
||||||
pub use migrations::{
|
pub use migrations::{
|
||||||
migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
|
migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
|
||||||
migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric, name_from_story_id,
|
migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric,
|
||||||
purge_done_stage_merge_jobs,
|
migrate_zombie_pipeline_rows, name_from_story_id, purge_done_stage_merge_jobs,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -60,6 +60,17 @@ pub enum ContentKey<'a> {
|
|||||||
/// completion. Read by `get_merge_status` to surface gate output for the
|
/// completion. Read by `get_merge_status` to surface gate output for the
|
||||||
/// "completed" state without a separate MergeJob CRDT register (story 1036).
|
/// "completed" state without a separate MergeJob CRDT register (story 1036).
|
||||||
MergeReport(&'a str),
|
MergeReport(&'a str),
|
||||||
|
/// Flag written by spawn.rs when a coder session exits with a non-zero exit
|
||||||
|
/// code (API error, network failure, or Claude-API-level budget exhaustion).
|
||||||
|
/// Prevents the stuck-respawn counter from incrementing for forced exits —
|
||||||
|
/// only self-exits with no file or read changes count toward the cap.
|
||||||
|
/// Consumed (read + deleted) by the commit-recovery path in pipeline advance.
|
||||||
|
CommitRecoveryForcedExit(&'a str),
|
||||||
|
/// Cumulative set of files read across all commit-recovery sessions for a
|
||||||
|
/// story, stored as a newline-separated sorted list. Used to detect whether
|
||||||
|
/// the agent made read-exploration progress even when the worktree diff did
|
||||||
|
/// not grow (story 1089, AC2). Cleared when a commit lands or the story blocks.
|
||||||
|
CommitRecoveryReadSet(&'a str),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> ContentKey<'a> {
|
impl<'a> ContentKey<'a> {
|
||||||
@@ -85,6 +96,10 @@ impl<'a> ContentKey<'a> {
|
|||||||
ContentKey::MergeFailureKind(id) => format!("{id}:merge_failure_kind"),
|
ContentKey::MergeFailureKind(id) => format!("{id}:merge_failure_kind"),
|
||||||
ContentKey::MergeSuccess(id) => format!("{id}:merge_success"),
|
ContentKey::MergeSuccess(id) => format!("{id}:merge_success"),
|
||||||
ContentKey::MergeReport(id) => format!("{id}:merge_report"),
|
ContentKey::MergeReport(id) => format!("{id}:merge_report"),
|
||||||
|
ContentKey::CommitRecoveryForcedExit(id) => {
|
||||||
|
format!("{id}:commit_recovery_forced_exit")
|
||||||
|
}
|
||||||
|
ContentKey::CommitRecoveryReadSet(id) => format!("{id}:commit_recovery_read_set"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -150,7 +165,9 @@ pub fn delete_content(key: ContentKey<'_>) {
|
|||||||
|
|
||||||
/// Ensure the in-memory content store is initialised.
|
/// Ensure the in-memory content store is initialised.
|
||||||
///
|
///
|
||||||
/// Safe to call multiple times — the `OnceLock` is set at most once.
|
/// In non-test builds: init-once via `OnceLock` (safe to call multiple times).
|
||||||
|
/// In test builds: always resets `CONTENT_STORE_TL` to an empty `HashMap` so
|
||||||
|
/// each test on the same thread starts with a clean store.
|
||||||
pub fn ensure_content_store() {
|
pub fn ensure_content_store() {
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
{
|
{
|
||||||
@@ -160,7 +177,11 @@ pub fn ensure_content_store() {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
{
|
{
|
||||||
CONTENT_STORE_TL.with(|lock| {
|
CONTENT_STORE_TL.with(|lock| {
|
||||||
if lock.get().is_none() {
|
if let Some(mutex) = lock.get() {
|
||||||
|
// Already initialised on this thread — reset to empty so the
|
||||||
|
// next test does not see content written by a previous test.
|
||||||
|
mutex.lock().unwrap().clear();
|
||||||
|
} else {
|
||||||
let _ = lock.set(Mutex::new(HashMap::new()));
|
let _ = lock.set(Mutex::new(HashMap::new()));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -188,6 +209,41 @@ pub(super) fn init_content_store(map: HashMap<String, String>) {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Regression: two sequential `ensure_content_store()` + write + read cycles
|
||||||
|
/// in the same test body must not see each other's content. Before the fix,
|
||||||
|
/// `ensure_content_store()` was a no-op on the second call (OnceLock gating),
|
||||||
|
/// so the second cycle could read items written in the first cycle.
|
||||||
|
#[test]
|
||||||
|
fn sequential_ensure_content_store_resets_state() {
|
||||||
|
// ── Cycle 1 ──────────────────────────────────────────────────────────
|
||||||
|
ensure_content_store();
|
||||||
|
write_content(ContentKey::Story("1111_cycle1"), "cycle-one body");
|
||||||
|
assert_eq!(
|
||||||
|
read_content(ContentKey::Story("1111_cycle1")).as_deref(),
|
||||||
|
Some("cycle-one body"),
|
||||||
|
"cycle 1: item must be readable after write"
|
||||||
|
);
|
||||||
|
|
||||||
|
// ── Cycle 2: reset, write a different item ────────────────────────────
|
||||||
|
ensure_content_store();
|
||||||
|
// Cycle-1 item must no longer be visible.
|
||||||
|
assert!(
|
||||||
|
read_content(ContentKey::Story("1111_cycle1")).is_none(),
|
||||||
|
"cycle 2: store must be empty; cycle-1 content must not bleed through"
|
||||||
|
);
|
||||||
|
write_content(ContentKey::Story("1111_cycle2"), "cycle-two body");
|
||||||
|
assert_eq!(
|
||||||
|
read_content(ContentKey::Story("1111_cycle2")).as_deref(),
|
||||||
|
Some("cycle-two body"),
|
||||||
|
"cycle 2: own item must be readable"
|
||||||
|
);
|
||||||
|
// And cycle-1 key must still be absent.
|
||||||
|
assert!(
|
||||||
|
read_content(ContentKey::Story("1111_cycle1")).is_none(),
|
||||||
|
"cycle 2: cycle-1 content must remain absent after cycle-2 write"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// AC 2 regression: writing under `ContentKey::Story` is not visible under
|
/// AC 2 regression: writing under `ContentKey::Story` is not visible under
|
||||||
/// `ContentKey::GateOutput` (and vice versa). The typed key namespace, not
|
/// `ContentKey::GateOutput` (and vice versa). The typed key namespace, not
|
||||||
/// runtime substring matching, enforces the separation.
|
/// runtime substring matching, enforces the separation.
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user