crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

huskies: merge 961

Browse Source
This commit is contained in:
dave
2026-05-13 11:22:57 +00:00
parent 78b1ecdc3c
commit 8b53e20ca9
38 changed files with 327 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/agents/pool/pipeline/advance/mod.rs
+23 -10
View File
@@ -78,10 +78,15 @@ impl AgentPool {
// The coder exited with uncommitted content but no commits.
// Check if this is already a second recovery attempt (the
// first recovery respawn also produced no commits).
let recovery_key = format!("{story_id}:commit_recovery_pending");
if crate::db::read_content(&recovery_key).is_some() {
if crate::db::read_content(crate::db::ContentKey::CommitRecoveryPending(
story_id,
))
.is_some()
{
// Second attempt still produced no commits → block.
crate::db::delete_content(&recovery_key);
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
story_id,
));
slog!(
"[pipeline] Coder '{agent_name}' (commit-recovery respawn) \
still produced no commits for '{story_id}'. Blocking story."
@@ -99,7 +104,10 @@ impl AgentPool {
} else {
// First occurrence: issue a commit-only recovery respawn.
// This does NOT consume a retry_count slot.
crate::db::write_content(&recovery_key, "1");
crate::db::write_content(
crate::db::ContentKey::CommitRecoveryPending(story_id),
"1",
);
slog!(
"[pipeline] Coder '{agent_name}' exited with uncommitted work \
for '{story_id}'. Issuing commit-only recovery respawn."
@@ -125,7 +133,9 @@ impl AgentPool {
}
} else if completion.gates_passed {
// Clear any stale recovery key when the coder succeeds normally.
crate::db::delete_content(&format!("{story_id}:commit_recovery_pending"));
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
story_id,
));
// Determine effective QA mode for this story.
let qa_mode = {
let item_type = crate::agents::lifecycle::item_type_from_id(story_id);
@@ -183,7 +193,9 @@ impl AgentPool {
} else {
// Clear any stale recovery key when gates fail normally (agent committed
// but the build is broken — treat as a standard retry, not a recovery).
crate::db::delete_content(&format!("{story_id}:commit_recovery_pending"));
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
story_id,
));
// Bug 645 / 668: Before retry/block, check if the agent left committed
// work AND the agent had a passing run_tests result captured during its
// session. An agent may crash mid-output (e.g. Claude Code CLI PTY write
@@ -195,8 +207,9 @@ impl AgentPool {
// whenever script/test exits 0 inside a story worktree. Consume the
// evidence here so it does not persist to the next agent session.
let has_test_evidence =
crate::db::read_content(&format!("{story_id}:run_tests_ok")).is_some();
crate::db::delete_content(&format!("{story_id}:run_tests_ok"));
crate::db::read_content(crate::db::ContentKey::RunTestsOk(story_id))
.is_some();
crate::db::delete_content(crate::db::ContentKey::RunTestsOk(story_id));
let work_survived = has_test_evidence
&& worktree_path.as_ref().is_some_and(|wt_path| {
crate::agents::gates::worktree_has_committed_work(wt_path)
@@ -258,7 +271,7 @@ impl AgentPool {
// Persist gate_output so the retry spawn can inject it into
// --append-system-prompt (story 881).
crate::db::write_content(
&format!("{story_id}:gate_output"),
crate::db::ContentKey::GateOutput(story_id),
&completion.gate_output,
);
// Increment retry count and check if blocked.
@@ -384,7 +397,7 @@ impl AgentPool {
// Persist gate_output so the retry spawn can inject it into
// --append-system-prompt (story 881).
crate::db::write_content(
&format!("{story_id}:gate_output"),
crate::db::ContentKey::GateOutput(story_id),
&completion.gate_output,
);
if let Some(reason) = should_block_story(story_id, config.max_retries, "qa") {
server/src/agents/pool/pipeline/advance/tests.rs
+9 -6
View File
@@ -17,7 +17,7 @@ async fn pipeline_advance_coder_gates_pass_server_qa_moves_to_merge() {
fs::create_dir_all(&current).unwrap();
fs::write(current.join("9908_story_server_qa.md"), "test").unwrap();
crate::db::ensure_content_store();
crate::db::write_content("9908_story_server_qa", "test");
crate::db::write_content(crate::db::ContentKey::Story("9908_story_server_qa"), "test");
let pool = AgentPool::new_test(3001);
pool.run_pipeline_advance(
@@ -39,7 +39,7 @@ async fn pipeline_advance_coder_gates_pass_server_qa_moves_to_merge() {
// With default qa: server, story skips QA and goes straight to 4_merge/
// Lifecycle moves now update the content store, not the filesystem.
assert!(
crate::db::read_content("9908_story_server_qa").is_some(),
crate::db::read_content(crate::db::ContentKey::Story("9908_story_server_qa")).is_some(),
"story should still exist in content store after move to merge"
);
}
@@ -61,7 +61,7 @@ async fn pipeline_advance_coder_gates_pass_agent_qa_moves_to_qa() {
.unwrap();
crate::db::ensure_content_store();
crate::db::write_content(
"9909_story_agent_qa",
crate::db::ContentKey::Story("9909_story_agent_qa"),
"---\nname: Test\nqa: agent\n---\ntest",
);
@@ -85,7 +85,7 @@ async fn pipeline_advance_coder_gates_pass_agent_qa_moves_to_qa() {
// With qa: agent, story should move to 3_qa/
// Lifecycle moves now update the content store, not the filesystem.
assert!(
crate::db::read_content("9909_story_agent_qa").is_some(),
crate::db::read_content(crate::db::ContentKey::Story("9909_story_agent_qa")).is_some(),
"story should still exist in content store after move to qa"
);
}
@@ -106,7 +106,10 @@ async fn pipeline_advance_qa_gates_pass_moves_story_to_merge() {
)
.unwrap();
crate::db::ensure_content_store();
crate::db::write_content("51_story_test", "---\nname: Test\nqa: server\n---\ntest");
crate::db::write_content(
crate::db::ContentKey::Story("51_story_test"),
"---\nname: Test\nqa: server\n---\ntest",
);
let pool = AgentPool::new_test(3001);
pool.run_pipeline_advance(
@@ -128,7 +131,7 @@ async fn pipeline_advance_qa_gates_pass_moves_story_to_merge() {
// Story should have moved to 4_merge/
// Lifecycle moves now update the content store, not the filesystem.
assert!(
crate::db::read_content("51_story_test").is_some(),
crate::db::read_content(crate::db::ContentKey::Story("51_story_test")).is_some(),
"story should still exist in content store after move to merge"
);
}
server/src/agents/pool/pipeline/advance/tests_regression.rs
+48 -18
View File
@@ -76,7 +76,7 @@ async fn mergemaster_blocks_and_sends_story_blocked_when_no_commits_ahead() {
// Story should still exist in the content store after moving to merge.
assert!(
crate::db::read_content("9919_story_no_commits").is_some(),
crate::db::read_content(crate::db::ContentKey::Story("9919_story_no_commits")).is_some(),
"story should remain in content store — not removed"
);
@@ -249,7 +249,7 @@ async fn stale_mergemaster_advance_for_done_story_is_noop() {
// Seed the story in 5_done via the DB, which also writes to the CRDT.
let story_id = "9929_story_zombie_merge";
let content = "---\nname: Zombie Merge Test\n---\n";
crate::db::write_content(story_id, content);
crate::db::write_content(crate::db::ContentKey::Story(story_id), content);
crate::db::write_item_with_content(
story_id,
"5_done",
@@ -387,7 +387,10 @@ async fn work_survived_advances_to_qa_instead_of_blocking() {
// Set up the story in the content store.
crate::db::ensure_content_store();
crate::db::write_content("9945_story_survived", "---\nname: Survived Test\n---\n");
crate::db::write_content(
crate::db::ContentKey::Story("9945_story_survived"),
"---\nname: Survived Test\n---\n",
);
crate::db::write_item_with_content(
"9945_story_survived",
"2_current",
@@ -397,7 +400,10 @@ async fn work_survived_advances_to_qa_instead_of_blocking() {
// Simulate a passing run_tests call during the agent's session (bug 668):
// the agent ran script/test, it passed, and the server captured the evidence.
crate::db::write_content("9945_story_survived:run_tests_ok", "1");
crate::db::write_content(
crate::db::ContentKey::RunTestsOk("9945_story_survived"),
"1",
);
let pool = AgentPool::new_test(3001);
@@ -421,7 +427,7 @@ async fn work_survived_advances_to_qa_instead_of_blocking() {
// Story should have advanced — content store should reflect the move.
// The work-survived check should have moved it to QA (or merge for
// server qa mode), NOT incremented retry_count.
let content = crate::db::read_content("9945_story_survived")
let content = crate::db::read_content(crate::db::ContentKey::Story("9945_story_survived"))
.expect("story should exist in content store");
assert!(
!content.contains("blocked"),
@@ -482,7 +488,10 @@ async fn no_committed_work_still_retries_and_blocks() {
// Set up the story with max_retries=1 so it blocks immediately.
crate::crdt_state::init_for_test();
crate::db::ensure_content_store();
crate::db::write_content("9946_story_nowork", "---\nname: No Work Test\n---\n");
crate::db::write_content(
crate::db::ContentKey::Story("9946_story_nowork"),
"---\nname: No Work Test\n---\n",
);
crate::db::write_item_with_content(
"9946_story_nowork",
"2_current",
@@ -609,7 +618,7 @@ async fn gates_failed_no_test_evidence_does_not_advance() {
// Set up the story with max_retries=1 so we can observe the retry/block.
crate::db::ensure_content_store();
crate::db::write_content(
"9947_story_no_evidence",
crate::db::ContentKey::Story("9947_story_no_evidence"),
"---\nname: No Evidence Test\n---\n",
);
crate::db::write_item_with_content(
@@ -620,7 +629,7 @@ async fn gates_failed_no_test_evidence_does_not_advance() {
);
// Explicitly ensure no test evidence exists for this story.
crate::db::delete_content("9947_story_no_evidence:run_tests_ok");
crate::db::delete_content(crate::db::ContentKey::RunTestsOk("9947_story_no_evidence"));
fs::create_dir_all(root.join(".huskies")).unwrap();
fs::write(
@@ -740,7 +749,7 @@ async fn gates_failed_with_test_evidence_and_committed_work_advances() {
crate::db::ensure_content_store();
crate::db::write_content(
"9948_story_with_evidence",
crate::db::ContentKey::Story("9948_story_with_evidence"),
"---\nname: With Evidence Test\n---\n",
);
crate::db::write_item_with_content(
@@ -752,7 +761,10 @@ async fn gates_failed_with_test_evidence_and_committed_work_advances() {
// Write the run_tests evidence — simulates the agent having called run_tests
// MCP and getting a passing result before it crashed.
crate::db::write_content("9948_story_with_evidence:run_tests_ok", "1");
crate::db::write_content(
crate::db::ContentKey::RunTestsOk("9948_story_with_evidence"),
"1",
);
let pool = AgentPool::new_test(3001);
@@ -774,7 +786,7 @@ async fn gates_failed_with_test_evidence_and_committed_work_advances() {
.await;
// Story should advance (not blocked, no retry_count).
let content = crate::db::read_content("9948_story_with_evidence")
let content = crate::db::read_content(crate::db::ContentKey::Story("9948_story_with_evidence"))
.expect("story must exist in content store");
assert!(
!content.contains("blocked"),
@@ -786,7 +798,10 @@ async fn gates_failed_with_test_evidence_and_committed_work_advances() {
);
// Evidence must be consumed (cleared) after use.
assert!(
crate::db::read_content("9948_story_with_evidence:run_tests_ok").is_none(),
crate::db::read_content(crate::db::ContentKey::RunTestsOk(
"9948_story_with_evidence"
))
.is_none(),
"run_tests evidence must be cleared after pipeline advance consumes it"
);
}
@@ -919,7 +934,9 @@ stage = "coder"
crate::db::ItemMeta::named("Recovery Test"),
);
// Ensure no stale recovery key exists.
crate::db::delete_content("9954_story_recovery:commit_recovery_pending");
crate::db::delete_content(crate::db::ContentKey::CommitRecoveryPending(
"9954_story_recovery",
));
let pool = AgentPool::new_test(3001);
@@ -966,7 +983,10 @@ stage = "coder"
// The recovery key must be set so a second failure triggers a block.
assert!(
crate::db::read_content("9954_story_recovery:commit_recovery_pending").is_some(),
crate::db::read_content(crate::db::ContentKey::CommitRecoveryPending(
"9954_story_recovery"
))
.is_some(),
"commit_recovery_pending key must be set after issuing recovery respawn"
);
}
@@ -1008,7 +1028,10 @@ stage = "coder"
// Simulate the recovery key already being set (first recovery respawn was
// issued previously).
crate::db::write_content("9955_story_recovery2:commit_recovery_pending", "1");
crate::db::write_content(
crate::db::ContentKey::CommitRecoveryPending("9955_story_recovery2"),
"1",
);
let pool = AgentPool::new_test(3001);
let mut rx = pool.watcher_tx.subscribe();
@@ -1052,7 +1075,10 @@ stage = "coder"
// The recovery key must be cleared after blocking.
assert!(
crate::db::read_content("9955_story_recovery2:commit_recovery_pending").is_none(),
crate::db::read_content(crate::db::ContentKey::CommitRecoveryPending(
"9955_story_recovery2"
))
.is_none(),
"commit_recovery_pending key must be cleared after blocking the story"
);
@@ -1128,7 +1154,10 @@ async fn coder_completion_with_test_evidence_and_zero_commits_does_not_advance()
// Simulate the agent having called run_tests with a passing result (bug-645
// evidence) — but the feature branch still has zero commits ahead of master.
crate::db::write_content("9953_story_zero_commits:run_tests_ok", "1");
crate::db::write_content(
crate::db::ContentKey::RunTestsOk("9953_story_zero_commits"),
"1",
);
// Write a project.toml with max_retries=1 so the story blocks immediately,
// giving us a clean assertion target (StoryBlocked event).
@@ -1193,7 +1222,8 @@ async fn coder_completion_with_test_evidence_and_zero_commits_does_not_advance()
// Test evidence must have been consumed (cleared) by the advance handler.
assert!(
crate::db::read_content("9953_story_zero_commits:run_tests_ok").is_none(),
crate::db::read_content(crate::db::ContentKey::RunTestsOk("9953_story_zero_commits"))
.is_none(),
"run_tests evidence must be cleared after pipeline advance consumes it"
);
}
server/src/agents/pool/pipeline/merge/tests.rs
+2 -2
View File
@@ -703,7 +703,7 @@ async fn server_side_merge_happy_path_advances_to_done() {
if report.success {
// story_archived may or may not be true depending on gate env,
// but merge_failure must NOT be in the content store.
let content = crate::db::read_content("757a_happy");
let content = crate::db::read_content(crate::db::ContentKey::Story("757a_happy"));
if let Some(c) = content {
assert!(
!c.contains("merge_failure"),
@@ -713,7 +713,7 @@ async fn server_side_merge_happy_path_advances_to_done() {
} else {
// Gate failure (no script/test) is acceptable in test env —
// but merge_failure should be written.
let content = crate::db::read_content("757a_happy");
let content = crate::db::read_content(crate::db::ContentKey::Story("757a_happy"));
if let Some(c) = content {
// merge_failure should be written for gate failures
assert!(