crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 26 Wiki Activity

huskies: merge 870

Browse Source
This commit is contained in:
dave
2026-04-29 15:17:47 +00:00
parent db65271587
commit 2655288412
11 changed files with 251 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/agents/lifecycle.rs
+3 -3
View File
@@ -166,7 +166,7 @@ pub fn move_story_to_done(story_id: &str) -> Result<(), String> {
"5_done",
&["6_archived"],
false,
&["merge_failure", "retry_count", "blocked"],
&["merge_failure", "blocked"],
)
.map(|_| ())
}
@@ -181,7 +181,7 @@ pub fn move_story_to_merge(story_id: &str) -> Result<(), String> {
"4_merge",
&["5_done", "6_archived"],
false,
&["retry_count", "blocked"],
&["blocked"],
)
.map(|_| ())
}
@@ -196,7 +196,7 @@ pub fn move_story_to_qa(story_id: &str) -> Result<(), String> {
"3_qa",
&["5_done", "6_archived"],
false,
&["retry_count", "blocked"],
&["blocked"],
)
.map(|_| ())
}
server/src/agents/pool/auto_assign/watchdog/tests/limits_tests.rs
+59 -10
View File
@@ -218,6 +218,7 @@ max_budget_usd = 5.00
#[test]
fn watchdog_marks_story_blocked_after_limit_termination() {
crate::db::ensure_content_store();
crate::crdt_state::init_for_test();
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -239,6 +240,18 @@ max_turns = 10
let story_id = "42_story_runaway";
let initial = "---\nname: Runaway Story\n---\n# Runaway Story\n";
crate::db::write_content(story_id, initial);
crate::crdt_state::write_item(
story_id,
"2_current",
Some("Runaway Story"),
None,
None,
None,
None,
None,
None,
None,
);
// 12 turns in a single session exceeds the configured max of 10.
write_fake_session_log(root, story_id, "coder-1", "sess-runaway", 12);
@@ -334,6 +347,7 @@ max_turns = 10
#[test]
fn per_session_counting_terminates_over_limit() {
crate::db::ensure_content_store();
crate::crdt_state::init_for_test();
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -352,6 +366,18 @@ max_turns = 10
let story_id = "story_e_per_session";
crate::db::write_content(story_id, "---\nname: Per-Session Test\n---\n");
crate::crdt_state::write_item(
story_id,
"2_current",
Some("Per-Session Test"),
None,
None,
None,
None,
None,
None,
None,
);
// Prior session with 5 turns (under limit alone).
write_fake_session_log(root, story_id, "coder-1", "old-sess", 5);
@@ -416,38 +442,55 @@ max_turns = 10
let story_id = "88_story_retry_watchdog";
let initial = "---\nname: Retry Test\n---\n";
crate::crdt_state::init_for_test();
crate::db::write_content(story_id, initial);
crate::crdt_state::write_item(
story_id,
"2_current",
Some("Retry Test"),
None,
None,
None,
None,
None,
None,
None,
);
// Session 1: exceeds limit → retry_count=1, NOT blocked.
// Session 1: exceeds limit → retry_count=1 in CRDT, NOT blocked.
{
write_fake_session_log(root, story_id, "coder-1", "session-1", 12);
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-1");
pool.run_watchdog_pass(Some(root));
let content = crate::db::read_content(story_id).unwrap();
assert!(
content.contains("retry_count: 1"),
"after session 1, retry_count should be 1 — got:\n{content}"
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
item.retry_count,
Some(1),
"after session 1, retry_count should be 1 in CRDT"
);
let content = crate::db::read_content(story_id).unwrap();
assert!(
!content.contains("blocked: true"),
"story should NOT be blocked after session 1"
);
}
// Session 2: exceeds limit → retry_count=2, NOT blocked.
// Session 2: exceeds limit → retry_count=2 in CRDT, NOT blocked.
{
write_fake_session_log(root, story_id, "coder-1", "session-2", 12);
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-2");
pool.run_watchdog_pass(Some(root));
let content = crate::db::read_content(story_id).unwrap();
assert!(
content.contains("retry_count: 2"),
"after session 2, retry_count should be 2 — got:\n{content}"
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
item.retry_count,
Some(2),
"after session 2, retry_count should be 2 in CRDT"
);
let content = crate::db::read_content(story_id).unwrap();
assert!(
!content.contains("blocked: true"),
"story should NOT be blocked after session 2"
@@ -466,5 +509,11 @@ max_turns = 10
content.contains("blocked: true"),
"story must be blocked after session 3 (retry_count=3 >= max_retries=3) — got:\n{content}"
);
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
item.retry_count,
Some(3),
"retry_count should be 3 in CRDT after session 3"
);
}
}
server/src/agents/pool/pipeline/advance/helpers.rs
+26 -25
View File
@@ -104,41 +104,42 @@ pub(crate) fn should_block_story(
max_retries: u32,
stage_label: &str,
) -> Option<String> {
use crate::io::story_metadata::{increment_retry_count_in_content, write_blocked_in_content};
use crate::io::story_metadata::write_blocked_in_content;
if max_retries == 0 {
return None;
}
if let Some(contents) = crate::db::read_content(story_id) {
let (updated, new_count) = increment_retry_count_in_content(&contents);
crate::db::write_content(story_id, &updated);
let stage = crate::pipeline_state::read_typed(story_id)
.ok()
.flatten()
.map(|i| i.stage.dir_name().to_string())
.unwrap_or_else(|| "2_current".to_string());
crate::db::write_item_with_content(story_id, &stage, &updated);
let new_count = crate::crdt_state::bump_retry_count(story_id) as u32;
if new_count == 0 {
slog_error!(
"[pipeline] Failed to bump retry_count for '{story_id}': item not found in CRDT"
);
return None;
}
if new_count >= max_retries {
slog_warn!(
"[pipeline] Story '{story_id}' reached retry limit ({new_count}/{max_retries}) \
at {stage_label} stage. Marking as blocked."
);
let blocked = write_blocked_in_content(&updated);
if new_count >= max_retries {
slog_warn!(
"[pipeline] Story '{story_id}' reached retry limit ({new_count}/{max_retries}) \
at {stage_label} stage. Marking as blocked."
);
if let Some(contents) = crate::db::read_content(story_id) {
let blocked = write_blocked_in_content(&contents);
crate::db::write_content(story_id, &blocked);
let stage = crate::pipeline_state::read_typed(story_id)
.ok()
.flatten()
.map(|i| i.stage.dir_name().to_string())
.unwrap_or_else(|| "2_current".to_string());
crate::db::write_item_with_content(story_id, &stage, &blocked);
Some(format!(
"Retry limit exceeded ({new_count}/{max_retries}) at {stage_label} stage"
))
} else {
slog!(
"[pipeline] Story '{story_id}' retry {new_count}/{max_retries} at {stage_label} stage."
);
None
}
Some(format!(
"Retry limit exceeded ({new_count}/{max_retries}) at {stage_label} stage"
))
} else {
slog_error!("[pipeline] Failed to read content for '{story_id}' to increment retry_count");
slog!(
"[pipeline] Story '{story_id}' retry {new_count}/{max_retries} at {stage_label} stage."
);
None
}
}
server/src/agents/pool/pipeline/advance/tests_regression.rs
+8 -5
View File
@@ -467,6 +467,7 @@ async fn no_committed_work_still_retries_and_blocks() {
.unwrap();
// Set up the story with max_retries=1 so it blocks immediately.
crate::crdt_state::init_for_test();
crate::db::ensure_content_store();
crate::db::write_content("9946_story_nowork", "---\nname: No Work Test\n---\n");
crate::db::write_item_with_content(
@@ -806,6 +807,7 @@ stage = "coder"
)
.unwrap();
crate::crdt_state::init_for_test();
crate::db::ensure_content_store();
crate::db::write_item_with_content(
"9950_story_warm_resume",
@@ -848,11 +850,12 @@ stage = "coder"
);
drop(agents);
// Retry counter must have been incremented (AC 3).
let content = crate::db::read_content("9950_story_warm_resume")
.expect("story must exist in content store");
// Retry counter must have been incremented (AC 3) — checked via CRDT.
let item =
crate::crdt_state::read_item("9950_story_warm_resume").expect("story must be in CRDT");
assert!(
content.contains("retry_count"),
"retry_count must be incremented after warm-resume: {content}"
item.retry_count.is_some_and(|rc| rc > 0),
"retry_count must be incremented after warm-resume: got {:?}",
item.retry_count
);
}