crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

Converted all external tool calling to async

Browse Source
This commit is contained in:
Timmy
2026-06-29 16:59:54 +01:00
parent 75f41088b1
commit feb35ddd10
49 changed files with 482 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/agents/pool/auto_assign/watchdog/tests/limits_tests.rs
+41 -41
View File
@@ -10,8 +10,8 @@ use crate::agents::{AgentEvent, AgentStatus, TerminationReason};
// ── Limit enforcement integration tests (bug 624) ────────────────────────
#[test]
fn watchdog_terminates_agent_exceeding_turn_limit() {
#[tokio::test]
async fn watchdog_terminates_agent_exceeding_turn_limit() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -37,12 +37,12 @@ max_turns = 10
);
let mut rx = tx.subscribe();
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(found >= 1, "watchdog should detect the over-limit agent");
// Agent should now be Failed with TurnLimit reason.
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_a", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
@@ -60,8 +60,8 @@ max_turns = 10
);
}
#[test]
fn watchdog_terminates_agent_exceeding_budget_limit() {
#[tokio::test]
async fn watchdog_terminates_agent_exceeding_budget_limit() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -87,11 +87,11 @@ max_budget_usd = 5.00
);
let mut rx = tx.subscribe();
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(found >= 1, "watchdog should detect the over-budget agent");
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_b", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
@@ -106,8 +106,8 @@ max_budget_usd = 5.00
assert!(matches!(event, AgentEvent::Error { .. }));
}
#[test]
fn watchdog_does_not_terminate_agent_under_limits() {
#[tokio::test]
async fn watchdog_does_not_terminate_agent_under_limits() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -133,11 +133,11 @@ max_budget_usd = 10.00
// has 25 turns < 50 so no violation).
pool.inject_test_agent_with_session("story_c", "coder-1", AgentStatus::Running, "sess-ok");
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert_eq!(found, 0, "agent under limits should not be terminated");
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_c", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(
@@ -153,8 +153,8 @@ max_budget_usd = 10.00
/// coder-1 with max_turns=50, max_budget_usd=5.00 ran 5.6× over the turn
/// limit (280 turns). The watchdog must terminate at the turn limit (turns
/// hit first in the observed trace), with reason TurnLimit.
#[test]
fn regression_bug624_coder1_story623_trajectory() {
#[tokio::test]
async fn regression_bug624_coder1_story623_trajectory() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -183,11 +183,11 @@ max_budget_usd = 5.00
);
let mut rx = tx.subscribe();
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(found >= 1, "watchdog must catch the turn-limit violation");
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_623", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
@@ -218,8 +218,8 @@ max_budget_usd = 5.00
///
/// This test seeds a single session that legitimately exceeds the limit
/// and uses `max_retries = 1` so that the first violation blocks.
#[test]
fn watchdog_marks_story_blocked_after_limit_termination() {
#[tokio::test]
async fn watchdog_marks_story_blocked_after_limit_termination() {
crate::db::ensure_content_store();
crate::crdt_state::init_for_test();
@@ -263,7 +263,7 @@ max_turns = 10
"sess-runaway",
);
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(found >= 1, "watchdog should detect the over-limit agent");
// With max_retries=1, the first violation blocks immediately via the state machine.
@@ -278,7 +278,7 @@ max_turns = 10
// Sanity: the agent itself is also Failed with the right reason.
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key(story_id, "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
@@ -297,8 +297,8 @@ max_turns = 10
/// fresh session_id whose log has fewer events than `max_turns`.
/// Assert the agent is NOT terminated (per-session count is under the
/// limit) AND the story is NOT marked blocked.
#[test]
fn per_session_counting_does_not_terminate_under_limit() {
#[tokio::test]
async fn per_session_counting_does_not_terminate_under_limit() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -323,14 +323,14 @@ max_turns = 10
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session("story_d", "coder-1", AgentStatus::Running, "new-sess");
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert_eq!(
found, 0,
"agent under per-session limit should NOT be terminated"
);
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_d", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(
@@ -345,8 +345,8 @@ max_turns = 10
/// Same setup as per_session_counting_does_not_terminate_under_limit, but
/// the new agent's own session log exceeds `max_turns`. Assert the agent
/// IS terminated AND (with max_retries=1) the story IS marked blocked.
#[test]
fn per_session_counting_terminates_over_limit() {
#[tokio::test]
async fn per_session_counting_terminates_over_limit() {
crate::db::ensure_content_store();
crate::crdt_state::init_for_test();
@@ -390,14 +390,14 @@ max_turns = 10
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "new-sess");
let mut rx = tx.subscribe();
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(
found >= 1,
"agent over per-session limit must be terminated"
);
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key(story_id, "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
@@ -423,8 +423,8 @@ max_turns = 10
/// `max_turns`. After session 1: retry_count=1, NOT blocked. After
/// session 2: retry_count=2, NOT blocked. After session 3:
/// retry_count=3 >= max_retries, story IS blocked.
#[test]
fn watchdog_retry_semantic_blocks_after_max_retries() {
#[tokio::test]
async fn watchdog_retry_semantic_blocks_after_max_retries() {
crate::db::ensure_content_store();
let tmp = tempfile::tempdir().unwrap();
@@ -453,7 +453,7 @@ max_turns = 10
write_fake_session_log(root, story_id, "coder-1", "session-1", 12);
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-1");
pool.run_watchdog_pass(Some(root));
pool.run_watchdog_pass(Some(root)).await;
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
@@ -473,7 +473,7 @@ max_turns = 10
write_fake_session_log(root, story_id, "coder-1", "session-2", 12);
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-2");
pool.run_watchdog_pass(Some(root));
pool.run_watchdog_pass(Some(root)).await;
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
@@ -493,7 +493,7 @@ max_turns = 10
write_fake_session_log(root, story_id, "coder-1", "session-3", 12);
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session(story_id, "coder-1", AgentStatus::Running, "session-3");
pool.run_watchdog_pass(Some(root));
pool.run_watchdog_pass(Some(root)).await;
let item = crate::crdt_state::read_item(story_id).expect("story must be in CRDT");
assert_eq!(
@@ -518,8 +518,8 @@ max_turns = 10
/// must not count against the watchdog's turn budget. A session log with
/// 5 tool turns and 30 narration turns reports turns_used == 5, so an
/// agent with max_tool_turns = 10 stays Running.
#[test]
fn watchdog_does_not_count_narration_only_turns() {
#[tokio::test]
async fn watchdog_does_not_count_narration_only_turns() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -542,13 +542,13 @@ max_turns = 200
let pool = AgentPool::new_test(3001);
pool.inject_test_agent_with_session("story_923", "coder-1", AgentStatus::Running, "sess-narr");
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert_eq!(
found, 0,
"agent must not be terminated: only 5 tool turns of a 10-turn budget"
);
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_923", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Running);
@@ -558,8 +558,8 @@ max_turns = 200
/// Story 923: max_tool_turns takes precedence over max_turns when both are
/// set. With max_tool_turns = 3 and max_turns = 200, an agent that has 4
/// tool turns is killed even though total turns (4) is far below max_turns.
#[test]
fn watchdog_max_tool_turns_overrides_max_turns() {
#[tokio::test]
async fn watchdog_max_tool_turns_overrides_max_turns() {
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
@@ -585,13 +585,13 @@ max_turns = 200
);
let mut rx = tx.subscribe();
let found = pool.run_watchdog_pass(Some(root));
let found = pool.run_watchdog_pass(Some(root)).await;
assert!(
found >= 1,
"watchdog must terminate when tool turns exceed max_tool_turns"
);
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("story_923b", "coder-1");
let agent = agents.get(&key).unwrap();
assert_eq!(agent.status, AgentStatus::Failed);
server/src/agents/pool/auto_assign/watchdog/tests/orphan_tests.rs
+9 -9
View File
@@ -20,18 +20,18 @@ async fn check_orphaned_agents_returns_count_of_orphaned_agents() {
pool.inject_test_agent_with_handle("story_a", "coder", AgentStatus::Running, h1);
pool.inject_test_agent_with_handle("story_b", "coder", AgentStatus::Running, h2);
let found = check_orphaned_agents(&pool.agents);
let found = check_orphaned_agents(&pool.agents).await;
assert_eq!(found, 2, "should detect both orphaned agents");
}
#[test]
fn check_orphaned_agents_returns_zero_when_no_orphans() {
#[tokio::test]
async fn check_orphaned_agents_returns_zero_when_no_orphans() {
let pool = AgentPool::new_test(3001);
// Inject agents in terminal states — not orphaned.
pool.inject_test_agent("story_a", "coder", AgentStatus::Completed);
pool.inject_test_agent("story_b", "qa", AgentStatus::Failed);
let found = check_orphaned_agents(&pool.agents);
let found = check_orphaned_agents(&pool.agents).await;
assert_eq!(
found, 0,
"no orphans should be detected for terminal agents"
@@ -53,10 +53,10 @@ async fn watchdog_detects_orphaned_running_agent() {
pool.inject_test_agent_with_handle("orphan_story", "coder", AgentStatus::Running, handle);
let mut rx = tx.subscribe();
pool.run_watchdog_once();
pool.run_watchdog_once().await;
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("orphan_story", "coder");
let agent = agents.get(&key).unwrap();
assert_eq!(
@@ -87,13 +87,13 @@ async fn watchdog_orphan_detection_returns_nonzero_enabling_auto_assign() {
// Before watchdog: agent is Running.
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("orphan_story", "coder");
assert_eq!(agents.get(&key).unwrap().status, AgentStatus::Running);
}
// Run watchdog pass — should return 1 (orphan found).
let found = check_orphaned_agents(&pool.agents);
let found = check_orphaned_agents(&pool.agents).await;
assert_eq!(
found, 1,
"watchdog must return 1 for a single orphaned agent"
@@ -101,7 +101,7 @@ async fn watchdog_orphan_detection_returns_nonzero_enabling_auto_assign() {
// After watchdog: agent is Failed.
{
let agents = pool.agents.lock().unwrap();
let agents = pool.agents.try_lock().unwrap();
let key = composite_key("orphan_story", "coder");
assert_eq!(
agents.get(&key).unwrap().status,