huskies: merge 1185 story Bounded auto-retry for GatesFailed merges before requiring human intervention
This commit is contained in:
@@ -0,0 +1,570 @@
|
||||
//! TransitionFired subscriber that auto-retries `GatesFailed` merge failures after a delay.
|
||||
//!
|
||||
//! Story 1185: ConflictDetected already gets an automatic recovery path via
|
||||
//! [`super::merge_failure_subscriber`] (mergemaster auto-spawn). `GatesFailed`
|
||||
//! previously just sat in `Stage::MergeFailure` until a human intervened or the
|
||||
//! auto-block subscriber ([`super::merge_failure_block_subscriber`]) blocked the
|
||||
//! story after `merge_failure_block_threshold` consecutive failures.
|
||||
//!
|
||||
//! This subscriber schedules a delayed re-trigger of the deterministic
|
||||
//! server-side merge for `GatesFailed` failures, sharing the same
|
||||
//! `merge_failure_block_threshold` budget the auto-block subscriber uses as its
|
||||
//! retry ceiling. Once that many consecutive `GatesFailed` failures have
|
||||
//! occurred, no further retry is scheduled and the story is left exactly where
|
||||
//! it is — the untouched auto-block subscriber independently counts the same
|
||||
//! consecutive-failure stream and transitions the story to `Stage::Blocked` at
|
||||
//! the same threshold, so behaviour after budget exhaustion is unchanged from
|
||||
//! today.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::io::watcher::WatcherEvent;
|
||||
use crate::pipeline_state::{MergeFailureKind, Stage, Status, StoryId, TransitionFired};
|
||||
use crate::slog;
|
||||
use crate::slog_warn;
|
||||
|
||||
use super::super::AgentPool;
|
||||
|
||||
/// Delay before an auto-retry re-triggers the server-side merge for a
|
||||
/// `GatesFailed` failure. Gives a moment for transient conditions (e.g. a
|
||||
/// flaky test) to clear before retrying the exact same commit — retrying
|
||||
/// instantly would just hammer the same deterministic failure.
|
||||
const AUTO_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Reconcile: no-op, matching [`super::merge_failure_block_subscriber`].
|
||||
///
|
||||
/// The per-story attempt counter is in-memory bookkeeping that cannot be
|
||||
/// reconstructed from CRDT state alone (only the current stage is stored, not
|
||||
/// how many times a story has already been auto-retried). The live subscriber
|
||||
/// reacting to each new transition is sufficient for eventual consistency.
|
||||
pub(crate) fn reconcile_merge_failure_retry() {}
|
||||
|
||||
/// Spawn a background task that auto-retries `GatesFailed` merge failures.
|
||||
///
|
||||
/// Subscribes to the pipeline transition broadcast channel and tracks a
|
||||
/// per-story auto-retry attempt counter, bounded by the same
|
||||
/// `merge_failure_block_threshold` budget the auto-block subscriber uses.
|
||||
pub(crate) fn spawn_merge_failure_retry_subscriber(pool: Arc<AgentPool>, project_root: PathBuf) {
|
||||
let mut rx = crate::pipeline_state::subscribe_transitions();
|
||||
tokio::spawn(async move {
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(fired) => {
|
||||
if let Some((attempt, budget)) =
|
||||
decide_retry(&project_root, &fired, &mut attempts)
|
||||
{
|
||||
schedule_auto_retry(
|
||||
Arc::clone(&pool),
|
||||
project_root.clone(),
|
||||
fired.story_id.0.clone(),
|
||||
attempt,
|
||||
budget,
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
slog_warn!(
|
||||
"[merge-retry-sub] Subscriber lagged, skipped {n} event(s). \
|
||||
Some GatesFailed stories may need a manual retry."
|
||||
);
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Pure decision function: given a fired transition and the running per-story
|
||||
/// attempt counter, decide whether an auto-retry should be scheduled.
|
||||
///
|
||||
/// Returns `Some((attempt, budget))` when a retry should fire — `attempt` is
|
||||
/// this retry's 1-based sequence number and `budget` is the shared
|
||||
/// `merge_failure_block_threshold`. Returns `None` when the story left
|
||||
/// `MergeFailure`, the failure kind isn't `GatesFailed`, or the budget is
|
||||
/// exhausted (`attempt >= budget`).
|
||||
fn decide_retry(
|
||||
project_root: &Path,
|
||||
fired: &TransitionFired,
|
||||
attempts: &mut HashMap<StoryId, u32>,
|
||||
) -> Option<(u32, u32)> {
|
||||
if fired.after.status() != Status::MergeFailure {
|
||||
attempts.remove(&fired.story_id);
|
||||
return None;
|
||||
}
|
||||
let Stage::MergeFailure { kind, .. } = &fired.after else {
|
||||
return None;
|
||||
};
|
||||
if !matches!(kind, MergeFailureKind::GatesFailed(_)) {
|
||||
attempts.remove(&fired.story_id);
|
||||
return None;
|
||||
}
|
||||
|
||||
let entry = attempts.entry(fired.story_id.clone()).or_insert(0);
|
||||
*entry += 1;
|
||||
let attempt = *entry;
|
||||
|
||||
let budget = load_threshold(project_root);
|
||||
if budget == 0 || attempt >= budget {
|
||||
slog!(
|
||||
"[merge-retry-sub] Story '{}' exhausted GatesFailed auto-retry budget \
|
||||
({attempt}/{budget}); parking in MergeFailure for human intervention.",
|
||||
fired.story_id.0
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
Some((attempt, budget))
|
||||
}
|
||||
|
||||
/// Spawn the actual delayed retry as a background task.
|
||||
///
|
||||
/// Kept separate from [`decide_retry`] so the counter/budget decision stays
|
||||
/// synchronous and unit-testable without waiting on a real timer.
|
||||
fn schedule_auto_retry(
|
||||
pool: Arc<AgentPool>,
|
||||
project_root: PathBuf,
|
||||
story_id: String,
|
||||
attempt: u32,
|
||||
budget: u32,
|
||||
) {
|
||||
slog!(
|
||||
"[merge-retry-sub] Story '{story_id}' GatesFailed (attempt {attempt}/{budget}); \
|
||||
scheduling auto-retry in {AUTO_RETRY_DELAY:?}."
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(AUTO_RETRY_DELAY).await;
|
||||
perform_auto_retry(&pool, &project_root, &story_id, attempt, budget).await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Re-check the story is still parked in `GatesFailed`, notify chat, and
|
||||
/// re-trigger the deterministic server-side merge.
|
||||
///
|
||||
/// The guard protects against the story having been fixed manually, blocked,
|
||||
/// or otherwise moved on while the retry delay elapsed.
|
||||
async fn perform_auto_retry(
|
||||
pool: &Arc<AgentPool>,
|
||||
project_root: &Path,
|
||||
story_id: &str,
|
||||
attempt: u32,
|
||||
budget: u32,
|
||||
) {
|
||||
let still_gates_failed = matches!(
|
||||
crate::pipeline_state::read_typed(story_id),
|
||||
Ok(Some(item)) if matches!(
|
||||
item.stage,
|
||||
Stage::MergeFailure {
|
||||
kind: MergeFailureKind::GatesFailed(_),
|
||||
..
|
||||
}
|
||||
)
|
||||
);
|
||||
if !still_gates_failed {
|
||||
slog!(
|
||||
"[merge-retry-sub] Story '{story_id}' left GatesFailed before auto-retry \
|
||||
({attempt}/{budget}) fired; skipping."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
slog!("[merge-retry-sub] Auto-retrying merge for '{story_id}' (attempt {attempt}/{budget}).");
|
||||
let _ = pool.watcher_tx.send(WatcherEvent::MergeAutoRetry {
|
||||
story_id: story_id.to_string(),
|
||||
attempt,
|
||||
budget,
|
||||
});
|
||||
pool.trigger_server_side_merge(project_root, story_id);
|
||||
}
|
||||
|
||||
/// Load the auto-retry budget from project config, falling back to the
|
||||
/// compiled default. Shares `merge_failure_block_threshold` with the
|
||||
/// auto-block subscriber (story 1185, AC2).
|
||||
fn load_threshold(project_root: &Path) -> u32 {
|
||||
crate::config::ProjectConfig::load(project_root)
|
||||
.map(|c| c.merge_failure_block_threshold)
|
||||
.unwrap_or(3)
|
||||
}
|
||||
|
||||
// ── Tests ──────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::pipeline_state::{BranchName, PipelineEvent, Stage, StoryId, TransitionFired};
|
||||
use std::num::NonZeroU32;
|
||||
use std::process::Command;
|
||||
|
||||
fn setup_project(tmp: &tempfile::TempDir) {
|
||||
let sk = tmp.path().join(".huskies");
|
||||
std::fs::create_dir_all(&sk).unwrap();
|
||||
std::fs::write(sk.join("project.toml"), "[[agent]]\nname = \"coder\"\n").unwrap();
|
||||
}
|
||||
|
||||
fn seed_at_merge(story_id: &str) {
|
||||
crate::crdt_state::init_for_test();
|
||||
crate::db::ensure_content_store();
|
||||
crate::db::write_item_with_content(
|
||||
story_id,
|
||||
"4_merge",
|
||||
"---\nname: Test\n---\n",
|
||||
crate::db::ItemMeta::named("Test"),
|
||||
);
|
||||
}
|
||||
|
||||
fn make_merge_failure_fired(story_id: &str, kind: MergeFailureKind) -> TransitionFired {
|
||||
TransitionFired {
|
||||
story_id: StoryId(story_id.to_string()),
|
||||
before: Stage::Merge {
|
||||
feature_branch: BranchName("feature/test".to_string()),
|
||||
commits_ahead: NonZeroU32::new(1).unwrap(),
|
||||
claim: None,
|
||||
retries: 0,
|
||||
server_start_time: None,
|
||||
},
|
||||
after: Stage::MergeFailure {
|
||||
kind: kind.clone(),
|
||||
feature_branch: BranchName("feature/test".to_string()),
|
||||
commits_ahead: NonZeroU32::new(1).unwrap(),
|
||||
},
|
||||
event: PipelineEvent::MergeFailed { kind },
|
||||
at: chrono::Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn make_coding_fired(story_id: &str) -> TransitionFired {
|
||||
TransitionFired {
|
||||
story_id: StoryId(story_id.to_string()),
|
||||
before: Stage::MergeFailure {
|
||||
kind: MergeFailureKind::GatesFailed("error".to_string()),
|
||||
feature_branch: BranchName("feature/test".to_string()),
|
||||
commits_ahead: NonZeroU32::new(1).unwrap(),
|
||||
},
|
||||
after: Stage::Coding {
|
||||
claim: None,
|
||||
plan: Default::default(),
|
||||
retries: 0,
|
||||
},
|
||||
event: PipelineEvent::FixupRequested,
|
||||
at: chrono::Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
// ── decide_retry: pure decision logic ───────────────────────────────────
|
||||
|
||||
/// AC1/AC6: a GatesFailed failure below the budget must be scheduled for retry.
|
||||
#[test]
|
||||
fn gates_failed_below_budget_schedules_retry() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
setup_project(&tmp);
|
||||
let story_id = "1185_retry_basic";
|
||||
seed_at_merge(story_id);
|
||||
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
let fired = make_merge_failure_fired(
|
||||
story_id,
|
||||
MergeFailureKind::GatesFailed("cargo test failed".to_string()),
|
||||
);
|
||||
|
||||
let decision = decide_retry(tmp.path(), &fired, &mut attempts);
|
||||
assert_eq!(
|
||||
decision,
|
||||
Some((1, 3)),
|
||||
"first GatesFailed failure (default threshold 3) must schedule attempt 1/3"
|
||||
);
|
||||
}
|
||||
|
||||
/// AC2/AC6: once the attempt count reaches the shared threshold, no further
|
||||
/// retry is scheduled — the story parks in MergeFailure exactly as today
|
||||
/// (the untouched block subscriber handles the eventual Blocked transition).
|
||||
#[test]
|
||||
fn budget_exhaustion_stops_scheduling_retries() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
setup_project(&tmp);
|
||||
let story_id = "1185_retry_exhausted";
|
||||
seed_at_merge(story_id);
|
||||
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
let kind = MergeFailureKind::GatesFailed("cargo test failed".to_string());
|
||||
|
||||
// Default threshold is 3: attempts 1 and 2 schedule a retry, attempt 3
|
||||
// (== budget) does not.
|
||||
let fired1 = make_merge_failure_fired(story_id, kind.clone());
|
||||
assert_eq!(
|
||||
decide_retry(tmp.path(), &fired1, &mut attempts),
|
||||
Some((1, 3))
|
||||
);
|
||||
|
||||
let fired2 = make_merge_failure_fired(story_id, kind.clone());
|
||||
assert_eq!(
|
||||
decide_retry(tmp.path(), &fired2, &mut attempts),
|
||||
Some((2, 3))
|
||||
);
|
||||
|
||||
let fired3 = make_merge_failure_fired(story_id, kind);
|
||||
assert_eq!(
|
||||
decide_retry(tmp.path(), &fired3, &mut attempts),
|
||||
None,
|
||||
"budget exhausted at attempt == threshold; must not schedule another retry"
|
||||
);
|
||||
}
|
||||
|
||||
/// AC3/AC6: ConflictDetected, EmptyDiff, NoCommits, and Other must never be
|
||||
/// auto-retried by this subscriber (ConflictDetected has its own mergemaster
|
||||
/// auto-spawn path; the rest require human intervention).
|
||||
#[test]
|
||||
fn non_gates_failed_kinds_are_not_retried() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
setup_project(&tmp);
|
||||
|
||||
for (story_id, kind) in [
|
||||
(
|
||||
"1185_retry_conflict",
|
||||
MergeFailureKind::ConflictDetected(None),
|
||||
),
|
||||
("1185_retry_emptydiff", MergeFailureKind::EmptyDiff),
|
||||
("1185_retry_nocommits", MergeFailureKind::NoCommits),
|
||||
(
|
||||
"1185_retry_other",
|
||||
MergeFailureKind::Other("unknown".to_string()),
|
||||
),
|
||||
] {
|
||||
seed_at_merge(story_id);
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
let fired = make_merge_failure_fired(story_id, kind);
|
||||
assert_eq!(
|
||||
decide_retry(tmp.path(), &fired, &mut attempts),
|
||||
None,
|
||||
"non-GatesFailed kind must not be scheduled for auto-retry"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Counter resets when the story leaves MergeFailure (e.g. FixupRequested),
|
||||
/// mirroring the auto-block subscriber's reset behaviour.
|
||||
#[test]
|
||||
fn counter_resets_on_non_merge_failure_transition() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
setup_project(&tmp);
|
||||
let story_id = "1185_retry_reset";
|
||||
seed_at_merge(story_id);
|
||||
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
let kind = MergeFailureKind::GatesFailed("error".to_string());
|
||||
|
||||
let fired1 = make_merge_failure_fired(story_id, kind.clone());
|
||||
decide_retry(tmp.path(), &fired1, &mut attempts);
|
||||
assert_eq!(attempts.get(&StoryId(story_id.to_string())), Some(&1));
|
||||
|
||||
let reset_fired = make_coding_fired(story_id);
|
||||
decide_retry(tmp.path(), &reset_fired, &mut attempts);
|
||||
assert!(
|
||||
!attempts.contains_key(&StoryId(story_id.to_string())),
|
||||
"counter must be cleared after non-MergeFailure transition"
|
||||
);
|
||||
|
||||
// A fresh failure after reset must start back at attempt 1.
|
||||
let fired2 = make_merge_failure_fired(story_id, kind);
|
||||
assert_eq!(
|
||||
decide_retry(tmp.path(), &fired2, &mut attempts),
|
||||
Some((1, 3))
|
||||
);
|
||||
}
|
||||
|
||||
/// threshold == 0 disables auto-retry entirely, mirroring the auto-block
|
||||
/// subscriber's "0 disables" convention.
|
||||
#[test]
|
||||
fn zero_threshold_disables_auto_retry() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let sk = tmp.path().join(".huskies");
|
||||
std::fs::create_dir_all(&sk).unwrap();
|
||||
// Top-level keys must precede `[[agent]]` — TOML parses anything after
|
||||
// the array-of-tables header as belonging to that table.
|
||||
std::fs::write(
|
||||
sk.join("project.toml"),
|
||||
"merge_failure_block_threshold = 0\n[[agent]]\nname = \"coder\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
let story_id = "1185_retry_zero_threshold";
|
||||
seed_at_merge(story_id);
|
||||
|
||||
let mut attempts: HashMap<StoryId, u32> = HashMap::new();
|
||||
let fired =
|
||||
make_merge_failure_fired(story_id, MergeFailureKind::GatesFailed("error".to_string()));
|
||||
assert_eq!(decide_retry(tmp.path(), &fired, &mut attempts), None);
|
||||
}
|
||||
|
||||
// ── perform_auto_retry: end-to-end success path ─────────────────────────
|
||||
|
||||
fn init_git_repo(repo: &std::path::Path) {
|
||||
Command::new("git")
|
||||
.args(["init"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "user.email", "test@test.com"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "user.name", "Test"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["commit", "--allow-empty", "-m", "init"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// AC5/AC6: a mid-budget auto-retry that succeeds must drive the story to
|
||||
/// `Stage::Done` via the same path a manual re-run uses
|
||||
/// (`trigger_server_side_merge` → `start_merge_agent_work` →
|
||||
/// `move_story_to_done`) — no 1178-style trap state.
|
||||
#[tokio::test]
|
||||
async fn perform_auto_retry_success_completes_the_story() {
|
||||
use std::fs;
|
||||
use tempfile::tempdir;
|
||||
|
||||
crate::crdt_state::init_for_test();
|
||||
let tmp = tempdir().unwrap();
|
||||
let repo = tmp.path();
|
||||
init_git_repo(repo);
|
||||
|
||||
let story_id = "1185_retry_success";
|
||||
let branch = format!("feature/story-{story_id}");
|
||||
Command::new("git")
|
||||
.args(["checkout", "-b", &branch])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
fs::write(repo.join("feature.txt"), "feature content").unwrap();
|
||||
Command::new("git")
|
||||
.args(["add", "."])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["commit", "-m", "add feature"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["checkout", "master"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
crate::db::ensure_content_store();
|
||||
crate::db::write_item_with_content(
|
||||
story_id,
|
||||
"4_merge",
|
||||
"---\nname: Retry Success Test\n---\n",
|
||||
crate::db::ItemMeta::named("Retry Success Test"),
|
||||
);
|
||||
crate::agents::lifecycle::transition_to_merge_failure(
|
||||
story_id,
|
||||
MergeFailureKind::GatesFailed("cargo test failed".to_string()),
|
||||
)
|
||||
.expect("seed MergeFailure(GatesFailed)");
|
||||
|
||||
let pool = Arc::new(AgentPool::new_test(3199));
|
||||
perform_auto_retry(&pool, repo, story_id, 1, 3).await;
|
||||
|
||||
// start_merge_agent_work transitions MergeFailure -> Merge synchronously
|
||||
// (transition_merge_failure_to_retry) and then runs the actual squash
|
||||
// merge as a background task; poll until the story reaches Done (or
|
||||
// times out). Merge is an expected in-flight state, not a terminal one
|
||||
// — only stop polling on Done or a re-observed MergeFailure/Blocked.
|
||||
// Generous deadline: the real git subprocess pipeline (branch
|
||||
// checkout, worktree add/remove, squash, cherry-pick, source-map
|
||||
// regen) can be slow under a loaded, parallel `cargo test` run
|
||||
// (matches the 5s+ headroom used by similar tests in
|
||||
// `agents/pool/pipeline/merge/tests.rs` and `pipeline/advance/tests_regression.rs`).
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
|
||||
loop {
|
||||
let item = crate::pipeline_state::read_typed(story_id)
|
||||
.expect("read")
|
||||
.expect("item");
|
||||
if matches!(item.stage, Stage::Done { .. }) {
|
||||
break;
|
||||
}
|
||||
if !matches!(item.stage, Stage::Merge { .. }) {
|
||||
panic!(
|
||||
"successful mid-budget auto-retry must complete the story via \
|
||||
the manual-rerun path; got {:?}",
|
||||
item.stage
|
||||
);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
panic!(
|
||||
"auto-retry did not reach Done in time; still {:?}",
|
||||
item.stage
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// The guard in `perform_auto_retry` must skip re-triggering when the story
|
||||
/// left GatesFailed while the retry delay was elapsing (e.g. blocked or
|
||||
/// fixed manually in the meantime).
|
||||
#[tokio::test]
|
||||
async fn perform_auto_retry_skips_when_story_left_gates_failed() {
|
||||
crate::crdt_state::init_for_test();
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let repo = tmp.path();
|
||||
|
||||
let story_id = "1185_retry_stale_guard";
|
||||
crate::db::ensure_content_store();
|
||||
crate::db::write_item_with_content(
|
||||
story_id,
|
||||
"4_merge",
|
||||
"---\nname: Stale Guard Test\n---\n",
|
||||
crate::db::ItemMeta::named("Stale Guard Test"),
|
||||
);
|
||||
crate::agents::lifecycle::transition_to_merge_failure(
|
||||
story_id,
|
||||
MergeFailureKind::GatesFailed("error".to_string()),
|
||||
)
|
||||
.expect("seed MergeFailure(GatesFailed)");
|
||||
|
||||
// Story moved on (e.g. blocked) before the delayed retry fired.
|
||||
crate::pipeline_state::apply_transition(
|
||||
story_id,
|
||||
PipelineEvent::Block {
|
||||
reason: "manual block".to_string(),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("block story");
|
||||
|
||||
let pool = Arc::new(AgentPool::new_test(3198));
|
||||
perform_auto_retry(&pool, repo, story_id, 1, 3).await;
|
||||
|
||||
// No new merge job should have been started — the story is still
|
||||
// Blocked. `transition_to_merge_failure` above already wrote a
|
||||
// "failed" job entry as part of seeding; the guard skipping the
|
||||
// retry means that entry must be untouched (in particular, never
|
||||
// flipped to "running" by `trigger_server_side_merge`).
|
||||
let item = crate::pipeline_state::read_typed(story_id)
|
||||
.expect("read")
|
||||
.expect("item");
|
||||
assert!(
|
||||
matches!(item.stage, Stage::Blocked { .. }),
|
||||
"guard must skip re-triggering once the story left GatesFailed: {:?}",
|
||||
item.stage
|
||||
);
|
||||
let job_status = crate::crdt_state::read_merge_job(story_id).map(|j| j.status);
|
||||
assert_ne!(
|
||||
job_status.as_deref(),
|
||||
Some("running"),
|
||||
"no new merge job should be started when the guard skips the retry"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ mod backlog;
|
||||
mod merge;
|
||||
/// TransitionFired subscriber that auto-blocks stories after N consecutive MergeFailure transitions.
|
||||
pub(crate) mod merge_failure_block_subscriber;
|
||||
/// TransitionFired subscriber that auto-retries GatesFailed merge failures after a delay.
|
||||
pub(crate) mod merge_failure_retry_subscriber;
|
||||
/// TransitionFired subscriber that auto-spawns mergemaster on ConflictDetected merge failures.
|
||||
pub(crate) mod merge_failure_subscriber;
|
||||
mod pipeline;
|
||||
@@ -22,6 +24,10 @@ pub(crate) use merge_failure_block_subscriber::reconcile_merge_failure_block;
|
||||
/// Re-export for `startup::tick_loop`.
|
||||
pub(crate) use merge_failure_block_subscriber::spawn_merge_failure_block_subscriber;
|
||||
/// Re-export for `startup::tick_loop`.
|
||||
pub(crate) use merge_failure_retry_subscriber::reconcile_merge_failure_retry;
|
||||
/// Re-export for `startup::tick_loop`.
|
||||
pub(crate) use merge_failure_retry_subscriber::spawn_merge_failure_retry_subscriber;
|
||||
/// Re-export for `startup::tick_loop`.
|
||||
pub(crate) use merge_failure_subscriber::reconcile_merge_failure;
|
||||
/// Re-export for `startup::tick_loop`.
|
||||
pub(crate) use merge_failure_subscriber::spawn_merge_failure_subscriber;
|
||||
|
||||
@@ -100,4 +100,15 @@ pub enum WatcherEvent {
|
||||
/// Human-readable item name.
|
||||
name: String,
|
||||
},
|
||||
/// A `GatesFailed` merge failure is being automatically retried after a
|
||||
/// delay, without human intervention (story 1185).
|
||||
/// Triggers a status notification to configured chat rooms.
|
||||
MergeAutoRetry {
|
||||
/// Work item ID (e.g. `"42_story_my_feature"`).
|
||||
story_id: String,
|
||||
/// This retry's attempt number (1-based).
|
||||
attempt: u32,
|
||||
/// Total auto-retry budget shared with the auto-block threshold.
|
||||
budget: u32,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ pub enum EventAction {
|
||||
},
|
||||
/// Post a new-item-created notification.
|
||||
NewItemCreated,
|
||||
/// Post a merge-auto-retry notification naming the attempt and budget.
|
||||
MergeAutoRetry,
|
||||
/// Log server-side only; do not post to chat (e.g. hard rate-limit blocks).
|
||||
LogOnly,
|
||||
/// Reload the project configuration.
|
||||
@@ -54,6 +56,7 @@ pub fn classify(event: &WatcherEvent) -> EventAction {
|
||||
EventAction::AgentCompleted { success: *success }
|
||||
}
|
||||
WatcherEvent::NewItemCreated { .. } => EventAction::NewItemCreated,
|
||||
WatcherEvent::MergeAutoRetry { .. } => EventAction::MergeAutoRetry,
|
||||
_ => EventAction::Skip,
|
||||
}
|
||||
}
|
||||
@@ -191,4 +194,14 @@ mod tests {
|
||||
};
|
||||
assert_eq!(classify(&event), EventAction::NewItemCreated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_auto_retry_is_classified_correctly() {
|
||||
let event = WatcherEvent::MergeAutoRetry {
|
||||
story_id: "1_story_foo".to_string(),
|
||||
attempt: 1,
|
||||
budget: 3,
|
||||
};
|
||||
assert_eq!(classify(&event), EventAction::MergeAutoRetry);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,6 +240,38 @@ pub fn format_new_item_notification(
|
||||
(plain, html)
|
||||
}
|
||||
|
||||
/// Format a merge-auto-retry notification message.
|
||||
///
|
||||
/// Sent when a `GatesFailed` merge failure is automatically retried after a
|
||||
/// delay (story 1185). Returns `(plain_text, html)` suitable for
|
||||
/// `ChatTransport::send_message`.
|
||||
pub fn format_merge_auto_retry_notification(
|
||||
item_id: &str,
|
||||
story_name: &str,
|
||||
attempt: u32,
|
||||
budget: u32,
|
||||
) -> (String, String) {
|
||||
let number = extract_item_number(item_id).unwrap_or(item_id);
|
||||
let effective_name = if story_name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(story_name)
|
||||
};
|
||||
let name_plain = effective_name.map(|n| format!("{n} ")).unwrap_or_default();
|
||||
let name_html = effective_name
|
||||
.map(|n| format!("<em>{n}</em> "))
|
||||
.unwrap_or_default();
|
||||
|
||||
let plain = format!(
|
||||
"\u{1f504} #{number} {name_plain}\u{2014} auto-retrying merge (attempt {attempt}/{budget})"
|
||||
);
|
||||
let html = format!(
|
||||
"\u{1f504} <strong>#{number}</strong> {name_html}\u{2014} auto-retrying merge \
|
||||
(attempt {attempt}/{budget})"
|
||||
);
|
||||
(plain, html)
|
||||
}
|
||||
|
||||
/// Maximum number of trailing gate-output lines included in a merge-failure
|
||||
/// chat notification.
|
||||
///
|
||||
@@ -595,6 +627,36 @@ mod tests {
|
||||
assert_eq!(plain, "\u{1F916} #42 \u{2014} coder-1 started");
|
||||
}
|
||||
|
||||
// ── format_merge_auto_retry_notification ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn format_merge_auto_retry_notification_with_story_name() {
|
||||
let (plain, html) =
|
||||
format_merge_auto_retry_notification("42_story_foo", "My Feature", 1, 3);
|
||||
assert_eq!(
|
||||
plain,
|
||||
"\u{1f504} #42 My Feature \u{2014} auto-retrying merge (attempt 1/3)"
|
||||
);
|
||||
assert_eq!(
|
||||
html,
|
||||
"\u{1f504} <strong>#42</strong> <em>My Feature</em> \u{2014} auto-retrying merge \
|
||||
(attempt 1/3)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_merge_auto_retry_notification_falls_back_to_number() {
|
||||
let (plain, html) = format_merge_auto_retry_notification("42_story_foo", "", 2, 3);
|
||||
assert_eq!(
|
||||
plain,
|
||||
"\u{1f504} #42 \u{2014} auto-retrying merge (attempt 2/3)"
|
||||
);
|
||||
assert_eq!(
|
||||
html,
|
||||
"\u{1f504} <strong>#42</strong> \u{2014} auto-retrying merge (attempt 2/3)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── truncate_gate_output ──────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -16,8 +16,9 @@ use super::super::filter::{AGENT_EVENT_DEBOUNCE, should_send_rate_limit};
|
||||
use super::super::format::{
|
||||
MERGE_FAILURE_TAIL_LINES, format_agent_completed_notification,
|
||||
format_agent_started_notification, format_blocked_notification, format_error_notification,
|
||||
format_new_item_notification, format_oauth_account_swapped, format_oauth_accounts_exhausted,
|
||||
format_rate_limit_notification, truncate_gate_output,
|
||||
format_merge_auto_retry_notification, format_new_item_notification,
|
||||
format_oauth_account_swapped, format_oauth_accounts_exhausted, format_rate_limit_notification,
|
||||
truncate_gate_output,
|
||||
};
|
||||
use super::super::route::rooms_for_notification;
|
||||
use super::{find_story_name_any_stage, read_story_name};
|
||||
@@ -295,6 +296,35 @@ pub fn spawn_notification_listener(
|
||||
}
|
||||
}
|
||||
}
|
||||
EventAction::MergeAutoRetry => {
|
||||
if !config.status_push_enabled {
|
||||
continue;
|
||||
}
|
||||
let WatcherEvent::MergeAutoRetry {
|
||||
ref story_id,
|
||||
attempt,
|
||||
budget,
|
||||
} = event
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let story_name = find_story_name_any_stage(&project_root, story_id);
|
||||
let (plain, html) = format_merge_auto_retry_notification(
|
||||
story_id,
|
||||
&story_name,
|
||||
attempt,
|
||||
budget,
|
||||
);
|
||||
slog!("[bot] Sending merge-auto-retry notification: {plain}");
|
||||
for room_id in &rooms_for_notification(&get_room_ids) {
|
||||
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
|
||||
slog!(
|
||||
"[bot] Failed to send merge-auto-retry notification \
|
||||
to {room_id}: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
EventAction::LogOnly => {
|
||||
// Hard-block: log server-side for debugging; do NOT post to chat.
|
||||
// Hard-block auto-resume is normal operation — the status command
|
||||
|
||||
@@ -39,6 +39,8 @@ pub fn watcher_event_to_response(e: WatcherEvent) -> Option<WsResponse> {
|
||||
WatcherEvent::AgentCompleted { .. } => None,
|
||||
// Creation notifications are forwarded to chat transports only; no WebSocket message.
|
||||
WatcherEvent::NewItemCreated { .. } => None,
|
||||
// Merge-auto-retry notifications are forwarded to chat transports only; no WebSocket message.
|
||||
WatcherEvent::MergeAutoRetry { .. } => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,15 @@ pub(crate) fn spawn_event_bridges(
|
||||
root.clone(),
|
||||
);
|
||||
|
||||
// GatesFailed auto-retry subscriber: re-triggers the server-side merge
|
||||
// after a delay for GatesFailed failures, bounded by the same
|
||||
// merge_failure_block_threshold budget the auto-block subscriber above
|
||||
// uses (story 1185).
|
||||
crate::agents::pool::auto_assign::spawn_merge_failure_retry_subscriber(
|
||||
Arc::clone(&agents),
|
||||
root.clone(),
|
||||
);
|
||||
|
||||
// Content-store GC subscriber: purges all ContentKey::* entries for a
|
||||
// story when it reaches a terminal stage, preventing zombie entries from
|
||||
// accumulating in the process heap (story 996).
|
||||
@@ -542,6 +551,9 @@ pub(crate) async fn run_reconcile_pass(
|
||||
// Merge-block: no-op (in-memory counter cannot be reconstructed from CRDT).
|
||||
crate::agents::pool::auto_assign::reconcile_merge_failure_block();
|
||||
|
||||
// Merge-retry: no-op (in-memory attempt counter cannot be reconstructed from CRDT).
|
||||
crate::agents::pool::auto_assign::reconcile_merge_failure_retry();
|
||||
|
||||
// Audit-log: no-op (historical replay would produce misleading entries).
|
||||
crate::pipeline_state::reconcile_audit_log();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user