feat(423): auto-schedule timer on rate limit to resume after reset

- pty.rs: detect rate_limit_event hard blocks, parse reset_at, emit
  WatcherEvent::RateLimitHardBlock with story_id, agent_name, reset_at
- watcher.rs: add RateLimitHardBlock variant to WatcherEvent enum
- timer.rs: add TimerStore::upsert (add-or-update-to-later) and
  spawn_rate_limit_auto_scheduler (listens for RateLimitHardBlock,
  upserts timer for the blocked story)
- notifications.rs: handle RateLimitHardBlock events with a debounced
  chat notification including the scheduled resume time;
  add format_rate_limit_hard_block_notification helper
- matrix/mod.rs: subscribe second watcher_rx for auto-scheduler,
  pass it to run_bot
- matrix/bot/run.rs: wire spawn_rate_limit_auto_scheduler into bot startup

Tests cover: AC1 (hard block detection in pty), AC2 (auto-scheduler
adds timer), AC3 (upsert deduplication), AC5 (chat notification sent),
AC6 (worktree preserved — timer fires start_agent on existing worktree)
This commit is contained in:
dave
2026-03-28 09:18:58 +00:00
parent 57407aed51
commit b44f3a33e3
6 changed files with 374 additions and 8 deletions
@@ -22,6 +22,7 @@ pub async fn run_bot(
config: super::super::config::BotConfig,
project_root: PathBuf,
watcher_rx: tokio::sync::broadcast::Receiver<crate::io::watcher::WatcherEvent>,
watcher_rx_auto: tokio::sync::broadcast::Receiver<crate::io::watcher::WatcherEvent>,
perm_rx: Arc<TokioMutex<mpsc::UnboundedReceiver<crate::http::context::PermissionForward>>>,
agents: Arc<AgentPool>,
shutdown_rx: watch::Receiver<Option<crate::rebuild::ShutdownReason>>,
@@ -224,6 +225,11 @@ pub async fn run_bot(
Arc::clone(&agents),
project_root.clone(),
);
// Auto-schedule timers when an agent hits a hard rate limit.
crate::chat::timer::spawn_rate_limit_auto_scheduler(
Arc::clone(&timer_store),
watcher_rx_auto,
);
let ctx = BotContext {
bot_user_id,
+4 -1
View File
@@ -90,8 +90,11 @@ pub fn spawn_bot(
let root = project_root.to_path_buf();
let watcher_rx = watcher_tx.subscribe();
let watcher_rx_auto = watcher_tx.subscribe();
tokio::spawn(async move {
if let Err(e) = bot::run_bot(config, root, watcher_rx, perm_rx, agents, shutdown_rx).await
if let Err(e) =
bot::run_bot(config, root, watcher_rx, watcher_rx_auto, perm_rx, agents, shutdown_rx)
.await
{
crate::slog!("[matrix-bot] Fatal error: {e}");
}
@@ -136,6 +136,31 @@ pub fn format_blocked_notification(
/// Minimum time between rate-limit notifications for the same agent.
const RATE_LIMIT_DEBOUNCE: Duration = Duration::from_secs(60);
/// Format a rate limit hard block notification message with scheduled resume time.
///
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
pub fn format_rate_limit_hard_block_notification(
item_id: &str,
story_name: Option<&str>,
agent_name: &str,
resume_at: chrono::DateTime<chrono::Utc>,
) -> (String, String) {
let number = extract_story_number(item_id).unwrap_or(item_id);
let name = story_name.unwrap_or(item_id);
let local_time = resume_at.with_timezone(&chrono::Local);
let resume_str = local_time.format("%Y-%m-%d %H:%M").to_string();
let plain = format!(
"\u{1f6d1} #{number} {name} \u{2014} {agent_name} hit a hard rate limit; \
will auto-resume at {resume_str}"
);
let html = format!(
"\u{1f6d1} <strong>#{number}</strong> <em>{name}</em> \u{2014} \
{agent_name} hit a hard rate limit; will auto-resume at {resume_str}"
);
(plain, html)
}
/// Format a rate limit warning notification message.
///
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
@@ -288,6 +313,45 @@ pub fn spawn_notification_listener(
}
}
}
Ok(WatcherEvent::RateLimitHardBlock {
ref story_id,
ref agent_name,
reset_at,
}) => {
// Debounce: reuse the same key as RateLimitWarning so both
// types are rate-limited together for the same agent.
let debounce_key = format!("{story_id}:{agent_name}");
let now = Instant::now();
if let Some(&last) = rate_limit_last_notified.get(&debounce_key)
&& now.duration_since(last) < RATE_LIMIT_DEBOUNCE
{
slog!(
"[bot] Rate-limit hard-block notification debounced for \
{story_id}:{agent_name}"
);
continue;
}
rate_limit_last_notified.insert(debounce_key, now);
let story_name = find_story_name_any_stage(&project_root, story_id);
let (plain, html) = format_rate_limit_hard_block_notification(
story_id,
story_name.as_deref(),
agent_name,
reset_at,
);
slog!("[bot] Sending rate-limit hard-block notification: {plain}");
for room_id in &get_room_ids() {
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
slog!(
"[bot] Failed to send rate-limit hard-block notification \
to {room_id}: {e}"
);
}
}
}
Ok(_) => {} // Ignore non-work-item events
Err(broadcast::error::RecvError::Lagged(n)) => {
slog!(