feat(423): auto-schedule timer on rate limit to resume after reset
- pty.rs: detect rate_limit_event hard blocks, parse reset_at, emit WatcherEvent::RateLimitHardBlock with story_id, agent_name, reset_at - watcher.rs: add RateLimitHardBlock variant to WatcherEvent enum - timer.rs: add TimerStore::upsert (add-or-update-to-later) and spawn_rate_limit_auto_scheduler (listens for RateLimitHardBlock, upserts timer for the blocked story) - notifications.rs: handle RateLimitHardBlock events with a debounced chat notification including the scheduled resume time; add format_rate_limit_hard_block_notification helper - matrix/mod.rs: subscribe second watcher_rx for auto-scheduler, pass it to run_bot - matrix/bot/run.rs: wire spawn_rate_limit_auto_scheduler into bot startup Tests cover: AC1 (hard block detection in pty), AC2 (auto-scheduler adds timer), AC3 (upsert deduplication), AC5 (chat notification sent), AC6 (worktree preserved — timer fires start_agent on existing worktree)
This commit is contained in:
@@ -22,6 +22,7 @@ pub async fn run_bot(
|
||||
config: super::super::config::BotConfig,
|
||||
project_root: PathBuf,
|
||||
watcher_rx: tokio::sync::broadcast::Receiver<crate::io::watcher::WatcherEvent>,
|
||||
watcher_rx_auto: tokio::sync::broadcast::Receiver<crate::io::watcher::WatcherEvent>,
|
||||
perm_rx: Arc<TokioMutex<mpsc::UnboundedReceiver<crate::http::context::PermissionForward>>>,
|
||||
agents: Arc<AgentPool>,
|
||||
shutdown_rx: watch::Receiver<Option<crate::rebuild::ShutdownReason>>,
|
||||
@@ -224,6 +225,11 @@ pub async fn run_bot(
|
||||
Arc::clone(&agents),
|
||||
project_root.clone(),
|
||||
);
|
||||
// Auto-schedule timers when an agent hits a hard rate limit.
|
||||
crate::chat::timer::spawn_rate_limit_auto_scheduler(
|
||||
Arc::clone(&timer_store),
|
||||
watcher_rx_auto,
|
||||
);
|
||||
|
||||
let ctx = BotContext {
|
||||
bot_user_id,
|
||||
|
||||
@@ -90,8 +90,11 @@ pub fn spawn_bot(
|
||||
|
||||
let root = project_root.to_path_buf();
|
||||
let watcher_rx = watcher_tx.subscribe();
|
||||
let watcher_rx_auto = watcher_tx.subscribe();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = bot::run_bot(config, root, watcher_rx, perm_rx, agents, shutdown_rx).await
|
||||
if let Err(e) =
|
||||
bot::run_bot(config, root, watcher_rx, watcher_rx_auto, perm_rx, agents, shutdown_rx)
|
||||
.await
|
||||
{
|
||||
crate::slog!("[matrix-bot] Fatal error: {e}");
|
||||
}
|
||||
|
||||
@@ -136,6 +136,31 @@ pub fn format_blocked_notification(
|
||||
/// Minimum time between rate-limit notifications for the same agent.
|
||||
const RATE_LIMIT_DEBOUNCE: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Format a rate limit hard block notification message with scheduled resume time.
|
||||
///
|
||||
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
|
||||
pub fn format_rate_limit_hard_block_notification(
|
||||
item_id: &str,
|
||||
story_name: Option<&str>,
|
||||
agent_name: &str,
|
||||
resume_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> (String, String) {
|
||||
let number = extract_story_number(item_id).unwrap_or(item_id);
|
||||
let name = story_name.unwrap_or(item_id);
|
||||
let local_time = resume_at.with_timezone(&chrono::Local);
|
||||
let resume_str = local_time.format("%Y-%m-%d %H:%M").to_string();
|
||||
|
||||
let plain = format!(
|
||||
"\u{1f6d1} #{number} {name} \u{2014} {agent_name} hit a hard rate limit; \
|
||||
will auto-resume at {resume_str}"
|
||||
);
|
||||
let html = format!(
|
||||
"\u{1f6d1} <strong>#{number}</strong> <em>{name}</em> \u{2014} \
|
||||
{agent_name} hit a hard rate limit; will auto-resume at {resume_str}"
|
||||
);
|
||||
(plain, html)
|
||||
}
|
||||
|
||||
/// Format a rate limit warning notification message.
|
||||
///
|
||||
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
|
||||
@@ -288,6 +313,45 @@ pub fn spawn_notification_listener(
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(WatcherEvent::RateLimitHardBlock {
|
||||
ref story_id,
|
||||
ref agent_name,
|
||||
reset_at,
|
||||
}) => {
|
||||
// Debounce: reuse the same key as RateLimitWarning so both
|
||||
// types are rate-limited together for the same agent.
|
||||
let debounce_key = format!("{story_id}:{agent_name}");
|
||||
let now = Instant::now();
|
||||
if let Some(&last) = rate_limit_last_notified.get(&debounce_key)
|
||||
&& now.duration_since(last) < RATE_LIMIT_DEBOUNCE
|
||||
{
|
||||
slog!(
|
||||
"[bot] Rate-limit hard-block notification debounced for \
|
||||
{story_id}:{agent_name}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
rate_limit_last_notified.insert(debounce_key, now);
|
||||
|
||||
let story_name = find_story_name_any_stage(&project_root, story_id);
|
||||
let (plain, html) = format_rate_limit_hard_block_notification(
|
||||
story_id,
|
||||
story_name.as_deref(),
|
||||
agent_name,
|
||||
reset_at,
|
||||
);
|
||||
|
||||
slog!("[bot] Sending rate-limit hard-block notification: {plain}");
|
||||
|
||||
for room_id in &get_room_ids() {
|
||||
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
|
||||
slog!(
|
||||
"[bot] Failed to send rate-limit hard-block notification \
|
||||
to {room_id}: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(_) => {} // Ignore non-work-item events
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
slog!(
|
||||
|
||||
Reference in New Issue
Block a user