63 lines
2.4 KiB
Rust
63 lines
2.4 KiB
Rust
//! Orphan detection: marks running agents whose backing task has exited.
|
|
|
|
use std::collections::HashMap;
|
|
use tokio::sync::Mutex;
|
|
use tokio::sync::broadcast;
|
|
|
|
use crate::agents::pool::StoryAgent;
|
|
use crate::agents::{AgentEvent, AgentStatus};
|
|
use crate::slog;
|
|
|
|
/// Scan the agent pool for Running entries whose backing tokio task has already
|
|
/// finished and mark them as Failed.
|
|
///
|
|
/// This handles the case where the PTY read loop or the spawned task exits
|
|
/// without updating the agent status — for example when the process is killed
|
|
/// externally and the PTY master fd returns EOF before our inactivity timeout
|
|
/// fires, but some other edge case prevents the normal cleanup path from running.
|
|
pub(super) async fn check_orphaned_agents(agents: &Mutex<HashMap<String, StoryAgent>>) -> usize {
|
|
let mut lock = agents.lock().await;
|
|
|
|
// Collect orphaned entries: Running or Pending agents whose task handle is finished.
|
|
// Pending agents can be orphaned if worktree creation panics before setting status.
|
|
let orphaned: Vec<(String, String, broadcast::Sender<AgentEvent>, AgentStatus)> = lock
|
|
.iter()
|
|
.filter_map(|(key, agent)| {
|
|
if matches!(agent.status, AgentStatus::Running | AgentStatus::Pending)
|
|
&& let Some(handle) = &agent.task_handle
|
|
&& handle.is_finished()
|
|
{
|
|
let story_id = key
|
|
.rsplit_once(':')
|
|
.map(|(s, _)| s.to_string())
|
|
.unwrap_or_else(|| key.clone());
|
|
return Some((
|
|
key.clone(),
|
|
story_id,
|
|
agent.tx.clone(),
|
|
agent.status.clone(),
|
|
));
|
|
}
|
|
None
|
|
})
|
|
.collect();
|
|
|
|
let count = orphaned.len();
|
|
for (key, story_id, tx, prev_status) in orphaned {
|
|
if let Some(agent) = lock.get_mut(&key) {
|
|
agent.status = AgentStatus::Failed;
|
|
slog!(
|
|
"[watchdog] Orphaned agent '{key}': task finished but status was {prev_status}. \
|
|
Marking Failed."
|
|
);
|
|
let _ = tx.send(AgentEvent::Error {
|
|
story_id,
|
|
agent_name: agent.agent_name.clone(),
|
|
message: "Agent process terminated unexpectedly (watchdog detected orphan)"
|
|
.to_string(),
|
|
});
|
|
}
|
|
}
|
|
count
|
|
}
|