huskies: merge 1163 story Replace perm_rx lock-as-presence-signal with a permission router

server/src/chat/transport/matrix/bot/context.rs

@@ -263,9 +263,9 @@ impl BotContext {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::service::permission_router::{PendingPermReplies, ResponderRegistry};
     use std::collections::HashMap;
     use std::path::PathBuf;
-    use tokio::sync::mpsc;
 
     fn make_user_id(s: &str) -> OwnedUserId {
         s.parse().unwrap()
@@ -273,15 +273,14 @@ mod tests {
 
     /// Build a test `Services` bundle with the given project root.
     fn test_services(project_root: PathBuf) -> Arc<Services> {
-        let (_perm_tx, perm_rx) = mpsc::unbounded_channel();
         Arc::new(Services {
             project_root,
             agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
             bot_name: "Assistant".to_string(),
             bot_user_id: "@bot:example.com".to_string(),
             ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
-            perm_rx: Arc::new(TokioMutex::new(perm_rx)),
-            pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
+            permission_registry: ResponderRegistry::new(),
+            pending_perm_replies: PendingPermReplies::new(),
             permission_timeout_secs: 120,
             status: Arc::new(crate::service::status::StatusBroadcaster::new()),
             chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),

server/src/chat/transport/matrix/bot/messages/on_room_message.rs

@@ -231,8 +231,12 @@ pub(in crate::chat::transport::matrix::bot) async fn on_room_message(
     // If there is a pending permission prompt for this room, interpret the
     // message as a yes/no response instead of starting a new chat.
     {
-        let mut pending = ctx.services.pending_perm_replies.lock().await;
-        if let Some(tx) = pending.remove(incoming_room_id.as_str()) {
+        if let Some(tx) = ctx
+            .services
+            .pending_perm_replies
+            .resolve_oldest(incoming_room_id.as_str())
+            .await
+        {
             let decision = if is_permission_approval(&body) {
                 PermissionDecision::Approve
             } else {

server/src/chat/transport/matrix/bot/mod.rs

@@ -9,8 +9,8 @@ pub mod history;
 pub mod mentions;
 /// Message handlers — processes incoming Matrix room messages.
 pub mod messages;
-/// Permission listener — holds perm_rx for the bot's lifetime and forwards
-/// permission requests to the configured Matrix room.
+/// Permission listener — registers as a permission responder for the bot's
+/// lifetime and forwards permission requests to the configured Matrix room.
 pub mod permission_listener;
 /// Bot run loop — the main async task that drives the Matrix sync loop.
 pub mod run;

server/src/chat/transport/matrix/bot/permission_listener.rs

@@ -1,11 +1,11 @@
-//! Background task that holds `perm_rx` for the bot's lifetime and forwards
-//! permission requests to the configured Matrix room.
+//! Background task that registers as a permission responder for the bot's
+//! lifetime and forwards permission requests to the configured Matrix room.
 //!
 //! Before story 884, each chat message handler acquired `perm_rx` for the
 //! duration of one chat_fut and dropped it afterwards. That meant whenever
 //! the bot wasn't actively responding, `prompt_permission` auto-denied any
 //! spawned coder bash call as "no interactive session" — making unattended
-//! coder work impossible. This task holds the lock continuously while the
+//! coder work impossible. This task stays registered continuously while the
 //! bot is connected, so requests can flow at any time.
 
 use crate::chat::ChatTransport;
@@ -20,13 +20,13 @@ use tokio::sync::Mutex as TokioMutex;
 
 use super::format::markdown_to_html;
 
-/// Spawn a background task that holds `services.perm_rx` for the bot's
-/// lifetime and forwards each incoming permission request to `target_room`
-/// as a chat message. Replies (yes/no) are resolved by the existing
-/// `on_room_message` handler via `pending_perm_replies`.
+/// Spawn a background task that registers with `services.permission_registry`
+/// for the bot's lifetime and forwards each incoming permission request to
+/// `target_room` as a chat message. Replies (yes/no) are resolved by the
+/// existing `on_room_message` handler via `pending_perm_replies`.
 ///
 /// Returns the JoinHandle so the caller can keep ownership; the task exits
-/// only when the `perm_rx` channel is closed (bot shutdown).
+/// only when its private channel closes (bot shutdown).
 pub fn spawn_permission_listener(
     services: Arc<Services>,
     transport: Arc<dyn ChatTransport>,
@@ -34,7 +34,7 @@ pub fn spawn_permission_listener(
     bot_sent_event_ids: Arc<TokioMutex<HashSet<OwnedEventId>>>,
 ) -> tokio::task::JoinHandle<()> {
     tokio::spawn(async move {
-        let mut perm_rx = services.perm_rx.lock().await;
+        let (_responder_guard, mut perm_rx) = services.permission_registry.register();
         let target_room_str = target_room.as_str().to_string();
         slog!("[matrix-bot] permission listener started; forwarding requests to {target_room_str}");
 
@@ -57,24 +57,33 @@ pub fn spawn_permission_listener(
             }
 
             // Store the MCP oneshot sender so on_room_message can resolve it
-            // when the user replies yes/no in the target room.
+            // when the user replies yes/no in the target room. Keyed by
+            // request_id (not just room) so a second concurrent request for
+            // the same room doesn't drop the first's sender.
             services
                 .pending_perm_replies
-                .lock()
-                .await
-                .insert(target_room.to_string(), perm_fwd.response_tx);
+                .insert(
+                    target_room.to_string(),
+                    perm_fwd.request_id.clone(),
+                    perm_fwd.response_tx,
+                )
+                .await;
 
             // Spawn a per-request timeout: auto-deny if the user does not
             // respond within `permission_timeout_secs`.
             let pending = Arc::clone(&services.pending_perm_replies);
             let timeout_room_key = target_room.to_string();
+            let timeout_request_id = perm_fwd.request_id.clone();
             let timeout_transport = Arc::clone(&transport);
             let timeout_room_str = target_room_str.clone();
             let timeout_sent_ids = Arc::clone(&bot_sent_event_ids);
             let timeout_secs = services.permission_timeout_secs;
             tokio::spawn(async move {
                 tokio::time::sleep(Duration::from_secs(timeout_secs)).await;
-                if let Some(tx) = pending.lock().await.remove(&timeout_room_key) {
+                if let Some(tx) = pending
+                    .remove_by_request_id(&timeout_room_key, &timeout_request_id)
+                    .await
+                {
                     let _ = tx.send(PermissionDecision::Deny);
                     let msg = "Permission request timed out — denied (fail-closed).";
                     let html = markdown_to_html(msg);
@@ -89,7 +98,7 @@ pub fn spawn_permission_listener(
             });
         }
 
-        slog!("[matrix-bot] permission listener exiting (perm_rx channel closed)");
+        slog!("[matrix-bot] permission listener exiting (channel closed)");
     })
 }
 
@@ -102,11 +111,11 @@ mod tests {
 
     use super::*;
     use crate::http::context::PermissionForward;
+    use crate::service::permission_router::PendingPermReplies;
     use crate::services::Services;
     use async_trait::async_trait;
     use serde_json::json;
-    use std::collections::HashMap;
-    use tokio::sync::{mpsc, oneshot};
+    use tokio::sync::oneshot;
 
     struct RecordingTransport {
         sent: Arc<std::sync::Mutex<Vec<(String, String)>>>,
@@ -138,26 +147,40 @@ mod tests {
         }
     }
 
-    fn test_services_with_tx() -> (Arc<Services>, mpsc::UnboundedSender<PermissionForward>) {
-        let (perm_tx, perm_rx) = mpsc::unbounded_channel();
-        let services = Arc::new(Services {
+    /// Poll `cond` until it returns `true` or `timeout` elapses, sleeping
+    /// briefly between checks. Used instead of a single fixed sleep so tests
+    /// don't flake under slow/loaded CI where a spawned task hasn't yet run.
+    async fn wait_until(mut cond: impl FnMut() -> bool, timeout: std::time::Duration) -> bool {
+        let start = tokio::time::Instant::now();
+        loop {
+            if cond() {
+                return true;
+            }
+            if start.elapsed() > timeout {
+                return false;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        }
+    }
+
+    fn test_services() -> Arc<Services> {
+        Arc::new(Services {
             project_root: std::path::PathBuf::from("/tmp/test"),
             agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
             bot_name: "Assistant".to_string(),
             bot_user_id: "@bot:example.com".to_string(),
             ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
-            perm_rx: Arc::new(TokioMutex::new(perm_rx)),
-            pending_perm_replies: Arc::new(TokioMutex::new(HashMap::new())),
+            permission_registry: crate::service::permission_router::ResponderRegistry::new(),
+            pending_perm_replies: PendingPermReplies::new(),
             permission_timeout_secs: 120,
             status: Arc::new(crate::service::status::StatusBroadcaster::new()),
             chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
-        });
-        (services, perm_tx)
+        })
     }
 
     #[tokio::test]
     async fn listener_forwards_request_to_target_room_when_no_chat_in_flight() {
-        let (services, perm_tx) = test_services_with_tx();
+        let services = test_services();
         let sent: Arc<std::sync::Mutex<Vec<(String, String)>>> =
             Arc::new(std::sync::Mutex::new(Vec::new()));
         let transport: Arc<dyn crate::chat::ChatTransport> = Arc::new(RecordingTransport {
@@ -173,21 +196,33 @@ mod tests {
             Arc::clone(&bot_sent_event_ids),
         );
 
-        // Yield so the listener task acquires perm_rx and starts recv'ing.
-        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        // Wait until the listener task registers and starts recv'ing.
+        assert!(
+            wait_until(
+                || !services.permission_registry.is_empty(),
+                std::time::Duration::from_secs(2)
+            )
+            .await,
+            "listener never registered as a responder"
+        );
 
         let (response_tx, _response_rx) = oneshot::channel();
-        perm_tx
-            .send(PermissionForward {
-                request_id: "req-1".to_string(),
-                tool_name: "Bash".to_string(),
-                tool_input: json!({"command": "cargo test"}),
-                response_tx,
-            })
-            .expect("send PermissionForward");
+        services.permission_registry.dispatch(PermissionForward {
+            request_id: "req-1".to_string(),
+            tool_name: "Bash".to_string(),
+            tool_input: json!({"command": "cargo test"}),
+            response_tx,
+        });
 
-        // Give the listener a moment to process.
-        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+        // Wait for the listener to process the forward.
+        assert!(
+            wait_until(
+                || !sent.lock().unwrap().is_empty(),
+                std::time::Duration::from_secs(2)
+            )
+            .await,
+            "listener never sent the permission prompt"
+        );
 
         // The transport must have received exactly one send_message to the
         // target room with the prompt content.
@@ -205,17 +240,126 @@ mod tests {
             recorded[0].1
         );
 
-        // pending_perm_replies must contain an entry keyed by the target room
-        // (so the user-reply handler can resolve the request when they reply).
-        let pending = services.pending_perm_replies.lock().await;
+        // pending_perm_replies must contain an entry resolvable for the target
+        // room (so the user-reply handler can resolve the request when they
+        // reply). The insert happens just after send_message returns, so poll
+        // briefly rather than assuming it's already visible.
+        let mut resolved = None;
+        for _ in 0..50 {
+            resolved = services
+                .pending_perm_replies
+                .resolve_oldest(target_room.as_str())
+                .await;
+            if resolved.is_some() {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        }
         assert!(
-            pending.contains_key(target_room.as_str()),
+            resolved.is_some(),
             "pending_perm_replies missing entry for target room"
         );
 
         // bot_sent_event_ids must have recorded the prompt's event ID so the
         // bot does not echo its own prompt back as user input.
-        let sent_ids = bot_sent_event_ids.lock().await;
-        assert_eq!(sent_ids.len(), 1, "expected one sent event ID recorded");
+        assert!(
+            wait_until(
+                || bot_sent_event_ids
+                    .try_lock()
+                    .map(|s| s.len() == 1)
+                    .unwrap_or(false),
+                std::time::Duration::from_secs(2)
+            )
+            .await,
+            "expected one sent event ID recorded"
+        );
+    }
+
+    /// Regression: two concurrent permission requests for the same target
+    /// room must both be resolvable — keying `pending_perm_replies` by
+    /// request_id (rather than overwriting a single room-keyed entry) means
+    /// the first request's oneshot sender is never dropped by the second.
+    #[tokio::test]
+    async fn two_concurrent_requests_for_same_room_both_resolve() {
+        let services = test_services();
+        let sent: Arc<std::sync::Mutex<Vec<(String, String)>>> =
+            Arc::new(std::sync::Mutex::new(Vec::new()));
+        let transport: Arc<dyn crate::chat::ChatTransport> = Arc::new(RecordingTransport {
+            sent: Arc::clone(&sent),
+        });
+        let target_room: OwnedRoomId = "!perm:example.com".parse().unwrap();
+        let bot_sent_event_ids = Arc::new(TokioMutex::new(HashSet::new()));
+
+        spawn_permission_listener(
+            Arc::clone(&services),
+            Arc::clone(&transport),
+            target_room.clone(),
+            Arc::clone(&bot_sent_event_ids),
+        );
+        assert!(
+            wait_until(
+                || !services.permission_registry.is_empty(),
+                std::time::Duration::from_secs(2)
+            )
+            .await,
+            "listener never registered as a responder"
+        );
+
+        let (tx1, rx1) = oneshot::channel();
+        services.permission_registry.dispatch(PermissionForward {
+            request_id: "req-a".to_string(),
+            tool_name: "Bash".to_string(),
+            tool_input: json!({}),
+            response_tx: tx1,
+        });
+        let (tx2, rx2) = oneshot::channel();
+        services.permission_registry.dispatch(PermissionForward {
+            request_id: "req-b".to_string(),
+            tool_name: "Write".to_string(),
+            tool_input: json!({}),
+            response_tx: tx2,
+        });
+
+        // Wait until the listener has processed both forwards.
+        assert!(
+            wait_until(
+                || sent.lock().unwrap().len() >= 2,
+                std::time::Duration::from_secs(2)
+            )
+            .await,
+            "listener never sent both permission prompts"
+        );
+
+        // Poll rather than assuming the pending_perm_replies insert (which
+        // happens just after send_message returns) is already visible.
+        let mut first = None;
+        for _ in 0..50 {
+            first = services
+                .pending_perm_replies
+                .resolve_oldest(target_room.as_str())
+                .await;
+            if first.is_some() {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        }
+        let first = first.expect("first request's sender must not have been dropped");
+        let _ = first.send(PermissionDecision::Approve);
+        assert_eq!(rx1.await.unwrap(), PermissionDecision::Approve);
+
+        let mut second = None;
+        for _ in 0..50 {
+            second = services
+                .pending_perm_replies
+                .resolve_oldest(target_room.as_str())
+                .await;
+            if second.is_some() {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        }
+        let second = second.expect("second request's sender must still be present");
+        let _ = second.send(PermissionDecision::Deny);
+        assert_eq!(rx2.await.unwrap(), PermissionDecision::Deny);
     }
 }

server/src/chat/transport/matrix/bot/run.rs

@@ -299,7 +299,7 @@ pub async fn run_bot(
     let bot_sent_event_ids: Arc<TokioMutex<HashSet<matrix_sdk::ruma::OwnedEventId>>> =
         Arc::new(TokioMutex::new(HashSet::new()));
 
-    // Spawn the permission listener: holds `perm_rx` for the bot's lifetime
+    // Spawn the permission listener: registers as a responder for the bot's lifetime
     // and forwards permission requests to the first configured room. Story
     // 884 — replaces the per-message lock acquire previously done in
     // handle_message.rs, so spawned coders' bash calls reach chat even when

server/src/chat/transport/matrix/health.rs

@@ -118,13 +118,14 @@ fn truncate_lines(mut lines: Vec<HealthLine>) -> Vec<HealthLine> {
 
 // ── Individual checks ────────────────────────────────────────────────────────
 
-/// Check the `perm_rx` receiver — PASS when the permission listener holds the lock,
-/// FAIL when no task is holding it (listener has died or was never started).
+/// Check the permission registry — PASS when at least one responder (e.g. the
+/// Matrix permission listener) is registered, FAIL when none is (listener has
+/// died or was never started).
 fn check_perm_rx(ctx: &BotContext) -> HealthLine {
-    if ctx.services.perm_rx.try_lock().is_err() {
-        HealthLine::pass("perm_rx")
+    if ctx.services.permission_registry.is_empty() {
+        HealthLine::fail("perm_rx", "no responder registered", "restart bot")
     } else {
-        HealthLine::fail("perm_rx", "listener not holding lock", "restart bot")
+        HealthLine::pass("perm_rx")
     }
 }
 
@@ -513,25 +514,22 @@ mod tests {
 
     #[tokio::test]
     async fn perm_rx_pass_when_locked() {
+        use crate::service::permission_router::{PendingPermReplies, ResponderRegistry};
         use crate::services::Services;
         use std::sync::Arc;
-        use tokio::sync::Mutex as TokioMutex;
 
-        let (perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
-        let perm_rx_arc = Arc::new(TokioMutex::new(perm_rx));
+        let registry = ResponderRegistry::new();
+        // Register a responder to simulate the permission listener being active.
+        let _guard_and_rx = registry.register();
 
-        // Acquire the lock to simulate the permission listener holding it.
-        let _guard = perm_rx_arc.try_lock().unwrap();
-
-        // Build a minimal services bundle referencing our locked perm_rx.
         let services = Arc::new(Services {
             project_root: std::path::PathBuf::from("/tmp"),
             agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
             bot_name: "test".to_string(),
             bot_user_id: "@bot:test".to_string(),
             ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
-            perm_rx: Arc::clone(&perm_rx_arc),
-            pending_perm_replies: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
+            permission_registry: registry,
+            pending_perm_replies: PendingPermReplies::new(),
             permission_timeout_secs: 120,
             status: Arc::new(crate::service::status::StatusBroadcaster::new()),
             chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
@@ -544,30 +542,25 @@ mod tests {
         assert_eq!(
             line.status,
             Status::Pass,
-            "perm_rx should PASS when a task holds the lock"
+            "perm_rx should PASS when a responder is registered"
         );
-
-        drop(perm_tx); // suppress unused warning
     }
 
     #[tokio::test]
     async fn perm_rx_fail_when_unlocked() {
+        use crate::service::permission_router::{PendingPermReplies, ResponderRegistry};
         use crate::services::Services;
         use std::sync::Arc;
-        use tokio::sync::Mutex as TokioMutex;
-
-        let (_perm_tx, perm_rx) = tokio::sync::mpsc::unbounded_channel();
-        let perm_rx_arc = Arc::new(TokioMutex::new(perm_rx));
-        // Lock is NOT held by anyone.
 
+        // No responder registered.
         let services = Arc::new(Services {
             project_root: std::path::PathBuf::from("/tmp"),
             agents: Arc::new(crate::agents::AgentPool::new_test(3000)),
             bot_name: "test".to_string(),
             bot_user_id: "@bot:test".to_string(),
             ambient_rooms: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
-            perm_rx: Arc::clone(&perm_rx_arc),
-            pending_perm_replies: Arc::new(TokioMutex::new(std::collections::HashMap::new())),
+            permission_registry: ResponderRegistry::new(),
+            pending_perm_replies: PendingPermReplies::new(),
             permission_timeout_secs: 120,
             status: Arc::new(crate::service::status::StatusBroadcaster::new()),
             chat_dispatcher: Arc::new(crate::chat::dispatcher::ChatDispatcher::new(1_500)),
@@ -579,7 +572,7 @@ mod tests {
         assert_eq!(
             line.status,
             Status::Fail,
-            "perm_rx should FAIL when no task holds the lock"
+            "perm_rx should FAIL when no responder is registered"
         );
     }