huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down

server/src/service/gateway/io.rs

@@ -507,15 +507,51 @@ pub fn init_wizard_state(path: &Path) {
 ///
 /// The task exits cleanly when the broadcast channel is closed (i.e. when
 /// `GatewayState` is dropped).
+/// Window within which identical disk-space warnings from different sleds are
+/// deduped into a single forwarded chat message (story 1200 AC4). Matches the
+/// sled-side default `rate_limit_secs`.
+const DISK_DEDUPE_WINDOW: std::time::Duration = std::time::Duration::from_secs(6 * 60 * 60);
+
+/// Returns the dedupe key for a [`crate::service::events::StoredEvent`], or
+/// `None` if the event type is never deduped (only disk-space events dedupe
+/// across sleds — story 1200 AC4).
+fn dedupe_key(event: &crate::service::events::StoredEvent) -> Option<String> {
+    match event {
+        crate::service::events::StoredEvent::DiskSpaceWarning { level, .. } => {
+            Some(format!("disk_warning:{level}"))
+        }
+        crate::service::events::StoredEvent::DiskSpaceRecovered { .. } => {
+            Some("disk_recovered".to_string())
+        }
+        _ => None,
+    }
+}
+
 pub fn spawn_gateway_broadcaster_forwarder(
     transport: std::sync::Arc<dyn crate::chat::ChatTransport>,
     room_ids: Vec<String>,
     mut rx: tokio::sync::broadcast::Receiver<super::GatewayStatusEvent>,
 ) {
     tokio::spawn(async move {
+        let mut last_forwarded: std::collections::HashMap<String, std::time::Instant> =
+            std::collections::HashMap::new();
         loop {
             match rx.recv().await {
                 Ok(event) => {
+                    if let Some(key) = dedupe_key(&event.event) {
+                        let now = std::time::Instant::now();
+                        let should_forward = match last_forwarded.get(&key) {
+                            None => true,
+                            Some(last) => now.duration_since(*last) >= DISK_DEDUPE_WINDOW,
+                        };
+                        if !should_forward {
+                            crate::slog!(
+                                "[gateway-forwarder] Deduping repeated {key} within window"
+                            );
+                            continue;
+                        }
+                        last_forwarded.insert(key, now);
+                    }
                     let (plain, html) =
                         super::polling::format_gateway_event(&event.project, &event.event);
                     for room_id in &room_ids {
@@ -839,4 +875,57 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         assert!(read_installed_manifest(dir.path(), "huskies-linux-arm64").is_none());
     }
+
+    // ── dedupe_key (story 1200 AC4) ──────────────────────────────────────────
+
+    #[test]
+    fn dedupe_key_disk_warning_keys_by_level() {
+        let warn = crate::service::events::StoredEvent::DiskSpaceWarning {
+            level: "warn".to_string(),
+            free_bytes: 1,
+            target_bytes: 0,
+            worktrees_bytes: 0,
+            host_id: "sled-a".to_string(),
+            timestamp_ms: 0,
+        };
+        let critical = crate::service::events::StoredEvent::DiskSpaceWarning {
+            level: "critical".to_string(),
+            free_bytes: 1,
+            target_bytes: 0,
+            worktrees_bytes: 0,
+            host_id: "sled-b".to_string(),
+            timestamp_ms: 0,
+        };
+        // Different sleds at the same level share a key (so they dedupe)...
+        assert_eq!(dedupe_key(&warn), dedupe_key(&warn));
+        // ...but different levels do not.
+        assert_ne!(dedupe_key(&warn), dedupe_key(&critical));
+    }
+
+    #[test]
+    fn dedupe_key_disk_recovered_has_a_single_shared_key() {
+        let a = crate::service::events::StoredEvent::DiskSpaceRecovered {
+            free_bytes: 1,
+            host_id: "sled-a".to_string(),
+            timestamp_ms: 0,
+        };
+        let b = crate::service::events::StoredEvent::DiskSpaceRecovered {
+            free_bytes: 2,
+            host_id: "sled-b".to_string(),
+            timestamp_ms: 1,
+        };
+        assert_eq!(dedupe_key(&a), dedupe_key(&b));
+    }
+
+    #[test]
+    fn dedupe_key_non_disk_events_are_not_deduped() {
+        let transition = crate::service::events::StoredEvent::StageTransition {
+            story_id: "1_story".to_string(),
+            story_name: String::new(),
+            from_stage: "2_current".to_string(),
+            to_stage: "3_qa".to_string(),
+            timestamp_ms: 0,
+        };
+        assert_eq!(dedupe_key(&transition), None);
+    }
 }

server/src/service/gateway/polling.rs

@@ -7,7 +7,8 @@
 use crate::pipeline_state::Stage;
 use crate::service::events::StoredEvent;
 use crate::service::notifications::{
-    format_blocked_notification, format_error_notification, format_stage_notification,
+    format_blocked_notification, format_disk_recovery_notification,
+    format_disk_warning_notification, format_error_notification, format_stage_notification,
 };
 
 /// Format a [`StoredEvent`] from a project into a gateway notification.
@@ -49,6 +50,31 @@ pub fn format_gateway_event(project_name: &str, event: &StoredEvent) -> (String,
             let (plain, html) = format_blocked_notification(story_id, story_name, reason);
             (format!("{prefix}{plain}"), format!("{prefix}{html}"))
         }
+        StoredEvent::DiskSpaceWarning {
+            level,
+            free_bytes,
+            target_bytes,
+            worktrees_bytes,
+            host_id,
+            ..
+        } => {
+            let (plain, html) = format_disk_warning_notification(
+                level,
+                host_id,
+                *free_bytes,
+                *target_bytes,
+                *worktrees_bytes,
+            );
+            (format!("{prefix}{plain}"), format!("{prefix}{html}"))
+        }
+        StoredEvent::DiskSpaceRecovered {
+            free_bytes,
+            host_id,
+            ..
+        } => {
+            let (plain, html) = format_disk_recovery_notification(host_id, *free_bytes);
+            (format!("{prefix}{plain}"), format!("{prefix}{html}"))
+        }
     }
 }
 
@@ -91,6 +117,14 @@ pub fn format_gateway_audit_line(project: &str, event: &StoredEvent) -> String {
                 "audit ts={ts} project={project} id={story_id} event=story_blocked reason={reason}"
             )
         }
+        StoredEvent::DiskSpaceWarning { level, host_id, .. } => {
+            format!(
+                "audit ts={ts} project={project} event=disk_space_warning level={level} host={host_id}"
+            )
+        }
+        StoredEvent::DiskSpaceRecovered { host_id, .. } => {
+            format!("audit ts={ts} project={project} event=disk_space_recovered host={host_id}")
+        }
     }
 }