huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down

2026-07-17 19:25:52 +00:00
parent 82865956d2
commit c1523e8acf
24 changed files with 1206 additions and 3 deletions
@@ -0,0 +1,144 @@
+//! Side effects for the disk-space watchdog: statvfs free-space reads,
+//! directory-size walks, and the periodic check-and-notify entry point.
+
+use std::ffi::CString;
+use std::os::unix::ffi::OsStrExt as _;
+use std::path::Path;
+use std::time::Instant;
+
+use tokio::sync::broadcast;
+
+use crate::config::DiskWatchConfig;
+use crate::io::watcher::WatcherEvent;
+use crate::service::status::{StatusBroadcaster, StatusEvent};
+
+use super::{DiskAction, DiskWatchState, decide_action};
+
+/// Read free bytes available on the filesystem containing `path`.
+pub fn free_space_bytes(path: &Path) -> std::io::Result<u64> {
+    let c_path = CString::new(path.as_os_str().as_bytes())
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+    // SAFETY: `c_path` is a valid NUL-terminated string and `stat` is a
+    // zero-initialized `libc::statvfs` passed by mutable reference, matching
+    // the `statvfs(2)` contract.
+    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
+    let ret = unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) };
+    if ret != 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(stat.f_bavail as u64 * stat.f_frsize as u64)
+}
+
+/// Recursively sum file sizes under `path`. Returns 0 if `path` does not exist.
+pub fn dir_size_bytes(path: &Path) -> u64 {
+    walkdir::WalkDir::new(path)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| e.file_type().is_file())
+        .filter_map(|e| e.metadata().ok())
+        .map(|m| m.len())
+        .sum()
+}
+
+/// Run one disk-space check against `workspace_root`, updating `state` and
+/// broadcasting a [`WatcherEvent`] (local chat notifications) and a
+/// [`StatusEvent`] (gateway relay, for cross-sled dedupe — story 1200 AC4)
+/// when a warning or recovery notification is due.
+///
+/// `host_id` identifies the sled that observed the reading (surfaced in the
+/// gateway dedupe key and chat message).
+pub fn check_and_notify(
+    workspace_root: &Path,
+    config: &DiskWatchConfig,
+    state: &mut DiskWatchState,
+    watcher_tx: &broadcast::Sender<WatcherEvent>,
+    status: &StatusBroadcaster,
+    host_id: &str,
+) {
+    let Ok(free_bytes) = free_space_bytes(workspace_root) else {
+        return;
+    };
+    let now = Instant::now();
+    match decide_action(free_bytes, config, state, now) {
+        DiskAction::None => {}
+        DiskAction::Warning(level) => {
+            let target_bytes = dir_size_bytes(&workspace_root.join("target"));
+            let worktrees_bytes = dir_size_bytes(&workspace_root.join(".huskies/worktrees"));
+            let level_str = level.as_str().to_string();
+            let _ = watcher_tx.send(WatcherEvent::DiskSpaceWarning {
+                level: level_str.clone(),
+                free_bytes,
+                target_bytes,
+                worktrees_bytes,
+                host_id: host_id.to_string(),
+            });
+            status.publish(StatusEvent::DiskSpaceWarning {
+                level: level_str,
+                free_bytes,
+                target_bytes,
+                worktrees_bytes,
+                host_id: host_id.to_string(),
+            });
+        }
+        DiskAction::Recovery => {
+            let _ = watcher_tx.send(WatcherEvent::DiskSpaceRecovered {
+                free_bytes,
+                host_id: host_id.to_string(),
+            });
+            status.publish(StatusEvent::DiskSpaceRecovered {
+                free_bytes,
+                host_id: host_id.to_string(),
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn free_space_bytes_reads_a_real_path() {
+        // The current directory always exists in test runs; just assert the
+        // call succeeds and returns a plausible non-zero value.
+        let free = free_space_bytes(Path::new(".")).expect("statvfs should succeed on cwd");
+        assert!(free > 0);
+    }
+
+    #[test]
+    fn dir_size_bytes_sums_files_in_a_temp_dir() {
+        let tmp = tempfile::tempdir().unwrap();
+        std::fs::write(tmp.path().join("a.txt"), b"hello").unwrap();
+        std::fs::write(tmp.path().join("b.txt"), b"world!").unwrap();
+        assert_eq!(dir_size_bytes(tmp.path()), 11);
+    }
+
+    #[test]
+    fn dir_size_bytes_returns_zero_for_missing_path() {
+        assert_eq!(dir_size_bytes(Path::new("/no/such/path/1200")), 0);
+    }
+
+    #[tokio::test]
+    async fn check_and_notify_sends_warning_event_on_low_space() {
+        let tmp = tempfile::tempdir().unwrap();
+        let (tx, mut rx) = broadcast::channel::<WatcherEvent>(4);
+        let status = StatusBroadcaster::new();
+        let mut status_sub = status.subscribe();
+        let config = DiskWatchConfig {
+            warn_gb: u64::MAX / 1_000_000_000, // force Warn on any real filesystem
+            critical_gb: 0,
+            rate_limit_secs: 21_600,
+            recovery_margin_pct: 10,
+        };
+        let mut state = DiskWatchState::default();
+        check_and_notify(tmp.path(), &config, &mut state, &tx, &status, "sled-a");
+        let event = rx.try_recv().expect("should broadcast a WatcherEvent");
+        assert!(matches!(event, WatcherEvent::DiskSpaceWarning { .. }));
+        let status_event =
+            tokio::time::timeout(std::time::Duration::from_millis(100), status_sub.recv())
+                .await
+                .expect("should not time out")
+                .expect("should publish a StatusEvent");
+        assert!(matches!(status_event, StatusEvent::DiskSpaceWarning { .. }));
+    }
+}
@@ -0,0 +1,330 @@
+//! Low-disk-space watchdog (story 1200) — checks free space on the
+//! `/workspace` filesystem each tick and decides whether to emit a
+//! rate-limited warn/critical warning or a one-shot recovery notice.
+//!
+//! Follows service-module conventions: this file holds pure classification
+//! and rate-limiting logic (no I/O, no `Instant::now()` calls — callers
+//! supply `now`); `io.rs` is the only place performing side effects
+//! (statvfs reads, directory-size walks, broadcasting `WatcherEvent`s).
+
+/// Side effects: free-space reads, directory-size walks, and the periodic
+/// check-and-notify entry point.
+pub mod io;
+
+use std::time::{Duration, Instant};
+
+use crate::config::DiskWatchConfig;
+
+/// Bytes per gigabyte, using the same decimal (GB, not GiB) convention as the
+/// `warn_gb` / `critical_gb` config fields.
+const BYTES_PER_GB: u64 = 1_000_000_000;
+
+/// Disk-space severity level relative to configured thresholds.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DiskLevel {
+    /// Free space is at or above the warn threshold.
+    Ok,
+    /// Free space is below `warn_gb` but at/above `critical_gb`.
+    Warn,
+    /// Free space is below `critical_gb`.
+    Critical,
+}
+
+impl DiskLevel {
+    /// Wire/display name used in notifications and gateway dedupe keys.
+    pub fn as_str(self) -> &'static str {
+        match self {
+            DiskLevel::Ok => "ok",
+            DiskLevel::Warn => "warn",
+            DiskLevel::Critical => "critical",
+        }
+    }
+}
+
+/// Classify `free_bytes` against the configured warn/critical thresholds.
+pub fn classify_level(free_bytes: u64, config: &DiskWatchConfig) -> DiskLevel {
+    if free_bytes < config.critical_gb * BYTES_PER_GB {
+        DiskLevel::Critical
+    } else if free_bytes < config.warn_gb * BYTES_PER_GB {
+        DiskLevel::Warn
+    } else {
+        DiskLevel::Ok
+    }
+}
+
+/// Returns `true` if free space has recovered enough to send a recovery
+/// notice: at or above `warn_gb * (1 + recovery_margin_pct / 100)`.
+pub fn is_recovered(free_bytes: u64, config: &DiskWatchConfig) -> bool {
+    let recovery_threshold =
+        config.warn_gb * BYTES_PER_GB * (100 + config.recovery_margin_pct) / 100;
+    free_bytes >= recovery_threshold
+}
+
+/// Returns `true` if a repeat notification for the same level should be
+/// sent, given when it was last sent (`None` if never) and the configured
+/// rate limit.
+pub fn should_send(last_sent: Option<Instant>, now: Instant, rate_limit_secs: u64) -> bool {
+    match last_sent {
+        None => true,
+        Some(last) => now.duration_since(last) >= Duration::from_secs(rate_limit_secs),
+    }
+}
+
+/// Per-sled in-memory state tracked across tick iterations: the last level
+/// observed, and when a notification was last sent for each level.
+#[derive(Debug, Default)]
+pub struct DiskWatchState {
+    /// The disk level observed on the previous check, `None` before the first check.
+    pub last_level: Option<DiskLevel>,
+    /// When a "warn" notification was last sent.
+    pub warn_last_sent: Option<Instant>,
+    /// When a "critical" notification was last sent.
+    pub critical_last_sent: Option<Instant>,
+}
+
+/// The next action to take based on a fresh disk-level observation.
+#[derive(Debug, PartialEq)]
+pub enum DiskAction {
+    /// No notification needed.
+    None,
+    /// Send a warn/critical warning notification.
+    Warning(DiskLevel),
+    /// Send a single recovery notice.
+    Recovery,
+}
+
+/// Decide what action to take given a fresh free-space reading, updating
+/// `state` in place. Pure aside from taking `now` as an argument (no direct
+/// I/O or wall-clock reads).
+pub fn decide_action(
+    free_bytes: u64,
+    config: &DiskWatchConfig,
+    state: &mut DiskWatchState,
+    now: Instant,
+) -> DiskAction {
+    let level = classify_level(free_bytes, config);
+    let was_persisting = matches!(
+        state.last_level,
+        Some(DiskLevel::Warn) | Some(DiskLevel::Critical)
+    );
+    state.last_level = Some(level);
+
+    match level {
+        DiskLevel::Ok => {
+            if was_persisting && is_recovered(free_bytes, config) {
+                state.warn_last_sent = None;
+                state.critical_last_sent = None;
+                DiskAction::Recovery
+            } else {
+                DiskAction::None
+            }
+        }
+        DiskLevel::Warn => {
+            if should_send(state.warn_last_sent, now, config.rate_limit_secs) {
+                state.warn_last_sent = Some(now);
+                DiskAction::Warning(DiskLevel::Warn)
+            } else {
+                DiskAction::None
+            }
+        }
+        DiskLevel::Critical => {
+            if should_send(state.critical_last_sent, now, config.rate_limit_secs) {
+                state.critical_last_sent = Some(now);
+                DiskAction::Warning(DiskLevel::Critical)
+            } else {
+                DiskAction::None
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn config() -> DiskWatchConfig {
+        DiskWatchConfig {
+            warn_gb: 50,
+            critical_gb: 20,
+            rate_limit_secs: 6 * 60 * 60,
+            recovery_margin_pct: 10,
+        }
+    }
+
+    const GB: u64 = 1_000_000_000;
+
+    // ── classify_level (threshold crossing) ───────────────────────────────
+
+    #[test]
+    fn classify_above_warn_is_ok() {
+        assert_eq!(classify_level(60 * GB, &config()), DiskLevel::Ok);
+    }
+
+    #[test]
+    fn classify_at_warn_boundary_is_warn() {
+        // Below warn_gb (50) counts as Warn; exactly at warn_gb is still Ok
+        // (the threshold is "below", not "at or below").
+        assert_eq!(classify_level(50 * GB, &config()), DiskLevel::Ok);
+        assert_eq!(classify_level(49 * GB, &config()), DiskLevel::Warn);
+    }
+
+    #[test]
+    fn classify_between_warn_and_critical_is_warn() {
+        assert_eq!(classify_level(30 * GB, &config()), DiskLevel::Warn);
+    }
+
+    #[test]
+    fn classify_at_critical_boundary() {
+        assert_eq!(classify_level(20 * GB, &config()), DiskLevel::Warn);
+        assert_eq!(classify_level(19 * GB, &config()), DiskLevel::Critical);
+    }
+
+    #[test]
+    fn classify_below_critical_is_critical() {
+        assert_eq!(classify_level(5 * GB, &config()), DiskLevel::Critical);
+    }
+
+    // ── is_recovered ───────────────────────────────────────────────────────
+
+    #[test]
+    fn is_recovered_requires_margin_above_warn() {
+        let cfg = config();
+        // warn_gb=50, margin=10% => recovery threshold = 55 GB.
+        assert!(!is_recovered(54 * GB, &cfg));
+        assert!(is_recovered(55 * GB, &cfg));
+        assert!(is_recovered(60 * GB, &cfg));
+    }
+
+    // ── should_send (rate limiting) ────────────────────────────────────────
+
+    #[test]
+    fn should_send_when_never_sent() {
+        assert!(should_send(None, Instant::now(), 21_600));
+    }
+
+    #[test]
+    fn should_not_send_within_rate_limit_window() {
+        let now = Instant::now();
+        let last = now - Duration::from_secs(100);
+        assert!(!should_send(Some(last), now, 21_600));
+    }
+
+    #[test]
+    fn should_send_after_rate_limit_window_expires() {
+        let now = Instant::now();
+        let last = now - Duration::from_secs(21_601);
+        assert!(should_send(Some(last), now, 21_600));
+    }
+
+    // ── decide_action (threshold crossing + rate limiting + recovery) ──────
+
+    #[test]
+    fn decide_action_first_warn_reading_sends_warning() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let action = decide_action(30 * GB, &cfg, &mut state, Instant::now());
+        assert_eq!(action, DiskAction::Warning(DiskLevel::Warn));
+    }
+
+    #[test]
+    fn decide_action_repeat_warn_within_window_is_suppressed() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        assert_eq!(
+            decide_action(30 * GB, &cfg, &mut state, now),
+            DiskAction::Warning(DiskLevel::Warn)
+        );
+        // Second check 1 minute later — same level, still within 6h window.
+        let later = now + Duration::from_secs(60);
+        assert_eq!(
+            decide_action(30 * GB, &cfg, &mut state, later),
+            DiskAction::None
+        );
+    }
+
+    #[test]
+    fn decide_action_repeat_warn_after_window_sends_again() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        decide_action(30 * GB, &cfg, &mut state, now);
+        let later = now + Duration::from_secs(6 * 60 * 60 + 1);
+        assert_eq!(
+            decide_action(30 * GB, &cfg, &mut state, later),
+            DiskAction::Warning(DiskLevel::Warn)
+        );
+    }
+
+    #[test]
+    fn decide_action_warn_and_critical_have_independent_rate_limits() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        assert_eq!(
+            decide_action(30 * GB, &cfg, &mut state, now),
+            DiskAction::Warning(DiskLevel::Warn)
+        );
+        // Dropping straight to critical moments later must still notify —
+        // it has its own rate-limit bucket, separate from warn's.
+        let later = now + Duration::from_secs(5);
+        assert_eq!(
+            decide_action(10 * GB, &cfg, &mut state, later),
+            DiskAction::Warning(DiskLevel::Critical)
+        );
+    }
+
+    #[test]
+    fn decide_action_recovery_after_persisting_warn() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        decide_action(30 * GB, &cfg, &mut state, now); // Warn
+        // Free space climbs back above the 55 GB recovery threshold.
+        assert_eq!(
+            decide_action(60 * GB, &cfg, &mut state, now),
+            DiskAction::Recovery
+        );
+    }
+
+    #[test]
+    fn decide_action_no_recovery_notice_without_prior_warning() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        // First-ever reading is healthy — must not fire a spurious recovery notice.
+        assert_eq!(
+            decide_action(60 * GB, &cfg, &mut state, Instant::now()),
+            DiskAction::None
+        );
+    }
+
+    #[test]
+    fn decide_action_recovery_is_sent_only_once() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        decide_action(30 * GB, &cfg, &mut state, now); // Warn
+        assert_eq!(
+            decide_action(60 * GB, &cfg, &mut state, now),
+            DiskAction::Recovery
+        );
+        // Still healthy on the next tick — no repeat recovery notice.
+        assert_eq!(
+            decide_action(60 * GB, &cfg, &mut state, now),
+            DiskAction::None
+        );
+    }
+
+    #[test]
+    fn decide_action_back_above_warn_but_below_margin_is_not_recovery() {
+        let cfg = config();
+        let mut state = DiskWatchState::default();
+        let now = Instant::now();
+        decide_action(30 * GB, &cfg, &mut state, now); // Warn
+        // 52 GB is above warn_gb (50) but below the 55 GB recovery margin.
+        assert_eq!(
+            decide_action(52 * GB, &cfg, &mut state, now),
+            DiskAction::None
+        );
+    }
+}