huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down

server/src/service/disk_watch/io.rs

@@ -0,0 +1,144 @@
+//! Side effects for the disk-space watchdog: statvfs free-space reads,
+//! directory-size walks, and the periodic check-and-notify entry point.
+
+use std::ffi::CString;
+use std::os::unix::ffi::OsStrExt as _;
+use std::path::Path;
+use std::time::Instant;
+
+use tokio::sync::broadcast;
+
+use crate::config::DiskWatchConfig;
+use crate::io::watcher::WatcherEvent;
+use crate::service::status::{StatusBroadcaster, StatusEvent};
+
+use super::{DiskAction, DiskWatchState, decide_action};
+
+/// Read free bytes available on the filesystem containing `path`.
+pub fn free_space_bytes(path: &Path) -> std::io::Result<u64> {
+    let c_path = CString::new(path.as_os_str().as_bytes())
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
+    // SAFETY: `c_path` is a valid NUL-terminated string and `stat` is a
+    // zero-initialized `libc::statvfs` passed by mutable reference, matching
+    // the `statvfs(2)` contract.
+    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
+    let ret = unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) };
+    if ret != 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(stat.f_bavail as u64 * stat.f_frsize as u64)
+}
+
+/// Recursively sum file sizes under `path`. Returns 0 if `path` does not exist.
+pub fn dir_size_bytes(path: &Path) -> u64 {
+    walkdir::WalkDir::new(path)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| e.file_type().is_file())
+        .filter_map(|e| e.metadata().ok())
+        .map(|m| m.len())
+        .sum()
+}
+
+/// Run one disk-space check against `workspace_root`, updating `state` and
+/// broadcasting a [`WatcherEvent`] (local chat notifications) and a
+/// [`StatusEvent`] (gateway relay, for cross-sled dedupe — story 1200 AC4)
+/// when a warning or recovery notification is due.
+///
+/// `host_id` identifies the sled that observed the reading (surfaced in the
+/// gateway dedupe key and chat message).
+pub fn check_and_notify(
+    workspace_root: &Path,
+    config: &DiskWatchConfig,
+    state: &mut DiskWatchState,
+    watcher_tx: &broadcast::Sender<WatcherEvent>,
+    status: &StatusBroadcaster,
+    host_id: &str,
+) {
+    let Ok(free_bytes) = free_space_bytes(workspace_root) else {
+        return;
+    };
+    let now = Instant::now();
+    match decide_action(free_bytes, config, state, now) {
+        DiskAction::None => {}
+        DiskAction::Warning(level) => {
+            let target_bytes = dir_size_bytes(&workspace_root.join("target"));
+            let worktrees_bytes = dir_size_bytes(&workspace_root.join(".huskies/worktrees"));
+            let level_str = level.as_str().to_string();
+            let _ = watcher_tx.send(WatcherEvent::DiskSpaceWarning {
+                level: level_str.clone(),
+                free_bytes,
+                target_bytes,
+                worktrees_bytes,
+                host_id: host_id.to_string(),
+            });
+            status.publish(StatusEvent::DiskSpaceWarning {
+                level: level_str,
+                free_bytes,
+                target_bytes,
+                worktrees_bytes,
+                host_id: host_id.to_string(),
+            });
+        }
+        DiskAction::Recovery => {
+            let _ = watcher_tx.send(WatcherEvent::DiskSpaceRecovered {
+                free_bytes,
+                host_id: host_id.to_string(),
+            });
+            status.publish(StatusEvent::DiskSpaceRecovered {
+                free_bytes,
+                host_id: host_id.to_string(),
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn free_space_bytes_reads_a_real_path() {
+        // The current directory always exists in test runs; just assert the
+        // call succeeds and returns a plausible non-zero value.
+        let free = free_space_bytes(Path::new(".")).expect("statvfs should succeed on cwd");
+        assert!(free > 0);
+    }
+
+    #[test]
+    fn dir_size_bytes_sums_files_in_a_temp_dir() {
+        let tmp = tempfile::tempdir().unwrap();
+        std::fs::write(tmp.path().join("a.txt"), b"hello").unwrap();
+        std::fs::write(tmp.path().join("b.txt"), b"world!").unwrap();
+        assert_eq!(dir_size_bytes(tmp.path()), 11);
+    }
+
+    #[test]
+    fn dir_size_bytes_returns_zero_for_missing_path() {
+        assert_eq!(dir_size_bytes(Path::new("/no/such/path/1200")), 0);
+    }
+
+    #[tokio::test]
+    async fn check_and_notify_sends_warning_event_on_low_space() {
+        let tmp = tempfile::tempdir().unwrap();
+        let (tx, mut rx) = broadcast::channel::<WatcherEvent>(4);
+        let status = StatusBroadcaster::new();
+        let mut status_sub = status.subscribe();
+        let config = DiskWatchConfig {
+            warn_gb: u64::MAX / 1_000_000_000, // force Warn on any real filesystem
+            critical_gb: 0,
+            rate_limit_secs: 21_600,
+            recovery_margin_pct: 10,
+        };
+        let mut state = DiskWatchState::default();
+        check_and_notify(tmp.path(), &config, &mut state, &tx, &status, "sled-a");
+        let event = rx.try_recv().expect("should broadcast a WatcherEvent");
+        assert!(matches!(event, WatcherEvent::DiskSpaceWarning { .. }));
+        let status_event =
+            tokio::time::timeout(std::time::Duration::from_millis(100), status_sub.recv())
+                .await
+                .expect("should not time out")
+                .expect("should publish a StatusEvent");
+        assert!(matches!(status_event, StatusEvent::DiskSpaceWarning { .. }));
+    }
+}