huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down

This commit is contained in:
Huskies Agent
2026-07-17 19:25:52 +00:00
parent 82865956d2
commit c1523e8acf
24 changed files with 1206 additions and 3 deletions
+25
View File
@@ -183,9 +183,15 @@ pub(crate) fn spawn_tick_loop(
})
.unwrap_or((30, std::time::Duration::from_secs(4 * 3600)));
let disk_watch_host_id =
std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string());
let disk_watch_watcher_tx = agents.watcher_tx();
let disk_watch_status = agents.status_broadcaster();
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
let mut tick_count: u64 = 0;
let mut disk_watch_state = service::disk_watch::DiskWatchState::default();
loop {
interval.tick().await;
tick_count = tick_count.wrapping_add(1);
@@ -218,6 +224,25 @@ pub(crate) fn spawn_tick_loop(
agents.reap_stale_merge_jobs();
}
// Low-disk-space watchdog (story 1200): check free space on the
// project workspace filesystem every 30 ticks (~30s) and emit a
// rate-limited warn/critical notification, or a recovery notice.
if tick_count.is_multiple_of(30)
&& let Some(ref r) = root
{
let disk_config = config::ProjectConfig::load(r)
.map(|c| c.disk_watch)
.unwrap_or_default();
service::disk_watch::io::check_and_notify(
r,
&disk_config,
&mut disk_watch_state,
&disk_watch_watcher_tx,
&disk_watch_status,
&disk_watch_host_id,
);
}
// Periodic reconciler: converge subscriber side effects so that
// Lagged broadcast events never leave state permanently diverged.
if tick_count.is_multiple_of(reconcile_interval)