huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down
This commit is contained in:
@@ -183,9 +183,15 @@ pub(crate) fn spawn_tick_loop(
|
||||
})
|
||||
.unwrap_or((30, std::time::Duration::from_secs(4 * 3600)));
|
||||
|
||||
let disk_watch_host_id =
|
||||
std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string());
|
||||
let disk_watch_watcher_tx = agents.watcher_tx();
|
||||
let disk_watch_status = agents.status_broadcaster();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
|
||||
let mut tick_count: u64 = 0;
|
||||
let mut disk_watch_state = service::disk_watch::DiskWatchState::default();
|
||||
loop {
|
||||
interval.tick().await;
|
||||
tick_count = tick_count.wrapping_add(1);
|
||||
@@ -218,6 +224,25 @@ pub(crate) fn spawn_tick_loop(
|
||||
agents.reap_stale_merge_jobs();
|
||||
}
|
||||
|
||||
// Low-disk-space watchdog (story 1200): check free space on the
|
||||
// project workspace filesystem every 30 ticks (~30s) and emit a
|
||||
// rate-limited warn/critical notification, or a recovery notice.
|
||||
if tick_count.is_multiple_of(30)
|
||||
&& let Some(ref r) = root
|
||||
{
|
||||
let disk_config = config::ProjectConfig::load(r)
|
||||
.map(|c| c.disk_watch)
|
||||
.unwrap_or_default();
|
||||
service::disk_watch::io::check_and_notify(
|
||||
r,
|
||||
&disk_config,
|
||||
&mut disk_watch_state,
|
||||
&disk_watch_watcher_tx,
|
||||
&disk_watch_status,
|
||||
&disk_watch_host_id,
|
||||
);
|
||||
}
|
||||
|
||||
// Periodic reconciler: converge subscriber side effects so that
|
||||
// Lagged broadcast events never leave state permanently diverged.
|
||||
if tick_count.is_multiple_of(reconcile_interval)
|
||||
|
||||
Reference in New Issue
Block a user