crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down

Browse Source
This commit is contained in:
Huskies Agent
2026-07-17 19:25:52 +00:00
parent 82865956d2
commit c1523e8acf
24 changed files with 1206 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/config/mod.rs
+61
View File
@@ -19,6 +19,9 @@ pub struct ProjectConfig {
pub agent: Vec<AgentConfig>,
#[serde(default)]
pub watcher: WatcherConfig,
/// Configuration for the low-disk-space watchdog (story 1200).
#[serde(default)]
pub disk_watch: DiskWatchConfig,
/// Project-wide default QA mode: "server", "agent", or "human".
/// Per-story `qa` front matter overrides this. Default: "server".
#[serde(default = "default_qa")]
@@ -245,6 +248,58 @@ fn default_max_mesh_peers() -> usize {
3
}
/// Configuration for the low-disk-space watchdog's free-space thresholds.
///
/// Sleds check free space on the `/workspace` filesystem each tick and
/// compare it against `warn_gb` / `critical_gb`, rate-limiting repeat
/// notifications per level to `rate_limit_secs` and sending a single
/// recovery notice once free space climbs back above
/// `warn_gb * (1 + recovery_margin_pct / 100)`.
#[derive(Debug, Clone, Deserialize, PartialEq)]
pub struct DiskWatchConfig {
/// Free space (in GB) below which a "warn" notification fires. Default: 50.
#[serde(default = "default_disk_warn_gb")]
pub warn_gb: u64,
/// Free space (in GB) below which a "critical" notification fires. Default: 20.
#[serde(default = "default_disk_critical_gb")]
pub critical_gb: u64,
/// Minimum time (in seconds) between repeat notifications at the same
/// level while the condition persists. Default: 21600 (6 hours).
#[serde(default = "default_disk_rate_limit_secs")]
pub rate_limit_secs: u64,
/// Percentage above `warn_gb` free space must climb before a single
/// recovery notice is sent. Default: 10.
#[serde(default = "default_disk_recovery_margin_pct")]
pub recovery_margin_pct: u64,
}
impl Default for DiskWatchConfig {
fn default() -> Self {
Self {
warn_gb: default_disk_warn_gb(),
critical_gb: default_disk_critical_gb(),
rate_limit_secs: default_disk_rate_limit_secs(),
recovery_margin_pct: default_disk_recovery_margin_pct(),
}
}
}
fn default_disk_warn_gb() -> u64 {
50
}
fn default_disk_critical_gb() -> u64 {
20
}
fn default_disk_rate_limit_secs() -> u64 {
6 * 60 * 60
}
fn default_disk_recovery_margin_pct() -> u64 {
10
}
/// Configuration for a project component (name, path, setup/teardown commands).
///
/// `deny_unknown_fields` turns a top-level setting that lands inside a
@@ -347,6 +402,8 @@ struct LegacyProjectConfig {
agent: Option<AgentConfig>,
#[serde(default)]
watcher: WatcherConfig,
#[serde(default)]
disk_watch: DiskWatchConfig,
#[serde(default = "default_qa")]
default_qa: String,
#[serde(default)]
@@ -385,6 +442,7 @@ impl Default for ProjectConfig {
runtime: None,
}],
watcher: WatcherConfig::default(),
disk_watch: DiskWatchConfig::default(),
default_qa: default_qa(),
default_coder_model: None,
max_coders: None,
@@ -475,6 +533,7 @@ impl ProjectConfig {
component: legacy.component,
agent: vec![agent],
watcher: legacy.watcher,
disk_watch: legacy.disk_watch,
default_qa: legacy.default_qa,
default_coder_model: legacy.default_coder_model,
max_coders: legacy.max_coders,
@@ -516,6 +575,7 @@ impl ProjectConfig {
component: legacy.component,
agent: vec![agent],
watcher: legacy.watcher,
disk_watch: legacy.disk_watch,
default_qa: legacy.default_qa,
default_coder_model: legacy.default_coder_model,
max_coders: legacy.max_coders,
@@ -545,6 +605,7 @@ impl ProjectConfig {
component: legacy.component,
agent: Vec::new(),
watcher: legacy.watcher,
disk_watch: legacy.disk_watch,
default_qa: legacy.default_qa,
default_coder_model: legacy.default_coder_model,
max_coders: legacy.max_coders,