huskies: merge 1200 story Low-disk warning: the fleet tells the operator before the disk takes it down
This commit is contained in:
@@ -213,6 +213,15 @@ pub async fn run(
|
|||||||
// Track which stories we've claimed so we can detect conflicts.
|
// Track which stories we've claimed so we can detect conflicts.
|
||||||
let mut our_claims: HashMap<String, f64> = HashMap::new();
|
let mut our_claims: HashMap<String, f64> = HashMap::new();
|
||||||
|
|
||||||
|
// Low-disk-space watchdog (story 1200 AC1): tracks rate-limit/recovery
|
||||||
|
// state across loop iterations. Thresholds come from the config loaded
|
||||||
|
// at startup; host_id identifies this sled in chat messages and the
|
||||||
|
// gateway dedupe key.
|
||||||
|
let mut disk_watch_state = crate::service::disk_watch::DiskWatchState::default();
|
||||||
|
let disk_watch_host_id =
|
||||||
|
crdt_state::our_node_id().unwrap_or_else(|| "unknown-host".to_string());
|
||||||
|
let disk_watch_status = agents.status_broadcaster();
|
||||||
|
|
||||||
// Main loop: heartbeat, scan, claim, detect conflicts.
|
// Main loop: heartbeat, scan, claim, detect conflicts.
|
||||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(SCAN_INTERVAL_SECS));
|
let mut interval = tokio::time::interval(std::time::Duration::from_secs(SCAN_INTERVAL_SECS));
|
||||||
loop {
|
loop {
|
||||||
@@ -221,6 +230,16 @@ pub async fn run(
|
|||||||
// Write heartbeat.
|
// Write heartbeat.
|
||||||
write_heartbeat(&rendezvous_url, port);
|
write_heartbeat(&rendezvous_url, port);
|
||||||
|
|
||||||
|
// Low-disk-space check (story 1200 AC1): every tick period.
|
||||||
|
crate::service::disk_watch::io::check_and_notify(
|
||||||
|
&project_root,
|
||||||
|
&config.disk_watch,
|
||||||
|
&mut disk_watch_state,
|
||||||
|
&watcher_tx,
|
||||||
|
&disk_watch_status,
|
||||||
|
&disk_watch_host_id,
|
||||||
|
);
|
||||||
|
|
||||||
// Scan CRDT for claimable work.
|
// Scan CRDT for claimable work.
|
||||||
scan_and_claim(&agents, &project_root, &mut our_claims).await;
|
scan_and_claim(&agents, &project_root, &mut our_claims).await;
|
||||||
|
|
||||||
|
|||||||
@@ -478,6 +478,7 @@ pub async fn run_health_check(ctx: &BotContext) -> String {
|
|||||||
lines.push(sync_line);
|
lines.push(sync_line);
|
||||||
lines.push(creds_line);
|
lines.push(creds_line);
|
||||||
lines.push(hash_line);
|
lines.push(hash_line);
|
||||||
|
lines.push(check_disk_space());
|
||||||
|
|
||||||
let lines = truncate_lines(lines);
|
let lines = truncate_lines(lines);
|
||||||
lines
|
lines
|
||||||
@@ -487,6 +488,26 @@ pub async fn run_health_check(ctx: &BotContext) -> String {
|
|||||||
.join("\n")
|
.join("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Report current free disk space on `/workspace` (story 1200 AC5).
|
||||||
|
fn check_disk_space() -> HealthLine {
|
||||||
|
match crate::service::disk_watch::io::free_space_bytes(std::path::Path::new("/workspace")) {
|
||||||
|
Ok(free_bytes) => {
|
||||||
|
let free_gb = free_bytes as f64 / 1_000_000_000.0;
|
||||||
|
HealthLine {
|
||||||
|
subsystem: "disk".to_string(),
|
||||||
|
status: Status::Pass,
|
||||||
|
detail: Some(format!("{free_gb:.1}GB free")),
|
||||||
|
hint: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => HealthLine::warn(
|
||||||
|
"disk",
|
||||||
|
"unable to read free space",
|
||||||
|
"check /workspace mount",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Utilities ────────────────────────────────────────────────────────────────
|
// ── Utilities ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Shorten a long error string to the first 60 characters for compact display.
|
/// Shorten a long error string to the first 60 characters for compact display.
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ pub struct ProjectConfig {
|
|||||||
pub agent: Vec<AgentConfig>,
|
pub agent: Vec<AgentConfig>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub watcher: WatcherConfig,
|
pub watcher: WatcherConfig,
|
||||||
|
/// Configuration for the low-disk-space watchdog (story 1200).
|
||||||
|
#[serde(default)]
|
||||||
|
pub disk_watch: DiskWatchConfig,
|
||||||
/// Project-wide default QA mode: "server", "agent", or "human".
|
/// Project-wide default QA mode: "server", "agent", or "human".
|
||||||
/// Per-story `qa` front matter overrides this. Default: "server".
|
/// Per-story `qa` front matter overrides this. Default: "server".
|
||||||
#[serde(default = "default_qa")]
|
#[serde(default = "default_qa")]
|
||||||
@@ -245,6 +248,58 @@ fn default_max_mesh_peers() -> usize {
|
|||||||
3
|
3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Configuration for the low-disk-space watchdog's free-space thresholds.
|
||||||
|
///
|
||||||
|
/// Sleds check free space on the `/workspace` filesystem each tick and
|
||||||
|
/// compare it against `warn_gb` / `critical_gb`, rate-limiting repeat
|
||||||
|
/// notifications per level to `rate_limit_secs` and sending a single
|
||||||
|
/// recovery notice once free space climbs back above
|
||||||
|
/// `warn_gb * (1 + recovery_margin_pct / 100)`.
|
||||||
|
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||||
|
pub struct DiskWatchConfig {
|
||||||
|
/// Free space (in GB) below which a "warn" notification fires. Default: 50.
|
||||||
|
#[serde(default = "default_disk_warn_gb")]
|
||||||
|
pub warn_gb: u64,
|
||||||
|
/// Free space (in GB) below which a "critical" notification fires. Default: 20.
|
||||||
|
#[serde(default = "default_disk_critical_gb")]
|
||||||
|
pub critical_gb: u64,
|
||||||
|
/// Minimum time (in seconds) between repeat notifications at the same
|
||||||
|
/// level while the condition persists. Default: 21600 (6 hours).
|
||||||
|
#[serde(default = "default_disk_rate_limit_secs")]
|
||||||
|
pub rate_limit_secs: u64,
|
||||||
|
/// Percentage above `warn_gb` free space must climb before a single
|
||||||
|
/// recovery notice is sent. Default: 10.
|
||||||
|
#[serde(default = "default_disk_recovery_margin_pct")]
|
||||||
|
pub recovery_margin_pct: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for DiskWatchConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
warn_gb: default_disk_warn_gb(),
|
||||||
|
critical_gb: default_disk_critical_gb(),
|
||||||
|
rate_limit_secs: default_disk_rate_limit_secs(),
|
||||||
|
recovery_margin_pct: default_disk_recovery_margin_pct(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_disk_warn_gb() -> u64 {
|
||||||
|
50
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_disk_critical_gb() -> u64 {
|
||||||
|
20
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_disk_rate_limit_secs() -> u64 {
|
||||||
|
6 * 60 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_disk_recovery_margin_pct() -> u64 {
|
||||||
|
10
|
||||||
|
}
|
||||||
|
|
||||||
/// Configuration for a project component (name, path, setup/teardown commands).
|
/// Configuration for a project component (name, path, setup/teardown commands).
|
||||||
///
|
///
|
||||||
/// `deny_unknown_fields` turns a top-level setting that lands inside a
|
/// `deny_unknown_fields` turns a top-level setting that lands inside a
|
||||||
@@ -347,6 +402,8 @@ struct LegacyProjectConfig {
|
|||||||
agent: Option<AgentConfig>,
|
agent: Option<AgentConfig>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
watcher: WatcherConfig,
|
watcher: WatcherConfig,
|
||||||
|
#[serde(default)]
|
||||||
|
disk_watch: DiskWatchConfig,
|
||||||
#[serde(default = "default_qa")]
|
#[serde(default = "default_qa")]
|
||||||
default_qa: String,
|
default_qa: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -385,6 +442,7 @@ impl Default for ProjectConfig {
|
|||||||
runtime: None,
|
runtime: None,
|
||||||
}],
|
}],
|
||||||
watcher: WatcherConfig::default(),
|
watcher: WatcherConfig::default(),
|
||||||
|
disk_watch: DiskWatchConfig::default(),
|
||||||
default_qa: default_qa(),
|
default_qa: default_qa(),
|
||||||
default_coder_model: None,
|
default_coder_model: None,
|
||||||
max_coders: None,
|
max_coders: None,
|
||||||
@@ -475,6 +533,7 @@ impl ProjectConfig {
|
|||||||
component: legacy.component,
|
component: legacy.component,
|
||||||
agent: vec![agent],
|
agent: vec![agent],
|
||||||
watcher: legacy.watcher,
|
watcher: legacy.watcher,
|
||||||
|
disk_watch: legacy.disk_watch,
|
||||||
default_qa: legacy.default_qa,
|
default_qa: legacy.default_qa,
|
||||||
default_coder_model: legacy.default_coder_model,
|
default_coder_model: legacy.default_coder_model,
|
||||||
max_coders: legacy.max_coders,
|
max_coders: legacy.max_coders,
|
||||||
@@ -516,6 +575,7 @@ impl ProjectConfig {
|
|||||||
component: legacy.component,
|
component: legacy.component,
|
||||||
agent: vec![agent],
|
agent: vec![agent],
|
||||||
watcher: legacy.watcher,
|
watcher: legacy.watcher,
|
||||||
|
disk_watch: legacy.disk_watch,
|
||||||
default_qa: legacy.default_qa,
|
default_qa: legacy.default_qa,
|
||||||
default_coder_model: legacy.default_coder_model,
|
default_coder_model: legacy.default_coder_model,
|
||||||
max_coders: legacy.max_coders,
|
max_coders: legacy.max_coders,
|
||||||
@@ -545,6 +605,7 @@ impl ProjectConfig {
|
|||||||
component: legacy.component,
|
component: legacy.component,
|
||||||
agent: Vec::new(),
|
agent: Vec::new(),
|
||||||
watcher: legacy.watcher,
|
watcher: legacy.watcher,
|
||||||
|
disk_watch: legacy.disk_watch,
|
||||||
default_qa: legacy.default_qa,
|
default_qa: legacy.default_qa,
|
||||||
default_coder_model: legacy.default_coder_model,
|
default_coder_model: legacy.default_coder_model,
|
||||||
max_coders: legacy.max_coders,
|
max_coders: legacy.max_coders,
|
||||||
|
|||||||
@@ -513,6 +513,176 @@ async fn broadcaster_forwarder_resubscribes_on_lag() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Story 1200 AC4: identical disk-space warnings arriving from different
|
||||||
|
/// sleds within the rate window must collapse into a single forwarded chat
|
||||||
|
/// message.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn broadcaster_forwarder_dedupes_identical_disk_warnings_from_different_sleds() {
|
||||||
|
use crate::chat::{ChatTransport, MessageId};
|
||||||
|
use crate::service::events::StoredEvent;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
type CallLog = Arc<std::sync::Mutex<Vec<(String, String)>>>;
|
||||||
|
|
||||||
|
struct MockTransport {
|
||||||
|
calls: CallLog,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl ChatTransport for MockTransport {
|
||||||
|
async fn send_message(
|
||||||
|
&self,
|
||||||
|
room_id: &str,
|
||||||
|
plain: &str,
|
||||||
|
_html: &str,
|
||||||
|
) -> Result<MessageId, String> {
|
||||||
|
self.calls
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push((room_id.to_string(), plain.to_string()));
|
||||||
|
Ok("id".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit_message(
|
||||||
|
&self,
|
||||||
|
_room_id: &str,
|
||||||
|
_id: &str,
|
||||||
|
_plain: &str,
|
||||||
|
_html: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send_typing(&self, _room_id: &str, _typing: bool) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let calls: CallLog = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||||
|
let transport = Arc::new(MockTransport {
|
||||||
|
calls: Arc::clone(&calls),
|
||||||
|
});
|
||||||
|
|
||||||
|
let (tx, rx) =
|
||||||
|
tokio::sync::broadcast::channel::<crate::service::gateway::GatewayStatusEvent>(16);
|
||||||
|
gateway::spawn_gateway_broadcaster_forwarder(
|
||||||
|
transport as Arc<dyn crate::chat::ChatTransport>,
|
||||||
|
vec!["!room:example.org".to_string()],
|
||||||
|
rx,
|
||||||
|
);
|
||||||
|
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
|
|
||||||
|
let warning = |host_id: &str| crate::service::gateway::GatewayStatusEvent {
|
||||||
|
project: host_id.to_string(),
|
||||||
|
event: StoredEvent::DiskSpaceWarning {
|
||||||
|
level: "warn".to_string(),
|
||||||
|
free_bytes: 45_000_000_000,
|
||||||
|
target_bytes: 10_000_000_000,
|
||||||
|
worktrees_bytes: 5_000_000_000,
|
||||||
|
host_id: host_id.to_string(),
|
||||||
|
timestamp_ms: 100,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Two different sleds both observe the same "warn" level within the
|
||||||
|
// dedupe window — only the first should be forwarded.
|
||||||
|
tx.send(warning("sled-a")).unwrap();
|
||||||
|
tx.send(warning("sled-b")).unwrap();
|
||||||
|
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
|
let messages = calls.lock().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
messages.len(),
|
||||||
|
1,
|
||||||
|
"Expected identical disk warnings from different sleds to dedupe to one message"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Non-disk-space events (e.g. stage transitions) must never be deduped, even
|
||||||
|
/// when several arrive back-to-back — only disk-space warnings/recoveries
|
||||||
|
/// share a dedupe key (story 1200 AC4).
|
||||||
|
#[tokio::test]
|
||||||
|
async fn broadcaster_forwarder_does_not_dedupe_non_disk_events() {
|
||||||
|
use crate::chat::{ChatTransport, MessageId};
|
||||||
|
use crate::service::events::StoredEvent;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
|
||||||
|
type CallLog = Arc<std::sync::Mutex<Vec<(String, String)>>>;
|
||||||
|
|
||||||
|
struct MockTransport {
|
||||||
|
calls: CallLog,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl ChatTransport for MockTransport {
|
||||||
|
async fn send_message(
|
||||||
|
&self,
|
||||||
|
room_id: &str,
|
||||||
|
plain: &str,
|
||||||
|
_html: &str,
|
||||||
|
) -> Result<MessageId, String> {
|
||||||
|
self.calls
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push((room_id.to_string(), plain.to_string()));
|
||||||
|
Ok("id".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit_message(
|
||||||
|
&self,
|
||||||
|
_room_id: &str,
|
||||||
|
_id: &str,
|
||||||
|
_plain: &str,
|
||||||
|
_html: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send_typing(&self, _room_id: &str, _typing: bool) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let calls: CallLog = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||||
|
let transport = Arc::new(MockTransport {
|
||||||
|
calls: Arc::clone(&calls),
|
||||||
|
});
|
||||||
|
|
||||||
|
let (tx, rx) =
|
||||||
|
tokio::sync::broadcast::channel::<crate::service::gateway::GatewayStatusEvent>(16);
|
||||||
|
gateway::spawn_gateway_broadcaster_forwarder(
|
||||||
|
transport as Arc<dyn crate::chat::ChatTransport>,
|
||||||
|
vec!["!room:example.org".to_string()],
|
||||||
|
rx,
|
||||||
|
);
|
||||||
|
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
|
|
||||||
|
let transition = |n: u64| crate::service::gateway::GatewayStatusEvent {
|
||||||
|
project: "p".to_string(),
|
||||||
|
event: StoredEvent::StageTransition {
|
||||||
|
story_id: format!("{n}_story"),
|
||||||
|
story_name: String::new(),
|
||||||
|
from_stage: "2_current".to_string(),
|
||||||
|
to_stage: "3_qa".to_string(),
|
||||||
|
timestamp_ms: n,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
tx.send(transition(1)).unwrap();
|
||||||
|
tx.send(transition(2)).unwrap();
|
||||||
|
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
|
||||||
|
let messages = calls.lock().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
messages.len(),
|
||||||
|
2,
|
||||||
|
"Non-disk events must not be deduped against each other"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ── BotConfig tests ─────────────────────────────────────────────────
|
// ── BotConfig tests ─────────────────────────────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -168,6 +168,28 @@ fn status_to_stored(event: StatusEvent) -> Option<StoredEvent> {
|
|||||||
reason,
|
reason,
|
||||||
timestamp_ms: now_ms,
|
timestamp_ms: now_ms,
|
||||||
}),
|
}),
|
||||||
|
StatusEvent::DiskSpaceWarning {
|
||||||
|
level,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
host_id,
|
||||||
|
} => Some(StoredEvent::DiskSpaceWarning {
|
||||||
|
level,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
host_id,
|
||||||
|
timestamp_ms: now_ms,
|
||||||
|
}),
|
||||||
|
StatusEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id,
|
||||||
|
} => Some(StoredEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id,
|
||||||
|
timestamp_ms: now_ms,
|
||||||
|
}),
|
||||||
// Rate-limit events have no StoredEvent equivalent — skip them.
|
// Rate-limit events have no StoredEvent equivalent — skip them.
|
||||||
StatusEvent::RateLimitWarning { .. } | StatusEvent::RateLimitHardBlock { .. } => None,
|
StatusEvent::RateLimitWarning { .. } | StatusEvent::RateLimitHardBlock { .. } => None,
|
||||||
}
|
}
|
||||||
@@ -252,6 +274,36 @@ mod tests {
|
|||||||
assert!(status_to_stored(ev).is_none());
|
assert!(status_to_stored(ev).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn status_to_stored_disk_space_warning() {
|
||||||
|
let ev = StatusEvent::DiskSpaceWarning {
|
||||||
|
level: "warn".into(),
|
||||||
|
free_bytes: 45_000_000_000,
|
||||||
|
target_bytes: 10_000_000_000,
|
||||||
|
worktrees_bytes: 5_000_000_000,
|
||||||
|
host_id: "sled-a".into(),
|
||||||
|
};
|
||||||
|
let stored = status_to_stored(ev).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
stored,
|
||||||
|
StoredEvent::DiskSpaceWarning { ref level, ref host_id, .. }
|
||||||
|
if level == "warn" && host_id == "sled-a"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn status_to_stored_disk_space_recovered() {
|
||||||
|
let ev = StatusEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes: 60_000_000_000,
|
||||||
|
host_id: "sled-a".into(),
|
||||||
|
};
|
||||||
|
let stored = status_to_stored(ev).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
stored,
|
||||||
|
StoredEvent::DiskSpaceRecovered { ref host_id, .. } if host_id == "sled-a"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn status_to_stored_rate_limit_hard_block_is_none() {
|
fn status_to_stored_rate_limit_hard_block_is_none() {
|
||||||
let ev = StatusEvent::RateLimitHardBlock {
|
let ev = StatusEvent::RateLimitHardBlock {
|
||||||
|
|||||||
@@ -111,4 +111,28 @@ pub enum WatcherEvent {
|
|||||||
/// Total auto-retry budget shared with the auto-block threshold.
|
/// Total auto-retry budget shared with the auto-block threshold.
|
||||||
budget: u32,
|
budget: u32,
|
||||||
},
|
},
|
||||||
|
/// Free disk space on the `/workspace` filesystem crossed a warn/critical
|
||||||
|
/// threshold (story 1200).
|
||||||
|
/// Triggers a rate-limited warning notification to configured chat rooms.
|
||||||
|
DiskSpaceWarning {
|
||||||
|
/// Severity level: `"warn"` or `"critical"`.
|
||||||
|
level: String,
|
||||||
|
/// Free space in bytes at the time of the check.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Size of the `target/` directory in bytes.
|
||||||
|
target_bytes: u64,
|
||||||
|
/// Size of the `.huskies/worktrees/` directory in bytes.
|
||||||
|
worktrees_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the reading.
|
||||||
|
host_id: String,
|
||||||
|
},
|
||||||
|
/// Free disk space recovered above the warn threshold plus its recovery
|
||||||
|
/// margin, after a warn/critical warning had been sent (story 1200).
|
||||||
|
/// Triggers a single recovery notification to configured chat rooms.
|
||||||
|
DiskSpaceRecovered {
|
||||||
|
/// Free space in bytes at the time of recovery.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the recovery.
|
||||||
|
host_id: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
//! Side effects for the disk-space watchdog: statvfs free-space reads,
|
||||||
|
//! directory-size walks, and the periodic check-and-notify entry point.
|
||||||
|
|
||||||
|
use std::ffi::CString;
|
||||||
|
use std::os::unix::ffi::OsStrExt as _;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
use crate::config::DiskWatchConfig;
|
||||||
|
use crate::io::watcher::WatcherEvent;
|
||||||
|
use crate::service::status::{StatusBroadcaster, StatusEvent};
|
||||||
|
|
||||||
|
use super::{DiskAction, DiskWatchState, decide_action};
|
||||||
|
|
||||||
|
/// Read free bytes available on the filesystem containing `path`.
|
||||||
|
pub fn free_space_bytes(path: &Path) -> std::io::Result<u64> {
|
||||||
|
let c_path = CString::new(path.as_os_str().as_bytes())
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
|
||||||
|
// SAFETY: `c_path` is a valid NUL-terminated string and `stat` is a
|
||||||
|
// zero-initialized `libc::statvfs` passed by mutable reference, matching
|
||||||
|
// the `statvfs(2)` contract.
|
||||||
|
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
|
||||||
|
let ret = unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) };
|
||||||
|
if ret != 0 {
|
||||||
|
return Err(std::io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
Ok(stat.f_bavail as u64 * stat.f_frsize as u64)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursively sum file sizes under `path`. Returns 0 if `path` does not exist.
|
||||||
|
pub fn dir_size_bytes(path: &Path) -> u64 {
|
||||||
|
walkdir::WalkDir::new(path)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.filter(|e| e.file_type().is_file())
|
||||||
|
.filter_map(|e| e.metadata().ok())
|
||||||
|
.map(|m| m.len())
|
||||||
|
.sum()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run one disk-space check against `workspace_root`, updating `state` and
|
||||||
|
/// broadcasting a [`WatcherEvent`] (local chat notifications) and a
|
||||||
|
/// [`StatusEvent`] (gateway relay, for cross-sled dedupe — story 1200 AC4)
|
||||||
|
/// when a warning or recovery notification is due.
|
||||||
|
///
|
||||||
|
/// `host_id` identifies the sled that observed the reading (surfaced in the
|
||||||
|
/// gateway dedupe key and chat message).
|
||||||
|
pub fn check_and_notify(
|
||||||
|
workspace_root: &Path,
|
||||||
|
config: &DiskWatchConfig,
|
||||||
|
state: &mut DiskWatchState,
|
||||||
|
watcher_tx: &broadcast::Sender<WatcherEvent>,
|
||||||
|
status: &StatusBroadcaster,
|
||||||
|
host_id: &str,
|
||||||
|
) {
|
||||||
|
let Ok(free_bytes) = free_space_bytes(workspace_root) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let now = Instant::now();
|
||||||
|
match decide_action(free_bytes, config, state, now) {
|
||||||
|
DiskAction::None => {}
|
||||||
|
DiskAction::Warning(level) => {
|
||||||
|
let target_bytes = dir_size_bytes(&workspace_root.join("target"));
|
||||||
|
let worktrees_bytes = dir_size_bytes(&workspace_root.join(".huskies/worktrees"));
|
||||||
|
let level_str = level.as_str().to_string();
|
||||||
|
let _ = watcher_tx.send(WatcherEvent::DiskSpaceWarning {
|
||||||
|
level: level_str.clone(),
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
host_id: host_id.to_string(),
|
||||||
|
});
|
||||||
|
status.publish(StatusEvent::DiskSpaceWarning {
|
||||||
|
level: level_str,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
host_id: host_id.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
DiskAction::Recovery => {
|
||||||
|
let _ = watcher_tx.send(WatcherEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id: host_id.to_string(),
|
||||||
|
});
|
||||||
|
status.publish(StatusEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id: host_id.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn free_space_bytes_reads_a_real_path() {
|
||||||
|
// The current directory always exists in test runs; just assert the
|
||||||
|
// call succeeds and returns a plausible non-zero value.
|
||||||
|
let free = free_space_bytes(Path::new(".")).expect("statvfs should succeed on cwd");
|
||||||
|
assert!(free > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dir_size_bytes_sums_files_in_a_temp_dir() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::write(tmp.path().join("a.txt"), b"hello").unwrap();
|
||||||
|
std::fs::write(tmp.path().join("b.txt"), b"world!").unwrap();
|
||||||
|
assert_eq!(dir_size_bytes(tmp.path()), 11);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dir_size_bytes_returns_zero_for_missing_path() {
|
||||||
|
assert_eq!(dir_size_bytes(Path::new("/no/such/path/1200")), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn check_and_notify_sends_warning_event_on_low_space() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let (tx, mut rx) = broadcast::channel::<WatcherEvent>(4);
|
||||||
|
let status = StatusBroadcaster::new();
|
||||||
|
let mut status_sub = status.subscribe();
|
||||||
|
let config = DiskWatchConfig {
|
||||||
|
warn_gb: u64::MAX / 1_000_000_000, // force Warn on any real filesystem
|
||||||
|
critical_gb: 0,
|
||||||
|
rate_limit_secs: 21_600,
|
||||||
|
recovery_margin_pct: 10,
|
||||||
|
};
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
check_and_notify(tmp.path(), &config, &mut state, &tx, &status, "sled-a");
|
||||||
|
let event = rx.try_recv().expect("should broadcast a WatcherEvent");
|
||||||
|
assert!(matches!(event, WatcherEvent::DiskSpaceWarning { .. }));
|
||||||
|
let status_event =
|
||||||
|
tokio::time::timeout(std::time::Duration::from_millis(100), status_sub.recv())
|
||||||
|
.await
|
||||||
|
.expect("should not time out")
|
||||||
|
.expect("should publish a StatusEvent");
|
||||||
|
assert!(matches!(status_event, StatusEvent::DiskSpaceWarning { .. }));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
//! Low-disk-space watchdog (story 1200) — checks free space on the
|
||||||
|
//! `/workspace` filesystem each tick and decides whether to emit a
|
||||||
|
//! rate-limited warn/critical warning or a one-shot recovery notice.
|
||||||
|
//!
|
||||||
|
//! Follows service-module conventions: this file holds pure classification
|
||||||
|
//! and rate-limiting logic (no I/O, no `Instant::now()` calls — callers
|
||||||
|
//! supply `now`); `io.rs` is the only place performing side effects
|
||||||
|
//! (statvfs reads, directory-size walks, broadcasting `WatcherEvent`s).
|
||||||
|
|
||||||
|
/// Side effects: free-space reads, directory-size walks, and the periodic
|
||||||
|
/// check-and-notify entry point.
|
||||||
|
pub mod io;
|
||||||
|
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use crate::config::DiskWatchConfig;
|
||||||
|
|
||||||
|
/// Bytes per gigabyte, using the same decimal (GB, not GiB) convention as the
|
||||||
|
/// `warn_gb` / `critical_gb` config fields.
|
||||||
|
const BYTES_PER_GB: u64 = 1_000_000_000;
|
||||||
|
|
||||||
|
/// Disk-space severity level relative to configured thresholds.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum DiskLevel {
|
||||||
|
/// Free space is at or above the warn threshold.
|
||||||
|
Ok,
|
||||||
|
/// Free space is below `warn_gb` but at/above `critical_gb`.
|
||||||
|
Warn,
|
||||||
|
/// Free space is below `critical_gb`.
|
||||||
|
Critical,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DiskLevel {
|
||||||
|
/// Wire/display name used in notifications and gateway dedupe keys.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
DiskLevel::Ok => "ok",
|
||||||
|
DiskLevel::Warn => "warn",
|
||||||
|
DiskLevel::Critical => "critical",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Classify `free_bytes` against the configured warn/critical thresholds.
|
||||||
|
pub fn classify_level(free_bytes: u64, config: &DiskWatchConfig) -> DiskLevel {
|
||||||
|
if free_bytes < config.critical_gb * BYTES_PER_GB {
|
||||||
|
DiskLevel::Critical
|
||||||
|
} else if free_bytes < config.warn_gb * BYTES_PER_GB {
|
||||||
|
DiskLevel::Warn
|
||||||
|
} else {
|
||||||
|
DiskLevel::Ok
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if free space has recovered enough to send a recovery
|
||||||
|
/// notice: at or above `warn_gb * (1 + recovery_margin_pct / 100)`.
|
||||||
|
pub fn is_recovered(free_bytes: u64, config: &DiskWatchConfig) -> bool {
|
||||||
|
let recovery_threshold =
|
||||||
|
config.warn_gb * BYTES_PER_GB * (100 + config.recovery_margin_pct) / 100;
|
||||||
|
free_bytes >= recovery_threshold
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if a repeat notification for the same level should be
|
||||||
|
/// sent, given when it was last sent (`None` if never) and the configured
|
||||||
|
/// rate limit.
|
||||||
|
pub fn should_send(last_sent: Option<Instant>, now: Instant, rate_limit_secs: u64) -> bool {
|
||||||
|
match last_sent {
|
||||||
|
None => true,
|
||||||
|
Some(last) => now.duration_since(last) >= Duration::from_secs(rate_limit_secs),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-sled in-memory state tracked across tick iterations: the last level
|
||||||
|
/// observed, and when a notification was last sent for each level.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct DiskWatchState {
|
||||||
|
/// The disk level observed on the previous check, `None` before the first check.
|
||||||
|
pub last_level: Option<DiskLevel>,
|
||||||
|
/// When a "warn" notification was last sent.
|
||||||
|
pub warn_last_sent: Option<Instant>,
|
||||||
|
/// When a "critical" notification was last sent.
|
||||||
|
pub critical_last_sent: Option<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The next action to take based on a fresh disk-level observation.
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
pub enum DiskAction {
|
||||||
|
/// No notification needed.
|
||||||
|
None,
|
||||||
|
/// Send a warn/critical warning notification.
|
||||||
|
Warning(DiskLevel),
|
||||||
|
/// Send a single recovery notice.
|
||||||
|
Recovery,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide what action to take given a fresh free-space reading, updating
|
||||||
|
/// `state` in place. Pure aside from taking `now` as an argument (no direct
|
||||||
|
/// I/O or wall-clock reads).
|
||||||
|
pub fn decide_action(
|
||||||
|
free_bytes: u64,
|
||||||
|
config: &DiskWatchConfig,
|
||||||
|
state: &mut DiskWatchState,
|
||||||
|
now: Instant,
|
||||||
|
) -> DiskAction {
|
||||||
|
let level = classify_level(free_bytes, config);
|
||||||
|
let was_persisting = matches!(
|
||||||
|
state.last_level,
|
||||||
|
Some(DiskLevel::Warn) | Some(DiskLevel::Critical)
|
||||||
|
);
|
||||||
|
state.last_level = Some(level);
|
||||||
|
|
||||||
|
match level {
|
||||||
|
DiskLevel::Ok => {
|
||||||
|
if was_persisting && is_recovered(free_bytes, config) {
|
||||||
|
state.warn_last_sent = None;
|
||||||
|
state.critical_last_sent = None;
|
||||||
|
DiskAction::Recovery
|
||||||
|
} else {
|
||||||
|
DiskAction::None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DiskLevel::Warn => {
|
||||||
|
if should_send(state.warn_last_sent, now, config.rate_limit_secs) {
|
||||||
|
state.warn_last_sent = Some(now);
|
||||||
|
DiskAction::Warning(DiskLevel::Warn)
|
||||||
|
} else {
|
||||||
|
DiskAction::None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DiskLevel::Critical => {
|
||||||
|
if should_send(state.critical_last_sent, now, config.rate_limit_secs) {
|
||||||
|
state.critical_last_sent = Some(now);
|
||||||
|
DiskAction::Warning(DiskLevel::Critical)
|
||||||
|
} else {
|
||||||
|
DiskAction::None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn config() -> DiskWatchConfig {
|
||||||
|
DiskWatchConfig {
|
||||||
|
warn_gb: 50,
|
||||||
|
critical_gb: 20,
|
||||||
|
rate_limit_secs: 6 * 60 * 60,
|
||||||
|
recovery_margin_pct: 10,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const GB: u64 = 1_000_000_000;
|
||||||
|
|
||||||
|
// ── classify_level (threshold crossing) ───────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_above_warn_is_ok() {
|
||||||
|
assert_eq!(classify_level(60 * GB, &config()), DiskLevel::Ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_at_warn_boundary_is_warn() {
|
||||||
|
// Below warn_gb (50) counts as Warn; exactly at warn_gb is still Ok
|
||||||
|
// (the threshold is "below", not "at or below").
|
||||||
|
assert_eq!(classify_level(50 * GB, &config()), DiskLevel::Ok);
|
||||||
|
assert_eq!(classify_level(49 * GB, &config()), DiskLevel::Warn);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_between_warn_and_critical_is_warn() {
|
||||||
|
assert_eq!(classify_level(30 * GB, &config()), DiskLevel::Warn);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_at_critical_boundary() {
|
||||||
|
assert_eq!(classify_level(20 * GB, &config()), DiskLevel::Warn);
|
||||||
|
assert_eq!(classify_level(19 * GB, &config()), DiskLevel::Critical);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_below_critical_is_critical() {
|
||||||
|
assert_eq!(classify_level(5 * GB, &config()), DiskLevel::Critical);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── is_recovered ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_recovered_requires_margin_above_warn() {
|
||||||
|
let cfg = config();
|
||||||
|
// warn_gb=50, margin=10% => recovery threshold = 55 GB.
|
||||||
|
assert!(!is_recovered(54 * GB, &cfg));
|
||||||
|
assert!(is_recovered(55 * GB, &cfg));
|
||||||
|
assert!(is_recovered(60 * GB, &cfg));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── should_send (rate limiting) ────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn should_send_when_never_sent() {
|
||||||
|
assert!(should_send(None, Instant::now(), 21_600));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn should_not_send_within_rate_limit_window() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let last = now - Duration::from_secs(100);
|
||||||
|
assert!(!should_send(Some(last), now, 21_600));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn should_send_after_rate_limit_window_expires() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let last = now - Duration::from_secs(21_601);
|
||||||
|
assert!(should_send(Some(last), now, 21_600));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── decide_action (threshold crossing + rate limiting + recovery) ──────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_first_warn_reading_sends_warning() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let action = decide_action(30 * GB, &cfg, &mut state, Instant::now());
|
||||||
|
assert_eq!(action, DiskAction::Warning(DiskLevel::Warn));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_repeat_warn_within_window_is_suppressed() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::Warning(DiskLevel::Warn)
|
||||||
|
);
|
||||||
|
// Second check 1 minute later — same level, still within 6h window.
|
||||||
|
let later = now + Duration::from_secs(60);
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, later),
|
||||||
|
DiskAction::None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_repeat_warn_after_window_sends_again() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now);
|
||||||
|
let later = now + Duration::from_secs(6 * 60 * 60 + 1);
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, later),
|
||||||
|
DiskAction::Warning(DiskLevel::Warn)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_warn_and_critical_have_independent_rate_limits() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::Warning(DiskLevel::Warn)
|
||||||
|
);
|
||||||
|
// Dropping straight to critical moments later must still notify —
|
||||||
|
// it has its own rate-limit bucket, separate from warn's.
|
||||||
|
let later = now + Duration::from_secs(5);
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(10 * GB, &cfg, &mut state, later),
|
||||||
|
DiskAction::Warning(DiskLevel::Critical)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_recovery_after_persisting_warn() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now); // Warn
|
||||||
|
// Free space climbs back above the 55 GB recovery threshold.
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(60 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::Recovery
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_no_recovery_notice_without_prior_warning() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
// First-ever reading is healthy — must not fire a spurious recovery notice.
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(60 * GB, &cfg, &mut state, Instant::now()),
|
||||||
|
DiskAction::None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_recovery_is_sent_only_once() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now); // Warn
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(60 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::Recovery
|
||||||
|
);
|
||||||
|
// Still healthy on the next tick — no repeat recovery notice.
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(60 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decide_action_back_above_warn_but_below_margin_is_not_recovery() {
|
||||||
|
let cfg = config();
|
||||||
|
let mut state = DiskWatchState::default();
|
||||||
|
let now = Instant::now();
|
||||||
|
decide_action(30 * GB, &cfg, &mut state, now); // Warn
|
||||||
|
// 52 GB is above warn_gb (50) but below the 55 GB recovery margin.
|
||||||
|
assert_eq!(
|
||||||
|
decide_action(52 * GB, &cfg, &mut state, now),
|
||||||
|
DiskAction::None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -45,6 +45,30 @@ pub enum StoredEvent {
|
|||||||
/// Unix timestamp in milliseconds when this event was recorded.
|
/// Unix timestamp in milliseconds when this event was recorded.
|
||||||
timestamp_ms: u64,
|
timestamp_ms: u64,
|
||||||
},
|
},
|
||||||
|
/// Free disk space on a sled crossed a warn/critical threshold (story 1200).
|
||||||
|
DiskSpaceWarning {
|
||||||
|
/// Severity level: `"warn"` or `"critical"`.
|
||||||
|
level: String,
|
||||||
|
/// Free space in bytes at the time of the check.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Size of the `target/` directory in bytes.
|
||||||
|
target_bytes: u64,
|
||||||
|
/// Size of the `.huskies/worktrees/` directory in bytes.
|
||||||
|
worktrees_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the reading.
|
||||||
|
host_id: String,
|
||||||
|
/// Unix timestamp in milliseconds when this event was recorded.
|
||||||
|
timestamp_ms: u64,
|
||||||
|
},
|
||||||
|
/// Free disk space recovered on a sled after a warn/critical warning (story 1200).
|
||||||
|
DiskSpaceRecovered {
|
||||||
|
/// Free space in bytes at the time of recovery.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the recovery.
|
||||||
|
host_id: String,
|
||||||
|
/// Unix timestamp in milliseconds when this event was recorded.
|
||||||
|
timestamp_ms: u64,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StoredEvent {
|
impl StoredEvent {
|
||||||
@@ -54,6 +78,8 @@ impl StoredEvent {
|
|||||||
StoredEvent::StageTransition { timestamp_ms, .. } => *timestamp_ms,
|
StoredEvent::StageTransition { timestamp_ms, .. } => *timestamp_ms,
|
||||||
StoredEvent::MergeFailure { timestamp_ms, .. } => *timestamp_ms,
|
StoredEvent::MergeFailure { timestamp_ms, .. } => *timestamp_ms,
|
||||||
StoredEvent::StoryBlocked { timestamp_ms, .. } => *timestamp_ms,
|
StoredEvent::StoryBlocked { timestamp_ms, .. } => *timestamp_ms,
|
||||||
|
StoredEvent::DiskSpaceWarning { timestamp_ms, .. } => *timestamp_ms,
|
||||||
|
StoredEvent::DiskSpaceRecovered { timestamp_ms, .. } => *timestamp_ms,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -507,15 +507,51 @@ pub fn init_wizard_state(path: &Path) {
|
|||||||
///
|
///
|
||||||
/// The task exits cleanly when the broadcast channel is closed (i.e. when
|
/// The task exits cleanly when the broadcast channel is closed (i.e. when
|
||||||
/// `GatewayState` is dropped).
|
/// `GatewayState` is dropped).
|
||||||
|
/// Window within which identical disk-space warnings from different sleds are
|
||||||
|
/// deduped into a single forwarded chat message (story 1200 AC4). Matches the
|
||||||
|
/// sled-side default `rate_limit_secs`.
|
||||||
|
const DISK_DEDUPE_WINDOW: std::time::Duration = std::time::Duration::from_secs(6 * 60 * 60);
|
||||||
|
|
||||||
|
/// Returns the dedupe key for a [`crate::service::events::StoredEvent`], or
|
||||||
|
/// `None` if the event type is never deduped (only disk-space events dedupe
|
||||||
|
/// across sleds — story 1200 AC4).
|
||||||
|
fn dedupe_key(event: &crate::service::events::StoredEvent) -> Option<String> {
|
||||||
|
match event {
|
||||||
|
crate::service::events::StoredEvent::DiskSpaceWarning { level, .. } => {
|
||||||
|
Some(format!("disk_warning:{level}"))
|
||||||
|
}
|
||||||
|
crate::service::events::StoredEvent::DiskSpaceRecovered { .. } => {
|
||||||
|
Some("disk_recovered".to_string())
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn spawn_gateway_broadcaster_forwarder(
|
pub fn spawn_gateway_broadcaster_forwarder(
|
||||||
transport: std::sync::Arc<dyn crate::chat::ChatTransport>,
|
transport: std::sync::Arc<dyn crate::chat::ChatTransport>,
|
||||||
room_ids: Vec<String>,
|
room_ids: Vec<String>,
|
||||||
mut rx: tokio::sync::broadcast::Receiver<super::GatewayStatusEvent>,
|
mut rx: tokio::sync::broadcast::Receiver<super::GatewayStatusEvent>,
|
||||||
) {
|
) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let mut last_forwarded: std::collections::HashMap<String, std::time::Instant> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
loop {
|
loop {
|
||||||
match rx.recv().await {
|
match rx.recv().await {
|
||||||
Ok(event) => {
|
Ok(event) => {
|
||||||
|
if let Some(key) = dedupe_key(&event.event) {
|
||||||
|
let now = std::time::Instant::now();
|
||||||
|
let should_forward = match last_forwarded.get(&key) {
|
||||||
|
None => true,
|
||||||
|
Some(last) => now.duration_since(*last) >= DISK_DEDUPE_WINDOW,
|
||||||
|
};
|
||||||
|
if !should_forward {
|
||||||
|
crate::slog!(
|
||||||
|
"[gateway-forwarder] Deduping repeated {key} within window"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_forwarded.insert(key, now);
|
||||||
|
}
|
||||||
let (plain, html) =
|
let (plain, html) =
|
||||||
super::polling::format_gateway_event(&event.project, &event.event);
|
super::polling::format_gateway_event(&event.project, &event.event);
|
||||||
for room_id in &room_ids {
|
for room_id in &room_ids {
|
||||||
@@ -839,4 +875,57 @@ mod tests {
|
|||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
assert!(read_installed_manifest(dir.path(), "huskies-linux-arm64").is_none());
|
assert!(read_installed_manifest(dir.path(), "huskies-linux-arm64").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── dedupe_key (story 1200 AC4) ──────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dedupe_key_disk_warning_keys_by_level() {
|
||||||
|
let warn = crate::service::events::StoredEvent::DiskSpaceWarning {
|
||||||
|
level: "warn".to_string(),
|
||||||
|
free_bytes: 1,
|
||||||
|
target_bytes: 0,
|
||||||
|
worktrees_bytes: 0,
|
||||||
|
host_id: "sled-a".to_string(),
|
||||||
|
timestamp_ms: 0,
|
||||||
|
};
|
||||||
|
let critical = crate::service::events::StoredEvent::DiskSpaceWarning {
|
||||||
|
level: "critical".to_string(),
|
||||||
|
free_bytes: 1,
|
||||||
|
target_bytes: 0,
|
||||||
|
worktrees_bytes: 0,
|
||||||
|
host_id: "sled-b".to_string(),
|
||||||
|
timestamp_ms: 0,
|
||||||
|
};
|
||||||
|
// Different sleds at the same level share a key (so they dedupe)...
|
||||||
|
assert_eq!(dedupe_key(&warn), dedupe_key(&warn));
|
||||||
|
// ...but different levels do not.
|
||||||
|
assert_ne!(dedupe_key(&warn), dedupe_key(&critical));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dedupe_key_disk_recovered_has_a_single_shared_key() {
|
||||||
|
let a = crate::service::events::StoredEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes: 1,
|
||||||
|
host_id: "sled-a".to_string(),
|
||||||
|
timestamp_ms: 0,
|
||||||
|
};
|
||||||
|
let b = crate::service::events::StoredEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes: 2,
|
||||||
|
host_id: "sled-b".to_string(),
|
||||||
|
timestamp_ms: 1,
|
||||||
|
};
|
||||||
|
assert_eq!(dedupe_key(&a), dedupe_key(&b));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dedupe_key_non_disk_events_are_not_deduped() {
|
||||||
|
let transition = crate::service::events::StoredEvent::StageTransition {
|
||||||
|
story_id: "1_story".to_string(),
|
||||||
|
story_name: String::new(),
|
||||||
|
from_stage: "2_current".to_string(),
|
||||||
|
to_stage: "3_qa".to_string(),
|
||||||
|
timestamp_ms: 0,
|
||||||
|
};
|
||||||
|
assert_eq!(dedupe_key(&transition), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,8 @@
|
|||||||
use crate::pipeline_state::Stage;
|
use crate::pipeline_state::Stage;
|
||||||
use crate::service::events::StoredEvent;
|
use crate::service::events::StoredEvent;
|
||||||
use crate::service::notifications::{
|
use crate::service::notifications::{
|
||||||
format_blocked_notification, format_error_notification, format_stage_notification,
|
format_blocked_notification, format_disk_recovery_notification,
|
||||||
|
format_disk_warning_notification, format_error_notification, format_stage_notification,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Format a [`StoredEvent`] from a project into a gateway notification.
|
/// Format a [`StoredEvent`] from a project into a gateway notification.
|
||||||
@@ -49,6 +50,31 @@ pub fn format_gateway_event(project_name: &str, event: &StoredEvent) -> (String,
|
|||||||
let (plain, html) = format_blocked_notification(story_id, story_name, reason);
|
let (plain, html) = format_blocked_notification(story_id, story_name, reason);
|
||||||
(format!("{prefix}{plain}"), format!("{prefix}{html}"))
|
(format!("{prefix}{plain}"), format!("{prefix}{html}"))
|
||||||
}
|
}
|
||||||
|
StoredEvent::DiskSpaceWarning {
|
||||||
|
level,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
host_id,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let (plain, html) = format_disk_warning_notification(
|
||||||
|
level,
|
||||||
|
host_id,
|
||||||
|
*free_bytes,
|
||||||
|
*target_bytes,
|
||||||
|
*worktrees_bytes,
|
||||||
|
);
|
||||||
|
(format!("{prefix}{plain}"), format!("{prefix}{html}"))
|
||||||
|
}
|
||||||
|
StoredEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let (plain, html) = format_disk_recovery_notification(host_id, *free_bytes);
|
||||||
|
(format!("{prefix}{plain}"), format!("{prefix}{html}"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +117,14 @@ pub fn format_gateway_audit_line(project: &str, event: &StoredEvent) -> String {
|
|||||||
"audit ts={ts} project={project} id={story_id} event=story_blocked reason={reason}"
|
"audit ts={ts} project={project} id={story_id} event=story_blocked reason={reason}"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
StoredEvent::DiskSpaceWarning { level, host_id, .. } => {
|
||||||
|
format!(
|
||||||
|
"audit ts={ts} project={project} event=disk_space_warning level={level} host={host_id}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
StoredEvent::DiskSpaceRecovered { host_id, .. } => {
|
||||||
|
format!("audit ts={ts} project={project} event=disk_space_recovered host={host_id}")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ pub mod bot_command;
|
|||||||
pub mod common;
|
pub mod common;
|
||||||
/// Diagnostics — server logs, CRDT dump, and permission management.
|
/// Diagnostics — server logs, CRDT dump, and permission management.
|
||||||
pub mod diagnostics;
|
pub mod diagnostics;
|
||||||
|
/// Low-disk-space watchdog — free-space threshold checks and notifications.
|
||||||
|
pub mod disk_watch;
|
||||||
/// Event-based pipeline triggers: register, list, cancel, and execute on TransitionFired events.
|
/// Event-based pipeline triggers: register, list, cancel, and execute on TransitionFired events.
|
||||||
pub mod event_triggers;
|
pub mod event_triggers;
|
||||||
/// Pipeline event buffer for SSE streaming.
|
/// Pipeline event buffer for SSE streaming.
|
||||||
|
|||||||
@@ -30,6 +30,13 @@ pub enum EventAction {
|
|||||||
NewItemCreated,
|
NewItemCreated,
|
||||||
/// Post a merge-auto-retry notification naming the attempt and budget.
|
/// Post a merge-auto-retry notification naming the attempt and budget.
|
||||||
MergeAutoRetry,
|
MergeAutoRetry,
|
||||||
|
/// Post a low-disk-space warning notification (story 1200).
|
||||||
|
DiskWarning {
|
||||||
|
/// Severity level: `"warn"` or `"critical"`.
|
||||||
|
level: String,
|
||||||
|
},
|
||||||
|
/// Post a disk-space recovery notification (story 1200).
|
||||||
|
DiskRecovery,
|
||||||
/// Log server-side only; do not post to chat (e.g. hard rate-limit blocks).
|
/// Log server-side only; do not post to chat (e.g. hard rate-limit blocks).
|
||||||
LogOnly,
|
LogOnly,
|
||||||
/// Reload the project configuration.
|
/// Reload the project configuration.
|
||||||
@@ -57,6 +64,10 @@ pub fn classify(event: &WatcherEvent) -> EventAction {
|
|||||||
}
|
}
|
||||||
WatcherEvent::NewItemCreated { .. } => EventAction::NewItemCreated,
|
WatcherEvent::NewItemCreated { .. } => EventAction::NewItemCreated,
|
||||||
WatcherEvent::MergeAutoRetry { .. } => EventAction::MergeAutoRetry,
|
WatcherEvent::MergeAutoRetry { .. } => EventAction::MergeAutoRetry,
|
||||||
|
WatcherEvent::DiskSpaceWarning { level, .. } => EventAction::DiskWarning {
|
||||||
|
level: level.clone(),
|
||||||
|
},
|
||||||
|
WatcherEvent::DiskSpaceRecovered { .. } => EventAction::DiskRecovery,
|
||||||
_ => EventAction::Skip,
|
_ => EventAction::Skip,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -272,6 +272,57 @@ pub fn format_merge_auto_retry_notification(
|
|||||||
(plain, html)
|
(plain, html)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Format a low-disk-space warning notification message (story 1200 AC3).
|
||||||
|
///
|
||||||
|
/// Includes free space, `target/` and `.huskies/worktrees/` directory sizes,
|
||||||
|
/// and names the `gc` tool as the first remediation step.
|
||||||
|
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
|
||||||
|
pub fn format_disk_warning_notification(
|
||||||
|
level: &str,
|
||||||
|
host_id: &str,
|
||||||
|
free_bytes: u64,
|
||||||
|
target_bytes: u64,
|
||||||
|
worktrees_bytes: u64,
|
||||||
|
) -> (String, String) {
|
||||||
|
let emoji = if level == "critical" {
|
||||||
|
"\u{1f6a8}" // 🚨
|
||||||
|
} else {
|
||||||
|
"\u{26a0}\u{fe0f}" // ⚠️
|
||||||
|
};
|
||||||
|
let free_gb = bytes_to_gb(free_bytes);
|
||||||
|
let target_gb = bytes_to_gb(target_bytes);
|
||||||
|
let worktrees_gb = bytes_to_gb(worktrees_bytes);
|
||||||
|
let plain = format!(
|
||||||
|
"{emoji} Low disk space on {host_id} ({level}): {free_gb:.1}GB free \
|
||||||
|
(target/ {target_gb:.1}GB, worktrees/ {worktrees_gb:.1}GB) \
|
||||||
|
\u{2014} first response: run the `gc` tool to reclaim space"
|
||||||
|
);
|
||||||
|
let html = format!(
|
||||||
|
"{emoji} Low disk space on <strong>{host_id}</strong> ({level}): {free_gb:.1}GB free \
|
||||||
|
(target/ {target_gb:.1}GB, worktrees/ {worktrees_gb:.1}GB) \
|
||||||
|
\u{2014} first response: run the <code>gc</code> tool to reclaim space"
|
||||||
|
);
|
||||||
|
(plain, html)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format a disk-space-recovered notification message (story 1200 AC2).
|
||||||
|
///
|
||||||
|
/// Sent once when free space climbs back above the configured recovery
|
||||||
|
/// margin after a warn/critical warning.
|
||||||
|
/// Returns `(plain_text, html)` suitable for `ChatTransport::send_message`.
|
||||||
|
pub fn format_disk_recovery_notification(host_id: &str, free_bytes: u64) -> (String, String) {
|
||||||
|
let free_gb = bytes_to_gb(free_bytes);
|
||||||
|
let plain = format!("\u{2705} Disk space recovered on {host_id}: {free_gb:.1}GB free");
|
||||||
|
let html =
|
||||||
|
format!("\u{2705} Disk space recovered on <strong>{host_id}</strong>: {free_gb:.1}GB free");
|
||||||
|
(plain, html)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert a byte count to gigabytes for display (story 1200).
|
||||||
|
fn bytes_to_gb(bytes: u64) -> f64 {
|
||||||
|
bytes as f64 / 1_000_000_000.0
|
||||||
|
}
|
||||||
|
|
||||||
/// Maximum number of trailing gate-output lines included in a merge-failure
|
/// Maximum number of trailing gate-output lines included in a merge-failure
|
||||||
/// chat notification.
|
/// chat notification.
|
||||||
///
|
///
|
||||||
@@ -813,4 +864,50 @@ mod tests {
|
|||||||
format_agent_completed_notification("42_story_foo", "", "coder-1", false);
|
format_agent_completed_notification("42_story_foo", "", "coder-1", false);
|
||||||
assert_eq!(plain, "\u{274C} #42 \u{2014} coder-1 failed");
|
assert_eq!(plain, "\u{274C} #42 \u{2014} coder-1 failed");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── format_disk_warning_notification ──────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_disk_warning_notification_warn_level_includes_content() {
|
||||||
|
let (plain, html) = format_disk_warning_notification(
|
||||||
|
"warn",
|
||||||
|
"sled-a",
|
||||||
|
45_000_000_000,
|
||||||
|
30_000_000_000,
|
||||||
|
10_000_000_000,
|
||||||
|
);
|
||||||
|
assert!(plain.contains("sled-a"));
|
||||||
|
assert!(plain.contains("45.0GB free"));
|
||||||
|
assert!(plain.contains("target/ 30.0GB"));
|
||||||
|
assert!(plain.contains("worktrees/ 10.0GB"));
|
||||||
|
assert!(plain.contains("`gc` tool"));
|
||||||
|
assert!(html.contains("<code>gc</code> tool"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_disk_warning_notification_critical_uses_distinct_emoji() {
|
||||||
|
let (plain, _html) =
|
||||||
|
format_disk_warning_notification("critical", "sled-b", 5_000_000_000, 0, 0);
|
||||||
|
assert!(plain.starts_with("\u{1f6a8}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_disk_warning_notification_warn_uses_warning_emoji() {
|
||||||
|
let (plain, _html) =
|
||||||
|
format_disk_warning_notification("warn", "sled-b", 45_000_000_000, 0, 0);
|
||||||
|
assert!(plain.starts_with("\u{26a0}\u{fe0f}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── format_disk_recovery_notification ─────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn format_disk_recovery_notification_includes_host_and_free_space() {
|
||||||
|
let (plain, html) = format_disk_recovery_notification("sled-a", 60_000_000_000);
|
||||||
|
assert_eq!(
|
||||||
|
plain,
|
||||||
|
"\u{2705} Disk space recovered on sled-a: 60.0GB free"
|
||||||
|
);
|
||||||
|
assert!(html.contains("<strong>sled-a</strong>"));
|
||||||
|
assert!(html.contains("60.0GB free"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ use super::super::events::classify;
|
|||||||
use super::super::filter::{AGENT_EVENT_DEBOUNCE, should_send_rate_limit};
|
use super::super::filter::{AGENT_EVENT_DEBOUNCE, should_send_rate_limit};
|
||||||
use super::super::format::{
|
use super::super::format::{
|
||||||
MERGE_FAILURE_TAIL_LINES, format_agent_completed_notification,
|
MERGE_FAILURE_TAIL_LINES, format_agent_completed_notification,
|
||||||
format_agent_started_notification, format_blocked_notification, format_error_notification,
|
format_agent_started_notification, format_blocked_notification,
|
||||||
|
format_disk_recovery_notification, format_disk_warning_notification, format_error_notification,
|
||||||
format_merge_auto_retry_notification, format_new_item_notification,
|
format_merge_auto_retry_notification, format_new_item_notification,
|
||||||
format_oauth_account_swapped, format_oauth_accounts_exhausted, format_rate_limit_notification,
|
format_oauth_account_swapped, format_oauth_accounts_exhausted, format_rate_limit_notification,
|
||||||
truncate_gate_output,
|
truncate_gate_output,
|
||||||
@@ -325,6 +326,59 @@ pub fn spawn_notification_listener(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
EventAction::DiskWarning { .. } => {
|
||||||
|
if !config.status_push_enabled {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let WatcherEvent::DiskSpaceWarning {
|
||||||
|
ref level,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
ref host_id,
|
||||||
|
} = event
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let (plain, html) = format_disk_warning_notification(
|
||||||
|
level,
|
||||||
|
host_id,
|
||||||
|
free_bytes,
|
||||||
|
target_bytes,
|
||||||
|
worktrees_bytes,
|
||||||
|
);
|
||||||
|
slog!("[bot] Sending disk-space warning notification: {plain}");
|
||||||
|
for room_id in &rooms_for_notification(&get_room_ids) {
|
||||||
|
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
|
||||||
|
slog!(
|
||||||
|
"[bot] Failed to send disk-space warning notification \
|
||||||
|
to {room_id}: {e}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EventAction::DiskRecovery => {
|
||||||
|
if !config.status_push_enabled {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let WatcherEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
ref host_id,
|
||||||
|
} = event
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let (plain, html) = format_disk_recovery_notification(host_id, free_bytes);
|
||||||
|
slog!("[bot] Sending disk-space recovery notification: {plain}");
|
||||||
|
for room_id in &rooms_for_notification(&get_room_ids) {
|
||||||
|
if let Err(e) = transport.send_message(room_id, &plain, &html).await {
|
||||||
|
slog!(
|
||||||
|
"[bot] Failed to send disk-space recovery notification \
|
||||||
|
to {room_id}: {e}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
EventAction::LogOnly => {
|
EventAction::LogOnly => {
|
||||||
// Hard-block: log server-side for debugging; do NOT post to chat.
|
// Hard-block: log server-side for debugging; do NOT post to chat.
|
||||||
// Hard-block auto-resume is normal operation — the status command
|
// Hard-block auto-resume is normal operation — the status command
|
||||||
|
|||||||
@@ -19,7 +19,8 @@ pub(super) mod io;
|
|||||||
pub(super) mod route;
|
pub(super) mod route;
|
||||||
|
|
||||||
pub use format::{
|
pub use format::{
|
||||||
format_blocked_notification, format_error_notification, format_stage_notification,
|
format_blocked_notification, format_disk_recovery_notification,
|
||||||
|
format_disk_warning_notification, format_error_notification, format_stage_notification,
|
||||||
};
|
};
|
||||||
pub use io::spawn_notification_listener;
|
pub use io::spawn_notification_listener;
|
||||||
pub use io::spawn_stage_notification_subscriber;
|
pub use io::spawn_stage_notification_subscriber;
|
||||||
|
|||||||
@@ -97,6 +97,22 @@ pub fn format_status_event(event: &StatusEvent) -> String {
|
|||||||
"\u{26d4} #{number} {name} \u{2014} {agent_name} hard rate-limited until {reset}"
|
"\u{26d4} #{number} {name} \u{2014} {agent_name} hard rate-limited until {reset}"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
StatusEvent::DiskSpaceWarning {
|
||||||
|
level,
|
||||||
|
free_bytes,
|
||||||
|
host_id,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let free_gb = *free_bytes as f64 / 1_000_000_000.0;
|
||||||
|
format!("\u{26a0}\u{fe0f} Low disk space on {host_id} ({level}): {free_gb:.1}GB free")
|
||||||
|
}
|
||||||
|
StatusEvent::DiskSpaceRecovered {
|
||||||
|
free_bytes,
|
||||||
|
host_id,
|
||||||
|
} => {
|
||||||
|
let free_gb = *free_bytes as f64 / 1_000_000_000.0;
|
||||||
|
format!("\u{2705} Disk space recovered on {host_id}: {free_gb:.1}GB free")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -100,6 +100,26 @@ pub enum StatusEvent {
|
|||||||
/// UTC instant at which the rate limit resets.
|
/// UTC instant at which the rate limit resets.
|
||||||
reset_at: DateTime<Utc>,
|
reset_at: DateTime<Utc>,
|
||||||
},
|
},
|
||||||
|
/// Free disk space on a sled crossed a warn/critical threshold (story 1200).
|
||||||
|
DiskSpaceWarning {
|
||||||
|
/// Severity level: `"warn"` or `"critical"`.
|
||||||
|
level: String,
|
||||||
|
/// Free space in bytes at the time of the check.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Size of the `target/` directory in bytes.
|
||||||
|
target_bytes: u64,
|
||||||
|
/// Size of the `.huskies/worktrees/` directory in bytes.
|
||||||
|
worktrees_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the reading.
|
||||||
|
host_id: String,
|
||||||
|
},
|
||||||
|
/// Free disk space recovered on a sled after a warn/critical warning (story 1200).
|
||||||
|
DiskSpaceRecovered {
|
||||||
|
/// Free space in bytes at the time of recovery.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Identifier of the sled that observed the recovery.
|
||||||
|
host_id: String,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Subscription ──────────────────────────────────────────────────────────────
|
// ── Subscription ──────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ pub fn watcher_event_to_response(e: WatcherEvent) -> Option<WsResponse> {
|
|||||||
WatcherEvent::NewItemCreated { .. } => None,
|
WatcherEvent::NewItemCreated { .. } => None,
|
||||||
// Merge-auto-retry notifications are forwarded to chat transports only; no WebSocket message.
|
// Merge-auto-retry notifications are forwarded to chat transports only; no WebSocket message.
|
||||||
WatcherEvent::MergeAutoRetry { .. } => None,
|
WatcherEvent::MergeAutoRetry { .. } => None,
|
||||||
|
// Disk-space events are forwarded to chat transports only; no WebSocket message (story 1200).
|
||||||
|
WatcherEvent::DiskSpaceWarning { .. } => None,
|
||||||
|
WatcherEvent::DiskSpaceRecovered { .. } => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -183,9 +183,15 @@ pub(crate) fn spawn_tick_loop(
|
|||||||
})
|
})
|
||||||
.unwrap_or((30, std::time::Duration::from_secs(4 * 3600)));
|
.unwrap_or((30, std::time::Duration::from_secs(4 * 3600)));
|
||||||
|
|
||||||
|
let disk_watch_host_id =
|
||||||
|
std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown-host".to_string());
|
||||||
|
let disk_watch_watcher_tx = agents.watcher_tx();
|
||||||
|
let disk_watch_status = agents.status_broadcaster();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
|
let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
|
||||||
let mut tick_count: u64 = 0;
|
let mut tick_count: u64 = 0;
|
||||||
|
let mut disk_watch_state = service::disk_watch::DiskWatchState::default();
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
tick_count = tick_count.wrapping_add(1);
|
tick_count = tick_count.wrapping_add(1);
|
||||||
@@ -218,6 +224,25 @@ pub(crate) fn spawn_tick_loop(
|
|||||||
agents.reap_stale_merge_jobs();
|
agents.reap_stale_merge_jobs();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Low-disk-space watchdog (story 1200): check free space on the
|
||||||
|
// project workspace filesystem every 30 ticks (~30s) and emit a
|
||||||
|
// rate-limited warn/critical notification, or a recovery notice.
|
||||||
|
if tick_count.is_multiple_of(30)
|
||||||
|
&& let Some(ref r) = root
|
||||||
|
{
|
||||||
|
let disk_config = config::ProjectConfig::load(r)
|
||||||
|
.map(|c| c.disk_watch)
|
||||||
|
.unwrap_or_default();
|
||||||
|
service::disk_watch::io::check_and_notify(
|
||||||
|
r,
|
||||||
|
&disk_config,
|
||||||
|
&mut disk_watch_state,
|
||||||
|
&disk_watch_watcher_tx,
|
||||||
|
&disk_watch_status,
|
||||||
|
&disk_watch_host_id,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Periodic reconciler: converge subscriber side effects so that
|
// Periodic reconciler: converge subscriber side effects so that
|
||||||
// Lagged broadcast events never leave state permanently diverged.
|
// Lagged broadcast events never leave state permanently diverged.
|
||||||
if tick_count.is_multiple_of(reconcile_interval)
|
if tick_count.is_multiple_of(reconcile_interval)
|
||||||
|
|||||||
@@ -194,6 +194,7 @@ mod tests {
|
|||||||
component: vec![],
|
component: vec![],
|
||||||
agent: vec![],
|
agent: vec![],
|
||||||
watcher: WatcherConfig::default(),
|
watcher: WatcherConfig::default(),
|
||||||
|
disk_watch: Default::default(),
|
||||||
default_qa: "server".to_string(),
|
default_qa: "server".to_string(),
|
||||||
default_coder_model: None,
|
default_coder_model: None,
|
||||||
max_coders: None,
|
max_coders: None,
|
||||||
|
|||||||
@@ -233,6 +233,7 @@ mod tests {
|
|||||||
component: vec![],
|
component: vec![],
|
||||||
agent: vec![],
|
agent: vec![],
|
||||||
watcher: WatcherConfig::default(),
|
watcher: WatcherConfig::default(),
|
||||||
|
disk_watch: Default::default(),
|
||||||
default_qa: "server".to_string(),
|
default_qa: "server".to_string(),
|
||||||
default_coder_model: None,
|
default_coder_model: None,
|
||||||
max_coders: None,
|
max_coders: None,
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ mod tests {
|
|||||||
component: vec![],
|
component: vec![],
|
||||||
agent: vec![],
|
agent: vec![],
|
||||||
watcher: WatcherConfig::default(),
|
watcher: WatcherConfig::default(),
|
||||||
|
disk_watch: Default::default(),
|
||||||
default_qa: "server".to_string(),
|
default_qa: "server".to_string(),
|
||||||
default_coder_model: None,
|
default_coder_model: None,
|
||||||
max_coders: None,
|
max_coders: None,
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ mod tests {
|
|||||||
component: vec![],
|
component: vec![],
|
||||||
agent: vec![],
|
agent: vec![],
|
||||||
watcher: WatcherConfig::default(),
|
watcher: WatcherConfig::default(),
|
||||||
|
disk_watch: Default::default(),
|
||||||
default_qa: "server".to_string(),
|
default_qa: "server".to_string(),
|
||||||
default_coder_model: None,
|
default_coder_model: None,
|
||||||
max_coders: None,
|
max_coders: None,
|
||||||
|
|||||||
Reference in New Issue
Block a user