From 2fb935e7264ea18ae02241a462251b37c87c6f02 Mon Sep 17 00:00:00 2001
From: dave <futurechimp@users.noreply.github.com>
Date: Wed, 20 May 2026 01:13:29 +0000
Subject: [PATCH] huskies: merge 1156 story Periodic liveness tick so runtime
 freezes have a precise timestamp

---
 server/src/config/mod.rs               | 10 ++++++++++
 server/src/main.rs                     |  9 +++++++++
 server/src/service/settings/project.rs |  1 +
 server/src/startup/tick_loop.rs        | 23 +++++++++++++++++++++++
 4 files changed, 43 insertions(+)

diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs
index 28d7bc74..d3ea29ec 100644
--- a/server/src/config/mod.rs
+++ b/server/src/config/mod.rs
@@ -167,6 +167,11 @@ pub struct WatcherConfig {
     /// state diverged. Default: 30 seconds.
     #[serde(default = "default_reconcile_interval_secs")]
     pub reconcile_interval_secs: u64,
+    /// How often (in seconds) the liveness tick task emits a heartbeat log line.
+    /// Reduce if 5-second granularity causes noise; increase to quiet the logs.
+    /// Default: 5 seconds.
+    #[serde(default = "default_liveness_tick_secs")]
+    pub liveness_tick_secs: u64,
 }
 
 impl Default for WatcherConfig {
@@ -175,6 +180,7 @@ impl Default for WatcherConfig {
             sweep_interval_secs: default_sweep_interval_secs(),
             done_retention_secs: default_done_retention_secs(),
             reconcile_interval_secs: default_reconcile_interval_secs(),
+            liveness_tick_secs: default_liveness_tick_secs(),
         }
     }
 }
@@ -191,6 +197,10 @@ fn default_reconcile_interval_secs() -> u64 {
     30
 }
 
+fn default_liveness_tick_secs() -> u64 {
+    5
+}
+
 fn default_qa() -> String {
     "server".to_string()
 }
diff --git a/server/src/main.rs b/server/src/main.rs
index 292e0813..61318e08 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -403,6 +403,15 @@ async fn main() -> Result<(), std::io::Error> {
         Some(event_buffer),
     );
 
+    // Permanent liveness heartbeat — stops when the tokio runtime freezes,
+    // giving a precise timestamp of the freeze from the last [liveness] line.
+    let liveness_tick_secs = startup_root
+        .as_ref()
+        .and_then(|r| config::ProjectConfig::load(r).ok())
+        .map(|c| c.watcher.liveness_tick_secs)
+        .unwrap_or(5);
+    startup::tick_loop::spawn_liveness_tick(liveness_tick_secs);
+
     // Unified 1-second background tick loop.
     startup::tick_loop::spawn_tick_loop(
         Arc::clone(&startup_agents),
diff --git a/server/src/service/settings/project.rs b/server/src/service/settings/project.rs
index 156bb421..a3e617db 100644
--- a/server/src/service/settings/project.rs
+++ b/server/src/service/settings/project.rs
@@ -192,6 +192,7 @@ mod tests {
                 sweep_interval_secs: 30,
                 done_retention_secs: 7200,
                 reconcile_interval_secs: 30,
+                liveness_tick_secs: 5,
             },
             ..Default::default()
         };
diff --git a/server/src/startup/tick_loop.rs b/server/src/startup/tick_loop.rs
index 9345a231..67c97ebc 100644
--- a/server/src/startup/tick_loop.rs
+++ b/server/src/startup/tick_loop.rs
@@ -226,6 +226,29 @@ pub(crate) fn spawn_tick_loop(
     });
 }
 
+/// Spawn a permanent background task that emits a liveness heartbeat log line
+/// every `interval_secs` seconds on the main tokio runtime.
+///
+/// Running on the main runtime (not `spawn_blocking`) means the heartbeat stops
+/// the moment the runtime freezes, giving an exact timestamp of the freeze from
+/// the last `[liveness]` line in the logs.
+pub(crate) fn spawn_liveness_tick(interval_secs: u64) {
+    let interval_secs = interval_secs.max(1);
+    crate::slog!("[liveness] liveness tick started; cadence={interval_secs}s");
+    tokio::spawn(async move {
+        let duration = std::time::Duration::from_secs(interval_secs);
+        let mut ticker = tokio::time::interval(duration);
+        loop {
+            ticker.tick().await;
+            let ts = std::time::UNIX_EPOCH
+                .elapsed()
+                .unwrap_or_default()
+                .as_secs();
+            crate::slog!("[liveness] tokio runtime alive ts={ts}");
+        }
+    });
+}
+
 /// Fire any due generic scheduled timers and re-arm recurring ones.
 ///
 /// Called every second from the unified tick loop.  Catch-up semantics: timers