huskies: merge 1128 story Bounded event queues + EventStreamGap sentinel + observability for context assembly

server/src/crdt_state/lww_maps/event_log.rs

@@ -12,6 +12,14 @@ use serde_json::json;
 use super::super::state::{apply_and_persist, get_crdt};
 use super::super::types::EventLogEntryCrdt;
 
+/// `pipeline_event` value used to mark a gap sentinel entry in the event log.
+///
+/// A gap sentinel is appended when the event-log subscriber detects that the
+/// broadcast channel dropped events (i.e. it received `RecvError::Lagged`).
+/// The `from_stage` and `to_stage` fields encode the logical EventId range
+/// `[from, to]` of the dropped events as decimal strings.
+pub const GAP_PIPELINE_EVENT: &str = "EventStreamGap";
+
 /// Raw event log entry extracted from the CRDT document.
 ///
 /// All fields are decoded to Rust primitives; entries with a missing or
@@ -86,6 +94,26 @@ pub fn append_event_log_entry(
     apply_and_persist(&mut state, |s| s.crdt.doc.event_log.insert(after, entry));
 }
 
+/// Append an `EventStreamGap` sentinel entry to the CRDT event log.
+///
+/// Called when the event-log broadcast subscriber detects that the channel
+/// dropped events (`RecvError::Lagged`).  `from_id` and `to_id` are the
+/// logical sequence numbers (in the per-sled event stream) of the first and
+/// last dropped events respectively.  The sentinel itself also consumes one
+/// CRDT `event_seq` slot so the monotonic counter remains contiguous across
+/// the gap.
+pub fn append_gap_log_entry(sled_id: &str, from_id: u64, to_id: u64) {
+    let timestamp = chrono::Utc::now().timestamp() as f64;
+    append_event_log_entry(
+        sled_id,
+        timestamp,
+        "",
+        &from_id.to_string(),
+        &to_id.to_string(),
+        GAP_PIPELINE_EVENT,
+    );
+}
+
 /// Read all event log entries from the CRDT document.
 ///
 /// Entries with a missing or empty `sled_id` are silently skipped.

server/src/crdt_state/lww_maps/llm_sessions.rs

@@ -14,6 +14,7 @@ use serde_json::json;
 
 use super::super::state::{apply_and_persist, get_crdt, rebuild_llm_session_index};
 use super::super::types::{LlmSessionCrdt, LlmSessionView};
+use super::event_log::GAP_PIPELINE_EVENT;
 
 /// Write or upsert an LLM session entry keyed by `session_id`.
 ///
@@ -110,7 +111,7 @@ pub fn assemble_and_advance_session(session_id: &str) -> Vec<String> {
     // Advance the high-water mark to the maximum new event_seq.
     let new_max_seq = new_events.iter().map(|(seq, ..)| *seq).max().unwrap_or(0);
     let mut new_high_water = current_high_water;
-    new_high_water.insert(local_sled_id, new_max_seq);
+    new_high_water.insert(local_sled_id.clone(), new_max_seq);
     let new_hw_json = serde_json::to_string(&new_high_water).unwrap_or_else(|_| "{}".to_string());
 
     // Upsert the session entry with the new high-water value.
@@ -135,14 +136,41 @@ pub fn assemble_and_advance_session(session_id: &str) -> Vec<String> {
         state.llm_session_index = rebuild_llm_session_index(&state.crdt);
     }
 
-    // Render each new event as a compact audit line.
+    // Observability: log event-log size and gap count for this sled.
+    let total_entries = state
+        .crdt
+        .doc
+        .event_log
+        .iter()
+        .filter(|e| matches!(e.sled_id.view(), JsonValue::String(s) if s == local_sled_id))
+        .count();
+    let gap_count = state
+        .crdt
+        .doc
+        .event_log
+        .iter()
+        .filter(|e| {
+            matches!(e.sled_id.view(), JsonValue::String(s) if s == local_sled_id)
+                && matches!(e.pipeline_event.view(), JsonValue::String(s) if s == GAP_PIPELINE_EVENT)
+        })
+        .count();
+    crate::slog!(
+        "[event-log] assemble session={session_id} sled_entries={total_entries} gap_count={gap_count}"
+    );
+
+    // Render each new event as a compact audit line; gap sentinels get a
+    // human-readable message so the LLM is never presented with raw field data.
     new_events
         .into_iter()
         .map(|(_, story_id, from_stage, to_stage, pipeline_event)| {
-            format!(
-                "pipeline_event story_id=\"{story_id}\" from=\"{from_stage}\" \
-                 to=\"{to_stage}\" event=\"{pipeline_event}\""
-            )
+            if pipeline_event == GAP_PIPELINE_EVENT {
+                format!("events between {from_stage} and {to_stage} were dropped")
+            } else {
+                format!(
+                    "pipeline_event story_id=\"{story_id}\" from=\"{from_stage}\" \
+                     to=\"{to_stage}\" event=\"{pipeline_event}\""
+                )
+            }
         })
         .collect()
 }

server/src/crdt_state/lww_maps/mod.rs

@@ -30,7 +30,10 @@ pub use active_agents::{
 pub use agent_throttle::{
     delete_agent_throttle, read_agent_throttle, read_all_agent_throttles, write_agent_throttle,
 };
-pub use event_log::{EventLogEntryRaw, append_event_log_entry, read_all_event_log_entries};
+pub use event_log::{
+    EventLogEntryRaw, GAP_PIPELINE_EVENT, append_event_log_entry, append_gap_log_entry,
+    read_all_event_log_entries,
+};
 pub use gateway_projects::{
     delete_gateway_project, read_all_gateway_projects, read_gateway_project, write_gateway_project,
 };

server/src/crdt_state/mod.rs

@@ -28,14 +28,14 @@ mod write;
 
 pub use gateway_config::{read_gateway_active_project, write_gateway_active_project};
 pub use lww_maps::{
-    EventLogEntryRaw, append_event_log_entry, assemble_and_advance_session, delete_active_agent,
-    delete_agent_throttle, delete_gateway_project, delete_merge_job, delete_test_job,
-    delete_token_usage, read_active_agent, read_agent_throttle, read_all_active_agents,
-    read_all_agent_throttles, read_all_event_log_entries, read_all_gateway_projects,
-    read_all_merge_jobs, read_all_test_jobs, read_all_token_usage, read_gateway_project,
-    read_llm_session, read_merge_job, read_test_job, read_token_usage, write_active_agent,
-    write_agent_throttle, write_gateway_project, write_llm_session, write_merge_job,
-    write_test_job, write_token_usage,
+    EventLogEntryRaw, GAP_PIPELINE_EVENT, append_event_log_entry, append_gap_log_entry,
+    assemble_and_advance_session, delete_active_agent, delete_agent_throttle,
+    delete_gateway_project, delete_merge_job, delete_test_job, delete_token_usage,
+    read_active_agent, read_agent_throttle, read_all_active_agents, read_all_agent_throttles,
+    read_all_event_log_entries, read_all_gateway_projects, read_all_merge_jobs, read_all_test_jobs,
+    read_all_token_usage, read_gateway_project, read_llm_session, read_merge_job, read_test_job,
+    read_token_usage, write_active_agent, write_agent_throttle, write_gateway_project,
+    write_llm_session, write_merge_job, write_test_job, write_token_usage,
 };
 pub use ops::{all_ops_json, apply_remote_op, ops_since, our_vector_clock, subscribe_ops};
 pub use presence::{