huskies: merge 1094 bug delete_story leaks zombie rows in pipeline_items shadow table — 176 tombstoned items still report non-terminal stages

server/src/crdt_state/mod.rs

@@ -54,10 +54,10 @@ pub use types::{
 };
 pub use write::{
     bump_retry_count, migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
-    migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric, name_from_story_id,
-    purge_done_stage_merge_jobs, set_agent, set_depends_on, set_epic, set_item_type, set_name,
-    set_origin, set_plan_state, set_qa_mode, set_resume_to, set_resume_to_raw, set_retry_count,
-    write_item,
+    migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric,
+    migrate_zombie_pipeline_rows, name_from_story_id, purge_done_stage_merge_jobs, set_agent,
+    set_depends_on, set_epic, set_item_type, set_name, set_origin, set_plan_state, set_qa_mode,
+    set_resume_to, set_resume_to_raw, set_retry_count, write_item,
 };
 
 #[cfg(test)]

server/src/crdt_state/write/migrations.rs

@@ -705,6 +705,59 @@ pub fn purge_done_stage_merge_jobs() {
     slog!("[crdt] Purged {count} stale MergeJob entries for terminal-stage stories");
 }
 
+/// Delete `pipeline_items` rows that correspond to CRDT-tombstoned stories.
+///
+/// Pre-1094 code deleted pipeline_items via a fire-and-forget channel that
+/// could be lost on an abrupt restart, leaving rows with non-terminal stage
+/// values for stories that no longer exist in the CRDT.  This migration
+/// removes those zombie rows on startup.
+///
+/// Idempotent: rows already absent are unaffected; running twice produces the
+/// same result.
+pub async fn migrate_zombie_pipeline_rows() {
+    let pool = match crate::db::get_shared_pool() {
+        Some(p) => p,
+        None => return,
+    };
+    let tombstone_ids = crate::crdt_state::tombstoned_ids();
+    sweep_zombie_rows(pool, &tombstone_ids).await;
+}
+
+/// Inner sweep used by [`migrate_zombie_pipeline_rows`] and its tests.
+///
+/// Deletes every `pipeline_items` row in `ids` whose stage is not already a
+/// terminal value.  Returns the number of rows deleted.
+#[cfg_attr(test, allow(dead_code))]
+pub(crate) async fn sweep_zombie_rows(pool: &sqlx::SqlitePool, ids: &[String]) -> u32 {
+    if ids.is_empty() {
+        return 0;
+    }
+    let mut cleaned = 0u32;
+    for story_id in ids {
+        match sqlx::query(
+            "DELETE FROM pipeline_items WHERE id = ?1 AND stage NOT IN \
+             ('done','archived','abandoned','superseded','rejected')",
+        )
+        .bind(story_id)
+        .execute(pool)
+        .await
+        {
+            Ok(r) if r.rows_affected() > 0 => cleaned += 1,
+            Ok(_) => {}
+            Err(e) => {
+                slog!(
+                    "[crdt] migrate_zombie_pipeline_rows: failed to delete '{}': {e}",
+                    story_id
+                );
+            }
+        }
+    }
+    if cleaned > 0 {
+        slog!("[crdt] Swept {cleaned} zombie pipeline_items rows for tombstoned stories");
+    }
+    cleaned
+}
+
 #[cfg(test)]
 mod merge_job_migration_tests {
     use super::super::super::state::init_for_test;
@@ -909,3 +962,100 @@ mod merge_job_migration_tests {
         migrate_merge_job(std::path::Path::new("/nonexistent/pipeline.db"));
     }
 }
+
+#[cfg(test)]
+mod zombie_row_migration_tests {
+    use super::super::super::state::init_for_test;
+    use super::*;
+    use sqlx::Row as _;
+
+    async fn make_pool() -> sqlx::SqlitePool {
+        let options = sqlx::sqlite::SqliteConnectOptions::new()
+            .filename(":memory:")
+            .create_if_missing(true);
+        let pool = sqlx::pool::PoolOptions::new()
+            .max_connections(1)
+            .connect_with(options)
+            .await
+            .unwrap();
+        sqlx::migrate!("./migrations").run(&pool).await.unwrap();
+        pool
+    }
+
+    async fn insert_row(pool: &sqlx::SqlitePool, story_id: &str, stage: &str) {
+        let now = chrono::Utc::now().to_rfc3339();
+        sqlx::query(
+            "INSERT INTO pipeline_items \
+             (id, name, stage, agent, retry_count, depends_on, content, created_at, updated_at) \
+             VALUES (?1, ?2, ?3, NULL, 0, NULL, NULL, ?4, ?4)",
+        )
+        .bind(story_id)
+        .bind(story_id)
+        .bind(stage)
+        .bind(&now)
+        .execute(pool)
+        .await
+        .unwrap();
+    }
+
+    async fn row_stage(pool: &sqlx::SqlitePool, story_id: &str) -> Option<String> {
+        sqlx::query("SELECT stage FROM pipeline_items WHERE id = ?1")
+            .bind(story_id)
+            .fetch_optional(pool)
+            .await
+            .unwrap()
+            .map(|r| r.get(0))
+    }
+
+    /// Bug 1094 regression: delete a story in `coding` stage, assert the
+    /// `pipeline_items` row is gone; then re-run the sweep and confirm no
+    /// further changes (idempotent).
+    #[tokio::test]
+    async fn sweep_removes_zombie_coding_row_and_is_idempotent() {
+        init_for_test();
+        let pool = make_pool().await;
+        let story_id = "1094_zombie_regression";
+
+        // Seed: insert a pipeline_items row in the "coding" stage.
+        insert_row(&pool, story_id, "coding").await;
+        assert_eq!(row_stage(&pool, story_id).await.as_deref(), Some("coding"));
+
+        // Tombstone the story in the CRDT (simulate evict_item outcome).
+        crate::crdt_state::write_item_str(
+            story_id,
+            "coding",
+            Some("Zombie regression story"),
+            None,
+            None,
+            None,
+        );
+        crate::crdt_state::evict_item(story_id).ok();
+
+        // Run the sweep — row must be deleted.
+        let deleted = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
+        assert_eq!(deleted, 1, "expected one zombie row to be cleaned");
+        assert!(
+            row_stage(&pool, story_id).await.is_none(),
+            "pipeline_items row must be gone after sweep"
+        );
+
+        // Re-run is a no-op (idempotent).
+        let second = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
+        assert_eq!(second, 0, "second sweep must be a no-op");
+    }
+
+    /// Rows already in a terminal stage must be left alone.
+    #[tokio::test]
+    async fn sweep_skips_terminal_stage_rows() {
+        let pool = make_pool().await;
+        let story_id = "1094_terminal_skip";
+        insert_row(&pool, story_id, "done").await;
+
+        let deleted = sweep_zombie_rows(&pool, &[story_id.to_string()]).await;
+        assert_eq!(deleted, 0, "terminal-stage row must not be deleted");
+        assert!(
+            row_stage(&pool, story_id).await.is_some(),
+            "terminal-stage row must survive sweep"
+        );
+    }
+}

server/src/crdt_state/write/mod.rs

@@ -18,6 +18,6 @@ pub use item::{
 pub use item::write_item_str;
 pub use migrations::{
     migrate_legacy_stage_strings, migrate_merge_job, migrate_names_from_slugs,
-    migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric, name_from_story_id,
-    purge_done_stage_merge_jobs,
+    migrate_node_claims_to_agent_claims, migrate_story_ids_to_numeric,
+    migrate_zombie_pipeline_rows, name_from_story_id, purge_done_stage_merge_jobs,
 };