From 63d5a500de0abd35679991bd207bd6e5a90b840b Mon Sep 17 00:00:00 2001 From: dave Date: Mon, 27 Apr 2026 20:05:12 +0000 Subject: [PATCH] huskies: merge 670_refactor_hoist_chat_history_persistence_into_a_shared_module_replaces_658 --- server/src/chat/history.rs | 190 ++++++++++++++++++ server/src/chat/mod.rs | 1 + server/src/chat/transport/discord/history.rs | 36 +--- .../src/chat/transport/matrix/bot/history.rs | 45 +---- server/src/chat/transport/slack/history.rs | 36 +--- server/src/chat/transport/whatsapp/history.rs | 36 +--- 6 files changed, 208 insertions(+), 136 deletions(-) create mode 100644 server/src/chat/history.rs diff --git a/server/src/chat/history.rs b/server/src/chat/history.rs new file mode 100644 index 00000000..df9fedba --- /dev/null +++ b/server/src/chat/history.rs @@ -0,0 +1,190 @@ +//! Shared chat history persistence used by all transport modules. +//! +//! Each transport (Slack, WhatsApp, Discord, Matrix) stores its conversation +//! history in a transport-specific JSON file but uses the same on-disk format +//! and the same load/save logic provided here. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::Mutex as TokioMutex; + +use crate::chat::transport::matrix::RoomConversation; +use crate::slog; + +/// Shared conversation history type used by all string-keyed chat transports. +#[allow(dead_code)] +pub type ChatConversationHistory = Arc>>; + +/// On-disk format for persisted conversation history. +#[derive(Serialize, Deserialize)] +struct PersistedHistory { + conversations: HashMap, +} + +/// Load conversation history from `project_root/relative_path`. +/// +/// Returns an empty map on any I/O or parse error. +/// +/// # Migration fallback +/// +/// Older transport-specific files stored the conversation map under a key that +/// varied by transport (`channels` for Slack/Discord, `senders` for WhatsApp, +/// `rooms` for Matrix). If deserialising the file as a [`PersistedHistory`] +/// (which expects the key `conversations`) fails, this function falls back to +/// parsing the file as a generic [`serde_json::Value`], locates the first +/// top-level object field whose value is itself an object (the conversation +/// map), and returns its contents deserialised as +/// `HashMap`. New writes always use the +/// `conversations` key. +pub fn load_chat_history( + project_root: &std::path::Path, + relative_path: &str, + log_tag: &str, +) -> HashMap { + let path = project_root.join(relative_path); + let data = match std::fs::read_to_string(&path) { + Ok(d) => d, + Err(_) => return HashMap::new(), + }; + // Try the new canonical format first. + if let Ok(p) = serde_json::from_str::(&data) { + return p.conversations; + } + // Migration fallback: find the first map-valued field, whatever its name. + match serde_json::from_str::(&data) { + Ok(value) => { + if let Some(obj) = value.as_object() { + for (_key, val) in obj { + if let Ok(map) = + serde_json::from_value::>(val.clone()) + { + return map; + } + } + } + slog!( + "[{}] Failed to parse history file: no valid conversation map found", + log_tag + ); + HashMap::new() + } + Err(e) => { + slog!("[{}] Failed to parse history file: {}", log_tag, e); + HashMap::new() + } + } +} + +/// Save conversation history to `project_root/relative_path` under the +/// `conversations` key. Errors are logged but not propagated. +pub fn save_chat_history( + project_root: &std::path::Path, + relative_path: &str, + log_tag: &str, + history: &HashMap, +) { + let persisted = PersistedHistory { + conversations: history.clone(), + }; + let path = project_root.join(relative_path); + match serde_json::to_string_pretty(&persisted) { + Ok(json) => { + if let Err(e) = std::fs::write(&path, json) { + slog!("[{}] Failed to write history file: {}", log_tag, e); + } + } + Err(e) => slog!("[{}] Failed to serialise history: {}", log_tag, e), + } +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::chat::transport::matrix::{ConversationEntry, ConversationRole}; + + #[test] + fn round_trip_save_load_preserves_history() { + let tmp = tempfile::tempdir().unwrap(); + let huskies_dir = tmp.path().join(".huskies"); + std::fs::create_dir_all(&huskies_dir).unwrap(); + + let mut history = HashMap::new(); + history.insert( + "room-1".to_string(), + RoomConversation { + session_id: Some("sess-xyz".to_string()), + entries: vec![ + ConversationEntry { + role: ConversationRole::User, + sender: "alice".to_string(), + content: "hello".to_string(), + }, + ConversationEntry { + role: ConversationRole::Assistant, + sender: String::new(), + content: "hi there!".to_string(), + }, + ], + }, + ); + + save_chat_history(tmp.path(), ".huskies/test_history.json", "test", &history); + let loaded = load_chat_history(tmp.path(), ".huskies/test_history.json", "test"); + + assert_eq!(loaded.len(), 1); + let conv = loaded.get("room-1").unwrap(); + assert_eq!(conv.session_id.as_deref(), Some("sess-xyz")); + assert_eq!(conv.entries.len(), 2); + assert_eq!(conv.entries[0].role, ConversationRole::User); + assert_eq!(conv.entries[0].content, "hello"); + assert_eq!(conv.entries[1].role, ConversationRole::Assistant); + assert_eq!(conv.entries[1].content, "hi there!"); + } + + #[test] + fn legacy_channels_key_loads_via_migration_fallback() { + let tmp = tempfile::tempdir().unwrap(); + let huskies_dir = tmp.path().join(".huskies"); + std::fs::create_dir_all(&huskies_dir).unwrap(); + + // Simulate a legacy Slack history file with the `channels` top-level key. + let legacy_json = r#"{ + "channels": { + "C01ABCDEF": { + "entries": [ + {"role": "user", "sender": "U123", "content": "legacy message"} + ] + } + } + }"#; + std::fs::write(huskies_dir.join("slack_history.json"), legacy_json).unwrap(); + + let loaded = load_chat_history(tmp.path(), ".huskies/slack_history.json", "slack"); + + assert_eq!(loaded.len(), 1); + let conv = loaded.get("C01ABCDEF").unwrap(); + assert_eq!(conv.entries.len(), 1); + assert_eq!(conv.entries[0].content, "legacy message"); + assert_eq!(conv.entries[0].role, ConversationRole::User); + } + + #[test] + fn load_returns_empty_when_file_missing() { + let tmp = tempfile::tempdir().unwrap(); + let loaded = load_chat_history(tmp.path(), ".huskies/missing.json", "test"); + assert!(loaded.is_empty()); + } + + #[test] + fn load_returns_empty_on_invalid_json() { + let tmp = tempfile::tempdir().unwrap(); + let huskies_dir = tmp.path().join(".huskies"); + std::fs::create_dir_all(&huskies_dir).unwrap(); + std::fs::write(huskies_dir.join("bad.json"), "not json {{{").unwrap(); + let loaded = load_chat_history(tmp.path(), ".huskies/bad.json", "test"); + assert!(loaded.is_empty()); + } +} diff --git a/server/src/chat/mod.rs b/server/src/chat/mod.rs index 6ffa63c1..2a7c129b 100644 --- a/server/src/chat/mod.rs +++ b/server/src/chat/mod.rs @@ -5,6 +5,7 @@ //! notifications) to work against any chat platform — Matrix, WhatsApp, etc. pub mod commands; +pub mod history; pub(crate) mod lookup; #[cfg(test)] pub(crate) mod test_helpers; diff --git a/server/src/chat/transport/discord/history.rs b/server/src/chat/transport/discord/history.rs index 9b8828aa..3a14a59b 100644 --- a/server/src/chat/transport/discord/history.rs +++ b/server/src/chat/transport/discord/history.rs @@ -1,40 +1,21 @@ //! Discord conversation history persistence. -use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex as TokioMutex; +use crate::chat::history::{load_chat_history, save_chat_history}; use crate::chat::transport::matrix::RoomConversation; -use crate::slog; /// Per-channel conversation history, keyed by channel ID. pub type DiscordConversationHistory = Arc>>; -/// On-disk format for persisted Discord conversation history. -#[derive(Serialize, Deserialize)] -struct PersistedDiscordHistory { - channels: HashMap, -} - /// Path to the persisted Discord conversation history file. const DISCORD_HISTORY_FILE: &str = ".huskies/discord_history.json"; /// Load Discord conversation history from disk. pub fn load_discord_history(project_root: &std::path::Path) -> HashMap { - let path = project_root.join(DISCORD_HISTORY_FILE); - let data = match std::fs::read_to_string(&path) { - Ok(d) => d, - Err(_) => return HashMap::new(), - }; - let persisted: PersistedDiscordHistory = match serde_json::from_str(&data) { - Ok(p) => p, - Err(e) => { - slog!("[discord] Failed to parse history file: {e}"); - return HashMap::new(); - } - }; - persisted.channels + load_chat_history(project_root, DISCORD_HISTORY_FILE, "discord") } /// Save Discord conversation history to disk. @@ -42,18 +23,7 @@ pub(super) fn save_discord_history( project_root: &std::path::Path, history: &HashMap, ) { - let persisted = PersistedDiscordHistory { - channels: history.clone(), - }; - let path = project_root.join(DISCORD_HISTORY_FILE); - match serde_json::to_string_pretty(&persisted) { - Ok(json) => { - if let Err(e) = std::fs::write(&path, json) { - slog!("[discord] Failed to write history file: {e}"); - } - } - Err(e) => slog!("[discord] Failed to serialise history: {e}"), - } + save_chat_history(project_root, DISCORD_HISTORY_FILE, "discord", history); } // ── Tests ─────────────────────────────────────────────────────────────── diff --git a/server/src/chat/transport/matrix/bot/history.rs b/server/src/chat/transport/matrix/bot/history.rs index 034ef50d..5c381e60 100644 --- a/server/src/chat/transport/matrix/bot/history.rs +++ b/server/src/chat/transport/matrix/bot/history.rs @@ -1,5 +1,5 @@ //! Matrix conversation history — per-room message history for LLM context. -use crate::slog; +use crate::chat::history::{load_chat_history, save_chat_history}; use matrix_sdk::ruma::OwnedRoomId; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -44,32 +44,13 @@ pub struct RoomConversation { /// event-handler tasks without blocking the sync loop. pub type ConversationHistory = Arc>>; -/// On-disk format for persisted conversation history. Room IDs are stored as -/// strings because `OwnedRoomId` does not implement `Serialize` as a map key. -#[derive(Serialize, Deserialize)] -pub(super) struct PersistedHistory { - pub rooms: HashMap, -} - /// Path to the persisted conversation history file relative to project root. pub(super) const HISTORY_FILE: &str = ".huskies/matrix_history.json"; /// Load conversation history from disk, returning an empty map on any error. pub fn load_history(project_root: &std::path::Path) -> HashMap { - let path = project_root.join(HISTORY_FILE); - let data = match std::fs::read_to_string(&path) { - Ok(d) => d, - Err(_) => return HashMap::new(), - }; - let persisted: PersistedHistory = match serde_json::from_str(&data) { - Ok(p) => p, - Err(e) => { - slog!("[matrix-bot] Failed to parse history file: {e}"); - return HashMap::new(); - } - }; - persisted - .rooms + let string_map = load_chat_history(project_root, HISTORY_FILE, "matrix-bot"); + string_map .into_iter() .filter_map(|(k, v)| k.parse::().ok().map(|room_id| (room_id, v))) .collect() @@ -80,21 +61,11 @@ pub fn save_history( project_root: &std::path::Path, history: &HashMap, ) { - let persisted = PersistedHistory { - rooms: history - .iter() - .map(|(k, v)| (k.to_string(), v.clone())) - .collect(), - }; - let path = project_root.join(HISTORY_FILE); - match serde_json::to_string_pretty(&persisted) { - Ok(json) => { - if let Err(e) = std::fs::write(&path, json) { - slog!("[matrix-bot] Failed to write history file: {e}"); - } - } - Err(e) => slog!("[matrix-bot] Failed to serialise history: {e}"), - } + let string_map: HashMap = history + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(); + save_chat_history(project_root, HISTORY_FILE, "matrix-bot", &string_map); } // --------------------------------------------------------------------------- diff --git a/server/src/chat/transport/slack/history.rs b/server/src/chat/transport/slack/history.rs index c4a1f3df..b3f0d145 100644 --- a/server/src/chat/transport/slack/history.rs +++ b/server/src/chat/transport/slack/history.rs @@ -1,40 +1,21 @@ //! Slack conversation history persistence. -use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex as TokioMutex; +use crate::chat::history::{load_chat_history, save_chat_history}; use crate::chat::transport::matrix::RoomConversation; -use crate::slog; /// Per-channel conversation history, keyed by channel ID. pub type SlackConversationHistory = Arc>>; -/// On-disk format for persisted Slack conversation history. -#[derive(Serialize, Deserialize)] -struct PersistedSlackHistory { - channels: HashMap, -} - /// Path to the persisted Slack conversation history file. const SLACK_HISTORY_FILE: &str = ".huskies/slack_history.json"; /// Load Slack conversation history from disk. pub fn load_slack_history(project_root: &std::path::Path) -> HashMap { - let path = project_root.join(SLACK_HISTORY_FILE); - let data = match std::fs::read_to_string(&path) { - Ok(d) => d, - Err(_) => return HashMap::new(), - }; - let persisted: PersistedSlackHistory = match serde_json::from_str(&data) { - Ok(p) => p, - Err(e) => { - slog!("[slack] Failed to parse history file: {e}"); - return HashMap::new(); - } - }; - persisted.channels + load_chat_history(project_root, SLACK_HISTORY_FILE, "slack") } /// Save Slack conversation history to disk. @@ -42,18 +23,7 @@ pub(super) fn save_slack_history( project_root: &std::path::Path, history: &HashMap, ) { - let persisted = PersistedSlackHistory { - channels: history.clone(), - }; - let path = project_root.join(SLACK_HISTORY_FILE); - match serde_json::to_string_pretty(&persisted) { - Ok(json) => { - if let Err(e) = std::fs::write(&path, json) { - slog!("[slack] Failed to write history file: {e}"); - } - } - Err(e) => slog!("[slack] Failed to serialise history: {e}"), - } + save_chat_history(project_root, SLACK_HISTORY_FILE, "slack", history); } // ── Tests ─────────────────────────────────────────────────────────────── diff --git a/server/src/chat/transport/whatsapp/history.rs b/server/src/chat/transport/whatsapp/history.rs index 1b07a313..5cb7ed83 100644 --- a/server/src/chat/transport/whatsapp/history.rs +++ b/server/src/chat/transport/whatsapp/history.rs @@ -1,11 +1,10 @@ //! WhatsApp conversation history — per-number message history and messaging window tracking. -use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Mutex as TokioMutex; +use crate::chat::history::{load_chat_history, save_chat_history}; use crate::chat::transport::matrix::RoomConversation; -use crate::slog; // ── Messaging window tracker ───────────────────────────────────────────── @@ -73,30 +72,12 @@ impl MessagingWindowTracker { /// Per-sender conversation history, keyed by phone number. pub type WhatsAppConversationHistory = Arc>>; -/// On-disk format for persisted WhatsApp conversation history. -#[derive(Serialize, Deserialize)] -struct PersistedWhatsAppHistory { - senders: HashMap, -} - /// Path to the persisted WhatsApp conversation history file. const WHATSAPP_HISTORY_FILE: &str = ".huskies/whatsapp_history.json"; /// Load WhatsApp conversation history from disk. pub fn load_whatsapp_history(project_root: &std::path::Path) -> HashMap { - let path = project_root.join(WHATSAPP_HISTORY_FILE); - let data = match std::fs::read_to_string(&path) { - Ok(d) => d, - Err(_) => return HashMap::new(), - }; - let persisted: PersistedWhatsAppHistory = match serde_json::from_str(&data) { - Ok(p) => p, - Err(e) => { - slog!("[whatsapp] Failed to parse history file: {e}"); - return HashMap::new(); - } - }; - persisted.senders + load_chat_history(project_root, WHATSAPP_HISTORY_FILE, "whatsapp") } /// Save WhatsApp conversation history to disk. @@ -104,18 +85,7 @@ pub(super) fn save_whatsapp_history( project_root: &std::path::Path, history: &HashMap, ) { - let persisted = PersistedWhatsAppHistory { - senders: history.clone(), - }; - let path = project_root.join(WHATSAPP_HISTORY_FILE); - match serde_json::to_string_pretty(&persisted) { - Ok(json) => { - if let Err(e) = std::fs::write(&path, json) { - slog!("[whatsapp] Failed to write history file: {e}"); - } - } - Err(e) => slog!("[whatsapp] Failed to serialise history: {e}"), - } + save_chat_history(project_root, WHATSAPP_HISTORY_FILE, "whatsapp", history); } // ── Tests ───────────────────────────────────────────────────────────────