crashlabs/huskies
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 26 Wiki Activity

huskies: merge 670_refactor_hoist_chat_history_persistence_into_a_shared_module_replaces_658

Browse Source
This commit is contained in:
dave
2026-04-27 20:05:12 +00:00
parent 615e1c7f73
commit 63d5a500de
6 changed files with 208 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/chat/history.rs
+190
View File
@@ -0,0 +1,190 @@
//! Shared chat history persistence used by all transport modules.
//!
//! Each transport (Slack, WhatsApp, Discord, Matrix) stores its conversation
//! history in a transport-specific JSON file but uses the same on-disk format
//! and the same load/save logic provided here.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Mutex as TokioMutex;
use crate::chat::transport::matrix::RoomConversation;
use crate::slog;
/// Shared conversation history type used by all string-keyed chat transports.
#[allow(dead_code)]
pub type ChatConversationHistory = Arc<TokioMutex<HashMap<String, RoomConversation>>>;
/// On-disk format for persisted conversation history.
#[derive(Serialize, Deserialize)]
struct PersistedHistory {
conversations: HashMap<String, RoomConversation>,
}
/// Load conversation history from `project_root/relative_path`.
///
/// Returns an empty map on any I/O or parse error.
///
/// # Migration fallback
///
/// Older transport-specific files stored the conversation map under a key that
/// varied by transport (`channels` for Slack/Discord, `senders` for WhatsApp,
/// `rooms` for Matrix). If deserialising the file as a [`PersistedHistory`]
/// (which expects the key `conversations`) fails, this function falls back to
/// parsing the file as a generic [`serde_json::Value`], locates the first
/// top-level object field whose value is itself an object (the conversation
/// map), and returns its contents deserialised as
/// `HashMap<String, RoomConversation>`. New writes always use the
/// `conversations` key.
pub fn load_chat_history(
project_root: &std::path::Path,
relative_path: &str,
log_tag: &str,
) -> HashMap<String, RoomConversation> {
let path = project_root.join(relative_path);
let data = match std::fs::read_to_string(&path) {
Ok(d) => d,
Err(_) => return HashMap::new(),
};
// Try the new canonical format first.
if let Ok(p) = serde_json::from_str::<PersistedHistory>(&data) {
return p.conversations;
}
// Migration fallback: find the first map-valued field, whatever its name.
match serde_json::from_str::<serde_json::Value>(&data) {
Ok(value) => {
if let Some(obj) = value.as_object() {
for (_key, val) in obj {
if let Ok(map) =
serde_json::from_value::<HashMap<String, RoomConversation>>(val.clone())
{
return map;
}
}
}
slog!(
"[{}] Failed to parse history file: no valid conversation map found",
log_tag
);
HashMap::new()
}
Err(e) => {
slog!("[{}] Failed to parse history file: {}", log_tag, e);
HashMap::new()
}
}
}
/// Save conversation history to `project_root/relative_path` under the
/// `conversations` key. Errors are logged but not propagated.
pub fn save_chat_history(
project_root: &std::path::Path,
relative_path: &str,
log_tag: &str,
history: &HashMap<String, RoomConversation>,
) {
let persisted = PersistedHistory {
conversations: history.clone(),
};
let path = project_root.join(relative_path);
match serde_json::to_string_pretty(&persisted) {
Ok(json) => {
if let Err(e) = std::fs::write(&path, json) {
slog!("[{}] Failed to write history file: {}", log_tag, e);
}
}
Err(e) => slog!("[{}] Failed to serialise history: {}", log_tag, e),
}
}
// ── Tests ───────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
use crate::chat::transport::matrix::{ConversationEntry, ConversationRole};
#[test]
fn round_trip_save_load_preserves_history() {
let tmp = tempfile::tempdir().unwrap();
let huskies_dir = tmp.path().join(".huskies");
std::fs::create_dir_all(&huskies_dir).unwrap();
let mut history = HashMap::new();
history.insert(
"room-1".to_string(),
RoomConversation {
session_id: Some("sess-xyz".to_string()),
entries: vec![
ConversationEntry {
role: ConversationRole::User,
sender: "alice".to_string(),
content: "hello".to_string(),
},
ConversationEntry {
role: ConversationRole::Assistant,
sender: String::new(),
content: "hi there!".to_string(),
},
],
},
);
save_chat_history(tmp.path(), ".huskies/test_history.json", "test", &history);
let loaded = load_chat_history(tmp.path(), ".huskies/test_history.json", "test");
assert_eq!(loaded.len(), 1);
let conv = loaded.get("room-1").unwrap();
assert_eq!(conv.session_id.as_deref(), Some("sess-xyz"));
assert_eq!(conv.entries.len(), 2);
assert_eq!(conv.entries[0].role, ConversationRole::User);
assert_eq!(conv.entries[0].content, "hello");
assert_eq!(conv.entries[1].role, ConversationRole::Assistant);
assert_eq!(conv.entries[1].content, "hi there!");
}
#[test]
fn legacy_channels_key_loads_via_migration_fallback() {
let tmp = tempfile::tempdir().unwrap();
let huskies_dir = tmp.path().join(".huskies");
std::fs::create_dir_all(&huskies_dir).unwrap();
// Simulate a legacy Slack history file with the `channels` top-level key.
let legacy_json = r#"{
"channels": {
"C01ABCDEF": {
"entries": [
{"role": "user", "sender": "U123", "content": "legacy message"}
]
}
}
}"#;
std::fs::write(huskies_dir.join("slack_history.json"), legacy_json).unwrap();
let loaded = load_chat_history(tmp.path(), ".huskies/slack_history.json", "slack");
assert_eq!(loaded.len(), 1);
let conv = loaded.get("C01ABCDEF").unwrap();
assert_eq!(conv.entries.len(), 1);
assert_eq!(conv.entries[0].content, "legacy message");
assert_eq!(conv.entries[0].role, ConversationRole::User);
}
#[test]
fn load_returns_empty_when_file_missing() {
let tmp = tempfile::tempdir().unwrap();
let loaded = load_chat_history(tmp.path(), ".huskies/missing.json", "test");
assert!(loaded.is_empty());
}
#[test]
fn load_returns_empty_on_invalid_json() {
let tmp = tempfile::tempdir().unwrap();
let huskies_dir = tmp.path().join(".huskies");
std::fs::create_dir_all(&huskies_dir).unwrap();
std::fs::write(huskies_dir.join("bad.json"), "not json {{{").unwrap();
let loaded = load_chat_history(tmp.path(), ".huskies/bad.json", "test");
assert!(loaded.is_empty());
}
}
server/src/chat/mod.rs
+1
View File
@@ -5,6 +5,7 @@
//! notifications) to work against any chat platform — Matrix, WhatsApp, etc.
pub mod commands;
pub mod history;
pub(crate) mod lookup;
#[cfg(test)]
pub(crate) mod test_helpers;
server/src/chat/transport/discord/history.rs
+3 -33
View File
@@ -1,40 +1,21 @@
//! Discord conversation history persistence.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Mutex as TokioMutex;
use crate::chat::history::{load_chat_history, save_chat_history};
use crate::chat::transport::matrix::RoomConversation;
use crate::slog;
/// Per-channel conversation history, keyed by channel ID.
pub type DiscordConversationHistory = Arc<TokioMutex<HashMap<String, RoomConversation>>>;
/// On-disk format for persisted Discord conversation history.
#[derive(Serialize, Deserialize)]
struct PersistedDiscordHistory {
channels: HashMap<String, RoomConversation>,
}
/// Path to the persisted Discord conversation history file.
const DISCORD_HISTORY_FILE: &str = ".huskies/discord_history.json";
/// Load Discord conversation history from disk.
pub fn load_discord_history(project_root: &std::path::Path) -> HashMap<String, RoomConversation> {
let path = project_root.join(DISCORD_HISTORY_FILE);
let data = match std::fs::read_to_string(&path) {
Ok(d) => d,
Err(_) => return HashMap::new(),
};
let persisted: PersistedDiscordHistory = match serde_json::from_str(&data) {
Ok(p) => p,
Err(e) => {
slog!("[discord] Failed to parse history file: {e}");
return HashMap::new();
}
};
persisted.channels
load_chat_history(project_root, DISCORD_HISTORY_FILE, "discord")
}
/// Save Discord conversation history to disk.
@@ -42,18 +23,7 @@ pub(super) fn save_discord_history(
project_root: &std::path::Path,
history: &HashMap<String, RoomConversation>,
) {
let persisted = PersistedDiscordHistory {
channels: history.clone(),
};
let path = project_root.join(DISCORD_HISTORY_FILE);
match serde_json::to_string_pretty(&persisted) {
Ok(json) => {
if let Err(e) = std::fs::write(&path, json) {
slog!("[discord] Failed to write history file: {e}");
}
}
Err(e) => slog!("[discord] Failed to serialise history: {e}"),
}
save_chat_history(project_root, DISCORD_HISTORY_FILE, "discord", history);
}
// ── Tests ───────────────────────────────────────────────────────────────
server/src/chat/transport/matrix/bot/history.rs
+6 -35
View File
@@ -1,5 +1,5 @@
//! Matrix conversation history — per-room message history for LLM context.
use crate::slog;
use crate::chat::history::{load_chat_history, save_chat_history};
use matrix_sdk::ruma::OwnedRoomId;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -44,32 +44,13 @@ pub struct RoomConversation {
/// event-handler tasks without blocking the sync loop.
pub type ConversationHistory = Arc<TokioMutex<HashMap<OwnedRoomId, RoomConversation>>>;
/// On-disk format for persisted conversation history. Room IDs are stored as
/// strings because `OwnedRoomId` does not implement `Serialize` as a map key.
#[derive(Serialize, Deserialize)]
pub(super) struct PersistedHistory {
pub rooms: HashMap<String, RoomConversation>,
}
/// Path to the persisted conversation history file relative to project root.
pub(super) const HISTORY_FILE: &str = ".huskies/matrix_history.json";
/// Load conversation history from disk, returning an empty map on any error.
pub fn load_history(project_root: &std::path::Path) -> HashMap<OwnedRoomId, RoomConversation> {
let path = project_root.join(HISTORY_FILE);
let data = match std::fs::read_to_string(&path) {
Ok(d) => d,
Err(_) => return HashMap::new(),
};
let persisted: PersistedHistory = match serde_json::from_str(&data) {
Ok(p) => p,
Err(e) => {
slog!("[matrix-bot] Failed to parse history file: {e}");
return HashMap::new();
}
};
persisted
.rooms
let string_map = load_chat_history(project_root, HISTORY_FILE, "matrix-bot");
string_map
.into_iter()
.filter_map(|(k, v)| k.parse::<OwnedRoomId>().ok().map(|room_id| (room_id, v)))
.collect()
@@ -80,21 +61,11 @@ pub fn save_history(
project_root: &std::path::Path,
history: &HashMap<OwnedRoomId, RoomConversation>,
) {
let persisted = PersistedHistory {
rooms: history
let string_map: HashMap<String, RoomConversation> = history
.iter()
.map(|(k, v)| (k.to_string(), v.clone()))
.collect(),
};
let path = project_root.join(HISTORY_FILE);
match serde_json::to_string_pretty(&persisted) {
Ok(json) => {
if let Err(e) = std::fs::write(&path, json) {
slog!("[matrix-bot] Failed to write history file: {e}");
}
}
Err(e) => slog!("[matrix-bot] Failed to serialise history: {e}"),
}
.collect();
save_chat_history(project_root, HISTORY_FILE, "matrix-bot", &string_map);
}
// ---------------------------------------------------------------------------
server/src/chat/transport/slack/history.rs
+3 -33
View File
@@ -1,40 +1,21 @@
//! Slack conversation history persistence.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Mutex as TokioMutex;
use crate::chat::history::{load_chat_history, save_chat_history};
use crate::chat::transport::matrix::RoomConversation;
use crate::slog;
/// Per-channel conversation history, keyed by channel ID.
pub type SlackConversationHistory = Arc<TokioMutex<HashMap<String, RoomConversation>>>;
/// On-disk format for persisted Slack conversation history.
#[derive(Serialize, Deserialize)]
struct PersistedSlackHistory {
channels: HashMap<String, RoomConversation>,
}
/// Path to the persisted Slack conversation history file.
const SLACK_HISTORY_FILE: &str = ".huskies/slack_history.json";
/// Load Slack conversation history from disk.
pub fn load_slack_history(project_root: &std::path::Path) -> HashMap<String, RoomConversation> {
let path = project_root.join(SLACK_HISTORY_FILE);
let data = match std::fs::read_to_string(&path) {
Ok(d) => d,
Err(_) => return HashMap::new(),
};
let persisted: PersistedSlackHistory = match serde_json::from_str(&data) {
Ok(p) => p,
Err(e) => {
slog!("[slack] Failed to parse history file: {e}");
return HashMap::new();
}
};
persisted.channels
load_chat_history(project_root, SLACK_HISTORY_FILE, "slack")
}
/// Save Slack conversation history to disk.
@@ -42,18 +23,7 @@ pub(super) fn save_slack_history(
project_root: &std::path::Path,
history: &HashMap<String, RoomConversation>,
) {
let persisted = PersistedSlackHistory {
channels: history.clone(),
};
let path = project_root.join(SLACK_HISTORY_FILE);
match serde_json::to_string_pretty(&persisted) {
Ok(json) => {
if let Err(e) = std::fs::write(&path, json) {
slog!("[slack] Failed to write history file: {e}");
}
}
Err(e) => slog!("[slack] Failed to serialise history: {e}"),
}
save_chat_history(project_root, SLACK_HISTORY_FILE, "slack", history);
}
// ── Tests ───────────────────────────────────────────────────────────────
server/src/chat/transport/whatsapp/history.rs
+3 -33
View File
@@ -1,11 +1,10 @@
//! WhatsApp conversation history — per-number message history and messaging window tracking.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Mutex as TokioMutex;
use crate::chat::history::{load_chat_history, save_chat_history};
use crate::chat::transport::matrix::RoomConversation;
use crate::slog;
// ── Messaging window tracker ─────────────────────────────────────────────
@@ -73,30 +72,12 @@ impl MessagingWindowTracker {
/// Per-sender conversation history, keyed by phone number.
pub type WhatsAppConversationHistory = Arc<TokioMutex<HashMap<String, RoomConversation>>>;
/// On-disk format for persisted WhatsApp conversation history.
#[derive(Serialize, Deserialize)]
struct PersistedWhatsAppHistory {
senders: HashMap<String, RoomConversation>,
}
/// Path to the persisted WhatsApp conversation history file.
const WHATSAPP_HISTORY_FILE: &str = ".huskies/whatsapp_history.json";
/// Load WhatsApp conversation history from disk.
pub fn load_whatsapp_history(project_root: &std::path::Path) -> HashMap<String, RoomConversation> {
let path = project_root.join(WHATSAPP_HISTORY_FILE);
let data = match std::fs::read_to_string(&path) {
Ok(d) => d,
Err(_) => return HashMap::new(),
};
let persisted: PersistedWhatsAppHistory = match serde_json::from_str(&data) {
Ok(p) => p,
Err(e) => {
slog!("[whatsapp] Failed to parse history file: {e}");
return HashMap::new();
}
};
persisted.senders
load_chat_history(project_root, WHATSAPP_HISTORY_FILE, "whatsapp")
}
/// Save WhatsApp conversation history to disk.
@@ -104,18 +85,7 @@ pub(super) fn save_whatsapp_history(
project_root: &std::path::Path,
history: &HashMap<String, RoomConversation>,
) {
let persisted = PersistedWhatsAppHistory {
senders: history.clone(),
};
let path = project_root.join(WHATSAPP_HISTORY_FILE);
match serde_json::to_string_pretty(&persisted) {
Ok(json) => {
if let Err(e) = std::fs::write(&path, json) {
slog!("[whatsapp] Failed to write history file: {e}");
}
}
Err(e) => slog!("[whatsapp] Failed to serialise history: {e}"),
}
save_chat_history(project_root, WHATSAPP_HISTORY_FILE, "whatsapp", history);
}
// ── Tests ───────────────────────────────────────────────────────────────