Story 18: Token-by-token streaming responses

- Backend: Added OllamaProvider::chat_stream() with newline-delimited JSON parsing
- Backend: Emit chat:token events for each token received from Ollama
- Backend: Added futures dependency and stream feature for reqwest
- Frontend: Added streamingContent state and chat:token event listener
- Frontend: Real-time token display with auto-scroll
- Frontend: Markdown and syntax highlighting support for streaming content
- Fixed all TypeScript errors (tsc --noEmit)
- Fixed all Biome warnings and errors
- Fixed all Clippy warnings
- Added comprehensive code quality documentation
- Added tsc --noEmit to verification checklist

Tested and verified:
- Tokens stream in real-time
- Auto-scroll works during streaming
- Tool calls interrupt streaming correctly
- Multi-turn conversations work
- Smooth performance with no lag
This commit is contained in:
Dave
2025-12-27 16:50:18 +00:00
parent bb700ce870
commit 64d1b788be
19 changed files with 1441 additions and 684 deletions

View File

@@ -1,14 +1,11 @@
use crate::commands::{fs, search, shell};
use crate::llm::ollama::OllamaProvider;
use crate::llm::prompts::SYSTEM_PROMPT;
use crate::llm::types::{
Message, ModelProvider, Role, ToolCall, ToolDefinition, ToolFunctionDefinition,
};
use crate::llm::types::{Message, Role, ToolCall, ToolDefinition, ToolFunctionDefinition};
use crate::state::SessionState;
use serde::Deserialize;
use serde_json::json;
use tauri::{AppHandle, Emitter, State};
use tokio::select;
#[derive(Deserialize)]
pub struct ProviderConfig {
@@ -26,12 +23,6 @@ pub async fn get_ollama_models(base_url: Option<String>) -> Result<Vec<String>,
OllamaProvider::get_models(&url).await
}
#[tauri::command]
pub async fn cancel_chat(state: State<'_, SessionState>) -> Result<(), String> {
state.cancel_tx.send(true).map_err(|e| e.to_string())?;
Ok(())
}
#[tauri::command]
pub async fn chat(
app: AppHandle,
@@ -39,18 +30,17 @@ pub async fn chat(
config: ProviderConfig,
state: State<'_, SessionState>,
) -> Result<Vec<Message>, String> {
// Reset cancellation flag at start
let _ = state.cancel_tx.send(false);
let mut cancel_rx = state.cancel_rx.clone();
// 1. Setup Provider
let provider: Box<dyn ModelProvider> = match config.provider.as_str() {
"ollama" => Box::new(OllamaProvider::new(
config
.base_url
.unwrap_or_else(|| "http://localhost:11434".to_string()),
)),
_ => return Err(format!("Unsupported provider: {}", config.provider)),
};
let base_url = config
.base_url
.clone()
.unwrap_or_else(|| "http://localhost:11434".to_string());
if config.provider.as_str() != "ollama" {
return Err(format!("Unsupported provider: {}", config.provider));
}
let provider = OllamaProvider::new(base_url);
// 2. Define Tools
let tool_defs = get_tool_definitions();
@@ -94,23 +84,11 @@ pub async fn chat(
}
turn_count += 1;
// Call LLM with cancellation support
let chat_future = provider.chat(&config.model, &current_history, tools);
let response = select! {
result = chat_future => {
result.map_err(|e| format!("LLM Error: {}", e))?
}
_ = cancel_rx.changed() => {
if *cancel_rx.borrow() {
return Err("Chat cancelled by user".to_string());
}
// False alarm, continue
provider.chat(&config.model, &current_history, tools)
.await
.map_err(|e| format!("LLM Error: {}", e))?
}
};
// Call LLM with streaming
let response = provider
.chat_stream(&app, &config.model, &current_history, tools)
.await
.map_err(|e| format!("LLM Error: {}", e))?;
// Process Response
if let Some(tool_calls) = response.tool_calls {