From 23beb61e355101b9ae4d99a44cd98354782a9401 Mon Sep 17 00:00:00 2001 From: lollinng Date: Fri, 19 Jun 2026 15:13:10 +0530 Subject: [PATCH] feat: add local Ollama + Claude-headless backends for embeddings and LLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets patcha run without the patcha-cloud login. Opt-in, config-selectable backends; defaults (fastembed embeddings, patcha-cloud LLM) unchanged. Embeddings (EMBEDDING_PROVIDER): - "ollama": Embedder gains an enum backend; the Ollama variant calls /api/embed over a blocking ureq client (the embedder runs inside block_in_place/ spawn_blocking and is dropped inside async, so a runtime-free client avoids drop panics). Fails fast if Ollama is unreachable or the model isn't 768-dim. LLM (LLM_PROVIDER), routed inside PatchaApiClient::chat_completion (no caller churn): - "ollama": OpenAI-compatible /v1/chat/completions, no auth. - "claude": Claude Code headless — shells out to `claude -p --system-prompt … --model --output-format text` via tokio::process, using the local Claude login (no API key). --system-prompt overrides Claude Code's default agent prompt with patcha's. Config (env-driven): EMBEDDING_PROVIDER, LLM_PROVIDER, OLLAMA_URL, OLLAMA_EMBEDDING_MODEL, OLLAMA_LLM_MODEL, CLAUDE_BIN, CLAUDE_MODEL. Cargo: add ureq for the blocking embed call. Verified end-to-end with no cloud login: nomic-embed-text (768) embeds -> sqlite-vec store -> semantic search returns correct ranked results; summarize generates via both Ollama (llama3.2) and Claude headless (Opus). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Tq11wGi71nR57bXi4bwcCz --- rust/Cargo.lock | 1 + rust/patcha/Cargo.toml | 7 +- rust/patcha/src/config.rs | 28 +++++++ rust/patcha/src/embedding.rs | 143 ++++++++++++++++++++++++++++++---- rust/patcha/src/llm/client.rs | 86 ++++++++++++++++++++ 5 files changed, 251 insertions(+), 14 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 170aea6..d5a5ca6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2480,6 +2480,7 @@ dependencies = [ "tower-http 0.5.2", "tracing", "tracing-subscriber", + "ureq", "uuid", "walkdir", ] diff --git a/rust/patcha/Cargo.toml b/rust/patcha/Cargo.toml index 44e9f38..6dfc632 100644 --- a/rust/patcha/Cargo.toml +++ b/rust/patcha/Cargo.toml @@ -19,9 +19,14 @@ axum = { version = "0.7", features = ["macros"] } tower = "0.4" tower-http = { version = "0.5", features = ["cors"] } -# HTTP client +# HTTP client (async — cloud + Ollama chat) reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false } +# Synchronous HTTP client for the local Ollama embedding backend. The embedder +# runs inside block_in_place/spawn_blocking everywhere, so a blocking client with +# no async runtime of its own is the simplest, panic-free choice. +ureq = { version = "2", features = ["tls"], default-features = false } + # Serialization serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/rust/patcha/src/config.rs b/rust/patcha/src/config.rs index 6740135..da336f8 100644 --- a/rust/patcha/src/config.rs +++ b/rust/patcha/src/config.rs @@ -21,6 +21,20 @@ pub struct Config { pub max_events_per_day: u32, pub max_embedding_tokens: usize, pub embedding_chunk_overlap: usize, + + // Pluggable backends. Defaults preserve existing behaviour. + // embedding_provider: "fastembed" (default, on-device) | "ollama" (local) + // llm_provider: "patcha" (cloud, default) | "ollama" (local) | "claude" (Claude Code headless) + pub embedding_provider: String, + pub llm_provider: String, + pub ollama_url: String, + pub ollama_embedding_model: String, + pub ollama_llm_model: String, + // Claude Code headless backend for llm_provider="claude": shells out to the + // `claude -p` CLI, using the local Claude login (no API key). + pub claude_bin: String, + pub claude_model: String, + pub max_pending_per_cycle: usize, pub working_memory_dedup_threshold: f32, pub daily_compaction_min_activities: usize, @@ -72,6 +86,13 @@ impl Default for Config { max_events_per_day: 10_000, max_embedding_tokens: 8191, embedding_chunk_overlap: 100, + embedding_provider: "fastembed".into(), + llm_provider: "patcha".into(), + ollama_url: "http://localhost:11434".into(), + ollama_embedding_model: "nomic-embed-text".into(), + ollama_llm_model: "llama3.2".into(), + claude_bin: "claude".into(), + claude_model: "opus".into(), max_pending_per_cycle: 500, working_memory_dedup_threshold: 0.95, daily_compaction_min_activities: 2, @@ -182,6 +203,13 @@ impl Config { max_events_per_day: env_u64("MAX_EVENTS_PER_DAY", 10_000) as u32, max_embedding_tokens: env_usize("MAX_EMBEDDING_TOKENS", 8191), embedding_chunk_overlap: env_usize("EMBEDDING_CHUNK_OVERLAP", 100), + embedding_provider: env_str("EMBEDDING_PROVIDER", "fastembed"), + llm_provider: env_str("LLM_PROVIDER", "patcha"), + ollama_url: env_str("OLLAMA_URL", "http://localhost:11434"), + ollama_embedding_model: env_str("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"), + ollama_llm_model: env_str("OLLAMA_LLM_MODEL", "llama3.2"), + claude_bin: env_str("CLAUDE_BIN", "claude"), + claude_model: env_str("CLAUDE_MODEL", "opus"), max_pending_per_cycle: env_usize("MAX_PENDING_PER_CYCLE", 500), working_memory_dedup_threshold: env_f32("WORKING_MEMORY_DEDUP_THRESHOLD", 0.95), daily_compaction_min_activities: env_usize("DAILY_COMPACTION_MIN_ACTIVITIES", 2), diff --git a/rust/patcha/src/embedding.rs b/rust/patcha/src/embedding.rs index 36b835c..b23019a 100644 --- a/rust/patcha/src/embedding.rs +++ b/rust/patcha/src/embedding.rs @@ -1,5 +1,5 @@ use crate::config::Config; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use fastembed::{EmbeddingModel, InitOptions, TextEmbedding}; use tiktoken_rs::cl100k_base; @@ -7,6 +7,10 @@ use tiktoken_rs::cl100k_base; // text, so we apply a 0.8 budget to avoid overflowing the model's native limit. const TOKEN_SAFETY: f64 = 0.8; +// The sqlite-vec schema fixes the embedding column at 768 dims (db/migrations.rs). +// Any backend must produce 768-dim vectors or upserts will fail. +const STORE_VECTOR_DIM: usize = 768; + const MODEL_MAX_TOKENS: &[(&str, usize)] = &[ ("BAAI/bge-base-en-v1.5", 512), ("BAAI/bge-small-en-v1.5", 512), @@ -16,15 +20,38 @@ const MODEL_MAX_TOKENS: &[(&str, usize)] = &[ // Embedder // --------------------------------------------------------------------------- +/// Pluggable embedding backend. `effective_max_tokens` / `chunk_overlap` stay +/// public fields so existing call sites keep compiling unchanged. pub struct Embedder { - model: TextEmbedding, + backend: Backend, /// Effective token budget for chunking (accounts for safety margin). pub effective_max_tokens: usize, pub chunk_overlap: usize, } +enum Backend { + /// On-device ONNX embeddings via fastembed-rs (default — no server needed). + Fastembed(Box), + /// Local embeddings served by an Ollama instance. + Ollama(OllamaBackend), +} + +struct OllamaBackend { + http: ureq::Agent, + /// Full endpoint, e.g. http://localhost:11434/api/embed + url: String, + model: String, +} + impl Embedder { pub fn new(cfg: &Config) -> Result { + match cfg.embedding_provider.to_lowercase().as_str() { + "ollama" => Self::new_ollama(cfg), + _ => Self::new_fastembed(cfg), + } + } + + fn new_fastembed(cfg: &Config) -> Result { let model_enum = resolve_model(&cfg.embedding_model_name); let init_opts = InitOptions::new(model_enum) @@ -40,31 +67,121 @@ impl Embedder { .map(|(_, lim)| *lim) .unwrap_or(512); - let budget = ((native_limit as f64) * TOKEN_SAFETY) as usize; - let effective = budget - .min(cfg.max_embedding_tokens) - .max(cfg.embedding_chunk_overlap + 1); - Ok(Self { - model, - effective_max_tokens: effective, + backend: Backend::Fastembed(Box::new(model)), + effective_max_tokens: effective_tokens(native_limit, cfg), chunk_overlap: cfg.embedding_chunk_overlap, }) } + fn new_ollama(cfg: &Config) -> Result { + let http = ureq::AgentBuilder::new() + .timeout(std::time::Duration::from_secs(120)) + .build(); + let backend = OllamaBackend { + http, + url: format!("{}/api/embed", cfg.ollama_url.trim_end_matches('/')), + model: cfg.ollama_embedding_model.clone(), + }; + // nomic-embed-text handles ~2048 tokens; cap by the configured budget. + let embedder = Self { + backend: Backend::Ollama(backend), + effective_max_tokens: effective_tokens(2048, cfg), + chunk_overlap: cfg.embedding_chunk_overlap, + }; + + // Fail fast with an actionable message if Ollama is unreachable or the + // model's dimension doesn't match the store's fixed 768-dim schema. + let probe = embedder.embed_one("dimension probe").context( + "could not embed via Ollama — is `ollama serve` running and the model pulled? \ + try: ollama pull ", + )?; + if probe.len() != STORE_VECTOR_DIM { + bail!( + "Ollama embedding model '{}' returns {}-dim vectors, but patcha's store is \ + fixed at {}-dim. Use a 768-dim model such as nomic-embed-text.", + cfg.ollama_embedding_model, + probe.len(), + STORE_VECTOR_DIM + ); + } + Ok(embedder) + } + /// Embed a single text; returns a 768-dim float vector. pub fn embed_one(&self, text: &str) -> Result> { - let mut results = self.model.embed(vec![text.to_owned()], None) - .context("fastembed embed_one failed")?; - Ok(results.remove(0)) + match &self.backend { + Backend::Fastembed(model) => { + let mut results = model + .embed(vec![text.to_owned()], None) + .context("fastembed embed_one failed")?; + Ok(results.remove(0)) + } + Backend::Ollama(b) => Ok(b.embed(vec![text.to_owned()])?.remove(0)), + } } /// Embed a batch of texts. The returned Vec has the same length as `texts`. pub fn embed_many(&self, texts: Vec) -> Result>> { - self.model.embed(texts, None).context("fastembed embed_many failed") + match &self.backend { + Backend::Fastembed(model) => { + model.embed(texts, None).context("fastembed embed_many failed") + } + Backend::Ollama(b) => b.embed(texts), + } } } +impl OllamaBackend { + fn embed(&self, texts: Vec) -> Result>> { + // Ollama's /api/embed accepts a string or an array in `input` and returns + // one vector per input under `embeddings`. + #[derive(serde::Serialize)] + struct Req<'a> { + model: &'a str, + input: Vec, + } + #[derive(serde::Deserialize)] + struct Resp { + embeddings: Vec>, + } + + let body = serde_json::to_string(&Req { + model: &self.model, + input: texts, + }) + .context("serialize ollama embed request")?; + + let resp = match self + .http + .post(&self.url) + .set("content-type", "application/json") + .send_string(&body) + { + Ok(r) => r, + Err(ureq::Error::Status(code, r)) => { + let msg = r.into_string().unwrap_or_default(); + bail!("ollama embed HTTP {code}: {msg}"); + } + Err(e) => { + return Err(anyhow::Error::new(e)) + .with_context(|| format!("ollama embed request to {} failed", self.url)) + } + }; + + let parsed: Resp = serde_json::from_reader(resp.into_reader()) + .context("failed to parse ollama embed response")?; + Ok(parsed.embeddings) + } +} + +fn effective_tokens(native_limit: usize, cfg: &Config) -> usize { + let budget = ((native_limit as f64) * TOKEN_SAFETY) as usize; + budget + .min(cfg.max_embedding_tokens) + .max(cfg.embedding_chunk_overlap + 1) +} + fn resolve_model(name: &str) -> EmbeddingModel { match name { "BAAI/bge-base-en-v1.5" => EmbeddingModel::BGEBaseENV15, diff --git a/rust/patcha/src/llm/client.rs b/rust/patcha/src/llm/client.rs index f0f3988..4066e75 100644 --- a/rust/patcha/src/llm/client.rs +++ b/rust/patcha/src/llm/client.rs @@ -63,6 +63,12 @@ pub struct PatchaApiClient { http: reqwest::Client, /// In-memory tokens (shared across clones via Arc) tokens: Arc>, + /// "patcha" (cloud, default), "ollama" (local), or "claude" (Claude Code headless). + provider: String, + ollama_url: String, + ollama_model: String, + claude_bin: String, + claude_model: String, } #[derive(Default, Clone)] @@ -86,6 +92,11 @@ impl PatchaApiClient { .build() .expect("reqwest client"), tokens: Arc::new(Mutex::new(Tokens { access, refresh })), + provider: cfg.llm_provider.to_lowercase(), + ollama_url: cfg.ollama_url.trim_end_matches('/').to_owned(), + ollama_model: cfg.ollama_llm_model.clone(), + claude_bin: cfg.claude_bin.clone(), + claude_model: cfg.claude_model.clone(), } } @@ -100,6 +111,13 @@ impl PatchaApiClient { user: &str, model: &str, ) -> Result { + if self.provider == "claude" { + return self.claude_headless(system, user).await; + } + if self.provider == "ollama" { + return self.ollama_chat(system, user).await; + } + let messages = vec![ serde_json::json!({"role": "system", "content": system}), serde_json::json!({"role": "user", "content": user}), @@ -120,6 +138,74 @@ impl PatchaApiClient { .context("empty choices in chat response") } + /// Chat completion against a local Ollama server via its OpenAI-compatible + /// `/v1/chat/completions` endpoint. No auth; the configured model is used + /// regardless of the `model` the caller passed (which targets the cloud). + async fn ollama_chat(&self, system: &str, user: &str) -> Result { + let body = serde_json::json!({ + "model": self.ollama_model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "stream": false, + }); + let url = format!("{}/v1/chat/completions", self.ollama_url); + let resp = self + .http + .post(&url) + .json(&body) + .send() + .await + .context("ollama chat request failed — is `ollama serve` running?")?; + if !resp.status().is_success() { + bail!("ollama chat HTTP {} ({url})", resp.status()); + } + let parsed: ChatResponse = resp + .json() + .await + .context("failed to parse ollama chat response")?; + parsed + .choices + .into_iter() + .next() + .map(|c| c.message.content) + .context("empty choices in ollama chat response") + } + + /// Chat completion via Claude Code in headless mode (`claude -p`), using the + /// local Claude login — no API key. `--system-prompt` overrides Claude Code's + /// default agent prompt with the caller's; the `model` arg (which targets the + /// patcha cloud) is ignored in favour of the configured Claude model. + async fn claude_headless(&self, system: &str, user: &str) -> Result { + let output = tokio::process::Command::new(&self.claude_bin) + .arg("-p") + .arg(user) + .arg("--system-prompt") + .arg(system) + .arg("--model") + .arg(&self.claude_model) + .arg("--output-format") + .arg("text") + .output() + .await + .with_context(|| { + format!( + "failed to run `{}` — is Claude Code installed and logged in? (`claude` on PATH)", + self.claude_bin + ) + })?; + if !output.status.success() { + let err = String::from_utf8_lossy(&output.stderr); + bail!("claude headless failed ({}): {}", output.status, err.trim()); + } + let text = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if text.is_empty() { + bail!("claude headless returned empty output"); + } + Ok(text) + } + /// Embed a batch of texts via the API (fallback; local fastembed is preferred). pub async fn embed_remote(&self, texts: Vec, model: &str) -> Result>> { let body = serde_json::json!({ "model": model, "input": texts });