diff --git a/crates/icm-store/src/opensearch.rs b/crates/icm-store/src/opensearch.rs deleted file mode 100644 index b5517262..00000000 --- a/crates/icm-store/src/opensearch.rs +++ /dev/null @@ -1,2012 +0,0 @@ -//! OpenSearch storage backend (issue #301, opt-in via `--features opensearch`). -//! -//! A search-native shared store: BM25 full-text and `knn_vector` HNSW -//! vector search live in one engine, so horizontally-scaled ICM replicas -//! share one memory store (a node-local SQLite file cannot be shared). -//! -//! Design notes: -//! -//! - **Blocking REST.** OpenSearch is an HTTP/JSON service, so this talks -//! to it with the blocking `ureq` client and `serde_json` bodies. The -//! store traits are synchronous, so — like the PostgreSQL backend — -//! there is no async runtime and no sync-over-async bridge. -//! - **Vector search** uses a `knn_vector` field (HNSW, cosine space); -//! similarity is reported from the kNN `_score`. -//! - **Full-text search** uses BM25 `match` queries; the hybrid path -//! blends normalized BM25 and vector scores 30/70 to match the SQLite -//! and PostgreSQL backends. -//! - **Connection** from `ICM_OPENSEARCH_URL` (e.g. `http://localhost:9200`), -//! with optional basic auth from `ICM_OPENSEARCH_USER` / -//! `ICM_OPENSEARCH_PASSWORD`. -//! -//! Scope mirrors the PostgreSQL backend: the full [`MemoryStore`] surface -//! plus the ancillary store/recall/hook tables (hook telemetry, the -//! extraction queue, code areas, key/value metadata). The heavier -//! subsystems (memoir graph, transcripts, structured facts, feedback, -//! pattern mining) return [`IcmError::Unsupported`]; they stay fully -//! available on the default SQLite backend. - -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::time::Duration; - -use base64::engine::general_purpose::STANDARD as B64; -use base64::Engine; -use chrono::{DateTime, TimeZone, Utc}; -use serde_json::{json, Value}; - -use icm_core::{ - Concept, ConceptLink, Embedder, Fact, FactsStats, FactsStore, Feedback, FeedbackStats, - FeedbackStore, IcmError, IcmResult, Importance, Label, Memoir, MemoirStats, MemoirStore, - Memory, MemorySource, MemoryStore, Message, PatternCluster, Relation, Role, Scope, Session, - StoreStats, TopicHealth, TranscriptHit, TranscriptStats, TranscriptStore, -}; - -// Shared public row types live in `crate::common` (issue #301) so every -// backend can be compiled into one binary without colliding definitions. -pub use crate::common::{CodeArea, HookEvent, HookEventInsert, HookStatsRow, PendingRow}; - -// --------------------------------------------------------------------------- -// Index names -// --------------------------------------------------------------------------- - -const IDX_MEMORIES: &str = "icm_memories"; -const IDX_METADATA: &str = "icm_metadata"; -const IDX_HOOKS: &str = "icm_hook_events"; -const IDX_PENDING: &str = "icm_pending_extractions"; -const IDX_CODE_AREAS: &str = "icm_code_areas"; - -/// Percent-encode a value for safe use as a single path segment in a REST -/// URL. Document ids are caller-controlled (`icm forget ` CLI, MCP -/// `icm_forget`, etc.) with no format constraint enforced anywhere in the -/// schema — without this, a crafted id containing `/`, `..`, `?`, or `#` -/// could redirect which REST endpoint is actually hit instead of just -/// addressing the intended document (audit finding). -fn url_encode_path_segment(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char); - } - _ => out.push_str(&format!("%{b:02X}")), - } - } - out -} - -// --------------------------------------------------------------------------- -// Pure helpers (self-contained, mirror the other backends) -// --------------------------------------------------------------------------- - -fn source_type(source: &MemorySource) -> &'static str { - match source { - MemorySource::ClaudeCode { .. } => "claude_code", - MemorySource::Conversation { .. } => "conversation", - MemorySource::Manual => "manual", - } -} - -fn source_data(source: &MemorySource) -> Option { - match source { - MemorySource::Manual => None, - other => serde_json::to_string(other).ok(), - } -} - -fn parse_source(source_type_str: &str, source_data_str: Option) -> MemorySource { - match source_type_str { - "manual" => MemorySource::Manual, - _ => source_data_str - .and_then(|d| serde_json::from_str(&d).ok()) - .unwrap_or(MemorySource::Manual), - } -} - -fn importance_rank(i: Importance) -> u8 { - match i { - Importance::Low => 0, - Importance::Medium => 1, - Importance::High => 2, - Importance::Critical => 3, - } -} - -fn max_importance(a: Importance, b: Importance) -> Importance { - if importance_rank(a) >= importance_rank(b) { - a - } else { - b - } -} - -/// SHA-256 over the normalized `(topic, summary)` pair, hex-encoded. -/// Normalization: trim + lowercase + collapse whitespace, joined by `\0`. -fn summary_hash(topic: &str, summary: &str) -> String { - use sha2::{Digest, Sha256}; - let topic_n = topic.trim().to_lowercase(); - let summary_n = summary - .split_whitespace() - .collect::>() - .join(" ") - .to_lowercase(); - let mut h = Sha256::new(); - h.update(topic_n.as_bytes()); - h.update(b"\0"); - h.update(summary_n.as_bytes()); - format!("{:x}", h.finalize()) -} - -/// Validate and normalize a memory before storing (mirror of the other -/// backends): non-empty topic/summary, generate an id if missing, and -/// stamp timestamps. -fn validate_and_normalize(mut memory: Memory) -> IcmResult { - if memory.topic.trim().is_empty() { - return Err(IcmError::InvalidInput("topic cannot be empty".into())); - } - if memory.summary.trim().is_empty() { - return Err(IcmError::InvalidInput("summary cannot be empty".into())); - } - if memory.id.trim().is_empty() { - memory.id = ulid::Ulid::new().to_string(); - } - memory.topic = memory.topic.trim().to_string(); - Ok(memory) -} - -fn parse_dt(s: &str) -> DateTime { - DateTime::parse_from_rfc3339(s) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()) -} - -// --------------------------------------------------------------------------- -// Store -// --------------------------------------------------------------------------- - -/// OpenSearch-backed store. Cheap to clone-free share via `&self`; every -/// method is a blocking REST round-trip. -pub struct OpenSearchStore { - agent: ureq::Agent, - base: String, - auth: Option, - embedding_dims: usize, - readonly: bool, -} - -impl OpenSearchStore { - fn conn_url() -> IcmResult { - std::env::var("ICM_OPENSEARCH_URL") - .or_else(|_| std::env::var("OPENSEARCH_URL")) - .map_err(|_| { - IcmError::Config( - "OpenSearch backend: set ICM_OPENSEARCH_URL to the cluster endpoint, \ - e.g. http://localhost:9200" - .into(), - ) - }) - } - - fn auth_header() -> Option { - let user = std::env::var("ICM_OPENSEARCH_USER").ok()?; - let pass = std::env::var("ICM_OPENSEARCH_PASSWORD").unwrap_or_default(); - let token = B64.encode(format!("{user}:{pass}")); - Some(format!("Basic {token}")) - } - - /// Perform a request, returning the parsed JSON body. `expected_404` - /// makes a 404 return `Ok(None)` instead of an error (used by `get`). - fn request( - &self, - method: &str, - path: &str, - body: Option, - allow_404: bool, - ) -> IcmResult> { - let url = format!( - "{}/{}", - self.base.trim_end_matches('/'), - path.trim_start_matches('/') - ); - let mut req = self.agent.request(method, &url); - if let Some(a) = &self.auth { - req = req.set("Authorization", a); - } - let resp = match body { - Some(b) => req.send_json(b), - None => req.call(), - }; - match resp { - Ok(r) => { - let v = r - .into_json::() - .map_err(|e| IcmError::Database(format!("opensearch decode: {e}")))?; - Ok(Some(v)) - } - Err(ureq::Error::Status(404, _)) if allow_404 => Ok(None), - Err(ureq::Error::Status(code, r)) => { - let txt = r.into_string().unwrap_or_default(); - Err(IcmError::Database(format!( - "opensearch {method} {path} -> {code}: {txt}" - ))) - } - Err(e) => Err(IcmError::Database(format!( - "opensearch {method} {path}: {e}" - ))), - } - } - - fn get_json(&self, path: &str) -> IcmResult> { - self.request("GET", path, None, true) - } - - fn post(&self, path: &str, body: Value) -> IcmResult { - self.request("POST", path, Some(body), false) - .map(|o| o.unwrap_or(Value::Null)) - } - - /// Open or create a store with the default embedding dimension. - pub fn new(_path: &Path) -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) - } - - /// Open or create a store with a specific embedding dimension. - pub fn with_dims(_path: &Path, embedding_dims: usize) -> IcmResult { - Self::connect(embedding_dims, false) - } - - /// Open the store read-only (issue #263). OpenSearch has no read-only - /// connection mode, so this just flags the store and makes mutating - /// methods error. - pub fn open_readonly(_path: &Path) -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, true) - } - - /// In-memory variant is not meaningful for a remote backend; connect - /// from the environment instead. - pub fn in_memory() -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) - } - - /// See [`Self::in_memory`]. - pub fn in_memory_with_dims(embedding_dims: usize) -> IcmResult { - Self::connect(embedding_dims, false) - } - - /// Read the stored embedding dimension without committing to a full - /// open. Returns `Ok(None)` when unreachable so callers can fall back. - pub fn read_stored_embedding_dims(_path: &Path) -> IcmResult> { - let Ok(url) = Self::conn_url() else { - return Ok(None); - }; - let agent = ureq::AgentBuilder::new() - .timeout(Duration::from_secs(10)) - .build(); - let store = OpenSearchStore { - agent, - base: url, - auth: Self::auth_header(), - embedding_dims: icm_core::DEFAULT_EMBEDDING_DIMS, - readonly: true, - }; - match store.get_metadata_int("embedding_dims") { - Ok(Some(v)) => Ok(Some(v as usize)), - _ => Ok(None), - } - } - - pub fn is_readonly(&self) -> bool { - self.readonly - } - - /// No-op on this backend (kept for API parity with the SQLite store). - pub fn ensure_vec_init() {} - - fn connect(requested_dims: usize, readonly: bool) -> IcmResult { - let url = Self::conn_url()?; - let agent = ureq::AgentBuilder::new() - .timeout(Duration::from_secs(30)) - .build(); - let store = OpenSearchStore { - agent, - base: url, - auth: Self::auth_header(), - embedding_dims: requested_dims, - readonly, - }; - // Probe connectivity early with a clear error. - store - .get_json("/") - .map_err(|e| IcmError::Database(format!("cannot reach OpenSearch: {e}")))?; - - // An existing database's stored dims are authoritative. - let dims = match store.get_metadata_int("embedding_dims")? { - Some(d) => d as usize, - None => requested_dims, - }; - let mut store = store; - store.embedding_dims = dims; - - if !readonly { - store.init_indices(dims)?; - store.set_metadata_int("embedding_dims", dims as i64)?; - } - Ok(store) - } - - fn index_exists(&self, idx: &str) -> IcmResult { - let url = format!("{}/{}", self.base.trim_end_matches('/'), idx); - let mut req = self.agent.request("HEAD", &url); - if let Some(a) = &self.auth { - req = req.set("Authorization", a); - } - match req.call() { - Ok(_) => Ok(true), - Err(ureq::Error::Status(404, _)) => Ok(false), - Err(e) => Err(IcmError::Database(format!("opensearch HEAD {idx}: {e}"))), - } - } - - fn create_index(&self, idx: &str, body: Value) -> IcmResult<()> { - if self.index_exists(idx)? { - return Ok(()); - } - // A racing replica may create it between the check and here; treat - // "resource_already_exists_exception" as success. - match self.request("PUT", idx, Some(body), false) { - Ok(_) => Ok(()), - Err(IcmError::Database(msg)) if msg.contains("resource_already_exists_exception") => { - Ok(()) - } - Err(e) => Err(e), - } - } - - fn init_indices(&self, dims: usize) -> IcmResult<()> { - if !(64..=4096).contains(&dims) { - return Err(IcmError::Config(format!( - "embedding_dims must be between 64 and 4096, got {dims}" - ))); - } - self.create_index( - IDX_MEMORIES, - json!({ - "settings": { "index": { "knn": true } }, - "mappings": { "properties": { - "created_at": {"type": "date"}, - "updated_at": {"type": "date"}, - "last_accessed": {"type": "date"}, - "access_count": {"type": "integer"}, - "weight": {"type": "float"}, - "topic": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 1024}}}, - "summary": {"type": "text"}, - "raw_excerpt": {"type": "text"}, - "keywords": {"type": "keyword"}, - "importance": {"type": "keyword"}, - "source_type": {"type": "keyword"}, - "source_data": {"type": "text", "index": false}, - "related_ids": {"type": "keyword"}, - "summary_hash": {"type": "keyword"}, - "embedding": { - "type": "knn_vector", - "dimension": dims, - "method": {"name": "hnsw", "space_type": "cosinesimil", "engine": "lucene"} - } - }} - }), - )?; - self.create_index(IDX_METADATA, json!({"mappings": {"properties": {"value": {"type": "double"}, "text_value": {"type": "keyword"}}}}))?; - self.create_index( - IDX_HOOKS, - json!({"mappings": {"properties": { - "id": {"type": "long"}, - "ts": {"type": "date"}, - "event": {"type": "keyword"}, - "project": {"type": "keyword"}, - "session_id": {"type": "keyword"}, - "tool_name": {"type": "keyword"}, - "duration_ms": {"type": "long"}, - "exit_code": {"type": "integer"}, - "payload_size": {"type": "long"}, - "note": {"type": "text"} - }}}), - )?; - self.create_index( - IDX_PENDING, - json!({"mappings": {"properties": { - "project": {"type": "keyword"}, - "tool_name": {"type": "keyword"}, - "raw_output": {"type": "text", "index": false}, - "captured_at": {"type": "date"} - }}}), - )?; - self.create_index( - IDX_CODE_AREAS, - json!({"mappings": {"properties": { - "project": {"type": "keyword"}, - "file_path": {"type": "keyword"}, - "description": {"type": "text"}, - "session_id": {"type": "keyword"}, - "tool_name": {"type": "keyword"}, - "touch_count": {"type": "long"}, - "first_touched_at": {"type": "date"}, - "last_touched_at": {"type": "date"} - }}}), - )?; - Ok(()) - } - - // --- metadata kv helpers --- - - fn get_metadata_int(&self, key: &str) -> IcmResult> { - let path = format!("{IDX_METADATA}/_doc/{key}"); - match self.get_json(&path)? { - Some(v) => Ok(v - .get("_source") - .and_then(|s| s.get("value")) - .and_then(|n| n.as_f64()) - .map(|f| f as i64)), - None => Ok(None), - } - } - - fn set_metadata_int(&self, key: &str, value: i64) -> IcmResult<()> { - let path = format!("{IDX_METADATA}/_doc/{key}?refresh=true"); - self.request("PUT", &path, Some(json!({"value": value})), false)?; - Ok(()) - } - - fn check_dims(&self, memory: &Memory) -> IcmResult<()> { - if let Some(emb) = memory.embedding.as_ref() { - if emb.len() != self.embedding_dims { - return Err(IcmError::InvalidInput(format!( - "embedding has {} dimensions, but this store uses {}", - emb.len(), - self.embedding_dims - ))); - } - } - Ok(()) - } - - // --- (de)serialization --- - - fn memory_to_source(memory: &Memory) -> Value { - let mut doc = json!({ - "created_at": memory.created_at.to_rfc3339(), - "updated_at": memory.updated_at.to_rfc3339(), - "last_accessed": memory.last_accessed.to_rfc3339(), - "access_count": memory.access_count, - "weight": memory.weight, - "topic": memory.topic, - "summary": memory.summary, - "raw_excerpt": memory.raw_excerpt, - "keywords": memory.keywords, - "importance": memory.importance.to_string(), - "source_type": source_type(&memory.source), - "source_data": source_data(&memory.source), - "related_ids": memory.related_ids, - "summary_hash": summary_hash(&memory.topic, &memory.summary), - }); - if let Some(emb) = memory.embedding.as_ref() { - doc["embedding"] = json!(emb); - } - doc - } - - fn source_to_memory(id: &str, src: &Value) -> Memory { - let get_str = |k: &str| { - src.get(k) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string() - }; - let opt_str = |k: &str| { - src.get(k) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .filter(|s| !s.is_empty()) - }; - let arr = |k: &str| { - src.get(k) - .and_then(|v| v.as_array()) - .map(|a| { - a.iter() - .filter_map(|x| x.as_str().map(|s| s.to_string())) - .collect::>() - }) - .unwrap_or_default() - }; - let importance = get_str("importance").parse().unwrap_or(Importance::Medium); - let source = parse_source(&get_str("source_type"), opt_str("source_data")); - let embedding = src.get("embedding").and_then(|v| v.as_array()).map(|a| { - a.iter() - .filter_map(|x| x.as_f64().map(|f| f as f32)) - .collect::>() - }); - Memory { - id: id.to_string(), - created_at: parse_dt(&get_str("created_at")), - updated_at: parse_dt(&get_str("updated_at")), - last_accessed: parse_dt(&get_str("last_accessed")), - access_count: src - .get("access_count") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as u32, - weight: src.get("weight").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32, - topic: get_str("topic"), - summary: get_str("summary"), - raw_excerpt: opt_str("raw_excerpt"), - keywords: arr("keywords"), - importance, - source, - related_ids: arr("related_ids"), - embedding, - scope: Scope::default(), - } - } - - /// Map a `_search` response's hits to memories paired with `_score`. - fn hits_to_scored(resp: &Value) -> Vec<(Memory, f32)> { - resp.get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .map(|hits| { - hits.iter() - .filter_map(|h| { - let id = h.get("_id")?.as_str()?; - let src = h.get("_source")?; - let score = h.get("_score").and_then(|s| s.as_f64()).unwrap_or(0.0) as f32; - Some((Self::source_to_memory(id, src), score)) - }) - .collect() - }) - .unwrap_or_default() - } - - fn hits_to_memories(resp: &Value) -> Vec { - Self::hits_to_scored(resp) - .into_iter() - .map(|(m, _)| m) - .collect() - } - - fn refresh_param(&self) -> &'static str { - // Force a refresh so writes are immediately visible to subsequent - // searches (dedup, counts, the multi-replica path). ICM writes are - // low-frequency curated memories, so the cost is acceptable. - "refresh=true" - } - - fn store_inner(&self, memory: &Memory) -> IcmResult { - let hash = summary_hash(&memory.topic, &memory.summary); - // Dedup: an existing memory with the same (topic, summary_hash) - // wins; merge importance (max) + keywords (union) + raw_excerpt - // (prefer new) and return the existing id. - // - // Audit finding: this used to ALSO filter on `topic.keyword` (an - // exact-byte-match `keyword` field, no normalizer) alongside - // `summary_hash` — but `summary_hash` already encodes the topic via - // Rust's Unicode-correct `to_lowercase()`, so storing topic="Kexa" - // then topic="kexa" with the same summary hashed identically but - // failed the exact `topic.keyword` filter, silently creating a - // second document instead of deduping (broader than the SQLite/ - // Postgres accented-topic case — this fires on ANY case - // difference). `summary_hash` alone is sufficient, matching the - // SQLite/Postgres fix. - let existing = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": 1, - "query": {"bool": {"filter": [ - {"term": {"summary_hash": hash}} - ]}} - }), - )?; - if let Some(hit) = existing - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .and_then(|a| a.first()) - { - let existing_id = hit - .get("_id") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let src = hit.get("_source").cloned().unwrap_or(Value::Null); - let existing_importance: Importance = src - .get("importance") - .and_then(|v| v.as_str()) - .unwrap_or("medium") - .parse() - .unwrap_or(Importance::Medium); - let merged_importance = max_importance(existing_importance, memory.importance); - let mut merged_keywords: Vec = src - .get("keywords") - .and_then(|v| v.as_array()) - .map(|a| { - a.iter() - .filter_map(|x| x.as_str().map(String::from)) - .collect() - }) - .unwrap_or_default(); - for kw in &memory.keywords { - if !merged_keywords.contains(kw) { - merged_keywords.push(kw.clone()); - } - } - let raw = memory.raw_excerpt.clone().or_else(|| { - src.get("raw_excerpt") - .and_then(|v| v.as_str()) - .map(String::from) - }); - self.request( - "POST", - &format!( - "{IDX_MEMORIES}/_update/{existing_id}?{}", - self.refresh_param() - ), - Some(json!({"doc": { - "importance": merged_importance.to_string(), - "keywords": merged_keywords, - "raw_excerpt": raw, - "updated_at": Utc::now().to_rfc3339(), - }})), - false, - )?; - return Ok(existing_id); - } - - self.request( - "PUT", - &format!( - "{IDX_MEMORIES}/_doc/{}?{}", - url_encode_path_segment(&memory.id), - self.refresh_param() - ), - Some(Self::memory_to_source(memory)), - false, - )?; - Ok(memory.id.clone()) - } -} - -impl MemoryStore for OpenSearchStore { - fn store(&self, memory: Memory) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("store".into())); - } - let memory = validate_and_normalize(memory)?; - self.check_dims(&memory)?; - self.store_inner(&memory) - } - - fn get(&self, id: &str) -> IcmResult> { - let path = format!("{IDX_MEMORIES}/_doc/{}", url_encode_path_segment(id)); - match self.get_json(&path)? { - Some(v) => { - if v.get("found").and_then(|f| f.as_bool()).unwrap_or(false) { - let src = v.get("_source").cloned().unwrap_or(Value::Null); - Ok(Some(Self::source_to_memory(id, &src))) - } else { - Ok(None) - } - } - None => Ok(None), - } - } - - fn update(&self, memory: &Memory) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("update".into())); - } - self.check_dims(memory)?; - let mut doc = Self::memory_to_source(memory); - doc["updated_at"] = json!(Utc::now().to_rfc3339()); - // Replace the document wholesale (index by id). - self.request( - "PUT", - &format!( - "{IDX_MEMORIES}/_doc/{}?{}", - url_encode_path_segment(&memory.id), - self.refresh_param() - ), - Some(doc), - false, - )?; - Ok(()) - } - - fn delete(&self, id: &str) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("delete".into())); - } - self.request( - "DELETE", - &format!( - "{IDX_MEMORIES}/_doc/{}?{}", - url_encode_path_segment(id), - self.refresh_param() - ), - None, - true, - )?; - - // Manual-testing finding (same class as the SQLite/Postgres - // backends): a deleted memory otherwise stays as a dangling entry - // in every other memory's `related_ids` forever. `related_ids` is - // mapped as a `keyword` array field, so a term query finds every - // document that references it and a Painless script strips it out - // in place. Best-effort: a failure here doesn't roll back the - // delete above (OpenSearch has no cross-document transaction to - // roll back into) — surfacing an error would make a successful - // delete look like it failed, so log and move on. - if let Err(e) = self.post( - &format!( - "{IDX_MEMORIES}/_update_by_query?conflicts=proceed&{}", - self.refresh_param() - ), - json!({ - "script": { - "source": "ctx._source.related_ids.removeIf(x -> x == params.deleted_id)", - "params": {"deleted_id": id} - }, - "query": {"term": {"related_ids": id}} - }), - ) { - tracing::warn!(error = %e, id, "failed to clean up dangling related_ids after delete"); - } - - Ok(()) - } - - fn search_by_keywords(&self, keywords: &[&str], limit: usize) -> IcmResult> { - if keywords.is_empty() { - return Ok(Vec::new()); - } - // Audit finding: unlike Postgres/SQLite, `limit` was never clamped - // here — a caller-supplied limit above OpenSearch's own - // `index.max_result_window` (default 10,000) returns a hard 400 - // error instead of gracefully truncating like the other backends. - let limit = limit.min(100); - let joined = keywords.join(" "); - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": limit, - "query": {"bool": {"should": [ - {"terms": {"keywords": keywords}}, - {"multi_match": {"query": joined, "fields": ["summary", "topic"]}} - ], "minimum_should_match": 1}} - }), - )?; - Ok(Self::hits_to_memories(&resp)) - } - - fn search_fts(&self, query: &str, limit: usize) -> IcmResult> { - if query.trim().is_empty() { - return Ok(Vec::new()); - } - let limit = limit.min(100); - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": limit, - "query": {"multi_match": { - "query": query, - "fields": ["summary^2", "topic", "keywords"] - }} - }), - )?; - Ok(Self::hits_to_memories(&resp)) - } - - fn search_by_embedding( - &self, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - let limit = limit.min(1000); - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": limit, - "query": {"knn": {"embedding": {"vector": embedding, "k": limit}}} - }), - )?; - Ok(Self::hits_to_scored(&resp)) - } - - fn search_hybrid( - &self, - query: &str, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - let limit = limit.min(1000); - let pool = limit * 4; - - // FTS candidates (BM25). - let mut fts_scores: HashMap = HashMap::new(); - let mut memories: HashMap = HashMap::new(); - if !query.trim().is_empty() { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": pool, - "query": {"multi_match": {"query": query, "fields": ["summary^2", "topic", "keywords"]}} - }), - )?; - for (m, s) in Self::hits_to_scored(&resp) { - fts_scores.insert(m.id.clone(), s); - memories.insert(m.id.clone(), m); - } - } - - // Vector candidates. - let mut vec_scores: HashMap = HashMap::new(); - for (m, s) in self.search_by_embedding(embedding, pool)? { - vec_scores.insert(m.id.clone(), s); - memories.entry(m.id.clone()).or_insert(m); - } - - // Min-max normalize each score family to [0, 1] before blending. - let norm = |scores: &HashMap| -> HashMap { - if scores.is_empty() { - return HashMap::new(); - } - let (mut lo, mut hi) = (f32::MAX, f32::MIN); - for &v in scores.values() { - lo = lo.min(v); - hi = hi.max(v); - } - let span = (hi - lo).max(f32::EPSILON); - scores - .iter() - .map(|(k, v)| (k.clone(), (v - lo) / span)) - .collect() - }; - let fts_n = norm(&fts_scores); - let vec_n = norm(&vec_scores); - - let mut scored: Vec<(String, f32)> = memories - .keys() - .map(|id| { - let f = fts_n.get(id).copied().unwrap_or(0.0); - let v = vec_n.get(id).copied().unwrap_or(0.0); - (id.clone(), 0.3 * f + 0.7 * v) - }) - .collect(); - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - scored.truncate(limit); - - Ok(scored - .into_iter() - .filter_map(|(id, s)| memories.remove(&id).map(|m| (m, s))) - .collect()) - } - - fn update_access(&self, id: &str) -> IcmResult<()> { - if self.readonly { - return Ok(()); - } - // Best-effort; a missing doc is not an error for recall bookkeeping. - let _ = self.request( - "POST", - &format!("{IDX_MEMORIES}/_update/{id}"), - Some(json!({ - "script": { - "lang": "painless", - "source": "ctx._source.access_count = (ctx._source.access_count == null ? 1 : ctx._source.access_count + 1); ctx._source.last_accessed = params.now;", - "params": {"now": Utc::now().to_rfc3339()} - } - })), - true, - )?; - Ok(()) - } - - fn batch_update_access(&self, ids: &[&str]) -> IcmResult { - if self.readonly || ids.is_empty() { - return Ok(0); - } - let resp = self.post( - &format!("{IDX_MEMORIES}/_update_by_query?{}&conflicts=proceed", self.refresh_param()), - json!({ - "query": {"ids": {"values": ids}}, - "script": { - "lang": "painless", - "source": "ctx._source.access_count = (ctx._source.access_count == null ? 1 : ctx._source.access_count + 1); ctx._source.last_accessed = params.now;", - "params": {"now": Utc::now().to_rfc3339()} - } - }), - )?; - Ok(resp.get("updated").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - fn apply_decay(&self, decay_factor: f32) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("decay".into())); - } - let resp = self.post( - &format!("{IDX_MEMORIES}/_update_by_query?{}&conflicts=proceed", self.refresh_param()), - json!({ - "query": {"bool": {"must_not": [{"term": {"importance": "critical"}}]}}, - // Audit finding: for `low` importance with low access count, - // the raw multiplier goes negative once decay_factor < 0.5 - // (still inside the CLI's own validated [0.0, 1.0) range) — - // same bug already fixed for SQLite/Postgres. Math.max is - // Painless's equivalent clamp. - "script": { - "lang": "painless", - "source": "double f = params.factor; String imp = ctx._source.importance; double mult = imp != null && imp.equals('high') ? 0.5 : (imp != null && imp.equals('low') ? 2.0 : 1.0); double ac = ctx._source.access_count == null ? 0 : ctx._source.access_count; if (ac > 5) ac = 5; double m = Math.max(0.0, 1.0 - (1.0 - f) * mult / (1.0 + ac * 0.1)); ctx._source.weight = ctx._source.weight * m;", - "params": {"factor": decay_factor as f64} - } - }), - )?; - Ok(resp.get("updated").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - fn prune(&self, weight_threshold: f32) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("prune".into())); - } - let resp = self.post( - &format!( - "{IDX_MEMORIES}/_delete_by_query?{}&conflicts=proceed", - self.refresh_param() - ), - json!({ - "query": {"bool": { - "must": [{"range": {"weight": {"lt": weight_threshold as f64}}}], - "must_not": [{"terms": {"importance": ["critical", "high"]}}] - }} - }), - )?; - Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - fn get_by_topic(&self, topic: &str) -> IcmResult> { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": 500, - "query": {"term": {"topic.keyword": topic}}, - "sort": [{"weight": "desc"}] - }), - )?; - Ok(Self::hits_to_memories(&resp)) - } - - fn list_all(&self) -> IcmResult> { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({"size": 10000, "query": {"match_all": {}}, "sort": [{"weight": "desc"}]}), - )?; - Ok(Self::hits_to_memories(&resp)) - } - - fn list_topics(&self) -> IcmResult> { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({"size": 0, "aggs": {"topics": {"terms": {"field": "topic.keyword", "size": 10000}}}}), - )?; - let mut out = bucket_counts(&resp, "topics"); - out.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(out) - } - - fn consolidate_topic(&self, topic: &str, consolidated: Memory) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("consolidate".into())); - } - // Audit findings, both fixed together: - // 1. `critical` memories are never deleted — same contract - // apply_decay/prune already honor, and the same fix already - // applied to SQLite/Postgres consolidate_topic. This delete - // query previously wiped critical memories in the topic too. - // 2. Still not atomic (no multi-document transaction in - // OpenSearch), but reordered to insert-then-delete: if - // `store_inner` fails (dimension mismatch, network blip, - // cluster unavailable), the originals are left untouched - // instead of being gone with no replacement ever written. The - // failure mode is now "harmless duplication a retry fixes", - // not data loss — the consolidated memory gets a fresh id, so - // it can't collide with any original. - let consolidated = validate_and_normalize(consolidated)?; - self.check_dims(&consolidated)?; - // Manual-testing finding: captured before store_inner/delete below - // (the deleted rows are gone afterward), so any *other* memory's - // related_ids pointing at them can be cleaned up — same dangling- - // reference bug already fixed for the single-id `delete`. - let deleted_ids: Vec = self - .get_by_topic(topic)? - .into_iter() - .filter(|m| m.importance != Importance::Critical) - .map(|m| m.id) - .collect(); - // `store_inner` returns the id actually used — the caller's fresh - // id on a normal insert, or an existing row's id if this exact - // (topic, summary_hash) happened to already exist (dedup merge). - // Either way it now has the SAME topic as the memories being - // consolidated, so it must be excluded from the delete below or it - // would delete the very memory it just wrote/merged into. - let consolidated_id = self.store_inner(&consolidated)?; - self.post( - &format!( - "{IDX_MEMORIES}/_delete_by_query?{}&conflicts=proceed", - self.refresh_param() - ), - json!({"query": {"bool": { - "must": [{"term": {"topic.keyword": topic}}], - "must_not": [ - {"term": {"importance": "critical"}}, - {"ids": {"values": [consolidated_id]}} - ] - }}}), - )?; - - if !deleted_ids.is_empty() { - if let Err(e) = self.post( - &format!( - "{IDX_MEMORIES}/_update_by_query?conflicts=proceed&{}", - self.refresh_param() - ), - json!({ - "script": { - "source": "ctx._source.related_ids.removeIf(x -> params.deleted_ids.contains(x))", - "params": {"deleted_ids": deleted_ids} - }, - "query": {"terms": {"related_ids": deleted_ids}} - }), - ) { - tracing::warn!(topic, error = %e, "consolidate_topic: failed to clean up dangling related_ids"); - } - } - - Ok(()) - } - - fn count(&self) -> IcmResult { - let resp = self.post(&format!("{IDX_MEMORIES}/_count"), json!({}))?; - Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - fn count_by_topic(&self, topic: &str) -> IcmResult { - let resp = self.post( - &format!("{IDX_MEMORIES}/_count"), - json!({"query": {"term": {"topic.keyword": topic}}}), - )?; - Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - fn stats(&self) -> IcmResult { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": 0, - "track_total_hits": true, - "aggs": { - "avg_w": {"avg": {"field": "weight"}}, - "topics": {"cardinality": {"field": "topic.keyword"}}, - "oldest": {"min": {"field": "created_at", "format": "date_time"}}, - "newest": {"max": {"field": "created_at", "format": "date_time"}} - } - }), - )?; - let total = resp - .get("hits") - .and_then(|h| h.get("total")) - .and_then(|t| t.get("value")) - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - let aggs = resp.get("aggregations").cloned().unwrap_or(Value::Null); - let avg_weight = aggs - .get("avg_w") - .and_then(|a| a.get("value")) - .and_then(|v| v.as_f64()) - .unwrap_or(0.0) as f32; - let total_topics = aggs - .get("topics") - .and_then(|a| a.get("value")) - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - let parse_agg_date = |name: &str| -> Option> { - aggs.get(name) - .and_then(|a| a.get("value_as_string")) - .and_then(|v| v.as_str()) - .map(parse_dt) - }; - Ok(StoreStats { - total_memories: total, - total_topics, - avg_weight, - oldest_memory: parse_agg_date("oldest"), - newest_memory: parse_agg_date("newest"), - }) - } - - fn topic_health(&self, topic: &str) -> IcmResult { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": 0, - "track_total_hits": true, - "query": {"term": {"topic.keyword": topic}}, - "aggs": { - "avg_w": {"avg": {"field": "weight"}}, - "avg_ac": {"avg": {"field": "access_count"}}, - "oldest": {"min": {"field": "created_at"}}, - "newest": {"max": {"field": "created_at"}}, - "last_acc": {"max": {"field": "last_accessed"}}, - "stale": {"filter": {"bool": {"must": [ - {"range": {"weight": {"lt": 0.5}}}, - {"range": {"last_accessed": {"lt": "now-14d"}}} - ]}}} - } - }), - )?; - let entry_count = resp - .get("hits") - .and_then(|h| h.get("total")) - .and_then(|t| t.get("value")) - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - if entry_count == 0 { - return Err(IcmError::NotFound(format!( - "no memories in topic '{topic}'" - ))); - } - let aggs = resp.get("aggregations").cloned().unwrap_or(Value::Null); - let avg_weight = aggs - .get("avg_w") - .and_then(|a| a.get("value")) - .and_then(|v| v.as_f64()) - .unwrap_or(0.0) as f32; - let avg_access_count = aggs - .get("avg_ac") - .and_then(|a| a.get("value")) - .and_then(|v| v.as_f64()) - .unwrap_or(0.0) as f32; - let stale_count = aggs - .get("stale") - .and_then(|a| a.get("doc_count")) - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - Ok(TopicHealth { - topic: topic.to_string(), - entry_count, - avg_weight, - avg_access_count, - oldest: agg_date(&aggs, "oldest"), - newest: agg_date(&aggs, "newest"), - last_accessed: agg_date(&aggs, "last_acc"), - stale_count, - needs_consolidation: entry_count > 5, - }) - } -} - -/// Read a `min`/`max` date aggregation into a `DateTime`. -/// -/// Prefers the ISO `value_as_string` OpenSearch returns and falls back to -/// the epoch-millis `value`. Returns `None` when the bucket is empty. -fn agg_date(aggs: &Value, name: &str) -> Option> { - let node = aggs.get(name)?; - if let Some(s) = node.get("value_as_string").and_then(|v| v.as_str()) { - if let Ok(dt) = DateTime::parse_from_rfc3339(s) { - return Some(dt.with_timezone(&Utc)); - } - } - let ms = node.get("value").and_then(|v| v.as_f64())?; - if ms <= 0.0 { - return None; - } - Utc.timestamp_millis_opt(ms as i64).single() -} - -/// Extract `(key, doc_count)` pairs from a terms aggregation. -fn bucket_counts(resp: &Value, agg: &str) -> Vec<(String, usize)> { - resp.get("aggregations") - .and_then(|a| a.get(agg)) - .and_then(|t| t.get("buckets")) - .and_then(|b| b.as_array()) - .map(|buckets| { - buckets - .iter() - .filter_map(|b| { - let key = b.get("key")?.as_str()?.to_string(); - let count = b.get("doc_count")?.as_u64()? as usize; - Some((key, count)) - }) - .collect() - }) - .unwrap_or_default() -} - -// --------------------------------------------------------------------------- -// Inherent methods used by the cli/mcp store/recall/hook path -// --------------------------------------------------------------------------- - -impl OpenSearchStore { - pub fn maybe_auto_decay(&self) -> IcmResult<()> { - if self.readonly { - return Ok(()); - } - // Atomic-ish claim via a scripted upsert on a metadata doc: only the - // caller that flips `changed` to true runs the decay. - let now_ms = Utc::now().timestamp_millis(); - let resp = self.post( - &format!("{IDX_METADATA}/_update/last_decay_at?{}&_source=true", self.refresh_param()), - json!({ - "scripted_upsert": true, - "upsert": {}, - "script": { - "lang": "painless", - "source": "if (ctx._source.value == null || params.now - ctx._source.value >= 86400000L) { ctx._source.value = params.now; ctx._source.changed = true; } else { ctx._source.changed = false; }", - "params": {"now": now_ms} - } - }), - )?; - let changed = resp - .get("get") - .and_then(|g| g.get("_source")) - .and_then(|s| s.get("changed")) - .and_then(|c| c.as_bool()) - .unwrap_or(false); - if changed { - self.apply_decay(0.95)?; - } - Ok(()) - } - - pub fn increment_hook_counter(&self) -> IcmResult { - let resp = self.post( - &format!("{IDX_METADATA}/_update/hook_counter?_source=true"), - json!({ - "scripted_upsert": true, - "upsert": {}, - "script": { - "lang": "painless", - "source": "ctx._source.value = (ctx._source.value == null ? 1 : ctx._source.value + 1);" - } - }), - )?; - Ok(resp - .get("get") - .and_then(|g| g.get("_source")) - .and_then(|s| s.get("value")) - .and_then(|v| v.as_f64()) - .unwrap_or(0.0) as usize) - } - - pub fn reset_hook_counter(&self) -> IcmResult<()> { - self.set_metadata_int("hook_counter", 0) - } - - pub fn enqueue_pending_extraction( - &self, - project: &str, - tool_name: &str, - raw_output: &str, - ) -> IcmResult { - let id = ulid::Ulid::new().to_string(); - self.request( - "PUT", - &format!("{IDX_PENDING}/_doc/{id}?{}", self.refresh_param()), - Some(json!({ - "project": project, - "tool_name": tool_name, - "raw_output": raw_output, - "captured_at": Utc::now().to_rfc3339() - })), - false, - )?; - Ok(id) - } - - pub fn list_pending_extractions(&self, limit: usize) -> IcmResult> { - let resp = self.post( - &format!("{IDX_PENDING}/_search"), - json!({"size": limit, "query": {"match_all": {}}, "sort": [{"captured_at": "asc"}]}), - )?; - let rows = resp - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .map(|hits| { - hits.iter() - .filter_map(|h| { - let id = h.get("_id")?.as_str()?.to_string(); - let s = h.get("_source")?; - Some(( - id, - s.get("project")?.as_str()?.to_string(), - s.get("tool_name")?.as_str()?.to_string(), - s.get("raw_output")?.as_str()?.to_string(), - s.get("captured_at")?.as_str()?.to_string(), - )) - }) - .collect() - }) - .unwrap_or_default(); - Ok(rows) - } - - pub fn delete_pending_extractions(&self, ids: &[String]) -> IcmResult { - if ids.is_empty() { - return Ok(0); - } - let resp = self.post( - &format!( - "{IDX_PENDING}/_delete_by_query?{}&conflicts=proceed", - self.refresh_param() - ), - json!({"query": {"ids": {"values": ids}}}), - )?; - Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - pub fn pending_extraction_count(&self) -> IcmResult { - let resp = self.post(&format!("{IDX_PENDING}/_count"), json!({}))?; - Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - pub fn upsert_code_area( - &self, - project: &str, - file_path: &str, - description: Option<&str>, - session_id: Option<&str>, - tool_name: Option<&str>, - ) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("upsert_code_area".into())); - } - let ts = Utc::now(); - let now = ts.to_rfc3339(); - let id = ts.timestamp_millis(); - // Deterministic id makes the same (project, file_path) a single row. - let key = B64.encode(format!("{project}\0{file_path}")); - self.request( - "POST", - &format!("{IDX_CODE_AREAS}/_update/{key}?{}", self.refresh_param()), - Some(json!({ - "scripted_upsert": true, - "upsert": { - "id": id, - "project": project, - "file_path": file_path, - "description": description, - "session_id": session_id, - "tool_name": tool_name, - "touch_count": 1, - "first_touched_at": now, - "last_touched_at": now - }, - "script": { - "lang": "painless", - "source": "ctx._source.touch_count = (ctx._source.touch_count == null ? 1 : ctx._source.touch_count + 1); ctx._source.last_touched_at = params.now; if (params.description != null) ctx._source.description = params.description; if (params.session_id != null) ctx._source.session_id = params.session_id; if (params.tool_name != null) ctx._source.tool_name = params.tool_name;", - "params": {"now": now, "description": description, "session_id": session_id, "tool_name": tool_name} - } - })), - false, - )?; - Ok(()) - } - - pub fn list_code_areas( - &self, - project: Option<&str>, - in_file: Option<&str>, - since: Option>, - limit: usize, - ) -> IcmResult> { - let mut filters: Vec = Vec::new(); - if let Some(p) = project { - filters.push(json!({"term": {"project": p}})); - } - if let Some(f) = in_file { - filters.push(json!({"wildcard": {"file_path": {"value": format!("*{f}*")}}})); - } - if let Some(s) = since { - filters.push(json!({"range": {"last_touched_at": {"gte": s.to_rfc3339()}}})); - } - let query = if filters.is_empty() { - json!({"match_all": {}}) - } else { - json!({"bool": {"filter": filters}}) - }; - let resp = self.post( - &format!("{IDX_CODE_AREAS}/_search"), - json!({"size": limit, "query": query, "sort": [{"last_touched_at": "desc"}]}), - )?; - let rows = resp - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .map(|hits| { - hits.iter() - .filter_map(|h| { - let s = h.get("_source")?; - Some(CodeArea { - id: s.get("id").and_then(|v| v.as_i64()).unwrap_or(0), - project: s.get("project")?.as_str()?.to_string(), - file_path: s.get("file_path")?.as_str()?.to_string(), - description: s - .get("description") - .and_then(|v| v.as_str()) - .map(String::from), - session_id: s - .get("session_id") - .and_then(|v| v.as_str()) - .map(String::from), - tool_name: s - .get("tool_name") - .and_then(|v| v.as_str()) - .map(String::from), - touch_count: s.get("touch_count").and_then(|v| v.as_i64()).unwrap_or(1), - first_touched_at: s - .get("first_touched_at") - .and_then(|v| v.as_str()) - .and_then(|x| DateTime::parse_from_rfc3339(x).ok()) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(Utc::now), - last_touched_at: s - .get("last_touched_at") - .and_then(|v| v.as_str()) - .and_then(|x| DateTime::parse_from_rfc3339(x).ok()) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(Utc::now), - }) - }) - .collect() - }) - .unwrap_or_default(); - Ok(rows) - } - - pub fn code_area_count(&self) -> IcmResult { - let resp = self.post(&format!("{IDX_CODE_AREAS}/_count"), json!({}))?; - Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - pub fn record_hook_event(&self, ev: &HookEventInsert) -> IcmResult { - let now = Utc::now(); - let id = now.timestamp_millis(); - let doc_id = ulid::Ulid::new().to_string(); - self.request( - "PUT", - &format!("{IDX_HOOKS}/_doc/{doc_id}"), - Some(json!({ - "id": id, - "ts": now.to_rfc3339(), - "event": ev.event, - "project": ev.project, - "session_id": ev.session_id, - "tool_name": ev.tool_name, - "duration_ms": ev.duration_ms, - "exit_code": ev.exit_code, - "payload_size": ev.payload_size, - "note": ev.note - })), - false, - )?; - Ok(id) - } - - pub fn hook_events_recent( - &self, - limit: usize, - event_filter: Option<&str>, - ) -> IcmResult> { - let query = match event_filter { - Some(e) => json!({"term": {"event": e}}), - None => json!({"match_all": {}}), - }; - let resp = self.post( - &format!("{IDX_HOOKS}/_search"), - json!({"size": limit, "query": query, "sort": [{"ts": "desc"}]}), - )?; - let rows = resp - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - .map(|hits| { - hits.iter() - .filter_map(|h| { - let s = h.get("_source")?; - Some(HookEvent { - id: s.get("id").and_then(|v| v.as_i64()).unwrap_or(0), - ts: parse_dt(s.get("ts").and_then(|v| v.as_str()).unwrap_or("")), - event: s - .get("event") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(), - project: s.get("project").and_then(|v| v.as_str()).map(String::from), - session_id: s - .get("session_id") - .and_then(|v| v.as_str()) - .map(String::from), - tool_name: s - .get("tool_name") - .and_then(|v| v.as_str()) - .map(String::from), - duration_ms: s.get("duration_ms").and_then(|v| v.as_i64()), - exit_code: s.get("exit_code").and_then(|v| v.as_i64()).unwrap_or(0) - as i32, - payload_size: s.get("payload_size").and_then(|v| v.as_i64()), - note: s.get("note").and_then(|v| v.as_str()).map(String::from), - }) - }) - .collect() - }) - .unwrap_or_default(); - Ok(rows) - } - - pub fn hook_stats(&self, since_rfc3339: &str) -> IcmResult> { - let resp = self.post( - &format!("{IDX_HOOKS}/_search"), - json!({ - "size": 0, - "query": {"range": {"ts": {"gte": since_rfc3339}}}, - "aggs": {"events": { - "terms": {"field": "event", "size": 1000}, - "aggs": { - "errs": {"filter": {"bool": {"must_not": [{"term": {"exit_code": 0}}]}}}, - "avg_dur": {"avg": {"field": "duration_ms"}}, - "pct": {"percentiles": {"field": "duration_ms", "percents": [50, 99]}} - } - }} - }), - )?; - let buckets = resp - .get("aggregations") - .and_then(|a| a.get("events")) - .and_then(|e| e.get("buckets")) - .and_then(|b| b.as_array()) - .cloned() - .unwrap_or_default(); - let mut out = Vec::new(); - for b in &buckets { - let pct = b.get("pct").and_then(|p| p.get("values")); - let p = |k: &str| { - pct.and_then(|v| v.get(k)) - .and_then(|v| v.as_f64()) - .filter(|f| f.is_finite()) - .unwrap_or(0.0) as i64 - }; - out.push(HookStatsRow { - event: b - .get("key") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(), - count: b.get("doc_count").and_then(|v| v.as_i64()).unwrap_or(0), - error_count: b - .get("errs") - .and_then(|e| e.get("doc_count")) - .and_then(|v| v.as_i64()) - .unwrap_or(0), - avg_duration_ms: b - .get("avg_dur") - .and_then(|a| a.get("value")) - .and_then(|v| v.as_f64()) - .unwrap_or(0.0), - p50_duration_ms: p("50.0"), - p99_duration_ms: p("99.0"), - }); - } - out.sort_by(|a, b| a.event.cmp(&b.event)); - Ok(out) - } - - pub fn prune_hook_events(&self, cutoff_rfc3339: &str) -> IcmResult { - let resp = self.post( - &format!( - "{IDX_HOOKS}/_delete_by_query?{}&conflicts=proceed", - self.refresh_param() - ), - json!({"query": {"range": {"ts": {"lt": cutoff_rfc3339}}}}), - )?; - Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - pub fn hook_event_count(&self) -> IcmResult { - let resp = self.post(&format!("{IDX_HOOKS}/_count"), json!({}))?; - Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) - } - - /// Auto-consolidation is not yet implemented on this backend; it is a - /// no-op (returns `false`) so the normal store path keeps working. - pub fn auto_consolidate(&self, _topic: &str, _threshold: usize) -> IcmResult { - Ok(false) - } - - /// See [`Self::auto_consolidate`]. - pub fn auto_consolidate_with_embedder( - &self, - _topic: &str, - _threshold: usize, - _embedder: Option<&dyn Embedder>, - ) -> IcmResult { - Ok(false) - } - - pub fn get_many(&self, ids: &[&str]) -> IcmResult> { - if ids.is_empty() { - return Ok(HashMap::new()); - } - let resp = self.post(&format!("{IDX_MEMORIES}/_mget"), json!({"ids": ids}))?; - let mut out = HashMap::new(); - if let Some(docs) = resp.get("docs").and_then(|d| d.as_array()) { - for d in docs { - if d.get("found").and_then(|f| f.as_bool()).unwrap_or(false) { - if let (Some(id), Some(src)) = - (d.get("_id").and_then(|v| v.as_str()), d.get("_source")) - { - out.insert(id.to_string(), Self::source_to_memory(id, src)); - } - } - } - } - Ok(out) - } - - pub fn get_by_topic_prefix(&self, topic: &str) -> IcmResult> { - let resp = self.post( - &format!("{IDX_MEMORIES}/_search"), - json!({ - "size": 500, - "query": {"prefix": {"topic.keyword": topic}}, - "sort": [{"weight": "desc"}] - }), - )?; - Ok(Self::hits_to_memories(&resp)) - } - - pub fn list_topics_with_prefix(&self, prefix: Option<&str>) -> IcmResult> { - let mut topics = self.list_topics()?; - if let Some(p) = prefix { - topics.retain(|(t, _)| t.starts_with(p)); - } - Ok(topics) - } - - /// Expand a result set with graph neighbours (related ids), applying a - /// hop discount. Pure logic over [`Self::get_many`]; identical to the - /// other backends. - pub fn expand_with_neighbors( - &self, - initial: &[(Memory, f32)], - max_neighbors: usize, - hop_discount: f32, - max_total: usize, - ) -> IcmResult> { - if max_neighbors == 0 || initial.is_empty() { - let mut out = initial.to_vec(); - out.truncate(max_total); - return Ok(out); - } - let initial_ids: HashSet = initial.iter().map(|(m, _)| m.id.clone()).collect(); - let mut candidates: Vec<(String, f32)> = Vec::new(); - let mut seen: HashSet = HashSet::new(); - for (m, score) in initial { - for rid in &m.related_ids { - if !initial_ids.contains(rid) && seen.insert(rid.clone()) { - candidates.push((rid.clone(), *score * hop_discount)); - if candidates.len() >= max_neighbors { - break; - } - } - } - if candidates.len() >= max_neighbors { - break; - } - } - - let neighbor_ids: Vec<&str> = candidates.iter().map(|(id, _)| id.as_str()).collect(); - let fetched = self.get_many(&neighbor_ids)?; - - let mut combined: Vec<(Memory, f32)> = initial.to_vec(); - for (id, score) in candidates { - if let Some(m) = fetched.get(&id) { - combined.push((m.clone(), score)); - } - } - combined.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - combined.truncate(max_total); - Ok(combined) - } - - /// Pattern mining is not implemented on this backend yet. - pub fn detect_patterns( - &self, - _topic: &str, - _min_cluster_size: usize, - ) -> IcmResult> { - Err(IcmError::Unsupported("detect_patterns".into())) - } - - /// See [`Self::detect_patterns`]. - pub fn extract_pattern_as_concept( - &self, - _cluster: &PatternCluster, - _memoir_id: &str, - ) -> IcmResult { - Err(IcmError::Unsupported("extract_pattern_as_concept".into())) - } -} - -// --------------------------------------------------------------------------- -// Subsystems not yet ported to this backend. They stay fully available on -// the default SQLite backend; here they fail cleanly with `Unsupported`. -// --------------------------------------------------------------------------- - -fn unsupported(op: &str) -> IcmResult { - Err(IcmError::Unsupported(format!( - "{op} (use the default SQLite backend)" - ))) -} - -impl MemoirStore for OpenSearchStore { - fn create_memoir(&self, _memoir: Memoir) -> IcmResult { - unsupported("memoir.create_memoir") - } - fn get_memoir(&self, _id: &str) -> IcmResult> { - unsupported("memoir.get_memoir") - } - fn get_memoir_by_name(&self, _name: &str) -> IcmResult> { - unsupported("memoir.get_memoir_by_name") - } - fn update_memoir(&self, _memoir: &Memoir) -> IcmResult<()> { - unsupported("memoir.update_memoir") - } - fn delete_memoir(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_memoir") - } - fn list_memoirs(&self) -> IcmResult> { - unsupported("memoir.list_memoirs") - } - fn add_concept(&self, _concept: Concept) -> IcmResult { - unsupported("memoir.add_concept") - } - fn get_concept(&self, _id: &str) -> IcmResult> { - unsupported("memoir.get_concept") - } - fn get_concept_by_name(&self, _memoir_id: &str, _name: &str) -> IcmResult> { - unsupported("memoir.get_concept_by_name") - } - fn update_concept(&self, _concept: &Concept) -> IcmResult<()> { - unsupported("memoir.update_concept") - } - fn delete_concept(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_concept") - } - fn list_concepts(&self, _memoir_id: &str) -> IcmResult> { - unsupported("memoir.list_concepts") - } - fn search_concepts_fts( - &self, - _memoir_id: &str, - _query: &str, - _limit: usize, - ) -> IcmResult> { - unsupported("memoir.search_concepts_fts") - } - fn search_concepts_by_label( - &self, - _memoir_id: &str, - _label: &Label, - _limit: usize, - ) -> IcmResult> { - unsupported("memoir.search_concepts_by_label") - } - fn search_all_concepts_fts(&self, _query: &str, _limit: usize) -> IcmResult> { - unsupported("memoir.search_all_concepts_fts") - } - fn refine_concept( - &self, - _id: &str, - _new_definition: &str, - _new_source_ids: &[String], - ) -> IcmResult<()> { - unsupported("memoir.refine_concept") - } - fn add_link(&self, _link: ConceptLink) -> IcmResult { - unsupported("memoir.add_link") - } - fn get_links_from(&self, _concept_id: &str) -> IcmResult> { - unsupported("memoir.get_links_from") - } - fn get_links_to(&self, _concept_id: &str) -> IcmResult> { - unsupported("memoir.get_links_to") - } - fn delete_link(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_link") - } - fn get_neighbors( - &self, - _concept_id: &str, - _relation: Option, - ) -> IcmResult> { - unsupported("memoir.get_neighbors") - } - fn get_neighborhood( - &self, - _concept_id: &str, - _depth: usize, - ) -> IcmResult<(Vec, Vec)> { - unsupported("memoir.get_neighborhood") - } - fn get_links_for_memoir(&self, _memoir_id: &str) -> IcmResult> { - unsupported("memoir.get_links_for_memoir") - } - fn memoir_stats(&self, _memoir_id: &str) -> IcmResult { - unsupported("memoir.memoir_stats") - } - fn batch_memoir_concept_counts(&self) -> IcmResult> { - unsupported("memoir.batch_memoir_concept_counts") - } -} - -impl FeedbackStore for OpenSearchStore { - fn store_feedback(&self, _feedback: Feedback) -> IcmResult { - unsupported("feedback.store_feedback") - } - fn search_feedback( - &self, - _query: &str, - _query_embedding: Option<&[f32]>, - _topic: Option<&str>, - _limit: usize, - ) -> IcmResult> { - unsupported("feedback.search_feedback") - } - fn list_feedback(&self, _topic: Option<&str>, _limit: usize) -> IcmResult> { - unsupported("feedback.list_feedback") - } - fn increment_applied(&self, _id: &str) -> IcmResult<()> { - unsupported("feedback.increment_applied") - } - fn delete_feedback(&self, _id: &str) -> IcmResult<()> { - unsupported("feedback.delete_feedback") - } - fn feedback_stats(&self) -> IcmResult { - unsupported("feedback.feedback_stats") - } -} - -impl FactsStore for OpenSearchStore { - fn set_fact( - &self, - _entity: &str, - _key: &str, - _value: &str, - _source: &str, - ) -> IcmResult { - unsupported("facts.set_fact") - } - fn get_fact(&self, _entity: &str, _key: &str) -> IcmResult> { - unsupported("facts.get_fact") - } - fn list_facts(&self, _entity: &str, _key_prefix: Option<&str>) -> IcmResult> { - unsupported("facts.list_facts") - } - fn history(&self, _entity: &str, _key: &str) -> IcmResult> { - unsupported("facts.history") - } - fn forget_fact(&self, _entity: &str, _key: &str) -> IcmResult { - unsupported("facts.forget_fact") - } - fn facts_stats(&self) -> IcmResult { - unsupported("facts.facts_stats") - } -} - -impl TranscriptStore for OpenSearchStore { - fn create_session( - &self, - _agent: &str, - _project: Option<&str>, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.create_session") - } - fn ensure_session( - &self, - _id: &str, - _agent: &str, - _project: Option<&str>, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.ensure_session") - } - fn get_session(&self, _id: &str) -> IcmResult> { - unsupported("transcript.get_session") - } - fn list_sessions(&self, _project: Option<&str>, _limit: usize) -> IcmResult> { - unsupported("transcript.list_sessions") - } - fn record_message( - &self, - _session_id: &str, - _role: Role, - _content: &str, - _tool_name: Option<&str>, - _tokens: Option, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.record_message") - } - fn list_session_messages( - &self, - _session_id: &str, - _limit: usize, - _offset: usize, - ) -> IcmResult> { - unsupported("transcript.list_session_messages") - } - fn search_transcripts( - &self, - _query: &str, - _session_id: Option<&str>, - _project: Option<&str>, - _limit: usize, - ) -> IcmResult> { - unsupported("transcript.search_transcripts") - } - fn forget_session(&self, _id: &str) -> IcmResult<()> { - unsupported("transcript.forget_session") - } - fn transcript_stats(&self) -> IcmResult { - unsupported("transcript.transcript_stats") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Audit regression: a memory id containing reserved URL characters - /// (e.g. `/`, `..`, `?`) was interpolated straight into the `_doc/{id}` - /// REST path, letting an attacker-controlled id redirect the request to - /// a different document/endpoint. - #[test] - fn test_url_encode_path_segment() { - assert_eq!( - url_encode_path_segment("abc-DEF_123.~"), - "abc-DEF_123.~", - "unreserved chars must pass through unchanged" - ); - assert_eq!(url_encode_path_segment("a/b"), "a%2Fb"); - assert_eq!(url_encode_path_segment("../secret"), "..%2Fsecret"); - assert_eq!(url_encode_path_segment("id?x=1"), "id%3Fx%3D1"); - assert_eq!(url_encode_path_segment("id#frag"), "id%23frag"); - assert_eq!(url_encode_path_segment("a b"), "a%20b"); - } - - /// Audit regression: `apply_decay`'s Painless script computed a raw - /// multiplier that goes negative for low-importance/low-access memories - /// at factor<0.5 (still inside the CLI's own validated range). This - /// mirrors the exact formula now wrapped in `Math.max(0.0, ...)`. - #[test] - fn test_apply_decay_formula_would_go_negative_without_clamp() { - let factor: f64 = 0.4; - let mult: f64 = 2.0; // low importance - let access: f64 = 0.0; - let raw = 1.0 - (1.0 - factor) * mult / (1.0 + access * 0.1); - assert!( - raw < 0.0, - "expected the pre-clamp formula to go negative, got {raw}" - ); - assert_eq!( - raw.max(0.0), - 0.0, - "Math.max(0.0, ...) must clamp this to 0.0" - ); - } -} diff --git a/crates/icm-store/src/opensearch/connection.rs b/crates/icm-store/src/opensearch/connection.rs new file mode 100644 index 00000000..b6320eb0 --- /dev/null +++ b/crates/icm-store/src/opensearch/connection.rs @@ -0,0 +1,301 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Connection setup, constructors, and index/schema migration. + +use super::*; + +impl OpenSearchStore { + pub(crate) fn conn_url() -> IcmResult { + std::env::var("ICM_OPENSEARCH_URL") + .or_else(|_| std::env::var("OPENSEARCH_URL")) + .map_err(|_| { + IcmError::Config( + "OpenSearch backend: set ICM_OPENSEARCH_URL to the cluster endpoint, \ + e.g. http://localhost:9200" + .into(), + ) + }) + } + + pub(crate) fn auth_header() -> Option { + let user = std::env::var("ICM_OPENSEARCH_USER").ok()?; + let pass = std::env::var("ICM_OPENSEARCH_PASSWORD").unwrap_or_default(); + let token = B64.encode(format!("{user}:{pass}")); + Some(format!("Basic {token}")) + } + + /// Perform a request, returning the parsed JSON body. `expected_404` + /// makes a 404 return `Ok(None)` instead of an error (used by `get`). + pub(crate) fn request( + &self, + method: &str, + path: &str, + body: Option, + allow_404: bool, + ) -> IcmResult> { + let url = format!( + "{}/{}", + self.base.trim_end_matches('/'), + path.trim_start_matches('/') + ); + let mut req = self.agent.request(method, &url); + if let Some(a) = &self.auth { + req = req.set("Authorization", a); + } + let resp = match body { + Some(b) => req.send_json(b), + None => req.call(), + }; + match resp { + Ok(r) => { + let v = r + .into_json::() + .map_err(|e| IcmError::Database(format!("opensearch decode: {e}")))?; + Ok(Some(v)) + } + Err(ureq::Error::Status(404, _)) if allow_404 => Ok(None), + Err(ureq::Error::Status(code, r)) => { + let txt = r.into_string().unwrap_or_default(); + Err(IcmError::Database(format!( + "opensearch {method} {path} -> {code}: {txt}" + ))) + } + Err(e) => Err(IcmError::Database(format!( + "opensearch {method} {path}: {e}" + ))), + } + } + + pub(crate) fn get_json(&self, path: &str) -> IcmResult> { + self.request("GET", path, None, true) + } + + pub(crate) fn post(&self, path: &str, body: Value) -> IcmResult { + self.request("POST", path, Some(body), false) + .map(|o| o.unwrap_or(Value::Null)) + } + + /// Open or create a store with the default embedding dimension. + pub fn new(_path: &Path) -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) + } + + /// Open or create a store with a specific embedding dimension. + pub fn with_dims(_path: &Path, embedding_dims: usize) -> IcmResult { + Self::connect(embedding_dims, false) + } + + /// Open the store read-only (issue #263). OpenSearch has no read-only + /// connection mode, so this just flags the store and makes mutating + /// methods error. + pub fn open_readonly(_path: &Path) -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, true) + } + + /// In-memory variant is not meaningful for a remote backend; connect + /// from the environment instead. + pub fn in_memory() -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) + } + + /// See [`Self::in_memory`]. + pub fn in_memory_with_dims(embedding_dims: usize) -> IcmResult { + Self::connect(embedding_dims, false) + } + + /// Read the stored embedding dimension without committing to a full + /// open. Returns `Ok(None)` when unreachable so callers can fall back. + pub fn read_stored_embedding_dims(_path: &Path) -> IcmResult> { + let Ok(url) = Self::conn_url() else { + return Ok(None); + }; + let agent = ureq::AgentBuilder::new() + .timeout(Duration::from_secs(10)) + .build(); + let store = OpenSearchStore { + agent, + base: url, + auth: Self::auth_header(), + embedding_dims: icm_core::DEFAULT_EMBEDDING_DIMS, + readonly: true, + }; + match store.get_metadata_int("embedding_dims") { + Ok(Some(v)) => Ok(Some(v as usize)), + _ => Ok(None), + } + } + + pub fn is_readonly(&self) -> bool { + self.readonly + } + + /// No-op on this backend (kept for API parity with the SQLite store). + pub fn ensure_vec_init() {} + + pub(crate) fn connect(requested_dims: usize, readonly: bool) -> IcmResult { + let url = Self::conn_url()?; + let agent = ureq::AgentBuilder::new() + .timeout(Duration::from_secs(30)) + .build(); + let store = OpenSearchStore { + agent, + base: url, + auth: Self::auth_header(), + embedding_dims: requested_dims, + readonly, + }; + // Probe connectivity early with a clear error. + store + .get_json("/") + .map_err(|e| IcmError::Database(format!("cannot reach OpenSearch: {e}")))?; + + // An existing database's stored dims are authoritative. + let dims = match store.get_metadata_int("embedding_dims")? { + Some(d) => d as usize, + None => requested_dims, + }; + let mut store = store; + store.embedding_dims = dims; + + if !readonly { + store.init_indices(dims)?; + store.set_metadata_int("embedding_dims", dims as i64)?; + } + Ok(store) + } + + pub(crate) fn index_exists(&self, idx: &str) -> IcmResult { + let url = format!("{}/{}", self.base.trim_end_matches('/'), idx); + let mut req = self.agent.request("HEAD", &url); + if let Some(a) = &self.auth { + req = req.set("Authorization", a); + } + match req.call() { + Ok(_) => Ok(true), + Err(ureq::Error::Status(404, _)) => Ok(false), + Err(e) => Err(IcmError::Database(format!("opensearch HEAD {idx}: {e}"))), + } + } + + pub(crate) fn create_index(&self, idx: &str, body: Value) -> IcmResult<()> { + if self.index_exists(idx)? { + return Ok(()); + } + // A racing replica may create it between the check and here; treat + // "resource_already_exists_exception" as success. + match self.request("PUT", idx, Some(body), false) { + Ok(_) => Ok(()), + Err(IcmError::Database(msg)) if msg.contains("resource_already_exists_exception") => { + Ok(()) + } + Err(e) => Err(e), + } + } + + pub(crate) fn init_indices(&self, dims: usize) -> IcmResult<()> { + if !(64..=4096).contains(&dims) { + return Err(IcmError::Config(format!( + "embedding_dims must be between 64 and 4096, got {dims}" + ))); + } + self.create_index( + IDX_MEMORIES, + json!({ + "settings": { "index": { "knn": true } }, + "mappings": { "properties": { + "created_at": {"type": "date"}, + "updated_at": {"type": "date"}, + "last_accessed": {"type": "date"}, + "access_count": {"type": "integer"}, + "weight": {"type": "float"}, + "topic": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 1024}}}, + "summary": {"type": "text"}, + "raw_excerpt": {"type": "text"}, + "keywords": {"type": "keyword"}, + "importance": {"type": "keyword"}, + "source_type": {"type": "keyword"}, + "source_data": {"type": "text", "index": false}, + "related_ids": {"type": "keyword"}, + "summary_hash": {"type": "keyword"}, + "embedding": { + "type": "knn_vector", + "dimension": dims, + "method": {"name": "hnsw", "space_type": "cosinesimil", "engine": "lucene"} + } + }} + }), + )?; + self.create_index(IDX_METADATA, json!({"mappings": {"properties": {"value": {"type": "double"}, "text_value": {"type": "keyword"}}}}))?; + self.create_index( + IDX_HOOKS, + json!({"mappings": {"properties": { + "id": {"type": "long"}, + "ts": {"type": "date"}, + "event": {"type": "keyword"}, + "project": {"type": "keyword"}, + "session_id": {"type": "keyword"}, + "tool_name": {"type": "keyword"}, + "duration_ms": {"type": "long"}, + "exit_code": {"type": "integer"}, + "payload_size": {"type": "long"}, + "note": {"type": "text"} + }}}), + )?; + self.create_index( + IDX_PENDING, + json!({"mappings": {"properties": { + "project": {"type": "keyword"}, + "tool_name": {"type": "keyword"}, + "raw_output": {"type": "text", "index": false}, + "captured_at": {"type": "date"} + }}}), + )?; + self.create_index( + IDX_CODE_AREAS, + json!({"mappings": {"properties": { + "project": {"type": "keyword"}, + "file_path": {"type": "keyword"}, + "description": {"type": "text"}, + "session_id": {"type": "keyword"}, + "tool_name": {"type": "keyword"}, + "touch_count": {"type": "long"}, + "first_touched_at": {"type": "date"}, + "last_touched_at": {"type": "date"} + }}}), + )?; + Ok(()) + } + + // metadata kv helpers + + pub(crate) fn get_metadata_int(&self, key: &str) -> IcmResult> { + let path = format!("{IDX_METADATA}/_doc/{key}"); + match self.get_json(&path)? { + Some(v) => Ok(v + .get("_source") + .and_then(|s| s.get("value")) + .and_then(|n| n.as_f64()) + .map(|f| f as i64)), + None => Ok(None), + } + } + + pub(crate) fn set_metadata_int(&self, key: &str, value: i64) -> IcmResult<()> { + let path = format!("{IDX_METADATA}/_doc/{key}?refresh=true"); + self.request("PUT", &path, Some(json!({"value": value})), false)?; + Ok(()) + } + + pub(crate) fn check_dims(&self, memory: &Memory) -> IcmResult<()> { + if let Some(emb) = memory.embedding.as_ref() { + if emb.len() != self.embedding_dims { + return Err(IcmError::InvalidInput(format!( + "embedding has {} dimensions, but this store uses {}", + emb.len(), + self.embedding_dims + ))); + } + } + Ok(()) + } +} diff --git a/crates/icm-store/src/opensearch/facts.rs b/crates/icm-store/src/opensearch/facts.rs new file mode 100644 index 00000000..ac636535 --- /dev/null +++ b/crates/icm-store/src/opensearch/facts.rs @@ -0,0 +1,32 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl FactsStore for OpenSearchStore { + fn set_fact( + &self, + _entity: &str, + _key: &str, + _value: &str, + _source: &str, + ) -> IcmResult { + unsupported("facts.set_fact") + } + fn get_fact(&self, _entity: &str, _key: &str) -> IcmResult> { + unsupported("facts.get_fact") + } + fn list_facts(&self, _entity: &str, _key_prefix: Option<&str>) -> IcmResult> { + unsupported("facts.list_facts") + } + fn history(&self, _entity: &str, _key: &str) -> IcmResult> { + unsupported("facts.history") + } + fn forget_fact(&self, _entity: &str, _key: &str) -> IcmResult { + unsupported("facts.forget_fact") + } + fn facts_stats(&self) -> IcmResult { + unsupported("facts.facts_stats") + } +} diff --git a/crates/icm-store/src/opensearch/feedback.rs b/crates/icm-store/src/opensearch/feedback.rs new file mode 100644 index 00000000..0583e963 --- /dev/null +++ b/crates/icm-store/src/opensearch/feedback.rs @@ -0,0 +1,32 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl FeedbackStore for OpenSearchStore { + fn store_feedback(&self, _feedback: Feedback) -> IcmResult { + unsupported("feedback.store_feedback") + } + fn search_feedback( + &self, + _query: &str, + _query_embedding: Option<&[f32]>, + _topic: Option<&str>, + _limit: usize, + ) -> IcmResult> { + unsupported("feedback.search_feedback") + } + fn list_feedback(&self, _topic: Option<&str>, _limit: usize) -> IcmResult> { + unsupported("feedback.list_feedback") + } + fn increment_applied(&self, _id: &str) -> IcmResult<()> { + unsupported("feedback.increment_applied") + } + fn delete_feedback(&self, _id: &str) -> IcmResult<()> { + unsupported("feedback.delete_feedback") + } + fn feedback_stats(&self) -> IcmResult { + unsupported("feedback.feedback_stats") + } +} diff --git a/crates/icm-store/src/opensearch/hooks.rs b/crates/icm-store/src/opensearch/hooks.rs new file mode 100644 index 00000000..be98f55a --- /dev/null +++ b/crates/icm-store/src/opensearch/hooks.rs @@ -0,0 +1,366 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Hook telemetry, the extraction queue, and code areas. + +use super::*; + +impl OpenSearchStore { + pub fn increment_hook_counter(&self) -> IcmResult { + let resp = self.post( + &format!("{IDX_METADATA}/_update/hook_counter?_source=true"), + json!({ + "scripted_upsert": true, + "upsert": {}, + "script": { + "lang": "painless", + "source": "ctx._source.value = (ctx._source.value == null ? 1 : ctx._source.value + 1);" + } + }), + )?; + Ok(resp + .get("get") + .and_then(|g| g.get("_source")) + .and_then(|s| s.get("value")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0) as usize) + } + + pub fn reset_hook_counter(&self) -> IcmResult<()> { + self.set_metadata_int("hook_counter", 0) + } + + pub fn enqueue_pending_extraction( + &self, + project: &str, + tool_name: &str, + raw_output: &str, + ) -> IcmResult { + let id = ulid::Ulid::new().to_string(); + self.request( + "PUT", + &format!("{IDX_PENDING}/_doc/{id}?{}", self.refresh_param()), + Some(json!({ + "project": project, + "tool_name": tool_name, + "raw_output": raw_output, + "captured_at": Utc::now().to_rfc3339() + })), + false, + )?; + Ok(id) + } + + pub fn list_pending_extractions(&self, limit: usize) -> IcmResult> { + let resp = self.post( + &format!("{IDX_PENDING}/_search"), + json!({"size": limit, "query": {"match_all": {}}, "sort": [{"captured_at": "asc"}]}), + )?; + let rows = resp + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + .map(|hits| { + hits.iter() + .filter_map(|h| { + let id = h.get("_id")?.as_str()?.to_string(); + let s = h.get("_source")?; + Some(( + id, + s.get("project")?.as_str()?.to_string(), + s.get("tool_name")?.as_str()?.to_string(), + s.get("raw_output")?.as_str()?.to_string(), + s.get("captured_at")?.as_str()?.to_string(), + )) + }) + .collect() + }) + .unwrap_or_default(); + Ok(rows) + } + + pub fn delete_pending_extractions(&self, ids: &[String]) -> IcmResult { + if ids.is_empty() { + return Ok(0); + } + let resp = self.post( + &format!( + "{IDX_PENDING}/_delete_by_query?{}&conflicts=proceed", + self.refresh_param() + ), + json!({"query": {"ids": {"values": ids}}}), + )?; + Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + pub fn pending_extraction_count(&self) -> IcmResult { + let resp = self.post(&format!("{IDX_PENDING}/_count"), json!({}))?; + Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + pub fn upsert_code_area( + &self, + project: &str, + file_path: &str, + description: Option<&str>, + session_id: Option<&str>, + tool_name: Option<&str>, + ) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("upsert_code_area".into())); + } + let ts = Utc::now(); + let now = ts.to_rfc3339(); + let id = ts.timestamp_millis(); + // Deterministic id makes the same (project, file_path) a single row. + let key = B64.encode(format!("{project}\0{file_path}")); + self.request( + "POST", + &format!("{IDX_CODE_AREAS}/_update/{key}?{}", self.refresh_param()), + Some(json!({ + "scripted_upsert": true, + "upsert": { + "id": id, + "project": project, + "file_path": file_path, + "description": description, + "session_id": session_id, + "tool_name": tool_name, + "touch_count": 1, + "first_touched_at": now, + "last_touched_at": now + }, + "script": { + "lang": "painless", + "source": "ctx._source.touch_count = (ctx._source.touch_count == null ? 1 : ctx._source.touch_count + 1); ctx._source.last_touched_at = params.now; if (params.description != null) ctx._source.description = params.description; if (params.session_id != null) ctx._source.session_id = params.session_id; if (params.tool_name != null) ctx._source.tool_name = params.tool_name;", + "params": {"now": now, "description": description, "session_id": session_id, "tool_name": tool_name} + } + })), + false, + )?; + Ok(()) + } + + pub fn list_code_areas( + &self, + project: Option<&str>, + in_file: Option<&str>, + since: Option>, + limit: usize, + ) -> IcmResult> { + let mut filters: Vec = Vec::new(); + if let Some(p) = project { + filters.push(json!({"term": {"project": p}})); + } + if let Some(f) = in_file { + filters.push(json!({"wildcard": {"file_path": {"value": format!("*{f}*")}}})); + } + if let Some(s) = since { + filters.push(json!({"range": {"last_touched_at": {"gte": s.to_rfc3339()}}})); + } + let query = if filters.is_empty() { + json!({"match_all": {}}) + } else { + json!({"bool": {"filter": filters}}) + }; + let resp = self.post( + &format!("{IDX_CODE_AREAS}/_search"), + json!({"size": limit, "query": query, "sort": [{"last_touched_at": "desc"}]}), + )?; + let rows = resp + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + .map(|hits| { + hits.iter() + .filter_map(|h| { + let s = h.get("_source")?; + Some(CodeArea { + id: s.get("id").and_then(|v| v.as_i64()).unwrap_or(0), + project: s.get("project")?.as_str()?.to_string(), + file_path: s.get("file_path")?.as_str()?.to_string(), + description: s + .get("description") + .and_then(|v| v.as_str()) + .map(String::from), + session_id: s + .get("session_id") + .and_then(|v| v.as_str()) + .map(String::from), + tool_name: s + .get("tool_name") + .and_then(|v| v.as_str()) + .map(String::from), + touch_count: s.get("touch_count").and_then(|v| v.as_i64()).unwrap_or(1), + first_touched_at: s + .get("first_touched_at") + .and_then(|v| v.as_str()) + .and_then(|x| DateTime::parse_from_rfc3339(x).ok()) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(Utc::now), + last_touched_at: s + .get("last_touched_at") + .and_then(|v| v.as_str()) + .and_then(|x| DateTime::parse_from_rfc3339(x).ok()) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(Utc::now), + }) + }) + .collect() + }) + .unwrap_or_default(); + Ok(rows) + } + + pub fn code_area_count(&self) -> IcmResult { + let resp = self.post(&format!("{IDX_CODE_AREAS}/_count"), json!({}))?; + Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + pub fn record_hook_event(&self, ev: &HookEventInsert) -> IcmResult { + let now = Utc::now(); + let id = now.timestamp_millis(); + let doc_id = ulid::Ulid::new().to_string(); + self.request( + "PUT", + &format!("{IDX_HOOKS}/_doc/{doc_id}"), + Some(json!({ + "id": id, + "ts": now.to_rfc3339(), + "event": ev.event, + "project": ev.project, + "session_id": ev.session_id, + "tool_name": ev.tool_name, + "duration_ms": ev.duration_ms, + "exit_code": ev.exit_code, + "payload_size": ev.payload_size, + "note": ev.note + })), + false, + )?; + Ok(id) + } + + pub fn hook_events_recent( + &self, + limit: usize, + event_filter: Option<&str>, + ) -> IcmResult> { + let query = match event_filter { + Some(e) => json!({"term": {"event": e}}), + None => json!({"match_all": {}}), + }; + let resp = self.post( + &format!("{IDX_HOOKS}/_search"), + json!({"size": limit, "query": query, "sort": [{"ts": "desc"}]}), + )?; + let rows = resp + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + .map(|hits| { + hits.iter() + .filter_map(|h| { + let s = h.get("_source")?; + Some(HookEvent { + id: s.get("id").and_then(|v| v.as_i64()).unwrap_or(0), + ts: parse_dt(s.get("ts").and_then(|v| v.as_str()).unwrap_or("")), + event: s + .get("event") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + project: s.get("project").and_then(|v| v.as_str()).map(String::from), + session_id: s + .get("session_id") + .and_then(|v| v.as_str()) + .map(String::from), + tool_name: s + .get("tool_name") + .and_then(|v| v.as_str()) + .map(String::from), + duration_ms: s.get("duration_ms").and_then(|v| v.as_i64()), + exit_code: s.get("exit_code").and_then(|v| v.as_i64()).unwrap_or(0) + as i32, + payload_size: s.get("payload_size").and_then(|v| v.as_i64()), + note: s.get("note").and_then(|v| v.as_str()).map(String::from), + }) + }) + .collect() + }) + .unwrap_or_default(); + Ok(rows) + } + + pub fn hook_stats(&self, since_rfc3339: &str) -> IcmResult> { + let resp = self.post( + &format!("{IDX_HOOKS}/_search"), + json!({ + "size": 0, + "query": {"range": {"ts": {"gte": since_rfc3339}}}, + "aggs": {"events": { + "terms": {"field": "event", "size": 1000}, + "aggs": { + "errs": {"filter": {"bool": {"must_not": [{"term": {"exit_code": 0}}]}}}, + "avg_dur": {"avg": {"field": "duration_ms"}}, + "pct": {"percentiles": {"field": "duration_ms", "percents": [50, 99]}} + } + }} + }), + )?; + let buckets = resp + .get("aggregations") + .and_then(|a| a.get("events")) + .and_then(|e| e.get("buckets")) + .and_then(|b| b.as_array()) + .cloned() + .unwrap_or_default(); + let mut out = Vec::new(); + for b in &buckets { + let pct = b.get("pct").and_then(|p| p.get("values")); + let p = |k: &str| { + pct.and_then(|v| v.get(k)) + .and_then(|v| v.as_f64()) + .filter(|f| f.is_finite()) + .unwrap_or(0.0) as i64 + }; + out.push(HookStatsRow { + event: b + .get("key") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + count: b.get("doc_count").and_then(|v| v.as_i64()).unwrap_or(0), + error_count: b + .get("errs") + .and_then(|e| e.get("doc_count")) + .and_then(|v| v.as_i64()) + .unwrap_or(0), + avg_duration_ms: b + .get("avg_dur") + .and_then(|a| a.get("value")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0), + p50_duration_ms: p("50.0"), + p99_duration_ms: p("99.0"), + }); + } + out.sort_by(|a, b| a.event.cmp(&b.event)); + Ok(out) + } + + pub fn prune_hook_events(&self, cutoff_rfc3339: &str) -> IcmResult { + let resp = self.post( + &format!( + "{IDX_HOOKS}/_delete_by_query?{}&conflicts=proceed", + self.refresh_param() + ), + json!({"query": {"range": {"ts": {"lt": cutoff_rfc3339}}}}), + )?; + Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + pub fn hook_event_count(&self) -> IcmResult { + let resp = self.post(&format!("{IDX_HOOKS}/_count"), json!({}))?; + Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } +} diff --git a/crates/icm-store/src/opensearch/maintenance.rs b/crates/icm-store/src/opensearch/maintenance.rs new file mode 100644 index 00000000..75ecc79e --- /dev/null +++ b/crates/icm-store/src/opensearch/maintenance.rs @@ -0,0 +1,56 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Decay/auto-consolidate maintenance methods. + +use super::*; + +// Inherent methods used by the cli/mcp store/recall/hook path + +impl OpenSearchStore { + pub fn maybe_auto_decay(&self) -> IcmResult<()> { + if self.readonly { + return Ok(()); + } + // Atomic-ish claim via a scripted upsert on a metadata doc: only the + // caller that flips `changed` to true runs the decay. + let now_ms = Utc::now().timestamp_millis(); + let resp = self.post( + &format!("{IDX_METADATA}/_update/last_decay_at?{}&_source=true", self.refresh_param()), + json!({ + "scripted_upsert": true, + "upsert": {}, + "script": { + "lang": "painless", + "source": "if (ctx._source.value == null || params.now - ctx._source.value >= 86400000L) { ctx._source.value = params.now; ctx._source.changed = true; } else { ctx._source.changed = false; }", + "params": {"now": now_ms} + } + }), + )?; + let changed = resp + .get("get") + .and_then(|g| g.get("_source")) + .and_then(|s| s.get("changed")) + .and_then(|c| c.as_bool()) + .unwrap_or(false); + if changed { + self.apply_decay(0.95)?; + } + Ok(()) + } + + /// Auto-consolidation is not yet implemented on this backend; it is a + /// no-op (returns `false`) so the normal store path keeps working. + pub fn auto_consolidate(&self, _topic: &str, _threshold: usize) -> IcmResult { + Ok(false) + } + + /// See [`Self::auto_consolidate`]. + pub fn auto_consolidate_with_embedder( + &self, + _topic: &str, + _threshold: usize, + _embedder: Option<&dyn Embedder>, + ) -> IcmResult { + Ok(false) + } +} diff --git a/crates/icm-store/src/opensearch/memoir.rs b/crates/icm-store/src/opensearch/memoir.rs new file mode 100644 index 00000000..809ca5c5 --- /dev/null +++ b/crates/icm-store/src/opensearch/memoir.rs @@ -0,0 +1,106 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl MemoirStore for OpenSearchStore { + fn create_memoir(&self, _memoir: Memoir) -> IcmResult { + unsupported("memoir.create_memoir") + } + fn get_memoir(&self, _id: &str) -> IcmResult> { + unsupported("memoir.get_memoir") + } + fn get_memoir_by_name(&self, _name: &str) -> IcmResult> { + unsupported("memoir.get_memoir_by_name") + } + fn update_memoir(&self, _memoir: &Memoir) -> IcmResult<()> { + unsupported("memoir.update_memoir") + } + fn delete_memoir(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_memoir") + } + fn list_memoirs(&self) -> IcmResult> { + unsupported("memoir.list_memoirs") + } + fn add_concept(&self, _concept: Concept) -> IcmResult { + unsupported("memoir.add_concept") + } + fn get_concept(&self, _id: &str) -> IcmResult> { + unsupported("memoir.get_concept") + } + fn get_concept_by_name(&self, _memoir_id: &str, _name: &str) -> IcmResult> { + unsupported("memoir.get_concept_by_name") + } + fn update_concept(&self, _concept: &Concept) -> IcmResult<()> { + unsupported("memoir.update_concept") + } + fn delete_concept(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_concept") + } + fn list_concepts(&self, _memoir_id: &str) -> IcmResult> { + unsupported("memoir.list_concepts") + } + fn search_concepts_fts( + &self, + _memoir_id: &str, + _query: &str, + _limit: usize, + ) -> IcmResult> { + unsupported("memoir.search_concepts_fts") + } + fn search_concepts_by_label( + &self, + _memoir_id: &str, + _label: &Label, + _limit: usize, + ) -> IcmResult> { + unsupported("memoir.search_concepts_by_label") + } + fn search_all_concepts_fts(&self, _query: &str, _limit: usize) -> IcmResult> { + unsupported("memoir.search_all_concepts_fts") + } + fn refine_concept( + &self, + _id: &str, + _new_definition: &str, + _new_source_ids: &[String], + ) -> IcmResult<()> { + unsupported("memoir.refine_concept") + } + fn add_link(&self, _link: ConceptLink) -> IcmResult { + unsupported("memoir.add_link") + } + fn get_links_from(&self, _concept_id: &str) -> IcmResult> { + unsupported("memoir.get_links_from") + } + fn get_links_to(&self, _concept_id: &str) -> IcmResult> { + unsupported("memoir.get_links_to") + } + fn delete_link(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_link") + } + fn get_neighbors( + &self, + _concept_id: &str, + _relation: Option, + ) -> IcmResult> { + unsupported("memoir.get_neighbors") + } + fn get_neighborhood( + &self, + _concept_id: &str, + _depth: usize, + ) -> IcmResult<(Vec, Vec)> { + unsupported("memoir.get_neighborhood") + } + fn get_links_for_memoir(&self, _memoir_id: &str) -> IcmResult> { + unsupported("memoir.get_links_for_memoir") + } + fn memoir_stats(&self, _memoir_id: &str) -> IcmResult { + unsupported("memoir.memoir_stats") + } + fn batch_memoir_concept_counts(&self) -> IcmResult> { + unsupported("memoir.batch_memoir_concept_counts") + } +} diff --git a/crates/icm-store/src/opensearch/memory.rs b/crates/icm-store/src/opensearch/memory.rs new file mode 100644 index 00000000..e75a6bc1 --- /dev/null +++ b/crates/icm-store/src/opensearch/memory.rs @@ -0,0 +1,658 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. + +use super::*; + +impl OpenSearchStore { + pub(crate) fn refresh_param(&self) -> &'static str { + // Force a refresh so writes are immediately visible to subsequent + // searches (dedup, counts, the multi-replica path). ICM writes are + // low-frequency curated memories, so the cost is acceptable. + "refresh=true" + } + + pub(crate) fn store_inner(&self, memory: &Memory) -> IcmResult { + let hash = summary_hash(&memory.topic, &memory.summary); + // Dedup: an existing memory with the same (topic, summary_hash) + // wins; merge importance (max) + keywords (union) + raw_excerpt + // (prefer new) and return the existing id. + // + // Audit finding: this used to ALSO filter on `topic.keyword` (an + // exact-byte-match `keyword` field, no normalizer) alongside + // `summary_hash` — but `summary_hash` already encodes the topic via + // Rust's Unicode-correct `to_lowercase()`, so storing topic="Kexa" + // then topic="kexa" with the same summary hashed identically but + // failed the exact `topic.keyword` filter, silently creating a + // second document instead of deduping (broader than the SQLite/ + // Postgres accented-topic case — this fires on ANY case + // difference). `summary_hash` alone is sufficient, matching the + // SQLite/Postgres fix. + let existing = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": 1, + "query": {"bool": {"filter": [ + {"term": {"summary_hash": hash}} + ]}} + }), + )?; + if let Some(hit) = existing + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + .and_then(|a| a.first()) + { + let existing_id = hit + .get("_id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let src = hit.get("_source").cloned().unwrap_or(Value::Null); + let existing_importance: Importance = src + .get("importance") + .and_then(|v| v.as_str()) + .unwrap_or("medium") + .parse() + .unwrap_or(Importance::Medium); + let merged_importance = max_importance(existing_importance, memory.importance); + let mut merged_keywords: Vec = src + .get("keywords") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + for kw in &memory.keywords { + if !merged_keywords.contains(kw) { + merged_keywords.push(kw.clone()); + } + } + let raw = memory.raw_excerpt.clone().or_else(|| { + src.get("raw_excerpt") + .and_then(|v| v.as_str()) + .map(String::from) + }); + self.request( + "POST", + &format!( + "{IDX_MEMORIES}/_update/{existing_id}?{}", + self.refresh_param() + ), + Some(json!({"doc": { + "importance": merged_importance.to_string(), + "keywords": merged_keywords, + "raw_excerpt": raw, + "updated_at": Utc::now().to_rfc3339(), + }})), + false, + )?; + return Ok(existing_id); + } + + self.request( + "PUT", + &format!( + "{IDX_MEMORIES}/_doc/{}?{}", + url_encode_path_segment(&memory.id), + self.refresh_param() + ), + Some(Self::memory_to_source(memory)), + false, + )?; + Ok(memory.id.clone()) + } +} + +impl MemoryStore for OpenSearchStore { + fn store(&self, memory: Memory) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("store".into())); + } + let memory = validate_and_normalize(memory)?; + self.check_dims(&memory)?; + self.store_inner(&memory) + } + + fn get(&self, id: &str) -> IcmResult> { + let path = format!("{IDX_MEMORIES}/_doc/{}", url_encode_path_segment(id)); + match self.get_json(&path)? { + Some(v) => { + if v.get("found").and_then(|f| f.as_bool()).unwrap_or(false) { + let src = v.get("_source").cloned().unwrap_or(Value::Null); + Ok(Some(Self::source_to_memory(id, &src))) + } else { + Ok(None) + } + } + None => Ok(None), + } + } + + fn update(&self, memory: &Memory) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("update".into())); + } + self.check_dims(memory)?; + let mut doc = Self::memory_to_source(memory); + doc["updated_at"] = json!(Utc::now().to_rfc3339()); + // Replace the document wholesale (index by id). + self.request( + "PUT", + &format!( + "{IDX_MEMORIES}/_doc/{}?{}", + url_encode_path_segment(&memory.id), + self.refresh_param() + ), + Some(doc), + false, + )?; + Ok(()) + } + + fn delete(&self, id: &str) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("delete".into())); + } + self.request( + "DELETE", + &format!( + "{IDX_MEMORIES}/_doc/{}?{}", + url_encode_path_segment(id), + self.refresh_param() + ), + None, + true, + )?; + + // Manual-testing finding (same class as the SQLite/Postgres + // backends): a deleted memory otherwise stays as a dangling entry + // in every other memory's `related_ids` forever. `related_ids` is + // mapped as a `keyword` array field, so a term query finds every + // document that references it and a Painless script strips it out + // in place. Best-effort: a failure here doesn't roll back the + // delete above (OpenSearch has no cross-document transaction to + // roll back into) — surfacing an error would make a successful + // delete look like it failed, so log and move on. + if let Err(e) = self.post( + &format!( + "{IDX_MEMORIES}/_update_by_query?conflicts=proceed&{}", + self.refresh_param() + ), + json!({ + "script": { + "source": "ctx._source.related_ids.removeIf(x -> x == params.deleted_id)", + "params": {"deleted_id": id} + }, + "query": {"term": {"related_ids": id}} + }), + ) { + tracing::warn!(error = %e, id, "failed to clean up dangling related_ids after delete"); + } + + Ok(()) + } + + fn search_by_keywords(&self, keywords: &[&str], limit: usize) -> IcmResult> { + if keywords.is_empty() { + return Ok(Vec::new()); + } + // Audit finding: unlike Postgres/SQLite, `limit` was never clamped + // here — a caller-supplied limit above OpenSearch's own + // `index.max_result_window` (default 10,000) returns a hard 400 + // error instead of gracefully truncating like the other backends. + let limit = limit.min(100); + let joined = keywords.join(" "); + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": limit, + "query": {"bool": {"should": [ + {"terms": {"keywords": keywords}}, + {"multi_match": {"query": joined, "fields": ["summary", "topic"]}} + ], "minimum_should_match": 1}} + }), + )?; + Ok(Self::hits_to_memories(&resp)) + } + + fn search_fts(&self, query: &str, limit: usize) -> IcmResult> { + if query.trim().is_empty() { + return Ok(Vec::new()); + } + let limit = limit.min(100); + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": limit, + "query": {"multi_match": { + "query": query, + "fields": ["summary^2", "topic", "keywords"] + }} + }), + )?; + Ok(Self::hits_to_memories(&resp)) + } + + fn search_by_embedding( + &self, + embedding: &[f32], + limit: usize, + ) -> IcmResult> { + let limit = limit.min(1000); + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": limit, + "query": {"knn": {"embedding": {"vector": embedding, "k": limit}}} + }), + )?; + Ok(Self::hits_to_scored(&resp)) + } + + fn search_hybrid( + &self, + query: &str, + embedding: &[f32], + limit: usize, + ) -> IcmResult> { + let limit = limit.min(1000); + let pool = limit * 4; + + // FTS candidates (BM25). + let mut fts_scores: HashMap = HashMap::new(); + let mut memories: HashMap = HashMap::new(); + if !query.trim().is_empty() { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": pool, + "query": {"multi_match": {"query": query, "fields": ["summary^2", "topic", "keywords"]}} + }), + )?; + for (m, s) in Self::hits_to_scored(&resp) { + fts_scores.insert(m.id.clone(), s); + memories.insert(m.id.clone(), m); + } + } + + // Vector candidates. + let mut vec_scores: HashMap = HashMap::new(); + for (m, s) in self.search_by_embedding(embedding, pool)? { + vec_scores.insert(m.id.clone(), s); + memories.entry(m.id.clone()).or_insert(m); + } + + // Min-max normalize each score family to [0, 1] before blending. + let norm = |scores: &HashMap| -> HashMap { + if scores.is_empty() { + return HashMap::new(); + } + let (mut lo, mut hi) = (f32::MAX, f32::MIN); + for &v in scores.values() { + lo = lo.min(v); + hi = hi.max(v); + } + let span = (hi - lo).max(f32::EPSILON); + scores + .iter() + .map(|(k, v)| (k.clone(), (v - lo) / span)) + .collect() + }; + let fts_n = norm(&fts_scores); + let vec_n = norm(&vec_scores); + + let mut scored: Vec<(String, f32)> = memories + .keys() + .map(|id| { + let f = fts_n.get(id).copied().unwrap_or(0.0); + let v = vec_n.get(id).copied().unwrap_or(0.0); + (id.clone(), 0.3 * f + 0.7 * v) + }) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + scored.truncate(limit); + + Ok(scored + .into_iter() + .filter_map(|(id, s)| memories.remove(&id).map(|m| (m, s))) + .collect()) + } + + fn update_access(&self, id: &str) -> IcmResult<()> { + if self.readonly { + return Ok(()); + } + // Best-effort; a missing doc is not an error for recall bookkeeping. + let _ = self.request( + "POST", + &format!("{IDX_MEMORIES}/_update/{id}"), + Some(json!({ + "script": { + "lang": "painless", + "source": "ctx._source.access_count = (ctx._source.access_count == null ? 1 : ctx._source.access_count + 1); ctx._source.last_accessed = params.now;", + "params": {"now": Utc::now().to_rfc3339()} + } + })), + true, + )?; + Ok(()) + } + + fn batch_update_access(&self, ids: &[&str]) -> IcmResult { + if self.readonly || ids.is_empty() { + return Ok(0); + } + let resp = self.post( + &format!("{IDX_MEMORIES}/_update_by_query?{}&conflicts=proceed", self.refresh_param()), + json!({ + "query": {"ids": {"values": ids}}, + "script": { + "lang": "painless", + "source": "ctx._source.access_count = (ctx._source.access_count == null ? 1 : ctx._source.access_count + 1); ctx._source.last_accessed = params.now;", + "params": {"now": Utc::now().to_rfc3339()} + } + }), + )?; + Ok(resp.get("updated").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + fn apply_decay(&self, decay_factor: f32) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("decay".into())); + } + let resp = self.post( + &format!("{IDX_MEMORIES}/_update_by_query?{}&conflicts=proceed", self.refresh_param()), + json!({ + "query": {"bool": {"must_not": [{"term": {"importance": "critical"}}]}}, + // Audit finding: for `low` importance with low access count, + // the raw multiplier goes negative once decay_factor < 0.5 + // (still inside the CLI's own validated [0.0, 1.0) range) — + // same bug already fixed for SQLite/Postgres. Math.max is + // Painless's equivalent clamp. + "script": { + "lang": "painless", + "source": "double f = params.factor; String imp = ctx._source.importance; double mult = imp != null && imp.equals('high') ? 0.5 : (imp != null && imp.equals('low') ? 2.0 : 1.0); double ac = ctx._source.access_count == null ? 0 : ctx._source.access_count; if (ac > 5) ac = 5; double m = Math.max(0.0, 1.0 - (1.0 - f) * mult / (1.0 + ac * 0.1)); ctx._source.weight = ctx._source.weight * m;", + "params": {"factor": decay_factor as f64} + } + }), + )?; + Ok(resp.get("updated").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + fn prune(&self, weight_threshold: f32) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("prune".into())); + } + let resp = self.post( + &format!( + "{IDX_MEMORIES}/_delete_by_query?{}&conflicts=proceed", + self.refresh_param() + ), + json!({ + "query": {"bool": { + "must": [{"range": {"weight": {"lt": weight_threshold as f64}}}], + "must_not": [{"terms": {"importance": ["critical", "high"]}}] + }} + }), + )?; + Ok(resp.get("deleted").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + fn get_by_topic(&self, topic: &str) -> IcmResult> { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": 500, + "query": {"term": {"topic.keyword": topic}}, + "sort": [{"weight": "desc"}] + }), + )?; + Ok(Self::hits_to_memories(&resp)) + } + + fn list_all(&self) -> IcmResult> { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({"size": 10000, "query": {"match_all": {}}, "sort": [{"weight": "desc"}]}), + )?; + Ok(Self::hits_to_memories(&resp)) + } + + fn list_topics(&self) -> IcmResult> { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({"size": 0, "aggs": {"topics": {"terms": {"field": "topic.keyword", "size": 10000}}}}), + )?; + let mut out = bucket_counts(&resp, "topics"); + out.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(out) + } + + fn consolidate_topic(&self, topic: &str, consolidated: Memory) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("consolidate".into())); + } + // Audit findings, both fixed together: + // 1. `critical` memories are never deleted — same contract + // apply_decay/prune already honor, and the same fix already + // applied to SQLite/Postgres consolidate_topic. This delete + // query previously wiped critical memories in the topic too. + // 2. Still not atomic (no multi-document transaction in + // OpenSearch), but reordered to insert-then-delete: if + // `store_inner` fails (dimension mismatch, network blip, + // cluster unavailable), the originals are left untouched + // instead of being gone with no replacement ever written. The + // failure mode is now "harmless duplication a retry fixes", + // not data loss — the consolidated memory gets a fresh id, so + // it can't collide with any original. + let consolidated = validate_and_normalize(consolidated)?; + self.check_dims(&consolidated)?; + // Manual-testing finding: captured before store_inner/delete below + // (the deleted rows are gone afterward), so any *other* memory's + // related_ids pointing at them can be cleaned up — same dangling- + // reference bug already fixed for the single-id `delete`. + let deleted_ids: Vec = self + .get_by_topic(topic)? + .into_iter() + .filter(|m| m.importance != Importance::Critical) + .map(|m| m.id) + .collect(); + // `store_inner` returns the id actually used — the caller's fresh + // id on a normal insert, or an existing row's id if this exact + // (topic, summary_hash) happened to already exist (dedup merge). + // Either way it now has the SAME topic as the memories being + // consolidated, so it must be excluded from the delete below or it + // would delete the very memory it just wrote/merged into. + let consolidated_id = self.store_inner(&consolidated)?; + self.post( + &format!( + "{IDX_MEMORIES}/_delete_by_query?{}&conflicts=proceed", + self.refresh_param() + ), + json!({"query": {"bool": { + "must": [{"term": {"topic.keyword": topic}}], + "must_not": [ + {"term": {"importance": "critical"}}, + {"ids": {"values": [consolidated_id]}} + ] + }}}), + )?; + + if !deleted_ids.is_empty() { + if let Err(e) = self.post( + &format!( + "{IDX_MEMORIES}/_update_by_query?conflicts=proceed&{}", + self.refresh_param() + ), + json!({ + "script": { + "source": "ctx._source.related_ids.removeIf(x -> params.deleted_ids.contains(x))", + "params": {"deleted_ids": deleted_ids} + }, + "query": {"terms": {"related_ids": deleted_ids}} + }), + ) { + tracing::warn!(topic, error = %e, "consolidate_topic: failed to clean up dangling related_ids"); + } + } + + Ok(()) + } + + fn count(&self) -> IcmResult { + let resp = self.post(&format!("{IDX_MEMORIES}/_count"), json!({}))?; + Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + fn count_by_topic(&self, topic: &str) -> IcmResult { + let resp = self.post( + &format!("{IDX_MEMORIES}/_count"), + json!({"query": {"term": {"topic.keyword": topic}}}), + )?; + Ok(resp.get("count").and_then(|v| v.as_u64()).unwrap_or(0) as usize) + } + + fn stats(&self) -> IcmResult { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": 0, + "track_total_hits": true, + "aggs": { + "avg_w": {"avg": {"field": "weight"}}, + "topics": {"cardinality": {"field": "topic.keyword"}}, + "oldest": {"min": {"field": "created_at", "format": "date_time"}}, + "newest": {"max": {"field": "created_at", "format": "date_time"}} + } + }), + )?; + let total = resp + .get("hits") + .and_then(|h| h.get("total")) + .and_then(|t| t.get("value")) + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + let aggs = resp.get("aggregations").cloned().unwrap_or(Value::Null); + let avg_weight = aggs + .get("avg_w") + .and_then(|a| a.get("value")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0) as f32; + let total_topics = aggs + .get("topics") + .and_then(|a| a.get("value")) + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + let parse_agg_date = |name: &str| -> Option> { + aggs.get(name) + .and_then(|a| a.get("value_as_string")) + .and_then(|v| v.as_str()) + .map(parse_dt) + }; + Ok(StoreStats { + total_memories: total, + total_topics, + avg_weight, + oldest_memory: parse_agg_date("oldest"), + newest_memory: parse_agg_date("newest"), + }) + } + + fn topic_health(&self, topic: &str) -> IcmResult { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": 0, + "track_total_hits": true, + "query": {"term": {"topic.keyword": topic}}, + "aggs": { + "avg_w": {"avg": {"field": "weight"}}, + "avg_ac": {"avg": {"field": "access_count"}}, + "oldest": {"min": {"field": "created_at"}}, + "newest": {"max": {"field": "created_at"}}, + "last_acc": {"max": {"field": "last_accessed"}}, + "stale": {"filter": {"bool": {"must": [ + {"range": {"weight": {"lt": 0.5}}}, + {"range": {"last_accessed": {"lt": "now-14d"}}} + ]}}} + } + }), + )?; + let entry_count = resp + .get("hits") + .and_then(|h| h.get("total")) + .and_then(|t| t.get("value")) + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + if entry_count == 0 { + return Err(IcmError::NotFound(format!( + "no memories in topic '{topic}'" + ))); + } + let aggs = resp.get("aggregations").cloned().unwrap_or(Value::Null); + let avg_weight = aggs + .get("avg_w") + .and_then(|a| a.get("value")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0) as f32; + let avg_access_count = aggs + .get("avg_ac") + .and_then(|a| a.get("value")) + .and_then(|v| v.as_f64()) + .unwrap_or(0.0) as f32; + let stale_count = aggs + .get("stale") + .and_then(|a| a.get("doc_count")) + .and_then(|v| v.as_u64()) + .unwrap_or(0) as usize; + Ok(TopicHealth { + topic: topic.to_string(), + entry_count, + avg_weight, + avg_access_count, + oldest: agg_date(&aggs, "oldest"), + newest: agg_date(&aggs, "newest"), + last_accessed: agg_date(&aggs, "last_acc"), + stale_count, + needs_consolidation: entry_count > 5, + }) + } +} + +/// Read a `min`/`max` date aggregation into a `DateTime`. +/// +/// Prefers the ISO `value_as_string` OpenSearch returns and falls back to +/// the epoch-millis `value`. Returns `None` when the bucket is empty. +fn agg_date(aggs: &Value, name: &str) -> Option> { + let node = aggs.get(name)?; + if let Some(s) = node.get("value_as_string").and_then(|v| v.as_str()) { + if let Ok(dt) = DateTime::parse_from_rfc3339(s) { + return Some(dt.with_timezone(&Utc)); + } + } + let ms = node.get("value").and_then(|v| v.as_f64())?; + if ms <= 0.0 { + return None; + } + Utc.timestamp_millis_opt(ms as i64).single() +} + +/// Extract `(key, doc_count)` pairs from a terms aggregation. +fn bucket_counts(resp: &Value, agg: &str) -> Vec<(String, usize)> { + resp.get("aggregations") + .and_then(|a| a.get(agg)) + .and_then(|t| t.get("buckets")) + .and_then(|b| b.as_array()) + .map(|buckets| { + buckets + .iter() + .filter_map(|b| { + let key = b.get("key")?.as_str()?.to_string(); + let count = b.get("doc_count")?.as_u64()? as usize; + Some((key, count)) + }) + .collect() + }) + .unwrap_or_default() +} diff --git a/crates/icm-store/src/opensearch/mod.rs b/crates/icm-store/src/opensearch/mod.rs new file mode 100644 index 00000000..3d7b4f34 --- /dev/null +++ b/crates/icm-store/src/opensearch/mod.rs @@ -0,0 +1,91 @@ +//! OpenSearch storage backend (issue #301, opt-in via `--features opensearch`). +//! +//! A search-native shared store: BM25 full-text and `knn_vector` HNSW +//! vector search live in one engine, so horizontally-scaled ICM replicas +//! share one memory store (a node-local SQLite file cannot be shared). +//! +//! Design notes: +//! +//! - **Blocking REST.** OpenSearch is an HTTP/JSON service, so this talks +//! to it with the blocking `ureq` client and `serde_json` bodies. The +//! store traits are synchronous, so — like the PostgreSQL backend — +//! there is no async runtime and no sync-over-async bridge. +//! - **Vector search** uses a `knn_vector` field (HNSW, cosine space); +//! similarity is reported from the kNN `_score`. +//! - **Full-text search** uses BM25 `match` queries; the hybrid path +//! blends normalized BM25 and vector scores 30/70 to match the SQLite +//! and PostgreSQL backends. +//! - **Connection** from `ICM_OPENSEARCH_URL` (e.g. `http://localhost:9200`), +//! with optional basic auth from `ICM_OPENSEARCH_USER` / +//! `ICM_OPENSEARCH_PASSWORD`. +//! +//! Scope mirrors the PostgreSQL backend: the full [`MemoryStore`] surface +//! plus the ancillary store/recall/hook tables (hook telemetry, the +//! extraction queue, code areas, key/value metadata). The heavier +//! subsystems (memoir graph, transcripts, structured facts, feedback, +//! pattern mining) return [`IcmError::Unsupported`]; they stay fully +//! available on the default SQLite backend. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::time::Duration; + +use base64::engine::general_purpose::STANDARD as B64; +use base64::Engine; +use chrono::{DateTime, TimeZone, Utc}; +use serde_json::{json, Value}; + +use icm_core::{ + Concept, ConceptLink, Embedder, Fact, FactsStats, FactsStore, Feedback, FeedbackStats, + FeedbackStore, IcmError, IcmResult, Importance, Label, Memoir, MemoirStats, MemoirStore, + Memory, MemorySource, MemoryStore, Message, PatternCluster, Relation, Role, Scope, Session, + StoreStats, TopicHealth, TranscriptHit, TranscriptStats, TranscriptStore, +}; + +// Shared public row types live in `crate::common` (issue #301) so every +// backend can be compiled into one binary without colliding definitions. +pub use crate::common::{CodeArea, HookEvent, HookEventInsert, HookStatsRow, PendingRow}; + +// Index names + +const IDX_MEMORIES: &str = "icm_memories"; +const IDX_METADATA: &str = "icm_metadata"; +const IDX_HOOKS: &str = "icm_hook_events"; +const IDX_PENDING: &str = "icm_pending_extractions"; +const IDX_CODE_AREAS: &str = "icm_code_areas"; + +// Store + +/// OpenSearch-backed store. Cheap to clone-free share via `&self`; every +/// method is a blocking REST round-trip. +pub struct OpenSearchStore { + agent: ureq::Agent, + base: String, + auth: Option, + embedding_dims: usize, + readonly: bool, +} + +// Subsystems not yet ported to this backend. They stay fully available on +// the default SQLite backend; here they fail cleanly with `Unsupported`. + +fn unsupported(op: &str) -> IcmResult { + Err(IcmError::Unsupported(format!( + "{op} (use the default SQLite backend)" + ))) +} + +mod connection; +mod facts; +mod feedback; +mod hooks; +mod maintenance; +mod memoir; +mod memory; +mod patterns; +mod rows; +#[cfg(test)] +mod tests; +mod transcript; + +pub(crate) use rows::*; diff --git a/crates/icm-store/src/opensearch/patterns.rs b/crates/icm-store/src/opensearch/patterns.rs new file mode 100644 index 00000000..9716ae88 --- /dev/null +++ b/crates/icm-store/src/opensearch/patterns.rs @@ -0,0 +1,111 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Neighbor expansion and pattern-mining helpers. + +use super::*; + +impl OpenSearchStore { + pub fn get_many(&self, ids: &[&str]) -> IcmResult> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + let resp = self.post(&format!("{IDX_MEMORIES}/_mget"), json!({"ids": ids}))?; + let mut out = HashMap::new(); + if let Some(docs) = resp.get("docs").and_then(|d| d.as_array()) { + for d in docs { + if d.get("found").and_then(|f| f.as_bool()).unwrap_or(false) { + if let (Some(id), Some(src)) = + (d.get("_id").and_then(|v| v.as_str()), d.get("_source")) + { + out.insert(id.to_string(), Self::source_to_memory(id, src)); + } + } + } + } + Ok(out) + } + + pub fn get_by_topic_prefix(&self, topic: &str) -> IcmResult> { + let resp = self.post( + &format!("{IDX_MEMORIES}/_search"), + json!({ + "size": 500, + "query": {"prefix": {"topic.keyword": topic}}, + "sort": [{"weight": "desc"}] + }), + )?; + Ok(Self::hits_to_memories(&resp)) + } + + pub fn list_topics_with_prefix(&self, prefix: Option<&str>) -> IcmResult> { + let mut topics = self.list_topics()?; + if let Some(p) = prefix { + topics.retain(|(t, _)| t.starts_with(p)); + } + Ok(topics) + } + + /// Expand a result set with graph neighbours (related ids), applying a + /// hop discount. Pure logic over [`Self::get_many`]; identical to the + /// other backends. + pub fn expand_with_neighbors( + &self, + initial: &[(Memory, f32)], + max_neighbors: usize, + hop_discount: f32, + max_total: usize, + ) -> IcmResult> { + if max_neighbors == 0 || initial.is_empty() { + let mut out = initial.to_vec(); + out.truncate(max_total); + return Ok(out); + } + let initial_ids: HashSet = initial.iter().map(|(m, _)| m.id.clone()).collect(); + let mut candidates: Vec<(String, f32)> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for (m, score) in initial { + for rid in &m.related_ids { + if !initial_ids.contains(rid) && seen.insert(rid.clone()) { + candidates.push((rid.clone(), *score * hop_discount)); + if candidates.len() >= max_neighbors { + break; + } + } + } + if candidates.len() >= max_neighbors { + break; + } + } + + let neighbor_ids: Vec<&str> = candidates.iter().map(|(id, _)| id.as_str()).collect(); + let fetched = self.get_many(&neighbor_ids)?; + + let mut combined: Vec<(Memory, f32)> = initial.to_vec(); + for (id, score) in candidates { + if let Some(m) = fetched.get(&id) { + combined.push((m.clone(), score)); + } + } + combined.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + combined.truncate(max_total); + Ok(combined) + } + + /// Pattern mining is not implemented on this backend yet. + pub fn detect_patterns( + &self, + _topic: &str, + _min_cluster_size: usize, + ) -> IcmResult> { + Err(IcmError::Unsupported("detect_patterns".into())) + } + + /// See [`Self::detect_patterns`]. + pub fn extract_pattern_as_concept( + &self, + _cluster: &PatternCluster, + _memoir_id: &str, + ) -> IcmResult { + Err(IcmError::Unsupported("extract_pattern_as_concept".into())) + } +} diff --git a/crates/icm-store/src/opensearch/rows.rs b/crates/icm-store/src/opensearch/rows.rs new file mode 100644 index 00000000..7e9ab048 --- /dev/null +++ b/crates/icm-store/src/opensearch/rows.rs @@ -0,0 +1,212 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Row/parse helpers and doc<->Memory mapping shared by every trait-impl +//! submodule here. + +use super::*; + +/// Percent-encode a value for safe use as a single path segment in a REST +/// URL. Document ids are caller-controlled (`icm forget ` CLI, MCP +/// `icm_forget`, etc.) with no format constraint enforced anywhere in the +/// schema — without this, a crafted id containing `/`, `..`, `?`, or `#` +/// could redirect which REST endpoint is actually hit instead of just +/// addressing the intended document (audit finding). +pub(crate) fn url_encode_path_segment(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +// Pure helpers (self-contained, mirror the other backends) + +pub(crate) fn source_type(source: &MemorySource) -> &'static str { + match source { + MemorySource::ClaudeCode { .. } => "claude_code", + MemorySource::Conversation { .. } => "conversation", + MemorySource::Manual => "manual", + } +} + +pub(crate) fn source_data(source: &MemorySource) -> Option { + match source { + MemorySource::Manual => None, + other => serde_json::to_string(other).ok(), + } +} + +pub(crate) fn parse_source(source_type_str: &str, source_data_str: Option) -> MemorySource { + match source_type_str { + "manual" => MemorySource::Manual, + _ => source_data_str + .and_then(|d| serde_json::from_str(&d).ok()) + .unwrap_or(MemorySource::Manual), + } +} + +pub(crate) fn importance_rank(i: Importance) -> u8 { + match i { + Importance::Low => 0, + Importance::Medium => 1, + Importance::High => 2, + Importance::Critical => 3, + } +} + +pub(crate) fn max_importance(a: Importance, b: Importance) -> Importance { + if importance_rank(a) >= importance_rank(b) { + a + } else { + b + } +} + +/// SHA-256 over the normalized `(topic, summary)` pair, hex-encoded. +/// Normalization: trim + lowercase + collapse whitespace, joined by `\0`. +pub(crate) fn summary_hash(topic: &str, summary: &str) -> String { + use sha2::{Digest, Sha256}; + let topic_n = topic.trim().to_lowercase(); + let summary_n = summary + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase(); + let mut h = Sha256::new(); + h.update(topic_n.as_bytes()); + h.update(b"\0"); + h.update(summary_n.as_bytes()); + format!("{:x}", h.finalize()) +} + +/// Validate and normalize a memory before storing (mirror of the other +/// backends): non-empty topic/summary, generate an id if missing, and +/// stamp timestamps. +pub(crate) fn validate_and_normalize(mut memory: Memory) -> IcmResult { + if memory.topic.trim().is_empty() { + return Err(IcmError::InvalidInput("topic cannot be empty".into())); + } + if memory.summary.trim().is_empty() { + return Err(IcmError::InvalidInput("summary cannot be empty".into())); + } + if memory.id.trim().is_empty() { + memory.id = ulid::Ulid::new().to_string(); + } + memory.topic = memory.topic.trim().to_string(); + Ok(memory) +} + +pub(crate) fn parse_dt(s: &str) -> DateTime { + DateTime::parse_from_rfc3339(s) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now()) +} + +impl OpenSearchStore { + // (de)serialization + + pub(crate) fn memory_to_source(memory: &Memory) -> Value { + let mut doc = json!({ + "created_at": memory.created_at.to_rfc3339(), + "updated_at": memory.updated_at.to_rfc3339(), + "last_accessed": memory.last_accessed.to_rfc3339(), + "access_count": memory.access_count, + "weight": memory.weight, + "topic": memory.topic, + "summary": memory.summary, + "raw_excerpt": memory.raw_excerpt, + "keywords": memory.keywords, + "importance": memory.importance.to_string(), + "source_type": source_type(&memory.source), + "source_data": source_data(&memory.source), + "related_ids": memory.related_ids, + "summary_hash": summary_hash(&memory.topic, &memory.summary), + }); + if let Some(emb) = memory.embedding.as_ref() { + doc["embedding"] = json!(emb); + } + doc + } + + pub(crate) fn source_to_memory(id: &str, src: &Value) -> Memory { + let get_str = |k: &str| { + src.get(k) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string() + }; + let opt_str = |k: &str| { + src.get(k) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .filter(|s| !s.is_empty()) + }; + let arr = |k: &str| { + src.get(k) + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(|s| s.to_string())) + .collect::>() + }) + .unwrap_or_default() + }; + let importance = get_str("importance").parse().unwrap_or(Importance::Medium); + let source = parse_source(&get_str("source_type"), opt_str("source_data")); + let embedding = src.get("embedding").and_then(|v| v.as_array()).map(|a| { + a.iter() + .filter_map(|x| x.as_f64().map(|f| f as f32)) + .collect::>() + }); + Memory { + id: id.to_string(), + created_at: parse_dt(&get_str("created_at")), + updated_at: parse_dt(&get_str("updated_at")), + last_accessed: parse_dt(&get_str("last_accessed")), + access_count: src + .get("access_count") + .and_then(|v| v.as_u64()) + .unwrap_or(0) as u32, + weight: src.get("weight").and_then(|v| v.as_f64()).unwrap_or(1.0) as f32, + topic: get_str("topic"), + summary: get_str("summary"), + raw_excerpt: opt_str("raw_excerpt"), + keywords: arr("keywords"), + importance, + source, + related_ids: arr("related_ids"), + embedding, + scope: Scope::default(), + } + } + + /// Map a `_search` response's hits to memories paired with `_score`. + pub(crate) fn hits_to_scored(resp: &Value) -> Vec<(Memory, f32)> { + resp.get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + .map(|hits| { + hits.iter() + .filter_map(|h| { + let id = h.get("_id")?.as_str()?; + let src = h.get("_source")?; + let score = h.get("_score").and_then(|s| s.as_f64()).unwrap_or(0.0) as f32; + Some((Self::source_to_memory(id, src), score)) + }) + .collect() + }) + .unwrap_or_default() + } + + pub(crate) fn hits_to_memories(resp: &Value) -> Vec { + Self::hits_to_scored(resp) + .into_iter() + .map(|(m, _)| m) + .collect() + } +} diff --git a/crates/icm-store/src/opensearch/tests.rs b/crates/icm-store/src/opensearch/tests.rs new file mode 100644 index 00000000..2d11539a --- /dev/null +++ b/crates/icm-store/src/opensearch/tests.rs @@ -0,0 +1,42 @@ +//! Test suite for the OpenSearch backend (`opensearch::tests`). + +use super::*; + +/// Audit regression: a memory id containing reserved URL characters +/// (e.g. `/`, `..`, `?`) was interpolated straight into the `_doc/{id}` +/// REST path, letting an attacker-controlled id redirect the request to +/// a different document/endpoint. +#[test] +fn test_url_encode_path_segment() { + assert_eq!( + url_encode_path_segment("abc-DEF_123.~"), + "abc-DEF_123.~", + "unreserved chars must pass through unchanged" + ); + assert_eq!(url_encode_path_segment("a/b"), "a%2Fb"); + assert_eq!(url_encode_path_segment("../secret"), "..%2Fsecret"); + assert_eq!(url_encode_path_segment("id?x=1"), "id%3Fx%3D1"); + assert_eq!(url_encode_path_segment("id#frag"), "id%23frag"); + assert_eq!(url_encode_path_segment("a b"), "a%20b"); +} + +/// Audit regression: `apply_decay`'s Painless script computed a raw +/// multiplier that goes negative for low-importance/low-access memories +/// at factor<0.5 (still inside the CLI's own validated range). This +/// mirrors the exact formula now wrapped in `Math.max(0.0, ...)`. +#[test] +fn test_apply_decay_formula_would_go_negative_without_clamp() { + let factor: f64 = 0.4; + let mult: f64 = 2.0; // low importance + let access: f64 = 0.0; + let raw = 1.0 - (1.0 - factor) * mult / (1.0 + access * 0.1); + assert!( + raw < 0.0, + "expected the pre-clamp formula to go negative, got {raw}" + ); + assert_eq!( + raw.max(0.0), + 0.0, + "Math.max(0.0, ...) must clamp this to 0.0" + ); +} diff --git a/crates/icm-store/src/opensearch/transcript.rs b/crates/icm-store/src/opensearch/transcript.rs new file mode 100644 index 00000000..0b442e19 --- /dev/null +++ b/crates/icm-store/src/opensearch/transcript.rs @@ -0,0 +1,65 @@ +//! OpenSearch backend -- split out of the former monolithic opensearch.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl TranscriptStore for OpenSearchStore { + fn create_session( + &self, + _agent: &str, + _project: Option<&str>, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.create_session") + } + fn ensure_session( + &self, + _id: &str, + _agent: &str, + _project: Option<&str>, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.ensure_session") + } + fn get_session(&self, _id: &str) -> IcmResult> { + unsupported("transcript.get_session") + } + fn list_sessions(&self, _project: Option<&str>, _limit: usize) -> IcmResult> { + unsupported("transcript.list_sessions") + } + fn record_message( + &self, + _session_id: &str, + _role: Role, + _content: &str, + _tool_name: Option<&str>, + _tokens: Option, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.record_message") + } + fn list_session_messages( + &self, + _session_id: &str, + _limit: usize, + _offset: usize, + ) -> IcmResult> { + unsupported("transcript.list_session_messages") + } + fn search_transcripts( + &self, + _query: &str, + _session_id: Option<&str>, + _project: Option<&str>, + _limit: usize, + ) -> IcmResult> { + unsupported("transcript.search_transcripts") + } + fn forget_session(&self, _id: &str) -> IcmResult<()> { + unsupported("transcript.forget_session") + } + fn transcript_stats(&self) -> IcmResult { + unsupported("transcript.transcript_stats") + } +} diff --git a/crates/icm-store/src/postgres.rs b/crates/icm-store/src/postgres.rs deleted file mode 100644 index a937a1e0..00000000 --- a/crates/icm-store/src/postgres.rs +++ /dev/null @@ -1,1910 +0,0 @@ -//! PostgreSQL storage backend (issue #301, opt-in via `--features postgres`). -//! -//! A node-local SQLite file cannot be shared between several ICM -//! processes or Kubernetes replicas. This backend runs the same memory -//! model over a network-accessible PostgreSQL database so every instance -//! reads and writes one shared store. PostgreSQL serialises concurrent -//! writers, so N replicas can `icm store` into the same memory safely. -//! -//! Design notes: -//! -//! - **Blocking client.** The store traits are synchronous -//! (`fn store(&self, ...) -> IcmResult<...>`), so we use the blocking -//! `postgres` crate. No async runtime, no sync-over-async bridge — the -//! client maps one-to-one onto the trait surface. -//! - **`pgvector` for embeddings.** Memory embeddings live in a -//! `vector(N)` column; KNN search uses the `<=>` cosine-distance -//! operator. Similarity is reported as `1 - distance` to match the -//! SQLite backend. -//! - **PostgreSQL full-text search** replaces SQLite FTS5: a generated -//! `tsvector` column (config `simple`, no stemming, to mirror FTS5's -//! unicode61 tokenizer) with a GIN index, queried via -//! `websearch_to_tsquery` so arbitrary user input is operator-safe. -//! - **Connection string** comes from `ICM_POSTGRES_URL` (or -//! `DATABASE_URL` as a fallback). The `&Path` arguments that the CLI -//! passes for the SQLite file are ignored. -//! -//! Scope of this first cut: the full [`MemoryStore`] surface (the core -//! shared-memory use case behind #301) plus the ancillary tables used by -//! the normal store/recall/hook path (hook telemetry, the extraction -//! queue, code areas, the key/value metadata). The heavier subsystems -//! (memoir graph, transcripts, structured facts, feedback, pattern -//! mining) return [`IcmError::Unsupported`] on this backend for now; -//! they remain fully available on the default SQLite backend. - -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::sync::{Mutex, MutexGuard}; - -use chrono::{DateTime, Utc}; -use postgres::types::ToSql; -use postgres::{Client, GenericClient, NoTls}; - -use icm_core::{ - Concept, ConceptLink, Embedder, Fact, FactsStats, FactsStore, Feedback, FeedbackStats, - FeedbackStore, IcmError, IcmResult, Importance, Label, Memoir, MemoirStats, MemoirStore, - Memory, MemorySource, MemoryStore, Message, PatternCluster, Relation, Role, Session, - StoreStats, TopicHealth, TranscriptHit, TranscriptStats, TranscriptStore, -}; - -// Shared public row types live in `crate::common` (issue #301) so every -// backend can be compiled into one binary without colliding definitions. -pub use crate::common::{CodeArea, HookEvent, HookEventInsert, HookStatsRow, PendingRow}; - -// --------------------------------------------------------------------------- -// Helpers (mirrored from the SQLite backend so behaviour matches) -// --------------------------------------------------------------------------- - -fn pg_err(e: postgres::Error) -> IcmError { - IcmError::Database(e.to_string()) -} - -fn lock_err() -> IcmError { - IcmError::Database("postgres client mutex poisoned".into()) -} - -fn source_type(source: &MemorySource) -> &'static str { - match source { - MemorySource::ClaudeCode { .. } => "claude_code", - MemorySource::Conversation { .. } => "conversation", - MemorySource::Manual => "manual", - } -} - -fn source_data(source: &MemorySource) -> Option { - match source { - MemorySource::Manual => None, - other => serde_json::to_string(other).ok(), - } -} - -fn parse_source(source_type_str: &str, source_data_str: Option) -> MemorySource { - match source_type_str { - "manual" => MemorySource::Manual, - _ => source_data_str - .and_then(|d| serde_json::from_str(&d).ok()) - .unwrap_or(MemorySource::Manual), - } -} - -fn importance_rank(i: Importance) -> u8 { - match i { - Importance::Critical => 4, - Importance::High => 3, - Importance::Medium => 2, - Importance::Low => 1, - } -} - -fn max_importance(a: Importance, b: Importance) -> Importance { - if importance_rank(a) >= importance_rank(b) { - a - } else { - b - } -} - -/// SHA-256 over the normalized `(topic, summary)` pair, hex-encoded. -/// Identical normalization to the SQLite backend so dedup hashes match. -fn summary_hash(topic: &str, summary: &str) -> String { - use sha2::{Digest, Sha256}; - let topic_n = topic.trim().to_lowercase(); - let summary_n: String = summary - .split_whitespace() - .collect::>() - .join(" ") - .to_lowercase(); - let mut h = Sha256::new(); - h.update(topic_n.as_bytes()); - h.update(b"\0"); - h.update(summary_n.as_bytes()); - format!("{:x}", h.finalize()) -} - -const MAX_SUMMARY_BYTES: usize = 64 * 1024; -const MAX_TOPIC_BYTES: usize = 256; - -/// Escape `%`, `_`, and the escape character itself so a value can be -/// safely wrapped in a LIKE/ILIKE pattern. Pair with `ESCAPE '\'` in the -/// SQL. Mirrors the SQLite backend's `escape_like_wildcards`. -fn escape_like_wildcards(s: &str) -> String { - s.replace('\\', "\\\\") - .replace('%', "\\%") - .replace('_', "\\_") -} - -/// Validate and normalize a `Memory` before insertion. Mirrors the -/// SQLite backend's `validate_and_normalize`. -fn validate_and_normalize(mut memory: Memory) -> IcmResult { - memory.topic = memory.topic.trim().to_string(); - - if memory.topic.is_empty() { - return Err(IcmError::InvalidInput("topic cannot be empty".into())); - } - if memory.summary.trim().is_empty() { - return Err(IcmError::InvalidInput("summary cannot be empty".into())); - } - if memory.topic.contains('\0') { - return Err(IcmError::InvalidInput( - "topic must not contain NUL bytes".into(), - )); - } - if memory.summary.contains('\0') { - return Err(IcmError::InvalidInput( - "summary must not contain NUL bytes".into(), - )); - } - if memory.topic.contains(['\n', '\r', '\t']) { - return Err(IcmError::InvalidInput( - "topic must not contain newline / CR / tab characters".into(), - )); - } - if memory.topic.len() > MAX_TOPIC_BYTES { - return Err(IcmError::InvalidInput(format!( - "topic exceeds {MAX_TOPIC_BYTES} bytes" - ))); - } - if memory.summary.len() > MAX_SUMMARY_BYTES { - return Err(IcmError::InvalidInput(format!( - "summary exceeds {MAX_SUMMARY_BYTES} bytes" - ))); - } - Ok(memory) -} - -const SELECT_COLS: &str = "id, created_at, updated_at, last_accessed, access_count, weight, \ - topic, summary, raw_excerpt, keywords, \ - importance, source_type, source_data, related_ids, embedding"; - -/// Map a `memories` row (selected via [`SELECT_COLS`]) to a [`Memory`]. -fn row_to_memory(row: &postgres::Row) -> Memory { - let keywords_json: Option = row.get(9); - let keywords: Vec = keywords_json - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default(); - - let importance_str: String = row.get(10); - let importance = importance_str.parse().unwrap_or(Importance::Medium); - - let source_type_str: String = row.get(11); - let source_data_str: Option = row.get(12); - let source = parse_source(&source_type_str, source_data_str); - - let related_json: Option = row.get(13); - let related_ids: Vec = related_json - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default(); - - let embedding: Option> = row - .get::<_, Option>(14) - .map(|v| v.as_slice().to_vec()); - - let access_count: i32 = row.get(4); - - Memory { - id: row.get(0), - created_at: row.get(1), - updated_at: row.get(2), - last_accessed: row.get(3), - access_count: access_count.max(0) as u32, - weight: row.get(5), - topic: row.get(6), - summary: row.get(7), - raw_excerpt: row.get(8), - keywords, - importance, - source, - related_ids, - embedding, - scope: icm_core::Scope::User, - } -} - -/// Insert a memory, or merge metadata into an existing duplicate. -/// -/// Dedup contract identical to the SQLite backend: a collision on -/// `summary_hash` alone (which already encodes the topic, Rust-side, -/// Unicode-correct) is ignored and the existing row's id is returned, after -/// merging the caller's importance (take max), keywords (union), and -/// `raw_excerpt` (prefer new) into it. -fn insert_or_merge_memory(c: &mut C, memory: &Memory) -> IcmResult { - let keywords_json = serde_json::to_string(&memory.keywords)?; - let related_json = serde_json::to_string(&memory.related_ids)?; - let st = source_type(&memory.source); - let sd = source_data(&memory.source); - let hash = summary_hash(&memory.topic, &memory.summary); - let importance = memory.importance.to_string(); - let access = memory.access_count as i32; - let emb: Option = memory - .embedding - .as_ref() - .map(|e| pgvector::Vector::from(e.clone())); - - let inserted = c - .query_opt( - "INSERT INTO memories - (id, created_at, updated_at, last_accessed, access_count, weight, - topic, summary, raw_excerpt, keywords, importance, - source_type, source_data, related_ids, summary_hash, embedding) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) - ON CONFLICT (summary_hash) WHERE summary_hash IS NOT NULL - DO NOTHING - RETURNING id", - &[ - &memory.id, - &memory.created_at, - &memory.updated_at, - &memory.last_accessed, - &access, - &memory.weight, - &memory.topic, - &memory.summary, - &memory.raw_excerpt, - &keywords_json, - &importance, - &st, - &sd, - &related_json, - &hash, - &emb, - ], - ) - .map_err(pg_err)?; - - if let Some(row) = inserted { - return Ok(row.get::<_, String>(0)); - } - - // Dedup hit: merge metadata into the existing row (mirrors SQLite). - let existing = c - .query_one( - "SELECT id, importance, keywords, raw_excerpt FROM memories - WHERE summary_hash = $1", - &[&hash], - ) - .map_err(pg_err)?; - - let existing_id: String = existing.get(0); - let existing_importance_str: String = existing.get(1); - let existing_keywords_json: Option = existing.get(2); - let existing_raw: Option = existing.get(3); - - let existing_importance: Importance = existing_importance_str - .parse() - .unwrap_or(Importance::Medium); - let merged_importance = max_importance(existing_importance, memory.importance); - - let existing_keywords: Vec = existing_keywords_json - .as_deref() - .and_then(|s| serde_json::from_str(s).ok()) - .unwrap_or_default(); - let mut merged_keywords = existing_keywords.clone(); - for kw in &memory.keywords { - if !merged_keywords.contains(kw) { - merged_keywords.push(kw.clone()); - } - } - - let merged_raw = memory.raw_excerpt.clone().or_else(|| existing_raw.clone()); - - let importance_changed = merged_importance != existing_importance; - let keywords_changed = merged_keywords != existing_keywords; - let raw_changed = merged_raw != existing_raw; - if importance_changed || keywords_changed || raw_changed { - let merged_keywords_json = serde_json::to_string(&merged_keywords)?; - c.execute( - "UPDATE memories - SET importance = $1, keywords = $2, raw_excerpt = $3, updated_at = $4 - WHERE id = $5", - &[ - &merged_importance.to_string(), - &merged_keywords_json, - &merged_raw, - &Utc::now(), - &existing_id, - ], - ) - .map_err(pg_err)?; - } - - Ok(existing_id) -} - -// --------------------------------------------------------------------------- -// PostgresStore -// --------------------------------------------------------------------------- - -/// PostgreSQL-backed store. See the module docs. -pub struct PostgresStore { - client: Mutex, - embedding_dims: usize, - readonly: bool, -} - -/// One-shot warning that auto-consolidation silently does nothing on this -/// backend (see [`PostgresStore::auto_consolidate`]). -fn warn_auto_consolidate_unsupported() { - static WARNED: std::sync::Once = std::sync::Once::new(); - WARNED.call_once(|| { - tracing::warn!( - "auto-consolidation is not implemented on the PostgreSQL backend; \ - topics will keep growing (auto_consolidate_enabled has no effect here)" - ); - }); -} - -impl PostgresStore { - fn conn(&self) -> IcmResult> { - self.client.lock().map_err(|_| lock_err()) - } - - /// Resolve the connection string from the environment. - fn conn_string() -> IcmResult { - std::env::var("ICM_POSTGRES_URL") - .or_else(|_| std::env::var("DATABASE_URL")) - .map_err(|_| { - IcmError::Config( - "PostgreSQL backend: set ICM_POSTGRES_URL (or DATABASE_URL) to the \ - connection string, e.g. postgres://user:pass@host:5432/icm" - .into(), - ) - }) - } - - /// Connect and run the idempotent schema migration. - /// - /// The `&Path` the CLI passes for the SQLite file is ignored; the - /// connection comes from the environment. `requested_dims` is used - /// only when the database is fresh — an existing database's stored - /// `embedding_dims` is authoritative so we never try to declare a - /// `vector(N)` column that disagrees with the live table. - fn connect(requested_dims: usize, readonly: bool) -> IcmResult { - let url = Self::conn_string()?; - let mut client = Client::connect(&url, NoTls) - .map_err(|e| IcmError::Database(format!("cannot connect to PostgreSQL: {e}")))?; - - let dims = init_schema(&mut client, requested_dims)?; - - Ok(Self { - client: Mutex::new(client), - embedding_dims: dims, - readonly, - }) - } - - /// Reject an embedding whose length disagrees with the column's - /// declared dimension, with a clearer message than the raw PostgreSQL - /// "expected N dimensions, not M" error. - fn check_dims(&self, memory: &Memory) -> IcmResult<()> { - if let Some(emb) = memory.embedding.as_ref() { - if emb.len() != self.embedding_dims { - return Err(IcmError::InvalidInput(format!( - "embedding has {} dimensions, but this store uses {}", - emb.len(), - self.embedding_dims - ))); - } - } - Ok(()) - } - - /// Open or create a store with the default embedding dimension. - pub fn new(_path: &Path) -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) - } - - /// Open or create a store with a specific embedding dimension. - pub fn with_dims(_path: &Path, embedding_dims: usize) -> IcmResult { - Self::connect(embedding_dims, false) - } - - /// Open the store in read-only mode (issue #263). The connection is - /// the same; write methods refuse, read-like side effects are skipped. - pub fn open_readonly(_path: &Path) -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, true) - } - - /// PostgreSQL has no in-memory mode; connect to the configured - /// database. Provided for API parity with the SQLite backend. - pub fn in_memory() -> IcmResult { - Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) - } - - /// See [`Self::in_memory`]. - pub fn in_memory_with_dims(embedding_dims: usize) -> IcmResult { - Self::connect(embedding_dims, false) - } - - /// PostgreSQL stores `embedding_dims` in `icm_metadata`, but unlike - /// SQLite it never destructively recreates the vector column, so the - /// pre-open peek the SQLite backend needs is unnecessary here. Always - /// returns `Ok(None)` so callers fall through to the normal open path. - pub fn read_stored_embedding_dims(_path: &Path) -> IcmResult> { - Ok(None) - } - - #[must_use] - pub fn is_readonly(&self) -> bool { - self.readonly - } - - /// No-op on PostgreSQL (the SQLite backend uses this to load the - /// `sqlite-vec` extension; `pgvector` lives server-side). - pub fn ensure_vec_init() {} - - /// Apply decay if more than 24 hours since the last run. Mirrors the - /// SQLite backend's atomic check-and-claim via `icm_metadata`. - pub fn maybe_auto_decay(&self) -> IcmResult<()> { - if self.readonly { - return Ok(()); - } - let now = Utc::now(); - let claimed = { - let mut c = self.conn()?; - c.execute( - "INSERT INTO icm_metadata (key, value) VALUES ('last_decay_at', $1) - ON CONFLICT (key) DO UPDATE SET value = $1 - WHERE icm_metadata.value IS NULL - OR ($1::timestamptz - icm_metadata.value::timestamptz) >= interval '1 day'", - &[&now.to_rfc3339()], - ) - .map_err(pg_err)? - }; - if claimed > 0 { - self.apply_decay(0.95)?; - } - Ok(()) - } - - /// Atomically increment the hook call counter and return the new value. - pub fn increment_hook_counter(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one( - "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '1') - ON CONFLICT (key) DO UPDATE SET value = ((icm_metadata.value::bigint) + 1)::text - RETURNING value::bigint", - &[], - ) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - /// Reset the hook call counter to 0. - pub fn reset_hook_counter(&self) -> IcmResult<()> { - let mut c = self.conn()?; - c.execute( - "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '0') - ON CONFLICT (key) DO UPDATE SET value = '0'", - &[], - ) - .map_err(pg_err)?; - Ok(()) - } - - // ── Async extraction queue ───────────────────────────────────────── - - /// Enqueue raw tool output for later LLM extraction. - pub fn enqueue_pending_extraction( - &self, - project: &str, - tool_name: &str, - raw_output: &str, - ) -> IcmResult { - let id = ulid::Ulid::new().to_string(); - let mut c = self.conn()?; - c.execute( - "INSERT INTO pending_extractions (id, project, tool_name, raw_output, captured_at) - VALUES ($1, $2, $3, $4, $5)", - &[&id, &project, &tool_name, &raw_output, &Utc::now()], - ) - .map_err(pg_err)?; - Ok(id) - } - - /// Pop up to `limit` oldest pending rows (FIFO by capture time). - pub fn list_pending_extractions(&self, limit: usize) -> IcmResult> { - let mut c = self.conn()?; - let rows = c - .query( - "SELECT id, project, tool_name, raw_output, captured_at - FROM pending_extractions - ORDER BY captured_at ASC - LIMIT $1", - &[&(limit as i64)], - ) - .map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| { - let captured: DateTime = row.get(4); - ( - row.get(0), - row.get(1), - row.get(2), - row.get(3), - captured.to_rfc3339(), - ) - }) - .collect()) - } - - /// Delete pending rows by id. Used after a worker has processed them. - pub fn delete_pending_extractions(&self, ids: &[String]) -> IcmResult { - if ids.is_empty() { - return Ok(0); - } - let ids_vec: Vec = ids.to_vec(); - let mut c = self.conn()?; - let n = c - .execute( - "DELETE FROM pending_extractions WHERE id = ANY($1)", - &[&ids_vec], - ) - .map_err(pg_err)?; - Ok(n as usize) - } - - /// Total rows currently waiting in the queue. - pub fn pending_extraction_count(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one("SELECT COUNT(*) FROM pending_extractions", &[]) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - // ── Code areas (issue #196) ──────────────────────────────────────── - - /// Insert or refresh a row for `(project, file_path)`. - pub fn upsert_code_area( - &self, - project: &str, - file_path: &str, - description: Option<&str>, - session_id: Option<&str>, - tool_name: Option<&str>, - ) -> IcmResult<()> { - let now = Utc::now(); - let mut c = self.conn()?; - c.execute( - "INSERT INTO code_areas - (project, file_path, description, session_id, tool_name, - touch_count, first_touched_at, last_touched_at) - VALUES ($1, $2, $3, $4, $5, 1, $6, $6) - ON CONFLICT (project, file_path) DO UPDATE SET - touch_count = code_areas.touch_count + 1, - last_touched_at = EXCLUDED.last_touched_at, - session_id = COALESCE(EXCLUDED.session_id, code_areas.session_id), - tool_name = COALESCE(EXCLUDED.tool_name, code_areas.tool_name), - description = COALESCE(EXCLUDED.description, code_areas.description)", - &[ - &project, - &file_path, - &description, - &session_id, - &tool_name, - &now, - ], - ) - .map_err(pg_err)?; - Ok(()) - } - - /// List code areas, optionally filtered, newest-touch first. - pub fn list_code_areas( - &self, - project: Option<&str>, - in_file: Option<&str>, - since: Option>, - limit: usize, - ) -> IcmResult> { - let mut sql = String::from( - "SELECT id, project, file_path, description, session_id, tool_name, - touch_count, first_touched_at, last_touched_at - FROM code_areas WHERE TRUE", - ); - let mut owned: Vec> = Vec::new(); - if let Some(p) = project { - owned.push(Box::new(p.to_string())); - sql.push_str(&format!(" AND project = ${}", owned.len())); - } - if let Some(f) = in_file { - owned.push(Box::new(f.to_string())); - let exact = owned.len(); - owned.push(Box::new(format!("%/{f}"))); - let suffix = owned.len(); - sql.push_str(&format!( - " AND (file_path = ${exact} OR file_path LIKE ${suffix})" - )); - } - if let Some(t) = since { - owned.push(Box::new(t)); - sql.push_str(&format!(" AND last_touched_at >= ${}", owned.len())); - } - owned.push(Box::new(limit as i64)); - sql.push_str(&format!( - " ORDER BY last_touched_at DESC LIMIT ${}", - owned.len() - )); - - let params: Vec<&(dyn ToSql + Sync)> = owned.iter().map(|b| b.as_ref()).collect(); - let mut c = self.conn()?; - let rows = c.query(&sql, ¶ms).map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| CodeArea { - id: row.get(0), - project: row.get(1), - file_path: row.get(2), - description: row.get(3), - session_id: row.get(4), - tool_name: row.get(5), - touch_count: row.get(6), - first_touched_at: row.get(7), - last_touched_at: row.get(8), - }) - .collect()) - } - - /// Total rows in `code_areas`. - pub fn code_area_count(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one("SELECT COUNT(*) FROM code_areas", &[]) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - // ── Hook telemetry ───────────────────────────────────────────────── - - /// Append one hook telemetry row, returning its id. - pub fn record_hook_event(&self, ev: &HookEventInsert) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one( - "INSERT INTO hook_events - (ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - RETURNING id", - &[ - &Utc::now(), - &ev.event, - &ev.project, - &ev.session_id, - &ev.tool_name, - &ev.duration_ms, - &ev.exit_code, - &ev.payload_size, - &ev.note, - ], - ) - .map_err(pg_err)?; - Ok(row.get(0)) - } - - /// Most recent `limit` hook events, newest first; optional event filter. - pub fn hook_events_recent( - &self, - limit: usize, - event_filter: Option<&str>, - ) -> IcmResult> { - let mut c = self.conn()?; - let rows = match event_filter { - Some(ev) => c.query( - "SELECT id, ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note - FROM hook_events WHERE event = $1 ORDER BY id DESC LIMIT $2", - &[&ev, &(limit as i64)], - ), - None => c.query( - "SELECT id, ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note - FROM hook_events ORDER BY id DESC LIMIT $1", - &[&(limit as i64)], - ), - } - .map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| HookEvent { - id: row.get(0), - ts: row.get(1), - event: row.get(2), - project: row.get(3), - session_id: row.get(4), - tool_name: row.get(5), - duration_ms: row.get(6), - exit_code: row.get(7), - payload_size: row.get(8), - note: row.get(9), - }) - .collect()) - } - - /// Per-event aggregate stats since `since_rfc3339`. - pub fn hook_stats(&self, since_rfc3339: &str) -> IcmResult> { - let since = DateTime::parse_from_rfc3339(since_rfc3339) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now() - chrono::Duration::days(7)); - let mut c = self.conn()?; - let rows = c - .query( - "SELECT event, - COUNT(*)::bigint, - COUNT(*) FILTER (WHERE exit_code <> 0)::bigint, - COALESCE(AVG(duration_ms), 0)::float8, - COALESCE(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms), 0)::float8, - COALESCE(percentile_cont(0.99) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 - FROM hook_events WHERE ts >= $1 - GROUP BY event ORDER BY event", - &[&since], - ) - .map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| { - let p50: f64 = row.get(4); - let p99: f64 = row.get(5); - HookStatsRow { - event: row.get(0), - count: row.get(1), - error_count: row.get(2), - avg_duration_ms: row.get(3), - p50_duration_ms: p50 as i64, - p99_duration_ms: p99 as i64, - } - }) - .collect()) - } - - /// Delete hook events older than `cutoff_rfc3339`. - pub fn prune_hook_events(&self, cutoff_rfc3339: &str) -> IcmResult { - let cutoff = DateTime::parse_from_rfc3339(cutoff_rfc3339) - .map(|d| d.with_timezone(&Utc)) - .map_err(|e| IcmError::InvalidInput(format!("invalid cutoff timestamp: {e}")))?; - let mut c = self.conn()?; - let n = c - .execute("DELETE FROM hook_events WHERE ts < $1", &[&cutoff]) - .map_err(pg_err)?; - Ok(n as usize) - } - - /// Total rows in `hook_events`. - pub fn hook_event_count(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one("SELECT COUNT(*) FROM hook_events", &[]) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - // ── Memory reads used by recall expansion ────────────────────────── - - /// Fetch many memories by id in one round-trip, deduplicated by id. - pub fn get_many(&self, ids: &[&str]) -> IcmResult> { - if ids.is_empty() { - return Ok(HashMap::new()); - } - let id_vec: Vec = ids.iter().map(|s| s.to_string()).collect(); - let mut c = self.conn()?; - let rows = c - .query( - &format!("SELECT {SELECT_COLS} FROM memories WHERE id = ANY($1)"), - &[&id_vec], - ) - .map_err(pg_err)?; - let mut map = HashMap::with_capacity(rows.len()); - for row in &rows { - let m = row_to_memory(row); - map.insert(m.id.clone(), m); - } - Ok(map) - } - - /// Expand a scored result set with one hop of related memories. - /// Backend-agnostic logic mirrored from the SQLite store. - pub fn expand_with_neighbors( - &self, - initial: &[(Memory, f32)], - max_neighbors: usize, - hop_discount: f32, - max_total: usize, - ) -> IcmResult> { - if max_neighbors == 0 || initial.is_empty() { - let mut out = initial.to_vec(); - out.truncate(max_total); - return Ok(out); - } - - let initial_ids: HashSet = initial.iter().map(|(m, _)| m.id.clone()).collect(); - - let mut candidates: Vec<(String, f32)> = Vec::new(); - let mut seen: HashSet = HashSet::new(); - 'outer: for (mem, score) in initial { - for neighbor_id in &mem.related_ids { - if candidates.len() >= max_neighbors { - break 'outer; - } - if initial_ids.contains(neighbor_id) || !seen.insert(neighbor_id.clone()) { - continue; - } - candidates.push((neighbor_id.clone(), *score)); - } - } - - let mut neighbors: Vec<(Memory, f32)> = Vec::new(); - if !candidates.is_empty() { - let ids: Vec<&str> = candidates.iter().map(|(id, _)| id.as_str()).collect(); - let fetched = self.get_many(&ids)?; - for (id, parent_score) in candidates { - if let Some(m) = fetched.get(&id) { - neighbors.push((m.clone(), parent_score * hop_discount)); - } - } - } - - let mut combined: Vec<(Memory, f32)> = initial.to_vec(); - combined.extend(neighbors); - combined.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - combined.truncate(max_total); - Ok(combined) - } - - /// Memories whose topic starts with `topic` (prefix match). - pub fn get_by_topic_prefix(&self, topic: &str) -> IcmResult> { - // Audit finding: `topic` (the literal prefix to match) was - // interpolated unescaped — a topic containing `%`/`_` turned into - // unintended wildcards within what's meant to be a literal prefix. - // The trailing `%` is the deliberate "starts with" wildcard and - // stays outside the escaped portion. - let pattern = format!("{}%", escape_like_wildcards(topic)); - let mut c = self.conn()?; - let rows = c - .query( - &format!( - "SELECT {SELECT_COLS} FROM memories WHERE topic LIKE $1 ESCAPE '\\' \ - ORDER BY weight DESC LIMIT 500" - ), - &[&pattern], - ) - .map_err(pg_err)?; - Ok(rows.iter().map(row_to_memory).collect()) - } - - /// Distinct topics (optionally prefix-filtered) with their counts. - pub fn list_topics_with_prefix(&self, prefix: Option<&str>) -> IcmResult> { - let mut c = self.conn()?; - let rows = match prefix { - Some(p) => { - let pattern = format!("{p}%"); - c.query( - "SELECT topic, COUNT(*)::bigint FROM memories WHERE topic LIKE $1 \ - GROUP BY topic ORDER BY topic", - &[&pattern], - ) - } - None => c.query( - "SELECT topic, COUNT(*)::bigint FROM memories GROUP BY topic ORDER BY topic", - &[], - ), - } - .map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| { - let n: i64 = row.get(1); - (row.get(0), n.max(0) as usize) - }) - .collect()) - } - - // ── Consolidation / patterns ─────────────────────────────────────── - - /// Auto-consolidation is not yet implemented on the PostgreSQL - /// backend; the call is a no-op (returns "did not consolidate") so the - /// normal store path is unaffected. Unlike the other parity gaps (which - /// return `Unsupported`), this one is silent by design — but the user - /// deserves to know their `auto_consolidate_enabled = true` does nothing - /// here, so warn once per process (audit finding). - pub fn auto_consolidate(&self, _topic: &str, _threshold: usize) -> IcmResult { - warn_auto_consolidate_unsupported(); - Ok(false) - } - - /// See [`Self::auto_consolidate`]. - pub fn auto_consolidate_with_embedder( - &self, - _topic: &str, - _threshold: usize, - _embedder: Option<&dyn Embedder>, - ) -> IcmResult { - warn_auto_consolidate_unsupported(); - Ok(false) - } - - /// Pattern mining is not yet available on the PostgreSQL backend. - pub fn detect_patterns( - &self, - _topic: &str, - _min_cluster_size: usize, - ) -> IcmResult> { - Err(IcmError::Unsupported( - "detect_patterns (use the default SQLite backend)".into(), - )) - } - - /// Pattern mining is not yet available on the PostgreSQL backend. - pub fn extract_pattern_as_concept( - &self, - _cluster: &PatternCluster, - _memoir_id: &str, - ) -> IcmResult { - Err(IcmError::Unsupported( - "extract_pattern_as_concept (use the default SQLite backend)".into(), - )) - } -} - -/// Idempotent schema creation. Returns the embedding dimension the table -/// is actually using (the stored value wins over `requested_dims` on an -/// existing database). -fn init_schema(client: &mut Client, requested_dims: usize) -> IcmResult { - if !(64..=4096).contains(&requested_dims) { - return Err(IcmError::Config(format!( - "embedding_dims must be between 64 and 4096, got {requested_dims}" - ))); - } - - client - .batch_execute("CREATE EXTENSION IF NOT EXISTS vector") - .map_err(|e| { - IcmError::Database(format!( - "cannot enable the pgvector extension (need it for embeddings): {e}" - )) - })?; - - client - .batch_execute( - "CREATE TABLE IF NOT EXISTS icm_metadata ( - key TEXT PRIMARY KEY, - value TEXT - )", - ) - .map_err(pg_err)?; - - // The stored dimension is authoritative on an existing database. - let stored: Option = client - .query_opt( - "SELECT value::bigint FROM icm_metadata WHERE key = 'embedding_dims'", - &[], - ) - .map_err(pg_err)? - .map(|row| row.get(0)); - let dims = stored.map(|d| d as usize).unwrap_or(requested_dims); - - // Migration: the unique index used to be `(LOWER(topic), summary_hash)`. - // `summary_hash` already encodes the topic via Rust's Unicode-correct - // `to_lowercase()`, while Postgres's `LOWER()` behavior depends on the - // cluster's locale (ASCII-only under `C`/POSIX, common in minimal - // Docker/CI images) — the composite key could let two rows with an - // identical `summary_hash` coexist whenever their topic's SQL-LOWER() - // forms differed under that locale (audit finding, same class already - // fixed for SQLite). `CREATE INDEX IF NOT EXISTS` would silently keep - // the old definition on an already-migrated DB (same index name), so - // drop it first if it still has the old column list. - let old_index_def: Option = client - .query_opt( - "SELECT indexdef FROM pg_indexes WHERE indexname = 'idx_memories_topic_hash'", - &[], - ) - .map_err(pg_err)? - .map(|row| row.get(0)); - if let Some(def) = old_index_def { - if def.to_lowercase().contains("lower(topic") { - client - .batch_execute("DROP INDEX idx_memories_topic_hash;") - .map_err(pg_err)?; - } - } - - client - .batch_execute(&format!( - "CREATE TABLE IF NOT EXISTS memories ( - id TEXT PRIMARY KEY, - created_at TIMESTAMPTZ NOT NULL, - updated_at TIMESTAMPTZ NOT NULL, - last_accessed TIMESTAMPTZ NOT NULL, - access_count INTEGER NOT NULL DEFAULT 0, - weight REAL NOT NULL DEFAULT 1.0, - topic TEXT NOT NULL, - summary TEXT NOT NULL, - raw_excerpt TEXT, - keywords TEXT, - importance TEXT NOT NULL, - source_type TEXT NOT NULL, - source_data TEXT, - related_ids TEXT, - summary_hash TEXT, - embedding vector({dims}), - fts tsvector GENERATED ALWAYS AS ( - to_tsvector('simple', - coalesce(topic, '') || ' ' || - coalesce(summary, '') || ' ' || - coalesce(keywords, '')) - ) STORED - ); - - CREATE INDEX IF NOT EXISTS idx_memories_topic ON memories(topic); - CREATE INDEX IF NOT EXISTS idx_memories_weight ON memories(weight); - CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at); - CREATE INDEX IF NOT EXISTS idx_memories_fts ON memories USING GIN (fts); - CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_topic_hash - ON memories (summary_hash) WHERE summary_hash IS NOT NULL; - - CREATE TABLE IF NOT EXISTS pending_extractions ( - id TEXT PRIMARY KEY, - project TEXT NOT NULL, - tool_name TEXT NOT NULL, - raw_output TEXT NOT NULL, - captured_at TIMESTAMPTZ NOT NULL - ); - - CREATE TABLE IF NOT EXISTS code_areas ( - id BIGSERIAL PRIMARY KEY, - project TEXT NOT NULL, - file_path TEXT NOT NULL, - description TEXT, - session_id TEXT, - tool_name TEXT, - touch_count BIGINT NOT NULL DEFAULT 1, - first_touched_at TIMESTAMPTZ NOT NULL, - last_touched_at TIMESTAMPTZ NOT NULL, - UNIQUE (project, file_path) - ); - - CREATE TABLE IF NOT EXISTS hook_events ( - id BIGSERIAL PRIMARY KEY, - ts TIMESTAMPTZ NOT NULL, - event TEXT NOT NULL, - project TEXT, - session_id TEXT, - tool_name TEXT, - duration_ms BIGINT, - exit_code INTEGER NOT NULL DEFAULT 0, - payload_size BIGINT, - note TEXT - ); - CREATE INDEX IF NOT EXISTS idx_hook_events_ts ON hook_events(ts); - CREATE INDEX IF NOT EXISTS idx_hook_events_event ON hook_events(event);" - )) - .map_err(pg_err)?; - - // A vector index needs a concrete dimension; create it after the - // table exists. HNSW is available in pgvector >= 0.5 (the images we - // target ship a newer version). - client - .batch_execute( - "CREATE INDEX IF NOT EXISTS idx_memories_embedding - ON memories USING hnsw (embedding vector_cosine_ops)", - ) - .map_err(pg_err)?; - - client - .execute( - "INSERT INTO icm_metadata (key, value) VALUES ('embedding_dims', $1) - ON CONFLICT (key) DO NOTHING", - &[&dims.to_string()], - ) - .map_err(pg_err)?; - - Ok(dims) -} - -// --------------------------------------------------------------------------- -// MemoryStore -// --------------------------------------------------------------------------- - -impl MemoryStore for PostgresStore { - fn store(&self, memory: Memory) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("store".into())); - } - let memory = validate_and_normalize(memory)?; - self.check_dims(&memory)?; - let mut c = self.conn()?; - let mut tx = c.transaction().map_err(pg_err)?; - let id = insert_or_merge_memory(&mut tx, &memory)?; - tx.commit().map_err(pg_err)?; - Ok(id) - } - - fn get(&self, id: &str) -> IcmResult> { - let mut c = self.conn()?; - let row = c - .query_opt( - &format!("SELECT {SELECT_COLS} FROM memories WHERE id = $1"), - &[&id], - ) - .map_err(pg_err)?; - Ok(row.as_ref().map(row_to_memory)) - } - - fn update(&self, memory: &Memory) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("update".into())); - } - self.check_dims(memory)?; - let keywords_json = serde_json::to_string(&memory.keywords)?; - let related_json = serde_json::to_string(&memory.related_ids)?; - let st = source_type(&memory.source); - let sd = source_data(&memory.source); - let hash = summary_hash(&memory.topic, &memory.summary); - let importance = memory.importance.to_string(); - let access = memory.access_count as i32; - let emb: Option = memory - .embedding - .as_ref() - .map(|e| pgvector::Vector::from(e.clone())); - - let mut c = self.conn()?; - let changed = c - .execute( - "UPDATE memories SET - updated_at = $2, last_accessed = $3, access_count = $4, weight = $5, - topic = $6, summary = $7, raw_excerpt = $8, keywords = $9, - importance = $10, source_type = $11, source_data = $12, related_ids = $13, - embedding = $14, summary_hash = $15 - WHERE id = $1", - &[ - &memory.id, - &memory.updated_at, - &memory.last_accessed, - &access, - &memory.weight, - &memory.topic, - &memory.summary, - &memory.raw_excerpt, - &keywords_json, - &importance, - &st, - &sd, - &related_json, - &emb, - &hash, - ], - ) - .map_err(pg_err)?; - if changed == 0 { - return Err(IcmError::NotFound(memory.id.clone())); - } - Ok(()) - } - - fn delete(&self, id: &str) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("delete".into())); - } - let mut c = self.conn()?; - let mut tx = c.transaction().map_err(pg_err)?; - let changed = tx - .execute("DELETE FROM memories WHERE id = $1", &[&id]) - .map_err(pg_err)?; - if changed == 0 { - return Err(IcmError::NotFound(id.to_string())); - } - // Manual-testing finding (same class as the SQLite backend): a - // deleted memory otherwise stays as a dangling entry in every - // other memory's `related_ids` forever. Strip it out. `LIKE` is a - // cheap prefilter; the JSON-quoted match guards against a ULID - // that happens to be a literal substring of another. - tx.execute( - "UPDATE memories - SET related_ids = ( - SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)::text - FROM jsonb_array_elements_text(related_ids::jsonb) AS elem - WHERE elem != $1 - ) - WHERE related_ids LIKE '%\"' || $1 || '\"%'", - &[&id], - ) - .map_err(pg_err)?; - tx.commit().map_err(pg_err)?; - Ok(()) - } - - fn search_by_keywords(&self, keywords: &[&str], limit: usize) -> IcmResult> { - if keywords.is_empty() { - return Ok(Vec::new()); - } - let keywords = &keywords[..keywords.len().min(50)]; - let limit = limit.min(100); - - let mut owned: Vec> = Vec::new(); - let mut where_parts: Vec = Vec::new(); - for k in keywords { - // Audit finding: a keyword containing `%`/`_` was interpolated - // straight into the ILIKE pattern unescaped (same bug already - // fixed for SQLite's search_by_keywords). Escape and declare the - // escape character explicitly. - owned.push(Box::new(format!("%{}%", escape_like_wildcards(k)))); - let p = owned.len(); - where_parts.push(format!( - "(keywords ILIKE ${p} ESCAPE '\\' OR summary ILIKE ${p} ESCAPE '\\' \ - OR topic ILIKE ${p} ESCAPE '\\')" - )); - } - owned.push(Box::new(limit as i64)); - let sql = format!( - "SELECT {SELECT_COLS} FROM memories WHERE {} ORDER BY weight DESC LIMIT ${}", - where_parts.join(" OR "), - owned.len() - ); - let params: Vec<&(dyn ToSql + Sync)> = owned.iter().map(|b| b.as_ref()).collect(); - let mut c = self.conn()?; - let rows = c.query(&sql, ¶ms).map_err(pg_err)?; - Ok(rows.iter().map(row_to_memory).collect()) - } - - fn search_fts(&self, query: &str, limit: usize) -> IcmResult> { - let limit = limit.min(100); - if query.trim().is_empty() { - return Ok(Vec::new()); - } - let mut c = self.conn()?; - let rows = c - .query( - &format!( - "SELECT {SELECT_COLS} FROM memories \ - WHERE fts @@ websearch_to_tsquery('simple', $1) \ - ORDER BY weight DESC LIMIT $2" - ), - &[&query, &(limit as i64)], - ) - .map_err(pg_err)?; - Ok(rows.iter().map(row_to_memory).collect()) - } - - fn search_by_embedding( - &self, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - // Found while auditing OpenSearch's equivalent (which had no clamp - // at all): this function was also missing one, unlike its sibling - // search functions in this same file. - let limit = limit.min(1000); - let qv = pgvector::Vector::from(embedding.to_vec()); - let mut c = self.conn()?; - let rows = c - .query( - &format!( - "SELECT {SELECT_COLS}, embedding <=> $1 AS distance FROM memories \ - WHERE embedding IS NOT NULL ORDER BY embedding <=> $1 LIMIT $2" - ), - &[&qv, &(limit as i64)], - ) - .map_err(pg_err)?; - Ok(rows - .iter() - .map(|row| { - let distance: f64 = row.get(15); - (row_to_memory(row), 1.0 - distance as f32) - }) - .collect()) - } - - fn search_hybrid( - &self, - query: &str, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - let limit = limit.min(1000); - let pool_size = limit * 4; - - // 1. FTS candidates (id + rank). Lock released at scope end. - let fts_pairs: Vec<(String, f64)> = if query.trim().is_empty() { - Vec::new() - } else { - let mut c = self.conn()?; - let rows = c - .query( - "SELECT id, ts_rank_cd(fts, websearch_to_tsquery('simple', $1))::float8 AS rank \ - FROM memories \ - WHERE fts @@ websearch_to_tsquery('simple', $1) \ - ORDER BY rank DESC LIMIT $2", - &[&query, &(pool_size as i64)], - ) - .map_err(pg_err)?; - rows.iter().map(|r| (r.get(0), r.get(1))).collect() - }; - - // 2. Vector candidates (full rows + similarity). - let vec_results = self.search_by_embedding(embedding, pool_size)?; - - // 3. Assemble memory objects and per-source scores. - let mut all_memories: HashMap = HashMap::new(); - let mut vec_scores: HashMap = HashMap::new(); - for (mem, sim) in vec_results { - vec_scores.insert(mem.id.clone(), sim); - all_memories.insert(mem.id.clone(), mem); - } - - // Normalize FTS ranks into 0..1 within the pool (higher is better). - let max_rank = fts_pairs.iter().map(|(_, r)| *r).fold(0.0_f64, f64::max); - let mut fts_scores: HashMap = HashMap::new(); - let missing: Vec = fts_pairs - .iter() - .filter(|(id, _)| !all_memories.contains_key(id)) - .map(|(id, _)| id.clone()) - .collect(); - if !missing.is_empty() { - let refs: Vec<&str> = missing.iter().map(|s| s.as_str()).collect(); - let fetched = self.get_many(&refs)?; - for (id, m) in fetched { - all_memories.insert(id, m); - } - } - for (id, rank) in fts_pairs { - let score = if max_rank > 0.0 { - (rank / max_rank) as f32 - } else { - 0.0 - }; - fts_scores.insert(id, score); - } - - // 4. Blend: 30% FTS + 70% vector (matches the SQLite backend). - let mut scored: Vec<(String, f32)> = all_memories - .keys() - .map(|id| { - let fts = fts_scores.get(id).copied().unwrap_or(0.0); - let vec = vec_scores.get(id).copied().unwrap_or(0.0); - (id.clone(), 0.3 * fts + 0.7 * vec) - }) - .collect(); - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - scored.truncate(limit); - - Ok(scored - .into_iter() - .filter_map(|(id, score)| all_memories.remove(&id).map(|m| (m, score))) - .collect()) - } - - fn update_access(&self, id: &str) -> IcmResult<()> { - if self.readonly { - return Ok(()); - } - let mut c = self.conn()?; - let changed = c - .execute( - "UPDATE memories SET last_accessed = $1, access_count = access_count + 1 \ - WHERE id = $2", - &[&Utc::now(), &id], - ) - .map_err(pg_err)?; - if changed == 0 { - return Err(IcmError::NotFound(id.to_string())); - } - Ok(()) - } - - fn batch_update_access(&self, ids: &[&str]) -> IcmResult { - if ids.is_empty() || self.readonly { - return Ok(0); - } - let id_vec: Vec = ids.iter().map(|s| s.to_string()).collect(); - let mut c = self.conn()?; - let changed = c - .execute( - "UPDATE memories SET last_accessed = $1, access_count = access_count + 1 \ - WHERE id = ANY($2)", - &[&Utc::now(), &id_vec], - ) - .map_err(pg_err)?; - Ok(changed as usize) - } - - fn apply_decay(&self, decay_factor: f32) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("apply_decay".into())); - } - // Access-aware decay, capped at 5 accesses (matches SQLite). - let mut c = self.conn()?; - let changed = c - .execute( - // `$1::float8` is explicit so PostgreSQL doesn't infer the - // parameter as `numeric`/`real` from a neighbouring operand - // and reject the `f64` we bind ("error serializing parameter"). - // Audit finding: for `low` importance with low access count, - // the raw multiplier goes negative once decay_factor < 0.5 - // (still inside the CLI's own validated [0.0, 1.0) range) — - // the same bug already fixed for SQLite (GREATEST here is - // Postgres's equivalent of SQLite's MAX). See store.rs - // apply_decay for the full derivation. - "UPDATE memories SET weight = weight * GREATEST(0.0, - 1.0 - (1.0 - $1::float8) * - CASE importance - WHEN 'high' THEN 0.5 - WHEN 'low' THEN 2.0 - ELSE 1.0 - END - / (1.0 + LEAST(access_count, 5) * 0.1) - ) - WHERE importance <> 'critical'", - &[&(decay_factor as f64)], - ) - .map_err(pg_err)?; - Ok(changed as usize) - } - - fn prune(&self, weight_threshold: f32) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("prune".into())); - } - let mut c = self.conn()?; - let changed = c - .execute( - "DELETE FROM memories \ - WHERE weight < $1::float8 AND importance NOT IN ('critical', 'high')", - &[&(weight_threshold as f64)], - ) - .map_err(pg_err)?; - Ok(changed as usize) - } - - fn list_all(&self) -> IcmResult> { - let mut c = self.conn()?; - let rows = c - .query( - &format!("SELECT {SELECT_COLS} FROM memories ORDER BY weight DESC LIMIT 10000"), - &[], - ) - .map_err(pg_err)?; - Ok(rows.iter().map(row_to_memory).collect()) - } - - fn get_by_topic(&self, topic: &str) -> IcmResult> { - let mut c = self.conn()?; - let rows = c - .query( - &format!( - "SELECT {SELECT_COLS} FROM memories WHERE topic = $1 \ - ORDER BY weight DESC LIMIT 500" - ), - &[&topic], - ) - .map_err(pg_err)?; - Ok(rows.iter().map(row_to_memory).collect()) - } - - fn list_topics(&self) -> IcmResult> { - self.list_topics_with_prefix(None) - } - - fn consolidate_topic(&self, topic: &str, consolidated: Memory) -> IcmResult<()> { - if self.readonly { - return Err(IcmError::ReadOnly("consolidate_topic".into())); - } - // The consolidated memory goes through the same validation as any - // other write — an MCP-provided consolidate summary previously - // bypassed every size/NUL check (same gap already closed on SQLite). - let consolidated = validate_and_normalize(consolidated)?; - let mut c = self.conn()?; - let mut tx = c.transaction().map_err(pg_err)?; - // Audit finding: `critical` memories are never deleted — same - // contract `apply_decay`/`prune` already honor, and the same fix - // already applied to SQLite's consolidate_topic. This unconditional - // DELETE previously wiped critical memories in a consolidated topic - // too. - // Manual-testing finding: captured before the delete, since - // afterward these rows are gone. Used to clean up any *other* - // memory's related_ids that pointed at them — same dangling- - // reference bug already fixed for the single-id `delete`. - let deleted_ids: Vec = tx - .query( - "SELECT id FROM memories WHERE topic = $1 AND importance <> 'critical'", - &[&topic], - ) - .map_err(pg_err)? - .iter() - .map(|row| row.get(0)) - .collect(); - - tx.execute( - "DELETE FROM memories WHERE topic = $1 AND importance <> 'critical'", - &[&topic], - ) - .map_err(pg_err)?; - - if !deleted_ids.is_empty() { - tx.execute( - "UPDATE memories - SET related_ids = ( - SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)::text - FROM jsonb_array_elements_text(related_ids::jsonb) AS elem - WHERE elem <> ALL($1::text[]) - ) - WHERE related_ids::jsonb ?| $1::text[]", - &[&deleted_ids], - ) - .map_err(pg_err)?; - } - - insert_or_merge_memory(&mut tx, &consolidated)?; - tx.commit().map_err(pg_err)?; - Ok(()) - } - - fn count(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one("SELECT COUNT(*) FROM memories", &[]) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - fn count_by_topic(&self, topic: &str) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one("SELECT COUNT(*) FROM memories WHERE topic = $1", &[&topic]) - .map_err(pg_err)?; - let n: i64 = row.get(0); - Ok(n.max(0) as usize) - } - - fn stats(&self) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one( - "SELECT COUNT(*)::bigint, COUNT(DISTINCT topic)::bigint, \ - COALESCE(AVG(weight), 0.0)::float8, MIN(created_at), MAX(created_at) \ - FROM memories", - &[], - ) - .map_err(pg_err)?; - let total: i64 = row.get(0); - let topics: i64 = row.get(1); - let avg: f64 = row.get(2); - Ok(StoreStats { - total_memories: total.max(0) as usize, - total_topics: topics.max(0) as usize, - avg_weight: avg as f32, - oldest_memory: row.get(3), - newest_memory: row.get(4), - }) - } - - fn topic_health(&self, topic: &str) -> IcmResult { - let mut c = self.conn()?; - let row = c - .query_one( - "SELECT COUNT(*)::bigint, - COALESCE(AVG(weight), 0)::float8, - COALESCE(AVG(access_count::float8), 0)::float8, - MIN(created_at), MAX(created_at), MAX(last_accessed), - COALESCE(SUM(CASE WHEN weight < 0.5 - AND (now() - last_accessed) > interval '14 days' - THEN 1 ELSE 0 END), 0)::bigint - FROM memories WHERE topic = $1", - &[&topic], - ) - .map_err(pg_err)?; - - let entry_count: i64 = row.get(0); - if entry_count == 0 { - return Err(IcmError::NotFound(format!("topic: {topic}"))); - } - let avg_weight: f64 = row.get(1); - let avg_access: f64 = row.get(2); - let stale: i64 = row.get(6); - - Ok(TopicHealth { - topic: topic.to_string(), - entry_count: entry_count.max(0) as usize, - avg_weight: avg_weight as f32, - avg_access_count: avg_access as f32, - oldest: row.get(3), - newest: row.get(4), - last_accessed: row.get(5), - needs_consolidation: entry_count > 5, - stale_count: stale.max(0) as usize, - }) - } -} - -// --------------------------------------------------------------------------- -// Unsupported subsystems on this backend (first cut, issue #301). -// -// These return `IcmError::Unsupported` so the binary keeps working for the -// core shared-memory use case while the heavier subsystems remain on the -// default SQLite backend. A follow-up can port them. -// --------------------------------------------------------------------------- - -fn unsupported(op: &str) -> IcmResult { - Err(IcmError::Unsupported(format!( - "{op} (use the default SQLite backend)" - ))) -} - -impl MemoirStore for PostgresStore { - fn create_memoir(&self, _memoir: Memoir) -> IcmResult { - unsupported("memoir.create_memoir") - } - fn get_memoir(&self, _id: &str) -> IcmResult> { - unsupported("memoir.get_memoir") - } - fn get_memoir_by_name(&self, _name: &str) -> IcmResult> { - unsupported("memoir.get_memoir_by_name") - } - fn update_memoir(&self, _memoir: &Memoir) -> IcmResult<()> { - unsupported("memoir.update_memoir") - } - fn delete_memoir(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_memoir") - } - fn list_memoirs(&self) -> IcmResult> { - unsupported("memoir.list_memoirs") - } - fn add_concept(&self, _concept: Concept) -> IcmResult { - unsupported("memoir.add_concept") - } - fn get_concept(&self, _id: &str) -> IcmResult> { - unsupported("memoir.get_concept") - } - fn get_concept_by_name(&self, _memoir_id: &str, _name: &str) -> IcmResult> { - unsupported("memoir.get_concept_by_name") - } - fn update_concept(&self, _concept: &Concept) -> IcmResult<()> { - unsupported("memoir.update_concept") - } - fn delete_concept(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_concept") - } - fn list_concepts(&self, _memoir_id: &str) -> IcmResult> { - unsupported("memoir.list_concepts") - } - fn search_concepts_fts( - &self, - _memoir_id: &str, - _query: &str, - _limit: usize, - ) -> IcmResult> { - unsupported("memoir.search_concepts_fts") - } - fn search_concepts_by_label( - &self, - _memoir_id: &str, - _label: &Label, - _limit: usize, - ) -> IcmResult> { - unsupported("memoir.search_concepts_by_label") - } - fn search_all_concepts_fts(&self, _query: &str, _limit: usize) -> IcmResult> { - unsupported("memoir.search_all_concepts_fts") - } - fn refine_concept( - &self, - _id: &str, - _new_definition: &str, - _new_source_ids: &[String], - ) -> IcmResult<()> { - unsupported("memoir.refine_concept") - } - fn add_link(&self, _link: ConceptLink) -> IcmResult { - unsupported("memoir.add_link") - } - fn get_links_from(&self, _concept_id: &str) -> IcmResult> { - unsupported("memoir.get_links_from") - } - fn get_links_to(&self, _concept_id: &str) -> IcmResult> { - unsupported("memoir.get_links_to") - } - fn delete_link(&self, _id: &str) -> IcmResult<()> { - unsupported("memoir.delete_link") - } - fn get_neighbors( - &self, - _concept_id: &str, - _relation: Option, - ) -> IcmResult> { - unsupported("memoir.get_neighbors") - } - fn get_neighborhood( - &self, - _concept_id: &str, - _depth: usize, - ) -> IcmResult<(Vec, Vec)> { - unsupported("memoir.get_neighborhood") - } - fn get_links_for_memoir(&self, _memoir_id: &str) -> IcmResult> { - unsupported("memoir.get_links_for_memoir") - } - fn memoir_stats(&self, _memoir_id: &str) -> IcmResult { - unsupported("memoir.memoir_stats") - } - fn batch_memoir_concept_counts(&self) -> IcmResult> { - unsupported("memoir.batch_memoir_concept_counts") - } -} - -impl FeedbackStore for PostgresStore { - fn store_feedback(&self, _feedback: Feedback) -> IcmResult { - unsupported("feedback.store_feedback") - } - fn search_feedback( - &self, - _query: &str, - _query_embedding: Option<&[f32]>, - _topic: Option<&str>, - _limit: usize, - ) -> IcmResult> { - unsupported("feedback.search_feedback") - } - fn list_feedback(&self, _topic: Option<&str>, _limit: usize) -> IcmResult> { - unsupported("feedback.list_feedback") - } - fn increment_applied(&self, _id: &str) -> IcmResult<()> { - unsupported("feedback.increment_applied") - } - fn delete_feedback(&self, _id: &str) -> IcmResult<()> { - unsupported("feedback.delete_feedback") - } - fn feedback_stats(&self) -> IcmResult { - unsupported("feedback.feedback_stats") - } -} - -impl FactsStore for PostgresStore { - fn set_fact( - &self, - _entity: &str, - _key: &str, - _value: &str, - _source: &str, - ) -> IcmResult { - unsupported("facts.set_fact") - } - fn get_fact(&self, _entity: &str, _key: &str) -> IcmResult> { - unsupported("facts.get_fact") - } - fn list_facts(&self, _entity: &str, _key_prefix: Option<&str>) -> IcmResult> { - unsupported("facts.list_facts") - } - fn history(&self, _entity: &str, _key: &str) -> IcmResult> { - unsupported("facts.history") - } - fn forget_fact(&self, _entity: &str, _key: &str) -> IcmResult { - unsupported("facts.forget_fact") - } - fn facts_stats(&self) -> IcmResult { - unsupported("facts.facts_stats") - } -} - -impl TranscriptStore for PostgresStore { - fn create_session( - &self, - _agent: &str, - _project: Option<&str>, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.create_session") - } - fn ensure_session( - &self, - _id: &str, - _agent: &str, - _project: Option<&str>, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.ensure_session") - } - fn get_session(&self, _id: &str) -> IcmResult> { - unsupported("transcript.get_session") - } - fn list_sessions(&self, _project: Option<&str>, _limit: usize) -> IcmResult> { - unsupported("transcript.list_sessions") - } - fn record_message( - &self, - _session_id: &str, - _role: Role, - _content: &str, - _tool_name: Option<&str>, - _tokens: Option, - _metadata: Option<&str>, - ) -> IcmResult { - unsupported("transcript.record_message") - } - fn list_session_messages( - &self, - _session_id: &str, - _limit: usize, - _offset: usize, - ) -> IcmResult> { - unsupported("transcript.list_session_messages") - } - fn search_transcripts( - &self, - _query: &str, - _session_id: Option<&str>, - _project: Option<&str>, - _limit: usize, - ) -> IcmResult> { - unsupported("transcript.search_transcripts") - } - fn forget_session(&self, _id: &str) -> IcmResult<()> { - unsupported("transcript.forget_session") - } - fn transcript_stats(&self) -> IcmResult { - unsupported("transcript.transcript_stats") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Audit regression: a keyword containing `%`/`_` was interpolated - /// straight into an ILIKE pattern unescaped, turning it into an - /// unintended wildcard. - #[test] - fn test_escape_like_wildcards() { - assert_eq!(escape_like_wildcards("100%"), "100\\%"); - assert_eq!(escape_like_wildcards("snake_case"), "snake\\_case"); - assert_eq!(escape_like_wildcards("back\\slash"), "back\\\\slash"); - assert_eq!(escape_like_wildcards("plain"), "plain"); - } - - /// Audit regression: `apply_decay`'s raw multiplier goes negative for - /// `low` importance + low access count at factor < 0.5 (still inside - /// the CLI's own validated [0.0, 1.0) range). This reproduces the exact - /// arithmetic the SQL `GREATEST(0.0, ...)` clamp now guards, as a plain - /// Rust assertion (no live Postgres needed to prove the formula itself - /// would go negative without the clamp). - #[test] - fn test_apply_decay_formula_would_go_negative_without_clamp() { - let factor: f64 = 0.4; - let mult: f64 = 2.0; // low importance - let access: f64 = 0.0; - let raw = 1.0 - (1.0 - factor) * mult / (1.0 + access * 0.1); - assert!( - raw < 0.0, - "expected the pre-clamp formula to go negative, got {raw}" - ); - assert_eq!( - raw.max(0.0), - 0.0, - "GREATEST(0.0, ...) must clamp this to 0.0" - ); - } -} diff --git a/crates/icm-store/src/postgres/connection.rs b/crates/icm-store/src/postgres/connection.rs new file mode 100644 index 00000000..114598fc --- /dev/null +++ b/crates/icm-store/src/postgres/connection.rs @@ -0,0 +1,260 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Connection setup, constructors, and schema migration. + +use super::*; + +impl PostgresStore { + pub(crate) fn conn(&self) -> IcmResult> { + self.client.lock().map_err(|_| lock_err()) + } + + /// Resolve the connection string from the environment. + pub(crate) fn conn_string() -> IcmResult { + std::env::var("ICM_POSTGRES_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .map_err(|_| { + IcmError::Config( + "PostgreSQL backend: set ICM_POSTGRES_URL (or DATABASE_URL) to the \ + connection string, e.g. postgres://user:pass@host:5432/icm" + .into(), + ) + }) + } + + /// Connect and run the idempotent schema migration. + /// + /// The `&Path` the CLI passes for the SQLite file is ignored; the + /// connection comes from the environment. `requested_dims` is used + /// only when the database is fresh — an existing database's stored + /// `embedding_dims` is authoritative so we never try to declare a + /// `vector(N)` column that disagrees with the live table. + pub(crate) fn connect(requested_dims: usize, readonly: bool) -> IcmResult { + let url = Self::conn_string()?; + let mut client = Client::connect(&url, NoTls) + .map_err(|e| IcmError::Database(format!("cannot connect to PostgreSQL: {e}")))?; + + let dims = init_schema(&mut client, requested_dims)?; + + Ok(Self { + client: Mutex::new(client), + embedding_dims: dims, + readonly, + }) + } + + /// Reject an embedding whose length disagrees with the column's + /// declared dimension, with a clearer message than the raw PostgreSQL + /// "expected N dimensions, not M" error. + pub(crate) fn check_dims(&self, memory: &Memory) -> IcmResult<()> { + if let Some(emb) = memory.embedding.as_ref() { + if emb.len() != self.embedding_dims { + return Err(IcmError::InvalidInput(format!( + "embedding has {} dimensions, but this store uses {}", + emb.len(), + self.embedding_dims + ))); + } + } + Ok(()) + } + + /// Open or create a store with the default embedding dimension. + pub fn new(_path: &Path) -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) + } + + /// Open or create a store with a specific embedding dimension. + pub fn with_dims(_path: &Path, embedding_dims: usize) -> IcmResult { + Self::connect(embedding_dims, false) + } + + /// Open the store in read-only mode (issue #263). The connection is + /// the same; write methods refuse, read-like side effects are skipped. + pub fn open_readonly(_path: &Path) -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, true) + } + + /// PostgreSQL has no in-memory mode; connect to the configured + /// database. Provided for API parity with the SQLite backend. + pub fn in_memory() -> IcmResult { + Self::connect(icm_core::DEFAULT_EMBEDDING_DIMS, false) + } + + /// See [`Self::in_memory`]. + pub fn in_memory_with_dims(embedding_dims: usize) -> IcmResult { + Self::connect(embedding_dims, false) + } + + /// PostgreSQL stores `embedding_dims` in `icm_metadata`, but unlike + /// SQLite it never destructively recreates the vector column, so the + /// pre-open peek the SQLite backend needs is unnecessary here. Always + /// returns `Ok(None)` so callers fall through to the normal open path. + pub fn read_stored_embedding_dims(_path: &Path) -> IcmResult> { + Ok(None) + } + + #[must_use] + pub fn is_readonly(&self) -> bool { + self.readonly + } + + /// No-op on PostgreSQL (the SQLite backend uses this to load the + /// `sqlite-vec` extension; `pgvector` lives server-side). + pub fn ensure_vec_init() {} +} + +/// Idempotent schema creation. Returns the embedding dimension the table +/// is actually using (the stored value wins over `requested_dims` on an +/// existing database). +fn init_schema(client: &mut Client, requested_dims: usize) -> IcmResult { + if !(64..=4096).contains(&requested_dims) { + return Err(IcmError::Config(format!( + "embedding_dims must be between 64 and 4096, got {requested_dims}" + ))); + } + + client + .batch_execute("CREATE EXTENSION IF NOT EXISTS vector") + .map_err(|e| { + IcmError::Database(format!( + "cannot enable the pgvector extension (need it for embeddings): {e}" + )) + })?; + + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS icm_metadata ( + key TEXT PRIMARY KEY, + value TEXT + )", + ) + .map_err(pg_err)?; + + // The stored dimension is authoritative on an existing database. + let stored: Option = client + .query_opt( + "SELECT value::bigint FROM icm_metadata WHERE key = 'embedding_dims'", + &[], + ) + .map_err(pg_err)? + .map(|row| row.get(0)); + let dims = stored.map(|d| d as usize).unwrap_or(requested_dims); + + // Migration: the unique index used to be `(LOWER(topic), summary_hash)`. + // `summary_hash` already encodes the topic via Rust's Unicode-correct + // `to_lowercase()`, while Postgres's `LOWER()` behavior depends on the + // cluster's locale (ASCII-only under `C`/POSIX, common in minimal + // Docker/CI images) — the composite key could let two rows with an + // identical `summary_hash` coexist whenever their topic's SQL-LOWER() + // forms differed under that locale (audit finding, same class already + // fixed for SQLite). `CREATE INDEX IF NOT EXISTS` would silently keep + // the old definition on an already-migrated DB (same index name), so + // drop it first if it still has the old column list. + let old_index_def: Option = client + .query_opt( + "SELECT indexdef FROM pg_indexes WHERE indexname = 'idx_memories_topic_hash'", + &[], + ) + .map_err(pg_err)? + .map(|row| row.get(0)); + if let Some(def) = old_index_def { + if def.to_lowercase().contains("lower(topic") { + client + .batch_execute("DROP INDEX idx_memories_topic_hash;") + .map_err(pg_err)?; + } + } + + client + .batch_execute(&format!( + "CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + last_accessed TIMESTAMPTZ NOT NULL, + access_count INTEGER NOT NULL DEFAULT 0, + weight REAL NOT NULL DEFAULT 1.0, + topic TEXT NOT NULL, + summary TEXT NOT NULL, + raw_excerpt TEXT, + keywords TEXT, + importance TEXT NOT NULL, + source_type TEXT NOT NULL, + source_data TEXT, + related_ids TEXT, + summary_hash TEXT, + embedding vector({dims}), + fts tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + coalesce(topic, '') || ' ' || + coalesce(summary, '') || ' ' || + coalesce(keywords, '')) + ) STORED + ); + + CREATE INDEX IF NOT EXISTS idx_memories_topic ON memories(topic); + CREATE INDEX IF NOT EXISTS idx_memories_weight ON memories(weight); + CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at); + CREATE INDEX IF NOT EXISTS idx_memories_fts ON memories USING GIN (fts); + CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_topic_hash + ON memories (summary_hash) WHERE summary_hash IS NOT NULL; + + CREATE TABLE IF NOT EXISTS pending_extractions ( + id TEXT PRIMARY KEY, + project TEXT NOT NULL, + tool_name TEXT NOT NULL, + raw_output TEXT NOT NULL, + captured_at TIMESTAMPTZ NOT NULL + ); + + CREATE TABLE IF NOT EXISTS code_areas ( + id BIGSERIAL PRIMARY KEY, + project TEXT NOT NULL, + file_path TEXT NOT NULL, + description TEXT, + session_id TEXT, + tool_name TEXT, + touch_count BIGINT NOT NULL DEFAULT 1, + first_touched_at TIMESTAMPTZ NOT NULL, + last_touched_at TIMESTAMPTZ NOT NULL, + UNIQUE (project, file_path) + ); + + CREATE TABLE IF NOT EXISTS hook_events ( + id BIGSERIAL PRIMARY KEY, + ts TIMESTAMPTZ NOT NULL, + event TEXT NOT NULL, + project TEXT, + session_id TEXT, + tool_name TEXT, + duration_ms BIGINT, + exit_code INTEGER NOT NULL DEFAULT 0, + payload_size BIGINT, + note TEXT + ); + CREATE INDEX IF NOT EXISTS idx_hook_events_ts ON hook_events(ts); + CREATE INDEX IF NOT EXISTS idx_hook_events_event ON hook_events(event);" + )) + .map_err(pg_err)?; + + // A vector index needs a concrete dimension; create it after the + // table exists. HNSW is available in pgvector >= 0.5 (the images we + // target ship a newer version). + client + .batch_execute( + "CREATE INDEX IF NOT EXISTS idx_memories_embedding + ON memories USING hnsw (embedding vector_cosine_ops)", + ) + .map_err(pg_err)?; + + client + .execute( + "INSERT INTO icm_metadata (key, value) VALUES ('embedding_dims', $1) + ON CONFLICT (key) DO NOTHING", + &[&dims.to_string()], + ) + .map_err(pg_err)?; + + Ok(dims) +} diff --git a/crates/icm-store/src/postgres/facts.rs b/crates/icm-store/src/postgres/facts.rs new file mode 100644 index 00000000..a8b8541c --- /dev/null +++ b/crates/icm-store/src/postgres/facts.rs @@ -0,0 +1,32 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl FactsStore for PostgresStore { + fn set_fact( + &self, + _entity: &str, + _key: &str, + _value: &str, + _source: &str, + ) -> IcmResult { + unsupported("facts.set_fact") + } + fn get_fact(&self, _entity: &str, _key: &str) -> IcmResult> { + unsupported("facts.get_fact") + } + fn list_facts(&self, _entity: &str, _key_prefix: Option<&str>) -> IcmResult> { + unsupported("facts.list_facts") + } + fn history(&self, _entity: &str, _key: &str) -> IcmResult> { + unsupported("facts.history") + } + fn forget_fact(&self, _entity: &str, _key: &str) -> IcmResult { + unsupported("facts.forget_fact") + } + fn facts_stats(&self) -> IcmResult { + unsupported("facts.facts_stats") + } +} diff --git a/crates/icm-store/src/postgres/feedback.rs b/crates/icm-store/src/postgres/feedback.rs new file mode 100644 index 00000000..96a78b44 --- /dev/null +++ b/crates/icm-store/src/postgres/feedback.rs @@ -0,0 +1,32 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl FeedbackStore for PostgresStore { + fn store_feedback(&self, _feedback: Feedback) -> IcmResult { + unsupported("feedback.store_feedback") + } + fn search_feedback( + &self, + _query: &str, + _query_embedding: Option<&[f32]>, + _topic: Option<&str>, + _limit: usize, + ) -> IcmResult> { + unsupported("feedback.search_feedback") + } + fn list_feedback(&self, _topic: Option<&str>, _limit: usize) -> IcmResult> { + unsupported("feedback.list_feedback") + } + fn increment_applied(&self, _id: &str) -> IcmResult<()> { + unsupported("feedback.increment_applied") + } + fn delete_feedback(&self, _id: &str) -> IcmResult<()> { + unsupported("feedback.delete_feedback") + } + fn feedback_stats(&self) -> IcmResult { + unsupported("feedback.feedback_stats") + } +} diff --git a/crates/icm-store/src/postgres/hooks.rs b/crates/icm-store/src/postgres/hooks.rs new file mode 100644 index 00000000..e16589c6 --- /dev/null +++ b/crates/icm-store/src/postgres/hooks.rs @@ -0,0 +1,335 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Hook telemetry, the extraction queue, and code areas. + +use super::*; + +impl PostgresStore { + /// Atomically increment the hook call counter and return the new value. + pub fn increment_hook_counter(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one( + "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '1') + ON CONFLICT (key) DO UPDATE SET value = ((icm_metadata.value::bigint) + 1)::text + RETURNING value::bigint", + &[], + ) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } + + /// Reset the hook call counter to 0. + pub fn reset_hook_counter(&self) -> IcmResult<()> { + let mut c = self.conn()?; + c.execute( + "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '0') + ON CONFLICT (key) DO UPDATE SET value = '0'", + &[], + ) + .map_err(pg_err)?; + Ok(()) + } + + // Async extraction queue + + /// Enqueue raw tool output for later LLM extraction. + pub fn enqueue_pending_extraction( + &self, + project: &str, + tool_name: &str, + raw_output: &str, + ) -> IcmResult { + let id = ulid::Ulid::new().to_string(); + let mut c = self.conn()?; + c.execute( + "INSERT INTO pending_extractions (id, project, tool_name, raw_output, captured_at) + VALUES ($1, $2, $3, $4, $5)", + &[&id, &project, &tool_name, &raw_output, &Utc::now()], + ) + .map_err(pg_err)?; + Ok(id) + } + + /// Pop up to `limit` oldest pending rows (FIFO by capture time). + pub fn list_pending_extractions(&self, limit: usize) -> IcmResult> { + let mut c = self.conn()?; + let rows = c + .query( + "SELECT id, project, tool_name, raw_output, captured_at + FROM pending_extractions + ORDER BY captured_at ASC + LIMIT $1", + &[&(limit as i64)], + ) + .map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| { + let captured: DateTime = row.get(4); + ( + row.get(0), + row.get(1), + row.get(2), + row.get(3), + captured.to_rfc3339(), + ) + }) + .collect()) + } + + /// Delete pending rows by id. Used after a worker has processed them. + pub fn delete_pending_extractions(&self, ids: &[String]) -> IcmResult { + if ids.is_empty() { + return Ok(0); + } + let ids_vec: Vec = ids.to_vec(); + let mut c = self.conn()?; + let n = c + .execute( + "DELETE FROM pending_extractions WHERE id = ANY($1)", + &[&ids_vec], + ) + .map_err(pg_err)?; + Ok(n as usize) + } + + /// Total rows currently waiting in the queue. + pub fn pending_extraction_count(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one("SELECT COUNT(*) FROM pending_extractions", &[]) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } + + // Code areas (issue #196) + + /// Insert or refresh a row for `(project, file_path)`. + pub fn upsert_code_area( + &self, + project: &str, + file_path: &str, + description: Option<&str>, + session_id: Option<&str>, + tool_name: Option<&str>, + ) -> IcmResult<()> { + let now = Utc::now(); + let mut c = self.conn()?; + c.execute( + "INSERT INTO code_areas + (project, file_path, description, session_id, tool_name, + touch_count, first_touched_at, last_touched_at) + VALUES ($1, $2, $3, $4, $5, 1, $6, $6) + ON CONFLICT (project, file_path) DO UPDATE SET + touch_count = code_areas.touch_count + 1, + last_touched_at = EXCLUDED.last_touched_at, + session_id = COALESCE(EXCLUDED.session_id, code_areas.session_id), + tool_name = COALESCE(EXCLUDED.tool_name, code_areas.tool_name), + description = COALESCE(EXCLUDED.description, code_areas.description)", + &[ + &project, + &file_path, + &description, + &session_id, + &tool_name, + &now, + ], + ) + .map_err(pg_err)?; + Ok(()) + } + + /// List code areas, optionally filtered, newest-touch first. + pub fn list_code_areas( + &self, + project: Option<&str>, + in_file: Option<&str>, + since: Option>, + limit: usize, + ) -> IcmResult> { + let mut sql = String::from( + "SELECT id, project, file_path, description, session_id, tool_name, + touch_count, first_touched_at, last_touched_at + FROM code_areas WHERE TRUE", + ); + let mut owned: Vec> = Vec::new(); + if let Some(p) = project { + owned.push(Box::new(p.to_string())); + sql.push_str(&format!(" AND project = ${}", owned.len())); + } + if let Some(f) = in_file { + owned.push(Box::new(f.to_string())); + let exact = owned.len(); + owned.push(Box::new(format!("%/{f}"))); + let suffix = owned.len(); + sql.push_str(&format!( + " AND (file_path = ${exact} OR file_path LIKE ${suffix})" + )); + } + if let Some(t) = since { + owned.push(Box::new(t)); + sql.push_str(&format!(" AND last_touched_at >= ${}", owned.len())); + } + owned.push(Box::new(limit as i64)); + sql.push_str(&format!( + " ORDER BY last_touched_at DESC LIMIT ${}", + owned.len() + )); + + let params: Vec<&(dyn ToSql + Sync)> = owned.iter().map(|b| b.as_ref()).collect(); + let mut c = self.conn()?; + let rows = c.query(&sql, ¶ms).map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| CodeArea { + id: row.get(0), + project: row.get(1), + file_path: row.get(2), + description: row.get(3), + session_id: row.get(4), + tool_name: row.get(5), + touch_count: row.get(6), + first_touched_at: row.get(7), + last_touched_at: row.get(8), + }) + .collect()) + } + + /// Total rows in `code_areas`. + pub fn code_area_count(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one("SELECT COUNT(*) FROM code_areas", &[]) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } + + // Hook telemetry + + /// Append one hook telemetry row, returning its id. + pub fn record_hook_event(&self, ev: &HookEventInsert) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one( + "INSERT INTO hook_events + (ts, event, project, session_id, tool_name, + duration_ms, exit_code, payload_size, note) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING id", + &[ + &Utc::now(), + &ev.event, + &ev.project, + &ev.session_id, + &ev.tool_name, + &ev.duration_ms, + &ev.exit_code, + &ev.payload_size, + &ev.note, + ], + ) + .map_err(pg_err)?; + Ok(row.get(0)) + } + + /// Most recent `limit` hook events, newest first; optional event filter. + pub fn hook_events_recent( + &self, + limit: usize, + event_filter: Option<&str>, + ) -> IcmResult> { + let mut c = self.conn()?; + let rows = match event_filter { + Some(ev) => c.query( + "SELECT id, ts, event, project, session_id, tool_name, + duration_ms, exit_code, payload_size, note + FROM hook_events WHERE event = $1 ORDER BY id DESC LIMIT $2", + &[&ev, &(limit as i64)], + ), + None => c.query( + "SELECT id, ts, event, project, session_id, tool_name, + duration_ms, exit_code, payload_size, note + FROM hook_events ORDER BY id DESC LIMIT $1", + &[&(limit as i64)], + ), + } + .map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| HookEvent { + id: row.get(0), + ts: row.get(1), + event: row.get(2), + project: row.get(3), + session_id: row.get(4), + tool_name: row.get(5), + duration_ms: row.get(6), + exit_code: row.get(7), + payload_size: row.get(8), + note: row.get(9), + }) + .collect()) + } + + /// Per-event aggregate stats since `since_rfc3339`. + pub fn hook_stats(&self, since_rfc3339: &str) -> IcmResult> { + let since = DateTime::parse_from_rfc3339(since_rfc3339) + .map(|d| d.with_timezone(&Utc)) + .unwrap_or_else(|_| Utc::now() - chrono::Duration::days(7)); + let mut c = self.conn()?; + let rows = c + .query( + "SELECT event, + COUNT(*)::bigint, + COUNT(*) FILTER (WHERE exit_code <> 0)::bigint, + COALESCE(AVG(duration_ms), 0)::float8, + COALESCE(percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms), 0)::float8, + COALESCE(percentile_cont(0.99) WITHIN GROUP (ORDER BY duration_ms), 0)::float8 + FROM hook_events WHERE ts >= $1 + GROUP BY event ORDER BY event", + &[&since], + ) + .map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| { + let p50: f64 = row.get(4); + let p99: f64 = row.get(5); + HookStatsRow { + event: row.get(0), + count: row.get(1), + error_count: row.get(2), + avg_duration_ms: row.get(3), + p50_duration_ms: p50 as i64, + p99_duration_ms: p99 as i64, + } + }) + .collect()) + } + + /// Delete hook events older than `cutoff_rfc3339`. + pub fn prune_hook_events(&self, cutoff_rfc3339: &str) -> IcmResult { + let cutoff = DateTime::parse_from_rfc3339(cutoff_rfc3339) + .map(|d| d.with_timezone(&Utc)) + .map_err(|e| IcmError::InvalidInput(format!("invalid cutoff timestamp: {e}")))?; + let mut c = self.conn()?; + let n = c + .execute("DELETE FROM hook_events WHERE ts < $1", &[&cutoff]) + .map_err(pg_err)?; + Ok(n as usize) + } + + /// Total rows in `hook_events`. + pub fn hook_event_count(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one("SELECT COUNT(*) FROM hook_events", &[]) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } +} diff --git a/crates/icm-store/src/postgres/maintenance.rs b/crates/icm-store/src/postgres/maintenance.rs new file mode 100644 index 00000000..b852f8eb --- /dev/null +++ b/crates/icm-store/src/postgres/maintenance.rs @@ -0,0 +1,31 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Decay/auto-consolidate maintenance methods. + +use super::*; + +impl PostgresStore { + /// Apply decay if more than 24 hours since the last run. Mirrors the + /// SQLite backend's atomic check-and-claim via `icm_metadata`. + pub fn maybe_auto_decay(&self) -> IcmResult<()> { + if self.readonly { + return Ok(()); + } + let now = Utc::now(); + let claimed = { + let mut c = self.conn()?; + c.execute( + "INSERT INTO icm_metadata (key, value) VALUES ('last_decay_at', $1) + ON CONFLICT (key) DO UPDATE SET value = $1 + WHERE icm_metadata.value IS NULL + OR ($1::timestamptz - icm_metadata.value::timestamptz) >= interval '1 day'", + &[&now.to_rfc3339()], + ) + .map_err(pg_err)? + }; + if claimed > 0 { + self.apply_decay(0.95)?; + } + Ok(()) + } +} diff --git a/crates/icm-store/src/postgres/memoir.rs b/crates/icm-store/src/postgres/memoir.rs new file mode 100644 index 00000000..21605921 --- /dev/null +++ b/crates/icm-store/src/postgres/memoir.rs @@ -0,0 +1,106 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl MemoirStore for PostgresStore { + fn create_memoir(&self, _memoir: Memoir) -> IcmResult { + unsupported("memoir.create_memoir") + } + fn get_memoir(&self, _id: &str) -> IcmResult> { + unsupported("memoir.get_memoir") + } + fn get_memoir_by_name(&self, _name: &str) -> IcmResult> { + unsupported("memoir.get_memoir_by_name") + } + fn update_memoir(&self, _memoir: &Memoir) -> IcmResult<()> { + unsupported("memoir.update_memoir") + } + fn delete_memoir(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_memoir") + } + fn list_memoirs(&self) -> IcmResult> { + unsupported("memoir.list_memoirs") + } + fn add_concept(&self, _concept: Concept) -> IcmResult { + unsupported("memoir.add_concept") + } + fn get_concept(&self, _id: &str) -> IcmResult> { + unsupported("memoir.get_concept") + } + fn get_concept_by_name(&self, _memoir_id: &str, _name: &str) -> IcmResult> { + unsupported("memoir.get_concept_by_name") + } + fn update_concept(&self, _concept: &Concept) -> IcmResult<()> { + unsupported("memoir.update_concept") + } + fn delete_concept(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_concept") + } + fn list_concepts(&self, _memoir_id: &str) -> IcmResult> { + unsupported("memoir.list_concepts") + } + fn search_concepts_fts( + &self, + _memoir_id: &str, + _query: &str, + _limit: usize, + ) -> IcmResult> { + unsupported("memoir.search_concepts_fts") + } + fn search_concepts_by_label( + &self, + _memoir_id: &str, + _label: &Label, + _limit: usize, + ) -> IcmResult> { + unsupported("memoir.search_concepts_by_label") + } + fn search_all_concepts_fts(&self, _query: &str, _limit: usize) -> IcmResult> { + unsupported("memoir.search_all_concepts_fts") + } + fn refine_concept( + &self, + _id: &str, + _new_definition: &str, + _new_source_ids: &[String], + ) -> IcmResult<()> { + unsupported("memoir.refine_concept") + } + fn add_link(&self, _link: ConceptLink) -> IcmResult { + unsupported("memoir.add_link") + } + fn get_links_from(&self, _concept_id: &str) -> IcmResult> { + unsupported("memoir.get_links_from") + } + fn get_links_to(&self, _concept_id: &str) -> IcmResult> { + unsupported("memoir.get_links_to") + } + fn delete_link(&self, _id: &str) -> IcmResult<()> { + unsupported("memoir.delete_link") + } + fn get_neighbors( + &self, + _concept_id: &str, + _relation: Option, + ) -> IcmResult> { + unsupported("memoir.get_neighbors") + } + fn get_neighborhood( + &self, + _concept_id: &str, + _depth: usize, + ) -> IcmResult<(Vec, Vec)> { + unsupported("memoir.get_neighborhood") + } + fn get_links_for_memoir(&self, _memoir_id: &str) -> IcmResult> { + unsupported("memoir.get_links_for_memoir") + } + fn memoir_stats(&self, _memoir_id: &str) -> IcmResult { + unsupported("memoir.memoir_stats") + } + fn batch_memoir_concept_counts(&self) -> IcmResult> { + unsupported("memoir.batch_memoir_concept_counts") + } +} diff --git a/crates/icm-store/src/postgres/memory.rs b/crates/icm-store/src/postgres/memory.rs new file mode 100644 index 00000000..e397aaf7 --- /dev/null +++ b/crates/icm-store/src/postgres/memory.rs @@ -0,0 +1,522 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. + +use super::*; + +// MemoryStore + +impl MemoryStore for PostgresStore { + fn store(&self, memory: Memory) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("store".into())); + } + let memory = validate_and_normalize(memory)?; + self.check_dims(&memory)?; + let mut c = self.conn()?; + let mut tx = c.transaction().map_err(pg_err)?; + let id = insert_or_merge_memory(&mut tx, &memory)?; + tx.commit().map_err(pg_err)?; + Ok(id) + } + + fn get(&self, id: &str) -> IcmResult> { + let mut c = self.conn()?; + let row = c + .query_opt( + &format!("SELECT {SELECT_COLS} FROM memories WHERE id = $1"), + &[&id], + ) + .map_err(pg_err)?; + Ok(row.as_ref().map(row_to_memory)) + } + + fn update(&self, memory: &Memory) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("update".into())); + } + self.check_dims(memory)?; + let keywords_json = serde_json::to_string(&memory.keywords)?; + let related_json = serde_json::to_string(&memory.related_ids)?; + let st = source_type(&memory.source); + let sd = source_data(&memory.source); + let hash = summary_hash(&memory.topic, &memory.summary); + let importance = memory.importance.to_string(); + let access = memory.access_count as i32; + let emb: Option = memory + .embedding + .as_ref() + .map(|e| pgvector::Vector::from(e.clone())); + + let mut c = self.conn()?; + let changed = c + .execute( + "UPDATE memories SET + updated_at = $2, last_accessed = $3, access_count = $4, weight = $5, + topic = $6, summary = $7, raw_excerpt = $8, keywords = $9, + importance = $10, source_type = $11, source_data = $12, related_ids = $13, + embedding = $14, summary_hash = $15 + WHERE id = $1", + &[ + &memory.id, + &memory.updated_at, + &memory.last_accessed, + &access, + &memory.weight, + &memory.topic, + &memory.summary, + &memory.raw_excerpt, + &keywords_json, + &importance, + &st, + &sd, + &related_json, + &emb, + &hash, + ], + ) + .map_err(pg_err)?; + if changed == 0 { + return Err(IcmError::NotFound(memory.id.clone())); + } + Ok(()) + } + + fn delete(&self, id: &str) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("delete".into())); + } + let mut c = self.conn()?; + let mut tx = c.transaction().map_err(pg_err)?; + let changed = tx + .execute("DELETE FROM memories WHERE id = $1", &[&id]) + .map_err(pg_err)?; + if changed == 0 { + return Err(IcmError::NotFound(id.to_string())); + } + // Manual-testing finding (same class as the SQLite backend): a + // deleted memory otherwise stays as a dangling entry in every + // other memory's `related_ids` forever. Strip it out. `LIKE` is a + // cheap prefilter; the JSON-quoted match guards against a ULID + // that happens to be a literal substring of another. + tx.execute( + "UPDATE memories + SET related_ids = ( + SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)::text + FROM jsonb_array_elements_text(related_ids::jsonb) AS elem + WHERE elem != $1 + ) + WHERE related_ids LIKE '%\"' || $1 || '\"%'", + &[&id], + ) + .map_err(pg_err)?; + tx.commit().map_err(pg_err)?; + Ok(()) + } + + fn search_by_keywords(&self, keywords: &[&str], limit: usize) -> IcmResult> { + if keywords.is_empty() { + return Ok(Vec::new()); + } + let keywords = &keywords[..keywords.len().min(50)]; + let limit = limit.min(100); + + let mut owned: Vec> = Vec::new(); + let mut where_parts: Vec = Vec::new(); + for k in keywords { + // Audit finding: a keyword containing `%`/`_` was interpolated + // straight into the ILIKE pattern unescaped (same bug already + // fixed for SQLite's search_by_keywords). Escape and declare the + // escape character explicitly. + owned.push(Box::new(format!("%{}%", escape_like_wildcards(k)))); + let p = owned.len(); + where_parts.push(format!( + "(keywords ILIKE ${p} ESCAPE '\\' OR summary ILIKE ${p} ESCAPE '\\' \ + OR topic ILIKE ${p} ESCAPE '\\')" + )); + } + owned.push(Box::new(limit as i64)); + let sql = format!( + "SELECT {SELECT_COLS} FROM memories WHERE {} ORDER BY weight DESC LIMIT ${}", + where_parts.join(" OR "), + owned.len() + ); + let params: Vec<&(dyn ToSql + Sync)> = owned.iter().map(|b| b.as_ref()).collect(); + let mut c = self.conn()?; + let rows = c.query(&sql, ¶ms).map_err(pg_err)?; + Ok(rows.iter().map(row_to_memory).collect()) + } + + fn search_fts(&self, query: &str, limit: usize) -> IcmResult> { + let limit = limit.min(100); + if query.trim().is_empty() { + return Ok(Vec::new()); + } + let mut c = self.conn()?; + let rows = c + .query( + &format!( + "SELECT {SELECT_COLS} FROM memories \ + WHERE fts @@ websearch_to_tsquery('simple', $1) \ + ORDER BY weight DESC LIMIT $2" + ), + &[&query, &(limit as i64)], + ) + .map_err(pg_err)?; + Ok(rows.iter().map(row_to_memory).collect()) + } + + fn search_by_embedding( + &self, + embedding: &[f32], + limit: usize, + ) -> IcmResult> { + // Found while auditing OpenSearch's equivalent (which had no clamp + // at all): this function was also missing one, unlike its sibling + // search functions in this same file. + let limit = limit.min(1000); + let qv = pgvector::Vector::from(embedding.to_vec()); + let mut c = self.conn()?; + let rows = c + .query( + &format!( + "SELECT {SELECT_COLS}, embedding <=> $1 AS distance FROM memories \ + WHERE embedding IS NOT NULL ORDER BY embedding <=> $1 LIMIT $2" + ), + &[&qv, &(limit as i64)], + ) + .map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| { + let distance: f64 = row.get(15); + (row_to_memory(row), 1.0 - distance as f32) + }) + .collect()) + } + + fn search_hybrid( + &self, + query: &str, + embedding: &[f32], + limit: usize, + ) -> IcmResult> { + let limit = limit.min(1000); + let pool_size = limit * 4; + + // 1. FTS candidates (id + rank). Lock released at scope end. + let fts_pairs: Vec<(String, f64)> = if query.trim().is_empty() { + Vec::new() + } else { + let mut c = self.conn()?; + let rows = c + .query( + "SELECT id, ts_rank_cd(fts, websearch_to_tsquery('simple', $1))::float8 AS rank \ + FROM memories \ + WHERE fts @@ websearch_to_tsquery('simple', $1) \ + ORDER BY rank DESC LIMIT $2", + &[&query, &(pool_size as i64)], + ) + .map_err(pg_err)?; + rows.iter().map(|r| (r.get(0), r.get(1))).collect() + }; + + // 2. Vector candidates (full rows + similarity). + let vec_results = self.search_by_embedding(embedding, pool_size)?; + + // 3. Assemble memory objects and per-source scores. + let mut all_memories: HashMap = HashMap::new(); + let mut vec_scores: HashMap = HashMap::new(); + for (mem, sim) in vec_results { + vec_scores.insert(mem.id.clone(), sim); + all_memories.insert(mem.id.clone(), mem); + } + + // Normalize FTS ranks into 0..1 within the pool (higher is better). + let max_rank = fts_pairs.iter().map(|(_, r)| *r).fold(0.0_f64, f64::max); + let mut fts_scores: HashMap = HashMap::new(); + let missing: Vec = fts_pairs + .iter() + .filter(|(id, _)| !all_memories.contains_key(id)) + .map(|(id, _)| id.clone()) + .collect(); + if !missing.is_empty() { + let refs: Vec<&str> = missing.iter().map(|s| s.as_str()).collect(); + let fetched = self.get_many(&refs)?; + for (id, m) in fetched { + all_memories.insert(id, m); + } + } + for (id, rank) in fts_pairs { + let score = if max_rank > 0.0 { + (rank / max_rank) as f32 + } else { + 0.0 + }; + fts_scores.insert(id, score); + } + + // 4. Blend: 30% FTS + 70% vector (matches the SQLite backend). + let mut scored: Vec<(String, f32)> = all_memories + .keys() + .map(|id| { + let fts = fts_scores.get(id).copied().unwrap_or(0.0); + let vec = vec_scores.get(id).copied().unwrap_or(0.0); + (id.clone(), 0.3 * fts + 0.7 * vec) + }) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + scored.truncate(limit); + + Ok(scored + .into_iter() + .filter_map(|(id, score)| all_memories.remove(&id).map(|m| (m, score))) + .collect()) + } + + fn update_access(&self, id: &str) -> IcmResult<()> { + if self.readonly { + return Ok(()); + } + let mut c = self.conn()?; + let changed = c + .execute( + "UPDATE memories SET last_accessed = $1, access_count = access_count + 1 \ + WHERE id = $2", + &[&Utc::now(), &id], + ) + .map_err(pg_err)?; + if changed == 0 { + return Err(IcmError::NotFound(id.to_string())); + } + Ok(()) + } + + fn batch_update_access(&self, ids: &[&str]) -> IcmResult { + if ids.is_empty() || self.readonly { + return Ok(0); + } + let id_vec: Vec = ids.iter().map(|s| s.to_string()).collect(); + let mut c = self.conn()?; + let changed = c + .execute( + "UPDATE memories SET last_accessed = $1, access_count = access_count + 1 \ + WHERE id = ANY($2)", + &[&Utc::now(), &id_vec], + ) + .map_err(pg_err)?; + Ok(changed as usize) + } + + fn apply_decay(&self, decay_factor: f32) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("apply_decay".into())); + } + // Access-aware decay, capped at 5 accesses (matches SQLite). + let mut c = self.conn()?; + let changed = c + .execute( + // `$1::float8` is explicit so PostgreSQL doesn't infer the + // parameter as `numeric`/`real` from a neighbouring operand + // and reject the `f64` we bind ("error serializing parameter"). + // Audit finding: for `low` importance with low access count, + // the raw multiplier goes negative once decay_factor < 0.5 + // (still inside the CLI's own validated [0.0, 1.0) range) — + // the same bug already fixed for SQLite (GREATEST here is + // Postgres's equivalent of SQLite's MAX). See store.rs + // apply_decay for the full derivation. + "UPDATE memories SET weight = weight * GREATEST(0.0, + 1.0 - (1.0 - $1::float8) * + CASE importance + WHEN 'high' THEN 0.5 + WHEN 'low' THEN 2.0 + ELSE 1.0 + END + / (1.0 + LEAST(access_count, 5) * 0.1) + ) + WHERE importance <> 'critical'", + &[&(decay_factor as f64)], + ) + .map_err(pg_err)?; + Ok(changed as usize) + } + + fn prune(&self, weight_threshold: f32) -> IcmResult { + if self.readonly { + return Err(IcmError::ReadOnly("prune".into())); + } + let mut c = self.conn()?; + let changed = c + .execute( + "DELETE FROM memories \ + WHERE weight < $1::float8 AND importance NOT IN ('critical', 'high')", + &[&(weight_threshold as f64)], + ) + .map_err(pg_err)?; + Ok(changed as usize) + } + + fn list_all(&self) -> IcmResult> { + let mut c = self.conn()?; + let rows = c + .query( + &format!("SELECT {SELECT_COLS} FROM memories ORDER BY weight DESC LIMIT 10000"), + &[], + ) + .map_err(pg_err)?; + Ok(rows.iter().map(row_to_memory).collect()) + } + + fn get_by_topic(&self, topic: &str) -> IcmResult> { + let mut c = self.conn()?; + let rows = c + .query( + &format!( + "SELECT {SELECT_COLS} FROM memories WHERE topic = $1 \ + ORDER BY weight DESC LIMIT 500" + ), + &[&topic], + ) + .map_err(pg_err)?; + Ok(rows.iter().map(row_to_memory).collect()) + } + + fn list_topics(&self) -> IcmResult> { + self.list_topics_with_prefix(None) + } + + fn consolidate_topic(&self, topic: &str, consolidated: Memory) -> IcmResult<()> { + if self.readonly { + return Err(IcmError::ReadOnly("consolidate_topic".into())); + } + // The consolidated memory goes through the same validation as any + // other write — an MCP-provided consolidate summary previously + // bypassed every size/NUL check (same gap already closed on SQLite). + let consolidated = validate_and_normalize(consolidated)?; + let mut c = self.conn()?; + let mut tx = c.transaction().map_err(pg_err)?; + // Audit finding: `critical` memories are never deleted — same + // contract `apply_decay`/`prune` already honor, and the same fix + // already applied to SQLite's consolidate_topic. This unconditional + // DELETE previously wiped critical memories in a consolidated topic + // too. + // Manual-testing finding: captured before the delete, since + // afterward these rows are gone. Used to clean up any *other* + // memory's related_ids that pointed at them — same dangling- + // reference bug already fixed for the single-id `delete`. + let deleted_ids: Vec = tx + .query( + "SELECT id FROM memories WHERE topic = $1 AND importance <> 'critical'", + &[&topic], + ) + .map_err(pg_err)? + .iter() + .map(|row| row.get(0)) + .collect(); + + tx.execute( + "DELETE FROM memories WHERE topic = $1 AND importance <> 'critical'", + &[&topic], + ) + .map_err(pg_err)?; + + if !deleted_ids.is_empty() { + tx.execute( + "UPDATE memories + SET related_ids = ( + SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)::text + FROM jsonb_array_elements_text(related_ids::jsonb) AS elem + WHERE elem <> ALL($1::text[]) + ) + WHERE related_ids::jsonb ?| $1::text[]", + &[&deleted_ids], + ) + .map_err(pg_err)?; + } + + insert_or_merge_memory(&mut tx, &consolidated)?; + tx.commit().map_err(pg_err)?; + Ok(()) + } + + fn count(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one("SELECT COUNT(*) FROM memories", &[]) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } + + fn count_by_topic(&self, topic: &str) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one("SELECT COUNT(*) FROM memories WHERE topic = $1", &[&topic]) + .map_err(pg_err)?; + let n: i64 = row.get(0); + Ok(n.max(0) as usize) + } + + fn stats(&self) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one( + "SELECT COUNT(*)::bigint, COUNT(DISTINCT topic)::bigint, \ + COALESCE(AVG(weight), 0.0)::float8, MIN(created_at), MAX(created_at) \ + FROM memories", + &[], + ) + .map_err(pg_err)?; + let total: i64 = row.get(0); + let topics: i64 = row.get(1); + let avg: f64 = row.get(2); + Ok(StoreStats { + total_memories: total.max(0) as usize, + total_topics: topics.max(0) as usize, + avg_weight: avg as f32, + oldest_memory: row.get(3), + newest_memory: row.get(4), + }) + } + + fn topic_health(&self, topic: &str) -> IcmResult { + let mut c = self.conn()?; + let row = c + .query_one( + "SELECT COUNT(*)::bigint, + COALESCE(AVG(weight), 0)::float8, + COALESCE(AVG(access_count::float8), 0)::float8, + MIN(created_at), MAX(created_at), MAX(last_accessed), + COALESCE(SUM(CASE WHEN weight < 0.5 + AND (now() - last_accessed) > interval '14 days' + THEN 1 ELSE 0 END), 0)::bigint + FROM memories WHERE topic = $1", + &[&topic], + ) + .map_err(pg_err)?; + + let entry_count: i64 = row.get(0); + if entry_count == 0 { + return Err(IcmError::NotFound(format!("topic: {topic}"))); + } + let avg_weight: f64 = row.get(1); + let avg_access: f64 = row.get(2); + let stale: i64 = row.get(6); + + Ok(TopicHealth { + topic: topic.to_string(), + entry_count: entry_count.max(0) as usize, + avg_weight: avg_weight as f32, + avg_access_count: avg_access as f32, + oldest: row.get(3), + newest: row.get(4), + last_accessed: row.get(5), + needs_consolidation: entry_count > 5, + stale_count: stale.max(0) as usize, + }) + } +} + +// Unsupported subsystems on this backend (first cut, issue #301). +// +// These return `IcmError::Unsupported` so the binary keeps working for the +// core shared-memory use case while the heavier subsystems remain on the +// default SQLite backend. A follow-up can port them. diff --git a/crates/icm-store/src/postgres/mod.rs b/crates/icm-store/src/postgres/mod.rs new file mode 100644 index 00000000..b22252bf --- /dev/null +++ b/crates/icm-store/src/postgres/mod.rs @@ -0,0 +1,96 @@ +//! PostgreSQL storage backend (issue #301, opt-in via `--features postgres`). +//! +//! A node-local SQLite file cannot be shared between several ICM +//! processes or Kubernetes replicas. This backend runs the same memory +//! model over a network-accessible PostgreSQL database so every instance +//! reads and writes one shared store. PostgreSQL serialises concurrent +//! writers, so N replicas can `icm store` into the same memory safely. +//! +//! Design notes: +//! +//! - **Blocking client.** The store traits are synchronous +//! (`fn store(&self, ...) -> IcmResult<...>`), so we use the blocking +//! `postgres` crate. No async runtime, no sync-over-async bridge — the +//! client maps one-to-one onto the trait surface. +//! - **`pgvector` for embeddings.** Memory embeddings live in a +//! `vector(N)` column; KNN search uses the `<=>` cosine-distance +//! operator. Similarity is reported as `1 - distance` to match the +//! SQLite backend. +//! - **PostgreSQL full-text search** replaces SQLite FTS5: a generated +//! `tsvector` column (config `simple`, no stemming, to mirror FTS5's +//! unicode61 tokenizer) with a GIN index, queried via +//! `websearch_to_tsquery` so arbitrary user input is operator-safe. +//! - **Connection string** comes from `ICM_POSTGRES_URL` (or +//! `DATABASE_URL` as a fallback). The `&Path` arguments that the CLI +//! passes for the SQLite file are ignored. +//! +//! Scope of this first cut: the full [`MemoryStore`] surface (the core +//! shared-memory use case behind #301) plus the ancillary tables used by +//! the normal store/recall/hook path (hook telemetry, the extraction +//! queue, code areas, the key/value metadata). The heavier subsystems +//! (memoir graph, transcripts, structured facts, feedback, pattern +//! mining) return [`IcmError::Unsupported`] on this backend for now; +//! they remain fully available on the default SQLite backend. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; + +use chrono::{DateTime, Utc}; +use postgres::types::ToSql; +use postgres::{Client, GenericClient, NoTls}; + +use icm_core::{ + Concept, ConceptLink, Embedder, Fact, FactsStats, FactsStore, Feedback, FeedbackStats, + FeedbackStore, IcmError, IcmResult, Importance, Label, Memoir, MemoirStats, MemoirStore, + Memory, MemorySource, MemoryStore, Message, PatternCluster, Relation, Role, Session, + StoreStats, TopicHealth, TranscriptHit, TranscriptStats, TranscriptStore, +}; + +// Shared public row types live in `crate::common` (issue #301) so every +// backend can be compiled into one binary without colliding definitions. +pub use crate::common::{CodeArea, HookEvent, HookEventInsert, HookStatsRow, PendingRow}; + +// Helpers (mirrored from the SQLite backend so behaviour matches) + +// PostgresStore + +/// PostgreSQL-backed store. See the module docs. +pub struct PostgresStore { + client: Mutex, + embedding_dims: usize, + readonly: bool, +} + +/// One-shot warning that auto-consolidation silently does nothing on this +/// backend (see [`PostgresStore::auto_consolidate`]). +fn warn_auto_consolidate_unsupported() { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + tracing::warn!( + "auto-consolidation is not implemented on the PostgreSQL backend; \ + topics will keep growing (auto_consolidate_enabled has no effect here)" + ); + }); +} + +fn unsupported(op: &str) -> IcmResult { + Err(IcmError::Unsupported(format!( + "{op} (use the default SQLite backend)" + ))) +} + +mod connection; +mod facts; +mod feedback; +mod hooks; +mod maintenance; +mod memoir; +mod memory; +mod patterns; +mod rows; +#[cfg(test)] +mod tests; +mod transcript; + +pub(crate) use rows::*; diff --git a/crates/icm-store/src/postgres/patterns.rs b/crates/icm-store/src/postgres/patterns.rs new file mode 100644 index 00000000..02b70715 --- /dev/null +++ b/crates/icm-store/src/postgres/patterns.rs @@ -0,0 +1,173 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Neighbor expansion and pattern-mining helpers. + +use super::*; + +impl PostgresStore { + // Memory reads used by recall expansion + + /// Fetch many memories by id in one round-trip, deduplicated by id. + pub fn get_many(&self, ids: &[&str]) -> IcmResult> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + let id_vec: Vec = ids.iter().map(|s| s.to_string()).collect(); + let mut c = self.conn()?; + let rows = c + .query( + &format!("SELECT {SELECT_COLS} FROM memories WHERE id = ANY($1)"), + &[&id_vec], + ) + .map_err(pg_err)?; + let mut map = HashMap::with_capacity(rows.len()); + for row in &rows { + let m = row_to_memory(row); + map.insert(m.id.clone(), m); + } + Ok(map) + } + + /// Expand a scored result set with one hop of related memories. + /// Backend-agnostic logic mirrored from the SQLite store. + pub fn expand_with_neighbors( + &self, + initial: &[(Memory, f32)], + max_neighbors: usize, + hop_discount: f32, + max_total: usize, + ) -> IcmResult> { + if max_neighbors == 0 || initial.is_empty() { + let mut out = initial.to_vec(); + out.truncate(max_total); + return Ok(out); + } + + let initial_ids: HashSet = initial.iter().map(|(m, _)| m.id.clone()).collect(); + + let mut candidates: Vec<(String, f32)> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + 'outer: for (mem, score) in initial { + for neighbor_id in &mem.related_ids { + if candidates.len() >= max_neighbors { + break 'outer; + } + if initial_ids.contains(neighbor_id) || !seen.insert(neighbor_id.clone()) { + continue; + } + candidates.push((neighbor_id.clone(), *score)); + } + } + + let mut neighbors: Vec<(Memory, f32)> = Vec::new(); + if !candidates.is_empty() { + let ids: Vec<&str> = candidates.iter().map(|(id, _)| id.as_str()).collect(); + let fetched = self.get_many(&ids)?; + for (id, parent_score) in candidates { + if let Some(m) = fetched.get(&id) { + neighbors.push((m.clone(), parent_score * hop_discount)); + } + } + } + + let mut combined: Vec<(Memory, f32)> = initial.to_vec(); + combined.extend(neighbors); + combined.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + combined.truncate(max_total); + Ok(combined) + } + + /// Memories whose topic starts with `topic` (prefix match). + pub fn get_by_topic_prefix(&self, topic: &str) -> IcmResult> { + // Audit finding: `topic` (the literal prefix to match) was + // interpolated unescaped — a topic containing `%`/`_` turned into + // unintended wildcards within what's meant to be a literal prefix. + // The trailing `%` is the deliberate "starts with" wildcard and + // stays outside the escaped portion. + let pattern = format!("{}%", escape_like_wildcards(topic)); + let mut c = self.conn()?; + let rows = c + .query( + &format!( + "SELECT {SELECT_COLS} FROM memories WHERE topic LIKE $1 ESCAPE '\\' \ + ORDER BY weight DESC LIMIT 500" + ), + &[&pattern], + ) + .map_err(pg_err)?; + Ok(rows.iter().map(row_to_memory).collect()) + } + + /// Distinct topics (optionally prefix-filtered) with their counts. + pub fn list_topics_with_prefix(&self, prefix: Option<&str>) -> IcmResult> { + let mut c = self.conn()?; + let rows = match prefix { + Some(p) => { + let pattern = format!("{p}%"); + c.query( + "SELECT topic, COUNT(*)::bigint FROM memories WHERE topic LIKE $1 \ + GROUP BY topic ORDER BY topic", + &[&pattern], + ) + } + None => c.query( + "SELECT topic, COUNT(*)::bigint FROM memories GROUP BY topic ORDER BY topic", + &[], + ), + } + .map_err(pg_err)?; + Ok(rows + .iter() + .map(|row| { + let n: i64 = row.get(1); + (row.get(0), n.max(0) as usize) + }) + .collect()) + } + + // Consolidation / patterns + + /// Auto-consolidation is not yet implemented on the PostgreSQL + /// backend; the call is a no-op (returns "did not consolidate") so the + /// normal store path is unaffected. Unlike the other parity gaps (which + /// return `Unsupported`), this one is silent by design — but the user + /// deserves to know their `auto_consolidate_enabled = true` does nothing + /// here, so warn once per process (audit finding). + pub fn auto_consolidate(&self, _topic: &str, _threshold: usize) -> IcmResult { + warn_auto_consolidate_unsupported(); + Ok(false) + } + + /// See [`Self::auto_consolidate`]. + pub fn auto_consolidate_with_embedder( + &self, + _topic: &str, + _threshold: usize, + _embedder: Option<&dyn Embedder>, + ) -> IcmResult { + warn_auto_consolidate_unsupported(); + Ok(false) + } + + /// Pattern mining is not yet available on the PostgreSQL backend. + pub fn detect_patterns( + &self, + _topic: &str, + _min_cluster_size: usize, + ) -> IcmResult> { + Err(IcmError::Unsupported( + "detect_patterns (use the default SQLite backend)".into(), + )) + } + + /// Pattern mining is not yet available on the PostgreSQL backend. + pub fn extract_pattern_as_concept( + &self, + _cluster: &PatternCluster, + _memoir_id: &str, + ) -> IcmResult { + Err(IcmError::Unsupported( + "extract_pattern_as_concept (use the default SQLite backend)".into(), + )) + } +} diff --git a/crates/icm-store/src/postgres/rows.rs b/crates/icm-store/src/postgres/rows.rs new file mode 100644 index 00000000..3c0996fd --- /dev/null +++ b/crates/icm-store/src/postgres/rows.rs @@ -0,0 +1,284 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Row/parse helpers shared by every trait-impl submodule here. + +use super::*; + +pub(crate) fn pg_err(e: postgres::Error) -> IcmError { + IcmError::Database(e.to_string()) +} + +pub(crate) fn lock_err() -> IcmError { + IcmError::Database("postgres client mutex poisoned".into()) +} + +pub(crate) fn source_type(source: &MemorySource) -> &'static str { + match source { + MemorySource::ClaudeCode { .. } => "claude_code", + MemorySource::Conversation { .. } => "conversation", + MemorySource::Manual => "manual", + } +} + +pub(crate) fn source_data(source: &MemorySource) -> Option { + match source { + MemorySource::Manual => None, + other => serde_json::to_string(other).ok(), + } +} + +pub(crate) fn parse_source(source_type_str: &str, source_data_str: Option) -> MemorySource { + match source_type_str { + "manual" => MemorySource::Manual, + _ => source_data_str + .and_then(|d| serde_json::from_str(&d).ok()) + .unwrap_or(MemorySource::Manual), + } +} + +pub(crate) fn importance_rank(i: Importance) -> u8 { + match i { + Importance::Critical => 4, + Importance::High => 3, + Importance::Medium => 2, + Importance::Low => 1, + } +} + +pub(crate) fn max_importance(a: Importance, b: Importance) -> Importance { + if importance_rank(a) >= importance_rank(b) { + a + } else { + b + } +} + +/// SHA-256 over the normalized `(topic, summary)` pair, hex-encoded. +/// Identical normalization to the SQLite backend so dedup hashes match. +pub(crate) fn summary_hash(topic: &str, summary: &str) -> String { + use sha2::{Digest, Sha256}; + let topic_n = topic.trim().to_lowercase(); + let summary_n: String = summary + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase(); + let mut h = Sha256::new(); + h.update(topic_n.as_bytes()); + h.update(b"\0"); + h.update(summary_n.as_bytes()); + format!("{:x}", h.finalize()) +} + +pub(crate) const MAX_SUMMARY_BYTES: usize = 64 * 1024; +pub(crate) const MAX_TOPIC_BYTES: usize = 256; + +/// Escape `%`, `_`, and the escape character itself so a value can be +/// safely wrapped in a LIKE/ILIKE pattern. Pair with `ESCAPE '\'` in the +/// SQL. Mirrors the SQLite backend's `escape_like_wildcards`. +pub(crate) fn escape_like_wildcards(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// Validate and normalize a `Memory` before insertion. Mirrors the +/// SQLite backend's `validate_and_normalize`. +pub(crate) fn validate_and_normalize(mut memory: Memory) -> IcmResult { + memory.topic = memory.topic.trim().to_string(); + + if memory.topic.is_empty() { + return Err(IcmError::InvalidInput("topic cannot be empty".into())); + } + if memory.summary.trim().is_empty() { + return Err(IcmError::InvalidInput("summary cannot be empty".into())); + } + if memory.topic.contains('\0') { + return Err(IcmError::InvalidInput( + "topic must not contain NUL bytes".into(), + )); + } + if memory.summary.contains('\0') { + return Err(IcmError::InvalidInput( + "summary must not contain NUL bytes".into(), + )); + } + if memory.topic.contains(['\n', '\r', '\t']) { + return Err(IcmError::InvalidInput( + "topic must not contain newline / CR / tab characters".into(), + )); + } + if memory.topic.len() > MAX_TOPIC_BYTES { + return Err(IcmError::InvalidInput(format!( + "topic exceeds {MAX_TOPIC_BYTES} bytes" + ))); + } + if memory.summary.len() > MAX_SUMMARY_BYTES { + return Err(IcmError::InvalidInput(format!( + "summary exceeds {MAX_SUMMARY_BYTES} bytes" + ))); + } + Ok(memory) +} + +pub(crate) const SELECT_COLS: &str = + "id, created_at, updated_at, last_accessed, access_count, weight, \ + topic, summary, raw_excerpt, keywords, \ + importance, source_type, source_data, related_ids, embedding"; + +/// Map a `memories` row (selected via [`SELECT_COLS`]) to a [`Memory`]. +pub(crate) fn row_to_memory(row: &postgres::Row) -> Memory { + let keywords_json: Option = row.get(9); + let keywords: Vec = keywords_json + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + let importance_str: String = row.get(10); + let importance = importance_str.parse().unwrap_or(Importance::Medium); + + let source_type_str: String = row.get(11); + let source_data_str: Option = row.get(12); + let source = parse_source(&source_type_str, source_data_str); + + let related_json: Option = row.get(13); + let related_ids: Vec = related_json + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + let embedding: Option> = row + .get::<_, Option>(14) + .map(|v| v.as_slice().to_vec()); + + let access_count: i32 = row.get(4); + + Memory { + id: row.get(0), + created_at: row.get(1), + updated_at: row.get(2), + last_accessed: row.get(3), + access_count: access_count.max(0) as u32, + weight: row.get(5), + topic: row.get(6), + summary: row.get(7), + raw_excerpt: row.get(8), + keywords, + importance, + source, + related_ids, + embedding, + scope: icm_core::Scope::User, + } +} + +/// Insert a memory, or merge metadata into an existing duplicate. +/// +/// Dedup contract identical to the SQLite backend: a collision on +/// `summary_hash` alone (which already encodes the topic, Rust-side, +/// Unicode-correct) is ignored and the existing row's id is returned, after +/// merging the caller's importance (take max), keywords (union), and +/// `raw_excerpt` (prefer new) into it. +pub(crate) fn insert_or_merge_memory( + c: &mut C, + memory: &Memory, +) -> IcmResult { + let keywords_json = serde_json::to_string(&memory.keywords)?; + let related_json = serde_json::to_string(&memory.related_ids)?; + let st = source_type(&memory.source); + let sd = source_data(&memory.source); + let hash = summary_hash(&memory.topic, &memory.summary); + let importance = memory.importance.to_string(); + let access = memory.access_count as i32; + let emb: Option = memory + .embedding + .as_ref() + .map(|e| pgvector::Vector::from(e.clone())); + + let inserted = c + .query_opt( + "INSERT INTO memories + (id, created_at, updated_at, last_accessed, access_count, weight, + topic, summary, raw_excerpt, keywords, importance, + source_type, source_data, related_ids, summary_hash, embedding) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) + ON CONFLICT (summary_hash) WHERE summary_hash IS NOT NULL + DO NOTHING + RETURNING id", + &[ + &memory.id, + &memory.created_at, + &memory.updated_at, + &memory.last_accessed, + &access, + &memory.weight, + &memory.topic, + &memory.summary, + &memory.raw_excerpt, + &keywords_json, + &importance, + &st, + &sd, + &related_json, + &hash, + &emb, + ], + ) + .map_err(pg_err)?; + + if let Some(row) = inserted { + return Ok(row.get::<_, String>(0)); + } + + // Dedup hit: merge metadata into the existing row (mirrors SQLite). + let existing = c + .query_one( + "SELECT id, importance, keywords, raw_excerpt FROM memories + WHERE summary_hash = $1", + &[&hash], + ) + .map_err(pg_err)?; + + let existing_id: String = existing.get(0); + let existing_importance_str: String = existing.get(1); + let existing_keywords_json: Option = existing.get(2); + let existing_raw: Option = existing.get(3); + + let existing_importance: Importance = existing_importance_str + .parse() + .unwrap_or(Importance::Medium); + let merged_importance = max_importance(existing_importance, memory.importance); + + let existing_keywords: Vec = existing_keywords_json + .as_deref() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or_default(); + let mut merged_keywords = existing_keywords.clone(); + for kw in &memory.keywords { + if !merged_keywords.contains(kw) { + merged_keywords.push(kw.clone()); + } + } + + let merged_raw = memory.raw_excerpt.clone().or_else(|| existing_raw.clone()); + + let importance_changed = merged_importance != existing_importance; + let keywords_changed = merged_keywords != existing_keywords; + let raw_changed = merged_raw != existing_raw; + if importance_changed || keywords_changed || raw_changed { + let merged_keywords_json = serde_json::to_string(&merged_keywords)?; + c.execute( + "UPDATE memories + SET importance = $1, keywords = $2, raw_excerpt = $3, updated_at = $4 + WHERE id = $5", + &[ + &merged_importance.to_string(), + &merged_keywords_json, + &merged_raw, + &Utc::now(), + &existing_id, + ], + ) + .map_err(pg_err)?; + } + + Ok(existing_id) +} diff --git a/crates/icm-store/src/postgres/tests.rs b/crates/icm-store/src/postgres/tests.rs new file mode 100644 index 00000000..80492e9a --- /dev/null +++ b/crates/icm-store/src/postgres/tests.rs @@ -0,0 +1,37 @@ +//! Test suite for the PostgreSQL backend (`postgres::tests`). + +use super::*; + +/// Audit regression: a keyword containing `%`/`_` was interpolated +/// straight into an ILIKE pattern unescaped, turning it into an +/// unintended wildcard. +#[test] +fn test_escape_like_wildcards() { + assert_eq!(escape_like_wildcards("100%"), "100\\%"); + assert_eq!(escape_like_wildcards("snake_case"), "snake\\_case"); + assert_eq!(escape_like_wildcards("back\\slash"), "back\\\\slash"); + assert_eq!(escape_like_wildcards("plain"), "plain"); +} + +/// Audit regression: `apply_decay`'s raw multiplier goes negative for +/// `low` importance + low access count at factor < 0.5 (still inside +/// the CLI's own validated [0.0, 1.0) range). This reproduces the exact +/// arithmetic the SQL `GREATEST(0.0, ...)` clamp now guards, as a plain +/// Rust assertion (no live Postgres needed to prove the formula itself +/// would go negative without the clamp). +#[test] +fn test_apply_decay_formula_would_go_negative_without_clamp() { + let factor: f64 = 0.4; + let mult: f64 = 2.0; // low importance + let access: f64 = 0.0; + let raw = 1.0 - (1.0 - factor) * mult / (1.0 + access * 0.1); + assert!( + raw < 0.0, + "expected the pre-clamp formula to go negative, got {raw}" + ); + assert_eq!( + raw.max(0.0), + 0.0, + "GREATEST(0.0, ...) must clamp this to 0.0" + ); +} diff --git a/crates/icm-store/src/postgres/transcript.rs b/crates/icm-store/src/postgres/transcript.rs new file mode 100644 index 00000000..58ea59f2 --- /dev/null +++ b/crates/icm-store/src/postgres/transcript.rs @@ -0,0 +1,65 @@ +//! PostgreSQL backend -- split out of the former monolithic postgres.rs. +//! +//! Unsupported on this backend (issue #301) -- see mod.rs. + +use super::*; + +impl TranscriptStore for PostgresStore { + fn create_session( + &self, + _agent: &str, + _project: Option<&str>, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.create_session") + } + fn ensure_session( + &self, + _id: &str, + _agent: &str, + _project: Option<&str>, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.ensure_session") + } + fn get_session(&self, _id: &str) -> IcmResult> { + unsupported("transcript.get_session") + } + fn list_sessions(&self, _project: Option<&str>, _limit: usize) -> IcmResult> { + unsupported("transcript.list_sessions") + } + fn record_message( + &self, + _session_id: &str, + _role: Role, + _content: &str, + _tool_name: Option<&str>, + _tokens: Option, + _metadata: Option<&str>, + ) -> IcmResult { + unsupported("transcript.record_message") + } + fn list_session_messages( + &self, + _session_id: &str, + _limit: usize, + _offset: usize, + ) -> IcmResult> { + unsupported("transcript.list_session_messages") + } + fn search_transcripts( + &self, + _query: &str, + _session_id: Option<&str>, + _project: Option<&str>, + _limit: usize, + ) -> IcmResult> { + unsupported("transcript.search_transcripts") + } + fn forget_session(&self, _id: &str) -> IcmResult<()> { + unsupported("transcript.forget_session") + } + fn transcript_stats(&self) -> IcmResult { + unsupported("transcript.transcript_stats") + } +} diff --git a/crates/icm-store/src/store.rs b/crates/icm-store/src/store.rs deleted file mode 100644 index 2929df56..00000000 --- a/crates/icm-store/src/store.rs +++ /dev/null @@ -1,8759 +0,0 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::num::NonZeroUsize; -use std::path::Path; -use std::sync::{Mutex, Once}; - -use chrono::{DateTime, Utc}; -use lru::LruCache; -use rusqlite::{ffi::sqlite3_auto_extension, params, Connection}; -use sha2::{Digest, Sha256}; -use zerocopy::IntoBytes; - -use icm_core::{ - Concept, ConceptLink, Embedder, Fact, FactsStats, FactsStore, Feedback, FeedbackStats, - FeedbackStore, IcmError, IcmResult, Importance, Label, Memoir, MemoirStats, MemoirStore, - Memory, MemorySource, MemoryStore, Message, PatternCluster, Relation, Role, Session, - StoreStats, TopicHealth, TranscriptHit, TranscriptStats, TranscriptStore, -}; - -use crate::schema::init_db_with_dims; - -/// Convert rusqlite::Error to IcmError::Database -pub(crate) fn db_err(e: rusqlite::Error) -> IcmError { - IcmError::Database(e.to_string()) -} - -/// True when a rusqlite error is a "no such table" (a legacy DB missing an -/// FTS shadow table we optionally rebuild — issue #313). -fn is_missing_table(e: &rusqlite::Error) -> bool { - e.to_string().contains("no such table") -} - -/// FTS5 shadow tables maintained by ICM, checked/rebuilt during repair (#313). -const FTS_TABLES: [&str; 4] = [ - "memories_fts", - "concepts_fts", - "feedback_fts", - "messages_fts", -]; - -// Shared public row types live in `crate::common` so all backends can be -// compiled into one binary without colliding definitions (issue #301). -pub use crate::common::{CodeArea, HookEvent, HookEventInsert, HookStatsRow, PendingRow}; - -/// Collect mapped rows into a Vec, converting rusqlite errors. -fn collect_rows( - rows: rusqlite::MappedRows<'_, impl FnMut(&rusqlite::Row<'_>) -> rusqlite::Result>, -) -> IcmResult> { - rows.collect::, _>>().map_err(db_err) -} - -static SQLITE_VEC_INIT: Once = Once::new(); - -fn ensure_sqlite_vec() { - SQLITE_VEC_INIT.call_once(|| unsafe { - #[allow(clippy::missing_transmute_annotations)] - sqlite3_auto_extension(Some(std::mem::transmute( - sqlite_vec::sqlite3_vec_init as *const (), - ))); - }); -} - -/// URI-encode a filesystem path for a SQLite `file:` URI so a backslash on -/// Windows or a `?`/`#`/`%` in a pathological filename can't break the parser. -fn encode_sqlite_uri_path(path: &Path) -> String { - path.to_string_lossy() - .chars() - .map(|c| match c { - '?' | '#' | '%' => format!("%{:02X}", c as u32), - // Normalize Windows backslashes; SQLite URIs accept "/". - '\\' => "/".into(), - other => other.to_string(), - }) - .collect() -} - -/// Open `path` strictly read-only. When `immutable` is set, add the -/// `immutable=1` URI flag — SQLite then assumes the file never changes and -/// touches no `-shm`/`-wal` sidecars, which is required on a `chmod -w` -/// parent directory (issue #263) but serves a permanently stale snapshot and -/// eventually reports spurious `SQLITE_CORRUPT` on a live DB (issue #319). -/// Plain `mode=ro` (immutable = false) is WAL-aware and sees committed -/// writes, at the cost of needing a writable directory for the sidecars. -fn open_readonly_uri(path: &Path, immutable: bool) -> IcmResult { - let encoded = encode_sqlite_uri_path(path); - let uri = if immutable { - format!("file:{encoded}?mode=ro&immutable=1") - } else { - format!("file:{encoded}?mode=ro") - }; - Connection::open_with_flags( - uri, - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY - | rusqlite::OpenFlags::SQLITE_OPEN_URI - | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, - ) - .map_err(|e| IcmError::Database(format!("cannot open database read-only: {e}"))) -} - -/// Open a long-lived read-only connection (issue #319). -/// -/// Prefer a normal WAL-aware `mode=ro` connection: it respects locking and -/// sees writes committed after it opened — the actual deployment model for -/// `icm --read-only serve`, where hooks keep writing the same DB. Fall back to -/// `immutable=1` only when the live open can't even read the DB, e.g. a -/// `chmod -w` sandbox where SQLite can't create the `-shm` sidecar for a -/// WAL-mode file (issue #263). The read probe is essential: on such a -/// directory the open may *succeed* yet the first real read fails, so opening -/// alone is not a sufficient signal. -fn open_readonly_connection(path: &Path) -> IcmResult { - if let Ok(conn) = open_readonly_uri(path, false) { - // Give a momentarily-locked writer time to release before deciding - // the live open is unusable. - let _ = conn.execute_batch("PRAGMA busy_timeout=30000;"); - // Exercise a real table read (touches the WAL/-shm path) — `SELECT 1` - // would not. - if conn - .query_row("SELECT count(*) FROM sqlite_master", [], |_| Ok(())) - .is_ok() - { - return Ok(conn); - } - } - // Live open unusable (e.g. a read-only sandbox dir, #263). Fall back to an - // immutable snapshot — but warn, because a long-lived reader on this - // connection will NOT see subsequent writes (the #319 staleness tradeoff). - tracing::warn!( - path = %path.display(), - "read-only DB opened immutable (sandbox fallback): writes committed after \ - this point will not be visible until the connection is reopened" - ); - open_readonly_uri(path, true) -} - -/// In-process LRU cache size for hot memories. Each entry is one -/// fully-hydrated `Memory` (incl. optional 384×f32 embedding ≈ 1.5KB), -/// so 256 entries cap RAM at ~400KB worst case. Helps long-running -/// processes (`icm serve`, TUI) where the same memories are read -/// repeatedly; zero benefit in one-shot CLI invocations beyond the -/// single recall flow. -const MEMORY_CACHE_CAP: usize = 256; - -pub struct SqliteStore { - conn: Connection, - cache: Mutex>, - /// `true` when opened through [`Self::open_readonly`]. Read-like - /// methods that would otherwise dirty the DB (auto-decay, - /// `update_access`) check this and skip silently; mutation methods - /// (`store`, `update`, `delete`, etc.) check this and return - /// `IcmError::ReadOnly`. Issue #263. - readonly: bool, -} - -impl SqliteStore { - pub fn new(path: &Path) -> IcmResult { - Self::with_dims(path, icm_core::DEFAULT_EMBEDDING_DIMS) - } - - /// Open an existing database in read-only mode (issue #263). - /// - /// Differences vs [`Self::with_dims`]: - /// - The parent directory is NOT created. - /// - The connection is opened with `SQLITE_OPEN_READ_ONLY` — SQLite - /// itself refuses any DDL/DML that the application might miss. - /// - No `PRAGMA journal_mode=WAL` (WAL requires writable access). - /// - No `init_db_with_dims` (schema migration would mutate the DB). - /// - /// Returns an error if the file is absent (caller may want to fall - /// through to writable mode then). Use [`std::path::Path::exists`] - /// at the call site if you need a missing-DB fast path. - pub fn open_readonly(path: &Path) -> IcmResult { - ensure_sqlite_vec(); - if !path.exists() { - return Err(IcmError::NotFound(format!( - "database not found at {}", - path.display() - ))); - } - let conn = open_readonly_connection(path)?; - // foreign_keys is a no-op for reads; busy_timeout is still useful - // when another writer holds the file. - conn.execute_batch("PRAGMA foreign_keys=ON; PRAGMA busy_timeout=30000;") - .map_err(db_err)?; - Ok(Self { - conn, - cache: Mutex::new(new_cache()), - readonly: true, - }) - } - - /// Open an existing database for maintenance — integrity check and - /// repair (issue #313). - /// - /// Writable (so `REINDEX` and FTS `'rebuild'` can run) but, unlike - /// [`Self::with_dims`], it deliberately does NOT: - /// - run `init_db_with_dims` — schema migration would fail on, or mutate, - /// a corrupt DB before it can even be inspected; - /// - switch `journal_mode` — a damaged file's on-disk format is left - /// exactly as found so recovery reasons about the real state. - /// - /// Returns [`IcmError::NotFound`] when the file is absent. - pub fn open_maintenance(path: &Path) -> IcmResult { - ensure_sqlite_vec(); - if !path.exists() { - return Err(IcmError::NotFound(format!( - "database not found at {}", - path.display() - ))); - } - let conn = Connection::open(path) - .map_err(|e| IcmError::Database(format!("cannot open database: {e}")))?; - conn.execute_batch("PRAGMA busy_timeout=30000;") - .map_err(db_err)?; - Ok(Self { - conn, - cache: Mutex::new(new_cache()), - readonly: false, - }) - } - - /// Check database integrity (issue #313) and return a list of problems. - /// A healthy database yields exactly `["ok"]`; a damaged one yields one - /// entry per problem. - /// - /// Two complementary checks run: - /// 1. `PRAGMA integrity_check` — structural b-tree / page validation. - /// This is what caught the shadow-table and index damage reported in - /// the incident (`btreeInitPage`, `wrong # of entries in index …`). - /// 2. FTS5 `'integrity-check'` per shadow table — validates each FTS - /// index's internal structure, complementing the structural pass for - /// damage confined to the FTS shadow tables. - /// - /// This never returns `Err`: even a failure to *run* a check (e.g. an FTS - /// vtable too damaged to instantiate) is recorded as a problem, so the - /// caller — `icm doctor` / `icm repair` — always gets a usable verdict on - /// a badly corrupt database instead of a propagated error. - pub fn integrity_check(&self) -> IcmResult> { - let mut problems = Vec::new(); - - // 1. Structural check. Record a run failure as a problem instead of - // aborting the whole verdict. - match self.run_integrity_pragma() { - Ok(lines) => problems.extend(lines.into_iter().filter(|l| l != "ok")), - Err(e) => problems.push(format!("integrity_check pragma failed: {e}")), - } - - // 2. Per-FTS-table consistency. - for table in FTS_TABLES { - // `rank = 1` makes FTS5 verify the index against the *content* - // table, not just its own internal structure. Without it, an - // index that is stale or out of step with the base table (e.g. - // an interrupted write) is reported as healthy. Requires - // SQLite ≥ 3.37 (bundled rusqlite is well past that). - let sql = format!("INSERT INTO {table}({table}, rank) VALUES('integrity-check', 1);"); - match self.conn.execute_batch(&sql) { - Ok(()) => {} - Err(e) if is_missing_table(&e) => {} // legacy DB without this table - Err(e) => problems.push(format!("fts5 {table}: {e}")), - } - } - - if problems.is_empty() { - Ok(vec!["ok".to_string()]) - } else { - Ok(problems) - } - } - - /// Structural-only integrity check (`PRAGMA integrity_check`), safe on a - /// read-only connection (issue #313 follow-up). Unlike - /// [`Self::integrity_check`] it does NOT run the FTS5 `'integrity-check'` - /// (which is an `INSERT` and needs a writable connection), so it never - /// mutates the DB or triggers a WAL checkpoint. Used by the read-only - /// inspection paths (`icm doctor`, `icm repair --dry-run`). A healthy DB - /// yields `["ok"]`. Never returns `Err`. - pub fn integrity_check_structural(&self) -> IcmResult> { - let problems: Vec = match self.run_integrity_pragma() { - Ok(lines) => lines.into_iter().filter(|l| l != "ok").collect(), - Err(e) => vec![format!("integrity_check pragma failed: {e}")], - }; - if problems.is_empty() { - Ok(vec!["ok".to_string()]) - } else { - Ok(problems) - } - } - - /// Run `PRAGMA integrity_check` and collect its result rows. - fn run_integrity_pragma(&self) -> IcmResult> { - let mut stmt = self - .conn - .prepare("PRAGMA integrity_check") - .map_err(db_err)?; - let rows = stmt - .query_map([], |row| row.get::<_, String>(0)) - .map_err(db_err)?; - let mut out = Vec::new(); - for r in rows { - out.push(r.map_err(db_err)?); - } - Ok(out) - } - - /// Rebuild the FTS5 shadow tables from their content tables and `REINDEX` - /// every b-tree index (issue #313). This repairs the most common - /// corruption class — damaged indexes / FTS shadow tables with intact - /// base tables — without touching row data. - /// - /// Best-effort by design: a shadow table too damaged for `'rebuild'` to - /// even instantiate is skipped rather than aborting the whole repair, and - /// `REINDEX` failure is tolerated too. The caller re-runs - /// [`Self::integrity_check`] afterwards and reports any damage that - /// survived, so nothing is silently claimed fixed. Returns the FTS tables - /// that were successfully rebuilt. - pub fn rebuild_search_indexes(&self) -> IcmResult> { - let mut rebuilt = Vec::new(); - for table in FTS_TABLES { - let sql = format!("INSERT INTO {table}({table}) VALUES('rebuild');"); - // A missing (legacy DB) or too-corrupt-to-instantiate shadow table - // is skipped rather than aborting; the post-repair integrity check - // surfaces any table that could not be restored. - if self.conn.execute_batch(&sql).is_ok() { - rebuilt.push(table.to_string()); - } - } - // May fail on a badly damaged b-tree; the post-repair integrity check - // reports whatever REINDEX could not fix. - let _ = self.conn.execute_batch("REINDEX;"); - Ok(rebuilt) - } - - /// True when the store was opened read-only (issue #263). Read-like - /// methods skip side-effect mutations; write methods return - /// `IcmError::ReadOnly`. - #[must_use] - pub fn is_readonly(&self) -> bool { - self.readonly - } - - /// Peek `icm_metadata.embedding_dims` without running any schema - /// migration. Returns `Ok(None)` when the DB file is absent, the - /// metadata table doesn't exist (legacy DB), or the row is missing. - /// - /// Use this *before* calling [`Self::with_dims`] when running in a - /// mode that must not trigger a destructive vector recreate — most - /// notably the `--no-embeddings` path (issue #267): if the caller - /// has no embedder loaded, `with_dims` would otherwise fall back to - /// `DEFAULT_EMBEDDING_DIMS`, mismatch the stored value, and silently - /// DROP `vec_memories` while NULL-ing every `memories.embedding`. - pub fn read_stored_embedding_dims(path: &Path) -> IcmResult> { - if !path.exists() { - return Ok(None); - } - // Open strictly immutable so this helper survives a `chmod -w` - // sandbox (issue #263 interaction). `SQLITE_OPEN_READ_ONLY` - // alone is NOT enough — SQLite still tries to create/update - // the `-shm` / `-wal` companion files for any WAL-mode DB, - // which fails when the parent directory is non-writable. - // The `immutable=1` URI flag tells SQLite the file will not - // change during the connection's lifetime and stops it from - // touching WAL infrastructure entirely. This is a one-shot probe - // (not a long-lived connection), so the staleness that #319 fixes - // for `open_readonly` does not apply here. - let conn = open_readonly_uri(path, true)?; - // Probe for the metadata table — legacy DBs predate it. - let has_table: bool = conn - .query_row( - "SELECT COUNT(*) > 0 FROM sqlite_master - WHERE type = 'table' AND name = 'icm_metadata'", - [], - |row| row.get(0), - ) - .map_err(db_err)?; - if !has_table { - return Ok(None); - } - let row: Option = conn - .query_row( - "SELECT value FROM icm_metadata WHERE key = 'embedding_dims'", - [], - |row| row.get(0), - ) - .optional() - .map_err(db_err)?; - Ok(row.and_then(|s| s.parse().ok())) - } - - /// Open or create a store with a specific embedding dimension. - pub fn with_dims(path: &Path, embedding_dims: usize) -> IcmResult { - ensure_sqlite_vec(); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| IcmError::Database(format!("cannot create db directory: {e}")))?; - } - let conn = Connection::open(path) - .map_err(|e| IcmError::Database(format!("cannot open database: {e}")))?; - // Schema/PRAGMA setup races with other processes opening the same - // brand-new DB simultaneously (found via real concurrent testing: - // 10 processes opening one fresh DB, several hung, others errored, - // zero succeeded). Both the WAL-mode switch (needs a brief - // exclusive lock to convert a fresh file — busy_timeout must be - // set first in the same batch, or this statement itself has no - // timeout active yet) and init_db_with_dims's schema creation - // (BEGIN IMMEDIATE-wrapped in schema.rs, but SQLite's FTS5 - // virtual-table module can still surface a transient error on the - // loser even so) are retried together here: whatever the winner - // already committed, a fresh attempt's PRAGMA + existence checks - // correctly see and no-op past it. Jittered, not just linear, - // backoff: a fixed schedule lets many racing processes retry in - // near-lockstep and collide again and again. - let mut last_err = None; - for attempt in 0..40u32 { - if attempt > 0 { - let base_ms = (attempt as u64).min(20) * 15; - let jitter_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| u64::from(d.subsec_nanos()) % 40) - .unwrap_or(0); - std::thread::sleep(std::time::Duration::from_millis(base_ms + jitter_ms)); - } - // A short busy_timeout during this retry loop, not the normal - // 30s: 30s is meant to tolerate *ordinary* write contention - // during real use (e.g. a hook write racing a consolidate), - // but stacked with up to 40 outer attempts here it turns into - // a potentially multi-minute worst case under real multi- - // process contention (measured: several real `icm` processes - // hung well past 60s with the 30s inner timeout) — the outer - // jittered loop is what actually provides the robustness here, - // so the inner SQLite-level wait only needs to be long enough - // to smooth over a single competing transaction, not to be a - // retry mechanism in its own right. Restored to 30s below once - // the schema is confirmed present. - let attempt_result = conn - .execute_batch( - "PRAGMA busy_timeout=1000; PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;", - ) - .map_err(db_err) - .and_then(|()| init_db_with_dims(&conn, embedding_dims)); - match attempt_result { - Ok(()) => { - last_err = None; - break; - } - Err(e) => { - let msg = e.to_string(); - let transient = msg.contains("vtable constructor failed") - || msg.contains("already exists") - || msg.contains("database is locked") - || msg.contains("database is busy"); - last_err = Some(e); - if !transient { - break; - } - } - } - } - if let Some(e) = last_err { - return Err(e); - } - conn.execute_batch("PRAGMA busy_timeout=30000;") - .map_err(db_err)?; - - Ok(Self { - conn, - cache: Mutex::new(new_cache()), - readonly: false, - }) - } - - /// Apply decay if more than 24 hours since last decay. - /// Called automatically on recall to avoid manual `icm decay` cron. - /// - /// No-op when the store is read-only (issue #263): recall must work - /// against a DB the process cannot write to, and the bookkeeping - /// writes here would otherwise abort the whole read with - /// "attempt to write a readonly database". - pub fn maybe_auto_decay(&self) -> IcmResult<()> { - if self.readonly { - return Ok(()); - } - let now = Utc::now(); - let now_str = now.to_rfc3339(); - - // Audit finding: this used to apply a flat 0.95 step whenever >= 1 - // day had passed, regardless of HOW MANY days had actually passed — - // a machine touched once a week decayed at 0.95/week (≈0.993/day) - // instead of the documented 0.95/day. Read the previous timestamp - // first so the step can be `0.95 ^ elapsed_days` (compounded), - // matching the documented per-day rate regardless of gaps between - // calls. (The 0.95 base itself stays hardcoded here — wiring the - // CLI's configurable `decay_rate` through to this crate is a - // separate, out-of-scope change.) - let last_decay_at: Option = self - .conn - .query_row( - "SELECT value FROM icm_metadata WHERE key = 'last_decay_at'", - [], - |row| row.get(0), - ) - .optional() - .map_err(db_err)?; - let elapsed_days = last_decay_at - .as_deref() - .and_then(|prev| prev.parse::>().ok()) - .map(|prev| (now - prev).num_seconds() as f64 / 86_400.0) - .filter(|d| d.is_finite() && *d > 0.0) - .unwrap_or(1.0); // first run ever: preserve the historical single-step behavior - - // Atomic check-and-update: only one caller wins the race. (A narrow - // window between the read above and this claim could let a losing - // racer's `elapsed_days` be computed from a slightly stale - // timestamp, but only one claim ever succeeds — a day or two of - // imprecision in a decay RATE is harmless, not worth a stricter - // compare-and-swap loop.) - let changed = self - .conn - .execute( - "INSERT INTO icm_metadata (key, value) VALUES ('last_decay_at', ?1) - ON CONFLICT(key) DO UPDATE SET value = ?1 - WHERE value IS NULL OR julianday(?1) - julianday(value) >= 1.0", - params![now_str], - ) - .map_err(db_err)?; - - if changed > 0 { - let factor = 0.95_f64.powf(elapsed_days) as f32; - self.apply_decay(factor)?; - } - - Ok(()) - } - - /// Atomically increment the hook call counter and return the new value. - pub fn increment_hook_counter(&self) -> IcmResult { - let count: usize = self - .conn - .query_row( - "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '1') - ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT) - RETURNING CAST(value AS INTEGER)", - [], - |row| row.get(0), - ) - .map_err(db_err)?; - Ok(count) - } - - /// Reset the hook call counter to 0. - pub fn reset_hook_counter(&self) -> IcmResult<()> { - self.conn - .execute( - "INSERT INTO icm_metadata (key, value) VALUES ('hook_counter', '0') - ON CONFLICT(key) DO UPDATE SET value = '0'", - [], - ) - .map_err(db_err)?; - Ok(()) - } - - // ── Async extraction queue ───────────────────────────────────────── - // - // Row tuple shape: `(id, project, tool_name, raw_output, captured_at)` - // - // When `[extraction.summarizer].provider` is set to something other - // than `"none"`, PostToolUse hooks INSERT raw tool output here in - // ~50ms (no embedder load) and a worker (`icm extract-pending` or - // the SessionEnd async fork) dequeues batches and runs the LLM CLI. - - /// Enqueue raw tool output for later LLM extraction. Returns the - /// generated row id so the caller can correlate logs. - pub fn enqueue_pending_extraction( - &self, - project: &str, - tool_name: &str, - raw_output: &str, - ) -> IcmResult { - let id = ulid::Ulid::new().to_string(); - let now = chrono::Utc::now().to_rfc3339(); - self.conn - .execute( - "INSERT INTO pending_extractions (id, project, tool_name, raw_output, captured_at) - VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![id, project, tool_name, raw_output, now], - ) - .map_err(db_err)?; - Ok(id) - } - - /// Pop up to `limit` oldest pending rows. Caller is expected to call - /// `delete_pending_extractions` after successful processing. - pub fn list_pending_extractions(&self, limit: usize) -> IcmResult> { - let mut stmt = self - .conn - .prepare( - "SELECT id, project, tool_name, raw_output, captured_at - FROM pending_extractions - ORDER BY captured_at ASC - LIMIT ?1", - ) - .map_err(db_err)?; - let rows = stmt - .query_map([limit as i64], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - )) - }) - .map_err(db_err)? - .collect::, _>>() - .map_err(db_err)?; - Ok(rows) - } - - /// Delete pending rows by id. Used after a worker has processed them. - pub fn delete_pending_extractions(&self, ids: &[String]) -> IcmResult { - if ids.is_empty() { - return Ok(0); - } - let placeholders = ids.iter().map(|_| "?").collect::>().join(","); - let sql = format!("DELETE FROM pending_extractions WHERE id IN ({placeholders})"); - let params: Vec<&dyn rusqlite::ToSql> = - ids.iter().map(|s| s as &dyn rusqlite::ToSql).collect(); - let n = self.conn.execute(&sql, params.as_slice()).map_err(db_err)?; - Ok(n) - } - - /// Total rows currently waiting in the queue. Used by `icm doctor`. - pub fn pending_extraction_count(&self) -> IcmResult { - let n: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM pending_extractions", [], |r| r.get(0)) - .map_err(db_err)?; - Ok(n as usize) - } - - // ── Code areas (auto-captured file edits — issue #196) ──────────── - // - // `cmd_hook_post` calls `upsert_code_area` whenever the upstream - // tool was Edit / Write / MultiEdit / NotebookEdit. Same project + - // file_path => touch_count++ via ON CONFLICT. - - /// Insert or refresh a row for `(project, file_path)`. On conflict - /// bumps `touch_count`, updates `last_touched_at`, refreshes - /// `session_id` / `tool_name`, and only overwrites `description` if - /// the caller passes `Some` (so the most recent meaningful hint - /// wins without clobbering an existing one with `None`). - pub fn upsert_code_area( - &self, - project: &str, - file_path: &str, - description: Option<&str>, - session_id: Option<&str>, - tool_name: Option<&str>, - ) -> IcmResult<()> { - let now = chrono::Utc::now().to_rfc3339(); - self.conn - .execute( - "INSERT INTO code_areas (project, file_path, description, - session_id, tool_name, touch_count, - first_touched_at, last_touched_at) - VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?6) - ON CONFLICT(project, file_path) DO UPDATE SET - touch_count = touch_count + 1, - last_touched_at = excluded.last_touched_at, - session_id = COALESCE(excluded.session_id, session_id), - tool_name = COALESCE(excluded.tool_name, tool_name), - description = COALESCE(excluded.description, description)", - rusqlite::params![project, file_path, description, session_id, tool_name, now], - ) - .map_err(db_err)?; - Ok(()) - } - - /// List code areas, optionally filtered by project / file_path / - /// since timestamp. `limit` caps the result count (use `usize::MAX` - /// to disable). Ordered by `last_touched_at DESC` so the freshest - /// edits come first. - pub fn list_code_areas( - &self, - project: Option<&str>, - in_file: Option<&str>, - since: Option>, - limit: usize, - ) -> IcmResult> { - let mut sql = String::from( - "SELECT id, project, file_path, description, session_id, tool_name, - touch_count, first_touched_at, last_touched_at - FROM code_areas - WHERE 1=1", - ); - let mut params: Vec> = Vec::new(); - if let Some(p) = project { - sql.push_str(" AND project = ?"); - params.push(Box::new(p.to_string())); - } - if let Some(f) = in_file { - // Match either an exact file_path or a path that ends with - // the provided fragment so users can pass a short suffix. - sql.push_str(" AND (file_path = ? OR file_path LIKE ?)"); - params.push(Box::new(f.to_string())); - params.push(Box::new(format!("%/{f}"))); - } - if let Some(t) = since { - sql.push_str(" AND last_touched_at >= ?"); - params.push(Box::new(t.to_rfc3339())); - } - sql.push_str(" ORDER BY last_touched_at DESC LIMIT ?"); - params.push(Box::new(limit as i64)); - - let mut stmt = self.conn.prepare(&sql).map_err(db_err)?; - let param_refs: Vec<&dyn rusqlite::ToSql> = params - .iter() - .map(|p| p.as_ref() as &dyn rusqlite::ToSql) - .collect(); - let rows = stmt - .query_map(param_refs.as_slice(), |row| { - let first: String = row.get(7)?; - let last: String = row.get(8)?; - Ok(CodeArea { - id: row.get(0)?, - project: row.get(1)?, - file_path: row.get(2)?, - description: row.get(3)?, - session_id: row.get(4)?, - tool_name: row.get(5)?, - touch_count: row.get(6)?, - first_touched_at: DateTime::parse_from_rfc3339(&first) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()), - last_touched_at: DateTime::parse_from_rfc3339(&last) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()), - }) - }) - .map_err(db_err)? - .collect::, _>>() - .map_err(db_err)?; - Ok(rows) - } - - /// Total rows in `code_areas`. Cheap; used by stats / doctor. - pub fn code_area_count(&self) -> IcmResult { - let n: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM code_areas", [], |r| r.get(0)) - .map_err(db_err)?; - Ok(n as usize) - } - - // ── Hook telemetry ───────────────────────────────────────────────── - // - // Every `icm hook ` fire writes one row to `hook_events`. Read - // back via `hook_events_recent` / `hook_stats`. Inserts are designed - // to be cheap (single statement, no FTS) so they stay well under the - // <50ms async-path budget. - - /// Append one hook telemetry row. Errors are swallowed by callers in - /// hook paths (logging must never block the user), but tests can - /// inspect the `Result`. - pub fn record_hook_event(&self, ev: &HookEventInsert) -> IcmResult { - let now = chrono::Utc::now().to_rfc3339(); - self.conn - .execute( - "INSERT INTO hook_events - (ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", - rusqlite::params![ - now, - ev.event, - ev.project, - ev.session_id, - ev.tool_name, - ev.duration_ms, - ev.exit_code, - ev.payload_size, - ev.note, - ], - ) - .map_err(db_err)?; - Ok(self.conn.last_insert_rowid()) - } - - /// Most recent `limit` hook events, newest first. Optional `event` - /// filter (e.g. `Some("end")` to see only SessionEnd hooks). - pub fn hook_events_recent( - &self, - limit: usize, - event_filter: Option<&str>, - ) -> IcmResult> { - let limit_i64 = limit as i64; - let row_to_event = |row: &rusqlite::Row<'_>| -> rusqlite::Result { - let ts_str: String = row.get(1)?; - let ts = chrono::DateTime::parse_from_rfc3339(&ts_str) - .map(|t| t.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()); - Ok(HookEvent { - id: row.get(0)?, - ts, - event: row.get(2)?, - project: row.get(3)?, - session_id: row.get(4)?, - tool_name: row.get(5)?, - duration_ms: row.get(6)?, - exit_code: row.get(7)?, - payload_size: row.get(8)?, - note: row.get(9)?, - }) - }; - match event_filter { - Some(e) => { - let mut stmt = self - .conn - .prepare( - "SELECT id, ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note - FROM hook_events - WHERE event = ?1 - ORDER BY id DESC - LIMIT ?2", - ) - .map_err(db_err)?; - let rows = stmt - .query_map(rusqlite::params![e, limit_i64], row_to_event) - .map_err(db_err)?; - collect_rows(rows) - } - None => { - let mut stmt = self - .conn - .prepare( - "SELECT id, ts, event, project, session_id, tool_name, - duration_ms, exit_code, payload_size, note - FROM hook_events - ORDER BY id DESC - LIMIT ?1", - ) - .map_err(db_err)?; - let rows = stmt - .query_map(rusqlite::params![limit_i64], row_to_event) - .map_err(db_err)?; - collect_rows(rows) - } - } - } - - /// Aggregate counts and latency percentiles per event type, over a - /// time window starting `since` (RFC3339). Used by `icm hook-stats`. - pub fn hook_stats(&self, since_rfc3339: &str) -> IcmResult> { - // Pull each event type and compute percentiles in Rust — SQLite - // has no native percentile function and the row count is small - // enough (~1k/day worst case) that an in-process sort is fine. - let mut stmt = self - .conn - .prepare( - "SELECT event, duration_ms, exit_code - FROM hook_events - WHERE ts >= ?1 - ORDER BY event", - ) - .map_err(db_err)?; - let rows = stmt - .query_map([since_rfc3339], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, Option>(1)?, - r.get::<_, i32>(2)?, - )) - }) - .map_err(db_err)?; - let mut by_event: std::collections::BTreeMap, i32)>> = - std::collections::BTreeMap::new(); - for r in rows { - let (ev, dur, exit) = r.map_err(db_err)?; - by_event.entry(ev).or_default().push((dur, exit)); - } - let mut out = Vec::with_capacity(by_event.len()); - for (event, mut items) in by_event { - let count = items.len() as i64; - let error_count = items.iter().filter(|(_, e)| *e != 0).count() as i64; - let mut durations: Vec = items.iter().filter_map(|(d, _)| *d).collect(); - durations.sort_unstable(); - let avg = if durations.is_empty() { - 0.0 - } else { - durations.iter().sum::() as f64 / durations.len() as f64 - }; - let p = |q: f64| -> i64 { - if durations.is_empty() { - 0 - } else { - let idx = ((durations.len() as f64 - 1.0) * q).round() as usize; - durations[idx.min(durations.len() - 1)] - } - }; - out.push(HookStatsRow { - event, - count, - error_count, - avg_duration_ms: avg, - p50_duration_ms: p(0.50), - p99_duration_ms: p(0.99), - }); - // Avoid clippy 'unused variable' on items after move - let _ = &mut items; - } - Ok(out) - } - - /// Delete hook telemetry rows older than `cutoff_rfc3339`. Used by an - /// optional retention pass (`icm hook-log --prune-older-than ...`). - pub fn prune_hook_events(&self, cutoff_rfc3339: &str) -> IcmResult { - let n = self - .conn - .execute( - "DELETE FROM hook_events WHERE ts < ?1", - rusqlite::params![cutoff_rfc3339], - ) - .map_err(db_err)?; - Ok(n) - } - - /// Total rows currently in `hook_events`. Used by tests and `icm doctor`. - pub fn hook_event_count(&self) -> IcmResult { - let n: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM hook_events", [], |r| r.get(0)) - .map_err(db_err)?; - Ok(n as usize) - } - - pub fn in_memory() -> IcmResult { - Self::in_memory_with_dims(icm_core::DEFAULT_EMBEDDING_DIMS) - } - - /// Open an in-memory store with a specific embedding dimension. - /// Useful for tests that exercise the dim-migration / dim-drift paths. - pub fn in_memory_with_dims(embedding_dims: usize) -> IcmResult { - ensure_sqlite_vec(); - let conn = Connection::open_in_memory() - .map_err(|e| IcmError::Database(format!("cannot open in-memory db: {e}")))?; - conn.execute_batch("PRAGMA foreign_keys=ON; PRAGMA busy_timeout=30000;") - .map_err(db_err)?; - init_db_with_dims(&conn, embedding_dims)?; - Ok(Self { - conn, - cache: Mutex::new(new_cache()), - readonly: false, - }) - } - - fn cache_get(&self, id: &str) -> Option { - self.cache.lock().ok().and_then(|mut c| c.get(id).cloned()) - } - - fn cache_put(&self, m: &Memory) { - if let Ok(mut c) = self.cache.lock() { - c.put(m.id.clone(), m.clone()); - } - } - - fn cache_invalidate(&self, id: &str) { - if let Ok(mut c) = self.cache.lock() { - c.pop(id); - } - } - - fn cache_invalidate_many(&self, ids: &[&str]) { - if let Ok(mut c) = self.cache.lock() { - for id in ids { - c.pop(*id); - } - } - } - - fn cache_clear(&self) { - if let Ok(mut c) = self.cache.lock() { - c.clear(); - } - } -} - -fn new_cache() -> LruCache { - let cap = NonZeroUsize::new(MEMORY_CACHE_CAP) - .expect("MEMORY_CACHE_CAP must be non-zero — see store.rs"); - LruCache::new(cap) -} - -// --------------------------------------------------------------------------- -// Memory helpers -// --------------------------------------------------------------------------- - -fn source_type(source: &MemorySource) -> &'static str { - match source { - MemorySource::ClaudeCode { .. } => "claude_code", - MemorySource::Conversation { .. } => "conversation", - MemorySource::Manual => "manual", - } -} - -fn source_data(source: &MemorySource) -> Option { - match source { - MemorySource::Manual => None, - other => serde_json::to_string(other).ok(), - } -} - -fn parse_source(source_type_str: &str, source_data_str: Option) -> MemorySource { - match source_type_str { - "manual" => MemorySource::Manual, - _ => source_data_str - .and_then(|d| serde_json::from_str(&d).ok()) - .unwrap_or(MemorySource::Manual), - } -} - -fn embedding_to_blob(embedding: &[f32]) -> Vec { - embedding.as_bytes().to_vec() -} - -fn blob_to_embedding(blob: &[u8]) -> Vec { - if !blob.len().is_multiple_of(4) { - tracing::warn!( - blob_size = blob.len(), - "embedding blob size not divisible by 4, truncating" - ); - } - blob.chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect() -} - -fn row_to_memory(row: &rusqlite::Row) -> rusqlite::Result { - // Column order: id(0), created_at(1), updated_at(2), last_accessed(3), - // access_count(4), weight(5), topic(6), summary(7), raw_excerpt(8), - // keywords(9), importance(10), source_type(11), source_data(12), - // related_ids(13), embedding(14) - let keywords_json: String = row.get::<_, Option>(9)?.unwrap_or_default(); - let keywords: Vec = serde_json::from_str(&keywords_json).unwrap_or_default(); - - let importance_str: String = row.get(10)?; - let importance = importance_str.parse().unwrap_or(Importance::Medium); - - let source_type_str: String = row.get(11)?; - let source_data_str: Option = row.get(12)?; - let source = parse_source(&source_type_str, source_data_str); - - let related_json: String = row.get::<_, Option>(13)?.unwrap_or_default(); - let related_ids: Vec = serde_json::from_str(&related_json).unwrap_or_default(); - - let embedding: Option> = row - .get::<_, Option>>(14)? - .map(|b| blob_to_embedding(&b)); - - let created_at_str: String = row.get(1)?; - let updated_at_str: String = row.get::<_, Option>(2)?.unwrap_or_default(); - let last_accessed_str: String = row.get(3)?; - - let created_at = parse_dt(&created_at_str); - - Ok(Memory { - id: row.get(0)?, - created_at, - updated_at: if updated_at_str.is_empty() { - created_at - } else { - parse_dt(&updated_at_str) - }, - last_accessed: parse_dt(&last_accessed_str), - access_count: row.get::<_, u32>(4)?, - weight: row.get(5)?, - topic: row.get(6)?, - summary: row.get(7)?, - raw_excerpt: row.get(8)?, - keywords, - importance, - source, - related_ids, - embedding, - scope: icm_core::Scope::User, // default for existing local memories - }) -} - -const SELECT_COLS: &str = "id, created_at, updated_at, last_accessed, access_count, weight, \ - topic, summary, raw_excerpt, keywords, \ - importance, source_type, source_data, related_ids, embedding"; - -/// Sanitize a query string for FTS5 MATCH. -/// -/// FTS5 treats characters like `-`, `*`, `"`, `:`, `^`, `+`, `~` as operators. -/// A query like `"sqlite-vec"` makes FTS5 interpret `-` as NOT and `vec` as a -/// column name, causing "no such column: vec". -/// -/// Escape `%`, `_`, and the escape character itself so a keyword can be -/// safely wrapped in a `%...%` LIKE pattern. Pair with `ESCAPE '\'` in the -/// SQL — without it, a keyword containing `%` matches every row and `_` -/// matches any single character (audit finding). -fn escape_like_wildcards(s: &str) -> String { - s.replace('\\', "\\\\") - .replace('%', "\\%") - .replace('_', "\\_") -} - -/// Cap on auxiliary metadata (transcript sessions/messages) - best-effort -/// truncation, not rejection, matching MAX_MESSAGE_BYTES's rationale. -const MAX_METADATA_BYTES: usize = 8 * 1024; - -/// Truncate `s` to at most `max` bytes without splitting a UTF-8 char. -fn truncate_at_char_boundary(s: &str, max: usize) -> &str { - if s.len() <= max { - return s; - } - let mut cut = max; - while !s.is_char_boundary(cut) { - cut -= 1; - } - &s[..cut] -} - -/// This function strips special chars and wraps each token in double quotes. -fn sanitize_fts_query(query: &str) -> String { - // Limit input length to prevent abuse (UTF-8 safe truncation) - let query = if query.len() > 10_000 { - let mut end = 10_000; - while end > 0 && !query.is_char_boundary(end) { - end -= 1; - } - &query[..end] - } else { - query - }; - - // Replace FTS5 operator chars with spaces, then quote each resulting token. - // FTS5 tokenizer (unicode61) splits on `-` too, so we must keep tokens separate. - let cleaned: String = query - .chars() - .map(|c| { - if matches!( - c, - '-' | '*' | '"' | '(' | ')' | '{' | '}' | ':' | '^' | '+' | '~' | '\\' - ) { - ' ' - } else { - c - } - }) - .collect(); - - let tokens: Vec = cleaned - .split_whitespace() - .filter(|w| !w.is_empty()) - .take(100) // Limit token count to prevent excessive query complexity - .map(|w| { - // Strip any remaining quotes from tokens before wrapping in quotes - let stripped = w.replace('"', ""); - format!("\"{stripped}\"") - }) - .collect(); - tokens.join(" ") -} - -/// Whether `e` is FTS5 rejecting a malformed MATCH query (e.g. "hello AND", -/// unbalanced parens) rather than a genuine database error. Used by -/// `search_transcripts` to degrade to "no results" instead of surfacing a -/// raw sqlite error, without pre-sanitizing the query text away from valid -/// FTS5 syntax (which callers rely on — see -/// `test_transcript_search_fts5_boolean_and_phrase`). -fn is_fts5_syntax_error(e: &rusqlite::Error) -> bool { - matches!( - e, - rusqlite::Error::SqliteFailure(_, Some(msg)) if msg.contains("fts5: syntax error") - ) -} - -// --------------------------------------------------------------------------- -// MemoryStore impl -// --------------------------------------------------------------------------- - -/// Maximum byte length of a stored summary. Audit finding: a transcript -/// containing a 1 MB unbroken text block landed as a single memory whose -/// summary was the full 1 MB blob. Caps the cost of a single bad write -/// (memory bloat, embedding compute, FTS5 index growth) to a generous -/// but bounded 64 KB. -const MAX_SUMMARY_BYTES: usize = 64 * 1024; - -/// Maximum byte length of a stored topic. Topics surface in `icm -/// topics` listings and as the routing key for project filters; a -/// thousand-byte topic is always a bug, never legitimate user input. -const MAX_TOPIC_BYTES: usize = 256; - -/// Validate and normalize a `Memory` before insertion. Trims topic -/// whitespace and rejects inputs that we know corrupt or break the -/// store: -/// -/// - Empty or whitespace-only `topic` / `summary` — these would surface -/// as blank rows in `icm topics` / `icm list` and pollute the FTS5 -/// index without conveying information. -/// - NUL byte (`\0`) in `topic` or `summary` — libsql binds text via a -/// NUL-terminated C string, so anything past the first `\0` is -/// silently dropped. Rather than silently truncate, refuse the -/// write so the caller knows. -/// - Newline / CR / tab in `topic` — these break the `icm topics` -/// tabular layout and could enable display-spoofing of topic names -/// (e.g. a topic that visually overlaps another in TUI/log output). -/// Allowed in `summary` since it's free-form prose. -/// - `topic` longer than `MAX_TOPIC_BYTES` or `summary` longer than -/// `MAX_SUMMARY_BYTES` — see the constant docs for rationale. -fn validate_and_normalize(mut memory: Memory) -> IcmResult { - memory.topic = memory.topic.trim().to_string(); - validate_fields(&memory.topic, &memory.summary)?; - Ok(memory) -} - -/// The borrowed core of [`validate_and_normalize`], shared with `update()` -/// (audit finding: the update path previously bypassed every size/content -/// check, so oversized or NUL-carrying payloads could enter the store by -/// storing small then updating big). -fn validate_fields(topic: &str, summary: &str) -> IcmResult<()> { - if topic.is_empty() { - return Err(IcmError::InvalidInput("topic cannot be empty".into())); - } - if summary.trim().is_empty() { - return Err(IcmError::InvalidInput("summary cannot be empty".into())); - } - if topic.contains('\0') { - return Err(IcmError::InvalidInput( - "topic must not contain NUL bytes".into(), - )); - } - if summary.contains('\0') { - return Err(IcmError::InvalidInput( - "summary must not contain NUL bytes".into(), - )); - } - if topic.contains(['\n', '\r', '\t']) { - return Err(IcmError::InvalidInput( - "topic must not contain newline / CR / tab characters".into(), - )); - } - if topic.len() > MAX_TOPIC_BYTES { - return Err(IcmError::InvalidInput(format!( - "topic exceeds {} bytes", - MAX_TOPIC_BYTES - ))); - } - if summary.len() > MAX_SUMMARY_BYTES { - return Err(IcmError::InvalidInput(format!( - "summary exceeds {} bytes", - MAX_SUMMARY_BYTES - ))); - } - Ok(()) -} - -/// Local total order on `Importance` (Critical > High > Medium > Low). -/// `Importance` does not implement `Ord` because the project did not -/// want to imply a globally meaningful ordering across all uses -/// (e.g. presentation, filtering). For the dedup-merge path we *do* -/// want to take the maximum so re-storing with a higher priority -/// upgrades the existing row. -fn importance_rank(i: Importance) -> u8 { - match i { - Importance::Critical => 4, - Importance::High => 3, - Importance::Medium => 2, - Importance::Low => 1, - } -} - -/// Return the higher-priority importance. Used by the dedup path so -/// `store(...)` semantics are "re-store with critical upgrades, never -/// downgrades". -fn max_importance(a: Importance, b: Importance) -> Importance { - if importance_rank(a) >= importance_rank(b) { - a - } else { - b - } -} - -/// SHA-256 over the normalized `(topic, summary)` pair, hex-encoded. -/// Normalization: trim + lowercase + collapse whitespace runs to single -/// spaces. Topic and summary are joined by `\0` to prevent boundary -/// ambiguity (e.g. `"a"|"bc"` vs `"ab"|"c"` would otherwise hash the -/// same). Used by the dedup `INSERT OR IGNORE` path. -pub(crate) fn summary_hash(topic: &str, summary: &str) -> String { - let topic_n = topic.trim().to_lowercase(); - let summary_n: String = summary - .split_whitespace() - .collect::>() - .join(" ") - .to_lowercase(); - let mut h = Sha256::new(); - h.update(topic_n.as_bytes()); - h.update(b"\0"); - h.update(summary_n.as_bytes()); - format!("{:x}", h.finalize()) -} - -impl SqliteStore { - /// Insert a memory into the database without transaction management. - /// Callers are responsible for wrapping this in a transaction. - /// - /// Dedup contract: an INSERT that collides with an existing memory on - /// `(topic, summary_hash)` is silently ignored, and the **existing** - /// row's id is returned. The caller's `memory.id` is forgotten in - /// that case. This keeps `store(...)` idempotent: writing the same - /// fact 100× ends up with one row, not 100. - fn store_inner(&self, memory: &Memory) -> IcmResult { - let keywords_json = serde_json::to_string(&memory.keywords)?; - let related_json = serde_json::to_string(&memory.related_ids)?; - let st = source_type(&memory.source); - let sd = source_data(&memory.source); - let emb_blob = memory.embedding.as_deref().map(embedding_to_blob); - let hash = summary_hash(&memory.topic, &memory.summary); - - let inserted = self - .conn - .execute( - "INSERT OR IGNORE INTO memories (id, created_at, updated_at, last_accessed, access_count, weight, - topic, summary, raw_excerpt, keywords, - importance, source_type, source_data, related_ids, embedding, summary_hash) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)", - params![ - memory.id, - memory.created_at.to_rfc3339(), - memory.updated_at.to_rfc3339(), - memory.last_accessed.to_rfc3339(), - memory.access_count, - memory.weight, - memory.topic, - memory.summary, - memory.raw_excerpt, - keywords_json, - memory.importance.to_string(), - st, - sd, - related_json, - emb_blob, - hash, - ], - ) - .map_err(db_err)?; - - if inserted == 0 { - // Dedup hit: a row with the same (topic, summary_hash) - // already exists. Audit #185 H2: the previous behaviour - // returned the existing id and silently dropped the - // caller's importance / keywords / raw_excerpt. So - // running `icm store -t T -c "X" -i medium` then `icm - // store -t T -c "X" -i critical` left the importance at - // medium without warning the user. - // - // New behaviour: merge the caller's metadata into the - // existing row before returning the id. - // - importance: take the max (critical > high > medium > - // low). Re-storing with a *higher* priority upgrades. - // Re-storing with a *lower* priority is a no-op so a - // careless write can't downgrade an already-flagged - // critical memory. - // - keywords: union, preserving existing order then - // appending new ones not already present. - // - raw_excerpt: prefer the new value if non-None, - // otherwise keep existing. - // - updated_at: bumped whenever any field actually changed. - let (existing_id, existing_importance_str, existing_keywords_json, existing_raw): ( - String, - String, - String, - Option, - ) = self - .conn - .query_row( - // Audit finding: `summary_hash` already encodes the topic - // (Rust `to_lowercase()`, full Unicode) as part of the - // hash input — an additional `LOWER(topic) = LOWER(?)` - // comparison here used SQLite's built-in `LOWER()`, - // which is ASCII-only and does not fold e.g. 'É' → 'é'. - // For an all-caps accented topic like "DÉCISIONS" that - // mismatch meant this SELECT could fail to find the row - // the `INSERT OR IGNORE` conflict was already about, - // even though `summary_hash` alone uniquely identifies - // it. `summary_hash` is sufficient on its own. - "SELECT id, importance, keywords, raw_excerpt FROM memories - WHERE summary_hash = ?1", - params![hash], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) - .map_err(db_err)?; - - let existing_importance: Importance = existing_importance_str - .parse() - .unwrap_or(Importance::Medium); - let merged_importance = max_importance(existing_importance, memory.importance); - - let existing_keywords: Vec = - serde_json::from_str(&existing_keywords_json).unwrap_or_default(); - let mut merged_keywords = existing_keywords.clone(); - for kw in &memory.keywords { - if !merged_keywords.contains(kw) { - merged_keywords.push(kw.clone()); - } - } - - let merged_raw = memory.raw_excerpt.clone().or(existing_raw.clone()); - - let importance_changed = merged_importance != existing_importance; - let keywords_changed = merged_keywords != existing_keywords; - let raw_changed = merged_raw != existing_raw; - if importance_changed || keywords_changed || raw_changed { - let merged_keywords_json = serde_json::to_string(&merged_keywords)?; - self.conn - .execute( - "UPDATE memories - SET importance = ?1, keywords = ?2, raw_excerpt = ?3, updated_at = ?4 - WHERE id = ?5", - params![ - merged_importance.to_string(), - merged_keywords_json, - merged_raw, - Utc::now().to_rfc3339(), - existing_id, - ], - ) - .map_err(db_err)?; - self.cache_invalidate(&existing_id); - } - - tracing::debug!( - topic = %memory.topic, - existing = %existing_id, - attempted = %memory.id, - imp_changed = importance_changed, - kw_changed = keywords_changed, - raw_changed = raw_changed, - "store: dedup'd duplicate memory (metadata merged)" - ); - return Ok(existing_id); - } - - // Sync to vec_memories for KNN search (only on a fresh insert). - if let Some(ref blob) = emb_blob { - self.conn - .execute( - "INSERT INTO vec_memories (memory_id, embedding) VALUES (?1, ?2)", - params![memory.id, blob], - ) - .map_err(db_err)?; - } - - Ok(memory.id.clone()) - } -} - -impl MemoryStore for SqliteStore { - fn store(&self, memory: Memory) -> IcmResult { - let memory = validate_and_normalize(memory)?; - - self.conn - .execute_batch("BEGIN IMMEDIATE;") - .map_err(db_err)?; - - match self.store_inner(&memory) { - Ok(id) => { - self.conn.execute_batch("COMMIT;").map_err(db_err)?; - Ok(id) - } - Err(e) => { - let _ = self.conn.execute_batch("ROLLBACK;"); - Err(e) - } - } - } - - fn get(&self, id: &str) -> IcmResult> { - if let Some(m) = self.cache_get(id) { - return Ok(Some(m)); - } - - let mut stmt = self - .conn - .prepare(&format!("SELECT {SELECT_COLS} FROM memories WHERE id = ?1")) - .map_err(db_err)?; - - let result = stmt - .query_row(params![id], row_to_memory) - .optional() - .map_err(db_err)?; - - if let Some(ref m) = result { - self.cache_put(m); - } - Ok(result) - } - - fn update(&self, memory: &Memory) -> IcmResult<()> { - // Same constraints as `store()` — without this, oversized or - // NUL-carrying payloads could bypass validation by storing small - // then updating big (audit finding). - validate_fields(&memory.topic, &memory.summary)?; - - let keywords_json = serde_json::to_string(&memory.keywords)?; - let related_json = serde_json::to_string(&memory.related_ids)?; - let st = source_type(&memory.source); - let sd = source_data(&memory.source); - let emb_blob = memory.embedding.as_deref().map(embedding_to_blob); - - // Recompute summary_hash on update — topic or summary may have - // changed, and the partial unique index on (topic, summary_hash) - // would otherwise reflect stale state. - let hash = summary_hash(&memory.topic, &memory.summary); - - // memories + vec_memories must move together: a failure between the - // row update and the vector sync would leave a memory invisible to - // (or stale in) vector search (audit finding — same pattern as - // `store()` / `consolidate_topic`). - self.conn - .execute_batch("BEGIN IMMEDIATE;") - .map_err(db_err)?; - - let result: IcmResult<()> = (|| { - let changed = self - .conn - .execute( - "UPDATE memories SET - updated_at = ?2, last_accessed = ?3, access_count = ?4, weight = ?5, - topic = ?6, summary = ?7, raw_excerpt = ?8, keywords = ?9, - importance = ?10, source_type = ?11, source_data = ?12, related_ids = ?13, - embedding = ?14, summary_hash = ?15 - WHERE id = ?1", - params![ - memory.id, - memory.updated_at.to_rfc3339(), - memory.last_accessed.to_rfc3339(), - memory.access_count, - memory.weight, - memory.topic, - memory.summary, - memory.raw_excerpt, - keywords_json, - memory.importance.to_string(), - st, - sd, - related_json, - emb_blob, - hash, - ], - ) - .map_err(db_err)?; - - if changed == 0 { - return Err(IcmError::NotFound(memory.id.clone())); - } - - // Sync vec_memories: always delete old, re-insert if embedding exists - self.conn - .execute( - "DELETE FROM vec_memories WHERE memory_id = ?1", - params![memory.id], - ) - .map_err(db_err)?; - if let Some(ref blob) = emb_blob { - self.conn - .execute( - "INSERT INTO vec_memories (memory_id, embedding) VALUES (?1, ?2)", - params![memory.id, blob], - ) - .map_err(db_err)?; - } - Ok(()) - })(); - - match result { - Ok(()) => { - self.conn.execute_batch("COMMIT;").map_err(db_err)?; - self.cache_invalidate(&memory.id); - Ok(()) - } - Err(e) => { - let _ = self.conn.execute_batch("ROLLBACK;"); - Err(e) - } - } - } - - fn delete(&self, id: &str) -> IcmResult<()> { - // Both deletes in one transaction so a failure can't strand an - // orphaned vector or a memory whose vector is gone (audit finding). - self.conn - .execute_batch("BEGIN IMMEDIATE;") - .map_err(db_err)?; - - let result: IcmResult<()> = (|| { - self.conn - .execute("DELETE FROM vec_memories WHERE memory_id = ?1", params![id]) - .map_err(db_err)?; - - let changed = self - .conn - .execute("DELETE FROM memories WHERE id = ?1", params![id]) - .map_err(db_err)?; - - if changed == 0 { - return Err(IcmError::NotFound(id.to_string())); - } - - // Manual-testing finding: deleting a memory left it as a - // dangling entry in every other memory's `related_ids` - // (auto-link back-references) forever — `expand_with_neighbors` - // tolerates the miss silently, but each stale id still spends a - // slot out of the caller's `max_neighbors` budget instead of - // surfacing a real, live neighbor, and any external consumer of - // the JSON export sees a reference to nothing. Strip the - // deleted id from every `related_ids` array that mentions it. - // The `LIKE` clause is a cheap prefilter (only rows that could - // possibly match do the JSON rewrite); it matches the - // JSON-quoted form specifically so a ULID that happens to be a - // literal substring of another can't cause a false hit. - self.conn - .execute( - "UPDATE memories - SET related_ids = ( - SELECT COALESCE(json_group_array(value), '[]') - FROM json_each(memories.related_ids) - WHERE value != ?1 - ) - WHERE related_ids LIKE '%\"' || ?1 || '\"%'", - params![id], - ) - .map_err(db_err)?; - - Ok(()) - })(); - - match result { - Ok(()) => { - self.conn.execute_batch("COMMIT;").map_err(db_err)?; - // The related_ids cleanup above can touch an arbitrary - // number of other rows, not just `id` — clear the whole - // cache rather than tracking which ones, so a cached - // neighbor's `related_ids` can't keep serving the - // just-deleted id after this returns. - self.cache_clear(); - Ok(()) - } - Err(e) => { - let _ = self.conn.execute_batch("ROLLBACK;"); - Err(e) - } - } - } - - fn search_by_keywords(&self, keywords: &[&str], limit: usize) -> IcmResult> { - if keywords.is_empty() { - return Ok(Vec::new()); - } - - // Cap keywords to avoid massive SQL generation - let keywords = &keywords[..keywords.len().min(50)]; - let limit = limit.min(100); - - // Audit finding: a keyword containing `%` or `_` was interpolated - // straight into the LIKE pattern unescaped. `%` matches every row - // (`"100%"` as a keyword becomes the pattern `%100%%%`, which - // degrades to "contains 100" at best and can blow up matching); - // `_` matches any single character (`"snake_case"` also matches - // "snakeXcase"). Both are plausible keywords coming from an LLM via - // MCP. Escape them and declare the escape character explicitly. - let where_parts: Vec = (0..keywords.len()) - .map(|i| { - let p = i + 1; - format!( - "(keywords LIKE ?{p} ESCAPE '\\' OR summary LIKE ?{p} ESCAPE '\\' \ - OR topic LIKE ?{p} ESCAPE '\\')" - ) - }) - .collect(); - let where_clause = where_parts.join(" OR "); - - let query = format!( - "SELECT {SELECT_COLS} FROM memories WHERE {where_clause} ORDER BY weight DESC LIMIT ?{}", - keywords.len() + 1 - ); - - let mut stmt = self.conn.prepare(&query).map_err(db_err)?; - - let mut param_values: Vec> = keywords - .iter() - .map(|k| { - Box::new(format!("%{}%", escape_like_wildcards(k))) - as Box - }) - .collect(); - param_values.push(Box::new(limit as i64)); - - let params_ref: Vec<&dyn rusqlite::types::ToSql> = - param_values.iter().map(|p| p.as_ref()).collect(); - - let rows = stmt - .query_map(params_ref.as_slice(), row_to_memory) - .map_err(db_err)?; - - collect_rows(rows) - } - - fn search_fts(&self, query: &str, limit: usize) -> IcmResult> { - let limit = limit.min(100); - let sanitized = sanitize_fts_query(query); - if sanitized.is_empty() { - return Ok(Vec::new()); - } - - let sql = format!( - "SELECT {SELECT_COLS} FROM memories - WHERE id IN ( - SELECT id FROM memories_fts WHERE memories_fts MATCH ?1 - ) - ORDER BY weight DESC - LIMIT ?2" - ); - - let mut stmt = self.conn.prepare(&sql).map_err(db_err)?; - - let rows = stmt - .query_map(params![sanitized, limit as i64], row_to_memory) - .map_err(db_err)?; - - collect_rows(rows) - } - - fn search_by_embedding( - &self, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - let query_blob = embedding_to_blob(embedding); - - // KNN query on vec0 virtual table (requires LIMIT in the query itself) - let mut knn_stmt = self - .conn - .prepare( - "SELECT memory_id, distance - FROM vec_memories - WHERE embedding MATCH ?1 - ORDER BY distance - LIMIT ?2", - ) - .map_err(db_err)?; - - let knn_rows: Vec<(String, f32)> = knn_stmt - .query_map(params![query_blob, limit as i64], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, f32>(1)?)) - }) - .map_err(db_err)? - .filter_map(|r| r.ok()) - .collect(); - - if knn_rows.is_empty() { - return Ok(Vec::new()); - } - - // Batch fetch all memories in one query - let placeholders: Vec = (1..=knn_rows.len()).map(|i| format!("?{i}")).collect(); - let sql = format!( - "SELECT {SELECT_COLS} FROM memories WHERE id IN ({})", - placeholders.join(", ") - ); - let mut stmt = self.conn.prepare(&sql).map_err(db_err)?; - - let ids: Vec<&str> = knn_rows.iter().map(|(id, _)| id.as_str()).collect(); - let params: Vec<&dyn rusqlite::types::ToSql> = ids - .iter() - .map(|id| id as &dyn rusqlite::types::ToSql) - .collect(); - - let rows = stmt.query_map(&*params, row_to_memory).map_err(db_err)?; - - let mut memory_map: std::collections::HashMap = HashMap::new(); - for row in rows.flatten() { - memory_map.insert(row.id.clone(), row); - } - - // Reassemble in KNN order with similarity scores - let results: Vec<(Memory, f32)> = knn_rows - .into_iter() - .filter_map(|(id, distance)| memory_map.remove(&id).map(|mem| (mem, 1.0 - distance))) - .collect(); - - Ok(results) - } - - fn search_hybrid( - &self, - query: &str, - embedding: &[f32], - limit: usize, - ) -> IcmResult> { - let limit = limit.min(1000); - let pool_size = limit * 4; - let sanitized = sanitize_fts_query(query); - - // 1. Get FTS results with rank scores - let fts_sql = - "SELECT m.id, m.created_at, m.updated_at, m.last_accessed, m.access_count, m.weight, \ - m.topic, m.summary, m.raw_excerpt, m.keywords, \ - m.importance, m.source_type, m.source_data, m.related_ids, m.embedding, \ - fts.rank \ - FROM memories_fts fts \ - JOIN memories m ON m.id = fts.id \ - WHERE memories_fts MATCH ?1 \ - ORDER BY fts.rank \ - LIMIT ?2"; - - let mut fts_scores: HashMap = HashMap::with_capacity(pool_size); - let mut all_memories: HashMap = HashMap::with_capacity(pool_size); - - if !sanitized.is_empty() { - if let Ok(mut stmt) = self.conn.prepare(fts_sql) { - if let Ok(rows) = stmt.query_map(params![sanitized, pool_size as i64], |row| { - let memory = row_to_memory(row)?; - let rank: f32 = row.get(15)?; - Ok((memory, rank)) - }) { - for row in rows.flatten() { - let (memory, rank) = row; - // FTS5 bm25 rank is <= 0, MORE negative = MORE - // relevant. `1.0 / (1.0 + |rank|)` inverted this: it - // DECREASES as relevance increases (audit finding, - // proven wrong e.g. rank=-4.83 (strong match) scored - // 0.17 while rank=-0.2 (weak match) scored 0.83). - // `|rank| / (1.0 + |rank|)` keeps the same bounded - // [0,1) shape but is correctly monotonically - // INCREASING in relevance. - let score = rank.abs() / (1.0 + rank.abs()); - fts_scores.insert(memory.id.clone(), score); - all_memories.insert(memory.id.clone(), memory); - } - } - } - } // sanitized.is_empty() - - // 2. Get vector results - let vec_results = self.search_by_embedding(embedding, pool_size)?; - let mut vec_scores: HashMap = HashMap::with_capacity(pool_size); - for (memory, similarity) in vec_results { - vec_scores.insert(memory.id.clone(), similarity); - all_memories.entry(memory.id.clone()).or_insert(memory); - } - - // 3. Combine scores: 30% FTS + 70% vector - let keys: Vec = all_memories.keys().cloned().collect(); - let mut scored: Vec<(String, f32)> = Vec::with_capacity(keys.len()); - for id in keys { - let fts_score = fts_scores.get(&id).copied().unwrap_or(0.0); - let vec_score = vec_scores.get(&id).copied().unwrap_or(0.0); - let combined = 0.3 * fts_score + 0.7 * vec_score; - scored.push((id, combined)); - } - - // Sort by combined score descending - scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - scored.truncate(limit); - - let results: Vec<(Memory, f32)> = scored - .into_iter() - .filter_map(|(id, score)| all_memories.remove(&id).map(|mem| (mem, score))) - .collect(); - - Ok(results) - } - - fn update_access(&self, id: &str) -> IcmResult<()> { - // Read-only short-circuit (issue #263): callers of recall expect - // this to be best-effort bookkeeping, not a hard precondition. - // Skipping silently lets `icm recall` work against a DB the - // process cannot write to. - if self.readonly { - return Ok(()); - } - let now = Utc::now().to_rfc3339(); - let changed = self - .conn - .execute( - "UPDATE memories SET last_accessed = ?1, access_count = access_count + 1 WHERE id = ?2", - params![now, id], - ) - .map_err(db_err)?; - - if changed == 0 { - return Err(IcmError::NotFound(id.to_string())); - } - self.cache_invalidate(id); - Ok(()) - } - - fn batch_update_access(&self, ids: &[&str]) -> IcmResult { - if ids.is_empty() { - return Ok(0); - } - if self.readonly { - // Same rationale as `update_access` (issue #263). - return Ok(0); - } - let now = Utc::now().to_rfc3339(); - let placeholders: Vec = (2..=ids.len() + 1).map(|i| format!("?{i}")).collect(); - let sql = format!( - "UPDATE memories SET last_accessed = ?1, access_count = access_count + 1 WHERE id IN ({})", - placeholders.join(", ") - ); - let mut params_vec: Vec> = - Vec::with_capacity(ids.len() + 1); - params_vec.push(Box::new(now)); - for id in ids { - params_vec.push(Box::new(id.to_string())); - } - let refs: Vec<&dyn rusqlite::types::ToSql> = - params_vec.iter().map(|p| p.as_ref()).collect(); - let changed = self.conn.execute(&sql, refs.as_slice()).map_err(db_err)?; - self.cache_invalidate_many(ids); - Ok(changed) - } - - fn apply_decay(&self, decay_factor: f32) -> IcmResult { - if self.readonly { - return Err(IcmError::ReadOnly("apply_decay".into())); - } - // Access-aware decay: frequently accessed memories decay slower. - // decay = base_rate * importance_multiplier / (1 + min(access_count, 5) * 0.1) - // - // Audit #185 H7: the access-count term used to be uncapped - // (`1 + access_count * 0.1`). A memory with `access_count=100` - // got a 11x slowdown on its decay, which made it effectively - // immune to pruning even at low importance. Anyone (or any - // bench loop, or any benign hook-driven recall pattern) that - // touched a memory many times pinned it near the top of the - // ranking forever — the same gaming class as the M01 issue - // the maintainer flagged earlier. - // - // Cap at 5 accesses → max 1.5x slowdown (33%). That preserves - // the original intent ("useful memories decay a bit slower") - // without giving any single memory infinite decay immunity. - // Critical-importance memories still skip decay entirely. - // - // Importance multipliers: - // critical: never decays (filtered by WHERE clause) - // high: 0.5x decay (half speed) - // medium: 1.0x decay (normal) - // low: 2.0x decay (double speed) - // Audit finding: for `low` importance (2x multiplier) with a - // low-access memory, the multiplier `1.0 - (1.0-f)*mult/denom` goes - // NEGATIVE once `f < 0.5` — `icm decay --factor 0.4` (accepted by - // the CLI's own `[0.0, 1.0)` validation) drove low-importance - // weights negative, putting them last in every `ORDER BY weight - // DESC` and prunable on the next pass. `MAX(0.0, ...)` clamps the - // multiplier at the SQL layer so weight can never go negative - // regardless of the caller (CLI, MCP, or any future direct caller - // that bypasses the CLI's own boundary check). - let changed = self - .conn - .execute( - "UPDATE memories SET weight = weight * MAX(0.0, - 1.0 - (1.0 - ?1) * - CASE importance - WHEN 'high' THEN 0.5 - WHEN 'low' THEN 2.0 - ELSE 1.0 - END - / (1.0 + MIN(access_count, 5) * 0.1) - ) - WHERE importance != 'critical'", - params![decay_factor], - ) - .map_err(db_err)?; - - // Decay touches every non-critical row's weight; can't selectively - // invalidate without re-reading rows, so just nuke the cache. - self.cache_clear(); - Ok(changed) - } - - fn prune(&self, weight_threshold: f32) -> IcmResult { - // Never prune critical or high importance memories. Both deletes in - // one transaction, and the vec_memories error is propagated instead - // of swallowed — a partial prune would leave orphaned vectors that - // keep matching KNN search for rows that no longer exist (audit - // finding). - self.conn - .execute_batch("BEGIN IMMEDIATE;") - .map_err(db_err)?; - - let result: IcmResult = (|| { - self.conn.execute( - "DELETE FROM vec_memories WHERE memory_id IN ( - SELECT id FROM memories WHERE weight < ?1 AND importance NOT IN ('critical', 'high') - )", - params![weight_threshold], - ) - .map_err(db_err)?; - - self.conn - .execute( - "DELETE FROM memories WHERE weight < ?1 AND importance NOT IN ('critical', 'high')", - params![weight_threshold], - ) - .map_err(db_err) - })(); - - match result { - Ok(changed) => { - self.conn.execute_batch("COMMIT;").map_err(db_err)?; - if changed > 0 { - self.cache_clear(); - } - Ok(changed) - } - Err(e) => { - let _ = self.conn.execute_batch("ROLLBACK;"); - Err(e) - } - } - } - - fn get_by_topic(&self, topic: &str) -> IcmResult> { - let mut stmt = self - .conn - .prepare(&format!( - "SELECT {SELECT_COLS} FROM memories WHERE topic = ?1 ORDER BY weight DESC LIMIT 500" - )) - .map_err(db_err)?; - - let rows = stmt - .query_map(params![topic], row_to_memory) - .map_err(db_err)?; - - collect_rows(rows) - } - - fn list_all(&self) -> IcmResult> { - let mut stmt = self - .conn - .prepare(&format!( - "SELECT {SELECT_COLS} FROM memories ORDER BY weight DESC LIMIT 10000" - )) - .map_err(db_err)?; - - let rows = stmt.query_map([], row_to_memory).map_err(db_err)?; - collect_rows(rows) - } - - fn list_topics(&self) -> IcmResult> { - let mut stmt = self - .conn - .prepare("SELECT topic, COUNT(*) FROM memories GROUP BY topic ORDER BY topic") - .map_err(db_err)?; - - let rows = stmt - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, usize>(1)?)) - }) - .map_err(db_err)?; - - collect_rows(rows) - } - - fn consolidate_topic(&self, topic: &str, consolidated: Memory) -> IcmResult<()> { - // The consolidated memory goes through the same validation as any - // other write — MCP `icm_memory_consolidate` passes a caller-provided - // summary that previously bypassed every size/content check. - let consolidated = validate_and_normalize(consolidated)?; - - self.conn - .execute_batch("BEGIN IMMEDIATE;") - .map_err(db_err)?; - - // Manual-testing finding: captured before the delete below, since - // afterward the rows (and thus this query) are gone. Used to clean - // up any *other* memory's related_ids that pointed at these — - // same dangling-reference bug already fixed for the single-id - // `delete`, reachable here too since this is a second, separate - // bulk-delete code path. - let deleted_ids: Vec = { - let mut stmt = self - .conn - .prepare("SELECT id FROM memories WHERE topic = ?1 AND importance != 'critical'") - .map_err(db_err)?; - let rows = stmt - .query_map(params![topic], |row| row.get::<_, String>(0)) - .map_err(db_err)?; - rows.collect::>>().map_err(db_err)? - }; - - // `critical` memories are never deleted — same contract as - // `apply_decay` and `prune`. Consolidation replaces the expendable - // tail of a topic, not its "never forget" entries (audit finding: - // this DELETE previously wiped critical memories too). - // Clean vec_memories for entries about to be deleted - if let Err(e) = self.conn.execute( - "DELETE FROM vec_memories WHERE memory_id IN ( - SELECT id FROM memories WHERE topic = ?1 AND importance != 'critical' - )", - params![topic], - ) { - tracing::warn!(topic, error = %e, "consolidate_topic: rolling back after vec_memories delete failed"); - let _ = self.conn.execute_batch("ROLLBACK;"); - return Err(IcmError::Database(e.to_string())); - } - - if let Err(e) = self.conn.execute( - "DELETE FROM memories WHERE topic = ?1 AND importance != 'critical'", - params![topic], - ) { - tracing::warn!(topic, error = %e, "consolidate_topic: rolling back after memories delete failed"); - let _ = self.conn.execute_batch("ROLLBACK;"); - return Err(IcmError::Database(e.to_string())); - } - - if !deleted_ids.is_empty() { - let ids_json = serde_json::to_string(&deleted_ids).map_err(IcmError::from)?; - if let Err(e) = self.conn.execute( - "UPDATE memories - SET related_ids = ( - SELECT COALESCE(json_group_array(value), '[]') - FROM json_each(memories.related_ids) - WHERE value NOT IN (SELECT value FROM json_each(?1)) - ) - WHERE EXISTS ( - SELECT 1 FROM json_each(memories.related_ids) - WHERE value IN (SELECT value FROM json_each(?1)) - )", - params![ids_json], - ) { - tracing::warn!(topic, error = %e, "consolidate_topic: rolling back after related_ids cleanup failed"); - let _ = self.conn.execute_batch("ROLLBACK;"); - return Err(IcmError::Database(e.to_string())); - } - } - - if let Err(e) = self.store_inner(&consolidated) { - tracing::warn!(topic, error = %e, "consolidate_topic: rolling back after store failed"); - let _ = self.conn.execute_batch("ROLLBACK;"); - return Err(e); - } - - // Rebuild FTS index to eliminate any ghost entries from the external - // content table. This guarantees search results stay consistent after - // bulk deletes (fixes #44). - if let Err(e) = self - .conn - .execute_batch("INSERT INTO memories_fts(memories_fts) VALUES('rebuild');") - { - tracing::warn!(topic, error = %e, "consolidate_topic: rolling back after FTS rebuild failed"); - let _ = self.conn.execute_batch("ROLLBACK;"); - return Err(IcmError::Database(e.to_string())); - } - - self.conn.execute_batch("COMMIT;").map_err(db_err)?; - // Bulk delete + re-insert touches arbitrarily many cached entries. - self.cache_clear(); - Ok(()) - } - - fn count(&self) -> IcmResult { - self.conn - .query_row("SELECT COUNT(*) FROM memories", [], |row| { - row.get::<_, usize>(0) - }) - .map_err(|e| IcmError::Database(e.to_string())) - } - - fn count_by_topic(&self, topic: &str) -> IcmResult { - self.conn - .query_row( - "SELECT COUNT(*) FROM memories WHERE topic = ?1", - params![topic], - |row| row.get::<_, usize>(0), - ) - .map_err(|e| IcmError::Database(e.to_string())) - } - - fn topic_health(&self, topic: &str) -> IcmResult { - let row = self - .conn - .query_row( - "SELECT - COUNT(*), - AVG(weight), - AVG(CAST(access_count AS REAL)), - MIN(created_at), - MAX(created_at), - MAX(last_accessed), - SUM(CASE WHEN weight < 0.5 - AND julianday('now') - julianday(last_accessed) > 14 - THEN 1 ELSE 0 END) - FROM memories WHERE topic = ?1", - params![topic], - |row| { - Ok(( - row.get::<_, usize>(0)?, - row.get::<_, f32>(1)?, - row.get::<_, f32>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, usize>(6)?, - )) - }, - ) - .map_err(db_err)?; - - let ( - entry_count, - avg_weight, - avg_access, - oldest_str, - newest_str, - last_accessed_str, - stale_count, - ) = row; - - if entry_count == 0 { - return Err(IcmError::NotFound(format!("topic: {topic}"))); - } - - let parse_dt = |s: &str| -> Option> { - match DateTime::parse_from_rfc3339(s) { - Ok(d) => Some(d.with_timezone(&Utc)), - Err(e) => { - tracing::warn!("invalid timestamp '{}': {}", s, e); - None - } - } - }; - - Ok(TopicHealth { - topic: topic.to_string(), - entry_count, - avg_weight, - avg_access_count: avg_access, - oldest: oldest_str.as_deref().and_then(parse_dt), - newest: newest_str.as_deref().and_then(parse_dt), - last_accessed: last_accessed_str.as_deref().and_then(parse_dt), - needs_consolidation: entry_count > 5, - stale_count, - }) - } - - fn stats(&self) -> IcmResult { - let (total_memories, total_topics, avg_weight, oldest_str, newest_str): ( - usize, - usize, - f32, - Option, - Option, - ) = self - .conn - .query_row( - "SELECT COUNT(*), COUNT(DISTINCT topic), COALESCE(AVG(weight), 0.0), \ - MIN(created_at), MAX(created_at) FROM memories", - [], - |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - )) - }, - ) - .map_err(db_err)?; - - let oldest_memory = oldest_str - .and_then(|s| DateTime::parse_from_rfc3339(&s).ok()) - .map(|d| d.with_timezone(&Utc)); - let newest_memory = newest_str - .and_then(|s| DateTime::parse_from_rfc3339(&s).ok()) - .map(|d| d.with_timezone(&Utc)); - - Ok(StoreStats { - total_memories, - total_topics, - avg_weight, - oldest_memory, - newest_memory, - }) - } -} - -// --------------------------------------------------------------------------- -// Memoir / Concept helpers -// --------------------------------------------------------------------------- - -fn parse_dt(s: &str) -> DateTime { - DateTime::parse_from_rfc3339(s) - .map(|d| d.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()) -} - -fn row_to_memoir(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Memoir { - id: row.get(0)?, - name: row.get(1)?, - description: row.get(2)?, - created_at: parse_dt(&row.get::<_, String>(3)?), - updated_at: parse_dt(&row.get::<_, String>(4)?), - consolidation_threshold: row.get::<_, u32>(5)?, - }) -} - -const MEMOIR_COLS: &str = "id, name, description, created_at, updated_at, consolidation_threshold"; - -fn row_to_concept(row: &rusqlite::Row) -> rusqlite::Result { - let labels_json: String = row.get(4)?; - let labels: Vec