From a75c443d8630a813d24f026fefb8c2b0b5c1477d Mon Sep 17 00:00:00 2001 From: Parth576 Date: Wed, 4 Mar 2026 23:55:28 -0500 Subject: [PATCH 1/5] feat(vectorstore): add in-memory VectorStore with Delete method and config flag Add InMemoryStore as a lightweight alternative to Qdrant for resource-constrained deployments. Add Delete method to VectorStore interface for post-analysis vector cleanup. Wire conditional store initialization via VECTOR_STORE env var (default: memory). - Add Delete(ctx, collectionID, url, contentHash) to VectorStore interface - Create InMemoryStore with sync.RWMutex thread safety - Add no-op Delete to QdrantStore (logs and returns nil) - Add VECTOR_STORE config field (memory|qdrant, default: memory) - Update main.go for conditional store initialization - Add Delete pass-through to RAG pipeline - Call Delete in analyzer after caching results - 13 unit tests for InMemoryStore (all pass with -race) Assisted by the code-assist SOP --- .../context.md | 26 +++ .../progress.md | 24 ++ backend/cmd/server/main.go | 36 ++- backend/internal/analyzer/analyzer.go | 5 + backend/internal/config/config.go | 2 + backend/internal/rag/pipeline.go | 5 + backend/internal/vectorstore/memory.go | 99 ++++++++ backend/internal/vectorstore/memory_test.go | 213 ++++++++++++++++++ backend/internal/vectorstore/qdrant.go | 11 + backend/internal/vectorstore/store.go | 19 ++ 10 files changed, 429 insertions(+), 11 deletions(-) create mode 100644 .agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/context.md create mode 100644 .agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/progress.md create mode 100644 backend/internal/vectorstore/memory.go create mode 100644 backend/internal/vectorstore/memory_test.go diff --git a/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/context.md b/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/context.md new file mode 100644 index 0000000..f37e874 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/context.md @@ -0,0 +1,26 @@ +# Context: In-Memory VectorStore and Config + +## Requirements +- Add `Delete` method to `VectorStore` interface +- Create `InMemoryStore` implementing full `VectorStore` interface +- Add `VECTOR_STORE` config flag (memory/qdrant, default: memory) +- Update `main.go` for conditional store initialization +- Add `Delete` to RAG pipeline (pass-through) +- Call `Delete` in analyzer after caching results + +## Key Files +- `backend/internal/vectorstore/store.go` - Interface + MockVectorStore +- `backend/internal/vectorstore/qdrant.go` - QdrantStore (add no-op Delete) +- `backend/internal/vectorstore/memory.go` - New InMemoryStore +- `backend/internal/config/config.go` - Add VectorStore field +- `backend/cmd/server/main.go` - Conditional wiring +- `backend/internal/rag/pipeline.go` - Add Delete pass-through +- `backend/internal/analyzer/analyzer.go` - Call Delete after caching + +## Patterns +- stdlib testing, no test framework +- `slog.Logger` for structured logging +- `sync.RWMutex` for thread safety +- Interface-based DI throughout +- `t.Setenv()` for config tests +- Mock types record calls for assertion diff --git a/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/progress.md b/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/progress.md new file mode 100644 index 0000000..4fcf5b7 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/step18-task01-in-memory-vectorstore/progress.md @@ -0,0 +1,24 @@ +# Progress: In-Memory VectorStore and Config + +## Setup +- [x] Documentation directory created +- [x] Context document created +- [x] Existing code reviewed + +## Implementation +- [x] Add Delete to VectorStore interface and MockVectorStore +- [x] Add no-op Delete to QdrantStore +- [x] Write InMemoryStore tests (TDD - RED) - 13 tests covering Upsert, Delete, Search, HealthCheck, thread safety, nil logger +- [x] Create InMemoryStore implementation (TDD - GREEN) +- [x] Add VectorStore config field (VECTOR_STORE env var, default "memory") +- [x] Update main.go wiring (conditional init, renamed constant to `collection`) +- [x] Add Delete to RAG pipeline (pass-through to store.Delete) +- [x] Call Delete in analyzer after caching (Stage 12, warn-only on error) +- [x] Run full test suite - ALL PASS with -race flag +- [x] Build compiles successfully + +## Decisions +- Delete in analyzer logs warning on error but does not fail the pipeline (non-critical cleanup) +- InMemoryStore.Search is a placeholder that returns filtered chunks without similarity scoring (Task 2) +- InMemoryStore.Delete matches on both URL AND contentHash (both must match) +- Used `kept := chunks[:0]` pattern for in-place filtering to reduce allocations diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index d23ad8c..ce3ffac 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net/http" "os" @@ -16,7 +17,7 @@ import ( "github.com/parth/smolterms/backend/internal/vectorstore" ) -const qdrantCollection = "smolterms" +const collection = "smolterms" func main() { cfg, err := config.Load() @@ -38,21 +39,34 @@ func main() { embedder := embedding.NewOpenAIClient(cfg, logger) logger.Info("initialized embedding client", "provider", "openai") - // Initialize Qdrant vector store. - store, err := vectorstore.NewQdrantStore(cfg, logger) - if err != nil { - logger.Error("failed to initialize qdrant vector store", "error", err) - os.Exit(1) + // Initialize vector store based on configuration. + var store vectorstore.VectorStore + var healthCheck func(ctx context.Context) error + + switch cfg.VectorStore { + case "qdrant": + qs, err := vectorstore.NewQdrantStore(cfg, logger) + if err != nil { + logger.Error("failed to initialize qdrant vector store", "error", err) + os.Exit(1) + } + store = qs + healthCheck = qs.HealthCheck + logger.Info("initialized vector store", "provider", "qdrant", "url", cfg.QdrantURL) + default: + ms := vectorstore.NewInMemoryStore(logger) + store = ms + healthCheck = ms.HealthCheck + logger.Info("initialized vector store", "provider", "memory") } - logger.Info("initialized vector store", "provider", "qdrant", "url", cfg.QdrantURL) // Initialize Anthropic LLM client. llmClient := llm.NewAnthropicClient(cfg.AnthropicAPIKey, logger) logger.Info("initialized llm client", "provider", "anthropic") // Initialize RAG pipeline with embedding client and vector store. - ragPipeline := rag.NewPipeline(embedder, store, logger, qdrantCollection) - logger.Info("initialized rag pipeline", "collection", qdrantCollection) + ragPipeline := rag.NewPipeline(embedder, store, logger, collection) + logger.Info("initialized rag pipeline", "collection", collection) // Initialize in-memory cache. memCache := cache.NewMemoryCache(cacheTTL, cacheTTL/2) @@ -62,8 +76,8 @@ func main() { analyzerPipeline := analyzer.NewAnalyzer(ragPipeline, llmClient, memCache, logger) logger.Info("initialized analyzer pipeline") - // Wire the router with the analyzer and Qdrant health check. - router := api.NewRouter(logger, cfg, analyzerPipeline, store.HealthCheck) + // Wire the router with the analyzer and store health check. + router := api.NewRouter(logger, cfg, analyzerPipeline, healthCheck) addr := ":" + cfg.Port logger.Info("starting server", "addr", addr) diff --git a/backend/internal/analyzer/analyzer.go b/backend/internal/analyzer/analyzer.go index 6f173be..5731399 100644 --- a/backend/internal/analyzer/analyzer.go +++ b/backend/internal/analyzer/analyzer.go @@ -199,5 +199,10 @@ func (a *Analyzer) Analyze(ctx context.Context, req AnalysisRequest) (*AnalysisR } } + // Stage 12: Clean up vectors from store + if err := a.rag.Delete(ctx, req.URL, contentHash); err != nil { + a.logger.Warn("vector delete failed", slog.String("url", req.URL), slog.String("content_hash", contentHash), slog.Any("error", err)) + } + return result, nil } diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 6281783..e8948ae 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -15,6 +15,7 @@ type Config struct { AnthropicAPIKey string OpenAIAPIKey string QdrantURL string + VectorStore string CacheDefaultTTL string CORSAllowedOrigins string RateLimitRequests int @@ -39,6 +40,7 @@ func Load() (*Config, error) { AnthropicAPIKey: os.Getenv("ANTHROPIC_API_KEY"), OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"), QdrantURL: getEnvOrDefault("QDRANT_URL", "localhost:6334"), + VectorStore: getEnvOrDefault("VECTOR_STORE", "memory"), CacheDefaultTTL: getEnvOrDefault("CACHE_DEFAULT_TTL", "720h"), CORSAllowedOrigins: getEnvOrDefault("CORS_ALLOWED_ORIGINS", "*"), RateLimitRequests: rateLimitRequests, diff --git a/backend/internal/rag/pipeline.go b/backend/internal/rag/pipeline.go index 8bb2360..82c31c5 100644 --- a/backend/internal/rag/pipeline.go +++ b/backend/internal/rag/pipeline.go @@ -123,6 +123,11 @@ func (p *Pipeline) Retrieve(ctx context.Context, query string, limit int, url st return deduped, nil } +// Delete removes all chunks matching the given url and contentHash from the vector store. +func (p *Pipeline) Delete(ctx context.Context, url string, contentHash string) error { + return p.store.Delete(ctx, p.collection, url, contentHash) +} + // deduplicateByText removes duplicate chunks by text content, // keeping the one with the higher similarity score. func deduplicateByText(chunks []vectorstore.Chunk) []vectorstore.Chunk { diff --git a/backend/internal/vectorstore/memory.go b/backend/internal/vectorstore/memory.go new file mode 100644 index 0000000..e4958db --- /dev/null +++ b/backend/internal/vectorstore/memory.go @@ -0,0 +1,99 @@ +package vectorstore + +import ( + "context" + "log/slog" + "sync" +) + +// InMemoryStore implements VectorStore using in-memory maps, providing a +// lightweight alternative to Qdrant for resource-constrained deployments. +type InMemoryStore struct { + mu sync.RWMutex + chunks map[string][]Chunk // keyed by collectionID + logger *slog.Logger +} + +// NewInMemoryStore creates an InMemoryStore ready for use. +func NewInMemoryStore(logger *slog.Logger) *InMemoryStore { + if logger == nil { + logger = slog.Default() + } + return &InMemoryStore{ + chunks: make(map[string][]Chunk), + logger: logger, + } +} + +// Upsert appends chunks to the named collection. +func (s *InMemoryStore) Upsert(ctx context.Context, collectionID string, chunks []Chunk) error { + if len(chunks) == 0 { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + + s.chunks[collectionID] = append(s.chunks[collectionID], chunks...) + s.logger.InfoContext(ctx, "upserted chunks", "collection", collectionID, "count", len(chunks)) + return nil +} + +// Search returns stored chunks for the named collection. This is a placeholder +// implementation that returns all matching chunks; dot product similarity search +// is implemented in Task 2. +func (s *InMemoryStore) Search(ctx context.Context, collectionID string, _ []float32, limit int, url string, contentHash string) ([]Chunk, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + all := s.chunks[collectionID] + if len(all) == 0 { + return []Chunk{}, nil + } + + var result []Chunk + for _, c := range all { + if url != "" && c.URL != url { + continue + } + if contentHash != "" && c.ContentHash != contentHash { + continue + } + result = append(result, c) + } + + if limit > 0 && len(result) > limit { + result = result[:limit] + } + + return result, nil +} + +// Delete removes all chunks matching the given url and contentHash within the collection. +func (s *InMemoryStore) Delete(ctx context.Context, collectionID string, url string, contentHash string) error { + s.mu.Lock() + defer s.mu.Unlock() + + chunks, ok := s.chunks[collectionID] + if !ok { + return nil + } + + kept := chunks[:0] + removed := 0 + for _, c := range chunks { + if c.URL == url && c.ContentHash == contentHash { + removed++ + continue + } + kept = append(kept, c) + } + + s.chunks[collectionID] = kept + s.logger.InfoContext(ctx, "deleted chunks", "collection", collectionID, "url", url, "content_hash", contentHash, "removed", removed) + return nil +} + +// HealthCheck always returns nil for the in-memory store. +func (s *InMemoryStore) HealthCheck(_ context.Context) error { + return nil +} diff --git a/backend/internal/vectorstore/memory_test.go b/backend/internal/vectorstore/memory_test.go new file mode 100644 index 0000000..06069a6 --- /dev/null +++ b/backend/internal/vectorstore/memory_test.go @@ -0,0 +1,213 @@ +package vectorstore + +import ( + "context" + "log/slog" + "sync" + "testing" +) + +func newTestMemoryStore() *InMemoryStore { + return NewInMemoryStore(slog.Default()) +} + +func TestInMemoryStore_ImplementsVectorStore(t *testing.T) { + var _ VectorStore = &InMemoryStore{} +} + +func TestInMemoryStore_Upsert_StoresChunks(t *testing.T) { + s := newTestMemoryStore() + chunks := []Chunk{ + {ID: "c1", Text: "chunk one", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{0.1, 0.2}}, + {ID: "c2", Text: "chunk two", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{0.3, 0.4}}, + } + + err := s.Upsert(context.Background(), "col1", chunks) + if err != nil { + t.Fatalf("Upsert() error = %v, want nil", err) + } + + // Verify chunks are stored by retrieving via Search (placeholder returns stored chunks) + got, err := s.Search(context.Background(), "col1", []float32{0.1, 0.2}, 10, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 2 { + t.Errorf("Search() returned %d chunks, want 2", len(got)) + } +} + +func TestInMemoryStore_Upsert_EmptyChunks(t *testing.T) { + s := newTestMemoryStore() + err := s.Upsert(context.Background(), "col1", []Chunk{}) + if err != nil { + t.Fatalf("Upsert() error = %v, want nil", err) + } +} + +func TestInMemoryStore_Upsert_MultipleCollections(t *testing.T) { + s := newTestMemoryStore() + + err := s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one"}}) + if err != nil { + t.Fatalf("Upsert col1 error = %v", err) + } + err = s.Upsert(context.Background(), "col2", []Chunk{{ID: "b", Text: "two"}}) + if err != nil { + t.Fatalf("Upsert col2 error = %v", err) + } + + got1, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + got2, _ := s.Search(context.Background(), "col2", nil, 10, "", "") + + if len(got1) != 1 { + t.Errorf("col1 chunks = %d, want 1", len(got1)) + } + if len(got2) != 1 { + t.Errorf("col2 chunks = %d, want 1", len(got2)) + } +} + +func TestInMemoryStore_Upsert_AppendsToExisting(t *testing.T) { + s := newTestMemoryStore() + + _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one"}}) + _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "b", Text: "two"}}) + + got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + if len(got) != 2 { + t.Errorf("chunks after two upserts = %d, want 2", len(got)) + } +} + +func TestInMemoryStore_Delete_RemovesMatchingChunks(t *testing.T) { + s := newTestMemoryStore() + + chunks := []Chunk{ + {ID: "c1", Text: "chunk one", URL: "https://example.com", ContentHash: "hash1"}, + {ID: "c2", Text: "chunk two", URL: "https://example.com", ContentHash: "hash1"}, + {ID: "c3", Text: "chunk three", URL: "https://other.com", ContentHash: "hash2"}, + } + _ = s.Upsert(context.Background(), "col1", chunks) + + err := s.Delete(context.Background(), "col1", "https://example.com", "hash1") + if err != nil { + t.Fatalf("Delete() error = %v, want nil", err) + } + + got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + if len(got) != 1 { + t.Fatalf("chunks after delete = %d, want 1", len(got)) + } + if got[0].ID != "c3" { + t.Errorf("remaining chunk ID = %q, want %q", got[0].ID, "c3") + } +} + +func TestInMemoryStore_Delete_NonExistentCollection(t *testing.T) { + s := newTestMemoryStore() + err := s.Delete(context.Background(), "nonexistent", "url", "hash") + if err != nil { + t.Errorf("Delete() on nonexistent collection error = %v, want nil", err) + } +} + +func TestInMemoryStore_Delete_NoMatchingChunks(t *testing.T) { + s := newTestMemoryStore() + _ = s.Upsert(context.Background(), "col1", []Chunk{ + {ID: "c1", Text: "chunk", URL: "https://example.com", ContentHash: "hash1"}, + }) + + err := s.Delete(context.Background(), "col1", "https://other.com", "other-hash") + if err != nil { + t.Fatalf("Delete() error = %v, want nil", err) + } + + got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + if len(got) != 1 { + t.Errorf("chunks after no-match delete = %d, want 1", len(got)) + } +} + +func TestInMemoryStore_Delete_MatchesURLAndContentHash(t *testing.T) { + s := newTestMemoryStore() + chunks := []Chunk{ + {ID: "c1", URL: "https://example.com", ContentHash: "hash1"}, + {ID: "c2", URL: "https://example.com", ContentHash: "hash2"}, // same URL, different hash + {ID: "c3", URL: "https://other.com", ContentHash: "hash1"}, // different URL, same hash + } + _ = s.Upsert(context.Background(), "col1", chunks) + + _ = s.Delete(context.Background(), "col1", "https://example.com", "hash1") + + got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + if len(got) != 2 { + t.Fatalf("chunks after delete = %d, want 2", len(got)) + } + for _, c := range got { + if c.ID == "c1" { + t.Error("chunk c1 should have been deleted") + } + } +} + +func TestInMemoryStore_Search_NonExistentCollection(t *testing.T) { + s := newTestMemoryStore() + got, err := s.Search(context.Background(), "nonexistent", []float32{0.1}, 10, "", "") + if err != nil { + t.Fatalf("Search() error = %v, want nil", err) + } + if len(got) != 0 { + t.Errorf("Search() returned %d chunks, want 0", len(got)) + } +} + +func TestInMemoryStore_HealthCheck(t *testing.T) { + s := newTestMemoryStore() + err := s.HealthCheck(context.Background()) + if err != nil { + t.Errorf("HealthCheck() error = %v, want nil", err) + } +} + +func TestInMemoryStore_ThreadSafety(t *testing.T) { + s := newTestMemoryStore() + var wg sync.WaitGroup + ctx := context.Background() + + // Run concurrent upserts and deletes + for i := 0; i < 50; i++ { + wg.Add(2) + go func() { + defer wg.Done() + _ = s.Upsert(ctx, "col1", []Chunk{ + {ID: "c", Text: "text", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{0.1}}, + }) + }() + go func() { + defer wg.Done() + _ = s.Delete(ctx, "col1", "https://example.com", "hash1") + }() + } + + // Also run concurrent searches + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = s.Search(ctx, "col1", []float32{0.1}, 10, "", "") + }() + } + + wg.Wait() + // No assertion needed — test passes if no race detected with -race flag +} + +func TestInMemoryStore_NewWithNilLogger(t *testing.T) { + s := NewInMemoryStore(nil) + // Should not panic + err := s.Upsert(context.Background(), "col1", []Chunk{{ID: "a"}}) + if err != nil { + t.Errorf("Upsert with nil logger error = %v", err) + } +} diff --git a/backend/internal/vectorstore/qdrant.go b/backend/internal/vectorstore/qdrant.go index d7245ae..7e7e8b0 100644 --- a/backend/internal/vectorstore/qdrant.go +++ b/backend/internal/vectorstore/qdrant.go @@ -213,6 +213,17 @@ func chunkFromPayload(payload map[string]*qdrant.Value, score float32) Chunk { return c } +// Delete is a no-op for QdrantStore. Qdrant manages its own storage lifecycle, +// so we skip deletion and only log for observability. +func (s *QdrantStore) Delete(ctx context.Context, collectionID string, url string, contentHash string) error { + s.logger.InfoContext(ctx, "delete skipped for qdrant store", + "collection", collectionID, + "url", url, + "content_hash", contentHash, + ) + return nil +} + // HealthCheck verifies Qdrant connectivity by issuing a lightweight // CollectionExists call. It returns nil on success or the underlying error. func (s *QdrantStore) HealthCheck(ctx context.Context) error { diff --git a/backend/internal/vectorstore/store.go b/backend/internal/vectorstore/store.go index a1944a8..602d0fa 100644 --- a/backend/internal/vectorstore/store.go +++ b/backend/internal/vectorstore/store.go @@ -24,6 +24,8 @@ type VectorStore interface { // When url is non-empty, results are filtered to chunks from that URL. // When contentHash is non-empty, results are further filtered to that content version. Search(ctx context.Context, collectionID string, query []float32, limit int, url string, contentHash string) ([]Chunk, error) + // Delete removes all chunks matching the given url and contentHash within the collection. + Delete(ctx context.Context, collectionID string, url string, contentHash string) error } // UpsertCall records a single call to MockVectorStore.Upsert for test assertion. @@ -41,6 +43,13 @@ type SearchCall struct { ContentHash string } +// DeleteCall records a single call to MockVectorStore.Delete for test assertion. +type DeleteCall struct { + CollectionID string + URL string + ContentHash string +} + // MockVectorStore is a configurable VectorStore implementation for use in tests. type MockVectorStore struct { // UpsertErr is returned by Upsert if non-nil. @@ -49,11 +58,15 @@ type MockVectorStore struct { SearchResult []Chunk // SearchErr is returned by Search if non-nil. SearchErr error + // DeleteErr is returned by Delete if non-nil. + DeleteErr error // UpsertCalls records every call made to Upsert. UpsertCalls []UpsertCall // SearchCalls records every call made to Search. SearchCalls []SearchCall + // DeleteCalls records every call made to Delete. + DeleteCalls []DeleteCall } // Upsert records the call and returns UpsertErr. @@ -70,3 +83,9 @@ func (m *MockVectorStore) Search(_ context.Context, collectionID string, query [ } return m.SearchResult, nil } + +// Delete records the call and returns DeleteErr. +func (m *MockVectorStore) Delete(_ context.Context, collectionID string, url string, contentHash string) error { + m.DeleteCalls = append(m.DeleteCalls, DeleteCall{CollectionID: collectionID, URL: url, ContentHash: contentHash}) + return m.DeleteErr +} From b2401613e724e1194991e3b969e8710b08a72e7b Mon Sep 17 00:00:00 2001 From: Parth576 Date: Thu, 5 Mar 2026 08:37:35 -0500 Subject: [PATCH 2/5] feat(vectorstore): implement dot product similarity search for InMemoryStore Replace placeholder Search() with brute-force dot product similarity: - Add dotProduct() helper for computing vector similarity - Filter chunks by URL and contentHash before scoring - Skip chunks with nil/empty vectors - Sort results by score descending, return top N - Log search operations with structured fields (collection, filters, results, latency_ms) - Handle edge cases: empty collection, limit <= 0, fewer results than limit Add 12 new tests covering all acceptance criteria and update existing tests to include vector data for compatibility with real search logic. Assisted by the code-assist SOP --- .../dot-product-search/context.md | 21 ++ .../dot-product-search/plan.md | 27 ++ .../dot-product-search/progress.md | 15 + backend/internal/vectorstore/memory.go | 61 +++- backend/internal/vectorstore/memory_test.go | 333 +++++++++++++++++- 5 files changed, 431 insertions(+), 26 deletions(-) create mode 100644 .agents/scratchpad/2026-02-15-smolterms/dot-product-search/context.md create mode 100644 .agents/scratchpad/2026-02-15-smolterms/dot-product-search/plan.md create mode 100644 .agents/scratchpad/2026-02-15-smolterms/dot-product-search/progress.md diff --git a/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/context.md b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/context.md new file mode 100644 index 0000000..a6207e8 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/context.md @@ -0,0 +1,21 @@ +# Context: Dot Product Similarity Search + +## Task +Implement the core similarity search logic for InMemoryStore: brute-force dot product over stored vectors, filtered by URL/contentHash, returning top N most similar chunks sorted by score descending. + +## Key Files +- `backend/internal/vectorstore/memory.go` — InMemoryStore with placeholder Search() +- `backend/internal/vectorstore/memory_test.go` — Existing tests from Task 1 +- `backend/internal/vectorstore/store.go` — Chunk struct (has Score field) and VectorStore interface + +## Patterns +- Uses `sync.RWMutex` for thread safety (RLock for reads, Lock for writes) +- Structured logging with `slog` including latency_ms +- Returns `[]Chunk{}` (not nil) for empty results +- Chunk.Score field is `float32` with `omitempty` JSON tag + +## Implementation Requirements +1. `dotProduct(a, b []float32) float32` — helper, unexported +2. `Search()` — filter by url/contentHash, compute dot product, set Score, sort descending, limit results +3. Edge cases: empty collection → `[]Chunk{}`, limit ≤ 0 → `[]Chunk{}`, nil/empty vectors → skip +4. Log: collectionID, filters, result count, latency_ms diff --git a/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/plan.md b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/plan.md new file mode 100644 index 0000000..5cc448e --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/plan.md @@ -0,0 +1,27 @@ +# Plan: Dot Product Similarity Search + +## Test Scenarios + +1. **TestDotProduct** — Known vectors, verify mathematical correctness +2. **TestDotProduct_ZeroVectors** — Zero vector dot product = 0 +3. **TestInMemoryStore_Search_TopN** — 20 chunks, limit=15, verify 15 returned sorted descending +4. **TestInMemoryStore_Search_URLFilter** — Only chunks from specified URL considered +5. **TestInMemoryStore_Search_ContentHashFilter** — Both URL and contentHash filter applied +6. **TestInMemoryStore_Search_EmptyResults** — Empty collection returns []Chunk{} (not nil) +7. **TestInMemoryStore_Search_LimitZeroOrNegative** — Returns empty slice +8. **TestInMemoryStore_Search_ScorePopulation** — Each returned chunk has Score set +9. **TestInMemoryStore_Search_SkipsNilVectors** — Chunks with nil/empty vectors skipped +10. **TestInMemoryStore_Search_LimitExceedsAvailable** — Returns all if fewer than limit +11. **TestInMemoryStore_Search_ConcurrentReads** — No race conditions with -race +12. **TestInMemoryStore_Search_Logging** — Structured log with correct fields + +## Implementation Plan + +1. Add `dotProduct` helper (unexported, simple loop) +2. Replace placeholder Search: + - Early return for limit ≤ 0 + - Get chunks, filter by url/contentHash + - Skip nil/empty vectors, compute dot product, set Score + - Sort with `slices.SortFunc` descending + - Truncate to limit + - Log with slog diff --git a/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/progress.md b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/progress.md new file mode 100644 index 0000000..da2ad87 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/dot-product-search/progress.md @@ -0,0 +1,15 @@ +# Progress: Dot Product Similarity Search + +## Setup +- [x] Created documentation directory +- [x] Reviewed task requirements and existing code + +## Implementation +- [x] Write TDD tests for search functionality (12 new tests) +- [x] Verify tests fail (RED phase) — dotProduct undefined +- [x] Implement dotProduct helper +- [x] Implement full Search method with filtering, scoring, sorting, logging +- [x] Fix existing Task 1 tests to include vectors on chunks +- [x] Verify tests pass (GREEN phase) — all 60+ tests pass +- [x] Run full backend test suite with -race — all packages pass +- [ ] Commit diff --git a/backend/internal/vectorstore/memory.go b/backend/internal/vectorstore/memory.go index e4958db..8b6369a 100644 --- a/backend/internal/vectorstore/memory.go +++ b/backend/internal/vectorstore/memory.go @@ -3,7 +3,9 @@ package vectorstore import ( "context" "log/slog" + "slices" "sync" + "time" ) // InMemoryStore implements VectorStore using in-memory maps, providing a @@ -38,10 +40,25 @@ func (s *InMemoryStore) Upsert(ctx context.Context, collectionID string, chunks return nil } -// Search returns stored chunks for the named collection. This is a placeholder -// implementation that returns all matching chunks; dot product similarity search -// is implemented in Task 2. -func (s *InMemoryStore) Search(ctx context.Context, collectionID string, _ []float32, limit int, url string, contentHash string) ([]Chunk, error) { +// dotProduct computes the dot product of two float32 vectors. +// For normalized vectors (unit length), this equals cosine similarity. +func dotProduct(a, b []float32) float32 { + var sum float32 + for i := range a { + sum += a[i] * b[i] + } + return sum +} + +// Search returns the top-limit chunks most similar to query in the given collection, +// using brute-force dot product similarity over the filtered subset of stored chunks. +func (s *InMemoryStore) Search(ctx context.Context, collectionID string, query []float32, limit int, url string, contentHash string) ([]Chunk, error) { + start := time.Now() + + if limit <= 0 { + return []Chunk{}, nil + } + s.mu.RLock() defer s.mu.RUnlock() @@ -50,7 +67,7 @@ func (s *InMemoryStore) Search(ctx context.Context, collectionID string, _ []flo return []Chunk{}, nil } - var result []Chunk + var scored []Chunk for _, c := range all { if url != "" && c.URL != url { continue @@ -58,14 +75,40 @@ func (s *InMemoryStore) Search(ctx context.Context, collectionID string, _ []flo if contentHash != "" && c.ContentHash != contentHash { continue } - result = append(result, c) + if len(c.Vector) == 0 { + continue + } + c.Score = dotProduct(query, c.Vector) + scored = append(scored, c) } - if limit > 0 && len(result) > limit { - result = result[:limit] + slices.SortFunc(scored, func(a, b Chunk) int { + if a.Score > b.Score { + return -1 + } + if a.Score < b.Score { + return 1 + } + return 0 + }) + + if len(scored) > limit { + scored = scored[:limit] } - return result, nil + if scored == nil { + scored = []Chunk{} + } + + s.logger.InfoContext(ctx, "search completed", + "collection", collectionID, + "url", url, + "content_hash", contentHash, + "results", len(scored), + "latency_ms", time.Since(start).Milliseconds(), + ) + + return scored, nil } // Delete removes all chunks matching the given url and contentHash within the collection. diff --git a/backend/internal/vectorstore/memory_test.go b/backend/internal/vectorstore/memory_test.go index 06069a6..fb1e0d8 100644 --- a/backend/internal/vectorstore/memory_test.go +++ b/backend/internal/vectorstore/memory_test.go @@ -1,8 +1,11 @@ package vectorstore import ( + "bytes" "context" + "fmt" "log/slog" + "math" "sync" "testing" ) @@ -47,18 +50,19 @@ func TestInMemoryStore_Upsert_EmptyChunks(t *testing.T) { func TestInMemoryStore_Upsert_MultipleCollections(t *testing.T) { s := newTestMemoryStore() + query := []float32{1, 0} - err := s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one"}}) + err := s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one", Vector: []float32{1, 0}}}) if err != nil { t.Fatalf("Upsert col1 error = %v", err) } - err = s.Upsert(context.Background(), "col2", []Chunk{{ID: "b", Text: "two"}}) + err = s.Upsert(context.Background(), "col2", []Chunk{{ID: "b", Text: "two", Vector: []float32{0, 1}}}) if err != nil { t.Fatalf("Upsert col2 error = %v", err) } - got1, _ := s.Search(context.Background(), "col1", nil, 10, "", "") - got2, _ := s.Search(context.Background(), "col2", nil, 10, "", "") + got1, _ := s.Search(context.Background(), "col1", query, 10, "", "") + got2, _ := s.Search(context.Background(), "col2", query, 10, "", "") if len(got1) != 1 { t.Errorf("col1 chunks = %d, want 1", len(got1)) @@ -70,11 +74,12 @@ func TestInMemoryStore_Upsert_MultipleCollections(t *testing.T) { func TestInMemoryStore_Upsert_AppendsToExisting(t *testing.T) { s := newTestMemoryStore() + query := []float32{1, 0} - _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one"}}) - _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "b", Text: "two"}}) + _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "a", Text: "one", Vector: []float32{1, 0}}}) + _ = s.Upsert(context.Background(), "col1", []Chunk{{ID: "b", Text: "two", Vector: []float32{0, 1}}}) - got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + got, _ := s.Search(context.Background(), "col1", query, 10, "", "") if len(got) != 2 { t.Errorf("chunks after two upserts = %d, want 2", len(got)) } @@ -82,11 +87,12 @@ func TestInMemoryStore_Upsert_AppendsToExisting(t *testing.T) { func TestInMemoryStore_Delete_RemovesMatchingChunks(t *testing.T) { s := newTestMemoryStore() + query := []float32{1, 0} chunks := []Chunk{ - {ID: "c1", Text: "chunk one", URL: "https://example.com", ContentHash: "hash1"}, - {ID: "c2", Text: "chunk two", URL: "https://example.com", ContentHash: "hash1"}, - {ID: "c3", Text: "chunk three", URL: "https://other.com", ContentHash: "hash2"}, + {ID: "c1", Text: "chunk one", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{1, 0}}, + {ID: "c2", Text: "chunk two", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{0.5, 0}}, + {ID: "c3", Text: "chunk three", URL: "https://other.com", ContentHash: "hash2", Vector: []float32{0.8, 0}}, } _ = s.Upsert(context.Background(), "col1", chunks) @@ -95,7 +101,7 @@ func TestInMemoryStore_Delete_RemovesMatchingChunks(t *testing.T) { t.Fatalf("Delete() error = %v, want nil", err) } - got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + got, _ := s.Search(context.Background(), "col1", query, 10, "", "") if len(got) != 1 { t.Fatalf("chunks after delete = %d, want 1", len(got)) } @@ -114,8 +120,9 @@ func TestInMemoryStore_Delete_NonExistentCollection(t *testing.T) { func TestInMemoryStore_Delete_NoMatchingChunks(t *testing.T) { s := newTestMemoryStore() + query := []float32{1, 0} _ = s.Upsert(context.Background(), "col1", []Chunk{ - {ID: "c1", Text: "chunk", URL: "https://example.com", ContentHash: "hash1"}, + {ID: "c1", Text: "chunk", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{1, 0}}, }) err := s.Delete(context.Background(), "col1", "https://other.com", "other-hash") @@ -123,7 +130,7 @@ func TestInMemoryStore_Delete_NoMatchingChunks(t *testing.T) { t.Fatalf("Delete() error = %v, want nil", err) } - got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + got, _ := s.Search(context.Background(), "col1", query, 10, "", "") if len(got) != 1 { t.Errorf("chunks after no-match delete = %d, want 1", len(got)) } @@ -131,16 +138,17 @@ func TestInMemoryStore_Delete_NoMatchingChunks(t *testing.T) { func TestInMemoryStore_Delete_MatchesURLAndContentHash(t *testing.T) { s := newTestMemoryStore() + query := []float32{1, 0} chunks := []Chunk{ - {ID: "c1", URL: "https://example.com", ContentHash: "hash1"}, - {ID: "c2", URL: "https://example.com", ContentHash: "hash2"}, // same URL, different hash - {ID: "c3", URL: "https://other.com", ContentHash: "hash1"}, // different URL, same hash + {ID: "c1", URL: "https://example.com", ContentHash: "hash1", Vector: []float32{1, 0}}, + {ID: "c2", URL: "https://example.com", ContentHash: "hash2", Vector: []float32{0.5, 0}}, // same URL, different hash + {ID: "c3", URL: "https://other.com", ContentHash: "hash1", Vector: []float32{0.8, 0}}, // different URL, same hash } _ = s.Upsert(context.Background(), "col1", chunks) _ = s.Delete(context.Background(), "col1", "https://example.com", "hash1") - got, _ := s.Search(context.Background(), "col1", nil, 10, "", "") + got, _ := s.Search(context.Background(), "col1", query, 10, "", "") if len(got) != 2 { t.Fatalf("chunks after delete = %d, want 2", len(got)) } @@ -211,3 +219,294 @@ func TestInMemoryStore_NewWithNilLogger(t *testing.T) { t.Errorf("Upsert with nil logger error = %v", err) } } + +// --- Dot Product and Search Tests (Task 2) --- + +func TestDotProduct(t *testing.T) { + // Two known normalized vectors: [1,0,0] · [0,1,0] = 0 + got := dotProduct([]float32{1, 0, 0}, []float32{0, 1, 0}) + if got != 0 { + t.Errorf("dotProduct orthogonal = %v, want 0", got) + } + + // Identical normalized vector: [1,0,0] · [1,0,0] = 1 + got = dotProduct([]float32{1, 0, 0}, []float32{1, 0, 0}) + if got != 1 { + t.Errorf("dotProduct identical = %v, want 1", got) + } + + // Known computation: [0.6, 0.8] · [0.8, 0.6] = 0.48 + 0.48 = 0.96 + got = dotProduct([]float32{0.6, 0.8}, []float32{0.8, 0.6}) + if diff := math.Abs(float64(got) - 0.96); diff > 1e-6 { + t.Errorf("dotProduct([0.6,0.8],[0.8,0.6]) = %v, want ~0.96", got) + } +} + +func TestDotProduct_ZeroVectors(t *testing.T) { + got := dotProduct([]float32{0, 0, 0}, []float32{0, 0, 0}) + if got != 0 { + t.Errorf("dotProduct zero vectors = %v, want 0", got) + } +} + +func TestInMemoryStore_Search_TopN(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + // Create 20 chunks with vectors that have known similarity to query + query := []float32{1, 0, 0} + var chunks []Chunk + for i := 0; i < 20; i++ { + // Vary the first component so scores differ + score := float32(i+1) / 20.0 + chunks = append(chunks, Chunk{ + ID: fmt.Sprintf("c%02d", i), + Text: fmt.Sprintf("chunk %d", i), + URL: "https://example.com", + ContentHash: "hash1", + Vector: []float32{score, 0, 0}, + }) + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", query, 15, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 15 { + t.Fatalf("Search() returned %d chunks, want 15", len(got)) + } + + // Verify sorted descending by score + for i := 1; i < len(got); i++ { + if got[i].Score > got[i-1].Score { + t.Errorf("chunk %d score %v > chunk %d score %v (not descending)", i, got[i].Score, i-1, got[i-1].Score) + } + } + + // The top result should be chunk 19 (highest first component) + if got[0].ID != "c19" { + t.Errorf("top result ID = %q, want %q", got[0].ID, "c19") + } +} + +func TestInMemoryStore_Search_URLFilter(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + chunks := []Chunk{ + {ID: "c1", URL: "https://a.com", ContentHash: "h1", Vector: []float32{1, 0}}, + {ID: "c2", URL: "https://a.com", ContentHash: "h1", Vector: []float32{0.9, 0}}, + {ID: "c3", URL: "https://b.com", ContentHash: "h2", Vector: []float32{0.95, 0}}, + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", []float32{1, 0}, 10, "https://a.com", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("Search() returned %d chunks, want 2 (URL-filtered)", len(got)) + } + for _, c := range got { + if c.URL != "https://a.com" { + t.Errorf("chunk %q URL = %q, want https://a.com", c.ID, c.URL) + } + } +} + +func TestInMemoryStore_Search_ContentHashFilter(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + chunks := []Chunk{ + {ID: "c1", URL: "https://a.com", ContentHash: "h1", Vector: []float32{1, 0}}, + {ID: "c2", URL: "https://a.com", ContentHash: "h2", Vector: []float32{0.9, 0}}, + {ID: "c3", URL: "https://a.com", ContentHash: "h1", Vector: []float32{0.8, 0}}, + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", []float32{1, 0}, 10, "https://a.com", "h1") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("Search() returned %d chunks, want 2 (contentHash-filtered)", len(got)) + } + for _, c := range got { + if c.ContentHash != "h1" { + t.Errorf("chunk %q contentHash = %q, want h1", c.ID, c.ContentHash) + } + } +} + +func TestInMemoryStore_Search_EmptyResults(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + // Empty collection + got, err := s.Search(ctx, "col1", []float32{1, 0}, 10, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if got == nil { + t.Fatal("Search() returned nil, want empty slice") + } + if len(got) != 0 { + t.Errorf("Search() returned %d, want 0", len(got)) + } + + // Filters match nothing + _ = s.Upsert(ctx, "col1", []Chunk{ + {ID: "c1", URL: "https://a.com", ContentHash: "h1", Vector: []float32{1, 0}}, + }) + got, err = s.Search(ctx, "col1", []float32{1, 0}, 10, "https://nonexistent.com", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if got == nil { + t.Fatal("Search() returned nil for no-match filter, want empty slice") + } + if len(got) != 0 { + t.Errorf("Search() returned %d for no-match filter, want 0", len(got)) + } +} + +func TestInMemoryStore_Search_LimitZeroOrNegative(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + _ = s.Upsert(ctx, "col1", []Chunk{ + {ID: "c1", URL: "https://a.com", Vector: []float32{1, 0}}, + }) + + for _, limit := range []int{0, -1, -100} { + got, err := s.Search(ctx, "col1", []float32{1, 0}, limit, "", "") + if err != nil { + t.Fatalf("Search(limit=%d) error = %v", limit, err) + } + if len(got) != 0 { + t.Errorf("Search(limit=%d) returned %d, want 0", limit, len(got)) + } + } +} + +func TestInMemoryStore_Search_ScorePopulation(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + chunks := []Chunk{ + {ID: "c1", URL: "https://a.com", Vector: []float32{1, 0}}, + {ID: "c2", URL: "https://a.com", Vector: []float32{0, 1}}, + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", []float32{1, 0}, 10, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + + for _, c := range got { + if c.ID == "c1" { + if diff := math.Abs(float64(c.Score) - 1.0); diff > 1e-6 { + t.Errorf("c1 Score = %v, want 1.0", c.Score) + } + } + if c.ID == "c2" { + if diff := math.Abs(float64(c.Score) - 0.0); diff > 1e-6 { + t.Errorf("c2 Score = %v, want 0.0", c.Score) + } + } + } +} + +func TestInMemoryStore_Search_SkipsNilVectors(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + chunks := []Chunk{ + {ID: "c1", URL: "https://a.com", Vector: []float32{1, 0}}, + {ID: "c2", URL: "https://a.com", Vector: nil}, + {ID: "c3", URL: "https://a.com", Vector: []float32{}}, + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", []float32{1, 0}, 10, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 1 { + t.Fatalf("Search() returned %d chunks, want 1 (skipping nil/empty vectors)", len(got)) + } + if got[0].ID != "c1" { + t.Errorf("got chunk ID = %q, want c1", got[0].ID) + } +} + +func TestInMemoryStore_Search_LimitExceedsAvailable(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + chunks := []Chunk{ + {ID: "c1", Vector: []float32{1, 0}}, + {ID: "c2", Vector: []float32{0.5, 0}}, + } + _ = s.Upsert(ctx, "col1", chunks) + + got, err := s.Search(ctx, "col1", []float32{1, 0}, 100, "", "") + if err != nil { + t.Fatalf("Search() error = %v", err) + } + if len(got) != 2 { + t.Errorf("Search() returned %d chunks, want 2 (all available)", len(got)) + } +} + +func TestInMemoryStore_Search_ConcurrentReads(t *testing.T) { + s := newTestMemoryStore() + ctx := context.Background() + + // Pre-populate + var chunks []Chunk + for i := 0; i < 100; i++ { + chunks = append(chunks, Chunk{ + ID: fmt.Sprintf("c%d", i), + URL: "https://example.com", + Vector: []float32{float32(i) / 100.0, 0, 0}, + }) + } + _ = s.Upsert(ctx, "col1", chunks) + + var wg sync.WaitGroup + query := []float32{1, 0, 0} + + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = s.Search(ctx, "col1", query, 10, "", "") + }() + } + wg.Wait() + // Passes if no race detected with -race flag +} + +func TestInMemoryStore_Search_Logging(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + s := NewInMemoryStore(logger) + ctx := context.Background() + + _ = s.Upsert(ctx, "col1", []Chunk{ + {ID: "c1", URL: "https://a.com", ContentHash: "h1", Vector: []float32{1, 0}}, + }) + + _, _ = s.Search(ctx, "col1", []float32{1, 0}, 10, "https://a.com", "h1") + + logOutput := buf.String() + for _, want := range []string{"collection", "col1", "url", "https://a.com", "content_hash", "h1", "results", "latency_ms"} { + if !bytes.Contains([]byte(logOutput), []byte(want)) { + t.Errorf("log output missing %q\nGot: %s", want, logOutput) + } + } +} From 582efd3e6e1d50bec30129fb6b7791cadd420c85 Mon Sep 17 00:00:00 2001 From: Parth576 Date: Thu, 5 Mar 2026 08:45:49 -0500 Subject: [PATCH 3/5] docs(config): put Qdrant behind Docker Compose profile, update docs for in-memory default - Add `profiles: [qdrant]` to Qdrant service in docker-compose.yml - Remove `depends_on: qdrant` and `QDRANT_URL` env override from backend service - Update README.md to document in-memory as default, Qdrant as optional via --profile - Update CLAUDE.md tech stack and build commands to reflect new defaults - Document VECTOR_STORE env var throughout Note: .env.example needs manual update to add VECTOR_STORE=memory (sandbox restriction). Assisted by the code-assist SOP --- .../context.md | 23 +++++ .../task-03-docker-compose-and-docs/plan.md | 30 ++++++ .../progress.md | 20 ++++ CLAUDE.md | 19 ++-- README.md | 94 ++++++++++++------- docker-compose.yml | 6 +- 6 files changed, 147 insertions(+), 45 deletions(-) create mode 100644 .agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/context.md create mode 100644 .agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/plan.md create mode 100644 .agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/progress.md diff --git a/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/context.md b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/context.md new file mode 100644 index 0000000..60a9723 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/context.md @@ -0,0 +1,23 @@ +# Context: Task 03 - Docker Compose Qdrant Profile & Documentation Updates + +## Requirements +1. Update `docker-compose.yml`: Add `profiles: [qdrant]` to Qdrant, remove `depends_on` and `QDRANT_URL` env override from backend +2. Update `README.md`: Reflect in-memory default, document Qdrant profile, add `VECTOR_STORE` env var +3. Update `.env.example`: Add `VECTOR_STORE=memory` with comments +4. Update `CLAUDE.md`: Update Build & Run and Tech Stack sections + +## Current State +- Config already supports `VECTOR_STORE` env var with `memory` default (config.go:43) +- main.go already switches between memory/qdrant based on `cfg.VectorStore` (main.go:46-61) +- InMemoryStore fully implemented and tested +- docker-compose.yml currently has hard `depends_on: qdrant` and `QDRANT_URL=qdrant:6334` override + +## Files to Modify +- `docker-compose.yml` (root) +- `README.md` (root) +- `.env.example` (root - sandbox restricted, may need alternative) +- `CLAUDE.md` (root) + +## Key Decisions +- No tests needed - this is config/docs only +- The `QDRANT_URL` env override removal means when running with `--profile qdrant`, user must set `VECTOR_STORE=qdrant` and `QDRANT_URL=qdrant:6334` in their `.env` diff --git a/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/plan.md b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/plan.md new file mode 100644 index 0000000..e14b918 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/plan.md @@ -0,0 +1,30 @@ +# Plan: Task 03 - Docker Compose & Docs + +## Test Strategy +No unit tests needed - this is purely configuration and documentation changes. Validation via `docker compose config`. + +## Implementation Plan + +### 1. docker-compose.yml +- Add `profiles: [qdrant]` to qdrant service +- Remove `depends_on: qdrant` from backend +- Remove `QDRANT_URL=qdrant:6334` environment override from backend + +### 2. README.md +- Update "How It Works" to mention vector store (in-memory default) +- Update "Local Development > Option A" to show `docker compose up` as default (backend only) +- Add Qdrant profile instructions: `docker compose --profile qdrant up` +- Add `VECTOR_STORE` to environment variables section +- Update Option C to reflect optional Qdrant +- Update health check response to not assume Qdrant + +### 3. .env.example +- Add `VECTOR_STORE=memory` with comment +- Add comment that `QDRANT_URL` only needed with `VECTOR_STORE=qdrant` + +### 4. CLAUDE.md +- Update Build & Run section +- Update Tech Stack to mention in-memory default + +### 5. Validation +- Run `docker compose config` to verify syntax diff --git a/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/progress.md b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/progress.md new file mode 100644 index 0000000..9232662 --- /dev/null +++ b/.agents/scratchpad/2026-02-15-smolterms/task-03-docker-compose-and-docs/progress.md @@ -0,0 +1,20 @@ +# Progress: Task 03 - Docker Compose & Docs + +## Setup +- [x] Created documentation directory +- [x] Read all relevant files +- [x] Created context.md + +## Implementation +- [x] Update docker-compose.yml - added `profiles: [qdrant]`, removed `depends_on` and `QDRANT_URL` env override +- [x] Update README.md - updated all sections to reflect in-memory default, Qdrant as optional +- [x] Update .env.example - **BLOCKED**: sandbox prevents read/write to .env.example (in deny list) +- [x] Update CLAUDE.md - updated Tech Stack, Build & Run, Data Flow, package layout, API endpoints +- [ ] Validate docker-compose.yml syntax - docker commands denied by sandbox +- [ ] Commit changes + +## Notes +- `.env.example` could not be updated due to sandbox read/write restrictions +- `docker compose config` could not be run due to sandbox restrictions +- `go test` could not be run due to sandbox Go cache restrictions +- All changes are config/docs only - no functional code changes diff --git a/CLAUDE.md b/CLAUDE.md index 7d076db..614e5c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,18 +16,19 @@ SmolTerms is a privacy policy and terms of service analyzer. A browser extension - **Extension:** Vanilla JS (no build system, no framework), Manifest V3, Firefox + Chrome - **LLM:** Anthropic API direct (Claude Sonnet 4.5), behind `LLMClient` interface for provider swapping - **Embeddings:** OpenAI `text-embedding-3-small` (1536 dimensions), behind `EmbeddingClient` interface -- **Vector DB:** Qdrant (official Go gRPC client), behind `VectorStore` interface +- **Vector Store:** In-memory (default), Qdrant gRPC (optional, via `VECTOR_STORE=qdrant`), behind `VectorStore` interface - **Caching:** go-cache in-memory (MVP), behind `Cache` interface for Redis swap later - **Configuration:** Environment variables only (12-factor app, `.env` file for local dev) -- **Infrastructure:** Docker Compose (Go backend + Qdrant) for local development +- **Infrastructure:** Docker Compose (Go backend; Qdrant optional via `--profile qdrant`) ## Build & Run Commands ```bash -docker-compose up # Start full dev stack (backend + Qdrant) -go run ./backend/cmd/server/main.go # Run backend directly -go test ./backend/... # Run all backend tests -go test ./backend/... -cover # Run tests with coverage +docker compose up # Start backend (in-memory vector store) +docker compose --profile qdrant up # Start backend + Qdrant +go run ./backend/cmd/server/main.go # Run backend directly +go test ./backend/... # Run all backend tests +go test ./backend/... -cover # Run tests with coverage ``` Extension: load unpacked in browser (no build step needed). @@ -40,7 +41,7 @@ Extension: load unpacked in browser (no build step needed). Extension click → Content script extracts main content HTML → Background worker POSTs to API → Backend: check cache (URL + content hash) → if miss: → HTML parse (goquery) → privacy policy detection → structure-aware text chunking (512 tokens target, 64 token overlap within sections) - → Embed chunks (OpenAI) → store in Qdrant + → Embed chunks (OpenAI) → store in vector store (in-memory default) → Single broad retrieval query → top 15-20 chunks → LLM analysis (Anthropic Claude) → structured scoring → cache result → return response ``` @@ -52,7 +53,7 @@ Extension click → Content script extracts main content HTML → Background wor | `api/` | HTTP handlers, CORS middleware, route definitions, JSON response helpers | | `extractor/` | HTML parsing (goquery), text chunking, privacy policy detection | | `embedding/` | EmbeddingClient interface, OpenAI implementation | -| `vectorstore/` | VectorStore interface, Qdrant gRPC implementation | +| `vectorstore/` | VectorStore interface, in-memory + Qdrant gRPC implementations | | `rag/` | RAG pipeline orchestration (store + retrieve) | | `llm/` | LLMClient interface, Anthropic implementation, prompt templates | | `analyzer/` | Full pipeline orchestration, scoring/aggregation, result types | @@ -74,7 +75,7 @@ Overall score = simple average. Risk levels: Low (8-10), Moderate (5-7.9), High ``` POST /api/v1/analyze # Submit HTML, get analysis back (synchronous) -GET /api/v1/health # Health check (backend + Qdrant status) +GET /api/v1/health # Health check (backend + vector store status) ``` ### Extension Structure (`extension/`) diff --git a/README.md b/README.md index d47407e..d7c538b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Browser Extension click -> Backend: check cache (URL + content hash) -> If cache miss: HTML parse -> privacy policy detection -> text chunking - -> Embed chunks (OpenAI) -> store in Qdrant + -> Embed chunks (OpenAI) -> store in vector store (in-memory default, Qdrant optional) -> Retrieve relevant chunks -> LLM analysis (Anthropic Claude) -> Structured scoring -> cache result -> return response ``` @@ -35,7 +35,7 @@ Five dimensions, equally weighted (20% each), rated 1-10 (higher = better for us | Method | Path | Description | |---|---|---| | `POST` | `/api/v1/analyze` | Submit HTML content for privacy analysis | -| `GET` | `/api/v1/health` | Health check (backend + Qdrant status) | +| `GET` | `/api/v1/health` | Health check (backend + vector store status) | --- @@ -153,8 +153,13 @@ ANTHROPIC_API_KEY=sk-ant-your-key-here # OpenAI API key (required) - get from https://platform.openai.com/api-keys OPENAI_API_KEY=sk-your-key-here -# Qdrant gRPC address (default: localhost:6334) -# When running with Docker/Podman Compose, this is overridden to qdrant:6334 +# Vector store backend: "memory" (default) or "qdrant" +# In-memory is the default and requires no additional setup. +# Set to "qdrant" to use Qdrant for persistent vector storage. +VECTOR_STORE=memory + +# Qdrant gRPC address (only needed when VECTOR_STORE=qdrant) +# When running with Docker/Podman Compose, set to qdrant:6334 QDRANT_URL=localhost:6334 # Cache TTL for analysis results (default: 720h = 30 days) @@ -171,7 +176,7 @@ CACHE_DEFAULT_TTL=720h Requires [Docker Engine](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) (v2+). -**Start the full stack (backend + Qdrant):** +**Start the backend (uses in-memory vector store by default):** ```bash docker compose up --build @@ -179,15 +184,23 @@ docker compose up --build This will: 1. Build the Go backend from `backend/Dockerfile` (multi-stage, distroless image) -2. Pull and start the `qdrant/qdrant` container -3. Wait for Qdrant's health check before starting the backend -4. Expose the backend on `http://localhost:8080` -5. Expose Qdrant's HTTP API on `http://localhost:6333` and gRPC on `localhost:6334` +2. Expose the backend on `http://localhost:8080` +3. Use the in-memory vector store (no external dependencies needed) + +**Start with Qdrant (for persistent vector storage):** + +```bash +docker compose --profile qdrant up --build +``` + +This additionally starts the Qdrant container. You must also set `VECTOR_STORE=qdrant` and `QDRANT_URL=qdrant:6334` in your `.env` file. **Run in the background:** ```bash docker compose up --build -d +# Or with Qdrant: +docker compose --profile qdrant up --build -d ``` **View logs:** @@ -195,13 +208,13 @@ docker compose up --build -d ```bash docker compose logs -f # all services docker compose logs -f backend # backend only -docker compose logs -f qdrant # qdrant only +docker compose logs -f qdrant # qdrant only (requires --profile qdrant) ``` **Stop everything:** ```bash -docker compose down # stop containers (data persists in volume) +docker compose down # stop containers docker compose down -v # stop containers AND remove Qdrant data volume ``` @@ -209,16 +222,26 @@ docker compose down -v # stop containers AND remove Qdrant data volume Requires [Podman](https://podman.io/docs/installation) and [Podman Compose](https://github.com/containers/podman-compose). -**Start the full stack:** +**Start the backend (uses in-memory vector store by default):** ```bash podman compose up --build ``` +**Start with Qdrant:** + +```bash +podman compose --profile qdrant up --build +``` + +Set `VECTOR_STORE=qdrant` and `QDRANT_URL=qdrant:6334` in your `.env` file when using Qdrant. + **Run in the background:** ```bash podman compose up --build -d +# Or with Qdrant: +podman compose --profile qdrant up --build -d ``` **View logs:** @@ -226,7 +249,7 @@ podman compose up --build -d ```bash podman compose logs -f podman compose logs -f backend -podman compose logs -f qdrant +podman compose logs -f qdrant # requires --profile qdrant ``` **Stop everything:** @@ -240,9 +263,17 @@ podman compose down -v # also remove Qdrant data volume ### Option C: Run Backend Directly (without containers) -If you prefer to run Go directly, you'll need Qdrant running separately. +Run the Go backend directly. By default it uses the in-memory vector store, so no external services are needed. + +**Run the backend:** -**1. Start Qdrant (pick one):** +```bash +go run ./backend/cmd/server/main.go +``` + +**(Optional) Use Qdrant for persistent vector storage:** + +Set `VECTOR_STORE=qdrant` and `QDRANT_URL=localhost:6334` in your `.env`, then start Qdrant: ```bash # Docker: @@ -254,13 +285,6 @@ podman run -d --name qdrant -p 6333:6333 -p 6334:6334 \ -v qdrant_data:/qdrant/storage qdrant/qdrant ``` -**2. Run the backend:** - -```bash -# Make sure .env is configured with QDRANT_URL=localhost:6334 -go run ./backend/cmd/server/main.go -``` - ### Verifying the Setup Once everything is running, check the health endpoint: @@ -269,13 +293,13 @@ Once everything is running, check the health endpoint: curl http://localhost:8080/api/v1/health ``` -Expected response: +Expected response (with in-memory vector store): ```json { "status": "healthy", "services": { - "qdrant": { "status": "healthy" }, + "vectorstore": { "status": "healthy" }, "anthropic": { "status": "configured" }, "openai": { "status": "configured" } } @@ -314,11 +338,11 @@ go test ./backend/... -cover ### Integration Tests Integration tests exercise the full pipeline against real services. They require: -- Running Qdrant instance +- Running Qdrant instance (or in-memory store depending on test) - Valid `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` in your environment ```bash -# Start Qdrant first (via Compose or standalone container) +# Start Qdrant if needed (via Compose or standalone container) # Run integration tests with the build tag: go test -tags=integration ./backend/internal/integration/... -v -timeout 120s @@ -367,7 +391,11 @@ cp .env.example .env **3. Start the services:** ```bash +# Backend only (in-memory vector store): docker compose up --build -d + +# With Qdrant (set VECTOR_STORE=qdrant and QDRANT_URL=qdrant:6334 in .env): +docker compose --profile qdrant up --build -d ``` **4. (Optional) Set up a reverse proxy:** @@ -418,13 +446,15 @@ ssh user@server # Set environment variables export ANTHROPIC_API_KEY="sk-ant-..." export OPENAI_API_KEY="sk-..." -export QDRANT_URL="localhost:6334" +# VECTOR_STORE defaults to "memory" — no Qdrant needed # Run /opt/smolterms/smolterms-server ``` -**3. Run Qdrant on the server:** +**3. (Optional) Run Qdrant for persistent vector storage:** + +Set `VECTOR_STORE=qdrant` and `QDRANT_URL=localhost:6334`, then: ```bash docker run -d --name qdrant \ @@ -465,10 +495,10 @@ sudo systemctl enable --now smolterms - [ ] Set `LOG_LEVEL=warn` or `LOG_LEVEL=error` for production - [ ] Set spending limits on both Anthropic and OpenAI dashboards - [ ] Put a reverse proxy (Nginx/Caddy) in front for TLS termination -- [ ] Secure Qdrant -- by default it has no authentication; consider binding to localhost only or adding an API key via [Qdrant security config](https://qdrant.tech/documentation/guides/security/) +- [ ] If using Qdrant (`VECTOR_STORE=qdrant`): secure it -- by default it has no authentication; consider binding to localhost only or adding an API key via [Qdrant security config](https://qdrant.tech/documentation/guides/security/) - [ ] Set up monitoring for the `/api/v1/health` endpoint - [ ] Configure firewall rules -- only expose ports 80/443 publicly, keep 8080/6333/6334 internal -- [ ] Back up the Qdrant volume periodically (or use [Qdrant snapshots](https://qdrant.tech/documentation/concepts/snapshots/)) +- [ ] If using Qdrant: back up the Qdrant volume periodically (or use [Qdrant snapshots](https://qdrant.tech/documentation/concepts/snapshots/)) - [ ] Consider rate limiting at the reverse proxy level to prevent abuse --- @@ -493,7 +523,7 @@ smolterms/ │ ├── types/ # Shared request/response types │ └── vectorstore/ # VectorStore interface + Qdrant impl ├── extension/ # Browser extension (Firefox + Chrome) -├── docker-compose.yml # Local dev: backend + Qdrant +├── docker-compose.yml # Local dev: backend (Qdrant via --profile qdrant) ├── .env.example # Environment variable template ├── go.mod └── go.sum @@ -506,7 +536,7 @@ smolterms/ | Backend | Go 1.22+, stdlib `net/http` | | LLM | Anthropic Claude Sonnet 4.5 | | Embeddings | OpenAI `text-embedding-3-small` (1536 dims) | -| Vector DB | Qdrant (gRPC) | +| Vector Store | In-memory (default), Qdrant gRPC (optional) | | Caching | go-cache (in-memory) | | Configuration | Environment variables (12-factor) | | Extension | Vanilla JS, Manifest V3 | diff --git a/docker-compose.yml b/docker-compose.yml index 71f2969..505675e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,16 +10,14 @@ services: - "8080:8080" env_file: - .env - environment: - - QDRANT_URL=qdrant:6334 - depends_on: - - qdrant qdrant: image: docker.io/qdrant/qdrant:latest security_opt: - seccomp=unconfined - label=disable + profiles: + - qdrant ports: - "6333:6333" - "6334:6334" From 156c72d8c5e5dd146aa78f1bd8a14e89979c2f26 Mon Sep 17 00:00:00 2001 From: Parth576 Date: Thu, 5 Mar 2026 08:50:11 -0500 Subject: [PATCH 4/5] update example env --- .env.example | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 01eaddf..d758a36 100644 --- a/.env.example +++ b/.env.example @@ -13,8 +13,13 @@ ANTHROPIC_API_KEY=sk-ant-your-key-here # OpenAI API key (required, used for embeddings) OPENAI_API_KEY=sk-your-key-here -# Qdrant gRPC address (default: localhost:6334) -QDRANT_URL=localhost:6334 +# Vector store backend: "memory" (default) or "qdrant" +# In-memory requires no external services. Use "qdrant" for persistent storage. +VECTOR_STORE=memory + +# Qdrant gRPC address (only needed when VECTOR_STORE=qdrant) +# Set to qdrant:6334 when using Docker/Podman Compose with --profile qdrant +#QDRANT_URL=localhost:6334 # Cache TTL for analysis results (default: 720h = 30 days) CACHE_DEFAULT_TTL=720h From d792d949a0e6a16f1d6c40549614db4ec165e68c Mon Sep 17 00:00:00 2001 From: Parth576 Date: Thu, 5 Mar 2026 08:51:56 -0500 Subject: [PATCH 5/5] update example env --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index d758a36..ae85ba8 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,7 @@ VECTOR_STORE=memory # Qdrant gRPC address (only needed when VECTOR_STORE=qdrant) # Set to qdrant:6334 when using Docker/Podman Compose with --profile qdrant -#QDRANT_URL=localhost:6334 +QDRANT_URL=localhost:6334 # Cache TTL for analysis results (default: 720h = 30 days) CACHE_DEFAULT_TTL=720h