diff --git a/.gitignore b/.gitignore index 1e1bedd..f265cb0 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ bench/results/ # Vault (local content) .napkin/ thread-draft.md +*.bun-build diff --git a/CLAUDE.md b/CLAUDE.md index e0ef41d..970ebdd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,8 @@ bun run check # Biome lint + format - `src/utils/frontmatter.ts` — YAML frontmatter parse/set/remove - `src/utils/config.ts` — Unified config (load/save/update, syncs to .obsidian/) - `src/utils/markdown.ts` — Extract headings, tasks, tags, links from markdown +- `src/utils/search-cache.ts` — Search index cache + whole-vault mtime fingerprint +- `src/utils/overview-cache.ts` — Overview result cache (fingerprint + options key) - `.pi/extensions/napkin-context/` — Pi extension: injects vault overview into system prompt - `.pi/extensions/distill/` — Pi extension: auto-distills conversations into vault @@ -31,8 +33,10 @@ bun run check # Biome lint + format ``` project/ - .napkin/ # napkin config + .napkin/ # napkin config + caches config.json # Unified config (syncs to .obsidian/) + search-cache.json # Auto-managed, safe to delete + overview-cache.json # Auto-managed, safe to delete .obsidian/ # Obsidian config (auto-generated) NAPKIN.md # Level 0 context note decisions/ # Template-defined dirs @@ -45,7 +49,9 @@ project/ - **Output triple**: Every command supports `--json`, `--quiet`, and human-readable output - **Vault auto-detect**: Walks up from cwd looking for `.napkin/` directory -- **`.napkin/` is config only**: Vault content lives in the project root, `.napkin/` holds `config.json` +- **`.napkin/` holds config + caches**: Vault content lives in the project root; `.napkin/` holds `config.json` and the auto-managed search/overview caches +- **Fingerprint caching**: search and overview cache their results keyed by vault file mtimes — any file change invalidates; caches are safe to delete +- **Search engine**: `@shift-labs/ferrosearch` — native, MiniSearch-compatible (index format included); `minisearch` remains a devDependency as the cache-migration test oracle - **File resolution**: positional `` or `--file` resolves by name (like wikilinks), `--path` requires exact path from vault root - **No Obsidian dependency**: Pure file-system operations on markdown files - **Progressive disclosure**: overview → search → read (4 levels, L0-L3) diff --git a/README.md b/README.md index de1f4cc..e823db5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ Every great idea started on a napkin. npm install -g napkin-ai ``` +Search runs on [ferrosearch](https://github.com/shift-labs-ai/ferrosearch), +a native MiniSearch-compatible engine with prebuilt binaries for macOS +(arm64, x64) and Linux (x64, arm64, glibc and musl). Other platforms build +it from source with Rust installed. + --- ## Quick Start @@ -113,8 +118,9 @@ See [`bench/README.md`](bench/README.md) for details and usage. ``` my-project/ - .napkin/ # napkin config + .napkin/ # napkin config + caches config.json # Unified config (syncs to .obsidian/) + *-cache.json # Search/overview caches (auto-managed) .obsidian/ # Obsidian config (auto-generated) NAPKIN.md # Context note (Level 0) decisions/ # Template-defined directories diff --git a/docs/configuration.md b/docs/configuration.md index 73dbef8..82b12f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -18,6 +18,7 @@ napkin config set --key search.limit --value 50 |-----|---------|-------------| | `overview.depth` | `3` | Max folder depth in vault map | | `overview.keywords` | `8` | Max TF-IDF keywords per folder | +| `overview.collapse` | `true` | Roll up numerous, lexically similar sibling folders into one row | ### search @@ -64,8 +65,13 @@ CLI flags > `config.json` > hardcoded defaults ``` project/ .napkin/ - config.json # This file - .obsidian/ # Auto-synced from config.json + config.json # This file + search-cache.json # Search index cache (auto-managed) + overview-cache.json # Overview result cache (auto-managed) + .obsidian/ # Auto-synced from config.json ``` +The cache files are keyed by a fingerprint of vault file mtimes and rebuild +automatically; they are safe to delete at any time. + Config is created on first `napkin config set` or `napkin init`. If the file doesn't exist, defaults are used. diff --git a/docs/overview-keyword-extraction.md b/docs/overview-keyword-extraction.md index b84ff71..99f065d 100644 --- a/docs/overview-keyword-extraction.md +++ b/docs/overview-keyword-extraction.md @@ -5,9 +5,12 @@ The `napkin overview` command generates a vault-wide index by extracting distinc ## Pipeline ``` -Files → Group by folder → Collect weighted text → Build TF → Compute IDF across folders → Score TF-IDF → Deduplicate bigrams → Top N keywords +Files → Group by folder → Collect weighted text → Build TF → Collapse homogeneous siblings → Compute IDF across folders → Score TF-IDF → Deduplicate bigrams → Top N keywords ``` +The whole pipeline runs behind an mtime-fingerprint cache (see +[Caching](#caching)); a cache hit skips everything below. + ### 1. Text Collection & Weighting Not all text is equal. Sources are weighted by signal strength: @@ -17,15 +20,21 @@ Not all text is equal. Sources are weighted by signal strength: | Headings | 3x | Curated by the author, high intent | | Filenames | 2x | Chosen names are strong signals | | Frontmatter title | 2x | Explicit metadata | +| Other frontmatter values | 2x | Explicit metadata (wikilink-only and date values excluded) | | Body text | 1x | Bulk content, noisier | ### 2. Noise Stripping Before tokenization, we strip: -- URLs (`https://...`) -- Emails +- URLs (`https://...`) and emails - Code blocks (fenced and inline) -- Hex hashes (commit SHAs, etc.) +- HTML tags and entities (residue of converted documents) +- Hex hashes (commit SHAs), dashed GUIDs (`AAAA1111-2222-...`), and + mixed-digit blobs (`INV20240915X`) — ID shrapnel from OCR'd PDFs and + DocuSign-style exports that would otherwise pollute keywords + +Each pattern only runs when a cheap necessary condition holds (an email +needs `@`, a URL needs `http`), so clean prose skips the regex scans. ### 3. Tokenization @@ -38,7 +47,16 @@ Two-word phrases are extracted alongside unigrams. Bigrams are kept only if: - They appear **2+ times** in the folder (otherwise likely noise) - The two words are **not identical** (filters "tbd tbd" type garbage) -### 5. TF-IDF Scoring +### 5. Homogeneous-Sibling Collapse + +Parents with 5+ children whose body-term distributions are lexically +similar (mean pairwise cosine ≥ 0.15 over top terms) are rendered as one +aggregate row — `imports/ (+6 similar subfolders)` — so imported document +dumps don't drown the overview. Similarity uses body text only, so shared +filename conventions cannot fake content homogeneity. Top-level folders +never collapse into the root. Disable with `--no-collapse`. + +### 6. TF-IDF Scoring Each folder is treated as a "document": @@ -47,16 +65,31 @@ Each folder is treated as a "document": The `1 +` dampening prevents over-penalizing terms that appear in a few folders. A word in 3 out of 9 folders still gets reasonable weight, while a word in all 9 gets suppressed. -### 6. Bigram Deduplication +Candidates are also filtered for corroboration — a term must appear outside +headings or in 2+ heading lines (single heading-only terms are usually +section labels like "Notes") — and terms matching the folder's own name +(including singular/plural variants) are excluded as redundant. + +### 7. Bigram Deduplication When a bigram is selected (e.g., "knowledge base"), its constituent unigrams ("knowledge", "base") are suppressed from the results. This prevents redundant keyword slots. +## Caching + +The final result is cached in `.napkin/overview-cache.json`, keyed by a +whole-vault fingerprint (file paths + mtimes) plus the resolved options. +Any file add, remove, or touch — including `NAPKIN.md` — invalidates it; +so does changing `depth`, `keywords`, or `collapse`. A cache hit costs one +stat pass instead of reading and tokenizing every note (~25ms vs ~1s on a +5,000-note vault). Corrupt cache files are ignored and rebuilt. + ## Configuration | Flag | Default | Description | |------|---------|-------------| | `--keywords ` | 8 | Max keywords per folder | | `--depth ` | 3 | Max folder depth to index | +| `--no-collapse` | collapse on | Disable homogeneous-sibling collapse | ## Example diff --git a/package.json b/package.json index ec36a06..10c9192 100644 --- a/package.json +++ b/package.json @@ -52,14 +52,15 @@ "@types/bun": "latest", "@types/js-yaml": "^4.0.9", "@types/node": "^25.6.0", + "minisearch": "^7.2.0", "typescript": "^5.8.0" }, "dependencies": { + "@shift-labs/ferrosearch": "^0.1.1", "chalk": "^5.6.2", "commander": "^14.0.3", "gray-matter": "^4.0.3", "jexl": "^2.3.0", - "minisearch": "^7.2.0", "sql.js": "^1.14.0" }, "optionalDependencies": { diff --git a/src/commands/overview.test.ts b/src/commands/overview.test.ts index ccc04d1..2cfe2c3 100644 --- a/src/commands/overview.test.ts +++ b/src/commands/overview.test.ts @@ -110,6 +110,57 @@ describe("overview", () => { vault.cleanup(); }); + test("warns for every file with identical malformed frontmatter", async () => { + // Regression: gray-matter's parse cache is poisoned by a failed parse, + // so a second file with byte-identical malformed frontmatter used to + // silently parse as empty data — no warning, wrong tags/keywords. + const badContent = + "---\ntags: [#twin, #copies]\n---\n# Twin\nIdentical broken note."; + const vault = createTempVault({ + "a/bad.md": badContent, + "b/bad.md": badContent, + }); + + const warnings: string[] = []; + const captured: unknown[] = []; + const origLog = console.log; + const origError = console.error; + + try { + console.error = (...args: unknown[]) => { + const msg = args.map(String).join(" "); + if (msg.includes("⚠")) warnings.push(msg); + }; + console.log = (...args: unknown[]) => { + captured.push(...args); + }; + await overview({ + vault: vault.path, + json: true, + quiet: false, + copy: false, + }); + } finally { + console.log = origLog; + console.error = origError; + } + + expect(warnings.length).toBe(2); + expect(warnings.join("\n")).toContain("a/bad.md"); + expect(warnings.join("\n")).toContain("b/bad.md"); + + // and neither file leaks tags from the unparsed frontmatter + const result = JSON.parse(captured[0] as string) as { + overview: Array<{ path: string; tags: string[] }>; + }; + for (const folder of result.overview) { + expect(folder.tags).not.toContain("twin"); + expect(folder.tags).not.toContain("copies"); + } + + vault.cleanup(); + }); + test("empty vault", async () => { const vault = createTempVault({}); diff --git a/src/core/__snapshots__/overview.golden.test.ts.snap b/src/core/__snapshots__/overview.golden.test.ts.snap new file mode 100644 index 0000000..e52160a --- /dev/null +++ b/src/core/__snapshots__/overview.golden.test.ts.snap @@ -0,0 +1,678 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`getOverview golden default options 1`] = ` +{ + "context": +"# Fixture project +Context note for the golden vault." +, + "overview": [ + { + "keywords": [ + "welcome", + "orientation", + "newcomers", + "fixture", + "vault", + ], + "notes": 1, + "path": "/", + "tags": [], + }, + { + "keywords": [ + "kubernetes", + "ingress", + "pod", + "autoscaling", + "policies", + "note", + "routing", + ], + "notes": 1, + "path": "areas/alpha", + "tags": [], + }, + { + "keywords": [ + "payroll", + "tax", + "withholding", + "tables", + "hourly", + "contractors", + "note", + ], + "notes": 1, + "path": "areas/beta", + "tags": [], + }, + { + "keywords": [ + "collimation", + "steps", + "reflector", + "note", + "telescope", + "optics", + ], + "notes": 1, + "path": "areas/delta", + "tags": [], + }, + { + "keywords": [ + "beehive", + "winterization", + "varroa", + "mite", + "treatment", + "note", + ], + "notes": 1, + "path": "areas/epsilon", + "tags": [], + }, + { + "keywords": [ + "sourdough", + "fermentation", + "hydration", + "ratios", + "note", + "schedules", + ], + "notes": 1, + "path": "areas/gamma", + "tags": [], + }, + { + "keywords": [ + "marathon", + "training", + "splits", + "lactate", + "threshold", + "pacing", + "note", + ], + "notes": 1, + "path": "areas/zeta", + "tags": [], + }, + { + "keywords": [ + "parking", + "lease", + "guarantee", + "agreement", + "lease agreement", + "envelope", + "bank", + "bank guarantee", + ], + "notes": 2, + "path": "contracts", + "tags": [], + }, + { + "keywords": [ + "outbox", + "transactional", + "braintree", + "transactional outbox", + "postgresql", + "adopt", + "adopt transactional", + "merchants", + ], + "notes": 3, + "path": "decisions", + "tags": [ + "adr", + "database", + "messaging", + ], + }, + { + "keywords": [ + "present", + "telescope", + "within", + "mentions", + "telescopes", + "twice", + "mounts", + "depth", + ], + "notes": 1, + "path": "deep/one", + "tags": [], + }, + { + "keywords": [ + "buried", + "folder", + "beyond", + "limit", + "depth", + ], + "notes": 1, + "path": "deep/one/two", + "tags": [], + }, + { + "collapsedFolders": 6, + "keywords": [ + "converted", + "document", + "converted document", + "contract", + "lease", + "suite", + "suite floor", + "schedule", + ], + "notes": 6, + "path": "imports", + "tags": [], + }, + { + "keywords": [ + "asha", + "lukas", + "engineering", + "boston", + "owns", + "staff", + "engineer", + "staff engineer", + ], + "notes": 3, + "path": "people", + "tags": [ + "leadership", + ], + }, + ], + "warnings": [ + "Skipping people/broken.md (malformed YAML frontmatter)", + ], +} +`; + +exports[`getOverview golden depth 3, keywords 8 1`] = ` +{ + "context": +"# Fixture project +Context note for the golden vault." +, + "overview": [ + { + "keywords": [ + "welcome", + "orientation", + "newcomers", + "fixture", + "vault", + ], + "notes": 1, + "path": "/", + "tags": [], + }, + { + "keywords": [ + "kubernetes", + "ingress", + "pod", + "autoscaling", + "policies", + "note", + "routing", + ], + "notes": 1, + "path": "areas/alpha", + "tags": [], + }, + { + "keywords": [ + "payroll", + "tax", + "withholding", + "tables", + "hourly", + "contractors", + "note", + ], + "notes": 1, + "path": "areas/beta", + "tags": [], + }, + { + "keywords": [ + "collimation", + "steps", + "reflector", + "note", + "telescope", + "optics", + ], + "notes": 1, + "path": "areas/delta", + "tags": [], + }, + { + "keywords": [ + "beehive", + "winterization", + "varroa", + "mite", + "treatment", + "note", + ], + "notes": 1, + "path": "areas/epsilon", + "tags": [], + }, + { + "keywords": [ + "sourdough", + "fermentation", + "hydration", + "ratios", + "note", + "schedules", + ], + "notes": 1, + "path": "areas/gamma", + "tags": [], + }, + { + "keywords": [ + "marathon", + "training", + "splits", + "lactate", + "threshold", + "pacing", + "note", + ], + "notes": 1, + "path": "areas/zeta", + "tags": [], + }, + { + "keywords": [ + "parking", + "lease", + "guarantee", + "agreement", + "lease agreement", + "envelope", + "bank", + "bank guarantee", + ], + "notes": 2, + "path": "contracts", + "tags": [], + }, + { + "keywords": [ + "outbox", + "transactional", + "braintree", + "transactional outbox", + "postgresql", + "adopt", + "adopt transactional", + "merchants", + ], + "notes": 3, + "path": "decisions", + "tags": [ + "adr", + "database", + "messaging", + ], + }, + { + "keywords": [ + "present", + "telescope", + "within", + "mentions", + "telescopes", + "twice", + "mounts", + "depth", + ], + "notes": 1, + "path": "deep/one", + "tags": [], + }, + { + "keywords": [ + "buried", + "folder", + "beyond", + "limit", + "depth", + ], + "notes": 1, + "path": "deep/one/two", + "tags": [], + }, + { + "collapsedFolders": 6, + "keywords": [ + "converted", + "document", + "converted document", + "contract", + "lease", + "suite", + "suite floor", + "schedule", + ], + "notes": 6, + "path": "imports", + "tags": [], + }, + { + "keywords": [ + "asha", + "lukas", + "engineering", + "boston", + "owns", + "staff", + "engineer", + "staff engineer", + ], + "notes": 3, + "path": "people", + "tags": [ + "leadership", + ], + }, + ], + "warnings": [ + "Skipping people/broken.md (malformed YAML frontmatter)", + ], +} +`; + +exports[`getOverview golden collapse disabled 1`] = ` +{ + "context": +"# Fixture project +Context note for the golden vault." +, + "overview": [ + { + "keywords": [ + "welcome", + "orientation", + "newcomers", + "fixture", + "vault", + ], + "notes": 1, + "path": "/", + "tags": [], + }, + { + "keywords": [ + "kubernetes", + "ingress", + "pod", + "autoscaling", + "policies", + "note", + "routing", + ], + "notes": 1, + "path": "areas/alpha", + "tags": [], + }, + { + "keywords": [ + "payroll", + "tax", + "withholding", + "tables", + "hourly", + "contractors", + "note", + ], + "notes": 1, + "path": "areas/beta", + "tags": [], + }, + { + "keywords": [ + "collimation", + "steps", + "reflector", + "note", + "telescope", + "optics", + ], + "notes": 1, + "path": "areas/delta", + "tags": [], + }, + { + "keywords": [ + "beehive", + "winterization", + "varroa", + "mite", + "treatment", + "note", + ], + "notes": 1, + "path": "areas/epsilon", + "tags": [], + }, + { + "keywords": [ + "sourdough", + "fermentation", + "hydration", + "ratios", + "note", + "schedules", + ], + "notes": 1, + "path": "areas/gamma", + "tags": [], + }, + { + "keywords": [ + "marathon", + "training", + "splits", + "lactate", + "threshold", + "pacing", + "note", + ], + "notes": 1, + "path": "areas/zeta", + "tags": [], + }, + { + "keywords": [ + "parking", + "lease", + "guarantee", + "agreement", + "lease agreement", + "envelope", + "bank", + "bank guarantee", + ], + "notes": 2, + "path": "contracts", + "tags": [], + }, + { + "keywords": [ + "outbox", + "transactional", + "braintree", + "transactional outbox", + "postgresql", + "adopt", + "adopt transactional", + "merchants", + ], + "notes": 3, + "path": "decisions", + "tags": [ + "adr", + "database", + "messaging", + ], + }, + { + "keywords": [ + "present", + "telescope", + "within", + "mentions", + "telescopes", + "twice", + "mounts", + "depth", + ], + "notes": 1, + "path": "deep/one", + "tags": [], + }, + { + "keywords": [ + "buried", + "folder", + "beyond", + "limit", + "depth", + ], + "notes": 1, + "path": "deep/one/two", + "tags": [], + }, + { + "keywords": [ + "contract", + "schedule", + "term", + "apply", + "stated", + "appendix", + "insurance", + "certificate", + ], + "notes": 1, + "path": "imports/tenant-0", + "tags": [], + }, + { + "keywords": [ + "contract", + "signature", + "page", + "attached", + "insurance", + "certificate", + "required", + "occupancy", + ], + "notes": 1, + "path": "imports/tenant-1", + "tags": [], + }, + { + "keywords": [ + "contract", + "lease", + "signature", + "page", + "attached", + "schedule", + "term", + "apply", + ], + "notes": 1, + "path": "imports/tenant-2", + "tags": [], + }, + { + "keywords": [ + "contract", + "schedule", + "term", + "apply", + "stated", + "appendix", + "insurance", + "certificate", + ], + "notes": 1, + "path": "imports/tenant-3", + "tags": [], + }, + { + "keywords": [ + "contract", + "signature", + "page", + "attached", + "insurance", + "certificate", + "required", + "occupancy", + ], + "notes": 1, + "path": "imports/tenant-4", + "tags": [], + }, + { + "keywords": [ + "contract", + "lease", + "signature", + "page", + "attached", + "schedule", + "term", + "apply", + ], + "notes": 1, + "path": "imports/tenant-5", + "tags": [], + }, + { + "keywords": [ + "asha", + "lukas", + "engineering", + "boston", + "owns", + "staff", + "engineer", + "staff engineer", + ], + "notes": 3, + "path": "people", + "tags": [ + "leadership", + ], + }, + ], + "warnings": [ + "Skipping people/broken.md (malformed YAML frontmatter)", + ], +} +`; diff --git a/src/core/__tests__/overview-original.ts b/src/core/__tests__/overview-original.ts new file mode 100644 index 0000000..054912c --- /dev/null +++ b/src/core/__tests__/overview-original.ts @@ -0,0 +1,699 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { loadConfig } from "../../utils/config.js"; +import { listFiles } from "../../utils/files.js"; +import { parseFrontmatter } from "../../utils/frontmatter.js"; +import { extractHeadings, extractTags } from "../../utils/markdown.js"; + +// ============================================================================ +// VERBATIM COPY of src/core/overview.ts BEFORE the performance refactor +// (git 36e88ab), used exclusively as the reference oracle in +// overview.equivalence.test.ts. Only the import paths above and the export +// block at the bottom of this file were changed. DO NOT "optimize" or edit +// this file — its entire value is that it is the original algorithm. +// ============================================================================ + +export interface OverviewFolder { + path: string; + notes: number; + keywords: string[]; + tags: string[]; + /** Number of subfolders rolled up into this row (homogeneous-sibling collapse). */ + collapsedFolders?: number; +} + +export interface VaultOverview { + context?: string; + overview: OverviewFolder[]; + warnings?: string[]; +} + +const CODE_BLOCK_RE = /```[\s\S]*?```/g; +const INLINE_CODE_RE = /`[^`]+`/g; +const URL_RE = /https?:\/\/[^\s)>\]]+/g; +const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; +const HEX_HASH_RE = /\b[a-f0-9]{8,}\b/g; +// Structured noise from imported/converted documents (OCR'd PDFs, DocuSign +// exports, HTML conversions). Stripped before tokenization so ID shrapnel +// ("DAB4-BCF3-..." → "dab", "bcf") never reaches keyword scoring. +const DASHED_HEX_RE = /\b[0-9a-f]{2,}(?:-[0-9a-f]{2,})+\b/gi; +const DIGIT_BLOB_RE = /\b(?=[0-9a-z]*\d)(?=[0-9a-z]*[a-z])[0-9a-z]{6,}\b/gi; +const HTML_TAG_RE = /<\/?[a-z][^>]*>/gi; +const HTML_ENTITY_RE = /&[a-z]+;|&#\d+;/gi; +const HEXLETTER_RUN_RE = /\b[a-f]{7,}\b/gi; + +// Homogeneous-sibling collapse: parents with at least this many children +// whose term distributions are at least this similar (mean pairwise cosine +// over top terms) are rendered as a single aggregate row. Tuned against a +// corpus of real-world agent vaults — imported document dumps sit at ~0.15–0.25 +// similarity, curated folder structures below ~0.05. +const COLLAPSE_MIN_CHILDREN = 5; +const COLLAPSE_SIMILARITY = 0.15; +const COLLAPSE_COSINE_TOP_TERMS = 60; +const COLLAPSE_PAIRWISE_CAP = 20; +const TOKEN_RE = /[a-z]{3,}/g; +const FRONTMATTER_RE = /^---[\s\S]*?---\n?/; +const ATX_HEADING_LINE_RE = /^#{1,6}\s+.+$/gm; +const WIKILINK_ONLY_RE = /^\[\[[^\]]+(?:\|[^\]]+)?\]\]$/; +const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}/; + +const STOP_WORDS = new Set([ + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", + "by", + "from", + "is", + "it", + "as", + "be", + "was", + "are", + "this", + "that", + "not", + "has", + "have", + "had", + "will", + "can", + "may", + "do", + "does", + "did", + "been", + "being", + "would", + "could", + "should", + "its", + "my", + "your", + "our", + "their", + "his", + "her", + "we", + "they", + "you", + "he", + "she", + "all", + "each", + "every", + "both", + "few", + "more", + "most", + "other", + "some", + "such", + "than", + "too", + "very", + "just", + "about", + "above", + "after", + "again", + "also", + "any", + "because", + "before", + "between", + "down", + "during", + "even", + "first", + "get", + "how", + "if", + "into", + "like", + "made", + "make", + "many", + "much", + "new", + "no", + "now", + "off", + "old", + "only", + "one", + "out", + "over", + "own", + "same", + "so", + "still", + "then", + "there", + "these", + "those", + "through", + "under", + "up", + "use", + "used", + "using", + "want", + "way", + "well", + "what", + "when", + "where", + "which", + "while", + "who", + "why", + "work", + "see", + "here", + "need", + "etc", + "two", + "next", + "per", + "via", + "vs", + "yet", + "ago", + "due", + "tbd", +]); // prettier-ignore + +interface WeightedText { + text: string; + weight: number; +} + +interface HeadingSignals { + lineCount: Map; + weightedTerms: Map; +} + +interface FolderData { + tf: Map; + /** + * Term frequencies from note content (bodies and heading lines, unweighted; + * no filename or title terms). Used for sibling-collapse similarity so + * shared naming conventions cannot fake content homogeneity. + */ + bodyTF: Map; + headingLineCount: Map; + hasNonHeading: Set; + tags: Set; + noteCount: number; +} + +function stripNoise(text: string): string { + return text + .replace(CODE_BLOCK_RE, "") + .replace(INLINE_CODE_RE, "") + .replace(URL_RE, "") + .replace(EMAIL_RE, "") + .replace(HTML_TAG_RE, " ") + .replace(HTML_ENTITY_RE, " ") + .replace(DASHED_HEX_RE, " ") + .replace(DIGIT_BLOB_RE, " ") + .replace(HEXLETTER_RUN_RE, " ") + .replace(HEX_HASH_RE, ""); +} + +function tokenize(text: string): string[] { + const cleaned = stripNoise(text); + return (cleaned.toLowerCase().match(TOKEN_RE) || []).filter( + (w) => !STOP_WORDS.has(w), + ); +} + +function extractBigrams(text: string): string[] { + const words = tokenize(text); + const bigrams: string[] = []; + for (let i = 0; i < words.length - 1; i++) { + bigrams.push(`${words[i]} ${words[i + 1]}`); + } + return bigrams; +} + +function terms(text: string): string[] { + return [...tokenize(text), ...extractBigrams(text)]; +} + +function addWeightedTerms( + target: Map, + sourceTerms: Iterable, + weight: number, +): void { + for (const term of sourceTerms) { + target.set(term, (target.get(term) || 0) + weight); + } +} + +function buildTF(sources: WeightedText[]): Map { + const freq = new Map(); + for (const { text, weight } of sources) { + addWeightedTerms(freq, terms(text), weight); + } + return freq; +} + +function folderKeywordTokens(folderPath: string): Set { + const tokens = new Set(); + for (const segment of folderPath.split("/")) { + for (const token of tokenize(segment)) { + tokens.add(token); + tokens.add(token.endsWith("s") ? token.slice(0, -1) : `${token}s`); + } + } + return tokens; +} + +function shouldSkipOverviewFile( + file: string, + folder: string, + templatesFolder: string, +): boolean { + const basename = path.basename(file); + const topLevelFolder = folder === "/" ? "" : folder.split("/")[0]; + + return ( + topLevelFolder === templatesFolder || + (folder === "/" && basename === "NAPKIN.md") || + basename === "_about.md" + ); +} + +function frontmatterText(properties: Record): string[] { + const values: string[] = []; + + const visit = (value: unknown) => { + if (typeof value === "string") values.push(value); + else if (Array.isArray(value)) value.forEach(visit); + }; + + for (const [key, value] of Object.entries(properties)) { + if (key === "title" || key === "tags") continue; + visit(value); + } + + return values.filter((value) => { + const trimmed = value.trim(); + return ( + trimmed.length > 0 && + !WIKILINK_ONLY_RE.test(trimmed) && + !ISO_DATE_PREFIX_RE.test(trimmed) + ); + }); +} + +function markdownBodyText(content: string): string { + return content.replace(FRONTMATTER_RE, "").replace(ATX_HEADING_LINE_RE, ""); +} + +function buildHeadingSignals(headings: Iterable): HeadingSignals { + const lineCount = new Map(); + const weightedTerms = new Map(); + const uniqueTerms = new Set(); + + for (const heading of headings) { + const seenInHeading = new Set(terms(heading)); + for (const term of seenInHeading) { + uniqueTerms.add(term); + lineCount.set(term, (lineCount.get(term) || 0) + 1); + } + } + + addWeightedTerms(weightedTerms, uniqueTerms, 3); + return { lineCount, weightedTerms }; +} + +function isCandidateKeyword( + term: string, + tf: number, + folderTokens: Set, + headingLineCount: Map, + hasNonHeading: Set, +): boolean { + if (term.includes(" ")) { + const [a, b] = term.split(" "); + if (tf < 2 || a === b) return false; + } else if (folderTokens.has(term)) { + return false; + } + + // Require corroboration: real keywords either appear outside headings or in + // multiple heading lines. Single heading-only terms are usually section labels. + return hasNonHeading.has(term) || (headingLineCount.get(term) || 0) >= 2; +} + +function extractKeywordsTFIDF( + folderTF: Map, + documentFrequency: Map, + totalFolders: number, + maxKeywords: number, + folderPath: string, + headingLineCount: Map, + hasNonHeading: Set, +): string[] { + const folderTokens = folderKeywordTokens(folderPath); + const scored: [string, number][] = []; + + for (const [term, tf] of folderTF) { + if ( + !isCandidateKeyword( + term, + tf, + folderTokens, + headingLineCount, + hasNonHeading, + ) + ) { + continue; + } + + const df = documentFrequency.get(term) || 1; + const idf = Math.log(1 + totalFolders / df); + scored.push([term, tf * idf]); + } + + const sorted = scored.sort((a, b) => b[1] - a[1]); + const selected: string[] = []; + const suppressed = new Set(); + + for (const [term] of sorted) { + if (selected.length >= maxKeywords) break; + if (suppressed.has(term)) continue; + + selected.push(term); + if (term.includes(" ")) { + for (const part of term.split(" ")) { + suppressed.add(part); + } + } + } + + return selected; +} + +/** Cosine similarity over the top-N terms of two TF maps. */ +function tfCosine(a: Map, b: Map): number { + const top = (m: Map) => + new Map( + [...m.entries()] + .sort((x, y) => y[1] - x[1]) + .slice(0, COLLAPSE_COSINE_TOP_TERMS), + ); + const ta = top(a); + const tb = top(b); + let dot = 0; + let na = 0; + let nb = 0; + for (const [, v] of ta) na += v * v; + for (const [, v] of tb) nb += v * v; + for (const [k, v] of ta) { + const w = tb.get(k); + if (w) dot += v * w; + } + if (na === 0 || nb === 0) return 0; + return dot / (Math.sqrt(na) * Math.sqrt(nb)); +} + +function mergeFolderData(items: FolderData[]): FolderData { + const tf = new Map(); + const bodyTF = new Map(); + const headingLineCount = new Map(); + const hasNonHeading = new Set(); + const tags = new Set(); + let noteCount = 0; + for (const d of items) { + for (const [k, v] of d.tf) tf.set(k, (tf.get(k) || 0) + v); + for (const [k, v] of d.bodyTF) bodyTF.set(k, (bodyTF.get(k) || 0) + v); + for (const [k, v] of d.headingLineCount) { + headingLineCount.set(k, (headingLineCount.get(k) || 0) + v); + } + for (const k of d.hasNonHeading) hasNonHeading.add(k); + for (const t of d.tags) tags.add(t); + noteCount += d.noteCount; + } + return { tf, bodyTF, headingLineCount, hasNonHeading, tags, noteCount }; +} + +/** + * Collapse numerous, lexically homogeneous sibling folders into their parent + * so repetitive subtrees (imported document dumps, per-entity folder fans) + * render as one aggregate row instead of dominating the overview. The vault + * root is never a collapse target: top-level folders are the taxonomy. + */ +function collapseHomogeneousSiblings(folderData: Map): { + data: Map; + collapsed: Map; +} { + const byParent = new Map(); + for (const folder of folderData.keys()) { + if (folder === "/") continue; + const idx = folder.lastIndexOf("/"); + const parent = idx === -1 ? "/" : folder.slice(0, idx); + if (!byParent.has(parent)) byParent.set(parent, []); + byParent.get(parent)?.push(folder); + } + + // Deepest parents first so collapses can cascade upward. + const parents = [...byParent.keys()].sort( + (a, b) => b.split("/").length - a.split("/").length, + ); + + const data = new Map(folderData); + const collapsed = new Map(); + + for (const parent of parents) { + if (parent === "/") continue; + const children = (byParent.get(parent) || []).filter((c) => data.has(c)); + if (children.length < COLLAPSE_MIN_CHILDREN) continue; + + const cap = Math.min(children.length, COLLAPSE_PAIRWISE_CAP); + let sum = 0; + let pairs = 0; + for (let i = 0; i < cap; i++) { + for (let j = i + 1; j < cap; j++) { + const a = data.get(children[i]); + const b = data.get(children[j]); + if (!a || !b) continue; + sum += tfCosine(a.bodyTF, b.bodyTF); + pairs++; + } + } + if (pairs === 0 || sum / pairs < COLLAPSE_SIMILARITY) continue; + + const toMerge: FolderData[] = []; + let mergedFolderCount = 0; + for (const child of children) { + const d = data.get(child); + if (!d) continue; + toMerge.push(d); + mergedFolderCount += 1 + (collapsed.get(child) || 0); + data.delete(child); + collapsed.delete(child); + } + const existing = data.get(parent); + const merged = existing + ? mergeFolderData([existing, ...toMerge]) + : mergeFolderData(toMerge); + data.set(parent, merged); + collapsed.set(parent, (collapsed.get(parent) || 0) + mergedFolderCount); + } + + return { data, collapsed }; +} + +function groupFilesByFolder( + files: string[], + templatesFolder: string, +): Map { + const folderFiles = new Map(); + + for (const file of files) { + const dir = path.dirname(file); + const folder = dir === "." ? "/" : dir; + if (shouldSkipOverviewFile(file, folder, templatesFolder)) continue; + + if (!folderFiles.has(folder)) folderFiles.set(folder, []); + folderFiles.get(folder)?.push(file); + } + + return folderFiles; +} + +function buildFolderData( + vaultPath: string, + folderFileList: string[], + warnings: string[], +): FolderData { + const allTags = new Set(); + const headings = new Set(); + const weightedSources: WeightedText[] = []; + const bodyTF = new Map(); + + for (const file of folderFileList) { + const content = fs.readFileSync(path.join(vaultPath, file), "utf-8"); + let properties: Record = {}; + + try { + ({ properties } = parseFrontmatter(content)); + } catch { + warnings.push(`Skipping ${file} (malformed YAML frontmatter)`); + continue; + } + + for (const tag of extractTags(content)) allTags.add(tag); + if (Array.isArray(properties.tags)) { + for (const tag of properties.tags) allTags.add(String(tag)); + } + + const fileHeadings = extractHeadings(content); + for (const heading of fileHeadings) { + headings.add(heading.text.trim()); + addWeightedTerms(bodyTF, terms(heading.text), 1); + } + + weightedSources.push({ text: path.basename(file, ".md"), weight: 2 }); + if (properties.title) { + weightedSources.push({ text: String(properties.title), weight: 2 }); + } + for (const value of frontmatterText(properties)) { + weightedSources.push({ text: value, weight: 2 }); + } + const body = markdownBodyText(content); + weightedSources.push({ text: body, weight: 1 }); + addWeightedTerms(bodyTF, terms(body), 1); + } + + const tf = buildTF(weightedSources); + const hasNonHeading = new Set(tf.keys()); + const headingSignals = buildHeadingSignals(headings); + + for (const [term, weight] of headingSignals.weightedTerms) { + tf.set(term, (tf.get(term) || 0) + weight); + } + + return { + tf, + bodyTF, + headingLineCount: headingSignals.lineCount, + hasNonHeading, + tags: allTags, + noteCount: folderFileList.length, + }; +} + +function buildOverviewFolders( + vaultPath: string, + maxDepth: number, + maxKeywords: number, + templatesFolder: string, + collapse: boolean, +): { folders: OverviewFolder[]; warnings: string[] } { + const files = listFiles(vaultPath, { ext: "md" }); + const warnings: string[] = []; + const folderFiles = groupFilesByFolder(files, templatesFolder); + let folderData = new Map(); + + for (const [folder, folderFileList] of folderFiles) { + const depth = folder === "/" ? 0 : folder.split("/").length; + if (depth > maxDepth) continue; + + folderData.set( + folder, + buildFolderData(vaultPath, folderFileList, warnings), + ); + } + + let collapsedCounts = new Map(); + if (collapse) { + const result = collapseHomogeneousSiblings(folderData); + folderData = result.data; + collapsedCounts = result.collapsed; + } + + const documentFrequency = new Map(); + for (const { tf } of folderData.values()) { + for (const term of tf.keys()) { + documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1); + } + } + + const totalFolders = folderData.size; + const folders: OverviewFolder[] = []; + + for (const [folder, data] of [...folderData.entries()].sort((a, b) => + a[0].localeCompare(b[0]), + )) { + folders.push({ + path: folder, + notes: data.noteCount, + keywords: extractKeywordsTFIDF( + data.tf, + documentFrequency, + totalFolders, + maxKeywords, + folder, + data.headingLineCount, + data.hasNonHeading, + ), + tags: [...data.tags].sort(), + ...(collapsedCounts.has(folder) + ? { collapsedFolders: collapsedCounts.get(folder) } + : {}), + }); + } + + return { folders, warnings }; +} + +export function getOverview( + contentPath: string, + configPath: string, + opts?: { depth?: number; keywords?: number; collapse?: boolean }, +): VaultOverview { + const config = loadConfig(configPath); + const maxDepth = opts?.depth ?? config.overview.depth; + const maxKeywords = opts?.keywords ?? config.overview.keywords; + const collapse = opts?.collapse ?? config.overview.collapse; + + const { folders, warnings } = buildOverviewFolders( + contentPath, + maxDepth, + maxKeywords, + config.templates.folder, + collapse, + ); + + const contextPath = path.join(contentPath, "NAPKIN.md"); + const context = fs.existsSync(contextPath) + ? fs.readFileSync(contextPath, "utf-8").trim() + : undefined; + + return { + ...(context ? { context } : {}), + overview: folders, + ...(warnings.length > 0 ? { warnings } : {}), + }; +} + +// Exported for differential testing only (see overview.equivalence.test.ts). +export { + getOverview as originalGetOverview, + stripNoise as originalStripNoise, + terms as originalTerms, +}; diff --git a/src/core/__tests__/overview.equivalence.test.ts b/src/core/__tests__/overview.equivalence.test.ts new file mode 100644 index 0000000..4a9ebb2 --- /dev/null +++ b/src/core/__tests__/overview.equivalence.test.ts @@ -0,0 +1,394 @@ +import { describe, expect, test } from "bun:test"; +import matter from "gray-matter"; +import { createTempVault } from "../../utils/test-helpers.js"; +import { _stripNoise, _termCounts, getOverview } from "../overview.js"; +import { + originalGetOverview, + originalStripNoise, + originalTerms, +} from "./overview-original.js"; + +/** + * Differential test suite: the optimized overview pipeline vs a verbatim + * copy of the pre-optimization implementation (overview-original.ts). + * + * The refactor made three equivalence claims, each verified here: + * 1. Guarded stripNoise ≡ unguarded stripNoise (guards are necessary + * conditions of their patterns; HEX_HASH may skip digitless text because + * HEXLETTER_RUN already removed pure-letter runs). + * 2. termCounts(text) ≡ occurrence-wise accumulation of terms(text), + * including Map INSERTION ORDER (keyword tie-breaking is a stable sort + * over Map iteration order, so order is behavior, not a detail). + * 3. The full pipeline produces byte-identical JSON for arbitrary vaults + * and options. + * + * Cache isolation: gray-matter memoizes by content string and caches the + * file object BEFORE parsing, so the first parse of malformed frontmatter + * throws while a later parse of the identical string silently returns empty + * data. Both implementations issue identical matter() call sequences, so + * they are equivalent given equal cache state — the harness clears the cache + * before every pipeline run to compare them from the same starting state. + */ + +function isolated(fn: () => T): T { + matter.clearCache(); + return fn(); +} + +// ─── deterministic PRNG ───────────────────────────────────────────── + +function mulberry32(seed: number): () => number { + let a = seed; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function pick(rand: () => number, arr: readonly T[]): T { + return arr[Math.floor(rand() * arr.length)]; +} + +function int(rand: () => number, min: number, max: number): number { + return min + Math.floor(rand() * (max - min + 1)); +} + +// ─── adversarial fragment pool ────────────────────────────────────── +// Every fragment targets a specific stripNoise pattern, a guard boundary, +// a tokenizer edge, or keyword-pipeline behavior. + +const FRAGMENTS: readonly string[] = [ + // hex-letter runs at the 6/7/8 length boundaries, case variants + "abcdef", + "abcdefa", + "deadbeef", + "DEADBEEF", + "deadbeefcafe", + "xabcdefgh", + "gabcdefabx", + // hex hashes with digits (8+ boundary) + "e5f6a7b8", + "a1b2c3d", + "cafe1234babe0000", + "0123456789abcdef", + // digit blobs (6+ mixed alnum) and short non-matches + "abc123def", + "ab12x", + "x1y2z3", + "INV20240915X", + "b2", + // dashed hex, including non-hex dashes and list dashes + "ab-cd", + "AB-CD-EF", + "a1-b2-c3", + "12-34-56", + "well-known", + "- list item", + "AAAA1111-2222-4333-ADAB-BCF123456789", + // emails and near-emails + "user@example.com", + "leasing@sub.example.co.uk", + "not @ email", + "@handle", + "a@b.co", + // URLs, case variants (URL_RE is case-sensitive) + "https://example.com/portal?id=99", + "http://x.y/z#frag", + "HTTP://UPPER.EXAMPLE.COM", + "httpish prose word", + // inline code and fences, closed and unclosed + "`inline code`", + "` unclosed backtick", + "```\nconst rent = base * 1.05;\n```", + "``` unclosed fence\ncode-ish 42", + // HTML tags and entities, real and fake + '
', + "

", + "< notatag", + "<3 hearts", + " ", + "&", + "{", + "& plain ampersand", + "¬arealentity", + // stopwords, short tokens, unicode + "the and with very just about", + "an ox is up", + "naïve café über résumé 東京", + // ordinary prose (keyword candidates, bigram material) + "kubernetes ingress routing policies", + "payroll tax withholding tables", + "telescope collimation reflector optics", + "transactional outbox relay latency", + "bank guarantee insurance certificate", + "lease agreement landlord tenant", + "Suite 100 on floor 3", + // markdown structure + "## Context", + "## Decision ", + "# Top Heading", + "[[Some Note|alias]]", + "2024-03-01 kickoff meeting", +]; + +const SEPARATORS: readonly string[] = [" ", "\n", ", ", ".\n\n", " — "]; + +function randomText(rand: () => number, maxFragments = 30): string { + const n = int(rand, 1, maxFragments); + const parts: string[] = []; + for (let i = 0; i < n; i++) { + parts.push(pick(rand, FRAGMENTS)); + if (i < n - 1) parts.push(pick(rand, SEPARATORS)); + } + return parts.join(""); +} + +// ─── layer 1: stripNoise guards ───────────────────────────────────── + +describe("stripNoise ≡ original", () => { + test("fixed adversarial cases (one per guard boundary)", () => { + const cases = [ + "", // empty + "plain prose with no noise at all", + "HTTP://UPPER.COM has http only lowercase-guarded", // URL guard vs case + "Http://Mixed.Com httpx", // "http" present, pattern can't match + "at sign only: a @ b", // "@" present, email can't match + "deadbeef", // pure-letter hex, no digit: HEX_HASH must still be stripped via HEXLETTER_RUN + "deadbeefX deadbeef1", // adjacent digit/no-digit hex + "xabcdefgh embedded run without word boundary", + "ab-cd but no digits anywhere", // DASHED_HEX without digits + "-- --- - dashes only", + "`` empty inline", // backtick present, empty inline can't match + "``` only one fence", + "&& & &; &#; entity near-misses", + "<> < > angle near-misses", + "١٢٣ unicode digits ٤٥٦", // \d is ASCII-only: hasDigit guard must not differ + "𝟏𝟐𝟑 mathematical digits", + ]; + for (const c of cases) { + expect(_stripNoise(c)).toBe(originalStripNoise(c)); + } + }); + + test("20,000 fuzzed strings", () => { + const rand = mulberry32(0x0135e); + for (let i = 0; i < 20_000; i++) { + const s = randomText(rand); + const got = _stripNoise(s); + const want = originalStripNoise(s); + if (got !== want) { + // fail with the offending input visible + expect({ input: s, got }).toEqual({ input: s, got: want }); + } + expect(got).toBe(want); + } + }); +}); + +// ─── layer 2: termCounts vs occurrence-wise terms() ───────────────── + +/** The original accumulation: +1 per occurrence, in occurrence order. */ +function originalCounts(text: string): Map { + const m = new Map(); + for (const t of originalTerms(text)) m.set(t, (m.get(t) || 0) + 1); + return m; +} + +describe("termCounts ≡ original occurrence accumulation", () => { + test("10,000 fuzzed strings: values AND insertion order", () => { + const rand = mulberry32(0x7e12); + for (let i = 0; i < 10_000; i++) { + const s = randomText(rand); + const got = _termCounts(s); + const want = originalCounts(s); + // insertion order is behavior: keyword tie-breaks depend on it + expect([...got.keys()]).toEqual([...want.keys()]); + expect([...got.values()]).toEqual([...want.values()]); + } + }); + + test("repeated terms keep first-occurrence position", () => { + const got = _termCounts("alpha beta alpha gamma beta alpha"); + expect([...got.entries()]).toEqual([ + ["alpha", 3], + ["beta", 2], + ["gamma", 1], + ["alpha beta", 1], + ["beta alpha", 2], + ["alpha gamma", 1], + ["gamma beta", 1], + ]); + }); +}); + +// ─── layer 3: weighted multi-source accumulation order ────────────── + +describe("weighted accumulation ≡ original buildTF", () => { + test("2,000 fuzzed source sequences: merged map order and values", () => { + const rand = mulberry32(0xacc); + for (let i = 0; i < 2_000; i++) { + const nSources = int(rand, 1, 6); + const sources: { text: string; weight: number }[] = []; + for (let j = 0; j < nSources; j++) { + sources.push({ + text: randomText(rand, 8), + weight: pick(rand, [1, 2, 3]), + }); + } + // original: occurrence-wise, source by source + const want = new Map(); + for (const { text, weight } of sources) { + for (const t of originalTerms(text)) { + want.set(t, (want.get(t) || 0) + weight); + } + } + // new: count once per source, merge unique entries + const got = new Map(); + for (const { text, weight } of sources) { + for (const [t, c] of _termCounts(text)) { + got.set(t, (got.get(t) || 0) + c * weight); + } + } + expect([...got.keys()]).toEqual([...want.keys()]); + expect([...got.values()]).toEqual([...want.values()]); + } + }); +}); + +// ─── layer 4: full pipeline on randomized vaults ──────────────────── + +const HEADING_POOL: readonly string[] = [ + "Context", + "Decision", + "Consequences", + "Overview", + "Notes ", // trailing space: exercises trimmed heading-cache key + "Notes", + "Kubernetes Setup", + "Bank Guarantee", +]; + +const FOLDER_POOL: readonly string[] = [ + "decisions", + "notes", + "people", + "contracts", + "areas/alpha", + "areas/beta", + "projects/web/frontend", + "deep/one/two/three", +]; + +function randomFrontmatter(rand: () => number): string { + if (rand() < 0.1) return "---\ntags: [#bad, #yaml]\n---\n"; // malformed + const lines = ["---"]; + if (rand() < 0.7) lines.push(`title: Title ${int(rand, 1, 5)} guarantee`); + if (rand() < 0.5) lines.push("tags: [adr, database]"); + if (rand() < 0.4) lines.push("status: accepted"); + if (rand() < 0.4) lines.push("date: 2024-03-01"); + if (rand() < 0.4) lines.push('related: "[[outbox]]"'); + if (rand() < 0.4) lines.push(`role: Staff Engineer ${pick(rand, FRAGMENTS)}`); + lines.push("---"); + return `${lines.join("\n")}\n`; +} + +function randomNote(rand: () => number): string { + let content = rand() < 0.7 ? randomFrontmatter(rand) : ""; + const nHeadings = int(rand, 0, 3); + for (let h = 0; h < nHeadings; h++) { + content += `${"#".repeat(int(rand, 1, 3))} ${pick(rand, HEADING_POOL)}\n`; + content += `${randomText(rand, 10)}\n`; + } + content += randomText(rand, 15); + return content; +} + +function randomVault(rand: () => number): Record { + const files: Record = {}; + if (rand() < 0.5) files["NAPKIN.md"] = "# Context\nGolden fuzz vault."; + if (rand() < 0.5) files["Templates/Decision.md"] = "# {{title}}\n## Context"; + if (rand() < 0.3) files["welcome.md"] = randomNote(rand); + + const nFiles = int(rand, 3, 20); + for (let i = 0; i < nFiles; i++) { + const folder = pick(rand, FOLDER_POOL); + files[`${folder}/note-${i}.md`] = randomNote(rand); + if (rand() < 0.15) files[`${folder}/_about.md`] = "# About\nScaffold."; + } + + // homogeneous sibling fan (collapse candidate) — shared boilerplate + if (rand() < 0.6) { + const fanSize = int(rand, 4, 8); // straddles COLLAPSE_MIN_CHILDREN=5 + const shared = + "Lease agreement between landlord and tenant.\n" + + "Bank guarantee and insurance certificate required."; + for (let i = 0; i < fanSize; i++) { + files[`imports/tenant-${i}/contract.md`] = + `# Converted document ${i}\n${shared}\nSuite ${100 + i}.`; + } + } + + // heterogeneous sibling fan (must NOT collapse) + if (rand() < 0.4) { + const topics = [ + "Kubernetes ingress routing and pod autoscaling.", + "Payroll tax withholding for hourly contractors.", + "Sourdough fermentation schedules and hydration.", + "Telescope collimation for reflector optics.", + "Beehive winterization and varroa treatment.", + "Marathon splits and lactate threshold pacing.", + ]; + topics.forEach((body, i) => { + files[`mixed/topic-${i}/note.md`] = `# Topic ${i}\n${body}`; + }); + } + + return files; +} + +describe("getOverview ≡ original on randomized vaults", () => { + test("60 fuzzed vaults × random options, byte-identical JSON", () => { + const rand = mulberry32(0x0a017); + for (let i = 0; i < 60; i++) { + const vault = createTempVault(randomVault(rand)); + try { + const opts = { + depth: int(rand, 1, 4), + keywords: int(rand, 3, 10), + collapse: rand() < 0.5, + }; + const got = isolated(() => + getOverview(vault.vaultPath, vault.vaultPath, opts), + ); + const want = isolated(() => + originalGetOverview(vault.vaultPath, vault.vaultPath, opts), + ); + expect(JSON.stringify(got, null, 1)).toBe( + JSON.stringify(want, null, 1), + ); + } finally { + vault.cleanup(); + } + } + }); + + test("empty vault and default options", () => { + const vault = createTempVault({}); + try { + expect( + JSON.stringify( + isolated(() => getOverview(vault.vaultPath, vault.vaultPath)), + ), + ).toBe( + JSON.stringify( + isolated(() => originalGetOverview(vault.vaultPath, vault.vaultPath)), + ), + ); + } finally { + vault.cleanup(); + } + }); +}); diff --git a/src/core/overview.golden.test.ts b/src/core/overview.golden.test.ts new file mode 100644 index 0000000..8a89141 --- /dev/null +++ b/src/core/overview.golden.test.ts @@ -0,0 +1,160 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { createTempVault } from "../utils/test-helpers.js"; +import { getOverview } from "./overview.js"; + +/** + * Characterization (golden) test for getOverview. + * + * The fixture vault deterministically exercises every code path in + * overview.ts: weighted TF sources (filename x2, title x2, frontmatter x2, + * body x1, headings x3), heading dedup across files, heading corroboration + * filtering, bigram extraction + unigram suppression, folder-token exclusion + * (singular/plural), noise stripping (code, URLs, emails, HTML, GUIDs, digit + * blobs, hex runs), scaffold skipping (Templates/, NAPKIN.md, _about.md), + * depth limiting, malformed-frontmatter warnings, homogeneous-sibling + * collapse, and heterogeneous siblings kept separate. + * + * Any behavior change in the overview pipeline must show up as a snapshot + * diff. Performance refactors must keep this snapshot byte-identical. + */ + +const FIXTURE: Record = { + // L0 context — rendered separately, excluded from folder rows + "NAPKIN.md": "# Fixture project\nContext note for the golden vault.", + // scaffold files — always skipped + "Templates/Decision.md": "# {{title}}\n## Context\n## Decision", + "decisions/_about.md": "# Decisions\nArchitecture Decision Records.", + + // root-level note + "welcome.md": "# Welcome\nOrientation for newcomers to the fixture vault.", + + // decisions/: repeated structural headings (Context/Decision/Consequences + // must be suppressed), bigrams, distinctive unigrams + "decisions/postgres.md": `--- +title: Use PostgreSQL +tags: [database, adr] +status: accepted +date: 2024-03-01 +related: "[[outbox]]" +--- +# Use PostgreSQL +## Context +Ledger writes need transactional storage with strict durability. +## Decision +Use PostgreSQL for balances and ledger entries. Connection pooling via pgbouncer. +## Consequences +We operate backups and vacuum schedules ourselves.`, + "decisions/outbox.md": `--- +title: Adopt transactional outbox +tags: [messaging] +--- +# Adopt transactional outbox +## Context +Kafka dual writes lost events during broker failover. +## Decision +Write outbox events inside the database transaction, relay them asynchronously. +## Consequences +Relay latency increases slightly. Transactional outbox rows need pruning.`, + "decisions/braintree.md": `# Deprecate Braintree +## Context +Braintree maintenance cost is high and the SDK is stale. +## Decision +Migrate merchants to Adyen over two quarters. +## Consequences +Two merchants need bespoke migration plans.`, + + // contracts/: converted-document noise that must never leak into keywords + "contracts/lease.md": `# Lease agreement +DocuSign Envelope ID: AAAA1111-2222-4333-ADAB-BCF123456789 +
 
+Tenant leases the third floor at https://example.com/portal?id=99 and pays +rent monthly. Contact leasing@example.com with hash deadbeefcafe1234. +Sublease requires landlord approval and a bank guarantee. Code \`rentCalc()\` +and block: +\`\`\`js +const rent = base * 1.05; // escalation +\`\`\` +Reference ab12cd34ef and invoice INV20240915X for the guarantee.`, + "contracts/parking.md": `# Parking addendum +Envelope ID: CCCC3333-4444-4555-FADE-CAB456789012 +Reserved parking slots on level B2. Guarantee covers parking fees and the +bank guarantee renews annually with the lease agreement.`, + + // people/: frontmatter values indexed, folder tokens (people/person) excluded + "people/asha.md": `--- +role: VP Engineering +location: Boston +tags: [leadership] +--- +# Asha Mehta +Owns platform strategy and the quarterly engineering roadmap.`, + "people/lukas.md": `--- +role: Staff Engineer +location: Berlin +--- +# Lukas Weber +Owns fleet dispatch and the routing engine internals.`, + + // malformed frontmatter — must warn and still count the note + "people/broken.md": `--- +tags: [#oops, #bad] +--- +# Broken note +This body is skipped for keywords but the note is counted.`, + + // deep/: exceeds default depth (2) at the third level + "deep/one/two/buried.md": "# Buried\nThis folder is beyond the depth limit.", + "deep/one/present.md": + "# Present\nWithin depth, mentions telescopes twice: telescope optics, telescope mounts.", +}; + +// imports/: six homogeneous siblings that must collapse into imports/ +const boilerplate = [ + "Lease agreement between landlord and tenant with signature page attached.", + "Rent schedule and lease term apply as stated in the appendix.", + "Bank guarantee and insurance certificate are required before occupancy.", +]; +for (let i = 0; i < 6; i++) { + const shared = boilerplate.filter((_, j) => j !== i % 3).join("\n"); + FIXTURE[`imports/tenant-${i}/contract.md`] = + `# Converted document ${i}\n${shared}\nSuite ${100 + i} on floor ${i}.`; +} + +// areas/: six heterogeneous siblings that must stay separate +const topics = [ + ["alpha", "Kubernetes ingress routing and pod autoscaling policies."], + ["beta", "Payroll tax withholding tables for hourly contractors."], + ["gamma", "Sourdough fermentation schedules and hydration ratios."], + ["delta", "Telescope collimation steps for reflector optics."], + ["epsilon", "Beehive winterization and varroa mite treatment."], + ["zeta", "Marathon training splits and lactate threshold pacing."], +] as const; +for (const [name, body] of topics) { + FIXTURE[`areas/${name}/note.md`] = `# ${name} notes\n${body}`; +} + +const vault = createTempVault(FIXTURE); +afterAll(() => vault.cleanup()); + +describe("getOverview golden", () => { + test("default options", () => { + const result = getOverview(vault.vaultPath, vault.vaultPath); + expect(result).toMatchSnapshot(); + }); + + test("depth 3, keywords 8", () => { + const result = getOverview(vault.vaultPath, vault.vaultPath, { + depth: 3, + keywords: 8, + }); + expect(result).toMatchSnapshot(); + }); + + test("collapse disabled", () => { + const result = getOverview(vault.vaultPath, vault.vaultPath, { + collapse: false, + depth: 3, + }); + expect(result).toMatchSnapshot(); + }); +}); diff --git a/src/core/overview.ts b/src/core/overview.ts index 7793b37..fe0226e 100644 --- a/src/core/overview.ts +++ b/src/core/overview.ts @@ -4,6 +4,11 @@ import { loadConfig } from "../utils/config.js"; import { listFiles } from "../utils/files.js"; import { parseFrontmatter } from "../utils/frontmatter.js"; import { extractHeadings, extractTags } from "../utils/markdown.js"; +import { + loadOverviewCache, + saveOverviewCache, +} from "../utils/overview-cache.js"; +import { computeFingerprint } from "../utils/search-cache.js"; export interface OverviewFolder { path: string; @@ -185,8 +190,9 @@ const STOP_WORDS = new Set([ "tbd", ]); // prettier-ignore -interface WeightedText { - text: string; +interface WeightedTerms { + /** Term → occurrence count for one source text. */ + counts: Map; weight: number; } @@ -209,18 +215,29 @@ interface FolderData { noteCount: number; } +const DIGIT_RE = /\d/; + +/** + * Each replace is guarded by a necessary condition of its pattern (an email + * must contain "@", a URL "http", ...) so clean prose skips the expensive + * regex scans entirely. Guards never change the result: when the guard is + * false the pattern cannot match. HEX_HASH_RE can skip when no digit remains + * because HEXLETTER_RUN_RE has already removed pure-letter hex runs ≥7. + */ function stripNoise(text: string): string { - return text - .replace(CODE_BLOCK_RE, "") - .replace(INLINE_CODE_RE, "") - .replace(URL_RE, "") - .replace(EMAIL_RE, "") - .replace(HTML_TAG_RE, " ") - .replace(HTML_ENTITY_RE, " ") - .replace(DASHED_HEX_RE, " ") - .replace(DIGIT_BLOB_RE, " ") - .replace(HEXLETTER_RUN_RE, " ") - .replace(HEX_HASH_RE, ""); + let out = text; + if (out.includes("```")) out = out.replace(CODE_BLOCK_RE, ""); + if (out.includes("`")) out = out.replace(INLINE_CODE_RE, ""); + if (out.includes("http")) out = out.replace(URL_RE, ""); + if (out.includes("@")) out = out.replace(EMAIL_RE, ""); + if (out.includes("<")) out = out.replace(HTML_TAG_RE, " "); + if (out.includes("&")) out = out.replace(HTML_ENTITY_RE, " "); + const hasDigit = DIGIT_RE.test(out); + if (out.includes("-")) out = out.replace(DASHED_HEX_RE, " "); + if (hasDigit) out = out.replace(DIGIT_BLOB_RE, " "); + out = out.replace(HEXLETTER_RUN_RE, " "); + if (hasDigit) out = out.replace(HEX_HASH_RE, ""); + return out; } function tokenize(text: string): string[] { @@ -230,17 +247,34 @@ function tokenize(text: string): string[] { ); } -function extractBigrams(text: string): string[] { - const words = tokenize(text); - const bigrams: string[] = []; - for (let i = 0; i < words.length - 1; i++) { - bigrams.push(`${words[i]} ${words[i + 1]}`); +/** + * Term → occurrence count for one text, from a single tokenize() pass. + * Unigrams are inserted before bigrams, each in first-occurrence order — + * the same insertion order occurrence-wise accumulation produced, so + * downstream keyword tie-breaking (stable sort over Map order) is unchanged. + */ +function termCounts(text: string): Map { + const tokens = tokenize(text); + const counts = new Map(); + for (const token of tokens) { + counts.set(token, (counts.get(token) || 0) + 1); } - return bigrams; + for (let i = 0; i < tokens.length - 1; i++) { + const bigram = `${tokens[i]} ${tokens[i + 1]}`; + counts.set(bigram, (counts.get(bigram) || 0) + 1); + } + return counts; } -function terms(text: string): string[] { - return [...tokenize(text), ...extractBigrams(text)]; +/** Merge per-text counts into an accumulator, scaled by an integer weight. */ +function mergeCounts( + target: Map, + counts: Map, + weight: number, +): void { + for (const [term, count] of counts) { + target.set(term, (target.get(term) || 0) + count * weight); + } } function addWeightedTerms( @@ -253,10 +287,10 @@ function addWeightedTerms( } } -function buildTF(sources: WeightedText[]): Map { +function buildTF(sources: WeightedTerms[]): Map { const freq = new Map(); - for (const { text, weight } of sources) { - addWeightedTerms(freq, terms(text), weight); + for (const source of sources) { + mergeCounts(freq, source.counts, source.weight); } return freq; } @@ -314,14 +348,15 @@ function markdownBodyText(content: string): string { return content.replace(FRONTMATTER_RE, "").replace(ATX_HEADING_LINE_RE, ""); } -function buildHeadingSignals(headings: Iterable): HeadingSignals { +function buildHeadingSignals( + headingCounts: Iterable>, +): HeadingSignals { const lineCount = new Map(); const weightedTerms = new Map(); const uniqueTerms = new Set(); - for (const heading of headings) { - const seenInHeading = new Set(terms(heading)); - for (const term of seenInHeading) { + for (const counts of headingCounts) { + for (const term of counts.keys()) { uniqueTerms.add(term); lineCount.set(term, (lineCount.get(term) || 0) + 1); } @@ -533,8 +568,10 @@ function buildFolderData( warnings: string[], ): FolderData { const allTags = new Set(); - const headings = new Set(); - const weightedSources: WeightedText[] = []; + // Term counts per unique heading text (first-seen order), computed once and + // reused for both bodyTF (per occurrence) and heading signals (per unique). + const headingCountCache = new Map>(); + const weightedSources: WeightedTerms[] = []; const bodyTF = new Map(); for (const file of folderFileList) { @@ -555,25 +592,36 @@ function buildFolderData( const fileHeadings = extractHeadings(content); for (const heading of fileHeadings) { - headings.add(heading.text.trim()); - addWeightedTerms(bodyTF, terms(heading.text), 1); + const key = heading.text.trim(); + let headingCounts = headingCountCache.get(key); + if (!headingCounts) { + headingCounts = termCounts(heading.text); + headingCountCache.set(key, headingCounts); + } + mergeCounts(bodyTF, headingCounts, 1); } - weightedSources.push({ text: path.basename(file, ".md"), weight: 2 }); + weightedSources.push({ + counts: termCounts(path.basename(file, ".md")), + weight: 2, + }); if (properties.title) { - weightedSources.push({ text: String(properties.title), weight: 2 }); + weightedSources.push({ + counts: termCounts(String(properties.title)), + weight: 2, + }); } for (const value of frontmatterText(properties)) { - weightedSources.push({ text: value, weight: 2 }); + weightedSources.push({ counts: termCounts(value), weight: 2 }); } - const body = markdownBodyText(content); - weightedSources.push({ text: body, weight: 1 }); - addWeightedTerms(bodyTF, terms(body), 1); + const bodyCounts = termCounts(markdownBodyText(content)); + weightedSources.push({ counts: bodyCounts, weight: 1 }); + mergeCounts(bodyTF, bodyCounts, 1); } const tf = buildTF(weightedSources); const hasNonHeading = new Set(tf.keys()); - const headingSignals = buildHeadingSignals(headings); + const headingSignals = buildHeadingSignals(headingCountCache.values()); for (const [term, weight] of headingSignals.weightedTerms) { tf.set(term, (tf.get(term) || 0) + weight); @@ -663,6 +711,19 @@ export function getOverview( const maxKeywords = opts?.keywords ?? config.overview.keywords; const collapse = opts?.collapse ?? config.overview.collapse; + // Whole-vault cache: one stat pass instead of reading + tokenizing every + // note. Any file add/remove/touch changes the fingerprint; NAPKIN.md is a + // vault .md file, so context changes invalidate too. Resolved options are + // part of the key because they change the result. + const fingerprint = computeFingerprint(contentPath); + const optionsKey = `${maxDepth}|${maxKeywords}|${collapse}|${config.templates.folder}`; + const cached = loadOverviewCache( + configPath, + fingerprint, + optionsKey, + ); + if (cached) return cached; + const { folders, warnings } = buildOverviewFolders( contentPath, maxDepth, @@ -676,9 +737,16 @@ export function getOverview( ? fs.readFileSync(contextPath, "utf-8").trim() : undefined; - return { + const result: VaultOverview = { ...(context ? { context } : {}), overview: folders, ...(warnings.length > 0 ? { warnings } : {}), }; + + saveOverviewCache(configPath, { fingerprint, optionsKey, result }); + return result; } + +// Exported for differential testing against the pre-optimization oracle +// (src/core/__tests__/overview.equivalence.test.ts). Not public API. +export { stripNoise as _stripNoise, termCounts as _termCounts }; diff --git a/src/core/search.ts b/src/core/search.ts index 9c4a03d..f9274e6 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -1,6 +1,6 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import MiniSearch from "minisearch"; +import { FerroSearch } from "@shift-labs/ferrosearch"; import { loadConfig } from "../utils/config.js"; import { listFiles, resolveFileLoose } from "../utils/files.js"; import { extractLinks } from "../utils/markdown.js"; @@ -33,6 +33,24 @@ interface DocRecord { mtime: number; } +/** The shape of one search hit; ferrosearch types results as `unknown`. */ +interface IndexHit { + id: number; + score: number; +} + +// Shared by indexing and cache loading: loadJson requires the exact options +// the index was serialized with, so there must be a single definition. +const INDEX_OPTIONS = { + fields: ["basename", "content"], + storeFields: ["file"], + searchOptions: { + boost: { basename: 2 }, + fuzzy: 0.2, + prefix: true, + }, +}; + function buildIndex(vaultPath: string, folder?: string) { const files = listFiles(vaultPath, { folder, ext: "md" }); @@ -44,16 +62,7 @@ function buildIndex(vaultPath: string, folder?: string) { return { id, file, basename, content, mtime: stat.mtimeMs }; }); - const index = new MiniSearch({ - fields: ["basename", "content"], - storeFields: ["file"], - searchOptions: { - boost: { basename: 2 }, - fuzzy: 0.2, - prefix: true, - }, - }); - + const index = new FerroSearch(INDEX_OPTIONS); index.addAll(docs); return { index, docs }; } @@ -141,20 +150,12 @@ export function searchVault( const fingerprint = computeFingerprint(contentPath, opts?.path); const cached = loadSearchCache(configPath, fingerprint); - let index: MiniSearch; + let index: FerroSearch; let docs: DocRecord[]; let backlinkCounts: Map; if (cached) { - index = MiniSearch.loadJSON(cached.index, { - fields: ["basename", "content"], - storeFields: ["file"], - searchOptions: { - boost: { basename: 2 }, - fuzzy: 0.2, - prefix: true, - }, - }); + index = FerroSearch.loadJson(cached.index, INDEX_OPTIONS); docs = cached.docs.map((d) => { const fullPath = path.join(contentPath, d.file); const content = fs.readFileSync(fullPath, "utf-8"); @@ -169,13 +170,15 @@ export function searchVault( saveSearchCache(configPath, { fingerprint, - index: JSON.stringify(index), + // ferrosearch has no toJSON, so JSON.stringify(index) would not work; + // toJsonString writes the MiniSearch version-2 format in one native pass. + index: index.toJsonString(), docs: docs.map(({ content: _, ...rest }) => rest), backlinkCounts: Object.fromEntries(backlinkCounts), }); } - const results = index.search(query); + const results = index.search(query) as IndexHit[]; const contextLines = opts?.snippetLines ?? config.search.snippetLines; const limit = opts?.limit ?? config.search.limit; diff --git a/src/main.ts b/src/main.ts index 0b53fc9..793ad13 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,9 @@ #!/usr/bin/env node -import { createRequire } from "node:module"; import { Command } from "commander"; +// A static JSON import bundles into compiled binaries (`build:bun`), where +// a runtime require("../package.json") cannot resolve. +import packageJson from "../package.json" with { type: "json" }; import { aliases } from "./commands/aliases.js"; import { baseCreate, baseQuery, bases, baseViews } from "./commands/bases.js"; import { bookmark, bookmarks } from "./commands/bookmarks.js"; @@ -61,8 +63,7 @@ import { update } from "./commands/update.js"; import { vault } from "./commands/vault.js"; import { wordcount } from "./commands/wordcount.js"; -const require = createRequire(import.meta.url); -const { version } = require("../package.json"); +const { version } = packageJson; const program = new Command(); diff --git a/src/utils/frontmatter.test.ts b/src/utils/frontmatter.test.ts index cbb7535..cfe40d6 100644 --- a/src/utils/frontmatter.test.ts +++ b/src/utils/frontmatter.test.ts @@ -25,6 +25,27 @@ describe("parseFrontmatter", () => { expect(result.properties).toEqual({}); expect(result.body).toContain("Body"); }); + + test("throws on malformed YAML", () => { + const bad = "---\ntags: [#malformed, #unique-a]\n---\nBody"; + expect(() => parseFrontmatter(bad)).toThrow(); + }); + + test("throws consistently on repeated parses of identical malformed YAML", () => { + // gray-matter caches the file object BEFORE parsing, so without eviction + // the second parse of the same string silently returns empty data. + const bad = "---\ntags: [#malformed, #unique-b]\n---\nBody"; + expect(() => parseFrontmatter(bad)).toThrow(); + expect(() => parseFrontmatter(bad)).toThrow(); + expect(() => parseFrontmatter(bad)).toThrow(); + }); + + test("valid content still parses after a malformed parse", () => { + const bad = "---\ntags: [#malformed, #unique-c]\n---\nBody"; + expect(() => parseFrontmatter(bad)).toThrow(); + const good = parseFrontmatter("---\ntitle: Fine\n---\nBody"); + expect(good.properties.title).toBe("Fine"); + }); }); describe("setProperty", () => { diff --git a/src/utils/frontmatter.ts b/src/utils/frontmatter.ts index a5923d3..8555aed 100644 --- a/src/utils/frontmatter.ts +++ b/src/utils/frontmatter.ts @@ -1,5 +1,24 @@ import matter from "gray-matter"; +// Runtime-only API not present in gray-matter's type declarations. +const matterCache = matter as unknown as { clearCache: () => void }; + +/** + * gray-matter caches the file object keyed by content BEFORE parsing it, so + * a failed parse leaves a poisoned entry: the next parse of a byte-identical + * string silently returns the cached, unparsed object (empty data, no error). + * Evict the cache when parsing throws so every parse of malformed content + * fails deterministically. + */ +function safeMatter(content: string): matter.GrayMatterFile { + try { + return matter(content); + } catch (err) { + matterCache.clearCache(); + throw err; + } +} + export interface ParsedFrontmatter { properties: Record; body: string; @@ -10,7 +29,7 @@ export interface ParsedFrontmatter { * Parse YAML frontmatter from markdown content. */ export function parseFrontmatter(content: string): ParsedFrontmatter { - const result = matter(content); + const result = safeMatter(content); return { properties: result.data, body: result.content, @@ -26,7 +45,7 @@ export function setProperty( name: string, value: unknown, ): string { - const result = matter(content); + const result = safeMatter(content); const data = { ...result.data, [name]: value }; return matter.stringify(result.content, data); } @@ -35,7 +54,7 @@ export function setProperty( * Remove a property from frontmatter. */ export function removeProperty(content: string, name: string): string { - const result = matter(content); + const result = safeMatter(content); const data = { ...result.data }; delete data[name]; // If no properties left, return just the body diff --git a/src/utils/overview-cache.test.ts b/src/utils/overview-cache.test.ts new file mode 100644 index 0000000..3ae08de --- /dev/null +++ b/src/utils/overview-cache.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { getOverview } from "../core/overview.js"; +import { createTempVault } from "./test-helpers.js"; + +/** + * The overview cache stores the final VaultOverview keyed by a whole-vault + * mtime fingerprint plus the resolved options. These tests observe caching + * strictly through getOverview behavior: + * - a frozen-mtime content rewrite must be INVISIBLE (cache hit, no re-read) + * - any mtime bump, file add, or file remove must be VISIBLE (recompute) + * - changed options must never be served another variant's cached result + */ + +const NOTE_A = "# Alpha\nkubernetes ingress routing policies cluster"; +const NOTE_A2 = "# Alpha\nsourdough fermentation hydration schedules levain"; +const NOTE_B = "# Beta\npayroll withholding contractors ledger invoices"; + +const FIXED_TIME = new Date("2024-06-01T12:00:00Z"); + +function freezeMtime(p: string): void { + fs.utimesSync(p, FIXED_TIME, FIXED_TIME); +} + +describe("overview cache", () => { + test("writes a cache file on first run", () => { + const vault = createTempVault({ "notes/a.md": NOTE_A }); + try { + getOverview(vault.vaultPath, vault.vaultPath); + expect( + fs.existsSync(path.join(vault.vaultPath, "overview-cache.json")), + ).toBe(true); + } finally { + vault.cleanup(); + } + }); + + test("cache hit: frozen-mtime rewrite is invisible", () => { + const vault = createTempVault({ "notes/a.md": NOTE_A }); + const notePath = path.join(vault.vaultPath, "notes/a.md"); + try { + freezeMtime(notePath); + const first = getOverview(vault.vaultPath, vault.vaultPath); + expect(first.overview[0].keywords).toContain("kubernetes"); + + // rewrite content but keep the identical mtime → fingerprint unchanged + fs.writeFileSync(notePath, NOTE_A2); + freezeMtime(notePath); + + const second = getOverview(vault.vaultPath, vault.vaultPath); + expect(second).toEqual(first); // served from cache, file not re-read + } finally { + vault.cleanup(); + } + }); + + test("mtime bump invalidates", () => { + const vault = createTempVault({ "notes/a.md": NOTE_A }); + const notePath = path.join(vault.vaultPath, "notes/a.md"); + try { + freezeMtime(notePath); + const first = getOverview(vault.vaultPath, vault.vaultPath); + expect(first.overview[0].keywords).toContain("kubernetes"); + + fs.writeFileSync(notePath, NOTE_A2); + const later = new Date(FIXED_TIME.getTime() + 5000); + fs.utimesSync(notePath, later, later); + + const second = getOverview(vault.vaultPath, vault.vaultPath); + expect(second.overview[0].keywords).toContain("sourdough"); + expect(second.overview[0].keywords).not.toContain("kubernetes"); + } finally { + vault.cleanup(); + } + }); + + test("added and removed files invalidate", () => { + const vault = createTempVault({ "notes/a.md": NOTE_A }); + const bPath = path.join(vault.vaultPath, "notes/b.md"); + try { + const first = getOverview(vault.vaultPath, vault.vaultPath); + expect(first.overview[0].notes).toBe(1); + + fs.writeFileSync(bPath, NOTE_B); + const second = getOverview(vault.vaultPath, vault.vaultPath); + expect(second.overview[0].notes).toBe(2); + + fs.rmSync(bPath); + const third = getOverview(vault.vaultPath, vault.vaultPath); + expect(third.overview[0].notes).toBe(1); + } finally { + vault.cleanup(); + } + }); + + test("different options are never served another variant's cache", () => { + const vault = createTempVault({ + "notes/a.md": + "# Alpha\nkubernetes ingress routing policies cluster autoscaling telemetry dashboards", + }); + try { + const five = getOverview(vault.vaultPath, vault.vaultPath, { + keywords: 5, + }); + expect(five.overview[0].keywords.length).toBe(5); + + const three = getOverview(vault.vaultPath, vault.vaultPath, { + keywords: 3, + }); + expect(three.overview[0].keywords.length).toBe(3); + + // and back: no stale first variant either + const fiveAgain = getOverview(vault.vaultPath, vault.vaultPath, { + keywords: 5, + }); + expect(fiveAgain.overview[0].keywords.length).toBe(5); + } finally { + vault.cleanup(); + } + }); + + test("corrupted cache file is ignored and rebuilt", () => { + const vault = createTempVault({ "notes/a.md": NOTE_A }); + const cachePath = path.join(vault.vaultPath, "overview-cache.json"); + try { + getOverview(vault.vaultPath, vault.vaultPath); + fs.writeFileSync(cachePath, "{not json!!"); + + const result = getOverview(vault.vaultPath, vault.vaultPath); + expect(result.overview[0].keywords).toContain("kubernetes"); + // cache restored to a valid state + const raw = JSON.parse(fs.readFileSync(cachePath, "utf-8")); + expect(typeof raw.fingerprint).toBe("string"); + } finally { + vault.cleanup(); + } + }); + + test("cached result includes context and warnings", () => { + const vault = createTempVault({ + "NAPKIN.md": "# Context note", + "notes/a.md": NOTE_A, + "notes/bad.md": "---\ntags: [#broken, #cache-test]\n---\n# Bad", + }); + try { + const first = getOverview(vault.vaultPath, vault.vaultPath); + const second = getOverview(vault.vaultPath, vault.vaultPath); + expect(second.context).toBe("# Context note"); + expect(second.warnings).toEqual([ + "Skipping notes/bad.md (malformed YAML frontmatter)", + ]); + expect(second).toEqual(first); + } finally { + vault.cleanup(); + } + }); +}); diff --git a/src/utils/overview-cache.ts b/src/utils/overview-cache.ts new file mode 100644 index 0000000..57d19fb --- /dev/null +++ b/src/utils/overview-cache.ts @@ -0,0 +1,43 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +const CACHE_FILE = "overview-cache.json"; + +export interface OverviewCacheData { + /** Whole-vault fingerprint (file paths + mtimes), see computeFingerprint. */ + fingerprint: string; + /** Resolved options the result was computed with (depth, keywords, ...). */ + optionsKey: string; + result: T; +} + +/** + * Load the cached overview result if both the vault fingerprint and the + * resolved options match. Returns null on miss, mismatch, or corruption. + * + * Single-entry cache, same trade-off as the search cache: the stored result + * is a few KB, and the dominant call pattern (agents re-running `napkin + * overview` with default options between reads) hits one variant. + */ +export function loadOverviewCache( + configPath: string, + fingerprint: string, + optionsKey: string, +): T | null { + try { + const raw = fs.readFileSync(path.join(configPath, CACHE_FILE), "utf-8"); + const data: OverviewCacheData = JSON.parse(raw); + if (data.fingerprint !== fingerprint) return null; + if (data.optionsKey !== optionsKey) return null; + return data.result; + } catch { + return null; + } +} + +export function saveOverviewCache( + configPath: string, + data: OverviewCacheData, +): void { + fs.writeFileSync(path.join(configPath, CACHE_FILE), JSON.stringify(data)); +} diff --git a/src/utils/search-cache.test.ts b/src/utils/search-cache.test.ts index ac8d51b..40b56c1 100644 --- a/src/utils/search-cache.test.ts +++ b/src/utils/search-cache.test.ts @@ -1,6 +1,8 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import * as fs from "node:fs"; import * as path from "node:path"; +import MiniSearch from "minisearch"; +import { searchVault } from "../core/search.js"; import { computeFingerprint, loadSearchCache, @@ -115,3 +117,49 @@ describe("saveSearchCache / loadSearchCache", () => { expect(loaded).toBeNull(); }); }); + +describe("minisearch cache migration", () => { + test("a cache blob written by minisearch loads and searches identically", () => { + // Vaults in the wild have search-cache.json blobs serialized by + // minisearch (napkin < ferrosearch swap). ferrosearch reads the same + // version-2 format, so old caches must keep working without a rebuild. + const fresh = searchVault(vault.vaultPath, vault.vaultPath, "alpha"); + expect(fresh.length).toBeGreaterThan(0); + + const files = ["README.md", "Projects/alpha.md", "Projects/beta.md"]; + const legacy = new MiniSearch({ + fields: ["basename", "content"], + storeFields: ["file"], + searchOptions: { boost: { basename: 2 }, fuzzy: 0.2, prefix: true }, + }); + legacy.addAll( + files.map((file, id) => ({ + id, + file, + basename: path.basename(file, ".md"), + content: fs.readFileSync(path.join(vault.vaultPath, file), "utf-8"), + })), + ); + + saveSearchCache(vault.vaultPath, { + fingerprint: computeFingerprint(vault.vaultPath), + index: JSON.stringify(legacy), // the old serialization call + docs: files.map((file, id) => ({ + id, + file, + basename: path.basename(file, ".md"), + mtime: fs.statSync(path.join(vault.vaultPath, file)).mtimeMs, + })), + backlinkCounts: {}, + }); + + const fromLegacyCache = searchVault( + vault.vaultPath, + vault.vaultPath, + "alpha", + ); + expect(fromLegacyCache.map((r) => [r.file, r.score])).toEqual( + fresh.map((r) => [r.file, r.score]), + ); + }); +});