Overview perf + caching, gray-matter fix, ferrosearch search engine

.gitignore

@@ -33,3 +33,4 @@ bench/results/
 # Vault (local content)
 .napkin/
 thread-draft.md
+*.bun-build

CLAUDE.md

@@ -22,6 +22,8 @@ bun run check                # Biome lint + format
 - `src/utils/frontmatter.ts` — YAML frontmatter parse/set/remove
 - `src/utils/config.ts` — Unified config (load/save/update, syncs to .obsidian/)
 - `src/utils/markdown.ts` — Extract headings, tasks, tags, links from markdown
+- `src/utils/search-cache.ts` — Search index cache + whole-vault mtime fingerprint
+- `src/utils/overview-cache.ts` — Overview result cache (fingerprint + options key)
 - `.pi/extensions/napkin-context/` — Pi extension: injects vault overview into system prompt
 - `.pi/extensions/distill/` — Pi extension: auto-distills conversations into vault
 
@@ -31,8 +33,10 @@ bun run check                # Biome lint + format
 
 ```
 project/
-  .napkin/                  # napkin config
+  .napkin/                  # napkin config + caches
     config.json             # Unified config (syncs to .obsidian/)
+    search-cache.json       # Auto-managed, safe to delete
+    overview-cache.json     # Auto-managed, safe to delete
   .obsidian/                # Obsidian config (auto-generated)
   NAPKIN.md                 # Level 0 context note
   decisions/                # Template-defined dirs
@@ -45,7 +49,9 @@ project/
 
 - **Output triple**: Every command supports `--json`, `--quiet`, and human-readable output
 - **Vault auto-detect**: Walks up from cwd looking for `.napkin/` directory
-- **`.napkin/` is config only**: Vault content lives in the project root, `.napkin/` holds `config.json`
+- **`.napkin/` holds config + caches**: Vault content lives in the project root; `.napkin/` holds `config.json` and the auto-managed search/overview caches
+- **Fingerprint caching**: search and overview cache their results keyed by vault file mtimes — any file change invalidates; caches are safe to delete
+- **Search engine**: `@shift-labs/ferrosearch` — native, MiniSearch-compatible (index format included); `minisearch` remains a devDependency as the cache-migration test oracle
 - **File resolution**: positional `<file>` or `--file` resolves by name (like wikilinks), `--path` requires exact path from vault root
 - **No Obsidian dependency**: Pure file-system operations on markdown files
 - **Progressive disclosure**: overview → search → read (4 levels, L0-L3)

README.md

@@ -8,6 +8,11 @@ Every great idea started on a napkin.
 npm install -g napkin-ai
 ```
 
+Search runs on [ferrosearch](https://github.com/shift-labs-ai/ferrosearch),
+a native MiniSearch-compatible engine with prebuilt binaries for macOS
+(arm64, x64) and Linux (x64, arm64, glibc and musl). Other platforms build
+it from source with Rust installed.
+
 ---
 
 ## Quick Start
@@ -113,8 +118,9 @@ See [`bench/README.md`](bench/README.md) for details and usage.
 
 ```
 my-project/
-  .napkin/                  # napkin config
+  .napkin/                  # napkin config + caches
     config.json             # Unified config (syncs to .obsidian/)
+    *-cache.json            # Search/overview caches (auto-managed)
   .obsidian/                # Obsidian config (auto-generated)
   NAPKIN.md                 # Context note (Level 0)
   decisions/                # Template-defined directories

docs/configuration.md

@@ -18,6 +18,7 @@ napkin config set --key search.limit --value 50
 |-----|---------|-------------|
 | `overview.depth` | `3` | Max folder depth in vault map |
 | `overview.keywords` | `8` | Max TF-IDF keywords per folder |
+| `overview.collapse` | `true` | Roll up numerous, lexically similar sibling folders into one row |
 
 ### search
 
@@ -64,8 +65,13 @@ CLI flags > `config.json` > hardcoded defaults
 ```
 project/
   .napkin/
-    config.json       # This file
-    .obsidian/        # Auto-synced from config.json
+    config.json            # This file
+    search-cache.json      # Search index cache (auto-managed)
+    overview-cache.json    # Overview result cache (auto-managed)
+    .obsidian/             # Auto-synced from config.json
 ```
 
+The cache files are keyed by a fingerprint of vault file mtimes and rebuild
+automatically; they are safe to delete at any time.
+
 Config is created on first `napkin config set` or `napkin init`. If the file doesn't exist, defaults are used.

docs/overview-keyword-extraction.md

@@ -5,9 +5,12 @@ The `napkin overview` command generates a vault-wide index by extracting distinc
 ## Pipeline
 
 ```
-Files → Group by folder → Collect weighted text → Build TF → Compute IDF across folders → Score TF-IDF → Deduplicate bigrams → Top N keywords
+Files → Group by folder → Collect weighted text → Build TF → Collapse homogeneous siblings → Compute IDF across folders → Score TF-IDF → Deduplicate bigrams → Top N keywords
 ```
 
+The whole pipeline runs behind an mtime-fingerprint cache (see
+[Caching](#caching)); a cache hit skips everything below.
+
 ### 1. Text Collection & Weighting
 
 Not all text is equal. Sources are weighted by signal strength:
@@ -17,15 +20,21 @@ Not all text is equal. Sources are weighted by signal strength:
 | Headings | 3x | Curated by the author, high intent |
 | Filenames | 2x | Chosen names are strong signals |
 | Frontmatter title | 2x | Explicit metadata |
+| Other frontmatter values | 2x | Explicit metadata (wikilink-only and date values excluded) |
 | Body text | 1x | Bulk content, noisier |
 
 ### 2. Noise Stripping
 
 Before tokenization, we strip:
-- URLs (`https://...`)
-- Emails
+- URLs (`https://...`) and emails
 - Code blocks (fenced and inline)
-- Hex hashes (commit SHAs, etc.)
+- HTML tags and entities (residue of converted documents)
+- Hex hashes (commit SHAs), dashed GUIDs (`AAAA1111-2222-...`), and
+  mixed-digit blobs (`INV20240915X`) — ID shrapnel from OCR'd PDFs and
+  DocuSign-style exports that would otherwise pollute keywords
+
+Each pattern only runs when a cheap necessary condition holds (an email
+needs `@`, a URL needs `http`), so clean prose skips the regex scans.
 
 ### 3. Tokenization
 
@@ -38,7 +47,16 @@ Two-word phrases are extracted alongside unigrams. Bigrams are kept only if:
 - They appear **2+ times** in the folder (otherwise likely noise)
 - The two words are **not identical** (filters "tbd tbd" type garbage)
 
-### 5. TF-IDF Scoring
+### 5. Homogeneous-Sibling Collapse
+
+Parents with 5+ children whose body-term distributions are lexically
+similar (mean pairwise cosine ≥ 0.15 over top terms) are rendered as one
+aggregate row — `imports/ (+6 similar subfolders)` — so imported document
+dumps don't drown the overview. Similarity uses body text only, so shared
+filename conventions cannot fake content homogeneity. Top-level folders
+never collapse into the root. Disable with `--no-collapse`.
+
+### 6. TF-IDF Scoring
 
 Each folder is treated as a "document":
 
@@ -47,16 +65,31 @@ Each folder is treated as a "document":
 
 The `1 +` dampening prevents over-penalizing terms that appear in a few folders. A word in 3 out of 9 folders still gets reasonable weight, while a word in all 9 gets suppressed.
 
-### 6. Bigram Deduplication
+Candidates are also filtered for corroboration — a term must appear outside
+headings or in 2+ heading lines (single heading-only terms are usually
+section labels like "Notes") — and terms matching the folder's own name
+(including singular/plural variants) are excluded as redundant.
+
+### 7. Bigram Deduplication
 
 When a bigram is selected (e.g., "knowledge base"), its constituent unigrams ("knowledge", "base") are suppressed from the results. This prevents redundant keyword slots.
 
+## Caching
+
+The final result is cached in `.napkin/overview-cache.json`, keyed by a
+whole-vault fingerprint (file paths + mtimes) plus the resolved options.
+Any file add, remove, or touch — including `NAPKIN.md` — invalidates it;
+so does changing `depth`, `keywords`, or `collapse`. A cache hit costs one
+stat pass instead of reading and tokenizing every note (~25ms vs ~1s on a
+5,000-note vault). Corrupt cache files are ignored and rebuilt.
+
 ## Configuration
 
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--keywords <n>` | 8 | Max keywords per folder |
 | `--depth <n>` | 3 | Max folder depth to index |
+| `--no-collapse` | collapse on | Disable homogeneous-sibling collapse |
 
 ## Example
 

package.json

@@ -52,14 +52,15 @@
     "@types/bun": "latest",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^25.6.0",
+    "minisearch": "^7.2.0",
     "typescript": "^5.8.0"
   },
   "dependencies": {
+    "@shift-labs/ferrosearch": "^0.1.1",
     "chalk": "^5.6.2",
     "commander": "^14.0.3",
     "gray-matter": "^4.0.3",
     "jexl": "^2.3.0",
-    "minisearch": "^7.2.0",
     "sql.js": "^1.14.0"
   },
   "optionalDependencies": {

src/commands/overview.test.ts

@@ -110,6 +110,57 @@ describe("overview", () => {
     vault.cleanup();
   });
 
+  test("warns for every file with identical malformed frontmatter", async () => {
+    // Regression: gray-matter's parse cache is poisoned by a failed parse,
+    // so a second file with byte-identical malformed frontmatter used to
+    // silently parse as empty data — no warning, wrong tags/keywords.
+    const badContent =
+      "---\ntags: [#twin, #copies]\n---\n# Twin\nIdentical broken note.";
+    const vault = createTempVault({
+      "a/bad.md": badContent,
+      "b/bad.md": badContent,
+    });
+
+    const warnings: string[] = [];
+    const captured: unknown[] = [];
+    const origLog = console.log;
+    const origError = console.error;
+
+    try {
+      console.error = (...args: unknown[]) => {
+        const msg = args.map(String).join(" ");
+        if (msg.includes("⚠")) warnings.push(msg);
+      };
+      console.log = (...args: unknown[]) => {
+        captured.push(...args);
+      };
+      await overview({
+        vault: vault.path,
+        json: true,
+        quiet: false,
+        copy: false,
+      });
+    } finally {
+      console.log = origLog;
+      console.error = origError;
+    }
+
+    expect(warnings.length).toBe(2);
+    expect(warnings.join("\n")).toContain("a/bad.md");
+    expect(warnings.join("\n")).toContain("b/bad.md");
+
+    // and neither file leaks tags from the unparsed frontmatter
+    const result = JSON.parse(captured[0] as string) as {
+      overview: Array<{ path: string; tags: string[] }>;
+    };
+    for (const folder of result.overview) {
+      expect(folder.tags).not.toContain("twin");
+      expect(folder.tags).not.toContain("copies");
+    }
+
+    vault.cleanup();
+  });
+
   test("empty vault", async () => {
     const vault = createTempVault({});