feat(core): make ingestion resumable

README.md

@@ -43,8 +43,10 @@ citation before you propose an edit.
 ```
 
 `rgr setup` creates ignored local state under `.ragmir/`, installs project-scoped native skills,
-and writes local MCP helpers. `rgr ingest` is incremental. The agent receives bounded passages with
-the source path, excerpt, chunk, line range, and PDF page when one is available.
+and writes local MCP helpers. `rgr ingest` is incremental and commits resumable batches of 25 files
+by default. Re-run the same command after an interruption to continue from the last committed
+batch. The agent receives bounded passages with the source path, excerpt, chunk, line range, and
+PDF page when one is available.
 
 Prefer a direct search? Run:
 
@@ -157,6 +159,18 @@ pnpm exec rgr ingest --rebuild
 The default `local-hash` provider is offline lexical/hash retrieval. Semantic mode uses
 Transformers.js and requires an explicit model download or a preloaded local model.
 
+### Resume a long ingestion
+
+```bash
+pnpm exec rgr ingest --batch-size 25
+pnpm exec rgr status --json
+```
+
+Ragmir records per-file progress atomically under ignored `.ragmir/storage/` state. Files from a
+committed batch are not parsed or embedded again after a restart. A full `--rebuild` writes to an
+isolated generation and activates it only after row and manifest validation, so an interrupted
+rebuild leaves the previous searchable index active.
+
 ### Search scanned PDFs
 
 ```bash

context7.json

@@ -26,6 +26,7 @@
     "Qwen and Gemma are optional Ragmir Chat profiles, never requirements of Core, the CLI, the TypeScript API, or MCP.",
     "The `local-hash` embedding provider (default) is a lexical sha256 embedding, not semantic; use `transformers` for semantic retrieval.",
     "Switching `embeddingProvider` requires `rgr ingest --rebuild`, since the two providers produce incompatible vectors.",
+    "Ingestion commits resumable file batches under ignored `.ragmir/storage/` state; use `rgr status --json` for progress, and expect `--rebuild` to preserve the active index until a staged generation is validated.",
     "Run `rgr doctor --fix` after upgrading or misconfiguration to repair scaffolding, `.gitignore` entries, and the agent skill install.",
     "Config resolves from the caller's working directory (`.ragmir/config.json`), never from the package install path.",
     "In a monorepo, the nearest configured ancestor is active; use `rgr bases --json` or explicit `--project-root` to verify routing.",

docs/api-reference.md

@@ -101,6 +101,7 @@ authentication, authorization, rate limits, and transport security.
 | Export | Purpose |
 | --- | --- |
 | `ingest(options?)` | Incrementally parse, redact, chunk, embed, and store selected files. |
+| `getIngestionProgress(config)` | Read durable progress for the latest ingestion run. |
 | `audit(cwd?)` | Compare files on disk with the current index. |
 | `previewChunks(options?)` | Return redacted chunks and distributions without writing an index. |
 | `search(query, options?)` | Return ranked cited passages. |
@@ -112,8 +113,10 @@ authentication, authorization, rate limits, and transport security.
 | `evaluateGoldenQueries(options)` | Score retrieval against a local golden-query file. |
 
 `SearchOptions` accepts `cwd`, `topK`, `contextRadius`, `includePaths`, `excludePaths`,
-`contextPaths`, `explain`, `signal`, and `timeoutMs`. `IngestOptions`, `ResearchOptions`, and
-`ExpandCitationOptions` also accept `signal` and `timeoutMs`. When explanation is enabled, each
+`contextPaths`, `explain`, `signal`, and `timeoutMs`. `IngestOptions` also accepts `rebuild`, a
+positive `batchSize` that defaults to 25 files, and an optional `onProgress` callback. Its durable
+progress contains the run ID, resume flag, last activity, chunk count, and per-stage file counts.
+`IngestOptions`, `ResearchOptions`, and `ExpandCitationOptions` accept `signal` and `timeoutMs`. When explanation is enabled, each
 result includes reciprocal-rank fusion contributions, one-based vector and lexical ranks, vector
 distance, lexical backend score, and matched query terms. `ExpandCitationOptions.contextRadius` is
 clamped to three chunks.
@@ -182,7 +185,7 @@ types that callers commonly compose explicitly.
 | Area | Exported types |
 | --- | --- |
 | Configuration | `Config`, `PrivacyProfile`, `RetrievalProfile` |
-| Ingestion | `IngestOptions`, `IngestResult`, `AuditReport`, `ChunkStats`, `IngestionLimitsReport`, `IndexManifest`, `IndexManifestFile`, `ParsedPage` |
+| Ingestion | `IngestOptions`, `IngestResult`, `IngestionProgress`, `IngestionFileStage`, `IngestionRunMode`, `IngestionRunStatus`, `AuditReport`, `ChunkStats`, `IngestionLimitsReport`, `IndexManifest`, `IndexManifestFile`, `ParsedPage` |
 | Preview | `PreviewChunksOptions`, `PreviewReport`, `PreviewFile`, `PreviewChunk` |
 | Retrieval | `SearchOptions`, `SearchResult`, `SearchContextChunk`, `SearchScoreExplanation`, `AskResult`, `CompactSearchResult`, `ExpandCitationOptions`, `ExpandedCitation` |
 | Research and evaluation | `ResearchOptions`, `ResearchReport`, `ResearchEvidence`, `CodeEvidence`, `SourceDiagnostics`, `SourceDuplicateCandidate`, `SourcePathCandidate`, `EvaluationOptions`, `EvaluationResult`, `EvaluationCaseResult`, `GoldenQuery` |

docs/cli-reference.md

@@ -17,13 +17,13 @@ rgr search "release decision"
 | `init` | Create basic local configuration only. |
 | `doctor [--fix]` | Check setup, index freshness, and safe repairs. |
 | `preview` | Parse, redact, and chunk selected sources without writing the index. |
-| `ingest [--rebuild]` | Index configured sources; rebuild after provider or chunking changes. |
+| `ingest [--rebuild] [--batch-size N]` | Index configured sources in resumable batches; rebuild after provider or chunking changes. |
 | `search <query>` | Return ranked cited passages. |
 | `ask <query>` | Return cited context without model synthesis. |
 | `research <query>` | Run an audit-backed multi-query retrieval pass. |
 | `audit [--unsupported]` | Compare sources with the index and list skipped files. |
 | `bases` | List root and nested monorepo bases and mark the active one. |
-| `status` | Show configuration and indexed chunk count. |
+| `status` | Show configuration, indexed chunk count, and the latest ingestion progress. |
 | `security-audit [--strict]` | Check local privacy and Git-ignore posture. |
 
 ## Sources and retrieval
@@ -46,6 +46,28 @@ agent context is limited.
 `preview` uses the active redaction and chunking configuration but never writes storage. `audit`
 reports min, mean, p50, p95, and max chunk sizes plus structural-context coverage.
 
+## Resumable ingestion
+
+```bash
+rgr ingest
+rgr status --json
+rgr ingest --batch-size 10
+```
+
+The default batch contains 25 files. After each batch, Ragmir atomically records per-file state and
+the current manifest under `.ragmir/storage/`. Starting `rgr ingest` again resumes a compatible
+interrupted run and processes only pending, failed, or changed files. Files already committed to
+the index are not parsed or embedded again.
+
+`rgr status --json` exposes the run ID, mode, status, resume flag, last activity, batch size, chunk
+count, and file counts for `pending`, `parsed`, `embedded`, `indexed`, and `error` states. The human
+output shows the same progress in a compact form.
+
+`rgr ingest --rebuild` writes batches into an isolated LanceDB generation. The existing index stays
+active until the new table and manifest pass row-count, checksum, and duplicate-ID validation. The
+final atomic manifest replacement activates the generation. Re-run the command after interruption
+to resume the staged generation.
+
 ## Monorepos
 
 ```bash

docs/troubleshooting.md

@@ -16,6 +16,16 @@ Run `rgr setup`. It creates `.ragmir/config.json`, local ignore rules, and optio
 
 Check `sources` with `rgr sources list`, then run `rgr ingest`. Use `rgr audit` to compare supported files with the index. Use `rgr ingest --rebuild` after changing embedding provider, model, or chunking.
 
+## Ingestion was interrupted
+
+Run `rgr status --json`, then start `rgr ingest` again. A compatible run resumes from its last
+committed file batch. Files in `parsed` or `embedded` state without a committed index write are
+retried; files already in `indexed` state are not parsed or embedded again. If source checksums or
+the indexing policy changed, Ragmir starts a new safe run instead.
+
+An interrupted `rgr ingest --rebuild` leaves the previous complete index active. Re-run the rebuild
+to continue its isolated generation.
+
 ## A PDF or image has no text
 
 `rgr ingest --json` reports `emptyTextFiles`. For scanned PDFs, run:

llms.txt

@@ -9,6 +9,9 @@ store.
 
 - Install: `pnpm add -D @jcode.labs/ragmir`
 - First use: `pnpm exec rgr setup && pnpm exec rgr ingest`
+- Resumable ingestion: `rgr ingest` commits 25 files per durable batch by default. Re-run it after an
+  interruption, inspect progress with `rgr status --json`, and use `--batch-size` to change the
+  checkpoint size. Rebuilds stay isolated until final manifest validation and activation.
 - CLI: `rgr preview`, `rgr search "query" --explain`, `rgr bases`, `rgr doctor`, `rgr audit`, and
   `rgr security-audit`
 - Core behavior: cited retrieval only. `ask` returns evidence, while synthesis belongs to the calling

packages/ragmir-core/README.md

@@ -84,8 +84,10 @@ npx rgr search "Which decision changed the rollout?"
 ```
 
 The project owns an ignored `.ragmir/` directory containing configuration and generated local
-state. Ingestion is incremental, and every result identifies the source path, chunk, line range, and
-PDF page when one is available.
+state. Ingestion is incremental and commits 25 files per resumable batch by default. Re-run the
+same command after an interruption, or inspect `npx rgr status --json` for the current run ID,
+counts, last activity, and resume flag. Every result identifies the source path, chunk, line range,
+and PDF page when one is available.
 
 ## CLI essentials
 
@@ -107,6 +109,10 @@ npx rgr research "deployment obligations" --compact
 npx rgr search "deployment decision" --json
 ```
 
+Use `npx rgr ingest --batch-size 10` when a smaller durable checkpoint is useful. Full rebuilds use
+an isolated local generation and switch the active manifest only after validation, so a failed or
+interrupted rebuild does not replace the last healthy index.
+
 `ask` returns cited retrieval context, not LLM synthesis. `research` performs an audit-backed,
 multi-query retrieval pass and reports missing or weak evidence.
 

packages/ragmir-core/src/cli.ts

@@ -25,6 +25,7 @@ import { evaluateGoldenQueries } from "./evaluate.js"
 import { countSkippedByReason } from "./files.js"
 import { getIndexFreshnessWarning, getLexicalScanWarning } from "./index-diagnostics.js"
 import { audit, ingest } from "./ingest.js"
+import { getIngestionProgress } from "./ingestion-state.js"
 import { initProject } from "./init.js"
 import { discoverKnowledgeBases, knowledgeBaseIdentity } from "./knowledge-bases.js"
 import { ingestionLimits } from "./limits.js"
@@ -377,46 +378,53 @@ program
   .command("ingest")
   .description("Parse changed documents, redact, chunk, embed locally, and update LanceDB.")
   .option("--rebuild", "Force a full local index rebuild instead of reusing unchanged rows.")
+  .option("--batch-size <number>", "Files committed per resumable batch.", parsePositiveInt)
   .option("--json", "Print machine-readable JSON.")
-  .action(async (options: { rebuild?: boolean; json?: boolean }, command: Command) => {
-    const cwd = projectRoot(command)
-    const ingestOptions: Parameters<typeof ingest>[0] = { cwd }
-    addOption(ingestOptions, "rebuild", options.rebuild)
-    const result = await ingest(ingestOptions)
-    if (options.json) {
-      console.log(JSON.stringify(result, null, 2))
-      if (result.errors.length > 0) {
-        process.exitCode = 1
+  .action(
+    async (
+      options: { rebuild?: boolean; batchSize?: number; json?: boolean },
+      command: Command,
+    ) => {
+      const cwd = projectRoot(command)
+      const ingestOptions: Parameters<typeof ingest>[0] = { cwd }
+      addOption(ingestOptions, "rebuild", options.rebuild)
+      addOption(ingestOptions, "batchSize", options.batchSize)
+      const result = await ingest(ingestOptions)
+      if (options.json) {
+        console.log(JSON.stringify(result, null, 2))
+        if (result.errors.length > 0) {
+          process.exitCode = 1
+        }
+        return
       }
-      return
-    }
 
-    console.log(
-      pc.green(
-        `Done. discoveredFiles=${result.discoveredFiles} supportedFiles=${result.supportedFiles} supportedBytes=${result.supportedBytes} largestFileBytes=${result.largestFileBytes} indexedFiles=${result.indexedFiles} rebuiltFiles=${result.rebuiltFiles} reusedFiles=${result.reusedFiles} chunks=${result.chunks} skippedFiles=${result.skippedFiles} unsupportedFiles=${result.unsupportedFiles} oversizedFiles=${result.oversizedFiles} sensitiveFiles=${result.sensitiveFiles} emptyTextFiles=${result.emptyTextFiles.length} redactions=${result.redactions} errors=${result.errors.length}`,
-      ),
-    )
-    printUnsupportedSummary(result.unsupportedExtensions)
-    printEmptyTextFiles(result.emptyTextFiles)
-    if (result.vectorIndexWarning) {
-      console.log(pc.yellow(result.vectorIndexWarning))
-    }
-    if (result.lexicalIndexWarning) {
-      console.log(pc.yellow(result.lexicalIndexWarning))
-    }
-    if (result.unsupportedFiles > 0 || result.oversizedFiles > 0 || result.sensitiveFiles > 0) {
-      const auditCommand = await rgrCommand(cwd, ["audit", "--unsupported"])
       console.log(
-        pc.yellow(`Some files were not indexed. Run \`${auditCommand.display}\` for details.`),
+        pc.green(
+          `Done. runId=${result.runId} resumed=${result.resumed} batchSize=${result.batchSize} discoveredFiles=${result.discoveredFiles} supportedFiles=${result.supportedFiles} supportedBytes=${result.supportedBytes} largestFileBytes=${result.largestFileBytes} indexedFiles=${result.indexedFiles} rebuiltFiles=${result.rebuiltFiles} reusedFiles=${result.reusedFiles} chunks=${result.chunks} skippedFiles=${result.skippedFiles} unsupportedFiles=${result.unsupportedFiles} oversizedFiles=${result.oversizedFiles} sensitiveFiles=${result.sensitiveFiles} emptyTextFiles=${result.emptyTextFiles.length} redactions=${result.redactions} errors=${result.errors.length}`,
+        ),
       )
-    }
-    for (const error of result.errors) {
-      console.error(pc.red(`  - ${error.path}: ${error.message}`))
-    }
-    if (result.errors.length > 0) {
-      process.exitCode = 1
-    }
-  })
+      printUnsupportedSummary(result.unsupportedExtensions)
+      printEmptyTextFiles(result.emptyTextFiles)
+      if (result.vectorIndexWarning) {
+        console.log(pc.yellow(result.vectorIndexWarning))
+      }
+      if (result.lexicalIndexWarning) {
+        console.log(pc.yellow(result.lexicalIndexWarning))
+      }
+      if (result.unsupportedFiles > 0 || result.oversizedFiles > 0 || result.sensitiveFiles > 0) {
+        const auditCommand = await rgrCommand(cwd, ["audit", "--unsupported"])
+        console.log(
+          pc.yellow(`Some files were not indexed. Run \`${auditCommand.display}\` for details.`),
+        )
+      }
+      for (const error of result.errors) {
+        console.error(pc.red(`  - ${error.path}: ${error.message}`))
+      }
+      if (result.errors.length > 0) {
+        process.exitCode = 1
+      }
+    },
+  )
 
 program
   .command("preview")
@@ -970,6 +978,7 @@ program
     const config = await loadConfig(cwd)
     const identity = knowledgeBaseIdentity(config.projectRoot)
     const rows = await countRows(config)
+    const ingestion = await getIngestionProgress(config)
     const status = {
       knowledgeBaseId: identity?.id ?? null,
       projectRoot: config.projectRoot,
@@ -999,6 +1008,7 @@ program
       legacyWordCommand: config.legacyWordCommand,
       legacyWordTimeoutMs: config.legacyWordTimeoutMs,
       chunksIndexed: rows,
+      ingestion,
     }
     if (options.json) {
       console.log(JSON.stringify(status, null, 2))
@@ -1033,6 +1043,17 @@ program
     console.log(`legacyWordCommand=${config.legacyWordCommand.join(" ")}`)
     console.log(`legacyWordTimeoutMs=${config.legacyWordTimeoutMs}`)
     console.log(`chunksIndexed=${rows}`)
+    if (ingestion) {
+      console.log(`ingestionRunId=${ingestion.runId}`)
+      console.log(`ingestionStatus=${ingestion.status}`)
+      console.log(`ingestionMode=${ingestion.mode}`)
+      console.log(`ingestionResumed=${ingestion.resumed}`)
+      console.log(`ingestionBatchSize=${ingestion.batchSize}`)
+      console.log(
+        `ingestionProgress=${ingestion.indexedFiles}/${ingestion.totalFiles} indexed, ${ingestion.errorFiles} errors, ${ingestion.pendingFiles} pending`,
+      )
+      console.log(`ingestionLastActivityAt=${ingestion.lastActivityAt}`)
+    }
   })
 
 program

packages/ragmir-core/src/destroy.test.ts

@@ -4,6 +4,7 @@ import os from "node:os"
 import path from "node:path"
 import { afterEach, describe, expect, it } from "vitest"
 import { destroyIndex } from "./destroy.js"
+import { createIngestionRunState, writeIngestionState } from "./ingestion-state.js"
 import { testConfig } from "./test-support/config.js"
 
 const tempDirs: string[] = []
@@ -39,6 +40,28 @@ describe("destroyIndex", () => {
     expect(result.removed).toBe(false)
   })
 
+  it("removes storage from an interrupted first run with valid ingestion state", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-destroy-ingestion-state-"))
+    tempDirs.push(root)
+    const config = testConfig(root)
+    await writeIngestionState(
+      createIngestionRunState({
+        mode: "incremental",
+        tableName: config.tableName,
+        previousTableName: null,
+        policyFingerprint: "test-policy",
+        batchSize: 25,
+        files: [],
+        reusablePaths: new Set(),
+        reusableChunkCounts: new Map(),
+      }),
+      config,
+    )
+
+    await expect(destroyIndex(root)).resolves.toMatchObject({ removed: true })
+    expect(existsSync(config.storageDir)).toBe(false)
+  })
+
   it("writes a destroy-index access log entry", async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-destroy-log-"))
     tempDirs.push(root)
@@ -67,7 +90,7 @@ describe("destroyIndex", () => {
     await mkdir(config.storageDir, { recursive: true })
     await writeFile(path.join(config.storageDir, "unrelated.txt"), "keep", "utf8")
 
-    await expect(destroyIndex(root)).rejects.toThrow("does not contain index-manifest.json")
+    await expect(destroyIndex(root)).rejects.toThrow("contains neither index-manifest.json")
     expect(existsSync(config.storageDir)).toBe(true)
   })
 })