From 38c2424c07e8ab8d80995264579bd5101aea95c2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste THERY Date: Tue, 14 Jul 2026 18:06:57 +0700 Subject: [PATCH] feat(core): add persistent Node.js client (#112) * feat(core): add persistent Node.js client * docs: document persistent Node.js integrations * fix(landing): align npm version accessible label --- AGENTS.md | 3 + CLAUDE.md | 3 + README.md | 47 ++++--- context7.json | 2 + docs/agent-integration.md | 6 + docs/api-reference.md | 59 +++++++- docs/configuration.md | 5 + llms.txt | 4 + package.json | 3 + packages/ragmir-core/README.md | 40 ++++-- .../examples/library-api-demo/README.md | 38 +++--- .../examples/library-api-demo/run.mjs | 54 ++++---- packages/ragmir-core/package.json | 5 + packages/ragmir-core/src/client.test.ts | 94 +++++++++++++ packages/ragmir-core/src/client.ts | 128 ++++++++++++++++++ packages/ragmir-core/src/errors.ts | 38 ++++++ .../ragmir-core/src/index-write-lock.test.ts | 65 +++++++++ packages/ragmir-core/src/index-write-lock.ts | 58 ++++++++ packages/ragmir-core/src/index.ts | 7 +- packages/ragmir-core/src/ingest.ts | 46 ++++++- packages/ragmir-core/src/mcp.test.ts | 21 ++- packages/ragmir-core/src/mcp.ts | 112 ++++++++------- packages/ragmir-core/src/operation.ts | 60 ++++++++ packages/ragmir-core/src/query.ts | 52 ++++++- packages/ragmir-core/src/research.ts | 46 +++++-- packages/ragmir-core/src/store.ts | 116 +++++++++++----- packages/ragmir-core/src/types.ts | 13 +- packages/ragmir-landing/messages/en.json | 14 +- packages/ragmir-landing/messages/fr.json | 14 +- packages/ragmir-landing/public/ai.txt | 4 + packages/ragmir-landing/public/llms.txt | 4 + .../src/components/sections/faq.astro | 1 + .../src/components/sections/footer.astro | 2 +- .../src/pages/[...locale]/index.astro | 1 + tests/public-api-consumer/consumer.ts | 22 +++ 35 files changed, 982 insertions(+), 205 deletions(-) create mode 100644 packages/ragmir-core/src/client.test.ts create mode 100644 packages/ragmir-core/src/client.ts create mode 100644 packages/ragmir-core/src/errors.ts create mode 100644 packages/ragmir-core/src/index-write-lock.test.ts create mode 100644 packages/ragmir-core/src/index-write-lock.ts create mode 100644 packages/ragmir-core/src/operation.ts diff --git a/AGENTS.md b/AGENTS.md index 9d9a0f8..b7d0fbc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,9 @@ - `packages/ragmir-tts` is the optional local/offline audio add-on used by `rgr audio`. - `packages/ragmir-landing` is a self-contained, telemetry-free Astro site. Keep it static, open-source focused, and free of vendor deployment configuration. - Ragmir Core stays retrieval-first: `local-hash` supports offline retrieval, `transformers` is the explicit semantic option, and local chat remains a separate add-on. +- Long-running Node.js processes use one `RagmirClient` per project root and close it during shutdown. Keep the top-level API for one-shot scripts. +- Ragmir does not provide an HTTP server or fixed port. A network-facing host owns transport security, authentication, authorization, and rate limits. +- Ingestion is serialized per local index inside one Node.js process; do not claim a distributed writer lock. - Public copy must lead with model-agnostic Core and the choice between the user's preferred AI or automation and a fully local consumer. Qwen and Gemma are optional Chat profiles, never Core or MCP requirements. ## Privacy and ingestion diff --git a/CLAUDE.md b/CLAUDE.md index 132fca8..df101b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,6 +32,9 @@ native desktop shell, or cloud-vendor deployment configuration. Keep OCR optiona local, remote model downloads explicit, and normal confidential retrieval offline. Describe Core as model-agnostic: users can connect their preferred AI or automation, or keep the consumer local. Qwen and Gemma are optional Chat profiles, never Core or MCP requirements. +For repeated retrieval in a stateful Node.js process, use one `RagmirClient` per project root and +close it during shutdown. Ragmir does not provide an HTTP server or fixed port; network-facing hosts +own transport security, authentication, authorization, and rate limits. # GitNexus — Code Intelligence diff --git a/README.md b/README.md index ea85387..0a39390 100644 --- a/README.md +++ b/README.md @@ -60,13 +60,14 @@ owns the source files. | Interface | Use it for | Result | | --- | --- | --- | | `rgr` CLI | Setup, ingest, search, audit, and maintenance | Human-readable or JSON output | -| TypeScript API | Embed retrieval in a Node.js application | Typed results with citations | +| TypeScript API | Embed retrieval in a script or stateful Node.js worker | Typed results with citations and explicit lifecycle | | Local MCP server | Give your preferred agent bounded project context | Read-focused retrieval tools | | Ragmir Chat | Keep answer generation on the workstation | Cited offline synthesis | | Ragmir TTS | Turn a text brief into audio | Local WAV or explicit online MP3 | Use the CLI or MCP for interactive agent work. Use the TypeScript API when a repeatable Node.js -process owns the control flow. +process owns the control flow. Ragmir does not open an HTTP port: applications own their network, +authentication, and authorization boundary. Ragmir Core stays retrieval-first. `ask()` returns cited context without calling an LLM. Local chat and audio are separate capabilities, so retrieval remains useful on machines that should not run a @@ -180,23 +181,37 @@ binary support. ## TypeScript API ```ts -import { ingest, search } from "@jcode.labs/ragmir" - -await ingest({ cwd: process.cwd() }) - -const results = await search("Which decision changed the rollout?", { - topK: 5, - explain: true, -}) - -for (const result of results) { - console.log(result.citation, result.text) +import { createRagmirClient, isRagmirError } from "@jcode.labs/ragmir" + +const ragmir = await createRagmirClient({ cwd: process.cwd() }) +try { + await ragmir.ingest({ timeoutMs: 120_000 }) + + const results = await ragmir.search("Which decision changed the rollout?", { + topK: 5, + explain: true, + timeoutMs: 10_000, + }) + + for (const result of results) { + console.log(result.citation, result.text) + } +} catch (error) { + if (isRagmirError(error)) console.error(error.code, error.retryable) + else throw error +} finally { + await ragmir.close() } ``` -Core also exports `previewChunks`, `ask`, `research`, `audit`, `doctor`, `securityAudit`, -`discoverKnowledgeBases`, bounded context helpers, `serveMcp`, and setup helpers. See the -[API reference](./docs/api-reference.md) for the public surface. +Reuse one client per project root in a long-running process. It keeps one local LanceDB connection, +serializes ingestion for the same index inside that process, accepts `AbortSignal` and `timeoutMs`, +and waits for active operations during `close()`. One-shot `ingest`, `search`, `ask`, and `research` +functions remain available for short scripts. + +Core also exports `previewChunks`, `audit`, `doctor`, `securityAudit`, bounded context helpers, +closeable MCP construction helpers, and setup helpers. See the [API reference](./docs/api-reference.md) +for the complete public surface. ## Privacy boundaries diff --git a/context7.json b/context7.json index 1c501a1..fbd1746 100644 --- a/context7.json +++ b/context7.json @@ -20,6 +20,8 @@ "rules": [ "Ragmir Core returns cited retrieval context only; it does not synthesize answers itself.", "Any compatible coding agent, script, CLI, TypeScript application, or MCP client can consume Core results; use a local consumer when no passage may leave the machine.", + "Use one `createRagmirClient()` per project root for repeated work in a stateful Node.js process; close it during shutdown and use the top-level functions for one-shot scripts.", + "Ragmir does not open an HTTP port; a network-facing host owns its transport, authentication, authorization, and rate limits.", "`rgr chat` is optional add-on generation via @jcode.labs/ragmir-chat; the core remains retrieval-only.", "Qwen and Gemma are optional Ragmir Chat profiles, never requirements of Core, the CLI, the TypeScript API, or MCP.", "The `local-hash` embedding provider (default) is a lexical sha256 embedding, not semantic; use `transformers` for semantic retrieval.", diff --git a/docs/agent-integration.md b/docs/agent-integration.md index aeb371c..43f924f 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -88,6 +88,12 @@ The generated helpers cover Claude Code, Codex, Kimi, OpenCode, and Cline. Other the same evidence through the CLI, TypeScript API, or any compatible MCP client. Hermes, n8n workers, CI jobs, and internal applications do not require a dedicated Ragmir model integration. +Embedding applications can call `createMcpServer(cwd)` to register a caller-owned transport, or +`connectMcpServer(transport, cwd)` to connect it and receive a closeable server handle. The standard +`serveMcp(cwd)` helper remains the simplest local stdio entry point. MCP cancellation propagates to +search, ask, research, and citation expansion. Ragmir does not open an HTTP port; applications that +expose a network transport own its authentication and authorization boundary. + ## Verify ```bash diff --git a/docs/api-reference.md b/docs/api-reference.md index cd75f5a..e595c48 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -35,6 +35,51 @@ for (const result of results) { Search results include `relativePath`, `citation`, `chunkIndex`, exact text, line ranges, page ranges when available, structural context, and optional score explanations. +### Persistent client for Node.js workers + +Use one client per project root when a stateful Node.js process performs repeated retrieval. The +client reuses one strongly consistent LanceDB connection and resolves its project root at creation. + +```ts +import { createRagmirClient, isRagmirError } from "@jcode.labs/ragmir" + +const controller = new AbortController() +const ragmir = await createRagmirClient({ cwd: process.cwd() }) + +try { + await ragmir.ingest({ signal: controller.signal, timeoutMs: 120_000 }) + const results = await ragmir.search("release approval", { + topK: 5, + signal: controller.signal, + timeoutMs: 10_000, + }) + console.log(results.map(({ citation }) => citation)) +} catch (error) { + if (isRagmirError(error)) { + console.error(error.code, error.retryable) + } else { + throw error + } +} finally { + await ragmir.close() +} +``` + +`RagmirClient` exposes `ingest`, `search`, `ask`, `research`, `expandCitation`, `status`, `sources`, +and an idempotent `close`. `close()` rejects new work, waits for active operations, then closes the +shared connection. Ingestion targeting the same storage directory is serialized inside one Node.js +process. Cancellation is cooperative between parsing, embedding, storage, and retrieval phases. + +`RagmirError.code` is one of `ABORTED`, `CLIENT_CLOSED`, `INTERNAL`, `INVALID_ARGUMENT`, or +`TIMEOUT`. `retryable` is true for cancellation and timeout errors. + +The writer queue coordinates clients inside one Node.js process. If several OS processes can ingest +the same storage directory, the host must elect one writer or serialize those ingestion jobs. + +This API targets stateful Node.js processes with a local filesystem. It is not an edge or stateless +serverless API, and Ragmir does not provide an HTTP listener. A network-facing application owns +authentication, authorization, rate limits, and transport security. + ### Project and source setup | Export | Purpose | @@ -65,9 +110,11 @@ when available, structural context, and optional score explanations. | `evaluateGoldenQueries(options)` | Score retrieval against a local golden-query file. | `SearchOptions` accepts `cwd`, `topK`, `contextRadius`, `includePaths`, `excludePaths`, -`contextPaths`, and `explain`. When explanation is enabled, each result includes reciprocal-rank -fusion contributions, one-based vector and lexical ranks, vector distance, lexical backend score, -and matched query terms. `ExpandCitationOptions.contextRadius` is clamped to three chunks. +`contextPaths`, `explain`, `signal`, and `timeoutMs`. `IngestOptions`, `ResearchOptions`, and +`ExpandCitationOptions` also accept `signal` and `timeoutMs`. When explanation is enabled, each +result includes reciprocal-rank fusion contributions, one-based vector and lexical ranks, vector +distance, lexical backend score, and matched query terms. `ExpandCitationOptions.contextRadius` is +clamped to three chunks. Structural context comes from Markdown headings or structured-data paths. It can improve candidate selection without changing the exact text, offsets, or citations returned to the caller. @@ -106,6 +153,8 @@ model download must be explicitly enabled before local inference can use it. | Export | Purpose | | --- | --- | +| `createMcpServer(cwd?)` | Construct the read-focused MCP server without selecting a transport. | +| `connectMcpServer(transport, cwd?)` | Connect a caller-owned MCP transport and return a closeable server handle. | | `serveMcp(cwd?)` | Start the local stdio MCP server. | | `installAgentSkills(options?)` | Install the canonical skill kit for selected native agents. | | `installSkill(options?)` | Install one bundled skill with ownership checks. | @@ -121,7 +170,7 @@ model download must be explicitly enabled before local inference can use it. New integrations should use `rgrCommand` and the `rgr` CLI name. MCP retrieval tools accept a `maxBytes` value below the configured `mcpMaxOutputBytes` ceiling. Search, ask, and research also accept compact output. Metrics are returned under `_meta["ragmir/output"]` and summarized by the -metadata-only usage report. +metadata-only usage report. MCP cancellation signals propagate into Core retrieval operations. ### Core type exports @@ -136,7 +185,7 @@ types that callers commonly compose explicitly. | Retrieval | `SearchOptions`, `SearchResult`, `SearchContextChunk`, `SearchScoreExplanation`, `AskResult`, `CompactSearchResult`, `ExpandCitationOptions`, `ExpandedCitation` | | Research and evaluation | `ResearchOptions`, `ResearchReport`, `ResearchEvidence`, `CodeEvidence`, `SourceDiagnostics`, `SourceDuplicateCandidate`, `SourcePathCandidate`, `EvaluationOptions`, `EvaluationResult`, `EvaluationCaseResult`, `GoldenQuery` | | Bases and sources | `KnowledgeBaseIdentity`, `KnowledgeBaseInfo`, `KnowledgeBaseInventory`, `KnowledgeBaseContextReport`, `KnowledgeBaseSourceCatalog`, `AddSourceEntriesOptions`, `AddSourceEntriesResult`, `SourceEntriesResult` | -| Operations | `DoctorReport`, `SecurityAuditReport`, `DestroyIndexResult`, `AccessLogAction`, `AccessLogUsageOptions`, `AccessLogUsageReport`, `McpOutputTool`, `McpOutputUsageReport`, `RedactionCount` | +| Operations | `RagmirClientOptions`, `OperationOptions`, `RagmirErrorCode`, `DoctorReport`, `SecurityAuditReport`, `DestroyIndexResult`, `AccessLogAction`, `AccessLogUsageOptions`, `AccessLogUsageReport`, `McpOutputTool`, `McpOutputUsageReport`, `RedactionCount` | | Embeddings and OCR | `EnableSemanticEmbeddingsResult`, `PullEmbeddingModelResult`, `ConfigurePdfOcrOptions`, `ConfigurePdfOcrResult`, `ExtractPdfPageOptions`, `OcrExecutableStatus`, `PdfOcrEngine`, `PdfOcrEngineSelection`, `PdfOcrStatus` | | Agent integration | `AgentHelperFile`, `AgentInstallMode`, `AgentInstallScope`, `AgentIntegrationReport`, `AgentSkillInstallation`, `AgentTarget`, `InstallAgentSkillsOptions`, `InstallAgentSkillsResult`, `InstallSkillOptions`, `InstallSkillResult`, `RagmirRunnerMode` | | Setup and commands | `SetupOptions`, `SetupResult`, `SetupSemanticResult`, `PackageManager`, `RagmirCommand`, `PromptRouteDecision`, `PromptRouteTool` | diff --git a/docs/configuration.md b/docs/configuration.md index ffb78ff..7ce7247 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,3 +70,8 @@ RAGMIR_MCP_MAX_OUTPUT_BYTES=16384 rgr mcp Environment overrides cover selected runtime settings such as models, retrieval limits, access logs, and extractor commands. Run `rgr status --json` to inspect the effective result. + +For a long-running process that hosts more than one isolated project workflow, create one +`RagmirClient` per project root and keep process-wide environment overrides stable after startup. +Close every client during shutdown. If several OS processes can ingest the same storage directory, +the host must coordinate a single writer. diff --git a/llms.txt b/llms.txt index 1165bb4..0d13c0e 100644 --- a/llms.txt +++ b/llms.txt @@ -15,6 +15,10 @@ store. agent or optional local chat add-on. - Consumer choice: use the AI or automation you already have through CLI, TypeScript, or MCP. Use a local consumer or optional Chat when no retrieved passage may leave the machine. +- Persistent API: `createRagmirClient()` reuses one local connection in a stateful Node.js worker, + supports cancellation and timeouts, and closes gracefully after active operations. +- Service boundary: Ragmir does not open an HTTP port. Network transports, authentication, and + authorization belong to the embedding application. - Default retrieval: local-hash, with no model download. Semantic embeddings and local GGUF chat are explicit opt-ins. - Optional packages: `@jcode.labs/ragmir-chat` for local cited GGUF synthesis and diff --git a/package.json b/package.json index 15e47e9..88da1c6 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,9 @@ "sovereign-rag", "mcp", "ai-agents", + "coding-agents", + "nodejs", + "typescript", "private-ai", "local-first", "knowledge-base" diff --git a/packages/ragmir-core/README.md b/packages/ragmir-core/README.md index 07a4d5d..aec0683 100644 --- a/packages/ragmir-core/README.md +++ b/packages/ragmir-core/README.md @@ -73,9 +73,9 @@ Use that command from a local shell script, a Node.js worker, or a CI step after and its local `.ragmir/` state. The process returns machine-readable cited passages; the workflow decides whether to continue, request human approval, or stop. -For long-running integrations, start the local stdio server with `npx rgr serve-mcp`. The MCP surface -is bounded and read-focused: status, source coverage, search, retrieval-only ask, research, exact -citation expansion, audit, evaluation, usage, and security checks. It never exposes index deletion. +For long-running agent integrations, start the local stdio server with `npx rgr serve-mcp`. For a +Node.js worker that owns the control flow, use `createRagmirClient()` and reuse one client per project +root. Ragmir does not open an HTTP port or define an authentication layer. ## Search directly from the CLI @@ -113,20 +113,29 @@ multi-query retrieval pass and reports missing or weak evidence. ## TypeScript API ```ts -import { ingest, search } from "@jcode.labs/ragmir" - -await ingest({ cwd: process.cwd() }) - -const results = await search("Which decision changed the rollout?", { - cwd: process.cwd(), - topK: 5, -}) - -for (const result of results) { - console.log(result.citation, result.text) +import { createRagmirClient } from "@jcode.labs/ragmir" + +const ragmir = await createRagmirClient({ cwd: process.cwd() }) +try { + await ragmir.ingest({ timeoutMs: 120_000 }) + + const results = await ragmir.search("Which decision changed the rollout?", { + topK: 5, + timeoutMs: 10_000, + }) + + for (const result of results) { + console.log(result.citation, result.text) + } +} finally { + await ragmir.close() } ``` +The persistent client reuses one local database connection, supports cooperative cancellation, and +waits for active work before shutdown. The top-level functions remain the smallest API for one-shot +scripts. + Frequently used exports: | Export | Purpose | @@ -134,11 +143,12 @@ Frequently used exports: | `setupProject`, `addSourceEntries` | Initialize project state and select files | | `discoverKnowledgeBases`, `knowledgeBaseIdentity` | Route root and nested monorepo bases | | `getKnowledgeBaseContext`, `getKnowledgeBaseSourceCatalog` | Give agents bounded readiness and source context | +| `createRagmirClient`, `RagmirClient` | Reuse local retrieval safely in a stateful Node.js process | | `ingest`, `audit` | Build the index and compare it with files on disk | | `previewChunks` | Inspect redacted chunks and distributions without writing storage | | `search`, `ask`, `research`, `expandCitation` | Retrieve or expand cited passages | | `doctor`, `securityAudit` | Inspect readiness and local privacy posture | -| `serveMcp` | Start the read-focused local MCP server | +| `createMcpServer`, `connectMcpServer`, `serveMcp` | Construct, connect, or start the read-focused local MCP server | | `configurePdfOcr`, `inspectPdfOcr` | Configure and inspect local PDF OCR | See the [complete API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md) diff --git a/packages/ragmir-core/examples/library-api-demo/README.md b/packages/ragmir-core/examples/library-api-demo/README.md index 4b80462..dcf91e1 100644 --- a/packages/ragmir-core/examples/library-api-demo/README.md +++ b/packages/ragmir-core/examples/library-api-demo/README.md @@ -9,12 +9,12 @@ prints cited results from a fictional local corpus. ## What it proves -The demo exercises four public operations in order: +The demo opens one persistent client and exercises four public operations in order: -1. `ingest({ cwd, rebuild: true })` parses, redacts, chunks, embeds, and stores the corpus. -2. `search(query, { cwd, topK })` returns ranked source passages. -3. `ask(query, { cwd, topK })` returns retrieval-only cited context without LLM synthesis. -4. `audit(cwd)` compares supported files with the current index. +1. `ragmir.ingest({ rebuild: true })` parses, redacts, chunks, embeds, and stores the corpus. +2. `ragmir.search(query, { topK })` returns ranked source passages. +3. `ragmir.ask(query, { topK })` returns retrieval-only cited context without LLM synthesis. +4. `ragmir.status()` reports the active knowledge base and indexed coverage. Node's package self-reference resolves `@jcode.labs/ragmir` to this checkout's local `packages/ragmir-core/dist` build. It never falls back to the npm-published version, so the result @@ -30,7 +30,7 @@ pnpm example ``` The root command builds the published packages required by Core and runs [`run.mjs`](./run.mjs). The -output is organized into `ingest`, `search`, `ask`, and `audit` sections so a reviewer can see each +output is organized into `ingest`, `search`, `ask`, and `status` sections so a reviewer can see each public step succeed. To rerun only the script after an existing build: @@ -46,25 +46,25 @@ node packages/ragmir-core/examples/library-api-demo/run.mjs The essential integration is intentionally small: ```js -import { ask, audit, ingest, search } from "@jcode.labs/ragmir" +import { createRagmirClient } from "@jcode.labs/ragmir" -await ingest({ cwd: projectRoot, rebuild: true }) +const ragmir = await createRagmirClient({ cwd: projectRoot }) +try { + await ragmir.ingest({ rebuild: true, timeoutMs: 30_000 }) -const results = await search("offline retrieval approval", { - cwd: projectRoot, - topK: 3, -}) + const results = await ragmir.search("offline retrieval approval", { topK: 3 }) -const context = await ask("What evidence supports offline operation?", { - cwd: projectRoot, - topK: 3, -}) + const context = await ragmir.ask("What evidence supports offline operation?", { topK: 3 }) -const report = await audit(projectRoot) + const status = await ragmir.status() +} finally { + await ragmir.close() +} ``` -All project-relative state is resolved from `cwd`. This is the key boundary for applications that -use Ragmir across more than one repository. +All project-relative state is resolved once from `cwd`. Reuse one client per project root in a +long-running Node.js process and close it during shutdown. For one-shot scripts, the top-level +`ingest`, `search`, `ask`, and `audit` functions remain available. ## Data used by the demo diff --git a/packages/ragmir-core/examples/library-api-demo/run.mjs b/packages/ragmir-core/examples/library-api-demo/run.mjs index 9420cae..de1b7e2 100644 --- a/packages/ragmir-core/examples/library-api-demo/run.mjs +++ b/packages/ragmir-core/examples/library-api-demo/run.mjs @@ -13,7 +13,7 @@ import path from "node:path" import { fileURLToPath } from "node:url" -import { ask, audit, ingest, search } from "@jcode.labs/ragmir" +import { createRagmirClient } from "@jcode.labs/ragmir" const here = path.dirname(fileURLToPath(import.meta.url)) @@ -28,31 +28,37 @@ function heading(title) { async function main() { console.log("Running @jcode.labs/ragmir from the local build against the synthetic corpus.") console.log(`corpus: ${corpus}`) + const ragmir = await createRagmirClient({ cwd: corpus }) + try { + heading("ingest") + const ingested = await ragmir.ingest({ rebuild: true, timeoutMs: 30_000 }) + console.log( + `indexed ${ingested.indexedFiles}/${ingested.supportedFiles} supported files, ` + + `${ingested.chunks} chunks, ${ingested.redactions} redactions`, + ) - heading("ingest") - const ingested = await ingest({ cwd: corpus, rebuild: true }) - console.log( - `indexed ${ingested.indexedFiles}/${ingested.supportedFiles} supported files, ` + - `${ingested.chunks} chunks, ${ingested.redactions} redactions`, - ) - - heading('search "offline retrieval approval"') - const passages = await search("offline retrieval approval", { cwd: corpus, topK: 3 }) - for (const passage of passages) { - console.log(`- ${passage.relativePath}#${passage.chunkIndex} (distance ${passage.distance ?? "n/a"})`) - } + heading('search "offline retrieval approval"') + const passages = await ragmir.search("offline retrieval approval", { topK: 3 }) + for (const passage of passages) { + console.log( + `- ${passage.relativePath}#${passage.chunkIndex} (distance ${passage.distance ?? "n/a"})`, + ) + } + + heading('ask "What evidence supports offline operation?"') + const answer = await ragmir.ask("What evidence supports offline operation?", { topK: 3 }) + console.log(`${answer.sources.length} cited sources`) + console.log(answer.answer) - heading('ask "What evidence supports offline operation?"') - const answer = await ask("What evidence supports offline operation?", { cwd: corpus, topK: 3 }) - console.log(`${answer.sources.length} cited sources`) - console.log(answer.answer) - - heading("audit") - const report = await audit(corpus) - console.log( - `${report.supportedFiles.length} supported, ${report.skippedFiles.length} skipped, ` + - `${report.totalChunks} chunks indexed`, - ) + heading("status") + const status = await ragmir.status() + console.log( + `${status.coverage.indexedFiles} indexed files, ` + + `${status.coverage.chunksIndexed} chunks`, + ) + } finally { + await ragmir.close() + } } main().catch((error) => { diff --git a/packages/ragmir-core/package.json b/packages/ragmir-core/package.json index d909c8e..ccd4d78 100644 --- a/packages/ragmir-core/package.json +++ b/packages/ragmir-core/package.json @@ -14,11 +14,16 @@ "open-source", "rag", "sovereign-rag", + "ai-agents", + "coding-agents", "knowledge-base", "confidential-data", "private-ai", "local-first", "mcp", + "mcp-server", + "nodejs", + "typescript", "transformers", "semantic-search", "lancedb" diff --git a/packages/ragmir-core/src/client.test.ts b/packages/ragmir-core/src/client.test.ts new file mode 100644 index 0000000..c961e6d --- /dev/null +++ b/packages/ragmir-core/src/client.test.ts @@ -0,0 +1,94 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, it } from "vitest" +import { createRagmirClient } from "./client.js" +import type { RagmirError } from "./errors.js" +import { initProject } from "./init.js" + +const tempDirs: string[] = [] + +afterEach(async () => { + for (const dir of tempDirs.splice(0)) { + await rm(dir, { recursive: true, force: true }) + } +}) + +async function projectWithEvidence(prefix: string): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), prefix)) + tempDirs.push(root) + await initProject(root) + await mkdir(path.join(root, ".ragmir", "raw"), { recursive: true }) + await writeFile( + path.join(root, ".ragmir", "raw", "decision.md"), + "The rollout requires human approval before production deployment.\n", + "utf8", + ) + return root +} + +describe("RagmirClient", () => { + it("should reuse one client for ingestion and cited retrieval", async () => { + const root = await projectWithEvidence("ragmir-client-") + const client = await createRagmirClient({ cwd: root }) + + const ingestion = await client.ingest() + const results = await client.search("production approval") + + expect(ingestion.indexedFiles).toBe(1) + expect(results[0]?.relativePath).toBe(".ragmir/raw/decision.md") + expect(results[0]?.citation).toContain("decision.md:L1-") + expect(client.isClosed).toBe(false) + + await client.close() + await client.close() + expect(client.isClosed).toBe(true) + await expect(client.search("approval")).rejects.toMatchObject({ + code: "CLIENT_CLOSED", + } satisfies Partial) + }) + + it("should serialize concurrent ingestion in one Node.js process", async () => { + const root = await projectWithEvidence("ragmir-client-concurrent-") + const firstClient = await createRagmirClient({ cwd: root }) + const secondClient = await createRagmirClient({ cwd: root }) + + const [first, second] = await Promise.all([firstClient.ingest(), secondClient.ingest()]) + const results = await secondClient.search("human approval") + + expect(first.rebuiltFiles).toBe(1) + expect(second.reusedFiles).toBe(1) + expect(results).toHaveLength(1) + + await Promise.all([firstClient.close(), secondClient.close()]) + }) + + it("should finish active work before closing its shared connection", async () => { + const root = await projectWithEvidence("ragmir-client-close-") + const client = await createRagmirClient({ cwd: root }) + + const ingestion = client.ingest() + const closing = client.close() + + await expect(ingestion).resolves.toMatchObject({ indexedFiles: 1 }) + await expect(closing).resolves.toBeUndefined() + expect(client.isClosed).toBe(true) + }) + + it("should expose stable abort and validation errors", async () => { + const root = await projectWithEvidence("ragmir-client-abort-") + const client = await createRagmirClient({ cwd: root }) + const controller = new AbortController() + controller.abort("cancelled by caller") + + await expect(client.search("approval", { signal: controller.signal })).rejects.toMatchObject({ + code: "ABORTED", + retryable: true, + } satisfies Partial) + await expect(client.search("approval", { timeoutMs: 0 })).rejects.toMatchObject({ + code: "INVALID_ARGUMENT", + } satisfies Partial) + + await client.close() + }) +}) diff --git a/packages/ragmir-core/src/client.ts b/packages/ragmir-core/src/client.ts new file mode 100644 index 0000000..bae9198 --- /dev/null +++ b/packages/ragmir-core/src/client.ts @@ -0,0 +1,128 @@ +import type { PathLike } from "node:fs" +import type { Connection } from "@lancedb/lancedb" +import { loadConfig } from "./config.js" +import { getKnowledgeBaseContext, getKnowledgeBaseSourceCatalog } from "./context-resources.js" +import { normalizeRagmirError, RagmirError } from "./errors.js" +import { ingestWithConfig } from "./ingest.js" +import { askWithConfig, expandCitationWithConfig, searchWithConfig } from "./query.js" +import { researchWithConfig } from "./research.js" +import { connectStore } from "./store.js" +import type { + AskResult, + Config, + ExpandCitationOptions, + ExpandedCitation, + IngestOptions, + IngestResult, + KnowledgeBaseContextReport, + KnowledgeBaseSourceCatalog, + ResearchOptions, + ResearchReport, + SearchOptions, + SearchResult, +} from "./types.js" + +export interface RagmirClientOptions { + cwd?: PathLike +} + +export class RagmirClient { + readonly projectRoot: string + private readonly config: Config + private readonly connection: Connection + private readonly activeOperations = new Set>() + private closed = false + private closePromise: Promise | undefined + + private constructor(config: Config, connection: Connection) { + this.config = config + this.connection = connection + this.projectRoot = config.projectRoot + } + + get isClosed(): boolean { + return this.closed + } + + static async create(options: RagmirClientOptions = {}): Promise { + try { + const config = await loadConfig(String(options.cwd ?? process.cwd())) + const connection = await connectStore(config) + return new RagmirClient(config, connection) + } catch (error) { + throw normalizeRagmirError(error) + } + } + + async ingest(options: Omit = {}): Promise { + return this.run(() => ingestWithConfig(this.config, options, this.connection)) + } + + async search(query: string, options: Omit = {}): Promise { + return this.run(() => searchWithConfig(query, options, this.config, this.connection)) + } + + async ask(query: string, options: Omit = {}): Promise { + return this.run(() => askWithConfig(query, options, this.config, this.connection)) + } + + async research( + query: string, + options: Omit = {}, + ): Promise { + return this.run(() => researchWithConfig(query, options, this.config, this.connection)) + } + + async expandCitation( + citation: string, + options: Omit = {}, + ): Promise { + return this.run(() => expandCitationWithConfig(citation, options, this.config, this.connection)) + } + + async status(): Promise { + return this.run(() => getKnowledgeBaseContext(this.projectRoot)) + } + + async sources(): Promise { + return this.run(() => getKnowledgeBaseSourceCatalog(this.projectRoot)) + } + + async close(): Promise { + if (!this.closePromise) { + this.closed = true + this.closePromise = (async () => { + await Promise.allSettled([...this.activeOperations]) + this.connection.close() + })() + } + await this.closePromise + } + + private assertOpen(): void { + if (this.closed || !this.connection.isOpen()) { + throw new RagmirError("CLIENT_CLOSED", "Ragmir client is closed.") + } + } + + private run(operation: () => Promise): Promise { + this.assertOpen() + const active = (async () => { + try { + return await operation() + } catch (error) { + throw normalizeRagmirError(error) + } + })() + this.activeOperations.add(active) + void active.then( + () => this.activeOperations.delete(active), + () => this.activeOperations.delete(active), + ) + return active + } +} + +export async function createRagmirClient(options: RagmirClientOptions = {}): Promise { + return RagmirClient.create(options) +} diff --git a/packages/ragmir-core/src/errors.ts b/packages/ragmir-core/src/errors.ts new file mode 100644 index 0000000..a7d4f0f --- /dev/null +++ b/packages/ragmir-core/src/errors.ts @@ -0,0 +1,38 @@ +export type RagmirErrorCode = + | "ABORTED" + | "CLIENT_CLOSED" + | "INTERNAL" + | "INVALID_ARGUMENT" + | "TIMEOUT" + +interface RagmirErrorOptions { + cause?: unknown + retryable?: boolean +} + +export class RagmirError extends Error { + readonly code: RagmirErrorCode + readonly retryable: boolean + + constructor(code: RagmirErrorCode, message: string, options: RagmirErrorOptions = {}) { + super(message, options.cause === undefined ? undefined : { cause: options.cause }) + this.name = "RagmirError" + this.code = code + this.retryable = options.retryable ?? false + } +} + +export function isRagmirError(error: unknown): error is RagmirError { + return error instanceof RagmirError +} + +export function normalizeRagmirError(error: unknown): RagmirError { + if (isRagmirError(error)) { + return error + } + return new RagmirError( + "INTERNAL", + error instanceof Error ? error.message : "Ragmir operation failed.", + { cause: error }, + ) +} diff --git a/packages/ragmir-core/src/index-write-lock.test.ts b/packages/ragmir-core/src/index-write-lock.test.ts new file mode 100644 index 0000000..f1f8f3d --- /dev/null +++ b/packages/ragmir-core/src/index-write-lock.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest" +import type { RagmirError } from "./errors.js" +import { withIndexWriteLock } from "./index-write-lock.js" + +describe("withIndexWriteLock", () => { + it("should let a queued caller abort without interrupting the active writer", async () => { + let markWriterStarted: (() => void) | undefined + const writerStarted = new Promise((resolve) => { + markWriterStarted = resolve + }) + let releaseWriter: (() => void) | undefined + const writerFinished = new Promise((resolve) => { + releaseWriter = resolve + }) + const active = withIndexWriteLock("project", undefined, async () => { + markWriterStarted?.() + await writerFinished + return "written" + }) + await writerStarted + + const controller = new AbortController() + const queued = withIndexWriteLock("project", controller.signal, async () => "unexpected") + + controller.abort() + await expect(queued).rejects.toMatchObject({ + code: "ABORTED", + } satisfies Partial) + + let nextWriterStarted = false + const next = withIndexWriteLock("project", undefined, async () => { + nextWriterStarted = true + return "next" + }) + await Promise.resolve() + expect(nextWriterStarted).toBe(false) + + releaseWriter?.() + await expect(active).resolves.toBe("written") + await expect(next).resolves.toBe("next") + }) + + it("should expose a timeout while waiting for the active writer", async () => { + let releaseWriter: (() => void) | undefined + const writerFinished = new Promise((resolve) => { + releaseWriter = resolve + }) + const active = withIndexWriteLock("project-timeout", undefined, async () => { + await writerFinished + return "written" + }) + const queued = withIndexWriteLock( + "project-timeout", + AbortSignal.timeout(5), + async () => "unexpected", + ) + + await expect(queued).rejects.toMatchObject({ + code: "TIMEOUT", + retryable: true, + } satisfies Partial) + releaseWriter?.() + await expect(active).resolves.toBe("written") + }) +}) diff --git a/packages/ragmir-core/src/index-write-lock.ts b/packages/ragmir-core/src/index-write-lock.ts new file mode 100644 index 0000000..51f4a91 --- /dev/null +++ b/packages/ragmir-core/src/index-write-lock.ts @@ -0,0 +1,58 @@ +import { throwIfAborted } from "./operation.js" + +const writeQueues = new Map>() + +export async function withIndexWriteLock( + key: string, + signal: AbortSignal | undefined, + operation: () => Promise, +): Promise { + const previous = writeQueues.get(key) ?? Promise.resolve() + let release: (() => void) | undefined + const current = new Promise((resolve) => { + release = resolve + }) + const tail = previous.then( + () => current, + () => current, + ) + writeQueues.set(key, tail) + void tail.then(() => { + if (writeQueues.get(key) === tail) { + writeQueues.delete(key) + } + }) + + try { + await waitForTurn(previous, signal) + throwIfAborted(signal) + return await operation() + } finally { + release?.() + } +} + +async function waitForTurn( + previous: Promise, + signal: AbortSignal | undefined, +): Promise { + if (!signal) { + await previous + return + } + throwIfAborted(signal) + + let onAbort: (() => void) | undefined + const aborted = new Promise((resolve) => { + onAbort = () => resolve() + signal.addEventListener("abort", onAbort, { once: true }) + }) + try { + await Promise.race([previous, aborted]) + throwIfAborted(signal) + } finally { + if (onAbort) { + signal.removeEventListener("abort", onAbort) + } + } +} diff --git a/packages/ragmir-core/src/index.ts b/packages/ragmir-core/src/index.ts index e986fef..02f9b35 100644 --- a/packages/ragmir-core/src/index.ts +++ b/packages/ragmir-core/src/index.ts @@ -1,4 +1,6 @@ export { accessLogUsageReport } from "./access-log.js" +export type { RagmirClientOptions } from "./client.js" +export { createRagmirClient, RagmirClient } from "./client.js" export { loadConfig } from "./config.js" export { getKnowledgeBaseContext, @@ -8,6 +10,8 @@ export { destroyIndex } from "./destroy.js" export { doctor } from "./doctor.js" export type { PullEmbeddingModelResult } from "./embeddings.js" export { clearTransformersCache, pullEmbeddingModel } from "./embeddings.js" +export type { RagmirErrorCode } from "./errors.js" +export { isRagmirError, normalizeRagmirError, RagmirError } from "./errors.js" export { evaluateGoldenQueries } from "./evaluate.js" export { getIndexFreshnessWarning, @@ -18,7 +22,7 @@ export { audit, ingest } from "./ingest.js" export { initProject } from "./init.js" export { discoverKnowledgeBases, knowledgeBaseIdentity } from "./knowledge-bases.js" export { ingestionLimits } from "./limits.js" -export { serveMcp } from "./mcp.js" +export { connectMcpServer, createMcpServer, serveMcp } from "./mcp.js" export type { ConfigurePdfOcrOptions, ConfigurePdfOcrResult, @@ -99,6 +103,7 @@ export type { KnowledgeBaseSourceCatalog, McpOutputTool, McpOutputUsageReport, + OperationOptions, ParsedPage, PreviewChunk, PreviewChunksOptions, diff --git a/packages/ragmir-core/src/ingest.ts b/packages/ragmir-core/src/ingest.ts index 50085cf..f9537b4 100644 --- a/packages/ragmir-core/src/ingest.ts +++ b/packages/ragmir-core/src/ingest.ts @@ -1,3 +1,4 @@ +import type { Connection } from "@lancedb/lancedb" import { recordAccess } from "./access-log.js" import { summarizeChunkStats } from "./chunk-stats.js" import { chunkDocument, chunkSearchText } from "./chunking.js" @@ -11,6 +12,8 @@ import { } from "./files.js" import { INDEX_SCHEMA_VERSION } from "./index-diagnostics.js" import { indexPolicyFingerprint } from "./index-policy.js" +import { withIndexWriteLock } from "./index-write-lock.js" +import { operationSignal, throwIfAborted } from "./operation.js" import { parseFile } from "./parsing.js" import { redactText, totalRedactions } from "./redaction.js" import { @@ -54,6 +57,27 @@ const MIRROR_PATH_PATTERNS = [ export async function ingest(options: IngestOptions = {}): Promise { const config = await loadConfig(String(options.cwd ?? process.cwd())) + return ingestWithConfig(config, options) +} + +export async function ingestWithConfig( + config: Config, + options: IngestOptions = {}, + connection?: Connection, +): Promise { + const signal = operationSignal(options) + return withIndexWriteLock(config.storageDir, signal, () => + ingestUnlocked(config, options, connection, signal), + ) +} + +async function ingestUnlocked( + config: Config, + options: IngestOptions, + connection: Connection | undefined, + signal: AbortSignal | undefined, +): Promise { + throwIfAborted(signal) const policyFingerprint = indexPolicyFingerprint(config) const existingManifest = await readIndexManifest(config) const manifestCompatible = @@ -61,7 +85,7 @@ export async function ingest(options: IngestOptions = {}): Promise existingManifest?.schemaVersion === INDEX_SCHEMA_VERSION && existingManifest.indexPolicyFingerprint === policyFingerprint && existingManifest.indexedFiles !== undefined - const existingTable = manifestCompatible ? await openRowsTable(config) : null + const existingTable = manifestCompatible ? await openRowsTable(config, connection) : null const storedEmptyFiles = manifestCompatible ? await readEmptyTextFiles(config) : [] const knownFiles = new Map( [...(existingManifest?.indexedFiles ?? []), ...storedEmptyFiles].map((file) => [ @@ -97,9 +121,10 @@ export async function ingest(options: IngestOptions = {}): Promise const redactionCounts: RedactionCount[] = [] const emptyTextFiles: string[] = [] - const results = await mapLimit(filesToIndex, config.ingestConcurrency, async (file) => { + const results = await mapLimit(filesToIndex, config.ingestConcurrency, signal, async (file) => { try { const parsed = await parseFile(file, config) + throwIfAborted(signal) const redacted = redactText(parsed.text, config) const chunks = chunkDocument( { ...parsed, text: redacted.text }, @@ -108,6 +133,7 @@ export async function ingest(options: IngestOptions = {}): Promise ) return { path: file.relativePath, chunks, redactions: redacted.counts, error: null } } catch (error) { + throwIfAborted(signal) return { path: file.relativePath, chunks: [], @@ -134,8 +160,10 @@ export async function ingest(options: IngestOptions = {}): Promise const rows: VectorRow[] = [] for (let i = 0; i < allChunks.length; i += config.embeddingBatchSize) { + throwIfAborted(signal) const batch = allChunks.slice(i, i + config.embeddingBatchSize) const embeddings = await embedTexts(batch.map(chunkSearchText), config) + throwIfAborted(signal) for (const [index, chunk] of batch.entries()) { const vector = embeddings[index] if (!vector) { @@ -162,14 +190,15 @@ export async function ingest(options: IngestOptions = {}): Promise ...filesToIndex.map((file) => file.relativePath), ...[...previousPaths].filter((relativePath) => !currentPaths.has(relativePath)), ] + throwIfAborted(signal) const writeResult = !canReuse || chunkCount === 0 - ? await writeRows(rows, config) + ? await writeRows(rows, config, connection) : replacePaths.length > 0 - ? await updateRows(rows, replacePaths, config) + ? await updateRows(rows, replacePaths, config, connection) : { vectorIndexWarning: null, lexicalIndexWarning: null } if (chunkCount > 0) { - const firstRow = rows[0] ?? (await firstStoredRow(config)) + const firstRow = rows[0] ?? (await firstStoredRow(config, connection)) if (!firstRow) { throw new Error("Cannot write an index manifest without indexed rows.") } @@ -258,8 +287,8 @@ function indexedFileRecords(rows: VectorRow[]): IndexManifestFile[] { return [...records.values()] } -async function firstStoredRow(config: Config): Promise { - const table = await openRowsTable(config) +async function firstStoredRow(config: Config, connection?: Connection): Promise { + const table = await openRowsTable(config, connection) if (!table) { return null } @@ -431,6 +460,7 @@ async function currentEmptyTextFiles( async function mapLimit( items: T[], concurrency: number, + signal: AbortSignal | undefined, worker: (item: T) => Promise, ): Promise { const results = new Array(items.length) @@ -438,11 +468,13 @@ async function mapLimit( async function run(): Promise { while (nextIndex < items.length) { + throwIfAborted(signal) const index = nextIndex nextIndex += 1 const item = items[index] if (item !== undefined) { results[index] = await worker(item) + throwIfAborted(signal) } } } diff --git a/packages/ragmir-core/src/mcp.test.ts b/packages/ragmir-core/src/mcp.test.ts index aa592c3..3049b2b 100644 --- a/packages/ragmir-core/src/mcp.test.ts +++ b/packages/ragmir-core/src/mcp.test.ts @@ -1,8 +1,14 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js" import { afterEach, describe, expect, it } from "vitest" -import { projectRelativeGoldenPath, resolveMcpProjectRoot, searchOptions } from "./mcp.js" +import { + connectMcpServer, + projectRelativeGoldenPath, + resolveMcpProjectRoot, + searchOptions, +} from "./mcp.js" const tempDirs: string[] = [] @@ -53,6 +59,19 @@ describe("resolveMcpProjectRoot", () => { }) }) +describe("connectMcpServer", () => { + it("should return a server handle that the embedding process can close", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-mcp-lifecycle-")) + tempDirs.push(root) + const [, serverTransport] = InMemoryTransport.createLinkedPair() + + const server = await connectMcpServer(serverTransport, root) + + expect(server.server).toBeDefined() + await expect(server.close()).resolves.toBeUndefined() + }) +}) + describe("searchOptions", () => { it("clamps requested topK to the configured mcpMaxTopK", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "ragmir-mcp-topk-")) diff --git a/packages/ragmir-core/src/mcp.ts b/packages/ragmir-core/src/mcp.ts index 33a4f0a..c393298 100644 --- a/packages/ragmir-core/src/mcp.ts +++ b/packages/ragmir-core/src/mcp.ts @@ -2,6 +2,7 @@ import { existsSync } from "node:fs" import path from "node:path" import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js" import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js" +import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js" import { z } from "zod" import { accessLogUsageReport, recordMcpOutput } from "./access-log.js" import { findProjectConfig, loadConfig } from "./config.js" @@ -84,7 +85,7 @@ const expandToolInputSchema = z.object({ maxBytes: z.number().int().min(MIN_MCP_OUTPUT_BYTES).optional(), }) -export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { +export function createMcpServer(cwd = resolveMcpProjectRoot()): McpServer { const server = new McpServer({ name: "ragmir", version: VERSION, @@ -185,30 +186,31 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { description: "Retrieve relevant passages from the local Ragmir knowledge base.", inputSchema: searchToolInputSchema, }, - async ({ - query, - topK, - contextRadius, - compact, - maxBytes, - includePaths, - excludePaths, - contextPaths, - explain, - }) => { - const config = await loadConfig(cwd) - const results = await search( + async ( + { query, - await searchOptions( - cwd, - topK, - contextRadius, - includePaths, - excludePaths, - contextPaths, - explain, - ), + topK, + contextRadius, + compact, + maxBytes, + includePaths, + excludePaths, + contextPaths, + explain, + }, + { signal }, + ) => { + const config = await loadConfig(cwd) + const options = await searchOptions( + cwd, + topK, + contextRadius, + includePaths, + excludePaths, + contextPaths, + explain, ) + const results = await search(query, { ...options, signal }) const compactResults = compactSearchResults(results) const compactOutput = config.privacyProfile === "strict" || compact === true const preferred: McpSearchPayload = compactOutput ? compactResults : results @@ -233,17 +235,20 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { description: "Return cited retrieval context for a question without calling an LLM.", inputSchema: askToolInputSchema, }, - async ({ - query, - topK, - contextRadius, - compact, - maxBytes, - includePaths, - excludePaths, - contextPaths, - explain, - }) => { + async ( + { + query, + topK, + contextRadius, + compact, + maxBytes, + includePaths, + excludePaths, + contextPaths, + explain, + }, + { signal }, + ) => { const config = await loadConfig(cwd) const options = await searchOptions( cwd, @@ -254,16 +259,17 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { contextPaths, explain, ) + const cancellableOptions = { ...options, signal } let fullPayload: AskResult if (config.privacyProfile === "strict") { - const results = await search(query, options) + const results = await search(query, cancellableOptions) fullPayload = { answer: "Strict privacy profile returns compact cited retrieval only.", sources: results, staleWarning: null, } } else { - fullPayload = await ask(query, options) + fullPayload = await ask(query, cancellableOptions) } const compactPayload: McpAskPayload = { answer: "Ragmir returns compact cited retrieval only. Expand a citation when needed.", @@ -293,16 +299,10 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { "Run an audit-backed multi-query research pass with cited evidence and optional code matches.", inputSchema: researchToolInputSchema, }, - async ({ - query, - topK, - includeCode, - compact, - maxBytes, - includePaths, - excludePaths, - contextPaths, - }) => { + async ( + { query, topK, includeCode, compact, maxBytes, includePaths, excludePaths, contextPaths }, + { signal }, + ) => { const config = await loadConfig(cwd) const options = await searchOptions( cwd, @@ -312,7 +312,7 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { excludePaths, contextPaths, ) - const researchOptions: Parameters[1] = { cwd } + const researchOptions: Parameters[1] = { cwd, signal } addOption(researchOptions, "topK", options.topK) addOption(researchOptions, "includeCode", includeCode) addOption(researchOptions, "includePaths", options.includePaths) @@ -343,10 +343,11 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { description: "Expand one Ragmir citation into a bounded exact passage window.", inputSchema: expandToolInputSchema, }, - async ({ citation, contextRadius, maxBytes }) => { + async ({ citation, contextRadius, maxBytes }, { signal }) => { const config = await loadConfig(cwd) const expanded = await expandCitation(citation, { cwd, + signal, ...(contextRadius === undefined ? {} : { contextRadius }), }) const bounded = budgetMcpJson({ @@ -444,7 +445,20 @@ export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { }, ) - await server.connect(new StdioServerTransport()) + return server +} + +export async function connectMcpServer( + transport: Transport, + cwd = resolveMcpProjectRoot(), +): Promise { + const server = createMcpServer(cwd) + await server.connect(transport) + return server +} + +export async function serveMcp(cwd = resolveMcpProjectRoot()): Promise { + await connectMcpServer(new StdioServerTransport(), cwd) } export function resolveMcpProjectRoot( diff --git a/packages/ragmir-core/src/operation.ts b/packages/ragmir-core/src/operation.ts new file mode 100644 index 0000000..2725f68 --- /dev/null +++ b/packages/ragmir-core/src/operation.ts @@ -0,0 +1,60 @@ +import { RagmirError } from "./errors.js" +import type { OperationOptions } from "./types.js" + +export function operationSignal(options: OperationOptions): AbortSignal | undefined { + const timeoutSignal = timeoutSignalFor(options.timeoutMs) + if (options.signal && timeoutSignal) { + return combineSignals(options.signal, timeoutSignal) + } + return options.signal ?? timeoutSignal +} + +function combineSignals(first: AbortSignal, second: AbortSignal): AbortSignal { + if (typeof AbortSignal.any === "function") { + return AbortSignal.any([first, second]) + } + + const controller = new AbortController() + const abort = (signal: AbortSignal): void => { + if (!controller.signal.aborted) { + controller.abort(signal.reason) + } + first.removeEventListener("abort", abortFirst) + second.removeEventListener("abort", abortSecond) + } + const abortFirst = (): void => abort(first) + const abortSecond = (): void => abort(second) + + if (first.aborted) { + abort(first) + } else if (second.aborted) { + abort(second) + } else { + first.addEventListener("abort", abortFirst, { once: true }) + second.addEventListener("abort", abortSecond, { once: true }) + } + return controller.signal +} + +export function throwIfAborted(signal: AbortSignal | undefined): void { + if (!signal?.aborted) { + return + } + const reason = signal.reason + const timeout = reason instanceof Error && reason.name === "TimeoutError" + throw new RagmirError( + timeout ? "TIMEOUT" : "ABORTED", + timeout ? "Ragmir operation timed out." : "Ragmir operation was aborted.", + { cause: reason, retryable: true }, + ) +} + +function timeoutSignalFor(timeoutMs: number | undefined): AbortSignal | undefined { + if (timeoutMs === undefined) { + return undefined + } + if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) { + throw new RagmirError("INVALID_ARGUMENT", "timeoutMs must be a positive integer.") + } + return AbortSignal.timeout(timeoutMs) +} diff --git a/packages/ragmir-core/src/query.ts b/packages/ragmir-core/src/query.ts index cae51b2..e20f3cc 100644 --- a/packages/ragmir-core/src/query.ts +++ b/packages/ragmir-core/src/query.ts @@ -1,13 +1,16 @@ +import type { Connection } from "@lancedb/lancedb" import { recordAccess } from "./access-log.js" import { loadConfig } from "./config.js" import { VECTOR_DISTANCE_METRIC } from "./defaults.js" import { embedText } from "./embeddings.js" import { getIndexFreshnessWarning } from "./index-diagnostics.js" +import { operationSignal, throwIfAborted } from "./operation.js" import { sanitizeRetrievalQuery } from "./query-sanitizer.js" import { openRowsTable, readIndexManifest } from "./store.js" import { tokenize } from "./text.js" import type { AskResult, + Config, ExpandCitationOptions, ExpandedCitation, RetrievalProfile, @@ -92,7 +95,20 @@ const FTS_SEARCH_COLUMNS = [...SEARCH_COLUMNS, "_score"] export async function search(query: string, options: SearchOptions = {}): Promise { const config = await loadConfig(String(options.cwd ?? process.cwd())) - const table = await openRowsTable(config) + return searchWithConfig(query, options, config) +} + +export async function searchWithConfig( + query: string, + options: SearchOptions, + config: Config, + connection?: Connection, + activeSignal?: AbortSignal, +): Promise { + const signal = activeSignal ?? operationSignal(options) + throwIfAborted(signal) + const table = await openRowsTable(config, connection) + throwIfAborted(signal) if (!table) { return [] } @@ -114,6 +130,7 @@ export async function search(query: string, options: SearchOptions = {}): Promis embedText(sanitized.query, config), lexicalCandidateRows(table, sanitized.query, config.hybridTextScanLimit, retrievalPredicate), ]) + throwIfAborted(signal) await assertVectorIndexCompatibility(config, vector.length) const vectorQuery = table.vectorSearch(vector).select(VECTOR_SEARCH_COLUMNS) const vectorRows = (await (retrievalPredicate @@ -122,6 +139,7 @@ export async function search(query: string, options: SearchOptions = {}): Promis ) .limit(vectorCandidateLimit(topK, config.retrievalProfile)) .toArray()) as SearchRow[] + throwIfAborted(signal) const rankedRows = rankHybridRows(sanitized.query, vectorRows, textRows) const relevantRows = config.embeddingProvider === "local-hash" @@ -134,6 +152,7 @@ export async function search(query: string, options: SearchOptions = {}): Promis rows, options.contextRadius ?? defaultContextRadius, ) + throwIfAborted(signal) const results = rows.map((row) => { const vectorDistance = typeof row.row._distance === "number" ? row.row._distance : null @@ -306,8 +325,20 @@ export function vectorCandidateLimit(topK: number, profile: RetrievalProfile = " export async function ask(query: string, options: SearchOptions = {}): Promise { const config = await loadConfig(String(options.cwd ?? process.cwd())) - const sources = await search(query, options) + return askWithConfig(query, options, config) +} + +export async function askWithConfig( + query: string, + options: SearchOptions, + config: Config, + connection?: Connection, +): Promise { + const signal = operationSignal(options) + throwIfAborted(signal) + const sources = await searchWithConfig(query, options, config, connection, signal) const staleWarning = await getIndexFreshnessWarning(config) + throwIfAborted(signal) if (sources.length === 0) { return { @@ -335,11 +366,23 @@ export async function expandCitation( citation: string, options: ExpandCitationOptions = {}, ): Promise { + const config = await loadConfig(String(options.cwd ?? process.cwd())) + return expandCitationWithConfig(citation, options, config) +} + +export async function expandCitationWithConfig( + citation: string, + options: ExpandCitationOptions, + config: Config, + connection?: Connection, +): Promise { + const signal = operationSignal(options) + throwIfAborted(signal) const requestedCitation = citation.trim() const target = parseCitationTarget(requestedCitation) const contextRadius = normalizeContextRadius(options.contextRadius) - const config = await loadConfig(String(options.cwd ?? process.cwd())) - const table = await openRowsTable(config) + const table = await openRowsTable(config, connection) + throwIfAborted(signal) if (!table) { return { requestedCitation, @@ -371,6 +414,7 @@ export async function expandCitation( `relativePath = ${sqlString(target.relativePath)} AND chunkIndex >= ${minimumChunkIndex} AND chunkIndex <= ${maximumChunkIndex}`, ) .toArray()) as SearchRow[] + throwIfAborted(signal) const targetRow = rows.find((row) => row.chunkIndex === target.chunkIndex) if (!targetRow) { return { diff --git a/packages/ragmir-core/src/research.ts b/packages/ragmir-core/src/research.ts index d983a2f..6e9771b 100644 --- a/packages/ragmir-core/src/research.ts +++ b/packages/ragmir-core/src/research.ts @@ -1,11 +1,13 @@ import { readFile } from "node:fs/promises" import path from "node:path" +import type { Connection } from "@lancedb/lancedb" import fg from "fast-glob" import { recordAccess } from "./access-log.js" import { loadConfig } from "./config.js" import { countSkippedByReason, DEFAULT_FAST_GLOB_IGNORES, isSensitiveFilePath } from "./files.js" import { audit } from "./ingest.js" -import { search } from "./query.js" +import { operationSignal, throwIfAborted } from "./operation.js" +import { searchWithConfig } from "./query.js" import { redactText } from "./redaction.js" import { securityAudit } from "./security.js" import { normalizeForMatch } from "./text.js" @@ -77,35 +79,55 @@ export async function research( query: string, options: ResearchOptions = {}, ): Promise { + const config = await loadConfig(String(options.cwd ?? process.cwd())) + return researchWithConfig(query, options, config) +} + +export async function researchWithConfig( + query: string, + options: ResearchOptions, + config: Config, + connection?: Connection, +): Promise { + const signal = operationSignal(options) + throwIfAborted(signal) const normalizedQuery = query.trim() if (!normalizedQuery) { throw new Error("Research query must not be empty.") } - const config = await loadConfig(String(options.cwd ?? process.cwd())) const topK = options.topK ?? config.topK const [auditReport, securityReport] = await Promise.all([ audit(config.projectRoot), securityAudit(config.projectRoot), ]) + throwIfAborted(signal) const generatedQueries = researchQueries(normalizedQuery) const includeCode = config.privacyProfile === "strict" ? false : options.includeCode !== false const perQueryTopK = Math.max(2, Math.ceil(topK / 2)) const searchResults = await Promise.all( generatedQueries.map(async (generatedQuery) => ({ query: generatedQuery, - results: await search(generatedQuery, { - cwd: config.projectRoot, - topK: perQueryTopK, - ...(options.includePaths ? { includePaths: options.includePaths } : {}), - ...(options.excludePaths ? { excludePaths: options.excludePaths } : {}), - ...(options.contextPaths ? { contextPaths: options.contextPaths } : {}), - }), + results: await searchWithConfig( + generatedQuery, + { + cwd: config.projectRoot, + topK: perQueryTopK, + ...(options.includePaths ? { includePaths: options.includePaths } : {}), + ...(options.excludePaths ? { excludePaths: options.excludePaths } : {}), + ...(options.contextPaths ? { contextPaths: options.contextPaths } : {}), + }, + config, + connection, + signal, + ), })), ) + throwIfAborted(signal) const evidence = mergeEvidence(searchResults).slice(0, topK) const codeEvidence = !includeCode ? [] - : await findCodeEvidence(config, normalizedQuery, DEFAULT_CODE_EVIDENCE_LIMIT) + : await findCodeEvidence(config, normalizedQuery, DEFAULT_CODE_EVIDENCE_LIMIT, signal) + throwIfAborted(signal) const unsupportedFiles = countSkippedByReason(auditReport.skippedFiles, "unsupported-extension") const oversizedFiles = countSkippedByReason(auditReport.skippedFiles, "oversized") const gaps = researchGaps({ @@ -251,7 +273,9 @@ async function findCodeEvidence( config: Config, query: string, limit: number, + signal: AbortSignal | undefined, ): Promise { + throwIfAborted(signal) const terms = meaningfulTerms(query) if (terms.length === 0) { return [] @@ -273,6 +297,7 @@ async function findCodeEvidence( const candidates: CodeEvidence[] = [] for (const entry of entries) { + throwIfAborted(signal) if (candidates.length >= candidateLimit) { break } @@ -288,6 +313,7 @@ async function findCodeEvidence( } const relativePath = path.relative(config.projectRoot, absolutePath) const content = await readFile(absolutePath, "utf8").catch(() => null) + throwIfAborted(signal) if (content === null) { continue } diff --git a/packages/ragmir-core/src/store.ts b/packages/ragmir-core/src/store.ts index 964bf2b..5d4f828 100644 --- a/packages/ragmir-core/src/store.ts +++ b/packages/ragmir-core/src/store.ts @@ -1,4 +1,5 @@ -import { readFile, rm, writeFile } from "node:fs/promises" +import { randomUUID } from "node:crypto" +import { readFile, rename, rm, writeFile } from "node:fs/promises" import path from "node:path" import * as lancedb from "@lancedb/lancedb" import { INDEX_MANIFEST_FILENAME } from "./defaults.js" @@ -20,39 +21,50 @@ export interface IndexWriteResult { lexicalIndexWarning: string | null } -export async function writeRows(rows: VectorRow[], config: Config): Promise { +export async function connectStore(config: Config): Promise { await ensurePrivateDirectory(config.storageDir) - const db = await lancedb.connect(config.storageDir) + return lancedb.connect(config.storageDir, { + readConsistencyInterval: 0, + }) +} - if (rows.length === 0) { - const tableNames = await db.tableNames() - if (tableNames.includes(config.tableName)) { - await db.dropTable(config.tableName) +export async function writeRows( + rows: VectorRow[], + config: Config, + connection?: lancedb.Connection, +): Promise { + return withConnection(config, connection, async (db) => { + if (rows.length === 0) { + const tableNames = await db.tableNames() + if (tableNames.includes(config.tableName)) { + await db.dropTable(config.tableName) + } + await rm(path.join(config.storageDir, INDEX_MANIFEST_FILENAME), { force: true }) + return { vectorIndexWarning: null, lexicalIndexWarning: null } } - await rm(path.join(config.storageDir, INDEX_MANIFEST_FILENAME), { force: true }) - return { vectorIndexWarning: null, lexicalIndexWarning: null } - } - const records = storedRows(rows) - const table = await db.createTable(config.tableName, records, { - mode: "overwrite", - }) + const records = storedRows(rows) + const table = await db.createTable(config.tableName, records, { + mode: "overwrite", + }) - const lexicalResult = await ensureLexicalIndex(table) - return { - vectorIndexWarning: null, - lexicalIndexWarning: lexicalResult.warning, - } + const lexicalResult = await ensureLexicalIndex(table) + return { + vectorIndexWarning: null, + lexicalIndexWarning: lexicalResult.warning, + } + }) } export async function updateRows( rows: VectorRow[], replacePaths: string[], config: Config, + connection?: lancedb.Connection, ): Promise { - const table = await openRowsTable(config) + const table = await openRowsTable(config, connection) if (!table) { - return writeRows(rows, config) + return writeRows(rows, config, connection) } for (const paths of batches([...new Set(replacePaths)], 200)) { @@ -95,10 +107,8 @@ async function ensureLexicalIndex(table: lancedb.Table): Promise { - await ensurePrivateDirectory(config.storageDir) const manifestPath = path.join(config.storageDir, INDEX_MANIFEST_FILENAME) - await writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8") - await hardenPrivateFile(manifestPath) + await writePrivateJsonAtomic(manifestPath, manifest, config.storageDir) } export async function readIndexManifest(config: Config): Promise { @@ -167,14 +177,12 @@ export async function writeEmptyTextFiles( return } - await ensurePrivateDirectory(config.storageDir) const sortedRecords = [...records].sort((a, b) => a.relativePath.localeCompare(b.relativePath)) - await writeFile( + await writePrivateJsonAtomic( manifestPath, - JSON.stringify({ version: 1, files: sortedRecords }, null, 2), - "utf8", + { version: 1, files: sortedRecords }, + config.storageDir, ) - await hardenPrivateFile(manifestPath) } export async function readEmptyTextFiles(config: Config): Promise { @@ -194,14 +202,17 @@ export async function readEmptyTextFiles(config: Config): Promise { - await ensurePrivateDirectory(config.storageDir) - const db = await lancedb.connect(config.storageDir) - const tableNames = await db.tableNames() - if (!tableNames.includes(config.tableName)) { - return null - } - return db.openTable(config.tableName) +export async function openRowsTable( + config: Config, + connection?: lancedb.Connection, +): Promise { + return withConnection(config, connection, async (db) => { + const tableNames = await db.tableNames() + if (!tableNames.includes(config.tableName)) { + return null + } + return db.openTable(config.tableName) + }) } export async function readRows(config: Config): Promise { @@ -293,3 +304,34 @@ function isEmptyTextFileRecord(value: unknown): value is EmptyTextFileRecord { function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error } + +async function withConnection( + config: Config, + connection: lancedb.Connection | undefined, + operation: (connection: lancedb.Connection) => Promise, +): Promise { + const activeConnection = connection ?? (await connectStore(config)) + try { + return await operation(activeConnection) + } finally { + if (!connection) { + activeConnection.close() + } + } +} + +async function writePrivateJsonAtomic( + targetPath: string, + value: unknown, + directory: string, +): Promise { + await ensurePrivateDirectory(directory) + const temporaryPath = `${targetPath}.${process.pid}.${randomUUID()}.tmp` + try { + await writeFile(temporaryPath, JSON.stringify(value, null, 2), "utf8") + await hardenPrivateFile(temporaryPath) + await rename(temporaryPath, targetPath) + } finally { + await rm(temporaryPath, { force: true }) + } +} diff --git a/packages/ragmir-core/src/types.ts b/packages/ragmir-core/src/types.ts index cfa9d68..f9f8ad3 100644 --- a/packages/ragmir-core/src/types.ts +++ b/packages/ragmir-core/src/types.ts @@ -220,7 +220,12 @@ export interface VectorRow extends TextChunk { embeddingModel: string } -export interface IngestOptions { +export interface OperationOptions { + signal?: AbortSignal + timeoutMs?: number +} + +export interface IngestOptions extends OperationOptions { cwd?: PathLike rebuild?: boolean } @@ -361,7 +366,7 @@ export interface KnowledgeBaseSourceCatalog { } } -export interface SearchOptions { +export interface SearchOptions extends OperationOptions { cwd?: PathLike topK?: number contextRadius?: number @@ -414,7 +419,7 @@ export interface SearchScoreExplanation { matchedTerms: string[] } -export interface ExpandCitationOptions { +export interface ExpandCitationOptions extends OperationOptions { cwd?: PathLike contextRadius?: number } @@ -459,7 +464,7 @@ export interface SourceDiagnostics { mirrorCandidates: SourcePathCandidate[] } -export interface ResearchOptions { +export interface ResearchOptions extends OperationOptions { cwd?: PathLike topK?: number includeCode?: boolean diff --git a/packages/ragmir-landing/messages/en.json b/packages/ragmir-landing/messages/en.json index 7485488..7fdc1f7 100644 --- a/packages/ragmir-landing/messages/en.json +++ b/packages/ragmir-landing/messages/en.json @@ -12,7 +12,7 @@ "language_label": "Language", "seo_home_title": "Ragmir: local RAG for coding agents and scripts", "seo_home_description": "Give coding agents and local scripts cited context from private specifications, monorepo docs, PDFs, spreadsheets, and synced folders.", - "seo_home_keywords": "Ragmir, local RAG, coding agents, AI coding agents, cited retrieval, private specifications, monorepo documentation, PDF search, XLSX search, synced Drive files, local automation scripts, MCP, TypeScript", + "seo_home_keywords": "Ragmir, local RAG, coding agents, AI coding agents, cited retrieval, private specifications, monorepo documentation, PDF search, XLSX search, synced Drive files, local automation scripts, persistent Node.js RAG client, local RAG API, MCP, TypeScript", "seo_author": "Jean-Baptiste Thery", "seo_robots": "index, follow", "seo_image_alt": "Ragmir local RAG with cited retrieval for coding agents", @@ -113,19 +113,19 @@ "features_install_label": "Install", "features_run_label": "Run", "features_core_title": "Ragmir Core", - "features_core_text": "Local RAG retrieves cited project evidence for coding agents.", - "features_core_item_1": "Preview redacted chunks before indexing.", - "features_core_item_2": "Explain rankings and filter by source or structure.", + "features_core_text": "Retrieve cited project evidence for agents, scripts, and Node.js workers.", + "features_core_item_1": "Preview redacted chunks and explain retrieval rankings.", + "features_core_item_2": "Reuse one persistent local client in a Node.js worker.", "features_core_item_3": "Return verifiable file, line, chunk, and PDF page citations.", "features_core_item_4": "Run offline and keep generated state under ignored .ragmir/.", "features_tts_title": "Ragmir TTS", - "features_tts_text": "Local TTS renders reviewed, cited project text as local audio.", + "features_tts_text": "Render reviewed, cited project text as local audio for agents and scripts.", "features_tts_item_1": "Render local text as offline WAV or explicit online MP3.", "features_tts_item_2": "Work from cited research and reports.", "features_tts_item_3": "Render WAV offline after model preload.", "features_tts_item_4": "Keep audio under ignored .ragmir/audio; Edge TTS stays opt-in.", "features_chat_title": "Ragmir Chat", - "features_chat_text": "Local Chat turns retrieved project evidence into cited answers.", + "features_chat_text": "Turn retrieved project evidence into cited local answers for agents and scripts.", "features_chat_item_1": "Ground every answer in cited Ragmir passages.", "features_chat_item_2": "Choose verified Qwen or Gemma profiles.", "features_chat_item_3": "Use Metal, CUDA, or Vulkan where supported.", @@ -221,6 +221,8 @@ "faq_role_answer": "Ragmir Core returns cited passages from your local files; the agent or application you choose decides what to do with them. Optional rgr chat adds cited answer generation from a verified local profile when the response must also stay on the workstation.", "faq_formats_question": "Which file formats can Ragmir index?", "faq_formats_answer": "Source code, text, Markdown, PDF, Office and OpenDocument, EPUB, HTML, CSV, JSON, and YAML, plus the custom text extensions you enable. Unsupported files are flagged explicitly, and scanned PDFs can use an optional OCR command.", + "faq_server_question": "Does Ragmir run as a web server or use a fixed port?", + "faq_server_answer": "No. Use the local stdio MCP server for coding agents or embed the TypeScript client in a stateful Node.js process. Ragmir opens no HTTP port; a network-facing application owns authentication, authorization, rate limits, and transport security.", "closing_eyebrow": "Start with the core", "closing_title": "Add local RAG to your agent. Keep your project files local.", "closing_text": "Connect cited passages to your preferred agent, or prepare a local Chat profile and keep retrieval and answer generation offline. No hosted Ragmir account is required.", diff --git a/packages/ragmir-landing/messages/fr.json b/packages/ragmir-landing/messages/fr.json index aed0ae8..ce2cd2b 100644 --- a/packages/ragmir-landing/messages/fr.json +++ b/packages/ragmir-landing/messages/fr.json @@ -12,7 +12,7 @@ "language_label": "Langue", "seo_home_title": "Ragmir : RAG local pour agents de code et scripts", "seo_home_description": "Donnez aux agents de code et scripts locaux un contexte cité issu de spécifications privées, docs de monorepo, PDF, tableurs et dossiers synchronisés.", - "seo_home_keywords": "Ragmir, RAG local, agents de code, agents IA de code, recherche citée, spécifications privées, documentation monorepo, recherche PDF, recherche XLSX, fichiers Drive synchronisés, scripts d'automatisation locaux, MCP, TypeScript", + "seo_home_keywords": "Ragmir, RAG local, agents de code, agents IA de code, recherche citée, spécifications privées, documentation monorepo, recherche PDF, recherche XLSX, fichiers Drive synchronisés, scripts d'automatisation locaux, client RAG Node.js persistant, API RAG locale, MCP, TypeScript", "seo_author": "Jean-Baptiste Thery", "seo_robots": "index, follow", "seo_image_alt": "Ragmir RAG local avec recherche citée pour agents de code", @@ -113,19 +113,19 @@ "features_install_label": "Installation", "features_run_label": "Commande", "features_core_title": "Ragmir Core", - "features_core_text": "Le RAG local récupère des preuves citées du projet pour les agents de code.", - "features_core_item_1": "Prévisualisez les segments masqués avant l'indexation.", - "features_core_item_2": "Expliquez les classements et filtrez par source ou structure.", + "features_core_text": "Récupérez les preuves citées du projet pour les agents, scripts et processus Node.js.", + "features_core_item_1": "Prévisualisez les segments masqués et expliquez le classement.", + "features_core_item_2": "Réutilisez un client local dans un processus Node.js persistant.", "features_core_item_3": "Obtenez des citations vérifiables vers le fichier, la ligne, le segment ou la page PDF.", "features_core_item_4": "Fonctionnez hors ligne et gardez l'état généré dans .ragmir/, ignoré par Git.", "features_tts_title": "Ragmir TTS", - "features_tts_text": "Le TTS local convertit un texte du projet, cité et validé, en audio local.", + "features_tts_text": "Convertissez un texte cité et validé en audio local pour les agents et les scripts.", "features_tts_item_1": "Convertissez du texte local en WAV hors ligne, ou en MP3 via le mode en ligne explicite.", "features_tts_item_2": "Partez de recherches et de rapports cités.", "features_tts_item_3": "Générez des WAV hors ligne après le préchargement du modèle.", "features_tts_item_4": "Gardez l'audio dans .ragmir/audio, ignoré par Git ; Edge TTS reste optionnel.", "features_chat_title": "Ragmir Chat", - "features_chat_text": "Le Chat local transforme les passages récupérés en réponses locales et citées.", + "features_chat_text": "Transformez les preuves du projet en réponses locales citées pour les agents et scripts.", "features_chat_item_1": "Fondez chaque réponse sur des passages Ragmir cités.", "features_chat_item_2": "Choisissez un profil Qwen ou Gemma vérifié.", "features_chat_item_3": "Utilisez Metal, CUDA ou Vulkan selon la machine.", @@ -221,6 +221,8 @@ "faq_role_answer": "Ragmir Core renvoie des passages cités depuis vos fichiers locaux ; l'agent ou l'application choisie décide quoi en faire. L'option rgr chat ajoute une génération de réponse citée avec un profil local vérifié lorsque la réponse doit aussi rester sur le poste.", "faq_formats_question": "Quels formats de fichiers Ragmir peut-il indexer ?", "faq_formats_answer": "Code source, texte, Markdown, PDF, Office et OpenDocument, EPUB, HTML, CSV, JSON et YAML, plus les extensions texte personnalisées que vous activez. Les fichiers non supportés sont signalés explicitement, et les PDF scannés peuvent utiliser une commande OCR optionnelle.", + "faq_server_question": "Ragmir fonctionne-t-il comme un serveur web avec un port fixe ?", + "faq_server_answer": "Non. Utilisez le serveur MCP local sur stdio pour les agents de code, ou intégrez le client TypeScript dans un processus Node.js avec état. Ragmir n'ouvre aucun port HTTP ; une application exposée au réseau gère l'authentification, les autorisations, les limites et la sécurité du transport.", "closing_eyebrow": "Commencer par le cœur", "closing_title": "Ajoutez un RAG local à votre agent. Gardez vos fichiers projet en local.", "closing_text": "Branchez les passages cités sur votre agent préféré, ou préparez un profil Chat local pour garder recherche et réponse hors ligne. Aucun compte Ragmir hébergé n'est requis.", diff --git a/packages/ragmir-landing/public/ai.txt b/packages/ragmir-landing/public/ai.txt index e950d41..f11a71e 100644 --- a/packages/ragmir-landing/public/ai.txt +++ b/packages/ragmir-landing/public/ai.txt @@ -9,6 +9,10 @@ MCP. - Core is retrieval-first. It does not call an LLM or synthesize answers. - Core is model-agnostic. Any compatible coding agent, script, CLI, TypeScript application, or MCP client can consume its cited results. +- For repeated work in a stateful Node.js process, use one `createRagmirClient()` per project root + and close it during shutdown. Use top-level functions for one-shot scripts. +- Do not describe Ragmir as an HTTP service or assign it a fixed port. Network-facing hosts own + transport security, authentication, authorization, and rate limits. - The default `local-hash` provider works offline with no model or API key. It is lexical/hash retrieval, not semantic embeddings. - Semantic retrieval, local GGUF chat, audio rendering, and PDF OCR are explicit optional paths. diff --git a/packages/ragmir-landing/public/llms.txt b/packages/ragmir-landing/public/llms.txt index 64e8410..636c394 100644 --- a/packages/ragmir-landing/public/llms.txt +++ b/packages/ragmir-landing/public/llms.txt @@ -9,6 +9,10 @@ passages through a CLI, TypeScript API, or local read-focused MCP server. - `@jcode.labs/ragmir` is retrieval-first. Core does not call an LLM or write generated answers. - Core is model-agnostic. The user can connect a preferred coding agent or script through CLI, TypeScript, or MCP. +- `createRagmirClient()` reuses one local connection for repeated work in a stateful Node.js process, + supports cancellation and timeouts, and closes after active operations finish. +- Ragmir does not open an HTTP port. Network transports, authentication, authorization, and rate + limits belong to the embedding application. - The default `local-hash` path works offline with no account, API key, or model download. - Semantic Transformers.js embeddings are an explicit opt-in and require a rebuilt index. - `@jcode.labs/ragmir-chat` adds optional cited synthesis with a local GGUF model. diff --git a/packages/ragmir-landing/src/components/sections/faq.astro b/packages/ragmir-landing/src/components/sections/faq.astro index 03a34e7..a6a6953 100644 --- a/packages/ragmir-landing/src/components/sections/faq.astro +++ b/packages/ragmir-landing/src/components/sections/faq.astro @@ -14,6 +14,7 @@ const faqItems = [ { question: t("faq_compare_question"), answer: t("faq_compare_answer") }, { question: t("faq_role_question"), answer: t("faq_role_answer") }, { question: t("faq_formats_question"), answer: t("faq_formats_answer") }, + { question: t("faq_server_question"), answer: t("faq_server_answer") }, ] --- diff --git a/packages/ragmir-landing/src/components/sections/footer.astro b/packages/ragmir-landing/src/components/sections/footer.astro index 0a59a53..31c5e8b 100644 --- a/packages/ragmir-landing/src/components/sections/footer.astro +++ b/packages/ragmir-landing/src/components/sections/footer.astro @@ -77,7 +77,7 @@ const marqueeCopy = href="https://www.npmjs.com/package/@jcode.labs/ragmir" target="_blank" rel="noopener noreferrer" - aria-label={`${t("footer_version_label")}: ${version}`} + aria-label={`${t("footer_version_label")}: v${version}`} class="font-semibold text-foreground no-underline transition hover:opacity-80" > v{version} diff --git a/packages/ragmir-landing/src/pages/[...locale]/index.astro b/packages/ragmir-landing/src/pages/[...locale]/index.astro index 415f727..8c83147 100644 --- a/packages/ragmir-landing/src/pages/[...locale]/index.astro +++ b/packages/ragmir-landing/src/pages/[...locale]/index.astro @@ -51,6 +51,7 @@ const structuredData = [ "confidential document search", "Model Context Protocol", "AI agent tooling", + "Node.js RAG library", ], sameAs: ["https://github.com/jcode-works"], }, diff --git a/tests/public-api-consumer/consumer.ts b/tests/public-api-consumer/consumer.ts index 48ad256..d1de0a2 100644 --- a/tests/public-api-consumer/consumer.ts +++ b/tests/public-api-consumer/consumer.ts @@ -1,13 +1,20 @@ import { + connectMcpServer, + createMcpServer, + createRagmirClient, enableSemanticEmbeddings, ingest, + isRagmirError, pullEmbeddingModel, redactText, search, type Config, type EnableSemanticEmbeddingsResult, type IngestOptions, + type OperationOptions, type PullEmbeddingModelResult, + type RagmirClient, + type RagmirErrorCode, type RedactionCount, type SearchOptions, } from "@jcode.labs/ragmir" @@ -21,6 +28,10 @@ import { renderSpeech, type RenderSpeechOptions } from "@jcode.labs/ragmir-tts" const cwd = process.cwd() const ingestOptions = { cwd, rebuild: false } satisfies IngestOptions const searchOptions = { cwd, topK: 5, explain: true } satisfies SearchOptions +const operationOptions = { + signal: AbortSignal.timeout(5_000), + timeoutMs: 10_000, +} satisfies OperationOptions const source = { relativePath: "docs/decision.md", chunkIndex: 0, @@ -43,6 +54,15 @@ const speechOptions = { void ingest(ingestOptions) void search("What changed?", searchOptions) +void createRagmirClient({ cwd }).then(async (client: RagmirClient) => { + await client.search("What changed?", operationOptions) + await client.close() +}) +void createMcpServer(cwd) +type McpTransport = Parameters[0] +declare const transport: McpTransport +void connectMcpServer(transport, cwd) +void isRagmirError(new Error("example")) void generateChatAnswer(chatOptions) void renderSpeech(speechOptions) @@ -50,7 +70,9 @@ declare const config: Config const semanticResult: Promise = enableSemanticEmbeddings(cwd) const pullResult: Promise = pullEmbeddingModel(config) const redactions: RedactionCount[] = redactText("example", config).counts +const errorCode: RagmirErrorCode = "TIMEOUT" void semanticResult void pullResult void redactions +void errorCode