ci: bootstrap RAG evaluation pipeline

.github/workflows/rag-evaluation.yml

@@ -0,0 +1,89 @@
+name: RAG evaluation bootstrap
+
+on:
+  pull_request:
+    paths:
+      - "services/chat/rag/**"
+      - "services/chat/scripts/run-rag-evaluation.ts"
+      - "services/chat/test/fixtures/rag-eval-set.json"
+      - "services/ragas-evaluator/**"
+      - "knowledge/**"
+      - ".github/workflows/rag-evaluation.yml"
+  workflow_dispatch:
+
+jobs:
+  bootstrap-evaluation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      RAGAS_MOCK: "true"
+      RAG_EVAL_BOOTSTRAP_MODE: "true"
+      RAGAS_SERVICE_URL: http://127.0.0.1:8000
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: services/ragas-evaluator/pyproject.toml
+
+      - name: Install JavaScript dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Install evaluator dependencies
+        working-directory: services/ragas-evaluator
+        run: python -m pip install -e ".[test]"
+
+      - name: Test evaluator service
+        working-directory: services/ragas-evaluator
+        run: python -m pytest
+
+      - name: Test offline RAG metrics and adapter
+        working-directory: services/chat
+        run: bun test test/chapter11-rag.spec.ts
+
+      - name: Start bootstrap evaluator
+        working-directory: services/ragas-evaluator
+        run: >-
+          python -m uvicorn ragas_evaluator.app:app
+          --host 127.0.0.1 --port 8000
+          > ragas-evaluator.log 2>&1 &
+
+      - name: Wait for evaluator health
+        shell: bash
+        run: |
+          for attempt in {1..30}; do
+            if curl --fail --silent http://127.0.0.1:8000/health; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "RAGAS evaluator did not become healthy"
+          exit 1
+
+      - name: Run bootstrap evaluation
+        working-directory: services/chat
+        run: bun run scripts/run-rag-evaluation.ts
+
+      - name: Upload evaluation report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rag-evaluation-bootstrap-report
+          path: artifacts/rag-evaluation/
+          if-no-files-found: warn
+
+      - name: Upload evaluator log on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ragas-evaluator-log
+          path: services/ragas-evaluator/ragas-evaluator.log
+          if-no-files-found: ignore
+

.gitignore

@@ -25,6 +25,10 @@
 # ─── Tests & coverage ─────────────────────────────────────────────────────────
 **/coverage/
 **/.nyc_output/
+artifacts/
+**/.venv/
+**/.pytest_cache/
+**/__pycache__/
 
 # ─── Environment variables ────────────────────────────────────────────────────
 # Remove the "!" line below if you use .env.example files and want to commit them

docs/rag-evaluation-bootstrap.md

@@ -0,0 +1,61 @@
+# RAG evaluation bootstrap
+
+This initial setup proves the CI wiring before a reviewed legal evaluation
+dataset and judge credentials are available.
+
+## What bootstrap mode proves
+
+The workflow exercises this complete path:
+
+1. Load `services/chat/test/fixtures/rag-eval-set.json`.
+2. Read each sample's explicitly synthetic `bootstrap` RAG output.
+3. Calculate offline Recall@K, MRR and NDCG@K.
+4. Call the Python evaluator through `POST /evaluate`.
+5. Write `artifacts/rag-evaluation/result.json` and `summary.md`.
+6. Compare every metric with `thresholds.json` and set an exit code.
+
+`RAGAS_MOCK=true` makes the evaluator return zero for each generation metric.
+All thresholds are initially zero. The report always records
+`bootstrapMode: true` and warns that the result is not a quality measurement.
+
+## Run locally
+
+Install the Python service once:
+
+```powershell
+cd services/ragas-evaluator
+python -m venv .venv
+.venv\Scripts\python -m pip install -e ".[test]"
+```
+
+Start it in one PowerShell terminal:
+
+```powershell
+$env:RAGAS_MOCK='true'
+.venv\Scripts\python -m uvicorn ragas_evaluator.app:app --host 127.0.0.1 --port 8000
+```
+
+Run the pipeline in a second terminal from `services/chat`:
+
+```powershell
+$env:RAG_EVAL_BOOTSTRAP_MODE='true'
+$env:RAGAS_MOCK='true'
+$env:RAGAS_SERVICE_URL='http://127.0.0.1:8000'
+bun run scripts/run-rag-evaluation.ts
+```
+
+## Move to real evaluation
+
+1. Replace the samples with reviewed legal questions and stable document IDs.
+2. Start the real NestJS service with the fixed evaluation corpus ingested.
+3. Set `RAG_EVAL_RAG_ENDPOINT` to its authenticated `/api/rag-demo/ask`
+   endpoint and provide `RAG_EVAL_RAG_TOKEN`.
+4. Set `RAG_EVAL_BOOTSTRAP_MODE=false` and `RAGAS_MOCK=false`.
+5. Configure the RAGAS model environment variables documented in
+   `services/ragas-evaluator/README.md`.
+6. Install the real evaluator dependencies with
+   `python -m pip install -e ".[real]"`.
+7. Raise the thresholds only after recording and reviewing a baseline.
+
+Exit codes are `0` for pass, `1` for a quality threshold failure and `2` for
+an evaluation infrastructure failure.

services/chat/package.json

@@ -8,6 +8,7 @@
 		"start": "NODE_ENV=production bun run dist/src/main.js",
 		"typecheck": "tsc --noEmit",
 		"lint": "tsc --noEmit",
+		"rag:evaluate:bootstrap": "RAG_EVAL_BOOTSTRAP_MODE=true RAGAS_MOCK=true bun run scripts/run-rag-evaluation.ts",
 		"db:generate": "prisma generate",
 		"db:migrate": "prisma migrate dev --name init",
 		"db:studio": "prisma studio"

services/chat/rag/evaluation/thresholds.json

@@ -0,0 +1,11 @@
+{
+  "recallAtK": 0,
+  "mrr": 0,
+  "ndcgAtK": 0,
+  "faithfulness": 0,
+  "answer_relevancy": 0,
+  "context_precision": 0,
+  "context_recall": 0,
+  "highRiskPassRate": 0
+}
+