From 46cbae0f4b9d3ba339357340eb49458665a6a504 Mon Sep 17 00:00:00 2001 From: meihua3 <91116974+CHJ30@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:23:45 +0800 Subject: [PATCH] ci: bootstrap RAG evaluation pipeline --- .github/workflows/rag-evaluation.yml | 89 +++++++ .gitignore | 4 + docs/rag-evaluation-bootstrap.md | 61 +++++ services/chat/package.json | 1 + services/chat/rag/evaluation/thresholds.json | 11 + services/chat/scripts/run-rag-evaluation.ts | 228 ++++++++++++++++++ services/chat/test/fixtures/rag-eval-set.json | 46 ++++ services/ragas-evaluator/README.md | 31 +++ services/ragas-evaluator/pyproject.toml | 29 +++ .../ragas_evaluator/__init__.py | 2 + .../ragas-evaluator/ragas_evaluator/app.py | 132 ++++++++++ services/ragas-evaluator/tests/test_app.py | 58 +++++ 12 files changed, 692 insertions(+) create mode 100644 .github/workflows/rag-evaluation.yml create mode 100644 docs/rag-evaluation-bootstrap.md create mode 100644 services/chat/rag/evaluation/thresholds.json create mode 100644 services/chat/scripts/run-rag-evaluation.ts create mode 100644 services/chat/test/fixtures/rag-eval-set.json create mode 100644 services/ragas-evaluator/README.md create mode 100644 services/ragas-evaluator/pyproject.toml create mode 100644 services/ragas-evaluator/ragas_evaluator/__init__.py create mode 100644 services/ragas-evaluator/ragas_evaluator/app.py create mode 100644 services/ragas-evaluator/tests/test_app.py diff --git a/.github/workflows/rag-evaluation.yml b/.github/workflows/rag-evaluation.yml new file mode 100644 index 0000000..8123ebc --- /dev/null +++ b/.github/workflows/rag-evaluation.yml @@ -0,0 +1,89 @@ +name: RAG evaluation bootstrap + +on: + pull_request: + paths: + - "services/chat/rag/**" + - "services/chat/scripts/run-rag-evaluation.ts" + - "services/chat/test/fixtures/rag-eval-set.json" + - "services/ragas-evaluator/**" + - "knowledge/**" + - ".github/workflows/rag-evaluation.yml" + workflow_dispatch: + +jobs: + bootstrap-evaluation: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + RAGAS_MOCK: "true" + RAG_EVAL_BOOTSTRAP_MODE: "true" + RAGAS_SERVICE_URL: http://127.0.0.1:8000 + + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: services/ragas-evaluator/pyproject.toml + + - name: Install JavaScript dependencies + run: bun install --frozen-lockfile + + - name: Install evaluator dependencies + working-directory: services/ragas-evaluator + run: python -m pip install -e ".[test]" + + - name: Test evaluator service + working-directory: services/ragas-evaluator + run: python -m pytest + + - name: Test offline RAG metrics and adapter + working-directory: services/chat + run: bun test test/chapter11-rag.spec.ts + + - name: Start bootstrap evaluator + working-directory: services/ragas-evaluator + run: >- + python -m uvicorn ragas_evaluator.app:app + --host 127.0.0.1 --port 8000 + > ragas-evaluator.log 2>&1 & + + - name: Wait for evaluator health + shell: bash + run: | + for attempt in {1..30}; do + if curl --fail --silent http://127.0.0.1:8000/health; then + exit 0 + fi + sleep 2 + done + echo "RAGAS evaluator did not become healthy" + exit 1 + + - name: Run bootstrap evaluation + working-directory: services/chat + run: bun run scripts/run-rag-evaluation.ts + + - name: Upload evaluation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: rag-evaluation-bootstrap-report + path: artifacts/rag-evaluation/ + if-no-files-found: warn + + - name: Upload evaluator log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ragas-evaluator-log + path: services/ragas-evaluator/ragas-evaluator.log + if-no-files-found: ignore + diff --git a/.gitignore b/.gitignore index 839b8c3..5125a3b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,10 @@ # ─── Tests & coverage ───────────────────────────────────────────────────────── **/coverage/ **/.nyc_output/ +artifacts/ +**/.venv/ +**/.pytest_cache/ +**/__pycache__/ # ─── Environment variables ──────────────────────────────────────────────────── # Remove the "!" line below if you use .env.example files and want to commit them diff --git a/docs/rag-evaluation-bootstrap.md b/docs/rag-evaluation-bootstrap.md new file mode 100644 index 0000000..ca7c7bf --- /dev/null +++ b/docs/rag-evaluation-bootstrap.md @@ -0,0 +1,61 @@ +# RAG evaluation bootstrap + +This initial setup proves the CI wiring before a reviewed legal evaluation +dataset and judge credentials are available. + +## What bootstrap mode proves + +The workflow exercises this complete path: + +1. Load `services/chat/test/fixtures/rag-eval-set.json`. +2. Read each sample's explicitly synthetic `bootstrap` RAG output. +3. Calculate offline Recall@K, MRR and NDCG@K. +4. Call the Python evaluator through `POST /evaluate`. +5. Write `artifacts/rag-evaluation/result.json` and `summary.md`. +6. Compare every metric with `thresholds.json` and set an exit code. + +`RAGAS_MOCK=true` makes the evaluator return zero for each generation metric. +All thresholds are initially zero. The report always records +`bootstrapMode: true` and warns that the result is not a quality measurement. + +## Run locally + +Install the Python service once: + +```powershell +cd services/ragas-evaluator +python -m venv .venv +.venv\Scripts\python -m pip install -e ".[test]" +``` + +Start it in one PowerShell terminal: + +```powershell +$env:RAGAS_MOCK='true' +.venv\Scripts\python -m uvicorn ragas_evaluator.app:app --host 127.0.0.1 --port 8000 +``` + +Run the pipeline in a second terminal from `services/chat`: + +```powershell +$env:RAG_EVAL_BOOTSTRAP_MODE='true' +$env:RAGAS_MOCK='true' +$env:RAGAS_SERVICE_URL='http://127.0.0.1:8000' +bun run scripts/run-rag-evaluation.ts +``` + +## Move to real evaluation + +1. Replace the samples with reviewed legal questions and stable document IDs. +2. Start the real NestJS service with the fixed evaluation corpus ingested. +3. Set `RAG_EVAL_RAG_ENDPOINT` to its authenticated `/api/rag-demo/ask` + endpoint and provide `RAG_EVAL_RAG_TOKEN`. +4. Set `RAG_EVAL_BOOTSTRAP_MODE=false` and `RAGAS_MOCK=false`. +5. Configure the RAGAS model environment variables documented in + `services/ragas-evaluator/README.md`. +6. Install the real evaluator dependencies with + `python -m pip install -e ".[real]"`. +7. Raise the thresholds only after recording and reviewing a baseline. + +Exit codes are `0` for pass, `1` for a quality threshold failure and `2` for +an evaluation infrastructure failure. diff --git a/services/chat/package.json b/services/chat/package.json index 3e16987..5feac5a 100644 --- a/services/chat/package.json +++ b/services/chat/package.json @@ -8,6 +8,7 @@ "start": "NODE_ENV=production bun run dist/src/main.js", "typecheck": "tsc --noEmit", "lint": "tsc --noEmit", + "rag:evaluate:bootstrap": "RAG_EVAL_BOOTSTRAP_MODE=true RAGAS_MOCK=true bun run scripts/run-rag-evaluation.ts", "db:generate": "prisma generate", "db:migrate": "prisma migrate dev --name init", "db:studio": "prisma studio" diff --git a/services/chat/rag/evaluation/thresholds.json b/services/chat/rag/evaluation/thresholds.json new file mode 100644 index 0000000..6850291 --- /dev/null +++ b/services/chat/rag/evaluation/thresholds.json @@ -0,0 +1,11 @@ +{ + "recallAtK": 0, + "mrr": 0, + "ndcgAtK": 0, + "faithfulness": 0, + "answer_relevancy": 0, + "context_precision": 0, + "context_recall": 0, + "highRiskPassRate": 0 +} + diff --git a/services/chat/scripts/run-rag-evaluation.ts b/services/chat/scripts/run-rag-evaluation.ts new file mode 100644 index 0000000..17d01ac --- /dev/null +++ b/services/chat/scripts/run-rag-evaluation.ts @@ -0,0 +1,228 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { mrr, ndcgAtK, recallAtK } from '../rag/evaluation/retrieval-metrics.js'; +import { runRagasEvaluation } from '../rag/evaluation/ragas-runner.js'; + +const RAGAS_METRICS = [ + 'faithfulness', + 'answer_relevancy', + 'context_precision', + 'context_recall', +] as const; + +interface BootstrapOutput { + answer: string; + contexts: string[]; + retrievedDocIds: string[]; +} + +interface EvaluationCase { + id: string; + category: string; + riskLevel: 'low' | 'medium' | 'high'; + question: string; + expectedDocIds: string[]; + groundTruth: string; + bootstrap?: BootstrapOutput; +} + +interface Thresholds { + recallAtK: number; + mrr: number; + ndcgAtK: number; + faithfulness: number; + answer_relevancy: number; + context_precision: number; + context_recall: number; + highRiskPassRate: number; +} + +interface RagOutput extends BootstrapOutput {} + +function isBootstrapMode(): boolean { + return process.env.RAG_EVAL_BOOTSTRAP_MODE?.toLowerCase() === 'true'; +} + +function assertStringArray(value: unknown, field: string): asserts value is string[] { + if (!Array.isArray(value) || value.some(item => typeof item !== 'string')) { + throw new Error(`${field} must be an array of strings`); + } +} + +function parseCases(value: unknown): EvaluationCase[] { + if (!Array.isArray(value) || value.length === 0) throw new Error('Evaluation dataset must not be empty'); + return value.map((item, index) => { + if (typeof item !== 'object' || item === null) throw new Error(`Sample ${index} must be an object`); + const sample = item as Record; + for (const field of ['id', 'category', 'riskLevel', 'question', 'groundTruth']) { + if (typeof sample[field] !== 'string' || sample[field] === '') { + throw new Error(`Sample ${index}.${field} must be a non-empty string`); + } + } + assertStringArray(sample.expectedDocIds, `Sample ${index}.expectedDocIds`); + return sample as unknown as EvaluationCase; + }); +} + +function parseThresholds(value: unknown): Thresholds { + if (typeof value !== 'object' || value === null) throw new Error('Thresholds must be an object'); + const data = value as Record; + const names: Array = [ + 'recallAtK', 'mrr', 'ndcgAtK', 'faithfulness', 'answer_relevancy', + 'context_precision', 'context_recall', 'highRiskPassRate', + ]; + for (const name of names) { + if (typeof data[name] !== 'number' || !Number.isFinite(data[name])) { + throw new Error(`Threshold ${name} must be a finite number`); + } + } + return data as unknown as Thresholds; +} + +async function queryRag(sample: EvaluationCase, bootstrapMode: boolean): Promise { + if (bootstrapMode) { + if (!sample.bootstrap) throw new Error(`Sample ${sample.id} has no bootstrap output`); + return sample.bootstrap; + } + + const endpoint = process.env.RAG_EVAL_RAG_ENDPOINT; + if (!endpoint) throw new Error('RAG_EVAL_RAG_ENDPOINT is required outside bootstrap mode'); + const headers: Record = { 'content-type': 'application/json' }; + if (process.env.RAG_EVAL_RAG_TOKEN) headers.authorization = `Bearer ${process.env.RAG_EVAL_RAG_TOKEN}`; + const response = await fetch(endpoint, { + method: 'POST', + headers, + body: JSON.stringify({ question: sample.question, topK: 5 }), + }); + if (!response.ok) throw new Error(`RAG endpoint returned HTTP ${response.status} for ${sample.id}`); + const data = await response.json() as { + answer?: unknown; + citations?: Array<{ documentId?: unknown; quote?: unknown }>; + }; + if (typeof data.answer !== 'string' || !Array.isArray(data.citations)) { + throw new Error(`RAG endpoint returned an invalid response for ${sample.id}`); + } + return { + answer: data.answer, + contexts: data.citations + .map(citation => citation.quote) + .filter((quote): quote is string => typeof quote === 'string'), + retrievedDocIds: data.citations + .map(citation => citation.documentId) + .filter((id): id is string => typeof id === 'string'), + }; +} + +function average(values: number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function markdownReport(result: Record, failures: string[]): string { + const summary = result.summary as Record; + const rows = Object.entries(summary) + .map(([metric, score]) => `| ${metric} | ${score.toFixed(4)} |`) + .join('\n'); + return [ + '# RAG evaluation report', + '', + `> Mode: **${result.bootstrapMode ? 'BOOTSTRAP (wiring only; not a quality result)' : 'REAL'}**`, + '', + `Generated: ${result.generatedAt}`, + '', + '| Metric | Score |', + '| --- | ---: |', + rows, + '', + failures.length ? `Failures: ${failures.join(', ')}` : 'All configured zero thresholds passed.', + '', + ].join('\n'); +} + +async function main(): Promise { + const scriptDirectory = dirname(fileURLToPath(import.meta.url)); + const repositoryRoot = resolve(scriptDirectory, '../../..'); + const datasetPath = process.env.RAG_EVAL_DATASET + ? resolve(process.env.RAG_EVAL_DATASET) + : resolve(scriptDirectory, '../test/fixtures/rag-eval-set.json'); + const thresholdsPath = process.env.RAG_EVAL_THRESHOLDS + ? resolve(process.env.RAG_EVAL_THRESHOLDS) + : resolve(scriptDirectory, '../rag/evaluation/thresholds.json'); + const artifactsDirectory = resolve(repositoryRoot, 'artifacts/rag-evaluation'); + const bootstrapMode = isBootstrapMode(); + + if (bootstrapMode && process.env.RAGAS_MOCK?.toLowerCase() !== 'true') { + throw new Error('Bootstrap evaluation requires RAGAS_MOCK=true on the evaluator service'); + } + + const cases = parseCases(JSON.parse(await readFile(datasetPath, 'utf8'))); + const thresholds = parseThresholds(JSON.parse(await readFile(thresholdsPath, 'utf8'))); + const evaluated = []; + + for (const sample of cases) { + const output = await queryRag(sample, bootstrapMode); + const retrievalApplicable = sample.expectedDocIds.length > 0; + evaluated.push({ + ...sample, + bootstrap: undefined, + ...output, + retrievalApplicable, + recallAtK: retrievalApplicable + ? recallAtK(output.retrievedDocIds, sample.expectedDocIds, 5) + : null, + ndcgAtK: retrievalApplicable + ? ndcgAtK(output.retrievedDocIds, sample.expectedDocIds, 5) + : null, + }); + } + + const ragas = await runRagasEvaluation({ + samples: evaluated.map(sample => ({ + question: sample.question, + answer: sample.answer, + contexts: sample.contexts, + ground_truth: sample.groundTruth, + })), + metrics: [...RAGAS_METRICS], + }); + if (ragas === null) throw new Error('RAGAS evaluator is unavailable; no threshold decision was made'); + + const retrievalSamples = evaluated.filter(sample => sample.retrievalApplicable); + const summary = { + recallAtK: average(retrievalSamples.map(sample => sample.recallAtK!)), + mrr: mrr( + retrievalSamples.map(sample => sample.retrievedDocIds), + retrievalSamples.map(sample => sample.expectedDocIds), + ), + ndcgAtK: average(retrievalSamples.map(sample => sample.ndcgAtK!)), + ...ragas, + highRiskPassRate: 0, + }; + const failures = Object.entries(thresholds) + .filter(([metric, threshold]) => (summary[metric as keyof typeof summary] ?? 0) < threshold) + .map(([metric, threshold]) => `${metric} < ${threshold}`); + const result = { + schemaVersion: 1, + bootstrapMode, + warning: bootstrapMode + ? 'Synthetic outputs and mock RAGAS scores validate wiring only; they do not measure RAG quality.' + : undefined, + generatedAt: new Date().toISOString(), + datasetPath, + thresholds, + summary, + failures, + samples: evaluated, + }; + + await mkdir(artifactsDirectory, { recursive: true }); + await writeFile(resolve(artifactsDirectory, 'result.json'), `${JSON.stringify(result, null, 2)}\n`, 'utf8'); + await writeFile(resolve(artifactsDirectory, 'summary.md'), markdownReport(result, failures), 'utf8'); + console.log(markdownReport(result, failures)); + if (failures.length > 0) process.exitCode = 1; +} + +main().catch(error => { + console.error(`[RAG evaluation infrastructure error] ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; +}); diff --git a/services/chat/test/fixtures/rag-eval-set.json b/services/chat/test/fixtures/rag-eval-set.json new file mode 100644 index 0000000..764e636 --- /dev/null +++ b/services/chat/test/fixtures/rag-eval-set.json @@ -0,0 +1,46 @@ +[ + { + "id": "bootstrap-civil-001", + "category": "合同责任", + "riskLevel": "low", + "question": "合同一方不履行合同义务时，通常应承担什么责任？", + "expectedDocIds": ["civil-code-bootstrap"], + "groundTruth": "当事人一方不履行合同义务或者履行合同义务不符合约定的，应依法承担继续履行、采取补救措施或者赔偿损失等违约责任。", + "bootstrap": { + "answer": "应依法承担继续履行、采取补救措施或者赔偿损失等违约责任。", + "contexts": [ + "演示上下文：当事人一方不履行合同义务或者履行合同义务不符合约定的，应承担继续履行、采取补救措施或者赔偿损失等违约责任。" + ], + "retrievedDocIds": ["civil-code-bootstrap"] + } + }, + { + "id": "bootstrap-civil-002", + "category": "民事权利", + "riskLevel": "low", + "question": "自然人的民事权利能力从什么时候开始？", + "expectedDocIds": ["civil-code-bootstrap"], + "groundTruth": "自然人的民事权利能力始于出生，终于死亡。", + "bootstrap": { + "answer": "自然人的民事权利能力始于出生，终于死亡。", + "contexts": [ + "演示上下文：自然人从出生时起到死亡时止，具有民事权利能力，依法享有民事权利，承担民事义务。" + ], + "retrievedDocIds": ["civil-code-bootstrap"] + } + }, + { + "id": "bootstrap-out-of-scope-001", + "category": "知识边界", + "riskLevel": "low", + "question": "请根据当前知识库说明某个虚构国家最新的公司税率。", + "expectedDocIds": [], + "groundTruth": "当前法律知识库没有足够资料，应明确说明无法回答。", + "bootstrap": { + "answer": "当前法律知识库资料不足，无法回答该问题。", + "contexts": [], + "retrievedDocIds": [] + } + } +] + diff --git a/services/ragas-evaluator/README.md b/services/ragas-evaluator/README.md new file mode 100644 index 0000000..ba7eac5 --- /dev/null +++ b/services/ragas-evaluator/README.md @@ -0,0 +1,31 @@ +# RAGAS evaluator + +CI-only HTTP wrapper around the Python RAGAS library. It is intentionally not +part of the NestJS production process. + +## Bootstrap mode + +Bootstrap mode validates the dataset -> HTTP -> report -> threshold pipeline +without an API key and always returns `0.0` for requested RAGAS metrics. These +scores are wiring checks, not quality measurements. + +```powershell +$env:RAGAS_MOCK='true' +python -m uvicorn ragas_evaluator.app:app --host 127.0.0.1 --port 8000 +``` + +## Real mode + +Set `RAGAS_MOCK=false` and configure: + +- `RAGAS_OPENAI_API_KEY` (or `OPENAI_API_KEY`) +- `RAGAS_OPENAI_BASE_URL` (optional) +- `RAGAS_JUDGE_MODEL` (defaults to `gpt-4o-mini`) +- `RAGAS_EMBEDDING_MODEL` (defaults to `text-embedding-3-small`) + +Install real-mode dependencies and test: + +```bash +python -m pip install -e ".[real,test]" +python -m pytest +``` diff --git a/services/ragas-evaluator/pyproject.toml b/services/ragas-evaluator/pyproject.toml new file mode 100644 index 0000000..3061c8e --- /dev/null +++ b/services/ragas-evaluator/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["hatchling==1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "autix-ragas-evaluator" +version = "0.1.0" +description = "CI-only RAGAS HTTP adapter for the Autix legal RAG system" +requires-python = ">=3.11" +dependencies = [ + "fastapi==0.139.2", + "uvicorn==0.51.0", +] + +[project.optional-dependencies] +real = [ + "openai==2.46.0", + "ragas==0.4.3", +] +test = [ + "httpx==0.28.1", + "pytest==9.1.1", +] + +[tool.hatch.build.targets.wheel] +packages = ["ragas_evaluator"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/ragas-evaluator/ragas_evaluator/__init__.py b/services/ragas-evaluator/ragas_evaluator/__init__.py new file mode 100644 index 0000000..8957b73 --- /dev/null +++ b/services/ragas-evaluator/ragas_evaluator/__init__.py @@ -0,0 +1,2 @@ +"""Autix RAGAS evaluator service.""" + diff --git a/services/ragas-evaluator/ragas_evaluator/app.py b/services/ragas-evaluator/ragas_evaluator/app.py new file mode 100644 index 0000000..0b775fd --- /dev/null +++ b/services/ragas-evaluator/ragas_evaluator/app.py @@ -0,0 +1,132 @@ +import math +import os +from collections.abc import Callable +from typing import Any + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field + + +SUPPORTED_METRICS = { + "faithfulness", + "answer_relevancy", + "context_precision", + "context_recall", +} + + +class EvaluationSample(BaseModel): + question: str = Field(min_length=1) + answer: str + contexts: list[str] + ground_truth: str + + +class EvaluationRequest(BaseModel): + samples: list[EvaluationSample] = Field(min_length=1) + metrics: list[str] = Field(min_length=1) + + +Evaluator = Callable[[EvaluationRequest], dict[str, float]] + + +def _mock_evaluate(request: EvaluationRequest) -> dict[str, float]: + """Exercise the HTTP/CI wiring without claiming meaningful quality scores.""" + return {metric: 0.0 for metric in request.metrics} + + +def _real_evaluate(request: EvaluationRequest) -> dict[str, float]: + # Imports stay lazy so bootstrap mode and health checks need no model client. + from datasets import Dataset + from openai import AsyncOpenAI, OpenAI + from ragas import evaluate + from ragas.embeddings import embedding_factory + from ragas.llms import llm_factory + from ragas.metrics import ( + answer_relevancy, + context_precision, + context_recall, + faithfulness, + ) + + metric_registry = { + "faithfulness": faithfulness, + "answer_relevancy": answer_relevancy, + "context_precision": context_precision, + "context_recall": context_recall, + } + api_key = os.environ.get("RAGAS_OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY") + if not api_key: + raise RuntimeError("RAGAS_OPENAI_API_KEY or OPENAI_API_KEY is required") + + base_url = os.environ.get("RAGAS_OPENAI_BASE_URL") or os.environ.get("OPENAI_BASE_URL") + async_client = AsyncOpenAI(api_key=api_key, base_url=base_url) + sync_client = OpenAI(api_key=api_key, base_url=base_url) + judge_model = os.environ.get("RAGAS_JUDGE_MODEL", "gpt-4o-mini") + embedding_model = os.environ.get("RAGAS_EMBEDDING_MODEL", "text-embedding-3-small") + + llm = llm_factory(judge_model, client=async_client) + embeddings = embedding_factory( + "openai", + model=embedding_model, + client=sync_client, + interface="modern", + ) + dataset = Dataset.from_list([sample.model_dump() for sample in request.samples]) + result = evaluate( + dataset=dataset, + metrics=[metric_registry[name] for name in request.metrics], + llm=llm, + embeddings=embeddings, + raise_exceptions=False, + ) + + frame = result.to_pandas() + scores: dict[str, float] = {} + for metric in request.metrics: + values = frame[metric].dropna() + score = float(values.mean()) if len(values) else 0.0 + scores[metric] = score if math.isfinite(score) else 0.0 + return scores + + +def create_app(evaluator: Evaluator | None = None) -> FastAPI: + app = FastAPI(title="Autix RAGAS Evaluator", version="0.1.0") + + @app.get("/health") + def health() -> dict[str, Any]: + return { + "status": "ok", + "mode": "bootstrap" if os.environ.get("RAGAS_MOCK", "false").lower() == "true" else "real", + } + + @app.post("/evaluate") + def run_evaluation(request: EvaluationRequest) -> dict[str, float]: + unknown = sorted(set(request.metrics) - SUPPORTED_METRICS) + if unknown: + raise HTTPException(status_code=400, detail=f"Unknown metrics: {unknown}") + selected_evaluator = evaluator + if selected_evaluator is None: + selected_evaluator = ( + _mock_evaluate + if os.environ.get("RAGAS_MOCK", "false").lower() == "true" + else _real_evaluate + ) + try: + scores = selected_evaluator(request) + except HTTPException: + raise + except Exception as error: + raise HTTPException(status_code=503, detail=f"Evaluation failed: {error}") from error + + if set(scores) != set(request.metrics): + raise HTTPException(status_code=503, detail="Evaluator returned an incomplete metric result") + if any(not isinstance(value, (int, float)) or not math.isfinite(value) for value in scores.values()): + raise HTTPException(status_code=503, detail="Evaluator returned a non-finite metric result") + return {name: float(value) for name, value in scores.items()} + + return app + + +app = create_app() + diff --git a/services/ragas-evaluator/tests/test_app.py b/services/ragas-evaluator/tests/test_app.py new file mode 100644 index 0000000..5f441cb --- /dev/null +++ b/services/ragas-evaluator/tests/test_app.py @@ -0,0 +1,58 @@ +from fastapi.testclient import TestClient + +from ragas_evaluator.app import EvaluationRequest, create_app + + +REQUEST = { + "samples": [ + { + "question": "合同一方不履行义务，应承担什么责任？", + "answer": "应承担违约责任。", + "contexts": ["当事人一方不履行合同义务的，应当承担违约责任。"], + "ground_truth": "应承担违约责任。", + } + ], + "metrics": ["faithfulness", "answer_relevancy"], +} + + +def test_health() -> None: + response = TestClient(create_app()).get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +def test_evaluate_uses_injected_evaluator() -> None: + def evaluator(request: EvaluationRequest) -> dict[str, float]: + assert request.samples[0].question.startswith("合同") + return {metric: 0.5 for metric in request.metrics} + + response = TestClient(create_app(evaluator)).post("/evaluate", json=REQUEST) + assert response.status_code == 200 + assert response.json() == {"faithfulness": 0.5, "answer_relevancy": 0.5} + + +def test_bootstrap_mode_returns_zero_scores(monkeypatch) -> None: + monkeypatch.setenv("RAGAS_MOCK", "true") + response = TestClient(create_app()).post("/evaluate", json=REQUEST) + assert response.status_code == 200 + assert response.json() == {"faithfulness": 0.0, "answer_relevancy": 0.0} + + +def test_unknown_metric_returns_400() -> None: + request = {**REQUEST, "metrics": ["not_a_metric"]} + response = TestClient(create_app()).post("/evaluate", json=request) + assert response.status_code == 400 + + +def test_empty_samples_returns_422() -> None: + request = {**REQUEST, "samples": []} + response = TestClient(create_app()).post("/evaluate", json=request) + assert response.status_code == 422 + + +def test_non_finite_result_returns_503() -> None: + response = TestClient(create_app(lambda _request: {"faithfulness": float("nan"), "answer_relevancy": 0.5})).post( + "/evaluate", json=REQUEST + ) + assert response.status_code == 503