raguide/Makefile at main · JavaLyHn/raguide · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# raguide Makefile — operator aliases (CLAUDE.md §9 §10)
#
# Eval-gate local parity:
#   make eval-infra   # boot postgres + qdrant + redis (idempotent)
#   make eval-gold    # run the gold-set (requires eval-infra)
#
# CI mirrors this exactly via .github/workflows/eval.yml `services:` block.

.PHONY: langfuse-up langfuse-down langfuse-logs langfuse-status \
        eval-infra eval-infra-down eval-gold eval-baseline eval-langfuse-dataset \
        eval-dataset-run ci-check help

help:
	@echo "Targets:"
	@echo "  eval-infra       boot postgres+qdrant+redis (required before eval-gold locally)"
	@echo "  eval-infra-down  stop the eval infra stack"
	@echo "  eval-gold        run the gold-set runner (writes report)"
	@echo "  eval-baseline    update the committed baseline (run after intentional behavior change)"
	@echo "  eval-langfuse-dataset  mirror tests/gold_set/*.yaml → Langfuse dataset 'raguide-gold-set' (items only)"
	@echo "  eval-dataset-run       run the gold-set + link each result to its dataset item as a Langfuse dataset run (closed loop; --ragas deferred, see target note)"
	@echo "  langfuse-up      boot the Langfuse 3.x stack"
	@echo "  ci-check         ruff + format + unit tests (the CI Gate-1 set)"

langfuse-up:
	docker compose -f infra/docker-compose.langfuse.yml up -d

langfuse-down:
	docker compose -f infra/docker-compose.langfuse.yml down

langfuse-logs:
	docker compose -f infra/docker-compose.langfuse.yml logs -f langfuse-web langfuse-worker

langfuse-status:
	docker compose -f infra/docker-compose.langfuse.yml ps

eval-infra:
	docker compose -f infra/docker-compose.yml up -d postgres qdrant redis

eval-infra-down:
	docker compose -f infra/docker-compose.yml stop postgres qdrant redis

eval-gold:
	uv run python tests/eval/run_gold_set.py --write-report

eval-baseline:
	uv run python tests/eval/run_gold_set.py --update-baseline

# Mirror the gold-set YAML into a Langfuse dataset (idempotent; YAML stays源真值).
eval-langfuse-dataset:
	uv run python -m raguide_eval.langfuse_dataset

# Closed loop: run the gold-set AND register each result as a Langfuse dataset run
# (each case's trace linked to its raguide-gold-set item + 5-dim/weighted scores).
# Pre-syncs items; needs real LANGFUSE_* env + eval-infra up.
#
# NOTE: the runner also accepts --ragas (goldset.ragas.* retrieval-quality metrics),
# but it is DEFERRED and intentionally NOT enabled here: ragas 0.2.15 needs
# langchain-core 0.3.x while the backend's langgraph 1.2.x needs core 1.x — they can't
# coexist in one venv, so ragas returns NaN→None (the nan-guard skips it gracefully).
# Re-enable --ragas once that conflict is resolved (upgrade ragas / isolate it).
eval-dataset-run:
	uv run python tests/eval/run_gold_set.py --dataset-run --write-report

ci-check:
	uv run --with ruff ruff check .
	uv run --with ruff ruff format --check .
	uv run pytest -m "not integration and not e2e" -v

# --- M2c Modular RAG inner pipeline ---

.PHONY: m2c-infra-up m2c-infra-down m2c-seed m2c-build-reranker-onnx

# One-time, pre-`m2c-infra-up`: convert BAAI/bge-reranker-v2-m3 safetensors
# to ONNX so the TEI cpu image (ORT backend) can load it. Required because
# TEI ships no native arm64 image (we run amd64 via Rosetta) and the cpu
# variant uses ORT, which needs ONNX. Weights stay from BAAI — only format
# changes. Idempotent: skip if output dir already populated.
m2c-build-reranker-onnx:
	@if [ -f infra/tei/bge-reranker-v2-m3-onnx/model.onnx ]; then \
	    echo "infra/tei/bge-reranker-v2-m3-onnx/ already populated — skip"; \
	else \
	    mkdir -p infra/tei/bge-reranker-v2-m3-onnx; \
	    uv run --with 'optimum[onnxruntime]>=1.20' --with 'transformers>=4.40' --with 'torch>=2.4' --with 'sentencepiece>=0.2' \
	        optimum-cli export onnx \
	            --model BAAI/bge-reranker-v2-m3 \
	            --task text-classification \
	            infra/tei/bge-reranker-v2-m3-onnx/; \
	fi

m2c-infra-up: m2c-build-reranker-onnx
	docker compose -f infra/docker-compose.yml up -d \
	    qdrant postgres redis searxng tantivy-server tei-reranker

m2c-infra-down:
	docker compose -f infra/docker-compose.yml stop \
	    tantivy-server tei-reranker

m2c-seed:
	uv run python scripts/seed_kb_demo.py

# Seed the persistent kb-pdf store (infra/kb_pdfs/, served at /static/kb-pdfs)
# from the committed data/m1/*.pdf so old test products show "查看源 PDF".
# Idempotent. New admin PDF uploads land here automatically.
.PHONY: kb-backfill-pdfs
kb-backfill-pdfs:
	bash infra/backfill-kb-pdfs.sh