diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9326774..99e2f2b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,9 +1,15 @@ name: Benchmark # The benchmark is deterministic and reproducible. This workflow re-verifies it -# weekly (and on demand), refreshes the published artifact, and commits the -# result back to the repo, where /api/benchmark-latest and /api/stats serve it. -# It is reproducibility verification, not synthetic daily activity. +# weekly (and on demand), refreshes the published artifact, and publishes the +# result to the unprotected `telemetry` data branch. The default branch is +# ruleset-protected and the default GITHUB_TOKEN cannot push to it, so the +# git-as-database loop lives on `telemetry`. The live stdlib endpoints +# (/api/stats, /api/benchmark-latest) read the freshest artifact from that +# branch at request time, falling back to the copy committed on main. +# +# History accumulates: the runner appends to the prior history, so we seed it +# from the telemetry branch before each run. on: schedule: @@ -11,7 +17,7 @@ on: workflow_dispatch: permissions: - contents: write + contents: write # push the data files to the telemetry branch jobs: benchmark: @@ -28,6 +34,14 @@ jobs: - name: Install package run: pip install -e . + - name: Seed accumulated history from telemetry branch + run: | + if git ls-remote --exit-code --heads origin telemetry >/dev/null 2>&1; then + git fetch origin telemetry --depth=1 + git show origin/telemetry:api/_benchmark_history.json \ + > api/_benchmark_history.json 2>/dev/null || true + fi + - name: Run benchmark run: python -m evalops_workbench.benchmark_runner @@ -51,17 +65,34 @@ jobs: print(f"artifact valid; {artifact['metrics']['n_cases']} cases; age {age:.0f}s") PY - - name: Commit results if changed + - name: Publish artifacts to telemetry branch run: | - git config user.name "eleventh-bot" - git config user.email "noreply@eleventh.dev" - git add api/_benchmark_latest.json api/_benchmark_history.json \ - examples/benchmark-v1/results examples/benchmark-v1/pinned-baseline.json - if git diff --cached --quiet; then - echo "No benchmark changes to commit." + stage="$(mktemp -d)" + mkdir -p "$stage/api" "$stage/examples" + cp api/_benchmark_latest.json api/_benchmark_history.json "$stage/api/" + cp -R examples/benchmark "$stage/examples/" 2>/dev/null || true + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + if git ls-remote --exit-code --heads origin telemetry >/dev/null 2>&1; then + git fetch origin telemetry --depth=1 + git worktree add -B telemetry .telemetry origin/telemetry + else + git worktree add --orphan -b telemetry .telemetry + fi + + mkdir -p .telemetry/api .telemetry/examples + cp "$stage/api/"* .telemetry/api/ + rm -rf .telemetry/examples/benchmark + cp -R "$stage/examples/benchmark" .telemetry/examples/ 2>/dev/null || true + + git -C .telemetry add api examples + if git -C .telemetry diff --cached --quiet; then + echo "No benchmark changes to publish." else - git commit -m "chore(benchmark): scheduled run [skip ci]" - git push + git -C .telemetry commit -m "telemetry: scheduled benchmark $(date -u +%Y-%m-%d)" + git -C .telemetry push origin telemetry fi - name: Live endpoint check (soft) @@ -73,5 +104,5 @@ jobs: echo "GET $url" curl -s --max-time 30 -A "Mozilla/5.0 ci" "$url" \ | python -c "import sys, json; d = json.load(sys.stdin); print(' ', {k: d.get(k) for k in ('system', 'mode', 'status', 'schema_version', 'benchmark_type')})" \ - || echo " (endpoint not reachable yet; redeploy may be in flight)" + || echo " (endpoint not reachable yet)" done diff --git a/api/benchmark-latest.py b/api/benchmark-latest.py index 299cee9..54c0998 100644 --- a/api/benchmark-latest.py +++ b/api/benchmark-latest.py @@ -10,6 +10,8 @@ from __future__ import annotations import json +import os +import urllib.request from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler from pathlib import Path @@ -20,11 +22,39 @@ SCHEMA_VERSION = 1 ARTIFACT_FILE = Path(__file__).parent / "_benchmark_latest.json" +# The scheduled benchmark publishes here because main is ruleset-protected. +_TELEMETRY_RAW_BASE = ( + "https://raw.githubusercontent.com/IgnazioDS/evalops-workbench/telemetry/api/" +) +_FETCH_TIMEOUT_S = 2.5 + def _now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def _fetch_remote_json(filename: str) -> dict[str, Any] | None: + """Best-effort fetch of the freshest artifact from the telemetry data branch. + + Returns None on any failure so the caller falls back to the committed copy. + Only runs in the deployed Vercel runtime; tests/local use the committed copy. + """ + if not os.environ.get("VERCEL"): + return None + try: + req = urllib.request.Request( + _TELEMETRY_RAW_BASE + filename, + headers={"User-Agent": f"{SYSTEM_SLUG}-telemetry"}, + ) + with urllib.request.urlopen(req, timeout=_FETCH_TIMEOUT_S) as resp: + if resp.status != 200: + return None + data = json.loads(resp.read().decode("utf-8")) + return data if isinstance(data, dict) else None + except Exception: # noqa: BLE001 - the contract forbids 5xx; fall back instead + return None + + def _pending_payload() -> dict[str, Any]: """Honest envelope for the window before the first run is published.""" return { @@ -39,6 +69,9 @@ def _pending_payload() -> dict[str, Any]: def build_response() -> dict[str, Any]: + remote = _fetch_remote_json("_benchmark_latest.json") + if isinstance(remote, dict): + return remote try: return json.loads(ARTIFACT_FILE.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError, ValueError): diff --git a/api/stats.py b/api/stats.py index c17d23b..6a23ade 100644 --- a/api/stats.py +++ b/api/stats.py @@ -15,6 +15,7 @@ import json import os +import urllib.request from datetime import datetime, timedelta, timezone from http.server import BaseHTTPRequestHandler from pathlib import Path @@ -27,6 +28,12 @@ HISTORY_FILE = Path(__file__).parent / "_benchmark_history.json" STATIC_FILE = Path(__file__).parent / "_telemetry_static.json" +# The scheduled benchmark publishes here because main is ruleset-protected. +_TELEMETRY_RAW_BASE = ( + "https://raw.githubusercontent.com/IgnazioDS/evalops-workbench/telemetry/api/" +) +_FETCH_TIMEOUT_S = 2.5 + # Sanity caps: never expose values larger than these (defence against a runaway # history file). The benchmark publishes one run per scheduled invocation. SAFETY_CAPS: dict[str, int] = { @@ -53,6 +60,27 @@ def _read_json(path: Path) -> Any: return None +def _fetch_remote_json(filename: str) -> Any: + """Best-effort fetch of the freshest artifact from the telemetry data branch. + + Returns None on any failure so the caller falls back to the committed copy. + Only runs in the deployed Vercel runtime; tests/local use the committed copy. + """ + if not os.environ.get("VERCEL"): + return None + try: + req = urllib.request.Request( + _TELEMETRY_RAW_BASE + filename, + headers={"User-Agent": f"{SYSTEM_SLUG}-telemetry"}, + ) + with urllib.request.urlopen(req, timeout=_FETCH_TIMEOUT_S) as resp: + if resp.status != 200: + return None + return json.loads(resp.read().decode("utf-8")) + except Exception: # noqa: BLE001 - the contract forbids 5xx; fall back instead + return None + + def _parse_iso(value: Any) -> datetime | None: if not isinstance(value, str): return None @@ -114,8 +142,14 @@ def _build_response() -> dict[str, Any]: static = _read_json(STATIC_FILE) or {} last_deployed_at = os.environ.get("VERCEL_GIT_COMMIT_AUTHOR_DATE") or static.get("built_at") - history = _read_json(HISTORY_FILE) - artifact = _read_json(ARTIFACT_FILE) + # Prefer the freshest artifact from the telemetry branch; fall back to the + # copy committed on main so a network blip degrades gracefully. + history = _fetch_remote_json("_benchmark_history.json") + if not isinstance(history, list): + history = _read_json(HISTORY_FILE) + artifact = _fetch_remote_json("_benchmark_latest.json") + if not isinstance(artifact, dict): + artifact = _read_json(ARTIFACT_FILE) if isinstance(history, list) and history: metrics = _metrics_from_history(history, now) diff --git a/src/evalops_workbench/benchmark_runner.py b/src/evalops_workbench/benchmark_runner.py index 1954979..eb983bd 100644 --- a/src/evalops_workbench/benchmark_runner.py +++ b/src/evalops_workbench/benchmark_runner.py @@ -38,6 +38,12 @@ GATE_KWARGS = {"max_regressions": 0, "max_score_drop": 0.0, "max_pass_rate_drop": 0.0} _RAW_BASE = "https://raw.githubusercontent.com/IgnazioDS/evalops-workbench/main" +# Runner-produced artifacts (report, per-run archive) are published to the +# unprotected telemetry branch, since the default branch is ruleset-protected. +# Static inputs (the fixture) stay on main. +_TELEMETRY_RAW_BASE = ( + "https://raw.githubusercontent.com/IgnazioDS/evalops-workbench/telemetry" +) _HISTORY_KEEP = 100 @@ -130,9 +136,9 @@ def _build_artifact(*, base, candidate, comparison, gate, generated_at, previous "max_pass_rate_drop": gate.max_pass_rate_drop, }, "artifact_urls": { - "report": f"{_RAW_BASE}/examples/benchmark/latest-report.md", + "report": f"{_TELEMETRY_RAW_BASE}/examples/benchmark/latest-report.md", "fixture": f"{_RAW_BASE}/{DATASET_REL}", - "run": f"{_RAW_BASE}/examples/benchmark/archive/{run_id}.json", + "run": f"{_TELEMETRY_RAW_BASE}/examples/benchmark/archive/{run_id}.json", }, "schema_version": SCHEMA_VERSION, "generated_at": generated_at,