From 7e9cf3c583eae9b0c69774bd60cca0638a90cd04 Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Wed, 10 Jun 2026 20:22:28 -0400 Subject: [PATCH 1/6] fix(adapter): surface OpenRouter error body instead of KeyError('choices') OpenRouter returns error objects with HTTP 200 for some failures (context length exceeded, provider moderation). The adapter crashed with an opaque KeyError('choices'); now it raises RuntimeError with the provider's actual error message. Found live when gpt-oss-120b (131k context) hit the ~184k-token operations tasks. Co-Authored-By: Claude Fable 5 --- runner/adapters/openrouter_adapter.py | 9 +++++++++ tests/unit/test_openrouter_adapter.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/runner/adapters/openrouter_adapter.py b/runner/adapters/openrouter_adapter.py index 2487a08..bc64b5b 100644 --- a/runner/adapters/openrouter_adapter.py +++ b/runner/adapters/openrouter_adapter.py @@ -557,6 +557,15 @@ def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: timestamp = now.strftime("%Y-%m-%dT%H:%M:%SZ") body: dict[str, Any] = response.json() + # OpenRouter can return an error object with HTTP 200 (e.g. context + # length exceeded, provider moderation) — surface its message instead + # of crashing with KeyError('choices'). + if "choices" not in body or not body["choices"]: + err = body.get("error") or {} + raise RuntimeError( + f"OpenRouter returned no choices for model {self._model!r}: " + f"{err.get('message') or json.dumps(body)[:300]}" + ) choice = body["choices"][0] raw_text: str = choice["message"]["content"] or "" finish_reason: str | None = choice.get("finish_reason") diff --git a/tests/unit/test_openrouter_adapter.py b/tests/unit/test_openrouter_adapter.py index 1eee7d8..f61fb9f 100644 --- a/tests/unit/test_openrouter_adapter.py +++ b/tests/unit/test_openrouter_adapter.py @@ -525,6 +525,29 @@ def test_non_200_raises_runtime_error(self) -> None: status_code=429, ) + def test_error_body_with_200_surfaces_provider_message(self) -> None: + """An HTTP-200 body without choices must raise with the provider's error message. + + OpenRouter returns error objects with HTTP 200 for some failures + (e.g. context length exceeded) — these must not crash with + KeyError('choices'). + """ + with pytest.raises(RuntimeError, match="maximum context length"): + self._run_with_mock( + mock_body={ + "error": { + "message": "This endpoint's maximum context length is 131072 tokens.", + "code": 400, + } + }, + status_code=200, + ) + + def test_empty_choices_list_raises_runtime_error(self) -> None: + """An HTTP-200 body with an empty choices list must raise RuntimeError.""" + with pytest.raises(RuntimeError, match="no choices"): + self._run_with_mock(mock_body={"choices": []}, status_code=200) + def test_missing_api_key_raises_environment_error(self, monkeypatch: Any) -> None: """Missing API key (no env var, no constructor arg) must raise EnvironmentError.""" monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) From 7f2f93d89731838a082fc9e373779192bbbce31b Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Thu, 11 Jun 2026 08:33:24 -0400 Subject: [PATCH 2/6] =?UTF-8?q?feat(scripts):=20backfill=5Ftasks=20?= =?UTF-8?q?=E2=80=94=20re-run=20only=20missing/failed=20tasks=20and=20merg?= =?UTF-8?q?e=20into=20existing=20row?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconstructs TaskRunResults from result.json + raw_outputs.jsonl, re-runs only the named task IDs (fixtures resolve via explicit pack_id), re-aggregates with summed judge cost, and rewrites the row in place with .bak backups. Clears stale failures.json when the row becomes complete. Validated end-to-end with the stub adapter (overwrite-merge and fresh-row paths). Co-Authored-By: Claude Fable 5 --- scripts/backfill_tasks.py | 229 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 scripts/backfill_tasks.py diff --git a/scripts/backfill_tasks.py b/scripts/backfill_tasks.py new file mode 100644 index 0000000..9db99e4 --- /dev/null +++ b/scripts/backfill_tasks.py @@ -0,0 +1,229 @@ +r"""backfill_tasks — re-run only the missing/failed tasks of an existing model row. + +Reads the model's existing ``result.json`` + ``raw_outputs.jsonl`` from +``//``, re-runs ONLY the requested task IDs against the +live pack, merges old and new per-task results, re-aggregates, and rewrites +both files in place (originals are backed up with a ``.bak`` suffix). + +Usage:: + + python -m scripts.backfill_tasks \ + --model anthropic/claude-sonnet-4.6 \ + --tasks T1-OPS-003,T1-OPS-004,T1-OPS-005,T1-OPS-006 \ + --pack operations --runs 5 --judge --out results/operations + +If the model has no existing row (e.g. gpt-5.5@xhigh), all tasks in +``--tasks`` are run and the merged row is just the new tasks — equivalent to +a fresh partial run. + +Judge cost in the merged ``cost_metrics`` is the sum of the old row's +``judge_cost_usd`` and the backfill's judge spend (token-level judge counts +are only available for the backfill portion and are reported as-is). +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path +from typing import Any + +from runner.aggregator import aggregate +from runner.dispatcher import TaskRunResult, load_pack, run_task +from runner.io import write_raw_outputs, write_result +from scripts.run_models import _build_adapter, _resolve_pack, _safe_slug + + +def _load_existing( + model_dir: Path, + skip_task_ids: set[str], +) -> tuple[list[TaskRunResult], float]: + """Reconstruct TaskRunResults for tasks NOT being backfilled. + + Args: + model_dir: Existing per-model result directory. + skip_task_ids: Task IDs being re-run (excluded from reconstruction). + + Returns: + ``(kept_results, old_judge_cost_usd)``. Empty list / 0.0 when no + prior row exists. + """ + result_path = model_dir / "result.json" + raw_path = model_dir / "raw_outputs.jsonl" + if not result_path.exists(): + return [], 0.0 + + result = json.loads(result_path.read_text(encoding="utf-8")) + + outputs_by_task: dict[str, list[dict[str, Any]]] = {} + if raw_path.exists(): + with raw_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + out = json.loads(line) + outputs_by_task.setdefault(out["task_id"], []).append(out) + + kept: list[TaskRunResult] = [] + for entry in result.get("per_task_scores", []): + tid = entry["task_id"] + if tid in skip_task_ids: + continue + kept.append( + TaskRunResult( + task_id=tid, + track=entry["track"], + pack_id=entry.get("pack_id"), + run_count=entry["run_count"], + outputs=outputs_by_task.get(tid, []), + scores=dict(entry["scores"]), + composite=entry.get("composite"), + scorer_flags=list(entry.get("scorer_flags") or []), + ) + ) + + old_judge_cost = (result.get("cost_metrics") or {}).get("judge_cost_usd") or 0.0 + return kept, float(old_judge_cost) + + +def main(argv: list[str] | None = None) -> int: + """CLI entry point: re-run the named tasks and merge into the existing row. + + Args: + argv: Argument list (defaults to ``sys.argv[1:]`` when ``None``). + + Returns: + Exit code: 0 on full success, 1 when any backfill task failed. + """ + parser = argparse.ArgumentParser( + prog="python -m scripts.backfill_tasks", + description="Re-run only the named tasks for one model row and merge results.", + ) + parser.add_argument("--model", required=True, help="Model ID (supports @ suffix).") + parser.add_argument( + "--tasks", required=True, metavar="T1,T2,...", help="Comma-separated task IDs to re-run." + ) + parser.add_argument("--pack", required=True, help="Pack short name or path.") + parser.add_argument("--adapter", default="openrouter") + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--judge", action="store_true") + parser.add_argument("--judge-model", default=None) + parser.add_argument("--temperature", type=float, default=1.0) + parser.add_argument("--out", required=True, help="Output root (same as the original run).") + parser.add_argument("--grade-version", default="0.1.0") + args = parser.parse_args(argv) + + wanted_ids = {t.strip() for t in args.tasks.split(",") if t.strip()} + if not wanted_ids: + print("ERROR: --tasks must be a non-empty comma-separated list.", file=sys.stderr) + return 1 + + pack_path, pack_id = _resolve_pack(args.pack) + tasks = [t for t in load_pack(pack_path) if t.get("task_id") in wanted_ids] + missing = wanted_ids - {t["task_id"] for t in tasks} + if missing: + print(f"ERROR: task IDs not found in pack: {sorted(missing)}", file=sys.stderr) + return 1 + + model_dir = Path(args.out) / _safe_slug(args.model) + kept_results, old_judge_cost = _load_existing(model_dir, wanted_ids) + print( + f"Backfilling {len(tasks)} task(s) for {args.model}; " + f"keeping {len(kept_results)} existing task result(s)." + ) + + adapter = _build_adapter(args.adapter, model=args.model, temperature=args.temperature) + + judge_client: Any | None = None + if args.judge: + from benchmark.rubrics.judge_client import JudgeClient, select_judge_model + + if args.judge_model: + judge_model, judge_effort = args.judge_model, None + else: + judge_model, judge_effort = select_judge_model(args.model) + print(f" judge: {judge_model}" + (f" (effort: {judge_effort})" if judge_effort else "")) + judge_client = JudgeClient(model=judge_model, reasoning_effort=judge_effort) + + new_results: list[TaskRunResult] = [] + failures: list[dict[str, str]] = [] + for i, task in enumerate(tasks, start=1): + tid = task["task_id"] + print(f" [{i}/{len(tasks)}] {tid} ...", end=" ", flush=True) + try: + tr = run_task( + task=task, + adapter=adapter, + runs=args.runs, + pack_id=pack_id, + judge_client=judge_client, + ) + except Exception as exc: # noqa: BLE001 + print(f"FAILED ({exc})", flush=True) + failures.append({"task_id": tid, "error": str(exc)}) + continue + comp = f"{tr.composite:.4f}" if tr.composite is not None else "n/a" + print(f"composite={comp}", flush=True) + new_results.append(tr) + + if not new_results: + print("No backfill task succeeded — leaving existing row untouched.", file=sys.stderr) + return 1 + + # Merge, preserving canonical pack order. + pack_order = {t["task_id"]: i for i, t in enumerate(load_pack(pack_path))} + merged = sorted( + kept_results + new_results, + key=lambda tr: pack_order.get(tr.task_id, 10_000), + ) + + judge_metrics: dict[str, Any] | None = None + if judge_client is not None: + judge_metrics = { + "cumulative_cost_usd": (judge_client.cumulative_cost_usd or 0.0) + old_judge_cost, + "cumulative_prompt_tokens": judge_client.cumulative_prompt_tokens, + "cumulative_completion_tokens": judge_client.cumulative_completion_tokens, + "judge_call_count": judge_client.judge_call_count, + } + elif old_judge_cost: + judge_metrics = {"cumulative_cost_usd": old_judge_cost} + + scorecard = aggregate( + task_results=merged, + model_id=args.model, + grade_version=args.grade_version, + judge_metrics=judge_metrics, + ) + + # Back up originals, then write merged row. + model_dir.mkdir(parents=True, exist_ok=True) + for name in ("result.json", "raw_outputs.jsonl", "failures.json"): + p = model_dir / name + if p.exists(): + shutil.copy2(p, p.with_suffix(p.suffix + ".bak")) + + all_outputs = [out for tr in merged for out in tr.outputs] + raw_path = write_raw_outputs(all_outputs, model_dir) + result_path = write_result(scorecard, model_dir) + + failures_path = model_dir / "failures.json" + if failures: + failures_path.write_text(json.dumps(failures, indent=2) + "\n", encoding="utf-8") + elif failures_path.exists(): + failures_path.unlink() # row is now complete — stale failure list removed + + overall = scorecard.get("overall_composite") + comp_str = f"{overall:.4f}" if overall is not None else "n/a" + print(f"\nMerged row: {len(merged)} task(s), composite={comp_str}") + print(f"Raw outputs → {raw_path}") + print(f"Result JSON → {result_path}") + if failures: + print(f"Backfill failures recorded → {failures_path}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 2287c267a0264fc9272f7f9672ddfc9737ada7e5 Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Thu, 11 Jun 2026 10:26:42 -0400 Subject: [PATCH 3/6] fix(adapter): retry transient transport errors and 429/5xx with backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mid-stream connection drops ('peer closed connection') killed whole tasks — three leaderboard rows were damaged this way. Both HTTP call sites (candidate run() and the shared post_chat_completion used by the judge) now retry up to GRADE_OPENROUTER_RETRIES attempts (default 3) with exponential backoff on httpx.TransportError and 429/500/502/503/504 responses. Timeouts still propagate unretried (each attempt already waits the full configured timeout). run()'s hardcoded 120s timeout now honors GRADE_OPENROUTER_TIMEOUT too. Co-Authored-By: Claude Fable 5 --- runner/adapters/openrouter_adapter.py | 91 +++++++++++++++++++++++++-- tests/unit/test_openrouter_adapter.py | 76 ++++++++++++++++++++++ 2 files changed, 162 insertions(+), 5 deletions(-) diff --git a/runner/adapters/openrouter_adapter.py b/runner/adapters/openrouter_adapter.py index bc64b5b..be60652 100644 --- a/runner/adapters/openrouter_adapter.py +++ b/runner/adapters/openrouter_adapter.py @@ -94,6 +94,85 @@ def _get_openrouter_timeout() -> float: return DEFAULT_OPENROUTER_TIMEOUT +#: Default number of HTTP attempts per call (1 initial + retries). +#: Override via the ``GRADE_OPENROUTER_RETRIES`` environment variable. +DEFAULT_OPENROUTER_ATTEMPTS: int = 3 + +#: HTTP statuses treated as transient and retried with backoff. +RETRYABLE_STATUS_CODES: tuple[int, ...] = (429, 500, 502, 503, 504) + + +def _get_openrouter_attempts() -> int: + """Return the configured number of HTTP attempts per OpenRouter call. + + Reads ``GRADE_OPENROUTER_RETRIES``. Falls back to + :data:`DEFAULT_OPENROUTER_ATTEMPTS` when unset or unparseable. + + Returns: + Attempt count as an :class:`int` (minimum 1). + """ + raw = os.environ.get("GRADE_OPENROUTER_RETRIES", "") + if raw: + try: + return max(1, int(raw)) + except ValueError: + pass + return DEFAULT_OPENROUTER_ATTEMPTS + + +def _post_with_retries( + httpx_mod: Any, + url: str, + headers: dict[str, str], + payload: dict[str, Any], + timeout: float, +) -> Any: + """POST with retries on transient transport errors and retryable statuses. + + Mid-stream connection drops (``peer closed connection``), connect/read + errors, and 429/5xx responses are retried with exponential backoff + (1s, 2s, ... capped at 8s). Timeouts are NOT retried — each attempt + already waits the full configured timeout, so callers keep their + existing fail-fast timeout semantics (raise / bump + ``GRADE_OPENROUTER_TIMEOUT`` instead). + + Args: + httpx_mod: The imported ``httpx`` module (passed in because the + adapter lazy-imports it). + url: Request URL. + headers: Request headers. + payload: JSON body. + timeout: Per-attempt timeout in seconds. + + Returns: + The ``httpx.Response`` of the first successful (or non-retryable) + attempt. + + Raises: + RuntimeError: When transport errors persist through all attempts. + httpx.TimeoutException: Propagated unretried. + """ + attempts = _get_openrouter_attempts() + response: Any = None + for attempt in range(attempts): + try: + response = httpx_mod.post(url, headers=headers, json=payload, timeout=timeout) + except httpx_mod.TimeoutException: + raise + except httpx_mod.TransportError as exc: + if attempt < attempts - 1: + time.sleep(min(2**attempt, 8)) + continue + raise RuntimeError( + f"OpenRouter transport error persisted through {attempts} attempts: {exc}" + ) from exc + if response.status_code in RETRYABLE_STATUS_CODES and attempt < attempts - 1: + time.sleep(min(2**attempt, 8)) + continue + return response + return response + + #: Seed model shorthand → OpenRouter model slug mapping. #: #: These are the launch leaderboard models. Pass a shorthand string as the @@ -540,11 +619,12 @@ def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: payload["reasoning"] = {"effort": self._reasoning_effort} t_start = time.monotonic() - response = httpx.post( + response = _post_with_retries( + httpx, f"{OPENROUTER_BASE_URL}/chat/completions", headers=headers, - json=payload, - timeout=120.0, + payload=payload, + timeout=_get_openrouter_timeout(), ) latency_ms = (time.monotonic() - t_start) * 1000.0 @@ -790,7 +870,8 @@ def post_chat_completion( timeout = _get_openrouter_timeout() try: - response = httpx.post( + response = _post_with_retries( + httpx, f"{OPENROUTER_BASE_URL}/chat/completions", headers={ "Authorization": f"Bearer {key}", @@ -798,7 +879,7 @@ def post_chat_completion( "HTTP-Referer": HTTP_REFERER, "X-Title": X_TITLE, }, - json=payload, + payload=payload, timeout=timeout, ) except httpx.TimeoutException as exc: diff --git a/tests/unit/test_openrouter_adapter.py b/tests/unit/test_openrouter_adapter.py index f61fb9f..414262e 100644 --- a/tests/unit/test_openrouter_adapter.py +++ b/tests/unit/test_openrouter_adapter.py @@ -145,10 +145,27 @@ def _inject_mock_httpx(mock_post: MagicMock) -> ModuleType: fake_httpx.post = mock_post # type: ignore[attr-defined] # Provide a minimal Client stub so _make_openrouter_http_client doesn't fail. fake_httpx.Client = MagicMock() # type: ignore[attr-defined] + # Exception hierarchy mirroring httpx (TimeoutException < TransportError) + # so the adapter's retry/except clauses resolve against the fake module. + fake_httpx.TransportError = _FakeTransportError # type: ignore[attr-defined] + fake_httpx.TimeoutException = _FakeTimeoutException # type: ignore[attr-defined] + fake_httpx.RemoteProtocolError = _FakeRemoteProtocolError # type: ignore[attr-defined] sys.modules["httpx"] = fake_httpx return fake_httpx +class _FakeTransportError(Exception): + """Stand-in for ``httpx.TransportError`` in the fake module.""" + + +class _FakeTimeoutException(_FakeTransportError): + """Stand-in for ``httpx.TimeoutException`` (subclasses TransportError).""" + + +class _FakeRemoteProtocolError(_FakeTransportError): + """Stand-in for ``httpx.RemoteProtocolError`` (peer closed connection).""" + + def _remove_mock_httpx() -> None: """Remove the fake httpx from sys.modules (cleanup after injection).""" sys.modules.pop("httpx", None) @@ -548,6 +565,65 @@ def test_empty_choices_list_raises_runtime_error(self) -> None: with pytest.raises(RuntimeError, match="no choices"): self._run_with_mock(mock_body={"choices": []}, status_code=200) + def test_transient_transport_error_is_retried(self, monkeypatch: Any) -> None: + """A mid-stream connection drop must be retried and then succeed. + + First post() raises RemoteProtocolError ('peer closed connection'), + second returns a valid response — run() must succeed with 2 calls. + """ + monkeypatch.setattr("time.sleep", lambda _s: None) + good = _make_mock_httpx_post(_MOCK_OR_RESPONSE)() # a response object + mock_post = MagicMock(side_effect=[_FakeRemoteProtocolError("peer closed"), good]) + _inject_mock_httpx(mock_post) + try: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + output = adapter.run(_VALID_TASK, run_index=0) + finally: + _remove_mock_httpx() + assert output["task_id"] == _VALID_TASK["task_id"] + assert mock_post.call_count == 2 + + def test_persistent_transport_error_raises_after_attempts(self, monkeypatch: Any) -> None: + """Transport errors on every attempt must raise RuntimeError, not crash opaquely.""" + monkeypatch.setattr("time.sleep", lambda _s: None) + mock_post = MagicMock(side_effect=_FakeRemoteProtocolError("peer closed")) + _inject_mock_httpx(mock_post) + try: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + with pytest.raises(RuntimeError, match="transport error persisted"): + adapter.run(_VALID_TASK, run_index=0) + finally: + _remove_mock_httpx() + assert mock_post.call_count == 3 # DEFAULT_OPENROUTER_ATTEMPTS + + def test_retryable_status_is_retried(self, monkeypatch: Any) -> None: + """A 502 response must be retried; a subsequent 200 succeeds.""" + monkeypatch.setattr("time.sleep", lambda _s: None) + bad = _make_mock_httpx_post({"error": {"message": "bad gateway"}}, status_code=502)() + good = _make_mock_httpx_post(_MOCK_OR_RESPONSE)() + mock_post = MagicMock(side_effect=[bad, good]) + _inject_mock_httpx(mock_post) + try: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + output = adapter.run(_VALID_TASK, run_index=0) + finally: + _remove_mock_httpx() + assert output["task_id"] == _VALID_TASK["task_id"] + assert mock_post.call_count == 2 + + def test_timeout_is_not_retried(self, monkeypatch: Any) -> None: + """Timeouts must propagate unretried (each attempt already waits in full).""" + monkeypatch.setattr("time.sleep", lambda _s: None) + mock_post = MagicMock(side_effect=_FakeTimeoutException("timed out")) + _inject_mock_httpx(mock_post) + try: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + with pytest.raises(_FakeTimeoutException): + adapter.run(_VALID_TASK, run_index=0) + finally: + _remove_mock_httpx() + assert mock_post.call_count == 1 + def test_missing_api_key_raises_environment_error(self, monkeypatch: Any) -> None: """Missing API key (no env var, no constructor arg) must raise EnvironmentError.""" monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) From ff79c42f922f864a7684b53f385a51250b2d48c8 Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Thu, 11 Jun 2026 17:33:27 -0400 Subject: [PATCH 4/6] updated readme --- README.md | 214 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 194 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 0b55551..763380f 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,86 @@ # GRADE -> **Private pre-release** — this repo is private/internal until launch; APIs and schemas are unstable. +**Grounded Reasoning & Analysis for Data in Education** -repository for pearl AI benchmark +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) +[![Version](https://img.shields.io/badge/version-0.1.0.dev0-orange)](pyproject.toml) +[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue)](pyproject.toml) + +GRADE is an open benchmark that measures how accurately and usefully AI systems analyze education program data. It was built by [Pearl](https://tutorwithpearl.com) — an AI tutoring platform working directly with school districts — to answer a practical question: **can AI produce education analytics that a program manager or district analyst could trust and act on?** + +--- + +## The Problem + +AI tools are being adopted across education to help program managers and analysts make sense of attendance records, session logs, outcome summaries, and research evidence. The quality of that analysis directly affects the decisions practitioners make about students. + +General-purpose benchmarks don't measure this. A model can excel at broad reasoning tasks while still hallucinating attendance rates, misattributing causation to correlated signals, or failing to flag that a subgroup is too small to report on. GRADE was built to fill that gap. + +--- + +## Leaderboard + +Results on the operations pack (11 tasks, 5 runs per task). Full 26-task results in progress. + +| Rank | Model | Composite | Grounding | Insight | Evidence | Calibration | Consistency | +|------|-------|:---------:|:---------:|:-------:|:--------:|:-----------:|:-----------:| +| 1 | `openai/gpt-5.5@high` | **68.3%** | 98.0% | 35.2% | 85.8% | 19.6% | 64.5% | +| 2 | `openai/gpt-5.5@medium` | **64.4%** | 87.5% | 41.7% | 75.3% | 18.0% | 66.6% | +| 3 | `openai/gpt-5.5@low` | **52.4%** | 62.4% | 35.7% | 59.3% | 23.2% | 64.2% | +| 4 | `anthropic/claude-opus-4.8` | **41.9%** | 45.5% | 30.1% | 31.2% | 27.3% | 75.5% | +| 5 | `anthropic/claude-haiku-4.5` | **37.9%** | 47.9% | 22.0% | 20.6% | 20.5% | 70.7% | +| 6 | `anthropic/claude-sonnet-4.6` | **37.5%** | 43.3% | 27.8% | 23.9% | 17.5% | 74.5% | +| 7 | `openai/gpt-oss-120b` | **25.2%** | 22.2% | 7.7% | 18.7% | 25.6% | 74.2% | +| 8 | `google/gemini-3.1-pro-preview` | **23.9%** | 23.9% | 7.9% | 17.6% | 27.4% | 60.3% | +| 9 | `nvidia/nemotron-3-ultra-550b-a55b` | **21.7%** | 22.4% | 5.4% | 19.6% | 13.4% | 70.6% | +| 10 | `google/gemini-3.5-flash` | **17.1%** | 11.1% | 6.2% | 8.9% | 27.8% | 60.3% | + +> Composite = 35% Grounding + 20% Insight + 15% Evidence + 15% Calibration + 10% Consistency + 5% Structure. +> Scores are computed on ground-truth-validated synthetic data. See [docs/methodology.md](docs/methodology.md) for caveats on partial runs and cross-family judging. + +**Key finding:** The hardest dimensions across every model are calibration (acknowledging data limitations and avoiding unsupported causal claims) and insight quality (interpreting signals rather than reciting numbers). Grounding accuracy varies most widely — the gap between GPT-5.5@high (98%) and Gemini 3.5 Flash (11%) on the same tasks reflects a real difference in whether models correctly read numbers from structured CSVs. + +--- + +## What GRADE Measures + +Five tracks across three synthetic fixture packs — 26 tasks total. + +| Track | Tasks | Core challenge | +|-------|------:|----------------| +| **1 · Grounded Retrieval & Computation** | 6 | Exact counts, rates, and date-filtered aggregates from CSVs — no inference required; the only failure mode is hallucination | +| **2 · Program Snapshot & Trend Interpretation** | 5 | Month-over-month summaries and anomaly detection with epistemic discipline: the data contains correlations that don't support causal claims | +| **3 · Operational Coaching & Recommendations** | 5 | Evidence-backed recommendations with entity IDs, numeric rates, and source-file citations — generic advice scores zero | +| **4 · Equity & Subgroup Interpretation** | 5 | Suppressed low-N groups, attendance and outcome disparities, and plain-language translation for practitioners | +| **5 · Program Effectiveness & Research Reasoning** | 5 | Synthesis of local fixture data with external research references, including contradictory findings | + +### Six Scoring Dimensions + +Every response is scored on six dimensions combined into a weighted composite: + +| Dimension | Weight | How it's scored | +|-----------|:------:|-----------------| +| Grounding Accuracy | **35%** | Deterministic: structured numeric outputs vs. gold facts with tolerance | +| Insight Quality | **20%** | LLM judge: coverage and depth of expected analytical findings | +| Evidence Linkage | **15%** | LLM judge: are claims traced to specific files and columns? | +| Calibration & Limitation Handling | **15%** | Deterministic + judge fallback: required caveats present, forbidden claims absent | +| Consistency | **10%** | Automated: finding/ranking/metric stability across 5 repeated runs | +| Structure & Usability | **5%** | LLM judge: format clarity and scannability | --- -## Quick start +## Quick Start + +No API key required to validate your installation. -Get from clone to a scored result in under 10 minutes — no API key required. +**Requirements:** Python 3.12+ ```bash git clone https://github.com/PearlEng/grade.git cd grade pip install -e . +# Smoke test with the built-in stub adapter (network-free, deterministic) python -m runner.cli \ --pack operations \ --adapter stub \ @@ -22,22 +88,9 @@ python -m runner.cli \ --out /tmp/grade_out ``` -The **stub adapter** is network-free and deterministic. It validates your -installation end-to-end without any model credentials. - -For a full walkthrough — including switching to a real model via OpenRouter, -rendering a Markdown scorecard, and writing your own adapter — see -[examples/quickstart.md](examples/quickstart.md). - -### Run all three task packs +Produces `raw_outputs.jsonl` (per-run model outputs) and `result.json` (aggregated scorecard) under `/tmp/grade_out`. -```bash -python -m runner.cli --pack operations --adapter stub --runs 1 --out /tmp/grade_ops -python -m runner.cli --pack outcomes --adapter stub --runs 1 --out /tmp/grade_out -python -m runner.cli --pack equity_research --adapter stub --runs 1 --out /tmp/grade_eq -``` - -### Use a real model (OpenRouter) +### Run a real model ```bash pip install -e ".[openrouter]" @@ -46,9 +99,130 @@ export OPENROUTER_API_KEY="sk-or-..." python -m runner.cli \ --pack operations \ --adapter openrouter \ - --model anthropic/claude-sonnet-4.6 \ + --model openai/gpt-4o \ --runs 5 \ --out /tmp/grade_real ``` +Any model available on [OpenRouter](https://openrouter.ai/models) works. A full 26-task × 5-run evaluation is 130 model calls and typically completes in under two hours. + +### Render a scorecard + +```bash +python -m benchmark.reports.scorecard \ + --results-dir /tmp/grade_real \ + --out /tmp/grade_reports +# → /tmp/grade_reports/scorecard.md (Markdown, per-track breakdown) +# → /tmp/grade_reports/result.json (machine-readable, result_schema) +``` + +For a full walkthrough — running all three packs, writing a custom adapter, and interpreting track profiles — see [examples/quickstart.md](examples/quickstart.md). + +--- + +## How It Works + +### Synthetic Fixtures + +All data is generated from a deterministic seed (42) and represents a fictional high-dosage tutoring program. Three packs are layered: + +| Pack | ID | Tracks | What's included | +|------|----|--------|-----------------| +| Operations | `pack_operations` | 1, 3 | Students, sessions, tutors, attendance, survey responses | +| Outcomes | `pack_outcomes` | 2 | All of Operations + month-over-month summary tables | +| Equity & Research | `pack_equity_research` | 4, 5 | All of Outcomes + subgroup summaries + 10 research references | + +No real students, schools, or programs are represented. Dataset cards: [`docs/dataset_cards/`](docs/dataset_cards/). + +### Scoring Pipeline + +For each (model × task) pair: + +1. **Adapter** calls the model N times (default: 5) with the task prompt and fixture context +2. **C1 — Grounding** compares `structured_metrics` in the response against `gold_facts` with numeric tolerance +3. **C2 — Rubric** passes the response to a judge model for quality dimensions (insight, evidence, structure) +4. **C3 — Claim validation** checks deterministically that required limitations are present and forbidden claims are absent +5. **C4 — Consistency** measures finding/ranking/metric stability across the N runs +6. **C9 — Aggregation** rolls up to task, track, and overall composites and writes `result.json` + +### Cross-Family Judging + +No judge ever evaluates a model from the same family. Claude-family models are judged by GPT-5.5 (xhigh reasoning effort); all others are judged by Claude Opus 4.8. This eliminates house-style bias in the 40% of the composite scored by an LLM judge. + +### Reproducibility + +Gold fact values are derived from `ground_truth.json` files computed from the generated data — not from the generator's input constants. Because the generator applies stochastic noise, realized values differ from spec targets. GRADE mitigates benchmark contamination by regenerating fixtures (new seed, recomputed `ground_truth.json`) for each numbered version; scores are only comparable within a version. + +--- + +## Pluggable Adapters + +Adapters implement a minimal Protocol: + +```python +class Adapter(Protocol): + name: str + def run(self, task: dict, run_index: int = 0) -> dict: ... +``` + +| Adapter | Flag | Purpose | +|---------|------|---------| +| Stub | `--adapter stub` | Deterministic, network-free — validates installation | +| OpenRouter | `--adapter openrouter` | Live inference for any OpenRouter model | + +To evaluate a model not on OpenRouter, implement the Protocol, register it in `runner/cli.py`, and pass `--adapter your_name`. See [examples/quickstart.md](examples/quickstart.md). + +--- + +## Repository Layout + +``` +grade/ +├── benchmark/ +│ ├── datagen/ # Synthetic fixture generator (seeded, deterministic) +│ ├── reports/ # Scorecard and leaderboard renderers +│ ├── rubrics/ # Scorer implementations (C1–C4, C9) +│ ├── schemas/ # JSON Schema contracts (task, output, result) +│ └── tasks/ # 26 task definitions across 3 JSONL packs +├── runner/ +│ ├── adapters/ # Adapter Protocol + built-in adapters +│ ├── cli.py # Entry point: python -m runner.cli +│ ├── dispatcher.py # Task loader and run coordinator +│ └── aggregator.py # Result aggregation +├── fixtures/ # Pre-generated synthetic data (3 packs) +├── results/ # Pre-computed results from model runs +├── examples/ # Quickstart guide and sample outputs +└── docs/ # Methodology, scoring reference, dataset cards +``` + +--- + +## Documentation + +| Document | Contents | +|----------|----------| +| [docs/methodology.md](docs/methodology.md) | Mission, task taxonomy, scoring design, limitations, judging policy | +| [docs/scoring.md](docs/scoring.md) | Full dimension definitions, aggregation rules, per-track rubric defaults | +| [docs/dataset_cards/](docs/dataset_cards/) | File inventories, column contracts, intentional signal descriptions per pack | +| [examples/quickstart.md](examples/quickstart.md) | End-to-end walkthrough: install → smoke test → real model → scorecard → custom adapter | +| [CONTRIBUTING.md](CONTRIBUTING.md) | How to add tasks, fixture packs, and adapter implementations | + --- + +## Contributing + +We welcome contributions — new tasks, fixture packs, adapter implementations, and methodology proposals. See [CONTRIBUTING.md](CONTRIBUTING.md) and [GOVERNANCE.md](GOVERNANCE.md). + +--- + +## About Pearl + +GRADE is built and maintained by [Pearl](https://tutorwithpearl.com), an AI tutoring platform that partners with school districts to deliver high-dosage tutoring programs. Pearl works with program managers and district analysts daily, and GRADE reflects the analytical questions and failure modes we encounter in practice. + +If you're a researcher, AI developer, or education technologist and want to discuss the benchmark or collaborate, reach out via GitHub Issues or at [hello@tutorwithpearl.com](mailto:hello@tutorwithpearl.com). + +--- + +## License + +Apache 2.0. See [LICENSE](LICENSE). From eb77c8acba2224bc59830592b3087d3943d4b9c7 Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Thu, 11 Jun 2026 21:59:51 -0400 Subject: [PATCH 5/6] docs: add citation infrastructure for research community - Add CITATION.cff so GitHub surfaces a "Cite this repository" button - Add ## Citing GRADE section to README with BibTeX entries (GitHub URL now, Zenodo DOI placeholder for v0.1.0 release) - Set repo topics: benchmark, llm-evaluation, education, ai-evaluation, nlp, education-analytics, grounding, open-benchmark Co-Authored-By: Claude Sonnet 4.6 --- CITATION.cff | 28 ++++++++++++++++++++++++++++ README.md | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..ae6d69b --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,28 @@ +cff-version: 1.2.0 +message: "If you use GRADE in your research, please cite it as below." +type: software +title: "GRADE: Grounded Reasoning & Analysis for Data in Education" +abstract: > + GRADE is an open benchmark that measures how accurately and usefully AI + systems analyze education program data. It evaluates models across five + tracks — grounded retrieval, trend interpretation, operational coaching, + equity and subgroup analysis, and research reasoning — using synthetic + fixture packs with deterministic ground truth. Scoring combines + deterministic fact-checking with cross-family LLM judging to eliminate + house-style bias. +authors: + - name: "Pearl" + website: "https://tutorwithpearl.com" +repository-code: "https://github.com/PearlEng/grade" +url: "https://tutorwithpearl.com" +license: Apache-2.0 +version: "0.1.0" +date-released: "2026-06-12" +keywords: + - benchmark + - education + - llm-evaluation + - ai-evaluation + - nlp + - grounding + - education-analytics diff --git a/README.md b/README.md index 763380f..2c0cf94 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,40 @@ If you're a researcher, AI developer, or education technologist and want to disc --- +## Citing GRADE + +If you use GRADE in your research, please cite the repository: + +```bibtex +@software{pearl2026grade, + author = {Pearl}, + title = {{GRADE}: Grounded Reasoning \& Analysis for Data in Education}, + year = {2026}, + publisher = {GitHub}, + version = {0.1.0}, + url = {https://github.com/PearlEng/grade}, + note = {Open benchmark for AI evaluation in education analytics} +} +``` + +A Zenodo DOI for stable versioned citation will be added at the v0.1.0 release. Once available, prefer the DOI-based entry: + +```bibtex +@software{pearl2026grade, + author = {Pearl}, + title = {{GRADE}: Grounded Reasoning \& Analysis for Data in Education}, + year = {2026}, + publisher = {Zenodo}, + version = {0.1.0}, + doi = {10.5281/zenodo.XXXXXXX}, + url = {https://doi.org/10.5281/zenodo.XXXXXXX} +} +``` + +GitHub also provides a formatted citation via the **Cite this repository** button in the sidebar (powered by [`CITATION.cff`](CITATION.cff)). + +--- + ## License Apache 2.0. See [LICENSE](LICENSE). From 2bccdacb786c2f3d5a7b1e6c8bc5d58567dae341 Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Fri, 12 Jun 2026 02:01:33 -0400 Subject: [PATCH 6/6] docs(readme): publish official three-pack leaderboard with coverage annotations Replaces the stale operations-only table (which contained truncation-artifact scores for Gemini/Nemotron) with the corrected 64k-budget results across all three packs, coverage column for partial/DNF rows, and revised key findings. Co-Authored-By: Claude Fable 5 --- README.md | 95 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2c0cf94..9d36a82 100644 --- a/README.md +++ b/README.md @@ -20,25 +20,82 @@ General-purpose benchmarks don't measure this. A model can excel at broad reason ## Leaderboard -Results on the operations pack (11 tasks, 5 runs per task). Full 26-task results in progress. - -| Rank | Model | Composite | Grounding | Insight | Evidence | Calibration | Consistency | -|------|-------|:---------:|:---------:|:-------:|:--------:|:-----------:|:-----------:| -| 1 | `openai/gpt-5.5@high` | **68.3%** | 98.0% | 35.2% | 85.8% | 19.6% | 64.5% | -| 2 | `openai/gpt-5.5@medium` | **64.4%** | 87.5% | 41.7% | 75.3% | 18.0% | 66.6% | -| 3 | `openai/gpt-5.5@low` | **52.4%** | 62.4% | 35.7% | 59.3% | 23.2% | 64.2% | -| 4 | `anthropic/claude-opus-4.8` | **41.9%** | 45.5% | 30.1% | 31.2% | 27.3% | 75.5% | -| 5 | `anthropic/claude-haiku-4.5` | **37.9%** | 47.9% | 22.0% | 20.6% | 20.5% | 70.7% | -| 6 | `anthropic/claude-sonnet-4.6` | **37.5%** | 43.3% | 27.8% | 23.9% | 17.5% | 74.5% | -| 7 | `openai/gpt-oss-120b` | **25.2%** | 22.2% | 7.7% | 18.7% | 25.6% | 74.2% | -| 8 | `google/gemini-3.1-pro-preview` | **23.9%** | 23.9% | 7.9% | 17.6% | 27.4% | 60.3% | -| 9 | `nvidia/nemotron-3-ultra-550b-a55b` | **21.7%** | 22.4% | 5.4% | 19.6% | 13.4% | 70.6% | -| 10 | `google/gemini-3.5-flash` | **17.1%** | 11.1% | 6.2% | 8.9% | 27.8% | 60.3% | - -> Composite = 35% Grounding + 20% Insight + 15% Evidence + 15% Calibration + 10% Consistency + 5% Structure. -> Scores are computed on ground-truth-validated synthetic data. See [docs/methodology.md](docs/methodology.md) for caveats on partial runs and cross-family judging. - -**Key finding:** The hardest dimensions across every model are calibration (acknowledging data limitations and avoiding unsupported causal claims) and insight quality (interpreting signals rather than reciting numbers). Grounding accuracy varies most widely — the gap between GPT-5.5@high (98%) and Gemini 3.5 Flash (11%) on the same tasks reflects a real difference in whether models correctly read numbers from structured CSVs. +Official run, June 10–12 2026. Three packs: **operations** (11 tasks × 5 runs), +**outcomes** (5 tasks × 1 run), **equity research** (10 tasks × 1 run). All rows +ran with a 65,536-token completion budget so reasoning never crowds out the +answer; judging is cross-family (no judge scores its own model family). Full +caveats in [docs/methodology.md](docs/methodology.md); run notes in +[RUN_FINDINGS.md](RUN_FINDINGS.md). + +### Operations (program-data analysis, 5 repetitions) + +| Rank | Model | Composite | Grounding | Insight | Evidence | Calibration | Consistency | Coverage | +|------|-------|:---------:|:---------:|:-------:|:--------:|:-----------:|:-----------:|:--------:| +| 1 | `openai/gpt-5.5@xhigh` | **66.9%** | 91.1% | 45.0% | 77.5% | 22.4% | 63.0% | full | +| 2 | `openai/gpt-5.5@high` | **65.4%** | 87.7% | 46.3% | 76.9% | 18.5% | 62.9% | full | +| 3 | `openai/gpt-5.5@medium` | **64.9%** | 88.6% | 44.2% | 73.4% | 17.8% | 66.2% | full | +| 4 | `google/gemini-3.1-pro-preview` | **63.5%** | 79.2% | 45.0% | 76.9% | 31.2% | 61.3% | 8/11 [1] | +| 5 | `google/gemini-3.5-flash` | **61.0%** | 79.5% | 44.7% | 69.4% | 23.6% | 61.2% | 10/11 [1] | +| 6 | `openai/gpt-5.5@low` | **52.4%** | 62.4% | 35.7% | 59.3% | 23.2% | 64.2% | full | +| 7 | `anthropic/claude-opus-4.8` | **41.9%** | 45.5% | 30.1% | 31.2% | 27.3% | 75.5% | full | +| 8 | `anthropic/claude-sonnet-4.6` | **38.3%** | 42.6% | 29.2% | 27.7% | 18.3% | 76.7% | full | +| 9 | `anthropic/claude-haiku-4.5` | **37.9%** | 47.9% | 22.0% | 20.6% | 20.5% | 70.7% | full | +| 10 | `openai/gpt-oss-120b` | **25.2%** | 22.2% | 7.7% | 18.7% | 25.6% | 74.2% | 3/11 DNF [2] | +| 11 | `nvidia/nemotron-3-ultra-550b-a55b` | **25.1%** | 30.2% | 7.3% | 26.9% | 8.3% | 67.6% | full | + +> [1] Remaining tasks in flight at publication; composite covers listed tasks only. +> [2] `gpt-oss-120b`'s 131k context cannot fit the ~190k-token data payloads on 8 of 11 tasks — reported as a finding, not ranked competitively. + +### Outcomes (trend analysis, single run) + +| Rank | Model | Composite | Grounding | Calibration | Model Cost | +|------|-------|:---------:|:---------:|:-----------:|:----------:| +| 1 | `openai/gpt-5.5@low` | **90.1%** | 100.0% | 59.3% | $0.17 | +| 2 | `openai/gpt-5.5@medium` | **90.0%** | 100.0% | 55.7% | $0.21 | +| 3 | `nvidia/nemotron-3-ultra-550b-a55b` | **88.3%** | 100.0% | 45.8% | $0.04 | +| 4 | `openai/gpt-5.5@xhigh` | **86.9%** | 100.0% | 45.0% | $0.67 | +| 5 | `openai/gpt-5.5@high` | **86.6%** | 100.0% | 39.2% | $0.51 | +| 6 | `anthropic/claude-opus-4.8` | **86.2%** | 100.0% | 46.0% | $0.30 | +| 7 | `openai/gpt-oss-120b` | **84.0%** | 100.0% | 39.9% | $0.00 | +| 8 | `anthropic/claude-sonnet-4.6` | **83.0%** | 100.0% | 43.6% | $0.14 | +| 9 | `google/gemini-3.5-flash` | **81.2%** | 100.0% | 29.9% | $0.17 | +| 10 | `google/gemini-3.1-pro-preview` | **80.5%** | 100.0% | 46.1% | $0.17 | +| 11 | `anthropic/claude-haiku-4.5` | **76.7%** | 90.0% | 43.8% | $0.05 | + +### Equity research (subgroup & research reasoning, single run) + +| Rank | Model | Composite | Grounding | Calibration | Model Cost | +|------|-------|:---------:|:---------:|:-----------:|:----------:| +| 1 | `openai/gpt-5.5@medium` | **83.0%** | 79.8% | 58.9% | $1.10 | +| 2 | `openai/gpt-5.5@xhigh` | **82.8%** | 81.5% | 53.1% | $2.45 | +| 3 | `nvidia/nemotron-3-ultra-550b-a55b` | **81.8%** | 78.5% | 52.8% | $0.10 | +| 4 | `openai/gpt-5.5@high` | **81.1%** | 77.7% | 52.8% | $1.97 | +| 5 | `openai/gpt-5.5@low` | **80.6%** | 76.0% | 58.3% | $0.64 | +| 6 | `anthropic/claude-opus-4.8` | **75.4%** | 69.8% | 49.7% | $0.95 | +| 7 | `google/gemini-3.1-pro-preview` | **73.8%** | 67.8% | 48.3% | $0.59 | +| 8 | `google/gemini-3.5-flash` | **73.0%** | 67.8% | 39.2% | $0.53 | +| 9 | `openai/gpt-oss-120b` | **70.0%** | 62.3% | 32.8% | $0.01 | +| 10 | `anthropic/claude-sonnet-4.6` | **68.7%** | 55.8% | 42.8% | $0.47 | +| 11 | `anthropic/claude-haiku-4.5` | **66.9%** | 66.0% | 35.8% | $0.17 | + +> Composite = 35% Grounding + 20% Insight + 15% Evidence + 15% Calibration + 10% Consistency + 5% Structure (consistency excluded and weights renormalized on single-run packs). + +**Key findings.** (1) *Reasoning effort pays only where tasks are hard:* GPT-5.5's +effort curve rises steeply from low→medium on heavy data analysis +(52%→65%) then saturates, and **inverts** on light trend summaries (low beats +xhigh). (2) *Scores are envelope-relative:* under a 4k token budget the +default-thinking models (Gemini, Nemotron) score near the floor because +reasoning consumes the budget before any answer appears — the same models are +mid-pack or better at 64k. Nemotron still exhausts even 64k on the largest +tasks (it reasons by exhaustive enumeration), so its operations score reflects +budget economics, not analytical ability — it places top-three on both smaller +packs. (3) *Accuracy and stability trade off:* GPT-5.5 leads grounding accuracy +while Claude models lead run-to-run consistency; Claude Haiku 4.5 delivers +~90% of Opus's operations composite at ~18% of its cost. (4) *Calibration is +universally weak* (18–31%): no model reliably acknowledges data limitations +without prompting — the clearest open problem this benchmark surfaces. + +--- ---