diff --git a/bench/README.md b/bench/README.md index d601382..b441b5f 100644 --- a/bench/README.md +++ b/bench/README.md @@ -43,19 +43,37 @@ python bench/run_skill_evals.py execute --skill skills/bully-init --only 1 # Different model python bench/run_skill_evals.py --executor-model claude-haiku-4-5-20251001 \ triggers --skill skills/bully-review + +# Error bars: run N epochs per eval -> every rate carries a 95% CI +python bench/run_skill_evals.py execute --skill skills/bully-author --epochs 5 + +# Control arm: also run without_skill -> paired per-eval pass_rate delta +python bench/run_skill_evals.py execute --skill skills/bully-author --epochs 5 --baseline ``` +LLM runs (executor and graders) are stochastic, so a single run is a point +estimate with unknown variance. `--epochs N` repeats each eval N times and +reports `mean [95% CI]`; default is `1` (legacy) but use `>=5` for any number +you intend to compare or trust. `--baseline` adds a `without_skill` arm so the +benchmark can attribute an effect to the skill rather than the base model. + Each invocation creates `bench/eval-runs//iteration-/` and writes: - `triggers.json` — triggering results (per-query pass/fail, trigger rate, false-positive rate) -- `eval--/with_skill/run-1/` +- `eval--//run-/` — one dir per epoch; + `` is `with_skill` (and `without_skill` under `--baseline`) - `outputs/` — files the executor wrote (the eval prompts reference paths under here) - `transcript.md` — readable conversation log - `stream.jsonl` — raw `claude -p --output-format stream-json` output - - `eval_metadata.json` — prompt, expectations, model, timestamp + - `eval_metadata.json` — prompt, expectations, model, configuration, run index, timestamp - `timing.json` — executor + grader durations - `grading.json` — grader's verdicts (skill-creator schema) -- `benchmark.json` / `benchmark.md` — aggregate stats across the iteration +- `benchmark.json` / `benchmark.md` — aggregate stats across the iteration: + per-eval and suite-level pass_rate as **mean [95% CI]** (two-stage — epochs + reduced per eval, then clustered across evals so N epochs don't pose as N + independent samples), an `errored_runs` count (grader failures are excluded + from rates, never scored as 0%), and a paired `baseline_delta` under `--baseline`. + Git SHA + executor/grader models are stamped into `metadata` for provenance. ## Fixture pollution gotcha @@ -74,6 +92,7 @@ python /path/to/skill-creator/eval-viewer/generate_review.py \ ## Caveats / known limits -- **No "without_skill" baseline.** This driver only runs `with_skill`. Producing a clean baseline requires running `claude -p` against a config that doesn't have the bully plugin loaded, which isn't trivial since the plugin is globally installed. Add a `--baseline` mode later if a delta becomes useful. +- **`--baseline` disables *all* skills, not just the target.** The control arm passes `--disallowedTools Skill` to the executor, which blocks every skill (the plugin is globally installed, so there's no clean per-skill unload). The paired delta therefore measures "this skill vs. no skills" — a slight over-attribution if another skill would also have helped. Fine for the skill-vs-no-skill signal; tighten if a specific skill is a known confound. +- **Default `--epochs 1` gives no error bars.** A single run reports `mean (n=1)` with a zero-width interval. That's a point estimate, not a measurement — pass `--epochs >=5` before comparing iterations or trusting a delta. - **Triggering detection is heuristic.** We look for the skill name in any tool call's serialized form. Claude Code's exact Skill-tool invocation shape may vary; if false negatives appear, refine `_detect_skill_invocation` in the driver. - **Grader is itself an LLM.** The grader can be wrong. For deterministic checks (exit codes, YAML key sets), encode them into the expectation text so the grader will shell out and verify. diff --git a/bench/run_skill_evals.py b/bench/run_skill_evals.py index 657a9c2..672abe1 100644 --- a/bench/run_skill_evals.py +++ b/bench/run_skill_evals.py @@ -37,6 +37,7 @@ import argparse import json +import math import re import shutil import statistics @@ -70,6 +71,56 @@ def _slug(s: str) -> str: return s[:60] or "eval" +# 95% two-sided normal critical value. Evals are stochastic, so every reported +# rate gets a CLT-based error bar per Anthropic's "Adding Error Bars to Evals" +# (arXiv 2411.00640) rather than a bare point estimate. +_Z95 = 1.959963984540054 + + +def mean_ci(xs: list[Any]) -> dict[str, Any]: + """Mean + 95% normal-approximation confidence interval for a sample. + + Returns ``{n, mean, stddev, sem, ci_low, ci_high}``. ``None`` entries are + dropped before computing. Distinguishes three regimes: + + - empty -> ``mean`` is ``None`` (an *absent* measurement, never ``0.0`` -- + this is what keeps a grader/infra failure from being scored as 0%). + - n == 1 -> zero-width CI (no variance information). + - n >= 2 -> ``sem = stdev / sqrt(n)``; CI is ``mean +/- 1.96 * sem``. + + Values are returned at full precision; rounding is a rendering concern. + """ + vals = [float(x) for x in xs if x is not None] + n = len(vals) + if n == 0: + return {"n": 0, "mean": None, "stddev": 0.0, "sem": 0.0, "ci_low": None, "ci_high": None} + mean = statistics.fmean(vals) + if n < 2: + return {"n": 1, "mean": mean, "stddev": 0.0, "sem": 0.0, "ci_low": mean, "ci_high": mean} + sd = statistics.stdev(vals) + sem = sd / math.sqrt(n) + return { + "n": n, + "mean": mean, + "stddev": sd, + "sem": sem, + "ci_low": mean - _Z95 * sem, + "ci_high": mean + _Z95 * sem, + } + + +def execution_plan(*, epochs: int, baseline: bool) -> list[tuple[str, int]]: + """The ordered ``(configuration, run_index)`` pairs to execute per eval. + + Without ``baseline`` this is just ``epochs`` runs of ``with_skill``. With + ``baseline`` each eval also runs ``without_skill`` (the skill disabled) so + the harness can attribute the skill's effect via a paired delta -- the + control arm Anthropic's skill-creator methodology treats as mandatory. + """ + configs = ["with_skill", "without_skill"] if baseline else ["with_skill"] + return [(cfg, i) for cfg in configs for i in range(1, epochs + 1)] + + def _next_iteration_dir(skill_dir: Path) -> Path: skill_dir.mkdir(parents=True, exist_ok=True) existing = sorted( @@ -374,15 +425,23 @@ def cmd_triggers(args: argparse.Namespace) -> int: # ----- execution eval --------------------------------------------------------- -def _seed_workspace(eval_dir: Path, skill_path: Path, files: list[str]) -> Path: - """Copy fixture files into the run's working directory. +def _seed_workspace( + eval_dir: Path, + skill_path: Path, + files: list[str], + *, + configuration: str = "with_skill", + run_index: int = 1, +) -> Path: + """Copy fixture files into one run's working directory. The fixture root mirrors the skill's layout: relative paths under skill_path/<...> get copied to ws/. The skill's prompts reference paths like 'evals/files//...' so the workspace - cwd must be the skill dir. + cwd must be the skill dir. ``configuration`` (with_skill / without_skill) + and ``run_index`` (the epoch) keep each run's outputs isolated. """ - ws = eval_dir / "with_skill" / "run-1" + ws = eval_dir / configuration / f"run-{run_index}" outputs = ws / "outputs" outputs.mkdir(parents=True, exist_ok=True) # Mirror the skill dir structure under outputs/ so the prompt's @@ -401,121 +460,173 @@ def _seed_workspace(eval_dir: Path, skill_path: Path, files: list[str]) -> Path: return ws -def cmd_execute(args: argparse.Namespace) -> int: - skill_path = Path(args.skill).resolve() - skill_name = skill_path.name - evals_path = skill_path / "evals" / "evals.json" - if not evals_path.exists(): - print(f"no evals.json at {evals_path}", file=sys.stderr) - return 2 - suite = json.loads(evals_path.read_text()) - iter_dir = _next_iteration_dir(RUNS_ROOT / skill_name) +def _execute_one_run( + ev: dict[str, Any], + *, + skill_path: Path, + skill_name: str, + eval_dir: Path, + configuration: str, + run_index: int, + args: argparse.Namespace, +) -> dict[str, Any]: + """Run one (eval, configuration, epoch): execute, grade, optionally score + quality, and return the per-run summary. Writes all artifacts under the + run's own ``/run-/`` directory so epochs and arms + never clobber each other. + + For the ``without_skill`` control arm the Skill tool is disallowed so the + base agent solves the task without the skill's guidance; the (skill-aware) + quality grader is skipped there since its protocol-adherence dimension is + only meaningful when the skill is loaded. + """ + eid = ev["id"] + is_multi_turn = bool(ev.get("turns")) + ws = _seed_workspace( + eval_dir, + skill_path, + ev.get("files", []), + configuration=configuration, + run_index=run_index, + ) + outputs = ws / "outputs" + transcript_path = ws / "transcript.md" + grading_path = ws / "grading.json" - only = set(args.only.split(",")) if args.only else None - summaries: list[dict[str, Any]] = [] - for ev in suite["evals"]: - eid = ev["id"] - if only and str(eid) not in only: - continue - slug = _slug(ev.get("name") or ev["prompt"]) - eval_dir = iter_dir / f"eval-{eid}-{slug}" - eval_dir.mkdir(parents=True, exist_ok=True) - ws = _seed_workspace(eval_dir, skill_path, ev.get("files", [])) - outputs = ws / "outputs" - transcript_path = ws / "transcript.md" - meta_path = ws / "eval_metadata.json" - timing_path = ws / "timing.json" - grading_path = ws / "grading.json" - - is_multi_turn = bool(ev.get("turns")) - meta_path.write_text( - json.dumps( - { - "eval_id": eid, - "eval_name": ev.get("name"), - "skill_name": skill_name, - "prompt": ev.get("prompt"), - "turns": ev.get("turns"), - "mode": "multi-turn" if is_multi_turn else "single-turn", - "files": ev.get("files", []), - "expected_output": ev.get("expected_output"), - "expectations": ev["expectations"], - "executor_model": args.executor_model, - "grader_model": args.grader_model, - "timestamp": _now_iso(), - }, - indent=2, - ) + (ws / "eval_metadata.json").write_text( + json.dumps( + { + "eval_id": eid, + "eval_name": ev.get("name"), + "skill_name": skill_name, + "configuration": configuration, + "run_index": run_index, + "prompt": ev.get("prompt"), + "turns": ev.get("turns"), + "mode": "multi-turn" if is_multi_turn else "single-turn", + "files": ev.get("files", []), + "expected_output": ev.get("expected_output"), + "expectations": ev["expectations"], + "executor_model": args.executor_model, + "grader_model": args.grader_model, + "timestamp": _now_iso(), + }, + indent=2, ) + ) - print( - f"\n[execute] eval-{eid} {slug} ({'multi-turn' if is_multi_turn else 'single-turn'})", - flush=True, + print(f" [{configuration} run-{run_index}] eval-{eid}", flush=True) + # Control arm gets the Skill tool disabled; both arms block AskUserQuestion. + disallowed = "AskUserQuestion,Skill" if configuration == "without_skill" else "AskUserQuestion" + executor_extra = [ + "--permission-mode", + args.executor_permission_mode, + "--disallowedTools", + disallowed, + ] + t0 = time.perf_counter() + if is_multi_turn: + events, per_turn, session_id, gate_results = _run_conversation( + ev["turns"], + model=args.executor_model, + cwd=outputs, + extra_args=executor_extra, + timeout_s=args.executor_timeout_s, + ) + (ws / "session_id.txt").write_text(session_id) + (ws / "turns.json").write_text( + json.dumps({"per_turn": per_turn, "gate_results": gate_results}, indent=2) ) - print(f" workspace: {outputs}", flush=True) - executor_extra = [ + stdout = "\n".join(json.dumps(e) for e in events) + stderr = "" + else: + stdout, stderr, _rc_exec = _claude_cmd( + ev["prompt"], + model=args.executor_model, + cwd=outputs, + extra_args=executor_extra, + timeout_s=args.executor_timeout_s, + ) + events = _parse_stream_events(stdout) + gate_results = [] + exec_elapsed = time.perf_counter() - t0 + transcript_path.write_text(_render_transcript(events)) + (ws / "stream.jsonl").write_text(stdout) + if stderr: + (ws / "executor.stderr.log").write_text(stderr) + + # Grader: a separate `claude -p` that reads the transcript + outputs and + # writes grading.json. + grader_prompt = ( + GRADER_PROMPT_PATH.read_text() + + "\n\n## Run inputs\n\n" + + json.dumps( + { + "skill_name": skill_name, + "eval_prompt": ev.get("prompt") or [t.get("user") for t in ev.get("turns", [])], + "expectations": ev["expectations"], + "transcript_path": str(transcript_path), + "outputs_dir": str(outputs), + "grading_path": str(grading_path), + }, + indent=2, + ) + ) + t1 = time.perf_counter() + g_stdout, g_stderr, g_rc = _claude_cmd( + grader_prompt, + model=args.grader_model, + cwd=REPO_ROOT, + extra_args=[ "--permission-mode", - args.executor_permission_mode, + "bypassPermissions", "--disallowedTools", "AskUserQuestion", - ] - t0 = time.perf_counter() - if is_multi_turn: - events, per_turn, session_id, gate_results = _run_conversation( - ev["turns"], - model=args.executor_model, - cwd=outputs, - extra_args=executor_extra, - timeout_s=args.executor_timeout_s, - ) - (ws / "session_id.txt").write_text(session_id) - (ws / "turns.json").write_text( - json.dumps( - { - "per_turn": per_turn, - "gate_results": gate_results, - }, - indent=2, - ) - ) - stdout = "\n".join(json.dumps(e) for e in events) - stderr = "" - else: - stdout, stderr, rc_exec = _claude_cmd( - ev["prompt"], - model=args.executor_model, - cwd=outputs, - extra_args=executor_extra, - timeout_s=args.executor_timeout_s, - ) - events = _parse_stream_events(stdout) - gate_results = [] - exec_elapsed = time.perf_counter() - t0 - transcript_path.write_text(_render_transcript(events)) - (ws / "stream.jsonl").write_text(stdout) - if stderr: - (ws / "executor.stderr.log").write_text(stderr) - - # Grader call. The grader is a separate `claude -p` invocation that - # reads the transcript + outputs and writes grading.json. - grader_prompt = ( - GRADER_PROMPT_PATH.read_text() + ], + timeout_s=args.grader_timeout_s, + ) + grader_elapsed = time.perf_counter() - t1 + if g_stderr: + (ws / "grader.stderr.log").write_text(g_stderr) + (ws / "grader.stream.jsonl").write_text(g_stdout) + + if grading_path.exists(): + grading = json.loads(grading_path.read_text()) + pr = grading.get("summary", {}).get("pass_rate", 0.0) + print(f" graded pass_rate={pr:.2f} exec={exec_elapsed:.0f}s", flush=True) + else: + # No grading.json => grader infra failure. This run is ERRORED, not 0%: + # pass_rate stays None so aggregate() excludes it instead of scoring it 0. + print( + f" WARN no grading.json -- grader failed (rc={g_rc}); run marked errored", flush=True + ) + grading = None + + # Quality grader (skill-aware; with_skill arm only). + quality_path = ws / "quality.json" + quality_elapsed = 0.0 + quality = None + if configuration == "with_skill" and not args.skip_quality and grading is not None: + quality_skill_path = REPO_ROOT / "skills" / skill_name + quality_prompt = ( + QUALITY_PROMPT_PATH.read_text() + "\n\n## Run inputs\n\n" + json.dumps( { "skill_name": skill_name, + "skill_path": str(quality_skill_path), "eval_prompt": ev.get("prompt") or [t.get("user") for t in ev.get("turns", [])], - "expectations": ev["expectations"], "transcript_path": str(transcript_path), "outputs_dir": str(outputs), "grading_path": str(grading_path), + "quality_path": str(quality_path), }, indent=2, ) ) - t1 = time.perf_counter() - g_stdout, g_stderr, g_rc = _claude_cmd( - grader_prompt, + t2 = time.perf_counter() + q_stdout, q_stderr, q_rc = _claude_cmd( + quality_prompt, model=args.grader_model, cwd=REPO_ROOT, extra_args=[ @@ -526,191 +637,291 @@ def cmd_execute(args: argparse.Namespace) -> int: ], timeout_s=args.grader_timeout_s, ) - grader_elapsed = time.perf_counter() - t1 - if g_stderr: - (ws / "grader.stderr.log").write_text(g_stderr) - (ws / "grader.stream.jsonl").write_text(g_stdout) - - # Inspect grading.json (grader is supposed to write it). - if grading_path.exists(): - grading = json.loads(grading_path.read_text()) - pr = grading.get("summary", {}).get("pass_rate", 0.0) - print( - f" graded pass_rate={pr:.2f} exec={exec_elapsed:.1f}s grade={grader_elapsed:.1f}s", - flush=True, - ) + quality_elapsed = time.perf_counter() - t2 + (ws / "quality.stream.jsonl").write_text(q_stdout) + if q_stderr: + (ws / "quality.stderr.log").write_text(q_stderr) + if quality_path.exists(): + quality = json.loads(quality_path.read_text()) + print(f" quality overall={quality.get('overall_score')}", flush=True) else: - print( - f" WARN no grading.json at {grading_path} -- grader may have failed (rc={g_rc})", - flush=True, - ) - grading = None - - # Quality grader (orthogonal-quality post-grading). - quality_path = ws / "quality.json" - quality_elapsed = 0.0 - quality = None - if not args.skip_quality and grading is not None: - skill_path = REPO_ROOT / "skills" / skill_name - quality_prompt = ( - QUALITY_PROMPT_PATH.read_text() - + "\n\n## Run inputs\n\n" - + json.dumps( - { - "skill_name": skill_name, - "skill_path": str(skill_path), - "eval_prompt": ev.get("prompt") - or [t.get("user") for t in ev.get("turns", [])], - "transcript_path": str(transcript_path), - "outputs_dir": str(outputs), - "grading_path": str(grading_path), - "quality_path": str(quality_path), - }, - indent=2, - ) - ) - t2 = time.perf_counter() - q_stdout, q_stderr, q_rc = _claude_cmd( - quality_prompt, - model=args.grader_model, - cwd=REPO_ROOT, - extra_args=[ - "--permission-mode", - "bypassPermissions", - "--disallowedTools", - "AskUserQuestion", - ], - timeout_s=args.grader_timeout_s, - ) - quality_elapsed = time.perf_counter() - t2 - (ws / "quality.stream.jsonl").write_text(q_stdout) - if q_stderr: - (ws / "quality.stderr.log").write_text(q_stderr) - if quality_path.exists(): - quality = json.loads(quality_path.read_text()) - qos = quality.get("overall_score") - print(f" quality overall={qos} ({quality_elapsed:.1f}s)", flush=True) - else: - print(f" WARN no quality.json (rc={q_rc})", flush=True) - - timing_path.write_text( - json.dumps( - { - "executor_duration_seconds": round(exec_elapsed, 2), - "grader_duration_seconds": round(grader_elapsed, 2), - "quality_grader_duration_seconds": round(quality_elapsed, 2), - "total_duration_seconds": round( - exec_elapsed + grader_elapsed + quality_elapsed, 2 - ), - }, - indent=2, - ) - ) + print(f" WARN no quality.json (rc={q_rc})", flush=True) - summaries.append( + (ws / "timing.json").write_text( + json.dumps( { - "eval_id": eid, - "eval_name": ev.get("name"), - "configuration": "with_skill", - "run_number": 1, - "mode": "multi-turn" if is_multi_turn else "single-turn", - "result": { - "pass_rate": (grading or {}).get("summary", {}).get("pass_rate"), - "passed": (grading or {}).get("summary", {}).get("passed"), - "failed": (grading or {}).get("summary", {}).get("failed"), - "total": (grading or {}).get("summary", {}).get("total"), - "time_seconds": round(exec_elapsed, 2), - "tool_calls": sum(_count_tool_calls(events).values()), - "errors": 0, - "quality_overall": (quality or {}).get("overall_score"), - "quality_scores": { - k: v.get("value") for k, v in (quality or {}).get("scores", {}).items() - }, - }, - "expectations": (grading or {}).get("expectations", []), - "tool_calls_breakdown": _count_tool_calls(events), - "gate_results": gate_results, - "quality_summary": (quality or {}).get("summary"), - } + "executor_duration_seconds": round(exec_elapsed, 2), + "grader_duration_seconds": round(grader_elapsed, 2), + "quality_grader_duration_seconds": round(quality_elapsed, 2), + "total_duration_seconds": round(exec_elapsed + grader_elapsed + quality_elapsed, 2), + }, + indent=2, ) + ) + + return { + "eval_id": eid, + "eval_name": ev.get("name"), + "configuration": configuration, + "run_number": run_index, + "mode": "multi-turn" if is_multi_turn else "single-turn", + "result": { + # None (not 0.0) when the grader failed: aggregate() treats this run + # as errored and excludes it from the rate. + "pass_rate": (grading or {}).get("summary", {}).get("pass_rate"), + "passed": (grading or {}).get("summary", {}).get("passed"), + "failed": (grading or {}).get("summary", {}).get("failed"), + "total": (grading or {}).get("summary", {}).get("total"), + "errored": grading is None, + "time_seconds": round(exec_elapsed, 2), + "tool_calls": sum(_count_tool_calls(events).values()), + "quality_overall": (quality or {}).get("overall_score"), + "quality_scores": { + k: v.get("value") for k, v in (quality or {}).get("scores", {}).items() + }, + }, + "expectations": (grading or {}).get("expectations", []), + "tool_calls_breakdown": _count_tool_calls(events), + "gate_results": gate_results, + "quality_summary": (quality or {}).get("summary"), + } + + +def cmd_execute(args: argparse.Namespace) -> int: + skill_path = Path(args.skill).resolve() + skill_name = skill_path.name + evals_path = skill_path / "evals" / "evals.json" + if not evals_path.exists(): + print(f"no evals.json at {evals_path}", file=sys.stderr) + return 2 + suite = json.loads(evals_path.read_text()) + iter_dir = _next_iteration_dir(RUNS_ROOT / skill_name) + + only = set(args.only.split(",")) if args.only else None + plan = execution_plan(epochs=args.epochs, baseline=args.baseline) + print( + f"plan: {len(plan)} run(s) per eval " + f"({args.epochs} epoch(s) x {len(set(c for c, _ in plan))} arm(s))", + flush=True, + ) + summaries: list[dict[str, Any]] = [] + for ev in suite["evals"]: + eid = ev["id"] + if only and str(eid) not in only: + continue + slug = _slug(ev.get("name") or ev["prompt"]) + eval_dir = iter_dir / f"eval-{eid}-{slug}" + eval_dir.mkdir(parents=True, exist_ok=True) + print(f"\n[execute] eval-{eid} {slug}", flush=True) + for configuration, run_index in plan: + summaries.append( + _execute_one_run( + ev, + skill_path=skill_path, + skill_name=skill_name, + eval_dir=eval_dir, + configuration=configuration, + run_index=run_index, + args=args, + ) + ) - # Aggregate across this iteration. - benchmark = _aggregate(skill_name, args.executor_model, summaries) + benchmark = aggregate( + skill_name, + args.executor_model, + args.grader_model, + summaries, + epochs=args.epochs, + git_sha=_git_sha(), + git_dirty=_git_dirty(), + ) (iter_dir / "benchmark.json").write_text(json.dumps(benchmark, indent=2)) (iter_dir / "benchmark.md").write_text(_render_benchmark_md(benchmark)) print(f"\nwrote {iter_dir / 'benchmark.json'}") + print(_render_benchmark_md(benchmark)) return 0 -def _aggregate(skill_name: str, executor_model: str, runs: list[dict[str, Any]]) -> dict[str, Any]: - by_config: dict[str, list[dict[str, Any]]] = {} +def aggregate( + skill_name: str, + executor_model: str, + grader_model: str, + runs: list[dict[str, Any]], + *, + epochs: int, + git_sha: str | None = None, + git_dirty: bool | None = None, +) -> dict[str, Any]: + """Two-stage aggregation of per-(eval, configuration, epoch) summaries. + + Stage 1 collapses an eval's epochs into one number (a model-stochasticity + CI). Stage 2 aggregates across evals -- clustering on the eval so N epochs + of one eval don't masquerade as N independent observations (the + standard-error inflation Anthropic's "Adding Error Bars to Evals" warns + about). Errored runs (``pass_rate is None``) are excluded from rates and + counted separately, so a grader/infra failure never reads as a 0% score. + When both ``with_skill`` and ``without_skill`` arms exist, a paired + per-eval delta quantifies the skill's effect. + """ + by_cfg_eval: dict[str, dict[Any, list[dict[str, Any]]]] = {} for r in runs: - by_config.setdefault(r["configuration"], []).append(r) - summary: dict[str, Any] = {} - for config, rs in by_config.items(): - rates = [r["result"].get("pass_rate") or 0.0 for r in rs] - times = [r["result"].get("time_seconds") or 0.0 for r in rs] - qualities = [ - r["result"].get("quality_overall") - for r in rs - if r["result"].get("quality_overall") is not None + by_cfg_eval.setdefault(r["configuration"], {}).setdefault(r["eval_id"], []).append(r) + + per_eval: list[dict[str, Any]] = [] + eval_mean: dict[tuple[str, Any], float | None] = {} + for cfg in sorted(by_cfg_eval): + for eid in sorted(by_cfg_eval[cfg]): + rows = by_cfg_eval[cfg][eid] + rates = [row["result"].get("pass_rate") for row in rows] + scored = [x for x in rates if x is not None] + pr = mean_ci(scored) + qci = mean_ci([row["result"].get("quality_overall") for row in rows]) + tci = mean_ci([row["result"].get("time_seconds") for row in rows]) + per_eval.append( + { + "eval_id": eid, + "eval_name": rows[0].get("eval_name"), + "configuration": cfg, + "mode": rows[0].get("mode"), + "epochs": len(rows), + "errored_epochs": len(rates) - len(scored), + "pass_rate": pr, + "quality_overall": qci if qci["n"] else None, + "time_seconds": tci, + } + ) + eval_mean[(cfg, eid)] = pr["mean"] + + run_summary: dict[str, Any] = {} + for cfg in sorted(by_cfg_eval): + rows = [pe for pe in per_eval if pe["configuration"] == cfg] + scored_means = [ + pe["pass_rate"]["mean"] for pe in rows if pe["pass_rate"]["mean"] is not None ] - summary[config] = { - "pass_rate": _stats(rates), - "time_seconds": _stats(times), - "quality_overall": _stats(qualities) if qualities else None, + q_means = [pe["quality_overall"]["mean"] for pe in rows if pe["quality_overall"]] + run_summary[cfg] = { + "pass_rate": mean_ci(scored_means), + "evals_scored": len(scored_means), + "errored_runs": sum(pe["errored_epochs"] for pe in rows), + "quality_overall": mean_ci(q_means) if q_means else None, + "time_seconds": mean_ci([pe["time_seconds"]["mean"] for pe in rows]), } + + baseline_delta = None + if {"with_skill", "without_skill"} <= set(by_cfg_eval): + common = sorted(set(by_cfg_eval["with_skill"]) & set(by_cfg_eval["without_skill"])) + deltas = [ + eval_mean[("with_skill", eid)] - eval_mean[("without_skill", eid)] + for eid in common + if eval_mean[("with_skill", eid)] is not None + and eval_mean[("without_skill", eid)] is not None + ] + baseline_delta = {"pass_rate": mean_ci(deltas), "n_evals_paired": len(deltas)} + return { "metadata": { "skill_name": skill_name, "executor_model": executor_model, + "grader_model": grader_model, "timestamp": _now_iso(), + "git_sha": git_sha, + "git_dirty": git_dirty, "evals_run": sorted({r["eval_id"] for r in runs}), - "runs_per_configuration": 1, + "epochs": epochs, + "configurations": sorted(by_cfg_eval), }, "runs": runs, - "run_summary": summary, + "per_eval": per_eval, + "run_summary": run_summary, + "baseline_delta": baseline_delta, } -def _stats(xs: list[float]) -> dict[str, float]: - if not xs: - return {"mean": 0, "stddev": 0, "min": 0, "max": 0} - return { - "mean": round(statistics.fmean(xs), 3), - "stddev": round(statistics.pstdev(xs), 3) if len(xs) > 1 else 0.0, - "min": round(min(xs), 3), - "max": round(max(xs), 3), - } +def _git_sha() -> str | None: + """Best-effort short SHA so a benchmark can be tied to a skill version.""" + try: + r = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=2, + cwd=str(REPO_ROOT), + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + return r.stdout.strip() or None if r.returncode == 0 else None + + +def _git_dirty() -> bool: + try: + r = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, + text=True, + timeout=2, + cwd=str(REPO_ROOT), + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + return bool(r.stdout.strip()) + + +def _fmt_ci(stat: dict[str, Any] | None, *, nd: int = 2, sign: bool = False) -> str: + """Render a mean_ci dict as 'mean [lo, hi] (n=k)', or '—' if unmeasured.""" + if not stat or stat.get("mean") is None: + return "—" + m = stat["mean"] + head = f"{m:+.{nd}f}" if sign else f"{m:.{nd}f}" + if stat["n"] < 2: + return f"{head} (n={stat['n']})" + return f"{head} [{stat['ci_low']:.{nd}f}, {stat['ci_high']:.{nd}f}] (n={stat['n']})" def _render_benchmark_md(b: dict[str, Any]) -> str: + m = b["metadata"] + git = f"{m.get('git_sha') or '?'}{' (dirty)' if m.get('git_dirty') else ''}" lines = [ - f"# Benchmark: {b['metadata']['skill_name']}", + f"# Benchmark: {m['skill_name']}", "", - f"- timestamp: {b['metadata']['timestamp']}", - f"- executor: {b['metadata']['executor_model']}", - f"- evals: {b['metadata']['evals_run']}", + f"- timestamp: {m['timestamp']}", + f"- executor: {m['executor_model']} grader: {m.get('grader_model', '?')}", + f"- git: {git}", + f"- epochs: {m.get('epochs')} configurations: {', '.join(m.get('configurations', []))}", + f"- evals: {m['evals_run']}", "", - "## Per-eval", + "## Per-eval (mean [95% CI] across epochs)", "", - "| eval_id | name | mode | pass_rate | quality | time_s | tool_calls |", - "|---|---|---|---|---|---|---|", + "| eval_id | name | config | mode | pass_rate | quality | time_s | epochs | errored |", + "|---|---|---|---|---|---|---|---|---|", ] - for r in b["runs"]: - res = r["result"] - qos = res.get("quality_overall") - qbits = res.get("quality_scores", {}) - qstr = ( - f"{qos}" + (f" ({','.join(f'{k[0]}={v}' for k, v in qbits.items())})" if qbits else "") - if qos is not None - else "-" + for pe in b.get("per_eval", []): + lines.append( + f"| {pe['eval_id']} | {pe.get('eval_name', '')} | {pe['configuration']} | " + f"{pe.get('mode', '')} | {_fmt_ci(pe['pass_rate'])} | " + f"{_fmt_ci(pe.get('quality_overall'))} | {_fmt_ci(pe['time_seconds'], nd=1)} | " + f"{pe['epochs']} | {pe['errored_epochs']} |" ) + lines += [ + "", + "## Suite (clustered on eval)", + "", + "| configuration | pass_rate | quality | evals_scored | errored_runs |", + "|---|---|---|---|---|", + ] + for cfg, s in b["run_summary"].items(): lines.append( - f"| {r['eval_id']} | {r.get('eval_name', '')} | {r.get('mode', 'single-turn')} | " - f"{res.get('pass_rate')} | {qstr} | " - f"{res.get('time_seconds')} | {res.get('tool_calls')} |" + f"| {cfg} | {_fmt_ci(s['pass_rate'])} | {_fmt_ci(s.get('quality_overall'))} | " + f"{s['evals_scored']} | {s['errored_runs']} |" ) + if b.get("baseline_delta"): + d = b["baseline_delta"] + lines += [ + "", + "## Skill effect (with_skill − without_skill, paired on eval)", + "", + f"- pass_rate uplift: **{_fmt_ci(d['pass_rate'], sign=True)}** " + f"over {d['n_evals_paired']} paired eval(s)", + ] return "\n".join(lines) + "\n" @@ -738,6 +949,19 @@ def main(argv: list[str] | None = None) -> int: pe = sub.add_parser("execute", help="Run execution-quality eval and grade it") pe.add_argument("--skill", required=True, help="path to skill dir, e.g. skills/bully-init") pe.add_argument("--only", help="comma-separated eval ids to run (default: all)") + pe.add_argument( + "--epochs", + type=int, + default=1, + help="runs per (eval, arm) so every rate carries a 95%% CI. Default 1 " + "(legacy point estimate); use >=5 for trustworthy error bars.", + ) + pe.add_argument( + "--baseline", + action="store_true", + help="also run a without_skill control arm (Skill tool disabled) and " + "report a paired per-eval pass_rate delta -- the skill's attributable effect.", + ) pe.add_argument( "--executor-timeout-s", type=float, diff --git a/pyproject.toml b/pyproject.toml index 5f99691..0fc2a4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,4 +58,4 @@ quote-style = "double" [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] +pythonpath = ["src", "bench"] diff --git a/tests/test_skill_evals.py b/tests/test_skill_evals.py new file mode 100644 index 0000000..eff9b57 --- /dev/null +++ b/tests/test_skill_evals.py @@ -0,0 +1,180 @@ +"""Tests for the skill eval harness's pure logic (`bench/run_skill_evals.py`). + +The skill-eval driver lives in `bench/` (added to pytest `pythonpath`). These +cover the P0 statistics + aggregation fixes — multi-epoch confidence intervals, +errored-run exclusion (an infra failure must NOT be scored as 0%), and the +baseline paired delta — none of which touch the `claude` subprocess, so they +run fast and offline. +""" + +from __future__ import annotations + +import math +import statistics + +import run_skill_evals as rse + +_Z95 = 1.959963984540054 + + +# ----- mean_ci ---------------------------------------------------------------- + + +def test_mean_ci_reports_mean_sem_and_95_ci(): + xs = [0.8, 1.0, 0.6, 1.0, 0.8] + s = rse.mean_ci(xs) + assert s["n"] == 5 + assert abs(s["mean"] - 0.84) < 1e-9 + sd = statistics.stdev(xs) + assert abs(s["stddev"] - sd) < 1e-9 + assert abs(s["sem"] - sd / math.sqrt(5)) < 1e-9 + # 95% CI half-width is 1.96 * SEM, symmetric around the mean. + assert abs((s["ci_high"] - s["mean"]) - _Z95 * s["sem"]) < 1e-9 + assert abs((s["mean"] - s["ci_low"]) - _Z95 * s["sem"]) < 1e-9 + + +def test_mean_ci_single_value_has_zero_width_ci(): + s = rse.mean_ci([0.5]) + assert s["n"] == 1 + assert s["mean"] == 0.5 + assert s["sem"] == 0.0 + assert s["ci_low"] == 0.5 and s["ci_high"] == 0.5 + + +def test_mean_ci_empty_returns_none_mean_not_zero(): + # The crux of the P0-C bug: an absent measurement is None, never 0.0. + s = rse.mean_ci([]) + assert s["n"] == 0 + assert s["mean"] is None + + +def test_mean_ci_ignores_none_entries(): + s = rse.mean_ci([1.0, None, 1.0]) + assert s["n"] == 2 + assert s["mean"] == 1.0 + + +# ----- execution_plan --------------------------------------------------------- + + +def test_execution_plan_single_config_is_n_epochs(): + assert rse.execution_plan(epochs=3, baseline=False) == [ + ("with_skill", 1), + ("with_skill", 2), + ("with_skill", 3), + ] + + +def test_execution_plan_baseline_adds_without_skill_arm(): + assert rse.execution_plan(epochs=2, baseline=True) == [ + ("with_skill", 1), + ("with_skill", 2), + ("without_skill", 1), + ("without_skill", 2), + ] + + +# ----- _seed_workspace -------------------------------------------------------- + + +def test_seed_workspace_uses_configuration_and_run_index(tmp_path): + skill = tmp_path / "skills" / "demo" + (skill / "evals" / "files").mkdir(parents=True) + (skill / "evals" / "files" / "x.txt").write_text("hi") + eval_dir = tmp_path / "eval-1" + ws = rse._seed_workspace( + eval_dir, + skill, + ["evals/files/x.txt"], + configuration="without_skill", + run_index=2, + ) + assert ws == eval_dir / "without_skill" / "run-2" + assert (ws / "outputs" / "evals" / "files" / "x.txt").read_text() == "hi" + + +# ----- aggregate -------------------------------------------------------------- + + +def _run(eval_id, config, pass_rate, *, quality=None, time_s=1.0, name="e"): + """A minimal per-(eval, config, epoch) summary as cmd_execute appends.""" + return { + "eval_id": eval_id, + "eval_name": name, + "configuration": config, + "mode": "single-turn", + "result": { + "pass_rate": pass_rate, + "quality_overall": quality, + "time_seconds": time_s, + }, + } + + +def test_aggregate_multi_epoch_pass_rate_has_ci(): + runs = [ + _run(1, "with_skill", 1.0), + _run(1, "with_skill", 0.5), + _run(1, "with_skill", 1.0), + ] + b = rse.aggregate("s", "exec", "grader", runs, epochs=3) + pe = next(p for p in b["per_eval"] if p["eval_id"] == 1) + assert abs(pe["pass_rate"]["mean"] - (2.5 / 3)) < 1e-9 + assert pe["pass_rate"]["n"] == 3 + assert pe["epochs"] == 3 + assert pe["errored_epochs"] == 0 + assert "with_skill" in b["run_summary"] + + +def test_aggregate_excludes_errored_epoch_from_rate(): + # One graded epoch (1.0) + one grader failure (None) -> mean is 1.0, not 0.5. + runs = [_run(1, "with_skill", 1.0), _run(1, "with_skill", None)] + b = rse.aggregate("s", "exec", "grader", runs, epochs=2) + pe = next(p for p in b["per_eval"] if p["eval_id"] == 1) + assert pe["pass_rate"]["mean"] == 1.0 + assert pe["pass_rate"]["n"] == 1 + assert pe["errored_epochs"] == 1 + + +def test_aggregate_all_errored_eval_is_none_not_zero(): + runs = [_run(1, "with_skill", None), _run(1, "with_skill", None)] + b = rse.aggregate("s", "exec", "grader", runs, epochs=2) + pe = next(p for p in b["per_eval"] if p["eval_id"] == 1) + assert pe["pass_rate"]["mean"] is None + assert pe["errored_epochs"] == 2 + # The suite number must not silently absorb the failure as a 0.0. + assert b["run_summary"]["with_skill"]["pass_rate"]["mean"] is None + assert b["run_summary"]["with_skill"]["evals_scored"] == 0 + assert b["run_summary"]["with_skill"]["errored_runs"] == 2 + + +def test_aggregate_suite_clusters_on_eval_not_epochs(): + # Two evals, one epoch each: suite mean is over 2 evals (clustered), not 2 rows. + runs = [ + _run(1, "with_skill", 1.0, name="a"), + _run(2, "with_skill", 0.0, name="b"), + ] + b = rse.aggregate("s", "exec", "grader", runs, epochs=1) + suite = b["run_summary"]["with_skill"]["pass_rate"] + assert abs(suite["mean"] - 0.5) < 1e-9 + assert suite["n"] == 2 + + +def test_aggregate_baseline_reports_paired_delta(): + runs = [ + _run(1, "with_skill", 1.0, name="a"), + _run(2, "with_skill", 1.0, name="b"), + _run(1, "without_skill", 0.0, name="a"), + _run(2, "without_skill", 0.5, name="b"), + ] + b = rse.aggregate("s", "exec", "grader", runs, epochs=1) + delta = b["baseline_delta"]["pass_rate"] + # Per-eval deltas [1.0, 0.5] -> mean uplift 0.75 over n=2 paired evals. + assert abs(delta["mean"] - 0.75) < 1e-9 + assert delta["n"] == 2 + assert b["metadata"]["configurations"] == ["with_skill", "without_skill"] + + +def test_aggregate_no_baseline_block_without_control_arm(): + b = rse.aggregate("s", "exec", "grader", [_run(1, "with_skill", 1.0)], epochs=1) + assert b["baseline_delta"] is None