From cdfbf5eb5916dfc00292b3106cfb3be9a9a01fba Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 13:08:09 -0300 Subject: [PATCH] feat(d2): report Hake normalized gain and use paired-difference significance Two statistical upgrades to the effectiveness (D2) dimension, both grounded in the deep-research survey (SkillsBench + Miller 2024, arXiv 2411.00640): - Normalized gain g = (skill - baseline) / (1 - baseline) corrects the raw lift for baseline headroom, so a small lift on a high baseline reads as the strong result it is. Reported in the scorecard D2 line and JSON. - Significance now runs on the per-eval PAIRED difference when >=2 evals are present in both configs: shared per-eval difficulty cancels, shrinking the SE so a real lift clears the noise with fewer trials. Falls back to the unpaired se_of_difference for a single eval (no between-eval variance to exploit). New stats helpers (normalized_gain, paired_diff) are pure and unit-tested; aggregate/dimensions/scorecard thread the new fields through. baseline_lift and its unpaired SE are unchanged for backward compatibility. --- skills/skill-validator/SKILL.md | 10 +++-- skills/skill-validator/scripts/aggregate.py | 46 +++++++++++++++++--- skills/skill-validator/scripts/dimensions.py | 3 ++ skills/skill-validator/scripts/scorecard.py | 3 ++ skills/skill-validator/scripts/stats.py | 33 ++++++++++++-- tests/test_aggregate.py | 45 +++++++++++++++++++ tests/test_scorecard.py | 26 +++++++++++ tests/test_stats.py | 19 ++++++++ 8 files changed, 172 insertions(+), 13 deletions(-) diff --git a/skills/skill-validator/SKILL.md b/skills/skill-validator/SKILL.md index 362ff88..15f7fa9 100644 --- a/skills/skill-validator/SKILL.md +++ b/skills/skill-validator/SKILL.md @@ -121,10 +121,12 @@ Make a workspace dir, run these stages (each has a guide in `agents/`), then let - **Batch** — rank many scorecards with `scripts/batch.py:rank_scorecards([...])`. - **Eval-viewer** — `scripts/benchmark_export.py` emits a skill-creator-compatible `benchmark.json` so runs render in that project's viewer. -- **SkillsBench-style paired evals** — when reporting D2 lift, preserve no-skill pass - rate, with-skill pass rate, absolute lift, and normalized gain; see - [`docs/references/skillsbench.md`](../../docs/references/skillsbench.md) for the - research note that motivates this future scorecard improvement. +- **SkillsBench-style D2 lift** — `aggregate.py` reports absolute lift **and Hake + normalized gain** `g = (skill − baseline) / (1 − baseline)`, which corrects for + baseline headroom, and decides significance from a **paired** per-eval difference + (Miller 2024) — shared per-eval difficulty cancels, so a real lift clears the noise + with fewer trials (falls back to the unpaired SE for a single eval). See + [`docs/references/skillsbench.md`](../../docs/references/skillsbench.md). - **Calibration** — tune dimension weights against a labeled corpus with `scripts/calibrate.py:suggest_weights(examples)`. diff --git a/skills/skill-validator/scripts/aggregate.py b/skills/skill-validator/scripts/aggregate.py index 83d8675..79567a7 100644 --- a/skills/skill-validator/scripts/aggregate.py +++ b/skills/skill-validator/scripts/aggregate.py @@ -2,12 +2,16 @@ Reads runs via the shared ``runs_io`` loader and computes, per configuration: a pass-rate summary (mean/stddev/std_error), tau-bench ``pass^k`` over evals, the -with-skill vs without-skill baseline lift, and whether that lift is significant. +with-skill vs without-skill baseline lift, its Hake normalized gain (headroom- +corrected), and whether that lift is significant. -Significance is one-sided (did the skill help beyond noise) and is computed from -the *unrounded* means/standard-errors so display rounding can't flip the verdict. -(``compare.compare_scorecards`` uses a two-sided test for per-dimension change, -which is a different question — a regression there is still a "significant" change.) +Significance is one-sided (did the skill help beyond noise) and is computed at +full precision so display rounding can't flip the verdict. It uses a *paired* +per-eval difference (Miller 2024) when >=2 evals appear in both configs — shared +per-eval difficulty cancels, lowering the SE — and falls back to the unpaired +``se_of_difference`` for a single eval. (``compare.compare_scorecards`` uses a +two-sided test for per-dimension change, a different question — a regression +there is still a "significant" change.) """ from __future__ import annotations @@ -29,6 +33,15 @@ def _successes_per_eval(runs: list[dict], threshold: float) -> list[tuple[int, i return list(by_eval.values()) +def _mean_pass_by_eval(runs: list[dict]) -> dict: + sums: dict = {} + counts: dict = {} + for r in runs: + sums[r["eval_id"]] = sums.get(r["eval_id"], 0.0) + r["pass_rate"] + counts[r["eval_id"]] = counts.get(r["eval_id"], 0) + 1 + return {e: sums[e] / counts[e] for e in sums} + + def aggregate(bench_dir, success_threshold: float = 1.0) -> dict: results = runs_io.load_runs(bench_dir) @@ -46,6 +59,7 @@ def aggregate(bench_dir, success_threshold: float = 1.0) -> dict: } lift = se_diff = significant = None + norm_gain = paired_lift = paired_se = None if "with_skill" in results and "without_skill" in results: w = [r["pass_rate"] for r in results["with_skill"]] b = [r["pass_rate"] for r in results["without_skill"]] @@ -53,13 +67,33 @@ def aggregate(bench_dir, success_threshold: float = 1.0) -> dict: raw_se = stats.se_of_difference(stats.std_error(w), stats.std_error(b)) lift = round(raw_lift, 4) se_diff = round(raw_se, 4) - significant = bool(raw_lift > raw_se) # decided at full precision + + gain = stats.normalized_gain(stats.mean(w), stats.mean(b)) + norm_gain = round(gain, 4) if gain is not None else None + + # Paired inference (Miller 2024): pair the per-eval means so shared + # per-eval difficulty cancels. Needs >=2 evals present in both configs + # for a between-eval SE; otherwise fall back to the unpaired within-eval + # SE (a single eval has no between-eval variance to exploit). + ew = _mean_pass_by_eval(results["with_skill"]) + eb = _mean_pass_by_eval(results["without_skill"]) + shared = sorted(ew.keys() & eb.keys()) + if len(shared) >= 2: + p_mean, p_se, _ = stats.paired_diff([(ew[e], eb[e]) for e in shared]) + paired_lift = round(p_mean, 4) + paired_se = round(p_se, 4) + significant = bool(p_mean > p_se) # decided at full precision + else: + significant = bool(raw_lift > raw_se) return { "configs": configs, "pass_hat_k": pass_hat, "baseline_lift": lift, "se_difference": se_diff, + "normalized_gain": norm_gain, + "paired_lift": paired_lift, + "paired_se": paired_se, "significant": significant, "success_threshold": success_threshold, } diff --git a/skills/skill-validator/scripts/dimensions.py b/skills/skill-validator/scripts/dimensions.py index 5a9213a..ec4f439 100644 --- a/skills/skill-validator/scripts/dimensions.py +++ b/skills/skill-validator/scripts/dimensions.py @@ -62,6 +62,9 @@ def behavioral_dims( detail["D2"] = { "pass_rate": agg["configs"]["with_skill"]["pass_rate"], "baseline_lift": agg.get("baseline_lift"), + "normalized_gain": agg.get("normalized_gain"), + "paired_lift": agg.get("paired_lift"), + "paired_se": agg.get("paired_se"), "significant": agg.get("significant"), } if agg.get("pass_hat_k", {}).get("with_skill"): diff --git a/skills/skill-validator/scripts/scorecard.py b/skills/skill-validator/scripts/scorecard.py index f15578a..fe586f6 100644 --- a/skills/skill-validator/scripts/scorecard.py +++ b/skills/skill-validator/scripts/scorecard.py @@ -144,6 +144,9 @@ def _dim_detail(dim: str, e: dict) -> str: lift = e.get("baseline_lift") if lift is not None: s += f", lift {lift * 100:+.0f}% ({'significant' if e.get('significant') else 'n.s.'})" + gain = e.get("normalized_gain") + if gain is not None: + s += f", gain {gain:+.2f}" return s if dim == "D3": ph = e.get("pass_hat_k", {}) diff --git a/skills/skill-validator/scripts/stats.py b/skills/skill-validator/scripts/stats.py index c2316ae..5c41fb0 100644 --- a/skills/skill-validator/scripts/stats.py +++ b/skills/skill-validator/scripts/stats.py @@ -1,9 +1,10 @@ """Pure statistics helpers for skval scoring. No third-party dependencies. Covers the variance/reliability machinery the -scorecard needs: sample mean/stddev/standard-error, a compact summary, and the -tau-bench ``pass^k`` reliability metric plus the standard error of a difference -(Miller 2024) used to decide whether a with-skill vs no-skill delta is real. +scorecard needs: sample mean/stddev/standard-error, a compact summary, the +tau-bench ``pass^k`` reliability metric, the standard error of a difference and +the ``paired_diff`` (Miller 2024) used to decide whether a with-skill vs no-skill +delta is real, and the Hake ``normalized_gain`` that corrects lift for headroom. """ from __future__ import annotations @@ -72,3 +73,29 @@ def pass_hat_k_over_evals(per_eval: Iterable[tuple[int, int]], k: int) -> float: def se_of_difference(se_a: float, se_b: float) -> float: """Standard error of a difference of two independent means: sqrt(se_a^2+se_b^2).""" return math.sqrt(se_a**2 + se_b**2) + + +def normalized_gain(pass_skill: float, pass_baseline: float) -> float | None: + """Hake normalized gain g = (skill - baseline) / (1 - baseline), in (-inf, 1]. + + Corrects the raw lift for baseline headroom: a +0.05 lift closes half the + remaining gap on a 0.9 baseline (g=0.5) but only a twelfth on a 0.4 baseline + (g≈0.083). ``None`` when the baseline is already 1.0 (no headroom, undefined). + """ + headroom = 1.0 - pass_baseline + if headroom <= 0: + return None + return (pass_skill - pass_baseline) / headroom + + +def paired_diff(pairs: Iterable[tuple[float, float]]) -> tuple[float, float, int]: + """Paired mean difference, its standard error, and the item count. + + ``pairs`` are (with_skill, without_skill) per matched item (eval). Inference + on the per-item differences rather than the two group means (Miller 2024) + cancels the shared per-item level, so when the two conditions are positively + correlated across items the SE is smaller than an unpaired ``se_of_difference`` + — more power to detect a real lift. SE is 0 for a single pair (no spread). + """ + diffs = [a - b for a, b in pairs] + return mean(diffs), std_error(diffs), len(diffs) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 2c15855..bda4b1b 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -3,6 +3,7 @@ import pytest import aggregate +import stats def _grading(pass_rate, total=2): @@ -64,3 +65,47 @@ def test_aggregate_no_baseline(tmp_path): agg = aggregate.aggregate(tmp_path) assert agg["baseline_lift"] is None assert agg["significant"] is None + assert agg["normalized_gain"] is None + assert agg["paired_lift"] is None + + +def test_aggregate_normalized_gain(tmp_path): + # with-skill mean 0.8, baseline mean 0.6 -> gain (0.8-0.6)/(1-0.6) = 0.5 + for r in (1, 2): + _mk_run(tmp_path, 0, "with_skill", r, _grading(0.8)) + _mk_run(tmp_path, 0, "without_skill", r, _grading(0.6)) + agg = aggregate.aggregate(tmp_path) + assert agg["baseline_lift"] == pytest.approx(0.2) + assert agg["normalized_gain"] == pytest.approx(0.5) + + +def test_aggregate_paired_significance_beats_unpaired(tmp_path): + # Two evals of very different difficulty; the skill adds a steady ~0.19 on + # each. The between-eval difficulty swamps an unpaired SE (verdict n.s.), + # but cancels in the paired difference (Miller 2024) -> significant. + for r in (1, 2): + _mk_run(tmp_path, 0, "with_skill", r, _grading(0.4)) # hard eval + _mk_run(tmp_path, 0, "without_skill", r, _grading(0.2)) + _mk_run(tmp_path, 1, "with_skill", r, _grading(0.9)) # easy eval + _mk_run(tmp_path, 1, "without_skill", r, _grading(0.72)) + agg = aggregate.aggregate(tmp_path) + # unpaired SE of the difference would not clear the lift... + unpaired = stats.se_of_difference( + stats.std_error([0.4, 0.4, 0.9, 0.9]), stats.std_error([0.2, 0.2, 0.72, 0.72]) + ) + assert agg["baseline_lift"] <= unpaired # unpaired alone would be n.s. + # ...but the paired inference exploits the correlation and clears it. + assert agg["paired_lift"] == pytest.approx(0.19) + assert agg["significant"] is True + + +def test_aggregate_paired_none_with_single_eval(tmp_path): + # One eval -> no between-eval variance to exploit; paired fields stay None + # and significance falls back to the unpaired within-eval SE. + for r in (1, 2, 3): + _mk_run(tmp_path, 0, "with_skill", r, _grading(1.0)) + _mk_run(tmp_path, 0, "without_skill", r, _grading(0.0)) + agg = aggregate.aggregate(tmp_path) + assert agg["paired_lift"] is None + assert agg["paired_se"] is None + assert agg["significant"] is True # unpaired fallback still fires diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py index c8abebc..59b86f4 100644 --- a/tests/test_scorecard.py +++ b/tests/test_scorecard.py @@ -36,6 +36,32 @@ def test_build_and_render(tmp_path): assert Path(m).exists() +def test_render_d2_shows_normalized_gain(): + sc = scorecard.build_scorecard( + provenance={"source": "x", "kind": "dir"}, + scoring_result={ + "score": 80, + "grade": "B", + "verdict": "Ship", + "dims": {"D2": 0.8}, + "weights_used": {"D2": 0.30}, + }, + d1_checks=[], + safety={"safety_pass": True, "findings": []}, + dimensions_detail={ + "D2": { + "pass_rate": {"mean": 0.8, "stddev": 0.1}, + "baseline_lift": 0.2, + "normalized_gain": 0.5, + "significant": True, + } + }, + ) + md = scorecard.render_markdown(sc) + assert "gain +0.50" in md + assert "lift +20%" in md + + def test_render_shows_classification(): sc = scorecard.build_scorecard( provenance={"source": "x", "kind": "dir"}, diff --git a/tests/test_stats.py b/tests/test_stats.py index 7673983..08aca3c 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -48,3 +48,22 @@ def test_pass_hat_k_over_evals(): def test_se_of_difference(): assert stats.se_of_difference(0.3, 0.4) == pytest.approx(0.5) + + +def test_normalized_gain(): + # Hake gain: same +0.05 absolute lift, but far more of the headroom closed + # on a high baseline -> higher normalized gain. + assert stats.normalized_gain(0.8, 0.6) == pytest.approx(0.5) # 0.2 / 0.4 + assert stats.normalized_gain(0.95, 0.9) == pytest.approx(0.5) # 0.05 / 0.1 + # a regression gives a negative gain + assert stats.normalized_gain(0.3, 0.5) == pytest.approx(-0.4) # -0.2 / 0.5 + # no headroom (baseline already maxed) -> undefined + assert stats.normalized_gain(0.5, 1.0) is None + + +def test_paired_diff(): + m, se, n = stats.paired_diff([(0.8, 0.5), (0.6, 0.4), (1.0, 0.9)]) + assert (m, n) == (pytest.approx(0.2), 3) # diffs 0.3, 0.2, 0.1 + assert se == pytest.approx(0.1 / math.sqrt(3)) # stddev 0.1 over sqrt(3) + # a single pair has no between-item variance -> se 0 + assert stats.paired_diff([(1.0, 0.0)]) == (pytest.approx(1.0), 0.0, 1)