feat(d2): normalized gain + paired-difference significance

skills/skill-validator/SKILL.md

@@ -121,10 +121,12 @@ Make a workspace dir, run these stages (each has a guide in `agents/`), then let
 - **Batch** — rank many scorecards with `scripts/batch.py:rank_scorecards([...])`.
 - **Eval-viewer** — `scripts/benchmark_export.py` emits a skill-creator-compatible
   `benchmark.json` so runs render in that project's viewer.
-- **SkillsBench-style paired evals** — when reporting D2 lift, preserve no-skill pass
-  rate, with-skill pass rate, absolute lift, and normalized gain; see
-  [`docs/references/skillsbench.md`](../../docs/references/skillsbench.md) for the
-  research note that motivates this future scorecard improvement.
+- **SkillsBench-style D2 lift** — `aggregate.py` reports absolute lift **and Hake
+  normalized gain** `g = (skill − baseline) / (1 − baseline)`, which corrects for
+  baseline headroom, and decides significance from a **paired** per-eval difference
+  (Miller 2024) — shared per-eval difficulty cancels, so a real lift clears the noise
+  with fewer trials (falls back to the unpaired SE for a single eval). See
+  [`docs/references/skillsbench.md`](../../docs/references/skillsbench.md).
 - **Calibration** — tune dimension weights against a labeled corpus with
   `scripts/calibrate.py:suggest_weights(examples)`.
 

skills/skill-validator/scripts/aggregate.py

@@ -2,12 +2,16 @@
 
 Reads runs via the shared ``runs_io`` loader and computes, per configuration: a
 pass-rate summary (mean/stddev/std_error), tau-bench ``pass^k`` over evals, the
-with-skill vs without-skill baseline lift, and whether that lift is significant.
+with-skill vs without-skill baseline lift, its Hake normalized gain (headroom-
+corrected), and whether that lift is significant.
 
-Significance is one-sided (did the skill help beyond noise) and is computed from
-the *unrounded* means/standard-errors so display rounding can't flip the verdict.
-(``compare.compare_scorecards`` uses a two-sided test for per-dimension change,
-which is a different question — a regression there is still a "significant" change.)
+Significance is one-sided (did the skill help beyond noise) and is computed at
+full precision so display rounding can't flip the verdict. It uses a *paired*
+per-eval difference (Miller 2024) when >=2 evals appear in both configs — shared
+per-eval difficulty cancels, lowering the SE — and falls back to the unpaired
+``se_of_difference`` for a single eval. (``compare.compare_scorecards`` uses a
+two-sided test for per-dimension change, a different question — a regression
+there is still a "significant" change.)
 """
 
 from __future__ import annotations
@@ -29,6 +33,15 @@ def _successes_per_eval(runs: list[dict], threshold: float) -> list[tuple[int, i
     return list(by_eval.values())
 
 
+def _mean_pass_by_eval(runs: list[dict]) -> dict:
+    sums: dict = {}
+    counts: dict = {}
+    for r in runs:
+        sums[r["eval_id"]] = sums.get(r["eval_id"], 0.0) + r["pass_rate"]
+        counts[r["eval_id"]] = counts.get(r["eval_id"], 0) + 1
+    return {e: sums[e] / counts[e] for e in sums}
+
+
 def aggregate(bench_dir, success_threshold: float = 1.0) -> dict:
     results = runs_io.load_runs(bench_dir)
 
@@ -46,20 +59,41 @@ def aggregate(bench_dir, success_threshold: float = 1.0) -> dict:
         }
 
     lift = se_diff = significant = None
+    norm_gain = paired_lift = paired_se = None
     if "with_skill" in results and "without_skill" in results:
         w = [r["pass_rate"] for r in results["with_skill"]]
         b = [r["pass_rate"] for r in results["without_skill"]]
         raw_lift = stats.mean(w) - stats.mean(b)
         raw_se = stats.se_of_difference(stats.std_error(w), stats.std_error(b))
         lift = round(raw_lift, 4)
         se_diff = round(raw_se, 4)
-        significant = bool(raw_lift > raw_se)  # decided at full precision
+
+        gain = stats.normalized_gain(stats.mean(w), stats.mean(b))
+        norm_gain = round(gain, 4) if gain is not None else None
+
+        # Paired inference (Miller 2024): pair the per-eval means so shared
+        # per-eval difficulty cancels. Needs >=2 evals present in both configs
+        # for a between-eval SE; otherwise fall back to the unpaired within-eval
+        # SE (a single eval has no between-eval variance to exploit).
+        ew = _mean_pass_by_eval(results["with_skill"])
+        eb = _mean_pass_by_eval(results["without_skill"])
+        shared = sorted(ew.keys() & eb.keys())
+        if len(shared) >= 2:
+            p_mean, p_se, _ = stats.paired_diff([(ew[e], eb[e]) for e in shared])
+            paired_lift = round(p_mean, 4)
+            paired_se = round(p_se, 4)
+            significant = bool(p_mean > p_se)  # decided at full precision
+        else:
+            significant = bool(raw_lift > raw_se)
 
     return {
         "configs": configs,
         "pass_hat_k": pass_hat,
         "baseline_lift": lift,
         "se_difference": se_diff,
+        "normalized_gain": norm_gain,
+        "paired_lift": paired_lift,
+        "paired_se": paired_se,
         "significant": significant,
         "success_threshold": success_threshold,
     }

skills/skill-validator/scripts/dimensions.py

@@ -62,6 +62,9 @@ def behavioral_dims(
         detail["D2"] = {
             "pass_rate": agg["configs"]["with_skill"]["pass_rate"],
             "baseline_lift": agg.get("baseline_lift"),
+            "normalized_gain": agg.get("normalized_gain"),
+            "paired_lift": agg.get("paired_lift"),
+            "paired_se": agg.get("paired_se"),
             "significant": agg.get("significant"),
         }
         if agg.get("pass_hat_k", {}).get("with_skill"):

skills/skill-validator/scripts/scorecard.py

@@ -144,6 +144,9 @@ def _dim_detail(dim: str, e: dict) -> str:
         lift = e.get("baseline_lift")
         if lift is not None:
             s += f", lift {lift * 100:+.0f}% ({'significant' if e.get('significant') else 'n.s.'})"
+        gain = e.get("normalized_gain")
+        if gain is not None:
+            s += f", gain {gain:+.2f}"
         return s
     if dim == "D3":
         ph = e.get("pass_hat_k", {})

skills/skill-validator/scripts/stats.py

@@ -1,9 +1,10 @@
 """Pure statistics helpers for skval scoring.
 
 No third-party dependencies. Covers the variance/reliability machinery the
-scorecard needs: sample mean/stddev/standard-error, a compact summary, and the
-tau-bench ``pass^k`` reliability metric plus the standard error of a difference
-(Miller 2024) used to decide whether a with-skill vs no-skill delta is real.
+scorecard needs: sample mean/stddev/standard-error, a compact summary, the
+tau-bench ``pass^k`` reliability metric, the standard error of a difference and
+the ``paired_diff`` (Miller 2024) used to decide whether a with-skill vs no-skill
+delta is real, and the Hake ``normalized_gain`` that corrects lift for headroom.
 """
 
 from __future__ import annotations
@@ -72,3 +73,29 @@ def pass_hat_k_over_evals(per_eval: Iterable[tuple[int, int]], k: int) -> float:
 def se_of_difference(se_a: float, se_b: float) -> float:
     """Standard error of a difference of two independent means: sqrt(se_a^2+se_b^2)."""
     return math.sqrt(se_a**2 + se_b**2)
+
+
+def normalized_gain(pass_skill: float, pass_baseline: float) -> float | None:
+    """Hake normalized gain g = (skill - baseline) / (1 - baseline), in (-inf, 1].
+
+    Corrects the raw lift for baseline headroom: a +0.05 lift closes half the
+    remaining gap on a 0.9 baseline (g=0.5) but only a twelfth on a 0.4 baseline
+    (g≈0.083). ``None`` when the baseline is already 1.0 (no headroom, undefined).
+    """
+    headroom = 1.0 - pass_baseline
+    if headroom <= 0:
+        return None
+    return (pass_skill - pass_baseline) / headroom
+
+
+def paired_diff(pairs: Iterable[tuple[float, float]]) -> tuple[float, float, int]:
+    """Paired mean difference, its standard error, and the item count.
+
+    ``pairs`` are (with_skill, without_skill) per matched item (eval). Inference
+    on the per-item differences rather than the two group means (Miller 2024)
+    cancels the shared per-item level, so when the two conditions are positively
+    correlated across items the SE is smaller than an unpaired ``se_of_difference``
+    — more power to detect a real lift. SE is 0 for a single pair (no spread).
+    """
+    diffs = [a - b for a, b in pairs]
+    return mean(diffs), std_error(diffs), len(diffs)

tests/test_aggregate.py

@@ -3,6 +3,7 @@
 import pytest
 
 import aggregate
+import stats
 
 
 def _grading(pass_rate, total=2):
@@ -64,3 +65,47 @@ def test_aggregate_no_baseline(tmp_path):
     agg = aggregate.aggregate(tmp_path)
     assert agg["baseline_lift"] is None
     assert agg["significant"] is None
+    assert agg["normalized_gain"] is None
+    assert agg["paired_lift"] is None
+
+
+def test_aggregate_normalized_gain(tmp_path):
+    # with-skill mean 0.8, baseline mean 0.6 -> gain (0.8-0.6)/(1-0.6) = 0.5
+    for r in (1, 2):
+        _mk_run(tmp_path, 0, "with_skill", r, _grading(0.8))
+        _mk_run(tmp_path, 0, "without_skill", r, _grading(0.6))
+    agg = aggregate.aggregate(tmp_path)
+    assert agg["baseline_lift"] == pytest.approx(0.2)
+    assert agg["normalized_gain"] == pytest.approx(0.5)
+
+
+def test_aggregate_paired_significance_beats_unpaired(tmp_path):
+    # Two evals of very different difficulty; the skill adds a steady ~0.19 on
+    # each. The between-eval difficulty swamps an unpaired SE (verdict n.s.),
+    # but cancels in the paired difference (Miller 2024) -> significant.
+    for r in (1, 2):
+        _mk_run(tmp_path, 0, "with_skill", r, _grading(0.4))  # hard eval
+        _mk_run(tmp_path, 0, "without_skill", r, _grading(0.2))
+        _mk_run(tmp_path, 1, "with_skill", r, _grading(0.9))  # easy eval
+        _mk_run(tmp_path, 1, "without_skill", r, _grading(0.72))
+    agg = aggregate.aggregate(tmp_path)
+    # unpaired SE of the difference would not clear the lift...
+    unpaired = stats.se_of_difference(
+        stats.std_error([0.4, 0.4, 0.9, 0.9]), stats.std_error([0.2, 0.2, 0.72, 0.72])
+    )
+    assert agg["baseline_lift"] <= unpaired  # unpaired alone would be n.s.
+    # ...but the paired inference exploits the correlation and clears it.
+    assert agg["paired_lift"] == pytest.approx(0.19)
+    assert agg["significant"] is True
+
+
+def test_aggregate_paired_none_with_single_eval(tmp_path):
+    # One eval -> no between-eval variance to exploit; paired fields stay None
+    # and significance falls back to the unpaired within-eval SE.
+    for r in (1, 2, 3):
+        _mk_run(tmp_path, 0, "with_skill", r, _grading(1.0))
+        _mk_run(tmp_path, 0, "without_skill", r, _grading(0.0))
+    agg = aggregate.aggregate(tmp_path)
+    assert agg["paired_lift"] is None
+    assert agg["paired_se"] is None
+    assert agg["significant"] is True  # unpaired fallback still fires

tests/test_scorecard.py

@@ -36,6 +36,32 @@ def test_build_and_render(tmp_path):
     assert Path(m).exists()
 
 
+def test_render_d2_shows_normalized_gain():
+    sc = scorecard.build_scorecard(
+        provenance={"source": "x", "kind": "dir"},
+        scoring_result={
+            "score": 80,
+            "grade": "B",
+            "verdict": "Ship",
+            "dims": {"D2": 0.8},
+            "weights_used": {"D2": 0.30},
+        },
+        d1_checks=[],
+        safety={"safety_pass": True, "findings": []},
+        dimensions_detail={
+            "D2": {
+                "pass_rate": {"mean": 0.8, "stddev": 0.1},
+                "baseline_lift": 0.2,
+                "normalized_gain": 0.5,
+                "significant": True,
+            }
+        },
+    )
+    md = scorecard.render_markdown(sc)
+    assert "gain +0.50" in md
+    assert "lift +20%" in md
+
+
 def test_render_shows_classification():
     sc = scorecard.build_scorecard(
         provenance={"source": "x", "kind": "dir"},

tests/test_stats.py

@@ -48,3 +48,22 @@ def test_pass_hat_k_over_evals():
 
 def test_se_of_difference():
     assert stats.se_of_difference(0.3, 0.4) == pytest.approx(0.5)
+
+
+def test_normalized_gain():
+    # Hake gain: same +0.05 absolute lift, but far more of the headroom closed
+    # on a high baseline -> higher normalized gain.
+    assert stats.normalized_gain(0.8, 0.6) == pytest.approx(0.5)  # 0.2 / 0.4
+    assert stats.normalized_gain(0.95, 0.9) == pytest.approx(0.5)  # 0.05 / 0.1
+    # a regression gives a negative gain
+    assert stats.normalized_gain(0.3, 0.5) == pytest.approx(-0.4)  # -0.2 / 0.5
+    # no headroom (baseline already maxed) -> undefined
+    assert stats.normalized_gain(0.5, 1.0) is None
+
+
+def test_paired_diff():
+    m, se, n = stats.paired_diff([(0.8, 0.5), (0.6, 0.4), (1.0, 0.9)])
+    assert (m, n) == (pytest.approx(0.2), 3)  # diffs 0.3, 0.2, 0.1
+    assert se == pytest.approx(0.1 / math.sqrt(3))  # stddev 0.1 over sqrt(3)
+    # a single pair has no between-item variance -> se 0
+    assert stats.paired_diff([(1.0, 0.0)]) == (pytest.approx(1.0), 0.0, 1)