From 98d6c48fa2d3034f9cd60be0dc25501c4439305d Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Thu, 9 Jul 2026 07:32:58 +0000 Subject: [PATCH 1/5] feat(crossover): report paired diffs for multiple arm contrasts Generalizes _paired to compute the within-seed paired adherence difference for a list of arm pairs, not just one, keyed by _vs_. build_dataset_multiseed and merge_seed_datasets now default to DEFAULT_PAIRS = (rac-vs-naive_rag, rac-vs-no_grounding) -- the pre-registered falsifier plus the parametric-memory floor -- while still accepting a single pair= for back-compat. A contrast whose arms aren't both in the sweep is silently skipped, so the default is safe for any arm subset. Each entry's across-seed diff_std is the noise term the forthcoming signal-to-noise report divides into. Results-neutral: no adherence number changes, only an additional paired series. --- decisiongrounding/runner/cli.py | 5 +- decisiongrounding/scoring/crossover.py | 87 +++++++++++++++++----- decisiongrounding/tests/test_real_curve.py | 15 ++++ 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/decisiongrounding/runner/cli.py b/decisiongrounding/runner/cli.py index e5261ad..5b0736b 100644 --- a/decisiongrounding/runner/cli.py +++ b/decisiongrounding/runner/cli.py @@ -46,6 +46,7 @@ from scenarios.loader import Scenario, load_pool, load_scenarios # noqa: E402 from scoring import aggregate, score # noqa: E402 from scoring.crossover import ( # noqa: E402 + DEFAULT_PAIRS, build_dataset, build_dataset_batched, build_dataset_multiseed, @@ -727,7 +728,7 @@ def _on_cell(rec: dict) -> None: pool=pool, pool_dir=pool_dir, scenarios_dir=args.scenarios, batched=batch, poll=args.poll, progress=_on_cell) dataset = merge_seed_datasets(existing, new_ps, list(arms), list(ns), - ("rac", "naive_rag")) + list(DEFAULT_PAIRS)) elif seeds is not None: if len(seeds) > 1: print(f" (multi-seed: {seeds}; reporting mean +/- 95% CI)", file=sys.stderr) @@ -735,7 +736,7 @@ def _on_cell(rec: dict) -> None: scenarios, arms=arms, ns=ns, seeds=seeds, answering_model_name=args.answering, embedder_spec=args.embedder, pool=pool, pool_dir=pool_dir, scenarios_dir=args.scenarios, - batched=batch, poll=args.poll, pair=("rac", "naive_rag"), progress=_on_cell, + batched=batch, poll=args.poll, pairs=list(DEFAULT_PAIRS), progress=_on_cell, resume=resume_cells) elif batch: dataset = build_dataset_batched( diff --git a/decisiongrounding/scoring/crossover.py b/decisiongrounding/scoring/crossover.py index e27db5e..3b7215e 100644 --- a/decisiongrounding/scoring/crossover.py +++ b/decisiongrounding/scoring/crossover.py @@ -617,26 +617,33 @@ def build_dataset_multiseed( scenarios_dir: str | None = None, batched: bool = False, poll: int = 20, - pair: tuple[str, str] = ("rac", "naive_rag"), + pair: "tuple[str, str] | None" = None, + pairs: "list[tuple[str, str]] | None" = None, progress: "Callable[[dict], None] | None" = None, resume: "dict[tuple[int, int, str, str], dict] | None" = None, ) -> dict: """Run the crossover over several seeds and aggregate per (arm, N) into mean +/- a t-based 95% CI. The plain fields stay the mean (backward - compatible); `_ci` / `_std` / `_values`, `n_seeds`, `seeds`, and a - paired `pair[0]`-vs-`pair[1]` adherence difference (`paired`) are added. + compatible); `_ci` / `_std` / `_values`, `n_seeds`, `seeds`, and + paired adherence differences (`paired`) are added. + + `pairs` selects the arm contrasts for the paired diffs (default + `DEFAULT_PAIRS`: rac-vs-naive_rag and rac-vs-no_grounding); a single + `pair=(a, b)` tuple is still accepted for back-compat. A contrast whose + arms aren't both in the sweep is skipped. Calls the single-seed builders per seed, so batched + multiseed compose. Offline runs (deterministic stub + embedder) show little spread; the aggregation is exercised regardless. """ + resolved = pairs if pairs is not None else ([pair] if pair is not None else DEFAULT_PAIRS) uniq = list(dict.fromkeys(int(s) for s in seeds)) or [0] per_seed = run_seeds( scenarios, arms, ns, uniq, answering_model_name=answering_model_name, embedder_spec=embedder_spec, pool=pool, pool_dir=pool_dir, scenarios_dir=scenarios_dir, batched=batched, poll=poll, progress=progress, resume=resume) - return _aggregate_seeds(per_seed, list(arms), list(ns), pair) + return _aggregate_seeds(per_seed, list(arms), list(ns), resolved) def run_seeds( @@ -710,10 +717,28 @@ def _aggregate_arm_points(arms, ns, cols, n_seeds) -> dict: return points -def _paired(per_seed, ns, pair) -> dict | None: - """Per-N paired adherence difference pair[0]-pair[1], differenced within each - seed (common random numbers), with its own CI.""" - a, b = pair +# The arm contrasts whose paired adherence difference (and thus signal-to-noise) +# the multi-seed crossover reports by default: the pre-registered falsifier +# (rac vs naive_rag, H1) and the parametric-memory floor (rac vs no_grounding, +# H2). A pair whose arms aren't both in the sweep is silently skipped, so this +# default is safe for any arm subset (offline runs without rac emit neither). +DEFAULT_PAIRS = (("rac", "naive_rag"), ("rac", "no_grounding")) + + +def _normalize_pairs(pairs) -> list[tuple[str, str]]: + """Accept a single ``(a, b)`` tuple or a list of them; return a list of + tuples. ``None``/empty -> ``[]``.""" + if not pairs: + return [] + if isinstance(pairs[0], str): # a bare (a, b) tuple + return [(pairs[0], pairs[1])] + return [(p[0], p[1]) for p in pairs] + + +def _paired_one(per_seed, ns, a, b) -> list | None: + """Per-N paired adherence difference a-b, differenced within each seed + (common random numbers), with its own CI. None when either arm is absent + from any seed's sweep.""" if not per_seed or not all(a in ds["arms"] and b in ds["arms"] for _, ds in per_seed): return None out = [] @@ -727,7 +752,23 @@ def _paired(per_seed, ns, pair) -> dict | None: s = summarize(diffs) out.append({"N": n, "diff_mean": s["mean"], "diff_ci": s["ci"], "diff_std": s["std"], "n": s["n"], "values": s["values"]}) - return {f"{a}_vs_{b}": out} if out else None + return out or None + + +def _paired(per_seed, ns, pairs) -> dict | None: + """Per-N within-seed paired adherence differences for one or more arm pairs. + + `pairs` accepts a single ``(a, b)`` tuple (back-compat) or a list of them. + Each pair whose both arms appear in every seed's sweep contributes a + ``"{a}_vs_{b}"`` entry; pairs with a missing arm are skipped. The + across-seed std of each entry (`diff_std`) is the noise term the + signal-to-noise report divides into (see `scoring.snr`).""" + result = {} + for a, b in _normalize_pairs(pairs): + series = _paired_one(per_seed, ns, a, b) + if series: + result[f"{a}_vs_{b}"] = series + return result or None def _agg_per_scenario(per_seed, arms) -> dict: @@ -755,7 +796,7 @@ def _agg_per_scenario(per_seed, arms) -> dict: return out -def _aggregate_seeds(per_seed, arms, ns, pair) -> dict: +def _aggregate_seeds(per_seed, arms, ns, pairs) -> dict: base = dict(per_seed[0][1]) cols = _columns_from_datasets(per_seed, arms, ns) base["arms"] = _aggregate_arm_points(arms, ns, cols, len(per_seed)) @@ -769,7 +810,7 @@ def _aggregate_seeds(per_seed, arms, ns, pair) -> dict: cells = [c for _, ds in per_seed for c in (ds.get("cells") or [])] base["cells"] = cells base["stats"] = stats_by_n(cells) if cells else None - paired = _paired(per_seed, ns, pair) + paired = _paired(per_seed, ns, pairs) if paired: base["paired"] = paired return base @@ -808,13 +849,19 @@ def merge_seed_datasets(existing: dict, new_per_seed, arms, ns, pair) -> dict: base = dict(existing) base["arms"] = _aggregate_arm_points(arms, ns, cols, len(all_seeds)) - # Paired: prior per-seed diffs + the new seeds' diffs. - a, b = pair - old_paired = (existing.get("paired") or {}).get(f"{a}_vs_{b}") - new_paired = (_paired(add, ns, pair) or {}).get(f"{a}_vs_{b}") - if old_paired is not None or new_paired is not None: - old_by_n = {e["N"]: e for e in (old_paired or [])} - new_by_n = {e["N"]: e for e in (new_paired or [])} + # Paired: prior per-seed diffs + the new seeds' diffs, for every contrast — + # the requested pairs plus any already present in `existing` (so an + # under-specified merge never silently drops a contrast). Arm names contain + # underscores, but "_vs_" is a safe delimiter for the existing keys. + requested = _normalize_pairs(pair) + keys = {f"{a}_vs_{b}": (a, b) for a, b in requested} + for k in (existing.get("paired") or {}): + keys.setdefault(k, tuple(k.split("_vs_"))) + new_paired = _paired(add, ns, list(keys.values())) or {} + merged_paired = {} + for key in keys: + old_by_n = {e["N"]: e for e in (existing.get("paired") or {}).get(key, [])} + new_by_n = {e["N"]: e for e in new_paired.get(key, [])} merged = [] for n in ns: vals = list(old_by_n.get(n, {}).get("values", [])) + list(new_by_n.get(n, {}).get("values", [])) @@ -823,7 +870,9 @@ def merge_seed_datasets(existing: dict, new_per_seed, arms, ns, pair) -> dict: merged.append({"N": n, "diff_mean": s["mean"], "diff_ci": s["ci"], "diff_std": s["std"], "n": s["n"], "values": s["values"]}) if merged: - base["paired"] = {f"{a}_vs_{b}": merged} + merged_paired[key] = merged + if merged_paired: + base["paired"] = merged_paired base["per_scenario"] = _merge_per_scenario(existing.get("per_scenario", {}), add, arms, n_old) base["errors"] = list(existing.get("errors", [])) + [e for _, ds in add for e in ds["errors"]] diff --git a/decisiongrounding/tests/test_real_curve.py b/decisiongrounding/tests/test_real_curve.py index 6acf556..67cf524 100644 --- a/decisiongrounding/tests/test_real_curve.py +++ b/decisiongrounding/tests/test_real_curve.py @@ -146,6 +146,21 @@ def test_build_dataset_multiseed_paired_difference(): assert len(e["diff_ci"]) == 2 and e["n"] == 2 and len(e["values"]) == 2 +def test_build_dataset_multiseed_reports_both_contrasts(): + # A three-arm sweep yields a paired diff for every requested contrast whose + # arms are present; a contrast with a missing arm is silently skipped. + sc = load_scenarios(_REAL) + ds = build_dataset_multiseed( + sc, arms=("context_dump", "naive_rag", "no_grounding"), ns=(3, 6), seeds=[0, 1], + pool=_pool(30), + pairs=[("context_dump", "naive_rag"), ("context_dump", "no_grounding"), + ("rac", "naive_rag")], # rac absent -> skipped + ) + assert set(ds["paired"]) == {"context_dump_vs_naive_rag", "context_dump_vs_no_grounding"} + for series in ds["paired"].values(): + assert [e["N"] for e in series] == [3, 6] + + def test_merge_seed_datasets_augment_equals_fresh_run(): sc = load_scenarios(_REAL) arms, ns = ("context_dump", "naive_rag"), (3, 6) From 0446de5733a979cb6a842297db30fbc30c7a895b Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Thu, 9 Jul 2026 07:34:38 +0000 Subject: [PATCH 2/5] feat(scoring): signal_to_noise() over the paired contrasts New scoring/snr.py computes SNR = |paired diff_mean| / across-seed diff_std per arm contrast and per N, with the article's core caveat baked in: noise cannot be estimated from one seed, so n_seeds<2 reports 'noise_not_estimable' rather than a fabricated ratio. Distinguishes noise-dominated (snr<1), clean separation (zero noise, nonzero signal), and no-effect (zero noise, zero signal). Headline SNR is taken at the largest N (where the thesis is adjudicated). Pure and results-neutral -- reads only the existing paired block. --- decisiongrounding/scoring/__init__.py | 2 + decisiongrounding/scoring/snr.py | 79 +++++++++++++++++++++++++++ decisiongrounding/tests/test_snr.py | 60 ++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 decisiongrounding/scoring/snr.py create mode 100644 decisiongrounding/tests/test_snr.py diff --git a/decisiongrounding/scoring/__init__.py b/decisiongrounding/scoring/__init__.py index 8ce2072..84046d0 100644 --- a/decisiongrounding/scoring/__init__.py +++ b/decisiongrounding/scoring/__init__.py @@ -10,10 +10,12 @@ t_critical_95, ) from .scorer import Score, score +from .snr import signal_to_noise __all__ = [ "Score", "score", + "signal_to_noise", "ArmMetrics", "aggregate", "adherence_variance", diff --git a/decisiongrounding/scoring/snr.py b/decisiongrounding/scoring/snr.py new file mode 100644 index 0000000..6fcdabc --- /dev/null +++ b/decisiongrounding/scoring/snr.py @@ -0,0 +1,79 @@ +"""Signal-to-noise for the crossover. + +Ai2's "Signal and Noise" framework (and OpenAI's coding-eval analysis) make the +same point: a benchmark result is only trustworthy when the effect it measures +is large relative to the run-to-run noise. Here the **signal** is the within-seed +paired adherence difference between two arms (`diff_mean` in +`dataset["paired"]`), and the **noise** is that difference's spread across seeds +(`diff_std`, under common random numbers). Their ratio is the signal-to-noise: + + SNR = |diff_mean| / diff_std + +An SNR below 1 means the between-arm gap is smaller than the seed-to-seed +wobble — noise-dominated, not a result to lean on. Crucially, **noise cannot be +estimated from a single seed**: with `n_seeds < 2` the SNR is reported as "not +estimable", never as a fabricated number. + +Pure and results-neutral: reads only the already-computed `paired` block, adds +nothing to the dataset. +""" + +from __future__ import annotations + + +def _snr_at(diff_mean: float, diff_std: "float | None", n_seeds: int) -> dict: + """Classify one (pair, N) cell into an SNR record.""" + signal = abs(diff_mean) + if n_seeds < 2 or diff_std is None: + # No repeated runs -> noise is unmeasured. The article's core caveat. + return {"signal": signal, "noise": diff_std, "snr": None, + "noise_dominated": False, "clean_separation": False, + "flag": "noise_not_estimable"} + if diff_std == 0: + if signal > 0: + # Every seed gave the same nonzero gap: maximally clean signal. + return {"signal": signal, "noise": 0.0, "snr": None, + "noise_dominated": False, "clean_separation": True, + "flag": "zero_noise_clean_separation"} + # Every seed a perfect tie: no effect and no variance. + return {"signal": 0.0, "noise": 0.0, "snr": None, + "noise_dominated": False, "clean_separation": False, + "flag": "no_effect"} + snr = signal / diff_std + return {"signal": signal, "noise": diff_std, "snr": snr, + "noise_dominated": snr < 1.0, "clean_separation": False, "flag": None} + + +def signal_to_noise(dataset: dict) -> dict: + """Signal-to-noise per arm contrast and per N, from `dataset["paired"]`. + + Returns:: + + {"n_seeds": int, + "pairs": {"_vs_": {"by_n": [{"N", "signal", "noise", "snr", + "noise_dominated", "clean_separation", + "flag"}, ...], + "headline_N": int|None, + "headline": |None}, + ...}} + + `.get`-guarded: a dataset without `paired` (single-arm, legacy, or + single-contrast) yields an empty `pairs`. The headline is taken at the + largest N — the buried-in-distractors corpus size where the thesis is + actually adjudicated. + """ + paired = dataset.get("paired") or {} + n_seeds = int(dataset.get("n_seeds", 1) or 1) + pairs: dict[str, dict] = {} + for key, series in paired.items(): + by_n = [] + for e in series: + rec = {"N": e["N"], **_snr_at(e["diff_mean"], e.get("diff_std"), n_seeds)} + by_n.append(rec) + headline = max(by_n, key=lambda r: r["N"]) if by_n else None + pairs[key] = { + "by_n": by_n, + "headline_N": headline["N"] if headline else None, + "headline": headline, + } + return {"n_seeds": n_seeds, "pairs": pairs} diff --git a/decisiongrounding/tests/test_snr.py b/decisiongrounding/tests/test_snr.py new file mode 100644 index 0000000..d9e7d46 --- /dev/null +++ b/decisiongrounding/tests/test_snr.py @@ -0,0 +1,60 @@ +"""signal_to_noise: SNR = |paired diff_mean| / across-seed diff_std, honestly +gated on >=2 seeds.""" + +from scoring.snr import signal_to_noise + + +def _ds(n_seeds, series, key="rac_vs_naive_rag"): + return {"n_seeds": n_seeds, "paired": {key: series}} + + +def _e(N, diff_mean, diff_std): + return {"N": N, "diff_mean": diff_mean, "diff_std": diff_std, + "diff_ci": [0, 0], "n": 3, "values": []} + + +def test_snr_is_signal_over_noise(): + ds = _ds(3, [_e(300, 0.40, 0.10)]) + rec = signal_to_noise(ds)["pairs"]["rac_vs_naive_rag"]["by_n"][0] + assert rec["signal"] == 0.40 and rec["noise"] == 0.10 + assert rec["snr"] == 4.0 and rec["noise_dominated"] is False and rec["flag"] is None + + +def test_noise_dominated_when_snr_below_one(): + rec = signal_to_noise(_ds(3, [_e(300, 0.05, 0.20)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0] + assert rec["snr"] == 0.25 and rec["noise_dominated"] is True + + +def test_single_seed_is_not_estimable(): + rec = signal_to_noise(_ds(1, [_e(300, 0.40, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0] + assert rec["snr"] is None and rec["flag"] == "noise_not_estimable" + assert rec["noise_dominated"] is False + + +def test_zero_noise_nonzero_signal_is_clean_separation(): + rec = signal_to_noise(_ds(4, [_e(300, 0.30, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0] + assert rec["snr"] is None and rec["clean_separation"] is True + assert rec["flag"] == "zero_noise_clean_separation" + + +def test_zero_noise_zero_signal_is_no_effect(): + rec = signal_to_noise(_ds(4, [_e(300, 0.0, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0] + assert rec["snr"] is None and rec["flag"] == "no_effect" + + +def test_headline_taken_at_largest_n(): + ds = _ds(3, [_e(10, 0.02, 0.2), _e(300, 0.5, 0.1)]) + pair = signal_to_noise(ds)["pairs"]["rac_vs_naive_rag"] + assert pair["headline_N"] == 300 and pair["headline"]["snr"] == 5.0 + + +def test_multiple_contrasts_and_missing_paired(): + ds = {"n_seeds": 3, "paired": { + "rac_vs_naive_rag": [_e(300, 0.4, 0.1)], + "rac_vs_no_grounding": [_e(300, 0.9, 0.05)], + }} + out = signal_to_noise(ds) + assert set(out["pairs"]) == {"rac_vs_naive_rag", "rac_vs_no_grounding"} + assert out["pairs"]["rac_vs_no_grounding"]["headline"]["snr"] == 18.0 + # legacy dataset with no paired block -> empty, not an error + assert signal_to_noise({"n_seeds": 1})["pairs"] == {} From 60cf8bdad39fa052c466827e1992d651c726fdd9 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Thu, 9 Jul 2026 07:37:02 +0000 Subject: [PATCH 3/5] feat(scoring): scenario_health() discrimination/validity audit New scoring/health.py classifies every crossover scenario as broken (ceiling arm context_dump never adheres -> unsolvable/mis-specified), contaminated (floor arm no_grounding adheres -> parametric memory suffices, doesn't test grounding), tie (no between-arm separation), or discriminating -- the analogue of OpenAI's 'X% of tasks are broken' ceiling analysis. unknown when neither control arm is in the sweep. Reports per-scenario rows + summary counts. Pure and results-neutral; reads only per_scenario. --- decisiongrounding/scoring/__init__.py | 2 + decisiongrounding/scoring/health.py | 106 ++++++++++++++++++ .../tests/test_scenario_health.py | 85 ++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 decisiongrounding/scoring/health.py create mode 100644 decisiongrounding/tests/test_scenario_health.py diff --git a/decisiongrounding/scoring/__init__.py b/decisiongrounding/scoring/__init__.py index 84046d0..4ae605d 100644 --- a/decisiongrounding/scoring/__init__.py +++ b/decisiongrounding/scoring/__init__.py @@ -9,6 +9,7 @@ summarize, t_critical_95, ) +from .health import scenario_health from .scorer import Score, score from .snr import signal_to_noise @@ -16,6 +17,7 @@ "Score", "score", "signal_to_noise", + "scenario_health", "ArmMetrics", "aggregate", "adherence_variance", diff --git a/decisiongrounding/scoring/health.py b/decisiongrounding/scoring/health.py new file mode 100644 index 0000000..d0f4b79 --- /dev/null +++ b/decisiongrounding/scoring/health.py @@ -0,0 +1,106 @@ +"""Per-scenario discrimination / validity audit. + +OpenAI's coding-eval analysis found ~20-30% of a popular benchmark's tasks were +broken — unsolvable, contaminated, or non-discriminating — silently capping the +ceiling. This audit surfaces the analogue for the crossover: for each scenario, +is it actually pulling its weight, or is it dead signal? + +Four classes, referencing the two controls already in the sweep — the ceiling +arm (`context_dump`, sees the whole corpus) and the floor arm (`no_grounding`, +parametric memory only): + +- **broken** — even the ceiling arm never adheres (likely mis-specified or + unsolvable from the corpus). +- **contaminated** — the floor arm adheres (the model answers from pretraining + memory, so the scenario doesn't test grounding — a real risk for public + PEPs/RFCs; synthetic scenarios are contamination-proof by construction). +- **tie** — no arm separates from any other at any N (degenerate, no signal). +- **discriminating** — otherwise (grounding separates from the floor). + +`unknown` when neither control arm is in the sweep. Pure and results-neutral: +reads only `dataset["per_scenario"]`. +""" + +from __future__ import annotations + +CEILING_ARM = "context_dump" +FLOOR_ARM = "no_grounding" +_ADHERE = 0.5 # majority; robust to multi-seed fractions and single-seed bools + + +def _adherence_by_n(per_scenario: dict, arm: str, sid: str) -> dict: + """{N: adherence} for one (arm, scenario); adherent may be a bool + (single-seed) or a fraction (multi-seed) — both coerce to float.""" + out = {} + for rec in per_scenario.get(arm, {}).get(sid, []): + v = rec.get("adherent") + if v is not None: + out[rec["N"]] = float(v) + return out + + +def _adheres_somewhere(by_n: dict) -> bool: + return any(v >= _ADHERE for v in by_n.values()) + + +def scenario_health(dataset: dict) -> dict: + """Classify every scenario in `dataset["per_scenario"]`. + + Returns:: + + {"scenarios": [{"scenario_id", "class", "ceiling_adherent", + "floor_adherent", "separates", "max_gap"}, ...], + "counts": {"discriminating", "broken", "contaminated", "tie", + "unknown"}, + "total": int, + "controls": {"ceiling": bool, "floor": bool}} + + `.get`-guarded: a dataset without `per_scenario` yields an empty audit. + """ + per = dataset.get("per_scenario") or {} + arms = list(per) + has_ceiling = CEILING_ARM in arms + has_floor = FLOOR_ARM in arms + sids = sorted({sid for arm in arms for sid in per.get(arm, {})}) + + scenarios = [] + counts = {"discriminating": 0, "broken": 0, "contaminated": 0, "tie": 0, "unknown": 0} + for sid in sids: + by_arm = {arm: _adherence_by_n(per, arm, sid) for arm in arms} + ns = sorted({n for b in by_arm.values() for n in b}) + + # Between-arm separation: the largest adherence gap at any shared N. + max_gap = 0.0 + for n in ns: + vals = [b[n] for b in by_arm.values() if n in b] + if len(vals) >= 2: + max_gap = max(max_gap, max(vals) - min(vals)) + separates = max_gap > 1e-9 + + ceiling_adherent = _adheres_somewhere(by_arm[CEILING_ARM]) if has_ceiling else None + floor_adherent = _adheres_somewhere(by_arm[FLOOR_ARM]) if has_floor else None + + if not (has_ceiling or has_floor): + cls = "unknown" + elif has_ceiling and not ceiling_adherent: + cls = "broken" + elif has_floor and floor_adherent: + cls = "contaminated" + elif not separates: + cls = "tie" + else: + cls = "discriminating" + + counts[cls] += 1 + scenarios.append({ + "scenario_id": sid, "class": cls, + "ceiling_adherent": ceiling_adherent, "floor_adherent": floor_adherent, + "separates": separates, "max_gap": max_gap, + }) + + return { + "scenarios": scenarios, + "counts": counts, + "total": len(sids), + "controls": {"ceiling": has_ceiling, "floor": has_floor}, + } diff --git a/decisiongrounding/tests/test_scenario_health.py b/decisiongrounding/tests/test_scenario_health.py new file mode 100644 index 0000000..f5549fe --- /dev/null +++ b/decisiongrounding/tests/test_scenario_health.py @@ -0,0 +1,85 @@ +"""scenario_health: per-scenario discrimination/validity classification.""" + +from scoring.health import scenario_health + + +def _rec(n, adherent): + return {"N": n, "adherent": adherent} + + +def _dataset(per_scenario): + return {"per_scenario": per_scenario} + + +def test_discriminating_scenario(): + # ceiling adheres, floor fails, arms separate. + ds = _dataset({ + "context_dump": {"s": [_rec(10, 1.0), _rec(300, 1.0)]}, + "naive_rag": {"s": [_rec(10, 1.0), _rec(300, 0.0)]}, + "no_grounding": {"s": [_rec(10, 0.0), _rec(300, 0.0)]}, + }) + h = scenario_health(ds) + assert h["counts"]["discriminating"] == 1 + row = h["scenarios"][0] + assert row["class"] == "discriminating" and row["separates"] is True + assert row["max_gap"] == 1.0 + + +def test_broken_scenario_ceiling_never_adheres(): + ds = _dataset({ + "context_dump": {"s": [_rec(10, 0.0), _rec(300, 0.0)]}, + "no_grounding": {"s": [_rec(10, 0.0), _rec(300, 0.0)]}, + }) + h = scenario_health(ds) + assert h["scenarios"][0]["class"] == "broken" + assert h["counts"]["broken"] == 1 + + +def test_contaminated_scenario_floor_adheres(): + # ceiling adheres AND floor also adheres -> parametric memory suffices. + ds = _dataset({ + "context_dump": {"s": [_rec(10, 1.0)]}, + "no_grounding": {"s": [_rec(10, 1.0)]}, + }) + h = scenario_health(ds) + assert h["scenarios"][0]["class"] == "contaminated" + + +def test_tie_scenario_no_separation(): + # Ceiling adheres, no floor arm to separate from, and the present arms + # never differ -> no between-arm signal. + ds = _dataset({ + "context_dump": {"s": [_rec(10, 1.0), _rec(300, 1.0)]}, + "naive_rag": {"s": [_rec(10, 1.0), _rec(300, 1.0)]}, + }) + h = scenario_health(ds) + assert h["scenarios"][0]["class"] == "tie" + assert h["scenarios"][0]["separates"] is False + + +def test_unknown_without_control_arms(): + ds = _dataset({ + "rac": {"s": [_rec(10, 1.0)]}, + "naive_rag": {"s": [_rec(10, 0.0)]}, + }) + h = scenario_health(ds) + assert h["scenarios"][0]["class"] == "unknown" + assert h["controls"] == {"ceiling": False, "floor": False} + + +def test_multi_seed_fractions_and_summary_counts(): + ds = _dataset({ + "context_dump": {"good": [_rec(300, 1.0)], "bad": [_rec(300, 0.0)]}, + "naive_rag": {"good": [_rec(300, 0.4)], "bad": [_rec(300, 0.0)]}, + "no_grounding": {"good": [_rec(300, 0.0)], "bad": [_rec(300, 0.0)]}, + }) + h = scenario_health(ds) + assert h["total"] == 2 + classes = {s["scenario_id"]: s["class"] for s in h["scenarios"]} + assert classes["good"] == "discriminating" # ceiling 1.0, floor 0, gap 0.6 + assert classes["bad"] == "broken" # ceiling never adheres + + +def test_legacy_dataset_without_per_scenario(): + h = scenario_health({}) + assert h["total"] == 0 and h["scenarios"] == [] From 966127f30a399f964f9fa7240fe5e61f47c20657 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Thu, 9 Jul 2026 07:45:12 +0000 Subject: [PATCH 4/5] feat(report): surface signal-to-noise + scenario-health in report and dashboard report.py gains a Signal-to-noise section (SNR per contrast per N, with noise-dominated cells flagged and a clear 'noise not estimable' notice for single-seed runs) and a Scenario-health section (the discrimination audit's summary counts + a table of broken/contaminated/tie scenarios), inserted after the per-scenario attribution. dashboard.py gains a matching 'Signal / noise' tab (s6); Reproduce->s7, Run->s8, with a new #t8/#s8 rule in the template CSS and _TABS kept in lockstep (guarded by a new test). All .get-guarded so legacy/single-seed/no-rac datasets render gracefully. --- decisiongrounding/runner/dashboard.py | 74 +++++++++++- .../runner/templates/dashboard.html | 3 +- decisiongrounding/scripts/report.py | 111 ++++++++++++++++++ decisiongrounding/tests/test_dashboard.py | 23 +++- decisiongrounding/tests/test_report_snr.py | 60 ++++++++++ 5 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 decisiongrounding/tests/test_report_snr.py diff --git a/decisiongrounding/runner/dashboard.py b/decisiongrounding/runner/dashboard.py index bbbc014..dca26f8 100644 --- a/decisiongrounding/runner/dashboard.py +++ b/decisiongrounding/runner/dashboard.py @@ -19,6 +19,8 @@ from scoring.charts import grouped_bar_chart, line_chart from scoring.cost import cost_by_arm, dollars +from scoring.health import scenario_health +from scoring.snr import signal_to_noise _DEFAULT_TEMPLATE = Path(__file__).resolve().parent / "templates" / "dashboard.html" @@ -267,6 +269,65 @@ def _scenarios_section(run): return matrix + "".join(drill) +def _snr_txt(rec: dict) -> str: + """Plain-text SNR cell (number, or why it isn't one).""" + snr = rec.get("snr") + if snr is not None: + return f"{snr:.2f}" + (" (noise-dominated)" if rec.get("noise_dominated") else "") + return {"zero_noise_clean_separation": "clean (σ=0)", + "no_effect": "0 (tie)"}.get(rec.get("flag"), "n/a") + + +def _signal_noise_section(dataset) -> str: + """The Signal / noise tab: SNR per contrast + the scenario-health audit.""" + snr = signal_to_noise(dataset) + health = scenario_health(dataset) + if not snr["pairs"] and not health["total"]: + return '

Needs a multi-seed crossover dataset.

' + out = [] + if snr["pairs"]: + out.append("

Signal-to-noise

") + if snr["n_seeds"] < 2: + out.append('

Only 1 seed — seed-to-seed noise is not ' + 'estimable, so SNR cannot be computed. Re-run with ' + '--seeds 0-4.

') + else: + out.append('

SNR = |between-arm adherence gap| ÷ ' + 'across-seed σ of that gap. SNR < 1 is ' + 'noise-dominated.

') + for key, pair in snr["pairs"].items(): + a, b = key.split("_vs_") + head = pair.get("headline") or {} + out.append(f'

{_esc(a)} vs {_esc(b)} — headline SNR ' + f'at N={pair.get("headline_N")}: {_esc(_snr_txt(head))}

') + rows = "".join( + f"{r['N']}{_f(r.get('signal'))}" + f"{_f(r.get('noise'))}{_esc(_snr_txt(r))}" + for r in pair["by_n"]) + out.append("" + f"{rows}" + "
Nsignalnoise σSNR
") + if health["total"]: + c = health["counts"] + out.append("

Scenario health

") + extra = f", {c['unknown']} unknown" if c["unknown"] else "" + out.append(f'

Of {health["total"]} scenarios: ' + f'{c["discriminating"]} discriminating, {c["broken"]} broken, ' + f'{c["contaminated"]} contaminated, {c["tie"]} tie{extra}.

') + flagged = [s for s in health["scenarios"] if s["class"] != "discriminating"] + if flagged: + rows = "".join( + f"{_esc(s['scenario_id'])}" + f"{_esc(s['class'])}{_f(s['max_gap'])}" + for s in flagged) + out.append("" + f"{rows}
scenarioclassmax arm gap
") + out.append('

broken = ceiling arm never adheres; ' + 'contaminated = parametric-memory floor already adheres; ' + 'tie = no arm separates.

') + return "".join(out) + + def _run_tab(paid: bool) -> str: if paid: real = ( @@ -287,7 +348,7 @@ def _run_tab(paid: bool) -> str: "DG_UI_ALLOW_PAID=1 (and ANTHROPIC_API_KEY) to enable; " "you'll still have to estimate and tick a confirmation before any spend.

") return ( - "

Run the benchmark

" + "

Run the benchmark

" "

Triggers the CLI on the server; the page refreshes when the run finishes.

" " " " " @@ -407,8 +468,10 @@ def render_main(run, dataset, cost_curve=None, *, live=False, paid_enabled=False secs.append('

Needs both rac and naive_rag in a crossover dataset.

') # 5 Scenarios secs.append(f'

Scenarios & failures

{_scenarios_section(run)}
') - # 6 Reproduce - secs.append('

Reproduce

'
+    # 6 Signal / noise (SNR + scenario-health audit)
+    secs.append(f'
{_signal_noise_section(dataset) if dataset else "
Needs a crossover dataset.
"}
')
+    # 7 Reproduce
+    secs.append('
Reproduce
'
                 'make real-crossover   # headline + adherence-vs-N over the real pool\n'
                 'make real-batch       # via the Batch API (~50% cost)\n\n'
                 'make ui               # serve this dashboard live at 127.0.0.1:8099\n'
@@ -424,9 +487,10 @@ def render_main(run, dataset, cost_curve=None, *, live=False, paid_enabled=False
     return "".join(secs)
 
 
-# Tab labels, in section order (s0..s6, +s7 Run when live). Kept beside the
+# Tab labels, in section order (s0..s7, +s8 Run when live). Kept beside the
 # renderer so the nav and the sections stay in lockstep.
-_TABS = ["Overview", "Leaderboard", "Curves", "Cost", "rac vs RAG", "Scenarios", "Reproduce"]
+_TABS = ["Overview", "Leaderboard", "Curves", "Cost", "rac vs RAG", "Scenarios",
+         "Signal / noise", "Reproduce"]
 
 
 def build_dashboard(run, dataset, cost_curve=None, *, live=False, paid_enabled=False, template=None):
diff --git a/decisiongrounding/runner/templates/dashboard.html b/decisiongrounding/runner/templates/dashboard.html
index 0251096..badf509 100644
--- a/decisiongrounding/runner/templates/dashboard.html
+++ b/decisiongrounding/runner/templates/dashboard.html
@@ -8,7 +8,7 @@
     [CHIPS]  - the config chips (answering model, embedder, scenarios)
     [BODY]   - the tab radios + nav + main, with the data-bound sections
   Point the server / static generator at a custom copy with --template or the
-  DG_UI_TEMPLATE env var. Tabs are pure CSS (#t0..#t7); the JS only activates
+  DG_UI_TEMPLATE env var. Tabs are pure CSS (#t0..#t8); the JS only activates
   when a live Run tab (#run-status) is present, so static snapshots stay inert.
 -->
 
@@ -43,6 +43,7 @@
 #t5:checked~nav label[for=t5]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t5:checked~main #s5{display:block}
 #t6:checked~nav label[for=t6]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t6:checked~main #s6{display:block}
 #t7:checked~nav label[for=t7]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t7:checked~main #s7{display:block}
+#t8:checked~nav label[for=t8]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t8:checked~main #s8{display:block}
 table{border-collapse:collapse;width:100%;margin:8px 0 18px;font-size:14px}
 th,td{border:1px solid var(--line);padding:7px 10px;text-align:right}
 th:first-child,td:first-child{text-align:left}
diff --git a/decisiongrounding/scripts/report.py b/decisiongrounding/scripts/report.py
index 8e3559a..807c970 100644
--- a/decisiongrounding/scripts/report.py
+++ b/decisiongrounding/scripts/report.py
@@ -29,6 +29,8 @@
 
 from scoring.charts import grouped_bar_chart, line_chart  # noqa: E402
 from scoring.cost import cost_by_arm, dollars  # noqa: E402
+from scoring.health import scenario_health  # noqa: E402
+from scoring.snr import signal_to_noise  # noqa: E402
 from scoring.stats import paired_significance  # noqa: E402
 
 _ARM_DESC = {
@@ -350,6 +352,109 @@ def _per_scenario_section(dataset: dict) -> list[str]:
     return parts
 
 
+def _snr_cell(rec: dict) -> str:
+    """Render one SNR record: a number (·* when noise-dominated), or the
+    reason it isn't a number."""
+    snr = rec.get("snr")
+    if snr is not None:
+        return f"{snr:.2f}" + ("*" if rec.get("noise_dominated") else "")
+    flag = rec.get("flag")
+    if flag == "zero_noise_clean_separation":
+        return "clean (σ=0)"
+    if flag == "no_effect":
+        return "0 (tie)"
+    return "n/a"
+
+
+def _snr_section(dataset: dict) -> list[str]:
+    """Signal-to-noise for each arm contrast: |paired Δ| / across-seed σ. The
+    honest headline of whether the crossover result is above run-to-run noise.
+
+    `.get`-guarded; needs the `paired` block. With <2 seeds, noise is not
+    estimable and the section says so rather than inventing a ratio.
+    """
+    snr = signal_to_noise(dataset)
+    if not snr["pairs"]:
+        return []
+    parts = ["## Signal-to-noise", ""]
+    if snr["n_seeds"] < 2:
+        parts += [
+            f"Only {snr['n_seeds']} seed — seed-to-seed noise is not estimable, so "
+            "signal-to-noise cannot be computed (you cannot measure noise from a "
+            "single run). Re-run with e.g. `--seeds 0-4` to report it.",
+            "",
+        ]
+        return parts
+    parts.append(
+        "SNR = |between-arm adherence gap| ÷ across-seed standard deviation of "
+        "that gap. **SNR < 1 (·\\*) is noise-dominated** — the gap is within the "
+        "seed-to-seed wobble and should not be leaned on.",
+    )
+    parts.append("")
+    for key, pair in snr["pairs"].items():
+        a, b = key.split("_vs_")
+        head = pair.get("headline") or {}
+        hn = pair.get("headline_N")
+        parts.append(f"**`{a}` vs `{b}`** — headline SNR at N={hn}: {_snr_cell(head)}")
+        parts.append("")
+        parts.append("| N | signal \\|Δ\\| | noise σ | SNR |")
+        parts.append("|---|--:|--:|--:|")
+        for rec in pair["by_n"]:
+            parts.append(
+                f"| {rec['N']} | {_fmt(rec.get('signal'))} | "
+                f"{_fmt(rec.get('noise'))} | {_snr_cell(rec)} |"
+            )
+        parts.append("")
+    parts.append(
+        "\\* noise-dominated (SNR < 1). `clean (σ=0)` = an identical gap every "
+        "seed; `0 (tie)` = no gap and no variance."
+    )
+    parts.append("")
+    return parts
+
+
+def _scenario_health_section(dataset: dict) -> list[str]:
+    """Per-scenario discrimination/validity audit — how many scenarios actually
+    carry signal vs. are broken, contaminated, or ties (OpenAI's broken-task
+    ceiling analysis, applied here). `.get`-guarded on `per_scenario`.
+    """
+    health = scenario_health(dataset)
+    if not health["total"]:
+        return []
+    c = health["counts"]
+    parts = ["## Scenario health (discrimination audit)", ""]
+    bits = [f"**{c['discriminating']} discriminating**", f"{c['broken']} broken",
+            f"{c['contaminated']} contaminated", f"{c['tie']} tie"]
+    if c["unknown"]:
+        bits.append(f"{c['unknown']} unknown")
+    parts.append(f"Of {health['total']} scenarios: " + ", ".join(bits) + ".")
+    if not health["controls"]["ceiling"]:
+        parts.append("_(no `context_dump` ceiling arm in the sweep — 'broken' "
+                     "cannot be detected.)_")
+    if not health["controls"]["floor"]:
+        parts.append("_(no `no_grounding` floor arm in the sweep — 'contaminated' "
+                     "cannot be detected.)_")
+    flagged = [s for s in health["scenarios"] if s["class"] != "discriminating"]
+    if flagged:
+        parts += ["", "| scenario | class | ceiling adheres | floor adheres | max arm gap |",
+                  "|---|---|:--:|:--:|--:|"]
+        _yn = {True: "yes", False: "no", None: "—"}
+        for s in flagged:
+            parts.append(
+                f"| `{s['scenario_id']}` | {s['class']} | "
+                f"{_yn[s['ceiling_adherent']]} | {_yn[s['floor_adherent']]} | "
+                f"{_fmt(s['max_gap'])} |"
+            )
+    parts.append("")
+    parts.append(
+        "_broken = the see-everything arm never adheres (likely mis-specified); "
+        "contaminated = the parametric-memory floor already adheres (doesn't test "
+        "grounding); tie = no arm separates from another._"
+    )
+    parts.append("")
+    return parts
+
+
 def _head_to_head(dataset: dict, run: dict) -> str:
     """rac vs naive_rag only — the comparison that actually adjudicates the thesis."""
     arms = dataset["arms"]
@@ -647,6 +752,12 @@ def build_report(run: dict, dataset: dict | None, cost_curve: dict | None, chart
     if dataset:
         parts += _per_scenario_section(dataset)
 
+    # 4a-bis. Signal-to-noise + scenario-health audit — is the result above the
+    # run-to-run noise, and how many scenarios actually carry signal?
+    if dataset:
+        parts += _snr_section(dataset)
+        parts += _scenario_health_section(dataset)
+
     # 4b. Pre-registered paired significance (when the artifacts carry stats).
     parts += _stats_section(run, dataset)
 
diff --git a/decisiongrounding/tests/test_dashboard.py b/decisiongrounding/tests/test_dashboard.py
index 9844c9f..60d21c8 100644
--- a/decisiongrounding/tests/test_dashboard.py
+++ b/decisiongrounding/tests/test_dashboard.py
@@ -64,12 +64,33 @@ def test_build_dashboard_renders_all_sections():
     out = build_dashboard(_run(), _dataset())
     _assert_html(out)
     for marker in ("Decision Grounding Bench", "Leaderboard", "Adherence vs N",
-                   "rac vs naive RAG", "Scenarios", "Reproduce", " noise-dominated, starred
+    assert "18.00" in out         # 0.90 / 0.05
+
+
+def test_snr_section_single_seed_says_not_estimable():
+    out = "\n".join(_snr_section(_paired_dataset(n_seeds=1)))
+    assert "not estimable" in out
+    assert "4.00" not in out
+
+
+def test_snr_section_absent_without_paired():
+    assert _snr_section({"n_seeds": 3}) == []
+
+
+def test_scenario_health_section_summary_and_flags():
+    ds = {"per_scenario": {
+        "context_dump": {"good": [{"N": 300, "adherent": 1.0}],
+                         "broke": [{"N": 300, "adherent": 0.0}],
+                         "contam": [{"N": 300, "adherent": 1.0}]},
+        "naive_rag": {"good": [{"N": 300, "adherent": 0.3}],
+                      "broke": [{"N": 300, "adherent": 0.0}],
+                      "contam": [{"N": 300, "adherent": 1.0}]},
+        "no_grounding": {"good": [{"N": 300, "adherent": 0.0}],
+                         "broke": [{"N": 300, "adherent": 0.0}],
+                         "contam": [{"N": 300, "adherent": 1.0}]},
+    }}
+    out = "\n".join(_scenario_health_section(ds))
+    assert "Scenario health" in out
+    assert "1 discriminating" in out and "1 broken" in out and "1 contaminated" in out
+    assert "`broke`" in out and "broken" in out
+    assert "`contam`" in out
+
+
+def test_scenario_health_absent_without_per_scenario():
+    assert _scenario_health_section({}) == []

From eb0d5b09a78ed8177cc46ef267675b7cb41627b9 Mon Sep 17 00:00:00 2001
From: Tom Ballard 
Date: Thu, 9 Jul 2026 07:49:06 +0000
Subject: [PATCH 5/5] docs(spec): signal-to-noise methodology note +
 repro/limitations wiring

New spec/methodology-signal-to-noise.md defines the SNR metric (signal =
within-seed paired diff, noise = its across-seed std; >=2 seeds required),
the four scenario-health classes, and the contamination defense (synthetic
scenarios are contamination-proof; the no_grounding floor detects it in
real ones), mapping the OpenAI and Ai2 lessons to the benchmark's
mechanisms. Additive methodology note -- changes no gold labels, gates, or
the frozen H1/H2 tests. The real-crossover Makefile target and run_real.sh
now flag that SEEDS>=2 is required for the signal-to-noise section (arms
already include rac + no_grounding, so both contrasts populate), and
report.py's Limitations section points readers to the methodology.
---
 decisiongrounding/Makefile                    |  5 +-
 decisiongrounding/scripts/report.py           |  9 ++
 decisiongrounding/scripts/run_real.sh         |  6 +-
 .../spec/methodology-signal-to-noise.md       | 90 +++++++++++++++++++
 4 files changed, 107 insertions(+), 3 deletions(-)
 create mode 100644 decisiongrounding/spec/methodology-signal-to-noise.md

diff --git a/decisiongrounding/Makefile b/decisiongrounding/Makefile
index 0a3a6c6..a49ade6 100644
--- a/decisiongrounding/Makefile
+++ b/decisiongrounding/Makefile
@@ -17,7 +17,10 @@ real-run:
 real-batch:
 	BATCH=1 ./scripts/run_real.sh
 # compare + the adherence-vs-N crossover over a real pool (set NS to taste; add
-# BATCH=1 for the Batch API, and SEEDS=0-4 for mean +/- CI error bars):
+# BATCH=1 for the Batch API, and SEEDS=0-4 for mean +/- CI error bars). The
+# default arms include rac + no_grounding, so both signal-to-noise contrasts
+# (rac-vs-naive_rag, rac-vs-no_grounding) populate; SEEDS>=2 is REQUIRED for the
+# signal-to-noise section (noise cannot be estimated from a single seed):
 real-crossover:
 	CROSSOVER=1 ./scripts/run_real.sh
 
diff --git a/decisiongrounding/scripts/report.py b/decisiongrounding/scripts/report.py
index 807c970..3796a71 100644
--- a/decisiongrounding/scripts/report.py
+++ b/decisiongrounding/scripts/report.py
@@ -606,6 +606,15 @@ def _mcnemar_line(dataset: dict, top) -> str | None:
   permit / cite / supersede), not prose quality.
 - **If naive RAG does not decay, the report says so.** The head-to-head verdict
   is computed from the numbers, not asserted.
+- **Signal is reported against noise, and errors are not failures.** The
+  Signal-to-noise section divides the between-arm gap by its across-seed spread
+  so a reader can see whether the result is above run-to-run noise (it needs
+  ≥2 seeds; a single-seed run says so rather than inventing a ratio). Cells that
+  errored or exceeded the context window are excluded from adherence and
+  reported as coverage — an infrastructure failure is never scored as a
+  behavioural one. The scenario-health audit flags any scenario that is broken,
+  contaminated (answerable from parametric memory), or non-discriminating.
+  Methodology: `spec/methodology-signal-to-noise.md`.
 """
 
 
diff --git a/decisiongrounding/scripts/run_real.sh b/decisiongrounding/scripts/run_real.sh
index 4406fa9..188ed6d 100755
--- a/decisiongrounding/scripts/run_real.sh
+++ b/decisiongrounding/scripts/run_real.sh
@@ -116,8 +116,10 @@ python3 -m runner.cli "$HEADLINE_CMD" \
 #    standard price, and it runs server-side so a client/container restart can't
 #    lose it — strongly recommended for the full N sweep).
 #    SEEDS (e.g. SEEDS=0-4) runs the sweep over several seeds and reports
-#    mean +/- 95% CI with a paired rac-vs-naive_rag difference. Cost multiplies
-#    with the number of seeds.
+#    mean +/- 95% CI with paired rac-vs-naive_rag and rac-vs-no_grounding
+#    differences. SEEDS with >=2 seeds is required for the report's
+#    signal-to-noise section (noise is not estimable from one seed). Cost
+#    multiplies with the number of seeds.
 if [ "$CROSSOVER" = "1" ]; then
   [ -d "$POOL/corpus" ] || python3 -m ingest.peps pool build --out "$POOL"
   echo "== crossover (real distractors) =="
diff --git a/decisiongrounding/spec/methodology-signal-to-noise.md b/decisiongrounding/spec/methodology-signal-to-noise.md
new file mode 100644
index 0000000..e897acd
--- /dev/null
+++ b/decisiongrounding/spec/methodology-signal-to-noise.md
@@ -0,0 +1,90 @@
+# Methodology — signal-to-noise and scenario health
+
+*Additive methodology note (not a pre-registration amendment). It documents how
+the reporting layer separates signal from noise; it changes no gold labels, no
+scoring rubric, and none of the confirmatory tests in
+`analysis-plan-amendment-1.md` / `-2.md` (both frozen).*
+
+## Why
+
+Two analyses of coding/LM evaluations converge on the same warning:
+
+- OpenAI, *["Separating signal from noise in coding evaluations"](https://openai.com/index/separating-signal-from-noise-coding-evaluations/)* — a
+  widely-used benchmark was found to have ~20–30% of tasks broken (unsolvable,
+  or verified against hidden assumptions), so failures reflected the harness,
+  not the model. A trustworthy eval must (a) separate genuine model failures
+  from harness/grading noise, and (b) know how many of its tasks actually carry
+  signal.
+- Ai2, *["Signal and Noise"](https://allenai.org/blog/signal-noise)* — a
+  benchmark is only reliable when its **signal** (spread between systems) is
+  large relative to its **noise** (run-to-run variance). They report a
+  signal-to-noise ratio and select high-SNR subtasks.
+
+This benchmark already avoids the largest noise source — grading — by scoring
+structurally and deterministically (no LLM judge). This note adds the two
+missing pieces: an explicit signal-to-noise ratio, and a per-scenario validity
+audit.
+
+## Signal-to-noise (SNR)
+
+For an arm contrast `(a, b)` at corpus size `N`:
+
+    signal = |mean over seeds of (adherence_a − adherence_b)|      # dataset["paired"][…]["diff_mean"]
+    noise  = std  over seeds of (adherence_a − adherence_b)        # …["diff_std"]
+    SNR    = signal / noise
+
+The difference is taken **within each seed** (common random numbers), so `noise`
+is the genuine seed-to-seed wobble of the *gap*, not of either arm alone.
+
+- **SNR < 1** — the between-arm gap is smaller than the run-to-run noise:
+  **noise-dominated**, not a result to lean on. Flagged with `*` in the report.
+- **noise cannot be estimated from one seed.** With `n_seeds < 2` the report
+  says "noise not estimable" and prints no ratio — the single most important
+  honesty guard, and the direct lesson from both articles. Run `SEEDS=0-4`
+  (`make real-crossover SEEDS=0-4`) to measure it.
+- Edge cases: an identical nonzero gap every seed is `clean (σ=0)` (maximal
+  signal); an identical zero gap every seed is `0 (tie)` (no effect).
+
+Reported for both pre-registered contrasts: `rac`-vs-`naive_rag` (H1, the
+falsifier) and `rac`-vs-`no_grounding` (H2, grounding vs the parametric-memory
+floor). Headline SNR is quoted at the largest `N` — the buried-in-distractors
+corpus size where the thesis is actually adjudicated. Implementation:
+`scoring/snr.py`.
+
+## Scenario health (discrimination / validity audit)
+
+Each scenario is classified against the two control arms already in the sweep —
+the ceiling (`context_dump`, sees the whole corpus) and the floor
+(`no_grounding`, parametric memory only):
+
+| class | condition | meaning |
+|---|---|---|
+| **broken** | ceiling never adheres at any N | likely mis-specified / unsolvable from the corpus |
+| **contaminated** | floor adheres | answerable from pretraining memory — doesn't test grounding |
+| **tie** | no arm separates from another at any N | degenerate, contributes no signal |
+| **discriminating** | otherwise | grounding separates from the floor |
+| **unknown** | neither control arm in the sweep | cannot audit |
+
+The summary count ("X of Y discriminating") is this benchmark's analogue of
+OpenAI's broken-task ceiling. Implementation: `scoring/health.py`.
+
+## Contamination defense
+
+Real PEPs/RFCs are in the answering model's pretraining, so it may "know" that
+PEP 440 supersedes PEP 386 independent of the grounding it is given. Two
+defenses, both already in the design and now surfaced:
+
+1. **Synthetic scenarios are contamination-proof by construction** — the
+   `scenarios/` bank invents fictional ADRs the model cannot have seen.
+2. **The `no_grounding` floor detects contamination in the real scenarios** —
+   if parametric memory alone adheres, the scenario is flagged `contaminated`
+   by the health audit and its adherence is not evidence that grounding helped.
+
+## Relationship to the confirmatory analysis
+
+SNR and scenario health are **descriptive reporting**, computed from the same
+`paired` / `per_scenario` data the frozen McNemar analysis already uses. They
+add no hypothesis and change no gate: H1/H2 remain exactly as pre-registered in
+amendment-1. They exist so a reader — or a hostile reviewer — can judge for
+themselves whether a reported gap is signal or noise, and how much of the
+roster is actually pulling its weight.