Triggers the CLI on the server; the page refreshes when the run finishes.
"
"Reproduce
'
+ # 6 Signal / noise (SNR + scenario-health audit)
+ secs.append(f'{_signal_noise_section(dataset) if dataset else "Needs a crossover dataset.
"}')
+ # 7 Reproduce
+ secs.append('Reproduce
'
'make real-crossover # headline + adherence-vs-N over the real pool\n'
'make real-batch # via the Batch API (~50% cost)\n\n'
'make ui # serve this dashboard live at 127.0.0.1:8099\n'
@@ -424,9 +487,10 @@ def render_main(run, dataset, cost_curve=None, *, live=False, paid_enabled=False
return "".join(secs)
-# Tab labels, in section order (s0..s6, +s7 Run when live). Kept beside the
+# Tab labels, in section order (s0..s7, +s8 Run when live). Kept beside the
# renderer so the nav and the sections stay in lockstep.
-_TABS = ["Overview", "Leaderboard", "Curves", "Cost", "rac vs RAG", "Scenarios", "Reproduce"]
+_TABS = ["Overview", "Leaderboard", "Curves", "Cost", "rac vs RAG", "Scenarios",
+ "Signal / noise", "Reproduce"]
def build_dashboard(run, dataset, cost_curve=None, *, live=False, paid_enabled=False, template=None):
diff --git a/decisiongrounding/runner/templates/dashboard.html b/decisiongrounding/runner/templates/dashboard.html
index 0251096..badf509 100644
--- a/decisiongrounding/runner/templates/dashboard.html
+++ b/decisiongrounding/runner/templates/dashboard.html
@@ -8,7 +8,7 @@
[CHIPS] - the config chips (answering model, embedder, scenarios)
[BODY] - the tab radios + nav + main, with the data-bound sections
Point the server / static generator at a custom copy with --template or the
- DG_UI_TEMPLATE env var. Tabs are pure CSS (#t0..#t7); the JS only activates
+ DG_UI_TEMPLATE env var. Tabs are pure CSS (#t0..#t8); the JS only activates
when a live Run tab (#run-status) is present, so static snapshots stay inert.
-->
@@ -43,6 +43,7 @@
#t5:checked~nav label[for=t5]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t5:checked~main #s5{display:block}
#t6:checked~nav label[for=t6]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t6:checked~main #s6{display:block}
#t7:checked~nav label[for=t7]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t7:checked~main #s7{display:block}
+#t8:checked~nav label[for=t8]{color:var(--fg);border-bottom-color:var(--fg);font-weight:600}#t8:checked~main #s8{display:block}
table{border-collapse:collapse;width:100%;margin:8px 0 18px;font-size:14px}
th,td{border:1px solid var(--line);padding:7px 10px;text-align:right}
th:first-child,td:first-child{text-align:left}
diff --git a/decisiongrounding/scoring/__init__.py b/decisiongrounding/scoring/__init__.py
index 8ce2072..4ae605d 100644
--- a/decisiongrounding/scoring/__init__.py
+++ b/decisiongrounding/scoring/__init__.py
@@ -9,11 +9,15 @@
summarize,
t_critical_95,
)
+from .health import scenario_health
from .scorer import Score, score
+from .snr import signal_to_noise
__all__ = [
"Score",
"score",
+ "signal_to_noise",
+ "scenario_health",
"ArmMetrics",
"aggregate",
"adherence_variance",
diff --git a/decisiongrounding/scoring/crossover.py b/decisiongrounding/scoring/crossover.py
index e27db5e..3b7215e 100644
--- a/decisiongrounding/scoring/crossover.py
+++ b/decisiongrounding/scoring/crossover.py
@@ -617,26 +617,33 @@ def build_dataset_multiseed(
scenarios_dir: str | None = None,
batched: bool = False,
poll: int = 20,
- pair: tuple[str, str] = ("rac", "naive_rag"),
+ pair: "tuple[str, str] | None" = None,
+ pairs: "list[tuple[str, str]] | None" = None,
progress: "Callable[[dict], None] | None" = None,
resume: "dict[tuple[int, int, str, str], dict] | None" = None,
) -> dict:
"""Run the crossover over several seeds and aggregate per (arm, N) into
mean +/- a t-based 95% CI. The plain fields stay the mean (backward
- compatible); `_ci` / `_std` / `_values`, `n_seeds`, `seeds`, and a
- paired `pair[0]`-vs-`pair[1]` adherence difference (`paired`) are added.
+ compatible); `_ci` / `_std` / `_values`, `n_seeds`, `seeds`, and
+ paired adherence differences (`paired`) are added.
+
+ `pairs` selects the arm contrasts for the paired diffs (default
+ `DEFAULT_PAIRS`: rac-vs-naive_rag and rac-vs-no_grounding); a single
+ `pair=(a, b)` tuple is still accepted for back-compat. A contrast whose
+ arms aren't both in the sweep is skipped.
Calls the single-seed builders per seed, so batched + multiseed compose.
Offline runs (deterministic stub + embedder) show little spread; the
aggregation is exercised regardless.
"""
+ resolved = pairs if pairs is not None else ([pair] if pair is not None else DEFAULT_PAIRS)
uniq = list(dict.fromkeys(int(s) for s in seeds)) or [0]
per_seed = run_seeds(
scenarios, arms, ns, uniq,
answering_model_name=answering_model_name, embedder_spec=embedder_spec,
pool=pool, pool_dir=pool_dir, scenarios_dir=scenarios_dir,
batched=batched, poll=poll, progress=progress, resume=resume)
- return _aggregate_seeds(per_seed, list(arms), list(ns), pair)
+ return _aggregate_seeds(per_seed, list(arms), list(ns), resolved)
def run_seeds(
@@ -710,10 +717,28 @@ def _aggregate_arm_points(arms, ns, cols, n_seeds) -> dict:
return points
-def _paired(per_seed, ns, pair) -> dict | None:
- """Per-N paired adherence difference pair[0]-pair[1], differenced within each
- seed (common random numbers), with its own CI."""
- a, b = pair
+# The arm contrasts whose paired adherence difference (and thus signal-to-noise)
+# the multi-seed crossover reports by default: the pre-registered falsifier
+# (rac vs naive_rag, H1) and the parametric-memory floor (rac vs no_grounding,
+# H2). A pair whose arms aren't both in the sweep is silently skipped, so this
+# default is safe for any arm subset (offline runs without rac emit neither).
+DEFAULT_PAIRS = (("rac", "naive_rag"), ("rac", "no_grounding"))
+
+
+def _normalize_pairs(pairs) -> list[tuple[str, str]]:
+ """Accept a single ``(a, b)`` tuple or a list of them; return a list of
+ tuples. ``None``/empty -> ``[]``."""
+ if not pairs:
+ return []
+ if isinstance(pairs[0], str): # a bare (a, b) tuple
+ return [(pairs[0], pairs[1])]
+ return [(p[0], p[1]) for p in pairs]
+
+
+def _paired_one(per_seed, ns, a, b) -> list | None:
+ """Per-N paired adherence difference a-b, differenced within each seed
+ (common random numbers), with its own CI. None when either arm is absent
+ from any seed's sweep."""
if not per_seed or not all(a in ds["arms"] and b in ds["arms"] for _, ds in per_seed):
return None
out = []
@@ -727,7 +752,23 @@ def _paired(per_seed, ns, pair) -> dict | None:
s = summarize(diffs)
out.append({"N": n, "diff_mean": s["mean"], "diff_ci": s["ci"],
"diff_std": s["std"], "n": s["n"], "values": s["values"]})
- return {f"{a}_vs_{b}": out} if out else None
+ return out or None
+
+
+def _paired(per_seed, ns, pairs) -> dict | None:
+ """Per-N within-seed paired adherence differences for one or more arm pairs.
+
+ `pairs` accepts a single ``(a, b)`` tuple (back-compat) or a list of them.
+ Each pair whose both arms appear in every seed's sweep contributes a
+ ``"{a}_vs_{b}"`` entry; pairs with a missing arm are skipped. The
+ across-seed std of each entry (`diff_std`) is the noise term the
+ signal-to-noise report divides into (see `scoring.snr`)."""
+ result = {}
+ for a, b in _normalize_pairs(pairs):
+ series = _paired_one(per_seed, ns, a, b)
+ if series:
+ result[f"{a}_vs_{b}"] = series
+ return result or None
def _agg_per_scenario(per_seed, arms) -> dict:
@@ -755,7 +796,7 @@ def _agg_per_scenario(per_seed, arms) -> dict:
return out
-def _aggregate_seeds(per_seed, arms, ns, pair) -> dict:
+def _aggregate_seeds(per_seed, arms, ns, pairs) -> dict:
base = dict(per_seed[0][1])
cols = _columns_from_datasets(per_seed, arms, ns)
base["arms"] = _aggregate_arm_points(arms, ns, cols, len(per_seed))
@@ -769,7 +810,7 @@ def _aggregate_seeds(per_seed, arms, ns, pair) -> dict:
cells = [c for _, ds in per_seed for c in (ds.get("cells") or [])]
base["cells"] = cells
base["stats"] = stats_by_n(cells) if cells else None
- paired = _paired(per_seed, ns, pair)
+ paired = _paired(per_seed, ns, pairs)
if paired:
base["paired"] = paired
return base
@@ -808,13 +849,19 @@ def merge_seed_datasets(existing: dict, new_per_seed, arms, ns, pair) -> dict:
base = dict(existing)
base["arms"] = _aggregate_arm_points(arms, ns, cols, len(all_seeds))
- # Paired: prior per-seed diffs + the new seeds' diffs.
- a, b = pair
- old_paired = (existing.get("paired") or {}).get(f"{a}_vs_{b}")
- new_paired = (_paired(add, ns, pair) or {}).get(f"{a}_vs_{b}")
- if old_paired is not None or new_paired is not None:
- old_by_n = {e["N"]: e for e in (old_paired or [])}
- new_by_n = {e["N"]: e for e in (new_paired or [])}
+ # Paired: prior per-seed diffs + the new seeds' diffs, for every contrast —
+ # the requested pairs plus any already present in `existing` (so an
+ # under-specified merge never silently drops a contrast). Arm names contain
+ # underscores, but "_vs_" is a safe delimiter for the existing keys.
+ requested = _normalize_pairs(pair)
+ keys = {f"{a}_vs_{b}": (a, b) for a, b in requested}
+ for k in (existing.get("paired") or {}):
+ keys.setdefault(k, tuple(k.split("_vs_")))
+ new_paired = _paired(add, ns, list(keys.values())) or {}
+ merged_paired = {}
+ for key in keys:
+ old_by_n = {e["N"]: e for e in (existing.get("paired") or {}).get(key, [])}
+ new_by_n = {e["N"]: e for e in new_paired.get(key, [])}
merged = []
for n in ns:
vals = list(old_by_n.get(n, {}).get("values", [])) + list(new_by_n.get(n, {}).get("values", []))
@@ -823,7 +870,9 @@ def merge_seed_datasets(existing: dict, new_per_seed, arms, ns, pair) -> dict:
merged.append({"N": n, "diff_mean": s["mean"], "diff_ci": s["ci"],
"diff_std": s["std"], "n": s["n"], "values": s["values"]})
if merged:
- base["paired"] = {f"{a}_vs_{b}": merged}
+ merged_paired[key] = merged
+ if merged_paired:
+ base["paired"] = merged_paired
base["per_scenario"] = _merge_per_scenario(existing.get("per_scenario", {}), add, arms, n_old)
base["errors"] = list(existing.get("errors", [])) + [e for _, ds in add for e in ds["errors"]]
diff --git a/decisiongrounding/scoring/health.py b/decisiongrounding/scoring/health.py
new file mode 100644
index 0000000..d0f4b79
--- /dev/null
+++ b/decisiongrounding/scoring/health.py
@@ -0,0 +1,106 @@
+"""Per-scenario discrimination / validity audit.
+
+OpenAI's coding-eval analysis found ~20-30% of a popular benchmark's tasks were
+broken — unsolvable, contaminated, or non-discriminating — silently capping the
+ceiling. This audit surfaces the analogue for the crossover: for each scenario,
+is it actually pulling its weight, or is it dead signal?
+
+Four classes, referencing the two controls already in the sweep — the ceiling
+arm (`context_dump`, sees the whole corpus) and the floor arm (`no_grounding`,
+parametric memory only):
+
+- **broken** — even the ceiling arm never adheres (likely mis-specified or
+ unsolvable from the corpus).
+- **contaminated** — the floor arm adheres (the model answers from pretraining
+ memory, so the scenario doesn't test grounding — a real risk for public
+ PEPs/RFCs; synthetic scenarios are contamination-proof by construction).
+- **tie** — no arm separates from any other at any N (degenerate, no signal).
+- **discriminating** — otherwise (grounding separates from the floor).
+
+`unknown` when neither control arm is in the sweep. Pure and results-neutral:
+reads only `dataset["per_scenario"]`.
+"""
+
+from __future__ import annotations
+
+CEILING_ARM = "context_dump"
+FLOOR_ARM = "no_grounding"
+_ADHERE = 0.5 # majority; robust to multi-seed fractions and single-seed bools
+
+
+def _adherence_by_n(per_scenario: dict, arm: str, sid: str) -> dict:
+ """{N: adherence} for one (arm, scenario); adherent may be a bool
+ (single-seed) or a fraction (multi-seed) — both coerce to float."""
+ out = {}
+ for rec in per_scenario.get(arm, {}).get(sid, []):
+ v = rec.get("adherent")
+ if v is not None:
+ out[rec["N"]] = float(v)
+ return out
+
+
+def _adheres_somewhere(by_n: dict) -> bool:
+ return any(v >= _ADHERE for v in by_n.values())
+
+
+def scenario_health(dataset: dict) -> dict:
+ """Classify every scenario in `dataset["per_scenario"]`.
+
+ Returns::
+
+ {"scenarios": [{"scenario_id", "class", "ceiling_adherent",
+ "floor_adherent", "separates", "max_gap"}, ...],
+ "counts": {"discriminating", "broken", "contaminated", "tie",
+ "unknown"},
+ "total": int,
+ "controls": {"ceiling": bool, "floor": bool}}
+
+ `.get`-guarded: a dataset without `per_scenario` yields an empty audit.
+ """
+ per = dataset.get("per_scenario") or {}
+ arms = list(per)
+ has_ceiling = CEILING_ARM in arms
+ has_floor = FLOOR_ARM in arms
+ sids = sorted({sid for arm in arms for sid in per.get(arm, {})})
+
+ scenarios = []
+ counts = {"discriminating": 0, "broken": 0, "contaminated": 0, "tie": 0, "unknown": 0}
+ for sid in sids:
+ by_arm = {arm: _adherence_by_n(per, arm, sid) for arm in arms}
+ ns = sorted({n for b in by_arm.values() for n in b})
+
+ # Between-arm separation: the largest adherence gap at any shared N.
+ max_gap = 0.0
+ for n in ns:
+ vals = [b[n] for b in by_arm.values() if n in b]
+ if len(vals) >= 2:
+ max_gap = max(max_gap, max(vals) - min(vals))
+ separates = max_gap > 1e-9
+
+ ceiling_adherent = _adheres_somewhere(by_arm[CEILING_ARM]) if has_ceiling else None
+ floor_adherent = _adheres_somewhere(by_arm[FLOOR_ARM]) if has_floor else None
+
+ if not (has_ceiling or has_floor):
+ cls = "unknown"
+ elif has_ceiling and not ceiling_adherent:
+ cls = "broken"
+ elif has_floor and floor_adherent:
+ cls = "contaminated"
+ elif not separates:
+ cls = "tie"
+ else:
+ cls = "discriminating"
+
+ counts[cls] += 1
+ scenarios.append({
+ "scenario_id": sid, "class": cls,
+ "ceiling_adherent": ceiling_adherent, "floor_adherent": floor_adherent,
+ "separates": separates, "max_gap": max_gap,
+ })
+
+ return {
+ "scenarios": scenarios,
+ "counts": counts,
+ "total": len(sids),
+ "controls": {"ceiling": has_ceiling, "floor": has_floor},
+ }
diff --git a/decisiongrounding/scoring/snr.py b/decisiongrounding/scoring/snr.py
new file mode 100644
index 0000000..6fcdabc
--- /dev/null
+++ b/decisiongrounding/scoring/snr.py
@@ -0,0 +1,79 @@
+"""Signal-to-noise for the crossover.
+
+Ai2's "Signal and Noise" framework (and OpenAI's coding-eval analysis) make the
+same point: a benchmark result is only trustworthy when the effect it measures
+is large relative to the run-to-run noise. Here the **signal** is the within-seed
+paired adherence difference between two arms (`diff_mean` in
+`dataset["paired"]`), and the **noise** is that difference's spread across seeds
+(`diff_std`, under common random numbers). Their ratio is the signal-to-noise:
+
+ SNR = |diff_mean| / diff_std
+
+An SNR below 1 means the between-arm gap is smaller than the seed-to-seed
+wobble — noise-dominated, not a result to lean on. Crucially, **noise cannot be
+estimated from a single seed**: with `n_seeds < 2` the SNR is reported as "not
+estimable", never as a fabricated number.
+
+Pure and results-neutral: reads only the already-computed `paired` block, adds
+nothing to the dataset.
+"""
+
+from __future__ import annotations
+
+
+def _snr_at(diff_mean: float, diff_std: "float | None", n_seeds: int) -> dict:
+ """Classify one (pair, N) cell into an SNR record."""
+ signal = abs(diff_mean)
+ if n_seeds < 2 or diff_std is None:
+ # No repeated runs -> noise is unmeasured. The article's core caveat.
+ return {"signal": signal, "noise": diff_std, "snr": None,
+ "noise_dominated": False, "clean_separation": False,
+ "flag": "noise_not_estimable"}
+ if diff_std == 0:
+ if signal > 0:
+ # Every seed gave the same nonzero gap: maximally clean signal.
+ return {"signal": signal, "noise": 0.0, "snr": None,
+ "noise_dominated": False, "clean_separation": True,
+ "flag": "zero_noise_clean_separation"}
+ # Every seed a perfect tie: no effect and no variance.
+ return {"signal": 0.0, "noise": 0.0, "snr": None,
+ "noise_dominated": False, "clean_separation": False,
+ "flag": "no_effect"}
+ snr = signal / diff_std
+ return {"signal": signal, "noise": diff_std, "snr": snr,
+ "noise_dominated": snr < 1.0, "clean_separation": False, "flag": None}
+
+
+def signal_to_noise(dataset: dict) -> dict:
+ """Signal-to-noise per arm contrast and per N, from `dataset["paired"]`.
+
+ Returns::
+
+ {"n_seeds": int,
+ "pairs": {"_vs_": {"by_n": [{"N", "signal", "noise", "snr",
+ "noise_dominated", "clean_separation",
+ "flag"}, ...],
+ "headline_N": int|None,
+ "headline": |None},
+ ...}}
+
+ `.get`-guarded: a dataset without `paired` (single-arm, legacy, or
+ single-contrast) yields an empty `pairs`. The headline is taken at the
+ largest N — the buried-in-distractors corpus size where the thesis is
+ actually adjudicated.
+ """
+ paired = dataset.get("paired") or {}
+ n_seeds = int(dataset.get("n_seeds", 1) or 1)
+ pairs: dict[str, dict] = {}
+ for key, series in paired.items():
+ by_n = []
+ for e in series:
+ rec = {"N": e["N"], **_snr_at(e["diff_mean"], e.get("diff_std"), n_seeds)}
+ by_n.append(rec)
+ headline = max(by_n, key=lambda r: r["N"]) if by_n else None
+ pairs[key] = {
+ "by_n": by_n,
+ "headline_N": headline["N"] if headline else None,
+ "headline": headline,
+ }
+ return {"n_seeds": n_seeds, "pairs": pairs}
diff --git a/decisiongrounding/scripts/report.py b/decisiongrounding/scripts/report.py
index 8e3559a..3796a71 100644
--- a/decisiongrounding/scripts/report.py
+++ b/decisiongrounding/scripts/report.py
@@ -29,6 +29,8 @@
from scoring.charts import grouped_bar_chart, line_chart # noqa: E402
from scoring.cost import cost_by_arm, dollars # noqa: E402
+from scoring.health import scenario_health # noqa: E402
+from scoring.snr import signal_to_noise # noqa: E402
from scoring.stats import paired_significance # noqa: E402
_ARM_DESC = {
@@ -350,6 +352,109 @@ def _per_scenario_section(dataset: dict) -> list[str]:
return parts
+def _snr_cell(rec: dict) -> str:
+ """Render one SNR record: a number (·* when noise-dominated), or the
+ reason it isn't a number."""
+ snr = rec.get("snr")
+ if snr is not None:
+ return f"{snr:.2f}" + ("*" if rec.get("noise_dominated") else "")
+ flag = rec.get("flag")
+ if flag == "zero_noise_clean_separation":
+ return "clean (σ=0)"
+ if flag == "no_effect":
+ return "0 (tie)"
+ return "n/a"
+
+
+def _snr_section(dataset: dict) -> list[str]:
+ """Signal-to-noise for each arm contrast: |paired Δ| / across-seed σ. The
+ honest headline of whether the crossover result is above run-to-run noise.
+
+ `.get`-guarded; needs the `paired` block. With <2 seeds, noise is not
+ estimable and the section says so rather than inventing a ratio.
+ """
+ snr = signal_to_noise(dataset)
+ if not snr["pairs"]:
+ return []
+ parts = ["## Signal-to-noise", ""]
+ if snr["n_seeds"] < 2:
+ parts += [
+ f"Only {snr['n_seeds']} seed — seed-to-seed noise is not estimable, so "
+ "signal-to-noise cannot be computed (you cannot measure noise from a "
+ "single run). Re-run with e.g. `--seeds 0-4` to report it.",
+ "",
+ ]
+ return parts
+ parts.append(
+ "SNR = |between-arm adherence gap| ÷ across-seed standard deviation of "
+ "that gap. **SNR < 1 (·\\*) is noise-dominated** — the gap is within the "
+ "seed-to-seed wobble and should not be leaned on.",
+ )
+ parts.append("")
+ for key, pair in snr["pairs"].items():
+ a, b = key.split("_vs_")
+ head = pair.get("headline") or {}
+ hn = pair.get("headline_N")
+ parts.append(f"**`{a}` vs `{b}`** — headline SNR at N={hn}: {_snr_cell(head)}")
+ parts.append("")
+ parts.append("| N | signal \\|Δ\\| | noise σ | SNR |")
+ parts.append("|---|--:|--:|--:|")
+ for rec in pair["by_n"]:
+ parts.append(
+ f"| {rec['N']} | {_fmt(rec.get('signal'))} | "
+ f"{_fmt(rec.get('noise'))} | {_snr_cell(rec)} |"
+ )
+ parts.append("")
+ parts.append(
+ "\\* noise-dominated (SNR < 1). `clean (σ=0)` = an identical gap every "
+ "seed; `0 (tie)` = no gap and no variance."
+ )
+ parts.append("")
+ return parts
+
+
+def _scenario_health_section(dataset: dict) -> list[str]:
+ """Per-scenario discrimination/validity audit — how many scenarios actually
+ carry signal vs. are broken, contaminated, or ties (OpenAI's broken-task
+ ceiling analysis, applied here). `.get`-guarded on `per_scenario`.
+ """
+ health = scenario_health(dataset)
+ if not health["total"]:
+ return []
+ c = health["counts"]
+ parts = ["## Scenario health (discrimination audit)", ""]
+ bits = [f"**{c['discriminating']} discriminating**", f"{c['broken']} broken",
+ f"{c['contaminated']} contaminated", f"{c['tie']} tie"]
+ if c["unknown"]:
+ bits.append(f"{c['unknown']} unknown")
+ parts.append(f"Of {health['total']} scenarios: " + ", ".join(bits) + ".")
+ if not health["controls"]["ceiling"]:
+ parts.append("_(no `context_dump` ceiling arm in the sweep — 'broken' "
+ "cannot be detected.)_")
+ if not health["controls"]["floor"]:
+ parts.append("_(no `no_grounding` floor arm in the sweep — 'contaminated' "
+ "cannot be detected.)_")
+ flagged = [s for s in health["scenarios"] if s["class"] != "discriminating"]
+ if flagged:
+ parts += ["", "| scenario | class | ceiling adheres | floor adheres | max arm gap |",
+ "|---|---|:--:|:--:|--:|"]
+ _yn = {True: "yes", False: "no", None: "—"}
+ for s in flagged:
+ parts.append(
+ f"| `{s['scenario_id']}` | {s['class']} | "
+ f"{_yn[s['ceiling_adherent']]} | {_yn[s['floor_adherent']]} | "
+ f"{_fmt(s['max_gap'])} |"
+ )
+ parts.append("")
+ parts.append(
+ "_broken = the see-everything arm never adheres (likely mis-specified); "
+ "contaminated = the parametric-memory floor already adheres (doesn't test "
+ "grounding); tie = no arm separates from another._"
+ )
+ parts.append("")
+ return parts
+
+
def _head_to_head(dataset: dict, run: dict) -> str:
"""rac vs naive_rag only — the comparison that actually adjudicates the thesis."""
arms = dataset["arms"]
@@ -501,6 +606,15 @@ def _mcnemar_line(dataset: dict, top) -> str | None:
permit / cite / supersede), not prose quality.
- **If naive RAG does not decay, the report says so.** The head-to-head verdict
is computed from the numbers, not asserted.
+- **Signal is reported against noise, and errors are not failures.** The
+ Signal-to-noise section divides the between-arm gap by its across-seed spread
+ so a reader can see whether the result is above run-to-run noise (it needs
+ ≥2 seeds; a single-seed run says so rather than inventing a ratio). Cells that
+ errored or exceeded the context window are excluded from adherence and
+ reported as coverage — an infrastructure failure is never scored as a
+ behavioural one. The scenario-health audit flags any scenario that is broken,
+ contaminated (answerable from parametric memory), or non-discriminating.
+ Methodology: `spec/methodology-signal-to-noise.md`.
"""
@@ -647,6 +761,12 @@ def build_report(run: dict, dataset: dict | None, cost_curve: dict | None, chart
if dataset:
parts += _per_scenario_section(dataset)
+ # 4a-bis. Signal-to-noise + scenario-health audit — is the result above the
+ # run-to-run noise, and how many scenarios actually carry signal?
+ if dataset:
+ parts += _snr_section(dataset)
+ parts += _scenario_health_section(dataset)
+
# 4b. Pre-registered paired significance (when the artifacts carry stats).
parts += _stats_section(run, dataset)
diff --git a/decisiongrounding/scripts/run_real.sh b/decisiongrounding/scripts/run_real.sh
index 4406fa9..188ed6d 100755
--- a/decisiongrounding/scripts/run_real.sh
+++ b/decisiongrounding/scripts/run_real.sh
@@ -116,8 +116,10 @@ python3 -m runner.cli "$HEADLINE_CMD" \
# standard price, and it runs server-side so a client/container restart can't
# lose it — strongly recommended for the full N sweep).
# SEEDS (e.g. SEEDS=0-4) runs the sweep over several seeds and reports
-# mean +/- 95% CI with a paired rac-vs-naive_rag difference. Cost multiplies
-# with the number of seeds.
+# mean +/- 95% CI with paired rac-vs-naive_rag and rac-vs-no_grounding
+# differences. SEEDS with >=2 seeds is required for the report's
+# signal-to-noise section (noise is not estimable from one seed). Cost
+# multiplies with the number of seeds.
if [ "$CROSSOVER" = "1" ]; then
[ -d "$POOL/corpus" ] || python3 -m ingest.peps pool build --out "$POOL"
echo "== crossover (real distractors) =="
diff --git a/decisiongrounding/spec/methodology-signal-to-noise.md b/decisiongrounding/spec/methodology-signal-to-noise.md
new file mode 100644
index 0000000..e897acd
--- /dev/null
+++ b/decisiongrounding/spec/methodology-signal-to-noise.md
@@ -0,0 +1,90 @@
+# Methodology — signal-to-noise and scenario health
+
+*Additive methodology note (not a pre-registration amendment). It documents how
+the reporting layer separates signal from noise; it changes no gold labels, no
+scoring rubric, and none of the confirmatory tests in
+`analysis-plan-amendment-1.md` / `-2.md` (both frozen).*
+
+## Why
+
+Two analyses of coding/LM evaluations converge on the same warning:
+
+- OpenAI, *["Separating signal from noise in coding evaluations"](https://openai.com/index/separating-signal-from-noise-coding-evaluations/)* — a
+ widely-used benchmark was found to have ~20–30% of tasks broken (unsolvable,
+ or verified against hidden assumptions), so failures reflected the harness,
+ not the model. A trustworthy eval must (a) separate genuine model failures
+ from harness/grading noise, and (b) know how many of its tasks actually carry
+ signal.
+- Ai2, *["Signal and Noise"](https://allenai.org/blog/signal-noise)* — a
+ benchmark is only reliable when its **signal** (spread between systems) is
+ large relative to its **noise** (run-to-run variance). They report a
+ signal-to-noise ratio and select high-SNR subtasks.
+
+This benchmark already avoids the largest noise source — grading — by scoring
+structurally and deterministically (no LLM judge). This note adds the two
+missing pieces: an explicit signal-to-noise ratio, and a per-scenario validity
+audit.
+
+## Signal-to-noise (SNR)
+
+For an arm contrast `(a, b)` at corpus size `N`:
+
+ signal = |mean over seeds of (adherence_a − adherence_b)| # dataset["paired"][…]["diff_mean"]
+ noise = std over seeds of (adherence_a − adherence_b) # …["diff_std"]
+ SNR = signal / noise
+
+The difference is taken **within each seed** (common random numbers), so `noise`
+is the genuine seed-to-seed wobble of the *gap*, not of either arm alone.
+
+- **SNR < 1** — the between-arm gap is smaller than the run-to-run noise:
+ **noise-dominated**, not a result to lean on. Flagged with `*` in the report.
+- **noise cannot be estimated from one seed.** With `n_seeds < 2` the report
+ says "noise not estimable" and prints no ratio — the single most important
+ honesty guard, and the direct lesson from both articles. Run `SEEDS=0-4`
+ (`make real-crossover SEEDS=0-4`) to measure it.
+- Edge cases: an identical nonzero gap every seed is `clean (σ=0)` (maximal
+ signal); an identical zero gap every seed is `0 (tie)` (no effect).
+
+Reported for both pre-registered contrasts: `rac`-vs-`naive_rag` (H1, the
+falsifier) and `rac`-vs-`no_grounding` (H2, grounding vs the parametric-memory
+floor). Headline SNR is quoted at the largest `N` — the buried-in-distractors
+corpus size where the thesis is actually adjudicated. Implementation:
+`scoring/snr.py`.
+
+## Scenario health (discrimination / validity audit)
+
+Each scenario is classified against the two control arms already in the sweep —
+the ceiling (`context_dump`, sees the whole corpus) and the floor
+(`no_grounding`, parametric memory only):
+
+| class | condition | meaning |
+|---|---|---|
+| **broken** | ceiling never adheres at any N | likely mis-specified / unsolvable from the corpus |
+| **contaminated** | floor adheres | answerable from pretraining memory — doesn't test grounding |
+| **tie** | no arm separates from another at any N | degenerate, contributes no signal |
+| **discriminating** | otherwise | grounding separates from the floor |
+| **unknown** | neither control arm in the sweep | cannot audit |
+
+The summary count ("X of Y discriminating") is this benchmark's analogue of
+OpenAI's broken-task ceiling. Implementation: `scoring/health.py`.
+
+## Contamination defense
+
+Real PEPs/RFCs are in the answering model's pretraining, so it may "know" that
+PEP 440 supersedes PEP 386 independent of the grounding it is given. Two
+defenses, both already in the design and now surfaced:
+
+1. **Synthetic scenarios are contamination-proof by construction** — the
+ `scenarios/` bank invents fictional ADRs the model cannot have seen.
+2. **The `no_grounding` floor detects contamination in the real scenarios** —
+ if parametric memory alone adheres, the scenario is flagged `contaminated`
+ by the health audit and its adherence is not evidence that grounding helped.
+
+## Relationship to the confirmatory analysis
+
+SNR and scenario health are **descriptive reporting**, computed from the same
+`paired` / `per_scenario` data the frozen McNemar analysis already uses. They
+add no hypothesis and change no gate: H1/H2 remain exactly as pre-registered in
+amendment-1. They exist so a reader — or a hostile reviewer — can judge for
+themselves whether a reported gap is signal or noise, and how much of the
+roster is actually pulling its weight.
diff --git a/decisiongrounding/tests/test_dashboard.py b/decisiongrounding/tests/test_dashboard.py
index 9844c9f..60d21c8 100644
--- a/decisiongrounding/tests/test_dashboard.py
+++ b/decisiongrounding/tests/test_dashboard.py
@@ -64,12 +64,33 @@ def test_build_dashboard_renders_all_sections():
out = build_dashboard(_run(), _dataset())
_assert_html(out)
for marker in ("Decision Grounding Bench", "Leaderboard", "Adherence vs N",
- "rac vs naive RAG", "Scenarios", "Reproduce", " skipped
+ )
+ assert set(ds["paired"]) == {"context_dump_vs_naive_rag", "context_dump_vs_no_grounding"}
+ for series in ds["paired"].values():
+ assert [e["N"] for e in series] == [3, 6]
+
+
def test_merge_seed_datasets_augment_equals_fresh_run():
sc = load_scenarios(_REAL)
arms, ns = ("context_dump", "naive_rag"), (3, 6)
diff --git a/decisiongrounding/tests/test_report_snr.py b/decisiongrounding/tests/test_report_snr.py
new file mode 100644
index 0000000..a01ebe0
--- /dev/null
+++ b/decisiongrounding/tests/test_report_snr.py
@@ -0,0 +1,60 @@
+"""report.py signal-to-noise + scenario-health sections."""
+
+from scripts.report import _scenario_health_section, _snr_section
+
+
+def _paired_dataset(n_seeds=3):
+ return {
+ "n_seeds": n_seeds,
+ "paired": {
+ "rac_vs_naive_rag": [
+ {"N": 10, "diff_mean": 0.02, "diff_std": 0.2, "diff_ci": [0, 0], "n": n_seeds, "values": []},
+ {"N": 300, "diff_mean": 0.40, "diff_std": 0.1, "diff_ci": [0, 0], "n": n_seeds, "values": []},
+ ],
+ "rac_vs_no_grounding": [
+ {"N": 300, "diff_mean": 0.90, "diff_std": 0.05, "diff_ci": [0, 0], "n": n_seeds, "values": []},
+ ],
+ },
+ }
+
+
+def test_snr_section_renders_both_contrasts_and_flags_noise():
+ out = "\n".join(_snr_section(_paired_dataset()))
+ assert "Signal-to-noise" in out
+ assert "`rac` vs `naive_rag`" in out and "`rac` vs `no_grounding`" in out
+ assert "4.00" in out # 0.40 / 0.10
+ assert "0.10*" in out # 0.02 / 0.20 -> noise-dominated, starred
+ assert "18.00" in out # 0.90 / 0.05
+
+
+def test_snr_section_single_seed_says_not_estimable():
+ out = "\n".join(_snr_section(_paired_dataset(n_seeds=1)))
+ assert "not estimable" in out
+ assert "4.00" not in out
+
+
+def test_snr_section_absent_without_paired():
+ assert _snr_section({"n_seeds": 3}) == []
+
+
+def test_scenario_health_section_summary_and_flags():
+ ds = {"per_scenario": {
+ "context_dump": {"good": [{"N": 300, "adherent": 1.0}],
+ "broke": [{"N": 300, "adherent": 0.0}],
+ "contam": [{"N": 300, "adherent": 1.0}]},
+ "naive_rag": {"good": [{"N": 300, "adherent": 0.3}],
+ "broke": [{"N": 300, "adherent": 0.0}],
+ "contam": [{"N": 300, "adherent": 1.0}]},
+ "no_grounding": {"good": [{"N": 300, "adherent": 0.0}],
+ "broke": [{"N": 300, "adherent": 0.0}],
+ "contam": [{"N": 300, "adherent": 1.0}]},
+ }}
+ out = "\n".join(_scenario_health_section(ds))
+ assert "Scenario health" in out
+ assert "1 discriminating" in out and "1 broken" in out and "1 contaminated" in out
+ assert "`broke`" in out and "broken" in out
+ assert "`contam`" in out
+
+
+def test_scenario_health_absent_without_per_scenario():
+ assert _scenario_health_section({}) == []
diff --git a/decisiongrounding/tests/test_scenario_health.py b/decisiongrounding/tests/test_scenario_health.py
new file mode 100644
index 0000000..f5549fe
--- /dev/null
+++ b/decisiongrounding/tests/test_scenario_health.py
@@ -0,0 +1,85 @@
+"""scenario_health: per-scenario discrimination/validity classification."""
+
+from scoring.health import scenario_health
+
+
+def _rec(n, adherent):
+ return {"N": n, "adherent": adherent}
+
+
+def _dataset(per_scenario):
+ return {"per_scenario": per_scenario}
+
+
+def test_discriminating_scenario():
+ # ceiling adheres, floor fails, arms separate.
+ ds = _dataset({
+ "context_dump": {"s": [_rec(10, 1.0), _rec(300, 1.0)]},
+ "naive_rag": {"s": [_rec(10, 1.0), _rec(300, 0.0)]},
+ "no_grounding": {"s": [_rec(10, 0.0), _rec(300, 0.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["counts"]["discriminating"] == 1
+ row = h["scenarios"][0]
+ assert row["class"] == "discriminating" and row["separates"] is True
+ assert row["max_gap"] == 1.0
+
+
+def test_broken_scenario_ceiling_never_adheres():
+ ds = _dataset({
+ "context_dump": {"s": [_rec(10, 0.0), _rec(300, 0.0)]},
+ "no_grounding": {"s": [_rec(10, 0.0), _rec(300, 0.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["scenarios"][0]["class"] == "broken"
+ assert h["counts"]["broken"] == 1
+
+
+def test_contaminated_scenario_floor_adheres():
+ # ceiling adheres AND floor also adheres -> parametric memory suffices.
+ ds = _dataset({
+ "context_dump": {"s": [_rec(10, 1.0)]},
+ "no_grounding": {"s": [_rec(10, 1.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["scenarios"][0]["class"] == "contaminated"
+
+
+def test_tie_scenario_no_separation():
+ # Ceiling adheres, no floor arm to separate from, and the present arms
+ # never differ -> no between-arm signal.
+ ds = _dataset({
+ "context_dump": {"s": [_rec(10, 1.0), _rec(300, 1.0)]},
+ "naive_rag": {"s": [_rec(10, 1.0), _rec(300, 1.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["scenarios"][0]["class"] == "tie"
+ assert h["scenarios"][0]["separates"] is False
+
+
+def test_unknown_without_control_arms():
+ ds = _dataset({
+ "rac": {"s": [_rec(10, 1.0)]},
+ "naive_rag": {"s": [_rec(10, 0.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["scenarios"][0]["class"] == "unknown"
+ assert h["controls"] == {"ceiling": False, "floor": False}
+
+
+def test_multi_seed_fractions_and_summary_counts():
+ ds = _dataset({
+ "context_dump": {"good": [_rec(300, 1.0)], "bad": [_rec(300, 0.0)]},
+ "naive_rag": {"good": [_rec(300, 0.4)], "bad": [_rec(300, 0.0)]},
+ "no_grounding": {"good": [_rec(300, 0.0)], "bad": [_rec(300, 0.0)]},
+ })
+ h = scenario_health(ds)
+ assert h["total"] == 2
+ classes = {s["scenario_id"]: s["class"] for s in h["scenarios"]}
+ assert classes["good"] == "discriminating" # ceiling 1.0, floor 0, gap 0.6
+ assert classes["bad"] == "broken" # ceiling never adheres
+
+
+def test_legacy_dataset_without_per_scenario():
+ h = scenario_health({})
+ assert h["total"] == 0 and h["scenarios"] == []
diff --git a/decisiongrounding/tests/test_snr.py b/decisiongrounding/tests/test_snr.py
new file mode 100644
index 0000000..d9e7d46
--- /dev/null
+++ b/decisiongrounding/tests/test_snr.py
@@ -0,0 +1,60 @@
+"""signal_to_noise: SNR = |paired diff_mean| / across-seed diff_std, honestly
+gated on >=2 seeds."""
+
+from scoring.snr import signal_to_noise
+
+
+def _ds(n_seeds, series, key="rac_vs_naive_rag"):
+ return {"n_seeds": n_seeds, "paired": {key: series}}
+
+
+def _e(N, diff_mean, diff_std):
+ return {"N": N, "diff_mean": diff_mean, "diff_std": diff_std,
+ "diff_ci": [0, 0], "n": 3, "values": []}
+
+
+def test_snr_is_signal_over_noise():
+ ds = _ds(3, [_e(300, 0.40, 0.10)])
+ rec = signal_to_noise(ds)["pairs"]["rac_vs_naive_rag"]["by_n"][0]
+ assert rec["signal"] == 0.40 and rec["noise"] == 0.10
+ assert rec["snr"] == 4.0 and rec["noise_dominated"] is False and rec["flag"] is None
+
+
+def test_noise_dominated_when_snr_below_one():
+ rec = signal_to_noise(_ds(3, [_e(300, 0.05, 0.20)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0]
+ assert rec["snr"] == 0.25 and rec["noise_dominated"] is True
+
+
+def test_single_seed_is_not_estimable():
+ rec = signal_to_noise(_ds(1, [_e(300, 0.40, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0]
+ assert rec["snr"] is None and rec["flag"] == "noise_not_estimable"
+ assert rec["noise_dominated"] is False
+
+
+def test_zero_noise_nonzero_signal_is_clean_separation():
+ rec = signal_to_noise(_ds(4, [_e(300, 0.30, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0]
+ assert rec["snr"] is None and rec["clean_separation"] is True
+ assert rec["flag"] == "zero_noise_clean_separation"
+
+
+def test_zero_noise_zero_signal_is_no_effect():
+ rec = signal_to_noise(_ds(4, [_e(300, 0.0, 0.0)]))["pairs"]["rac_vs_naive_rag"]["by_n"][0]
+ assert rec["snr"] is None and rec["flag"] == "no_effect"
+
+
+def test_headline_taken_at_largest_n():
+ ds = _ds(3, [_e(10, 0.02, 0.2), _e(300, 0.5, 0.1)])
+ pair = signal_to_noise(ds)["pairs"]["rac_vs_naive_rag"]
+ assert pair["headline_N"] == 300 and pair["headline"]["snr"] == 5.0
+
+
+def test_multiple_contrasts_and_missing_paired():
+ ds = {"n_seeds": 3, "paired": {
+ "rac_vs_naive_rag": [_e(300, 0.4, 0.1)],
+ "rac_vs_no_grounding": [_e(300, 0.9, 0.05)],
+ }}
+ out = signal_to_noise(ds)
+ assert set(out["pairs"]) == {"rac_vs_naive_rag", "rac_vs_no_grounding"}
+ assert out["pairs"]["rac_vs_no_grounding"]["headline"]["snr"] == 18.0
+ # legacy dataset with no paired block -> empty, not an error
+ assert signal_to_noise({"n_seeds": 1})["pairs"] == {}