From 5816531414dab6d3a643931fd567eec70b9d5f90 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 11:12:49 -0400 Subject: [PATCH 001/245] Flip PPI t-interval/logit-t wrappers to power-tuning by default _ppi_single_t_interval/_ppi_paired_t_interval/_ppi_single_logit_t/ _ppi_paired_logit_t hardcoded power_tune=False with no way to override it, unlike every other paired/single PPI wrapper in this module (Tango and the joint bootstrap were both flipped to power_tune=True earlier this week). Since PPI_AUTO_METHOD_TABLE routes continuous/likert (bounded_01) and unbounded numeric data to these exact methods under method="auto", compare() was silently using un-tuned (lambda=1) PPI for that entire data-kind branch. Found via simulations/harness/cases/compare_e2e.py's new power-comparison row: continuous/likert PPI power was landing BELOW both the raw (no-PPI) estimator and the human-labeled-subset-only baseline at every (k, N, frac) cell checked, sometimes by 40-60 points. Reproduced directly against es.compare() outside the harness (bypassing all harness scoring/aggregation code) to confirm it wasn't a harness bug: with power_tune=False, PPI's CI was 3.5x wider than the raw estimator's and non-significant (p=0.59) where raw scores were highly significant (p<0.001) on identical data. Forcing power_tune=True on the same data narrows the CI 5x, beats the raw estimator, and lambda correctly shrinks to ~0.2 instead of being pinned at 1. Exposes power_tune as a real parameter (matching the rest of the module) rather than an inline literal, and flips the default to True. Updated the two TestWrapperEquivalence tests in test_ppi_ci_methods.py that compared the wrapper's old implicit default against an explicit power_tune=False call to _analytic_mean_correct -- pinned power_tune=False on both sides so they still test genuine wrapper-delegation equivalence rather than reverting this fix. Full regression suite (test_ppi_ci_methods, test_alignment, test_ppi_core, test_ppi_corrections, test_compound_ppi_fwer): 450/450 passed. Co-Authored-By: Claude Sonnet 5 --- evalstats/tests/__init__.py | 36 ++++++++++++++++++++++++++++-------- tests/test_ppi_ci_methods.py | 4 ++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/evalstats/tests/__init__.py b/evalstats/tests/__init__.py index 9faa277..b660d50 100644 --- a/evalstats/tests/__init__.py +++ b/evalstats/tests/__init__.py @@ -2389,7 +2389,7 @@ def _svar(x: np.ndarray) -> float: ) -def _ppi_single_t_interval(a: np.ndarray, a_lab: np.ndarray, alpha: float): +def _ppi_single_t_interval(a: np.ndarray, a_lab: np.ndarray, alpha: float, power_tune: bool = True): """PPI correction for a single-sample mean estimand ``mean(a)`` on an unbounded numeric scale, via the closed-form (no-bootstrap) analytic construction -- evalstats.ppi._analytic_mean_correct -- applied at @@ -2403,6 +2403,10 @@ def _ppi_single_t_interval(a: np.ndarray, a_lab: np.ndarray, alpha: float): A position is included in the labeled set only when ``a_lab[i]`` is non-NaN. + + ``power_tune`` mirrors :func:`evalstats.ppi.correct`'s parameter of the + same name (see its docstring's "PPI++ power-tuning" section) -- default + True, matching every other paired/single PPI wrapper in this module. """ from evalstats.ppi import _analytic_mean_correct @@ -2415,10 +2419,12 @@ def _ppi_single_t_interval(a: np.ndarray, a_lab: np.ndarray, alpha: float): values_lab_llm = all_values[mask] values_lab_true = np.asarray(a_lab, dtype=float)[mask] - return _analytic_mean_correct(values_lab_true, values_lab_llm, values_unlab, alpha, power_tune=False) + return _analytic_mean_correct(values_lab_true, values_lab_llm, values_unlab, alpha, power_tune=power_tune) -def _ppi_paired_t_interval(a: np.ndarray, b: np.ndarray, a_lab: np.ndarray, b_lab: np.ndarray, alpha: float): +def _ppi_paired_t_interval( + a: np.ndarray, b: np.ndarray, a_lab: np.ndarray, b_lab: np.ndarray, alpha: float, power_tune: bool = True, +): """PPI correction for a paired mean-difference estimand ``mean(a_i - b_i)`` on an unbounded numeric scale, via the closed-form (no- bootstrap) analytic construction -- the closed-form analogue of @@ -2430,6 +2436,10 @@ def _ppi_paired_t_interval(a: np.ndarray, b: np.ndarray, a_lab: np.ndarray, b_la Pairing is by array position, matching _ppi_paired_bootstrap_t; a position is included in the labeled set only when *both* ``a_lab[i]`` and ``b_lab[i]`` are non-NaN. + + ``power_tune`` mirrors :func:`evalstats.ppi.correct`'s parameter of the + same name -- default True, matching every other paired/single PPI + wrapper in this module. """ from evalstats.ppi import _analytic_mean_correct @@ -2444,10 +2454,12 @@ def _ppi_paired_t_interval(a: np.ndarray, b: np.ndarray, a_lab: np.ndarray, b_la diffs_lab_llm = all_diffs[mask] diffs_lab_true = (a_lab - b_lab)[mask] - return _analytic_mean_correct(diffs_lab_true, diffs_lab_llm, diffs_unlab, alpha, power_tune=False) + return _analytic_mean_correct(diffs_lab_true, diffs_lab_llm, diffs_unlab, alpha, power_tune=power_tune) -def _ppi_single_logit_t(a: np.ndarray, a_lab: np.ndarray, alpha: float, lo: float = 0.0, hi: float = 1.0): +def _ppi_single_logit_t( + a: np.ndarray, a_lab: np.ndarray, alpha: float, lo: float = 0.0, hi: float = 1.0, power_tune: bool = True, +): """PPI correction for a single-sample mean estimand on a [lo, hi]- bounded numeric scale (continuous/likert/grades), via the closed-form logit-t construction -- evalstats.ppi._analytic_logit_t_correct. @@ -2465,6 +2477,10 @@ def _ppi_single_logit_t(a: np.ndarray, a_lab: np.ndarray, alpha: float, lo: floa A position is included in the labeled set only when ``a_lab[i]`` is non-NaN. + + ``power_tune`` mirrors :func:`evalstats.ppi.correct`'s parameter of the + same name -- default True, matching every other paired/single PPI + wrapper in this module. """ from evalstats.ppi import _analytic_logit_t_correct @@ -2478,13 +2494,13 @@ def _ppi_single_logit_t(a: np.ndarray, a_lab: np.ndarray, alpha: float, lo: floa values_lab_true = np.asarray(a_lab, dtype=float)[mask] return _analytic_logit_t_correct( - values_lab_true, values_lab_llm, values_unlab, alpha, power_tune=False, lo=lo, hi=hi, + values_lab_true, values_lab_llm, values_unlab, alpha, power_tune=power_tune, lo=lo, hi=hi, ) def _ppi_paired_logit_t( a: np.ndarray, b: np.ndarray, a_lab: np.ndarray, b_lab: np.ndarray, alpha: float, - lo: float = 0.0, hi: float = 1.0, + lo: float = 0.0, hi: float = 1.0, power_tune: bool = True, ): """PPI correction for a paired mean-difference estimand on a [lo, hi]- bounded numeric scale, via the closed-form logit-t construction. A @@ -2497,6 +2513,10 @@ def _ppi_paired_logit_t( Pairing is by array position, matching _ppi_paired_t_interval; a position is included in the labeled set only when *both* ``a_lab[i]`` and ``b_lab[i]`` are non-NaN. + + ``power_tune`` mirrors :func:`evalstats.ppi.correct`'s parameter of the + same name -- default True, matching every other paired/single PPI + wrapper in this module. """ from evalstats.ppi import _analytic_logit_t_correct @@ -2515,7 +2535,7 @@ def _ppi_paired_logit_t( diff_lo, diff_hi = -diff_span, diff_span return _analytic_logit_t_correct( - diffs_lab_true, diffs_lab_llm, diffs_unlab, alpha, power_tune=False, lo=diff_lo, hi=diff_hi, + diffs_lab_true, diffs_lab_llm, diffs_unlab, alpha, power_tune=power_tune, lo=diff_lo, hi=diff_hi, ) diff --git a/tests/test_ppi_ci_methods.py b/tests/test_ppi_ci_methods.py index 4a06fe8..50d3a77 100644 --- a/tests/test_ppi_ci_methods.py +++ b/tests/test_ppi_ci_methods.py @@ -85,7 +85,7 @@ def test_single_t_interval_matches_analytic_mean_correct(self): a_lab = _split_labels(rng, truth, llm, n_lab=30) mask = ~np.isnan(a_lab) - r = _ppi_single_t_interval(llm, a_lab, alpha=0.05) + r = _ppi_single_t_interval(llm, a_lab, alpha=0.05, power_tune=False) expected = _analytic_mean_correct( np.asarray(a_lab, dtype=float)[mask], llm[mask], llm[~mask], alpha=0.05, power_tune=False, ) @@ -106,7 +106,7 @@ def test_paired_t_interval_matches_analytic_mean_correct(self): b_lab[~np.isnan(a_lab)] = truth_b[~np.isnan(a_lab)] mask = ~np.isnan(a_lab) & ~np.isnan(b_lab) - r = _ppi_paired_t_interval(llm_a, llm_b, a_lab, b_lab, alpha=0.05) + r = _ppi_paired_t_interval(llm_a, llm_b, a_lab, b_lab, alpha=0.05, power_tune=False) diffs = llm_a - llm_b expected = _analytic_mean_correct( (a_lab - b_lab)[mask], diffs[mask], diffs[~mask], alpha=0.05, power_tune=False, From 91aaee5f6b0bdeb88118aac409bb5a0fa7f2188f Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 13:45:07 -0400 Subject: [PATCH 002/245] Fix paired-diff CI collapse on rounded/quantized data (likert small-N, continuous ties) Rounding a continuous latent value before differencing two highly correlated (shared-item) arms can cancel most of the real per-item diff variance: most items round identically in both arms (diff=0), and only boundary-adjacent items show a nonzero diff. At small N it's plausible no sampled items are boundary-adjacent, collapsing the sample's diff variance to ~0 and making any variance-based CI catastrophically overconfident. Confirmed via simulations/investigate_likert_family_wise_smalln.py: logit_t's family-wise coverage (Sidak-widened, k=10) was 14.5% at n=10 vs. 95% nominal. Fixes this via dithering: logit_t_dither/smooth_bootstrap_dither add independent jitter to each arm's raw values before differencing, then clip back to scale. The jitter width is auto-detected per rep from the data's own quantization grid (_detect_dither_halfwidth in ci_paired.py) rather than hardcoded -- a fixed +-0.5 (right for likert's integer rounding) was tried on continuous's [0,1] scale too and was wrong there (half the entire range), causing a boundary-clipping bias that got worse with N rather than converging. Data-driven detection returns 0.0 (no jitter) on genuinely continuous data and the correct grid width on anything discretized, including "continuous"-labeled data that's secretly coarse (e.g. a judge emitting only a handful of distinct values). Also: - Wire an interval-score bundle into compare_e2e.py's oracle/subset estimators, matching the rest of the harness's interval_score convention, and fix a continuous-only degenerate-zero-variance bug in those same estimators (icc=1.0's deterministic per-item shift gives exactly-constant diffs pre-rounding) via a small oracle noise floor. - ci_paired.py's LaTeX overall-summary table: split "numeric" into separate bin/cont/lik rows (was already binary/numeric; likert and continuous behave too differently to average together), and sort rows by eval-type block (all bin, then cont, then lik) instead of interleaving by method, with a midrule between blocks. - investigate_continuous_family_wise_smalln.py: standalone confirmation that continuous data does NOT suffer the same family-wise small-N collapse as likert (no rounding to cancel against). Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/ci_paired.py | 225 ++++++++++-- simulations/harness/cases/compare_e2e.py | 91 ++++- simulations/harness/methods.py | 59 +++- ...vestigate_continuous_family_wise_smalln.py | 235 +++++++++++++ .../investigate_likert_family_wise_smalln.py | 328 ++++++++++++++++++ 5 files changed, 900 insertions(+), 38 deletions(-) create mode 100644 simulations/investigate_continuous_family_wise_smalln.py create mode 100644 simulations/investigate_likert_family_wise_smalln.py diff --git a/simulations/harness/cases/ci_paired.py b/simulations/harness/cases/ci_paired.py index 97e3d27..d62247d 100644 --- a/simulations/harness/cases/ci_paired.py +++ b/simulations/harness/cases/ci_paired.py @@ -96,7 +96,7 @@ ) from evalstats.core.stats_utils import interval_score, rescaled_ci -from ..latex_tables import booktabs_table, escape_latex, eval_type_label, eval_type_group +from ..latex_tables import booktabs_table, escape_latex from ..scenarios import CIPairSource, EVAL_TYPES, EVAL_TYPE_SCALE_BOUNDS from ..scenarios.synthetic import ( SCENARIO_SUITES, @@ -118,6 +118,9 @@ BAYES_PAIR_PAIRED, WALD_PAIR_INDEP, PAIRWISE_EXTRA_METHODS, + LOGIT_T_DITHER, + SMOOTH_BOOTSTRAP_DITHER, + DITHER_EXTRA_METHODS, PAIR_DIFF_NESTED_METHODS, BOOTSTRAP_DIFF_NESTED, BAYES_DIFF_NESTED, @@ -318,6 +321,38 @@ def _pairwise_ci( return float(np.percentile(boot_stats, 100 * alpha / 2)), float(np.percentile(boot_stats, 100 * (1 - alpha / 2))) +def _detect_dither_halfwidth(pooled: np.ndarray) -> float: + """Auto-detect a rounding/quantization grid step from pooled raw arm + values (both arms, one rep) and return half that step -- the dither + half-width needed to reconstruct the pre-quantization variance that a + paired diff of two highly-correlated arms can lose to rounding + cancellation (see LOGIT_T_DITHER's docstring). Data-driven rather than + eval_type-driven: labeled "continuous" data that's actually coarse + (e.g. a judge that only emits a handful of distinct values) gets + detected and dithered correctly; genuinely continuous data (no + recurring gap) returns 0.0, meaning "don't dither" -- unlike a + hardcoded width, this can't apply a jitter mismatched to the data's + real resolution and reintroduce the boundary-clipping bias that broke + continuous coverage when a flat +-0.5 was tried there (see + add_dither_extras's comment).""" + uniq = np.unique(pooled) + if uniq.size < 3: + return 0.0 + gaps = np.diff(uniq) + gaps = gaps[gaps > 1e-9] + if gaps.size < 2: + return 0.0 + rounded = np.round(gaps, decimals=6) + grid_vals, counts = np.unique(rounded, return_counts=True) + best = np.argmax(counts) + step, count = grid_vals[best], counts[best] + # Require the dominant gap to recur often enough to look like a real + # grid, not coincidental spacing among genuinely continuous draws. + if count < max(3, 0.2 * gaps.size): + return 0.0 + return float(step) / 2.0 + + def _run_cell( source_obj: CIPairSource, n: int, n_reps: int, n_bootstrap: int, bayes_n: int, alpha: float, runs: int, statistic: str, seed, method_names: frozenset[str] | None = None, @@ -341,6 +376,23 @@ def _want(method_name: str) -> bool: active_bootstrap_methods = [m for m in METHODS if _want(m.name)] active_pairwise_extras = [m for m in PAIRWISE_EXTRA_METHODS if _want(m.name)] add_pairwise_extras = statistic == "mean" and source_obj.eval_type != "binary" and bool(active_pairwise_extras) + active_dither_extras = [m for m in DITHER_EXTRA_METHODS if _want(m.name)] + # Non-binary; the actual jitter width is auto-detected per rep from the + # data itself (_detect_dither_halfwidth), not hardcoded. The motivating + # mechanism (rounding-driven diff cancellation between two paired, + # highly-correlated arms) needs a quantization grid to undo -- a fixed + # +-0.5 (right for likert's integer rounding) was tried on continuous's + # [0,1] scale too and was WRONG there (half the entire range), causing + # heavy boundary clipping and a bias that got worse with N (coverage + # 0.936 -> 0.800, n=10 -> n=100, nested screening). Detecting the grid + # from the data instead of assuming one from eval_type fixes that AND + # generalizes: labeled-"continuous" data that's actually coarse (a judge + # emitting only a handful of distinct values) gets dithered correctly, + # while genuinely continuous data detects no grid and the dither variant + # safely reduces to its base method (identical CI, no bias introduced). + add_dither_extras = ( + statistic == "mean" and source_obj.eval_type != "binary" and bool(active_dither_extras) + ) add_newcombe = source_obj.eval_type == "binary" and statistic == "mean" and _want(NEWCOMBE.name) add_tango = source_obj.eval_type == "binary" and statistic == "mean" and _want(TANGO.name) add_tango_scc = source_obj.eval_type == "binary" and statistic == "mean" and _want(TANGO_SCC.name) @@ -351,6 +403,8 @@ def _want(method_name: str) -> bool: active_methods = list(active_bootstrap_methods) if add_pairwise_extras: active_methods += active_pairwise_extras + if add_dither_extras: + active_methods += active_dither_extras if add_newcombe: active_methods.append(NEWCOMBE) if add_tango: @@ -425,6 +479,44 @@ def _record(method, ci_low: float, ci_high: float) -> None: total_t_sq[method] += _el * _el _record(method, ci_low, ci_high) + if add_dither_extras: + # Independent U(-half, +half) jitter per arm (not on the diff + # directly) then clip back to the scale, where half is detected + # per rep from the data's own quantization grid (0.0 -- i.e. no + # jitter -- if none is detected) -- see LOGIT_T_DITHER's + # docstring for why this specifically targets the paired-diff + # rounding-cancellation pathology, not just "add some noise." + _scale_lo, _scale_hi = EVAL_TYPE_SCALE_BOUNDS[source_obj.eval_type] + _half = _detect_dither_halfwidth(np.concatenate([a.ravel(), b.ravel()])) + if _half > 0: + a_dither = np.clip(a + rng.uniform(-_half, _half, size=a.shape), _scale_lo, _scale_hi) + b_dither = np.clip(b + rng.uniform(-_half, _half, size=b.shape), _scale_lo, _scale_hi) + else: + a_dither, b_dither = a, b + pair_diffs_dither = a_dither.mean(axis=1) - b_dither.mean(axis=1) + obs_dither = float(np.mean(pair_diffs_dither)) + diff_span_dither = _scale_hi - _scale_lo + diff_lo_dither, diff_hi_dither = -diff_span_dither, diff_span_dither + for method in active_dither_extras: + _t0 = time.perf_counter() + try: + if method is LOGIT_T_DITHER: + ci_low, ci_high = rescaled_ci( + logit_t_ci_1d, pair_diffs_dither, alpha, diff_lo_dither, diff_hi_dither, + ) + else: # SMOOTH_BOOTSTRAP_DITHER + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + boot_stats = smooth_bootstrap_means_1d(pair_diffs_dither, n_bootstrap, rng, statistic=statistic) + ci_low = float(np.percentile(boot_stats, 100 * alpha / 2)) + ci_high = float(np.percentile(boot_stats, 100 * (1 - alpha / 2))) + except Exception: + ci_low = ci_high = obs_dither + _el = time.perf_counter() - _t0 + total_t[method] += _el + total_t_sq[method] += _el * _el + _record(method, ci_low, ci_high) + if add_newcombe: _t0 = time.perf_counter() try: @@ -642,6 +734,7 @@ def _want(method_name: str) -> bool: active_methods += [m for m in BINARY_PAIR_NESTED_METHODS if _want(m.name)] else: active_methods += [m for m in (LOGIT_T, NIG, EL) if _want(m.name)] + active_methods += [m for m in DITHER_EXTRA_METHODS if _want(m.name)] covered: dict = {m: 0 for m in active_methods} total_w: dict = {m: 0.0 for m in active_methods} @@ -718,6 +811,46 @@ def _record(method, ci_low: float, ci_high: float) -> None: total_t_sq[method] += _el * _el _record(method, ci_low, ci_high) + # -- logit_t_dither/smooth_bootstrap_dither on cell-mean diffs, + # non-binary -- same fix as _run_cell's flat-mode add_dither_extras + # block, see LOGIT_T_DITHER's and _detect_dither_halfwidth's + # docstrings: the jitter width is auto-detected per rep from the + # data's own quantization grid (0.0, i.e. no jitter, if none is + # found), not a hardcoded +-0.5 -- a fixed width calibrated to + # likert's integer rounding was tried on continuous's own scale too + # and caused a bias that got WORSE with N (coverage 0.936 -> 0.800, + # n=10 -> n=100). Like logit_t/nig/el above, these have no full-N-x-R + # nested variant -- they operate on the same cell-mean-reduced + # diffs, just computed from independently dithered a/b first. + if not is_binary: + _half = _detect_dither_halfwidth(np.concatenate([a.ravel(), b.ravel()])) + if _half > 0: + a_dither = np.clip(a + rng.uniform(-_half, _half, size=a.shape), _scale_lo, _scale_hi) + b_dither = np.clip(b + rng.uniform(-_half, _half, size=b.shape), _scale_lo, _scale_hi) + else: + a_dither, b_dither = a, b + cell_diffs_dither = a_dither.mean(axis=1) - b_dither.mean(axis=1) + obs_diff_dither = float(np.mean(cell_diffs_dither)) + for method in (LOGIT_T_DITHER, SMOOTH_BOOTSTRAP_DITHER): + if not _want(method.name): + continue + _t0 = time.perf_counter() + try: + if method is LOGIT_T_DITHER: + ci_low, ci_high = rescaled_ci(logit_t_ci_1d, cell_diffs_dither, alpha, diff_lo, diff_hi) + else: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + boot_stats = smooth_bootstrap_means_1d(cell_diffs_dither, n_bootstrap, rng, statistic="mean") + ci_low = float(np.percentile(boot_stats, 100 * alpha / 2)) + ci_high = float(np.percentile(boot_stats, 100 * (1 - alpha / 2))) + except Exception: + ci_low = ci_high = obs_diff_dither + _el = time.perf_counter() - _t0 + total_t[method] += _el + total_t_sq[method] += _el * _el + _record(method, ci_low, ci_high) + # -- Nested pairwise diff methods (full N x R pair matrices) -- if run_bootstrap: for method, fn in [ @@ -1027,10 +1160,18 @@ def mean_cov(et, m, n): row += " " + (" " * 7 if np.isnan(cov) else f"{cov:.3f}{_cov_marker(cov, target)}".ljust(8)) print(row) - # Split into three OVERALL SUMMARY tables -- binary, continuous [0,1], and - # numeric (likert + grades averaged together) -- since these data types - # are answered by very different method families and a single pooled - # table obscures which methods actually perform best for which type. + # Split into per-eval-type OVERALL SUMMARY tables -- binary, continuous + # [0,1], likert, and grades all separate -- since these data types are + # answered by very different method families/scales and a pooled table + # obscures which methods actually perform best for which type. Likert + # and grades used to be pooled together as one "numeric" table; kept + # separate now since likert was found (2026-08-11) to have materially + # different small-N paired-diff behavior than continuous/grades (see + # LOGIT_T_DITHER's docstring) -- pooling would hide exactly that, + # and concretely mixes likert's 1-5-scale widths with grades' 0-100- + # scale widths, an even more obviously incomparable pair. Official runs + # never include grades (see official_args()), so in practice this only + # ever prints 3 tables there. sizes_present = sorted({r.n for r in non_null}) _print_overall_summary_table( "OVERALL SUMMARY -- BINARY (averaged across sources)", @@ -1041,8 +1182,12 @@ def mean_cov(et, m, n): ["continuous"], non_null, agg, agg_counts, target, sizes_present, ) _print_overall_summary_table( - "OVERALL SUMMARY -- NUMERIC: LIKERT + GRADES (averaged across sources)", - ["likert", "grades"], non_null, agg, agg_counts, target, sizes_present, + "OVERALL SUMMARY -- LIKERT (averaged across sources)", + ["likert"], non_null, agg, agg_counts, target, sizes_present, + ) + _print_overall_summary_table( + "OVERALL SUMMARY -- GRADES (averaged across sources)", + ["grades"], non_null, agg, agg_counts, target, sizes_present, ) null_results = [r for r in results if r.is_null] @@ -1074,6 +1219,22 @@ def mean_cov(et, m, n): print() +def _report_eval_type_group(et: str) -> str: + """Short per-eval-type group label for ci_paired.py's own summary/LaTeX + tables -- FINER than the shared latex_tables.eval_type_group (which only + splits binary vs. a single "numeric" bucket covering continuous+likert+ + grades together). Likert and continuous were found (2026-08-11, + LOGIT_T_DITHER's investigation) to have materially different small-N + paired-diff behavior -- pooling them under one "numeric" Cov/Width/Score + number hides exactly the distinction that investigation surfaced (and + concretely mixes likert's 1-5-scale widths with grades' 0-100-scale + widths, an even more obviously incomparable pair). Local to this file + rather than changed in latex_tables.py, since that utility is shared + with cases/ci_single.py, which hasn't been checked for the same + per-numeric-type distinction and shouldn't be changed without that.""" + return {"binary": "bin", "continuous": "cont", "likert": "lik", "grades": "grades"}.get(et, et) + + def latex_overall_summary(results: list[SimResult], alpha: float, n_reps: int) -> str: """LaTeX booktabs version of print_report's OVERALL SUMMARY block (non-null rows only), plus one coverage column per sample size actually @@ -1081,44 +1242,47 @@ def latex_overall_summary(results: list[SimResult], alpha: float, n_reps: int) - collapses across n and can hide miscalibration that only shows up at small or large sample sizes. - Methods that ran on both eval-type groups (binary and numeric) get two - rows -- " (binary)" and " (numeric)" -- each computed - from only that group's data, rather than one row averaging across both. - Averaging Cov/Width/Score across binary and numeric data mixes two - different scales/regimes into a number that isn't comparable to any - group-pure method's row; an "all" Eval-types value was a symptom of - exactly this, not a meaningful category of its own, so no row's Eval - types column is ever "all" here. Mirrors cases/ci_single.py's identical - fix -- see that module's latex_overall_summary for the full writeup. - """ + Methods that ran on more than one eval type get one row PER eval type -- + " (bin)"/" (cont)"/" (lik)" -- each computed + from only that type's own data, rather than one row averaging across + incomparable scales/regimes (see _report_eval_type_group's docstring for + why this is now a 3-way split, not the 2-way binary/numeric split an + earlier version of this function used).""" target = 1.0 - alpha non_null = [r for r in results if not r.is_null] - eval_types_present = {et for et in EVAL_TYPES if any(r.eval_type == et for r in non_null)} method_labels = [m.name for m in order_present_methods({r.method for r in non_null})] sizes_present = sorted({r.n for r in non_null}) agg: dict[tuple, list[tuple[float, float, float]]] = defaultdict(list) agg_counts: dict[tuple, tuple[int, int]] = defaultdict(lambda: (0, 0)) - method_group_types: dict[tuple[str, str], set[str]] = defaultdict(set) for r in non_null: - g = eval_type_group(r.eval_type) + g = _report_eval_type_group(r.eval_type) cov = r.covered / r.n_reps width = r.total_width / r.n_reps score = r.total_score / r.n_reps agg[(g, r.method, r.n)].append((cov, width, score)) c_prev, t_prev = agg_counts[(g, r.method, r.n)] agg_counts[(g, r.method, r.n)] = (c_prev + r.covered, t_prev + r.n_reps) - method_group_types[(r.method, g)].add(r.eval_type) method_groups: dict[str, set[str]] = defaultdict(set) for (g, m, _n) in agg: method_groups[m].add(g) + group_order = ["bin", "cont", "lik", "grades"] + groups_present = sorted( + {g for methods in method_groups.values() for g in methods}, + key=lambda g: group_order.index(g) if g in group_order else len(group_order), + ) + rows = [] - for m in method_labels: - groups = sorted(method_groups[m]) - multi_group = len(groups) > 1 - for g in groups: + rule_before = set() + for g in groups_present: + if rows: + rule_before.add(len(rows)) + for m in method_labels: + if g not in method_groups[m]: + continue + multi_group = len(method_groups[m]) > 1 per_n_vals: dict[tuple[str, int], list[tuple[float, float, float]]] = defaultdict(list) all_counts: dict[str, tuple[int, int]] = defaultdict(lambda: (0, 0)) per_n_counts: dict[tuple[str, int], tuple[int, int]] = defaultdict(lambda: (0, 0)) @@ -1135,10 +1299,9 @@ def latex_overall_summary(results: list[SimResult], alpha: float, n_reps: int) - c_tot, t_tot = all_counts[m] _, _, lo, hi = _mc_proportion_stats(c_tot, t_tot) avg_ms, se_ms = _time_stats( - [r for r in non_null if r.method == m and eval_type_group(r.eval_type) == g] + [r for r in non_null if r.method == m and _report_eval_type_group(r.eval_type) == g] ) time_str = f"${avg_ms:.3f} \\pm {se_ms:.3f}$" if np.isfinite(avg_ms) else "-" - et_label = eval_type_label(method_group_types[(m, g)], eval_types_present) label = f"{escape_latex(m)} ({g})" if multi_group else escape_latex(m) row = [ label, @@ -1147,7 +1310,7 @@ def latex_overall_summary(results: list[SimResult], alpha: float, n_reps: int) - f"{mw:.4f}" if np.isfinite(mw) else "-", f"{ms:.4f}" if np.isfinite(ms) else "-", time_str, - et_label, + g, ] for n in sizes_present: c_n, t_n = per_n_counts.get((m, n), (0, 0)) @@ -1159,13 +1322,15 @@ def latex_overall_summary(results: list[SimResult], alpha: float, n_reps: int) - caption=( f"ci\\_paired: overall CI coverage summary (nominal {target:.0%}, reps/cell={n_reps}). " "Score is the interval score (width + $\\frac{2}{\\alpha}\\times$miss-distance; lower is better). " - "Methods tested on both binary and numeric data are reported as two rows, one per eval-type " - "group, so no row averages across incomparable scales." + "Methods tested on more than one eval type are reported as one row per type " + "(bin/cont/lik), so no row averages across incomparable scales. Rows are grouped by " + "eval type (all bin, then all cont, then all lik) so methods are comparable within a block." ), label="tab:ci_paired_overall", columns=["Method", "Coverage", "95\\% MC band", "Mean width", "Score", "Time (ms)", "Eval types"] + [f"n={n}" for n in sizes_present], rows=rows, + rule_before=rule_before, ) diff --git a/simulations/harness/cases/compare_e2e.py b/simulations/harness/cases/compare_e2e.py index 83adba1..c481fd2 100644 --- a/simulations/harness/cases/compare_e2e.py +++ b/simulations/harness/cases/compare_e2e.py @@ -71,6 +71,7 @@ import evalstats as es from evalstats.alignment import validate_alignment +from evalstats.core.stats_utils import interval_score from ..latex_tables import booktabs_table, escape_latex from ..scenarios import EVAL_TYPE_SCALE_BOUNDS @@ -137,6 +138,31 @@ def _effect_step_for(eval_type: str, frac: float) -> float: DEFAULT_SIZES = [15, 30, 60, 100, 200, 400, 800] DEFAULT_AGREEMENT_RATE = 0.85 # fixed judge quality -- matches this session's one-off investigations +# Oracle/subset-only reference estimators (see CompareE2EResult) feed TRUTH +# values directly to compare(), with NO noise -- correct in principle (an +# "oracle" IS the ground truth), but a real bug for CONTINUOUS data +# specifically: sample_group_truth's icc=1.0 applies `effects=` as a +# deterministic, per-item-IDENTICAL shift (see that function's own +# docstring: "icc=1.0 means no noise at all"), so for un-rounded continuous +# values the per-item PAIRED DIFFERENCE between any two arms is exactly +# constant across every item (confirmed directly: sample std of diffs was +# 1.1e-17, i.e. zero) -- any variance-based CI (logit_t, smooth_bootstrap, +# and evalstats' own default construction) built from that collapses to a +# near-zero-width interval, giving spuriously ~100% power (a symptom of +# gross overconfidence, not genuine statistical strength) rather than a +# real small-vs-large-N story. NOT an issue for binary (each item's {0,1} +# realization is its own independent Bernoulli draw, not a deterministic +# shift of a shared latent value) or likert (rounding to the integer scale +# breaks the exact constancy for items near a rounding boundary) -- both +# already have genuine non-degenerate per-item diff variance under icc=1.0. +# Fix: apply a SMALL amount of realistic labeler noise (not the LLM judge's +# own DEFAULT_AGREEMENT_RATE-level noise -- an "oracle" should still be +# near-perfect) when building oracle/subset's continuous scores specifically, +# just enough to break the exact-zero-variance degeneracy. Zero-mean +# (Gaussian) for continuous, so truth_means stays the correct, unbiased +# coverage-check reference -- no change needed there. +ORACLE_NOISE_AGREEMENT_RATE = 0.99 + # compare()'s own default is n_bootstrap=10_000 (evalstats/core/router.py's # analyze()) -- this dominates per-call cost far more than n_mc (which floors # at max(n_mc, 1000) for the PPI/Pareto path regardless of what's passed, so @@ -205,6 +231,12 @@ class CompareE2EResult: marginal_covered/marginal_total's coverage. bundle.robustness's per-arm CI is NOT multiplicity-adjusted by k (see CompareE2EResult module notes), so unlike pairwise_covered below this needs no k-based split.""" + marginal_score_sum: float = 0.0 + """Sum of evalstats.core.stats_utils.interval_score(ci_low, ci_high, + true_value, alpha) across the same checks as marginal_total -- the SAME + metric ci_single.py/ci_paired.py report as "Score" (width + (2/alpha) * + miss-distance when uncovered, lower is better), so this is + directly comparable across the harness, not a compare_e2e-only number.""" pairwise_covered: int = 0 pairwise_total: int = 0 """Total pairwise-CI checks across all C(k,2) pairs and all successful @@ -216,6 +248,13 @@ class CompareE2EResult: artifact of FWER widening. k==2 rows report each pair's OWN calibration with no correction confound (Sidak's alpha_adj reduces to plain alpha when there's only 1 pair).""" + pairwise_width_sum: float = 0.0 + pairwise_score_sum: float = 0.0 + """Sum of pairwise CI width / interval_score across the same checks as + pairwise_total. Same k==2-vs-k>2 split applies: k==2 rows give the + uncorrected per-pair width/score baseline; k>2 rows give the FWER-widened + per-pair width/score -- the direct "what does simultaneous protection + cost in width/score" comparison, on the SAME scale ci_paired.py uses.""" family_covered: int = 0 family_total: int = 0 """Family-wise (simultaneous) coverage: a rep counts as 'covered' only if @@ -359,22 +398,28 @@ def _score_bundle(bundle, true_means: np.ndarray, k: int, alpha: float, is_null: marginal_total = marginal_covered = 0 marginal_width_sum = 0.0 + marginal_score_sum = 0.0 for i, lbl in enumerate(labels): lbl_s = str(lbl) ci_lo, ci_hi = rob.ci_low[i], rob.ci_high[i] if np.isfinite(ci_lo) and np.isfinite(ci_hi): marginal_total += 1 marginal_width_sum += ci_hi - ci_lo + marginal_score_sum += interval_score(ci_lo, ci_hi, label_to_true[lbl_s], alpha) if ci_lo <= label_to_true[lbl_s] <= ci_hi: marginal_covered += 1 pairwise_total = pairwise_covered = 0 + pairwise_width_sum = 0.0 + pairwise_score_sum = 0.0 any_sig = False extreme_p = None all_pairs_covered = True for (a, b), pr in bundle.pairwise.results.items(): true_diff = label_to_true[str(a)] - label_to_true[str(b)] pairwise_total += 1 + pairwise_width_sum += pr.ci_high - pr.ci_low + pairwise_score_sum += interval_score(pr.ci_low, pr.ci_high, true_diff, alpha) pair_covered = pr.ci_low <= true_diff <= pr.ci_high if pair_covered: pairwise_covered += 1 @@ -390,8 +435,9 @@ def _score_bundle(bundle, true_means: np.ndarray, k: int, alpha: float, is_null: return dict( marginal_covered=marginal_covered, marginal_total=marginal_total, - marginal_width_sum=marginal_width_sum, + marginal_width_sum=marginal_width_sum, marginal_score_sum=marginal_score_sum, pairwise_covered=pairwise_covered, pairwise_total=pairwise_total, + pairwise_width_sum=pairwise_width_sum, pairwise_score_sum=pairwise_score_sum, family_covered=(1 if all_pairs_covered else 0), family_total=1, any_reject=any_reject, extreme_reject=extreme_reject, ) @@ -498,8 +544,11 @@ def _run_cell( result.marginal_covered += sc["marginal_covered"] result.marginal_total += sc["marginal_total"] result.marginal_width_sum += sc["marginal_width_sum"] + result.marginal_score_sum += sc["marginal_score_sum"] result.pairwise_covered += sc["pairwise_covered"] result.pairwise_total += sc["pairwise_total"] + result.pairwise_width_sum += sc["pairwise_width_sum"] + result.pairwise_score_sum += sc["pairwise_score_sum"] result.family_covered += sc["family_covered"] result.family_total += sc["family_total"] result.any_reject += sc["any_reject"] @@ -513,7 +562,16 @@ def _run_cell( # oracle_n_ok/subset_n_ok, the denominators _aggregate_group uses. if compute_reference and ppi_frac is None: try: - oracle_bundle = _run_truth_only_compare(truth, rng, score_range, n_bootstrap) + # Continuous only: see ORACLE_NOISE_AGREEMENT_RATE's docstring + # -- raw truth has an exactly-zero-variance paired diff under + # icc=1.0's deterministic shift, degenerating any CI built + # from it. Binary/likert don't need this (already + # non-degenerate) and stay on raw truth. + oracle_scores = ( + _apply_judge_noise(truth, eval_type, rng, ORACLE_NOISE_AGREEMENT_RATE) + if eval_type == "continuous" else truth + ) + oracle_bundle = _run_truth_only_compare(oracle_scores, rng, score_range, n_bootstrap) if oracle_bundle is not None: osc = _score_bundle(oracle_bundle, truth_means, k, alpha, is_null) result.oracle_marginal_covered += osc["marginal_covered"] @@ -529,7 +587,12 @@ def _run_cell( pass elif compute_reference: try: - subset_bundle = _run_truth_only_compare(truth[:, labeled_items], rng, score_range, n_bootstrap) + subset_truth = truth[:, labeled_items] + subset_scores = ( + _apply_judge_noise(subset_truth, eval_type, rng, ORACLE_NOISE_AGREEMENT_RATE) + if eval_type == "continuous" else subset_truth + ) + subset_bundle = _run_truth_only_compare(subset_scores, rng, score_range, n_bootstrap) if subset_bundle is not None: ssc = _score_bundle(subset_bundle, truth_means, k, alpha, is_null) result.subset_marginal_covered += ssc["marginal_covered"] @@ -704,6 +767,12 @@ def _aggregate_group(rows: list[CompareE2EResult]) -> dict: marg_cov_den = sum(r.marginal_total for r in rows) pair_cov_den = sum(r.pairwise_total for r in k2_rows) fam_cov_den = sum(r.family_total for r in kgt2_rows) + # Width/score are per-PAIR quantities (unlike family_covered/family_total, + # the per-REP "ALL pairs held" event) -- k>2's per-pair width/score reuses + # pairwise_width_sum/pairwise_total filtered to k>2 rows, giving the + # direct "what does FWER widening cost in width/score" comparison against + # k==2's own pairwise_width_sum/pairwise_total. + fam_pair_den = sum(r.pairwise_total for r in kgt2_rows) type1_den = sum(r.n_reps - r.n_errors for r in null_rows) power_den = sum(r.n_reps - r.n_errors for r in eff_rows) # Reference-estimator power/Type-I: oracle_n_ok is only nonzero on @@ -717,8 +786,13 @@ def _aggregate_group(rows: list[CompareE2EResult]) -> dict: return dict( marg_cov=(sum(r.marginal_covered for r in rows) / marg_cov_den) if marg_cov_den else float("nan"), marg_width=(sum(r.marginal_width_sum for r in rows) / marg_cov_den) if marg_cov_den else float("nan"), + marg_score=(sum(r.marginal_score_sum for r in rows) / marg_cov_den) if marg_cov_den else float("nan"), pair_cov=(sum(r.pairwise_covered for r in k2_rows) / pair_cov_den) if pair_cov_den else float("nan"), + pair_width=(sum(r.pairwise_width_sum for r in k2_rows) / pair_cov_den) if pair_cov_den else float("nan"), + pair_score=(sum(r.pairwise_score_sum for r in k2_rows) / pair_cov_den) if pair_cov_den else float("nan"), fam_cov=(sum(r.family_covered for r in kgt2_rows) / fam_cov_den) if fam_cov_den else float("nan"), + fam_width=(sum(r.pairwise_width_sum for r in kgt2_rows) / fam_pair_den) if fam_pair_den else float("nan"), + fam_score=(sum(r.pairwise_score_sum for r in kgt2_rows) / fam_pair_den) if fam_pair_den else float("nan"), type1=(sum(r.any_reject for r in null_rows) / type1_den) if type1_den else float("nan"), power=(sum(r.extreme_reject for r in eff_rows) / power_den) if power_den else float("nan"), oracle_type1=(sum(r.oracle_any_reject for r in null_rows) / oracle_type1_den) if oracle_type1_den else float("nan"), @@ -1079,8 +1153,8 @@ def save_results_artifacts( writer = csv.writer(handle) writer.writerow([ "eval_type", "shape_label", "k", "n_items", "ppi_config", "is_null", "n_reps", "n_errors", - "marginal_covered", "marginal_total", "marginal_coverage", "marginal_mean_width", - "pairwise_covered", "pairwise_total", "pairwise_coverage", + "marginal_covered", "marginal_total", "marginal_coverage", "marginal_mean_width", "marginal_mean_score", + "pairwise_covered", "pairwise_total", "pairwise_coverage", "pairwise_mean_width", "pairwise_mean_score", "family_covered", "family_total", "family_coverage", "any_reject", "extreme_reject", "type1_rate", "power_rate", "oracle_n_ok", "oracle_type1_rate", "oracle_power_rate", @@ -1090,7 +1164,10 @@ def save_results_artifacts( n_ok = r.n_reps - r.n_errors marg_cov = r.marginal_covered / r.marginal_total if r.marginal_total else float("nan") marg_width = r.marginal_width_sum / r.marginal_total if r.marginal_total else float("nan") + marg_score = r.marginal_score_sum / r.marginal_total if r.marginal_total else float("nan") pair_cov = r.pairwise_covered / r.pairwise_total if r.pairwise_total else float("nan") + pair_width = r.pairwise_width_sum / r.pairwise_total if r.pairwise_total else float("nan") + pair_score = r.pairwise_score_sum / r.pairwise_total if r.pairwise_total else float("nan") fam_cov = r.family_covered / r.family_total if r.family_total else float("nan") type1 = r.any_reject / n_ok if (r.is_null and n_ok) else float("nan") power = r.extreme_reject / n_ok if (not r.is_null and n_ok) else float("nan") @@ -1100,8 +1177,8 @@ def save_results_artifacts( subset_power = r.subset_extreme_reject / r.subset_n_ok if (not r.is_null and r.subset_n_ok) else float("nan") writer.writerow([ r.eval_type, r.shape_label, r.k, r.n_items, r.ppi_config, r.is_null, r.n_reps, r.n_errors, - r.marginal_covered, r.marginal_total, f"{marg_cov:.6f}", f"{marg_width:.6f}", - r.pairwise_covered, r.pairwise_total, f"{pair_cov:.6f}", + r.marginal_covered, r.marginal_total, f"{marg_cov:.6f}", f"{marg_width:.6f}", f"{marg_score:.6f}", + r.pairwise_covered, r.pairwise_total, f"{pair_cov:.6f}", f"{pair_width:.6f}", f"{pair_score:.6f}", r.family_covered, r.family_total, f"{fam_cov:.6f}", r.any_reject, r.extreme_reject, f"{type1:.6f}", f"{power:.6f}", r.oracle_n_ok, f"{oracle_type1:.6f}", f"{oracle_power:.6f}", diff --git a/simulations/harness/methods.py b/simulations/harness/methods.py index d826c77..9e7268d 100644 --- a/simulations/harness/methods.py +++ b/simulations/harness/methods.py @@ -75,6 +75,52 @@ def __format__(self, format_spec: str) -> str: logit_t (rescaled_ci recentres a paired diff near 0.5 regardless of raw skew, where order=2's boundary-only correction never activates).""" +LOGIT_T_DITHER = Method("logit_t_dither", "#ceb483") # pastel tint of LOGIT_T's #a6761d +SMOOTH_BOOTSTRAP_DITHER = Method("smooth_bootstrap_dither", "#c4abdb") # pastel tint of SMOOTH_BOOTSTRAP's #9467bd +"""ci_paired.py-only, non-binary eval types (see that file's +add_dither_extras): the SAME logit_t/smooth_bootstrap paired-diff CI, but +with U(-half, +half) jitter added independently to each arm's raw values +before differencing (then clipped back to the scale), where half is +auto-detected per rep from the data's own quantization grid via +_detect_dither_halfwidth -- 0.0 (no jitter) if none is found. Fixes a real, severe +small-N pathology distinct from LOGIT_T_2ND's: on a PAIRED diff of two +highly-correlated (shared-item) LIKERT arms, rounding mostly cancels +between arms -- most items round to the identical integer in both arms +(diff=0), and only the rare item whose latent value sits near a rounding +boundary shows a nonzero diff. At small N it's entirely plausible NONE of +the sampled items are boundary-adjacent, so the sample's diffs come out +literally constant, collapsing the sample variance to ~0 regardless of the +(real, nonzero) population-level diff variance -- any variance-based CI +built from that is catastrophically overconfident. Confirmed via +simulations/investigate_likert_family_wise_smalln.py: plain logit_t's +family-wise (Sidak-widened, k=10 arms) coverage was 14.5% at n=10 (vs. 95% +nominal); logit_t_dither recovered to a stable ~92% across n=10-60. NOT the +same mechanism LOGIT_T_2ND targets (that's single-sample boundary-hugging +skew) and does NOT help ci_single -- see that file's own likert check, +which showed plain logit_t already well-calibrated there (worst case 93.8% +at n=10) -- this is a paired-diff-specific pathology. nig_ci_1d fixes the +SAME failure via a wider prior instead, but was found to cost FAR more: +near-zero power at small N/moderate k (0.2% at k=3, n=10) vs. dithering's +much smaller power cost, because nig's conservatism is unconditional while +dithering targets the actual missing variance directly. + +Also tried on CONTINUOUS data with a hardcoded +-0.5 jitter (for +transparency/direct comparison against likert) and that was BROKEN: +-0.5 +is calibrated to undo exactly one unit of INTEGER rounding, but on +continuous's own [0, 1]-scale data it's HALF the entire range, causing +heavy boundary clipping and a systematic bias in the mean. Unlike random +noise, that bias doesn't shrink with N while the CI does, so coverage got +WORSE as N grows rather than converging: 0.936 -> 0.800 (n=10 -> n=100) in +nested-mode screening. Replacing the hardcoded width with +_detect_dither_halfwidth's data-driven detection fixes this generally: it +returns 0.0 (no jitter, dither variant reduces exactly to its base method) +on genuinely continuous data with no recurring gap, so it's now safe to +run on any non-binary type, and it ALSO catches the case a fixed +eval_type check never could -- data labeled "continuous" that's actually +coarse in practice (e.g. a judge that only emits a handful of distinct +values), which would otherwise silently re-trigger the same rounding- +cancellation pathology likert has.""" + BINARY_SINGLE_EXTRA_METHODS = [WILSON, JEFFREYS, WALD, CLOPPER_PEARSON, BAYES_SINGLE] CONTINUOUS_EXTRA_METHODS = [BETA, LOGIT_T, NIG, EL] CONTINUOUS_EXTRA_METHODS_WITH_LOGIT_T_2ND = [BETA, LOGIT_T, LOGIT_T_2ND, NIG, EL] @@ -168,6 +214,17 @@ def __format__(self, format_spec: str) -> str: BAYES_PAIR_PAIRED = Method("bayes_paired_comp", "#98df8a") WALD_PAIR_INDEP = Method("wald_indep", "#7f7f7f") # same grey as ci_single's WALD -- both are the naive baseline PAIRWISE_EXTRA_METHODS = [T_INTERVAL, LOGIT_T, NIG, EL] +DITHER_EXTRA_METHODS = [LOGIT_T_DITHER, SMOOTH_BOOTSTRAP_DITHER] +"""ci_paired.py-only, non-binary eval types -- see LOGIT_T_DITHER's +docstring. Structurally a SEPARATE list from PAIRWISE_EXTRA_METHODS (not +folded into it) since the actual jitter is data-gated (auto-detected per +rep, a no-op when the data shows no quantization grid), but runs BY +DEFAULT for all non-binary cells whenever --methods doesn't exclude them -- +same default-inclusion behavior as PAIRWISE_EXTRA_METHODS itself +(ci_paired.py's `_want` returns True for everything when --methods is +unset), NOT the hidden opt-in-only precedent LOGIT_T_2ND uses. Pass +--methods without these two names to exclude them if only comparing the +pre-existing battery.""" BINARY_PAIRWISE_EXTRA_METHODS = [NEWCOMBE, BAYES_PAIR_INDEP, BAYES_PAIR_PAIRED, WALD_PAIR_INDEP] # --------------------------------------------------------------------------- @@ -474,7 +531,7 @@ def __format__(self, format_spec: str) -> str: REPORT_METHOD_ORDER: list[Method] = BOOTSTRAP_METHODS + [ T_INTERVAL, WILSON, JEFFREYS, NEWCOMBE, TANGO, TANGO_SCC, WALD, CLOPPER_PEARSON, BAYES_SINGLE, BAYES_PAIR_INDEP, BAYES_PAIR_PAIRED, WALD_PAIR_INDEP, -] + CONTINUOUS_EXTRA_METHODS + [LOGIT_T_2ND] + NESTED_METHODS + BINARY_FLAT_METHODS + BINARY_NESTED_METHODS + ( +] + CONTINUOUS_EXTRA_METHODS + [LOGIT_T_2ND] + DITHER_EXTRA_METHODS + NESTED_METHODS + BINARY_FLAT_METHODS + BINARY_NESTED_METHODS + ( PAIR_DIFF_NESTED_METHODS + [TANGO_FLAT, NEWCOMBE_FLAT] + BINARY_PAIR_NESTED_METHODS ) + [ diff --git a/simulations/investigate_continuous_family_wise_smalln.py b/simulations/investigate_continuous_family_wise_smalln.py new file mode 100644 index 0000000..8af40f6 --- /dev/null +++ b/simulations/investigate_continuous_family_wise_smalln.py @@ -0,0 +1,235 @@ +"""One-off investigation (2026-08-11): sanity-check counterpart to +investigate_likert_family_wise_smalln.py -- does CONTINUOUS data show the +same small-N family-wise coverage collapse likert does, under the same +logit_t/nig/smooth_bootstrap comparison? + +Motivation: simulations/harness/cases/compare_e2e.py found likert's +family-wise coverage collapsing badly at small n_items (65% vs 95% nominal +at n=15, worsening to ~18% at k=10 for the worst shape), but explicitly +NOT present in continuous data at the same n_items using the SAME +underlying CI method (logit_t) -- see that harness case's own investigation +notes. investigate_likert_family_wise_smalln.py then confirmed (after +fixing a self-referential-coverage bug) that this is real and severe, and +that nig fixes it completely at a real score cost concentrated exactly +where the failure is worst. + +This script re-runs the identical methodology on continuous data, as a +direct check that continuous truly doesn't share the failure (ruling out +"logit_t is broken at small N in general" in favor of "likert's +discreteness/quantization is the specific driver") -- not just trusting +the earlier compare_e2e finding, but re-verifying it with the same +Sidak-widened, swappable-ci_func, bug-fixed methodology used for likert. + +Not part of the harness / --official-tests: standalone Monte Carlo script. +Run directly: + + .venv/bin/python simulations/investigate_continuous_family_wise_smalln.py +""" + +from __future__ import annotations + +import time +import warnings +from itertools import combinations +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from evalstats.core.paired import _sidak_simultaneous_cis +from evalstats.core.resampling import logit_t_ci_1d, nig_ci_1d, smooth_bootstrap_means_1d +from evalstats.core.stats_utils import interval_score, rescaled_ci +from simulations.harness.scenarios.synthetic import ( + CONTINUOUS_SHAPES, _jb_effect_magnitude, _tier_shapes, sample_group_truth, +) + +ALPHA = 0.05 +N_VALUES = [10, 15, 20, 30, 60] +K_VALUES = [3, 5, 10] +N_REPS = 300 +N_BOOTSTRAP = 1000 # per-pair resample count for smooth_bootstrap's ci_func +SEED = 20260811 +TRUE_MEAN_MC_N = 200_000 # matches compare_e2e's _TRUE_MEAN_MC_N convention + +# Standard-tier only -- matches compare_e2e's own shape catalog exactly. +CONTINUOUS_SHAPES = _tier_shapes(CONTINUOUS_SHAPES, "standard") +CONTINUOUS_SCALE = (0.0, 1.0) +EFFECT_FRAC = 0.15 # matches compare_e2e's DEFAULT_EFFECT_FRAC + +METHODS = ["logit_t", "nig", "smooth_bootstrap"] + + +def build_ci_func(method: str, rng: np.random.Generator): + lo, hi = CONTINUOUS_SCALE + span = hi - lo + diff_lo, diff_hi = -span, span + if method == "logit_t": + return lambda diffs, alpha: rescaled_ci(logit_t_ci_1d, diffs, alpha, diff_lo, diff_hi) + if method == "nig": + return lambda diffs, alpha: rescaled_ci(nig_ci_1d, diffs, alpha, diff_lo, diff_hi) + if method == "smooth_bootstrap": + def _ci(diffs, alpha): + boot = smooth_bootstrap_means_1d(diffs, N_BOOTSTRAP, rng, statistic="mean") + return (float(np.percentile(boot, 100 * alpha / 2)), + float(np.percentile(boot, 100 * (1 - alpha / 2)))) + return _ci + raise ValueError(method) + + +def family_wise_cis(diffs_by_arm: np.ndarray, labels: list[str], ci_func, alpha: float): + """diffs_by_arm: (k, n) truth array, one row per arm, paired by column + (item) index. Returns {(a,b): (lo,hi)} for every pair, Sidak-widened.""" + pairs = list(combinations(labels, 2)) + idx = {lbl: i for i, lbl in enumerate(labels)} + results = { + (a, b): SimpleNamespace(per_input_diffs=diffs_by_arm[idx[a]] - diffs_by_arm[idx[b]]) + for a, b in pairs + } + return _sidak_simultaneous_cis(results=results, pairs=pairs, ci=1.0 - alpha, ci_func=ci_func) + + +def run() -> pd.DataFrame: + rng = np.random.default_rng(SEED) + rows = [] + t0 = time.time() + total_cells = len(CONTINUOUS_SHAPES) * len(K_VALUES) * len(N_VALUES) + cell_i = 0 + for shape in CONTINUOUS_SHAPES: + for k in K_VALUES: + labels = [f"M{i}" for i in range(k)] + effect_step = _jb_effect_magnitude("continuous", EFFECT_FRAC) + effects = np.arange(k, dtype=float) * effect_step + # TRUE population means -- large separate MC draw, NOT the small + # test sample's own mean (see investigate_likert_..._smalln.py's + # docstring for why that was a real bug in the first version). + true_means = sample_group_truth( + shape, TRUE_MEAN_MC_N, 1, k, 1.0, rng, effects=effects, + )[:, :, 0].mean(axis=1) + for n_items in N_VALUES: + cell_i += 1 + per_method = {m: dict(covered=0, total=0, width_sum=0.0, score_sum=0.0, n=0) for m in METHODS} + ci_funcs = {m: build_ci_func(m, rng) for m in METHODS} + for _rep in range(N_REPS): + truth = sample_group_truth(shape, n_items, 1, k, 1.0, rng, effects=effects)[:, :, 0] + for method in METHODS: + cis = family_wise_cis(truth, labels, ci_funcs[method], ALPHA) + all_covered = True + for (a, b), (lo, hi) in cis.items(): + true_diff = true_means[labels.index(a)] - true_means[labels.index(b)] + covered = lo <= true_diff <= hi + if not covered: + all_covered = False + per_method[method]["width_sum"] += hi - lo + per_method[method]["score_sum"] += interval_score(lo, hi, true_diff, ALPHA) + per_method[method]["n"] += 1 + per_method[method]["total"] += 1 + if all_covered: + per_method[method]["covered"] += 1 + for method in METHODS: + d = per_method[method] + rows.append(dict( + shape=shape.label, k=k, n_items=n_items, method=method, + family_coverage=d["covered"] / d["total"], + mean_width=d["width_sum"] / d["n"], + mean_score=d["score_sum"] / d["n"], + )) + elapsed = time.time() - t0 + print(f"\r cell {cell_i}/{total_cells} ({elapsed:.0f}s elapsed)", end="", flush=True) + print() + return pd.DataFrame(rows) + + +METHOD_COLORS = {"logit_t": "#a6761d", "nig": "#888888", "smooth_bootstrap": "#9467bd"} + + +def save_by_k_violin_plot(df: pd.DataFrame, out_dir: str, run_stem: str) -> list[str]: + """Same structure as investigate_likert_family_wise_smalln.py's plot: + one ROW per k, one violin per method at each n, each dot one shape.""" + import matplotlib.pyplot as plt + import seaborn as sns + + target = 1.0 - ALPHA + ks = sorted(df["k"].unique()) + ns = sorted(df["n_items"].unique()) + n_order = [str(n) for n in ns] + df = df.copy() + df["n_label"] = df["n_items"].astype(str) + + out_paths: list[str] = [] + for metric, ylabel, fname_suffix in [ + ("family_coverage", "Family-wise coverage per shape\n(ALL C(k,2) pairs simultaneously covered)", "by_k_violin_coverage"), + ("mean_score", "Mean interval score per shape\n(per pair, lower=better)", "by_k_violin_score"), + ]: + fig, axes = plt.subplots(len(ks), 1, figsize=(1.4 * len(ns) + 3.0, 4.2 * len(ks)), squeeze=False) + for row_idx, k in enumerate(ks): + ax = axes[row_idx][0] + k_df = df[df["k"] == k] + sns.violinplot( + data=k_df, x="n_label", y=metric, order=n_order, hue="method", hue_order=METHODS, + palette=METHOD_COLORS, cut=0, inner="quartile", linewidth=0.8, dodge=True, alpha=0.35, ax=ax, + ) + sns.stripplot( + data=k_df, x="n_label", y=metric, order=n_order, hue="method", hue_order=METHODS, + palette=METHOD_COLORS, size=4, alpha=0.6, dodge=True, jitter=0.15, + linewidth=0.4, edgecolor="white", legend=False, ax=ax, + ) + if metric == "family_coverage": + ax.axhline(target, linestyle="--", color="tab:cyan", linewidth=1.2, zorder=0) + + handles, _ = ax.get_legend_handles_labels() + ax.legend( + handles=handles[:len(METHODS)], title="Method", fontsize=8, title_fontsize=9, + loc="upper left", bbox_to_anchor=(1.01, 1.0), borderaxespad=0.0, + ) + ax.set_xlabel("n_items (per arm)") + ax.set_ylabel(ylabel) + ax.set_title(f"k={k} ({k * (k - 1) // 2} pairs)") + + fig.suptitle( + f"Continuous family-wise {'coverage' if metric == 'family_coverage' else 'interval score'} vs. n_items, by arm count (k)\n" + f"{run_stem} | reps={N_REPS} | alpha={ALPHA}", + fontsize=12, + ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=r".*tight_layout.*", category=UserWarning) + fig.tight_layout(rect=(0, 0, 1, 0.96)) + out_path = str(Path(out_dir) / f"{run_stem}_{fname_suffix}.png") + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=150, bbox_inches="tight") + plt.close(fig) + out_paths.append(out_path) + return out_paths + + +if __name__ == "__main__": + df = run() + out_path = "simulations/out/investigate_continuous_family_wise_smalln_results.csv" + df.to_csv(out_path, index=False) + print(f"\nSaved: {out_path}\n") + + run_stem = f"investigate_continuous_family_wise_smalln_reps{N_REPS}_{time.strftime('%Y%m%d_%H%M%S')}" + plot_paths = save_by_k_violin_plot(df, "simulations/out/plots", run_stem) + for p in plot_paths: + print(f"Saved plot: {p}") + + print("=" * 100) + print("Family-wise coverage, mean width, mean interval score -- pooled across all continuous shapes") + print(f"(alpha={ALPHA}, nominal target={1 - ALPHA:.0%}, reps={N_REPS})") + print("=" * 100) + for k in K_VALUES: + print(f"\n--- k={k} ---") + g = df[df.k == k].groupby(["n_items", "method"]).agg( + family_coverage=("family_coverage", "mean"), + mean_width=("mean_width", "mean"), + mean_score=("mean_score", "mean"), + ).reset_index() + piv_cov = g.pivot(index="n_items", columns="method", values="family_coverage")[METHODS] + piv_width = g.pivot(index="n_items", columns="method", values="mean_width")[METHODS] + piv_score = g.pivot(index="n_items", columns="method", values="mean_score")[METHODS] + print("Coverage:") + print(piv_cov.to_string(float_format=lambda x: f"{x:.3f}")) + print("Mean width:") + print(piv_width.to_string(float_format=lambda x: f"{x:.3f}")) + print("Mean score (lower=better):") + print(piv_score.to_string(float_format=lambda x: f"{x:.3f}")) diff --git a/simulations/investigate_likert_family_wise_smalln.py b/simulations/investigate_likert_family_wise_smalln.py new file mode 100644 index 0000000..82cf017 --- /dev/null +++ b/simulations/investigate_likert_family_wise_smalln.py @@ -0,0 +1,328 @@ +"""One-off investigation (2026-08-11): does swapping the per-pair CI method +under Sidak family-wise widening fix likert's small-N family-wise coverage +collapse found by simulations/harness/cases/compare_e2e.py? + +Real finding that motivated this (compare_e2e, reps=50, full standard-tier +likert shape catalog): family-wise (k>2, ALL C(k,2) pairs simultaneously +covered) coverage for likert data at n_items=15 was 65.0% against a 95% +nominal target, worsening sharply with arm count (down to 18% at k=10 for +the worst shape, likert-bimodal). NOT present in continuous data using the +same underlying CI method (logit_t) or in binary data (Tango) -- see that +harness case's own investigation notes. A direct single-pair check +(`ci_paired --eval-types likert --methods logit_t nig smooth_bootstrap +--sizes 10 15 20 30 60 --reps 500`) shows the per-pair MinCov (worst single +scenario) is only 83-92% at n=15, not catastrophic -- so the family-wise +"ALL k*(k-1)/2 pairs must hold" requirement is amplifying an already-shaky +per-pair CI, not itself introducing a new bug. This script isolates whether +a per-pair method with a better worst-case (nig) or the current default +(logit_t) or a resampling alternative (smooth_bootstrap) actually fixes the +FAMILY-WISE number once Sidak widening (the auto-resolved method for +n_items<30 numeric data, confirmed via evalstats.config. +resolve_auto_simultaneous_ci_method) is layered on top -- and at what width/ +score cost, since coverage alone isn't the goal (nominal-or-slightly-above, +not maximally conservative). + +Reuses compare_e2e's own data generation (sample_group_truth + the same +population-SD-standardized effect size convention) so this reproduces the +exact regime the collapse was found in, rather than a different synthetic +setup that might not reproduce it. + +Not part of the harness / --official-tests: standalone Monte Carlo script. +Run directly: + + .venv/bin/python simulations/investigate_likert_family_wise_smalln.py +""" + +from __future__ import annotations + +import time +import warnings +from itertools import combinations +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd + +from evalstats.core.paired import ( + PairedDiffResult, _joint_bootstrap_scaled_simultaneous_cis, _sidak_simultaneous_cis, +) +from evalstats.core.resampling import logit_t_ci_1d, nig_ci_1d, smooth_bootstrap_means_1d +from evalstats.core.stats_utils import interval_score, rescaled_ci +from simulations.harness.scenarios.synthetic import ( + LIKERT_SHAPES, _jb_effect_magnitude, _tier_shapes, sample_group_truth, +) + +ALPHA = 0.05 +N_VALUES = [10, 15, 20, 30, 60] +K_VALUES = [3, 5, 10] +N_REPS = 300 +N_BOOTSTRAP = 1000 # per-pair resample count for smooth_bootstrap's ci_func, and + # the joint-bootstrap widening's own resample count for logit_t_boot +SEED = 20260811 +TRUE_MEAN_MC_N = 200_000 # matches compare_e2e's _TRUE_MEAN_MC_N convention + +# Standard-tier only -- matches compare_e2e's own shape catalog exactly, so +# this reproduces the exact regime the collapse was found in (LIKERT_SHAPES +# alone also includes "expanded"-tier shapes compare_e2e never tested). +LIKERT_SHAPES = _tier_shapes(LIKERT_SHAPES, "standard") +LIKERT_SCALE = (1.0, 5.0) +EFFECT_FRAC = 0.15 # matches compare_e2e's DEFAULT_EFFECT_FRAC + +# logit_t_boot (joint-bootstrap widening instead of Sidak, same logit_t +# per-pair CI) was tested and RULED OUT: coverage tracked plain logit_t +# almost exactly at every k/n (confirmed directly -- see this script's git +# history / the session that built it). Sidak's independence assumption +# isn't the culprit; the per-pair CI construction itself is. Replaced here +# with two DITHERING variants, which target the actual diagnosed mechanism +# directly: likert's rounding to an integer scale erases real underlying +# variability (a continuous latent value near a rounding boundary could +# have rounded either way), so at small N the sample of rounded diffs is +# often literally constant or near-constant (confirmed: smooth_bootstrap's +# own fallback warning fires on "sample std=0" repeatedly in this exact +# regime) -- both logit_t's normal-approximation variance estimate and +# smooth_bootstrap's KDE step then badly underestimate the true uncertainty. +# Dithering (adding U(-0.5, +0.5) jitter to each item's rounded value before +# differencing, then clipping back to the scale) is the standard technique +# for recovering a plausible pre-rounding continuous approximation -- zero- +# mean and symmetric, so it doesn't bias the estimate, but it un-collapses +# the degenerate sample distribution that's breaking both methods. +METHODS = ["logit_t", "nig", "smooth_bootstrap", "logit_t_dither", "smooth_bootstrap_dither"] +WIDENING = {m: "sidak" for m in METHODS} +DITHER_METHODS = {"logit_t_dither", "smooth_bootstrap_dither"} + + +def dither(truth: np.ndarray, rng: np.random.Generator) -> np.ndarray: + lo, hi = LIKERT_SCALE + return np.clip(truth + rng.uniform(-0.5, 0.5, size=truth.shape), lo, hi) + + +def build_ci_func(method: str, rng: np.random.Generator): + lo, hi = LIKERT_SCALE + span = hi - lo + diff_lo, diff_hi = -span, span + if method in ("logit_t", "logit_t_dither"): + return lambda diffs, alpha: rescaled_ci(logit_t_ci_1d, diffs, alpha, diff_lo, diff_hi) + if method == "nig": + return lambda diffs, alpha: rescaled_ci(nig_ci_1d, diffs, alpha, diff_lo, diff_hi) + if method in ("smooth_bootstrap", "smooth_bootstrap_dither"): + def _ci(diffs, alpha): + boot = smooth_bootstrap_means_1d(diffs, N_BOOTSTRAP, rng, statistic="mean") + return (float(np.percentile(boot, 100 * alpha / 2)), + float(np.percentile(boot, 100 * (1 - alpha / 2)))) + return _ci + raise ValueError(method) + + +def family_wise_cis( + diffs_by_arm: np.ndarray, labels: list[str], ci_func, alpha: float, + widening: str, rng: np.random.Generator, +): + """diffs_by_arm: (k, n) truth array, one row per arm, paired by column + (item) index. Returns {(a,b): (lo,hi)} for every pair, widened by either + Sidak (assumes independent pairs) or joint bootstrap (models the real + cross-pair correlation from shared items).""" + pairs = list(combinations(labels, 2)) + idx = {lbl: i for i, lbl in enumerate(labels)} + if widening == "sidak": + results = { + (a, b): SimpleNamespace(per_input_diffs=diffs_by_arm[idx[a]] - diffs_by_arm[idx[b]]) + for a, b in pairs + } + return _sidak_simultaneous_cis(results=results, pairs=pairs, ci=1.0 - alpha, ci_func=ci_func) + if widening == "boot": + results = {} + for a, b in pairs: + diffs = diffs_by_arm[idx[a]] - diffs_by_arm[idx[b]] + results[(a, b)] = PairedDiffResult( + template_a=a, template_b=b, + point_diff=float(diffs.mean()), std_diff=float(diffs.std(ddof=1)) if len(diffs) > 1 else 0.0, + ci_low=float("nan"), ci_high=float("nan"), + p_value=float("nan"), test_method="logit_t", + n_inputs=len(diffs), per_input_diffs=diffs, + ) + return _joint_bootstrap_scaled_simultaneous_cis( + scores=diffs_by_arm, results=results, pairs=pairs, labels=labels, + ci=1.0 - alpha, n_bootstrap=N_BOOTSTRAP, rng=rng, ci_func=ci_func, statistic="mean", + ) + raise ValueError(widening) + + +def run() -> pd.DataFrame: + rng = np.random.default_rng(SEED) + rows = [] + t0 = time.time() + total_cells = len(LIKERT_SHAPES) * len(K_VALUES) * len(N_VALUES) + cell_i = 0 + for shape in LIKERT_SHAPES: + for k in K_VALUES: + labels = [f"M{i}" for i in range(k)] + effect_step = _jb_effect_magnitude("likert", EFFECT_FRAC, scale_bounds=LIKERT_SCALE) + effects = np.arange(k, dtype=float) * effect_step + # TRUE population means for this (shape, k, effects) -- a large, + # separate Monte Carlo draw, NOT the small test sample's own mean + # (which a CI trivially contains almost by construction, since + # it's built from and centered near that same sample -- this was + # the bug in the first version of this script: it silently tested + # self-consistency, not calibration against the actual truth). + true_means = sample_group_truth( + shape, TRUE_MEAN_MC_N, 1, k, 1.0, rng, effects=effects, + )[:, :, 0].mean(axis=1) + extreme_pair = (labels[0], labels[-1]) + for n_items in N_VALUES: + cell_i += 1 + per_method = { + m: dict(covered=0, total=0, width_sum=0.0, score_sum=0.0, n=0, extreme_reject=0) + for m in METHODS + } + ci_funcs = {m: build_ci_func(m, rng) for m in METHODS} + for _rep in range(N_REPS): + truth = sample_group_truth(shape, n_items, 1, k, 1.0, rng, effects=effects)[:, :, 0] + for method in METHODS: + method_truth = dither(truth, rng) if method in DITHER_METHODS else truth + cis = family_wise_cis(method_truth, labels, ci_funcs[method], ALPHA, WIDENING[method], rng) + all_covered = True + for (a, b), (lo, hi) in cis.items(): + true_diff = true_means[labels.index(a)] - true_means[labels.index(b)] + covered = lo <= true_diff <= hi + if not covered: + all_covered = False + per_method[method]["width_sum"] += hi - lo + per_method[method]["score_sum"] += interval_score(lo, hi, true_diff, ALPHA) + per_method[method]["n"] += 1 + # Power: does the (family-wise-widened) CI for the + # extreme pair (largest true gap, M0 vs M{k-1}) + # exclude zero -- the CI-duality equivalent of a + # FWER-corrected p-value < alpha for that pair. + if (a, b) == extreme_pair and (lo > 0.0 or hi < 0.0): + per_method[method]["extreme_reject"] += 1 + per_method[method]["total"] += 1 + if all_covered: + per_method[method]["covered"] += 1 + for method in METHODS: + d = per_method[method] + if d["n"] == 0: + print(f"\n WARNING: {shape.label} k={k} n_items={n_items} method={method} -- " + f"family_wise_cis returned 0 pairs across all {N_REPS} reps, skipping cell") + continue + rows.append(dict( + shape=shape.label, k=k, n_items=n_items, method=method, + family_coverage=d["covered"] / d["total"], + mean_width=d["width_sum"] / d["n"], + mean_score=d["score_sum"] / d["n"], + power=d["extreme_reject"] / d["total"], + )) + elapsed = time.time() - t0 + print(f"\r cell {cell_i}/{total_cells} ({elapsed:.0f}s elapsed)", end="", flush=True) + print() + return pd.DataFrame(rows) + + +METHOD_COLORS = { + "logit_t": "#a6761d", "nig": "#888888", "smooth_bootstrap": "#9467bd", + "logit_t_dither": "#1f77b4", "smooth_bootstrap_dither": "#d62728", +} + + +def save_by_k_violin_plot(df: pd.DataFrame, out_dir: str, run_stem: str) -> list[str]: + """Grouped violin plots of per-shape family-wise coverage and interval + score vs. n_items -- one ROW per k (not one column per eval_type, like + ci_paired.py's by-n-violin-plot, since eval_type is fixed at likert here + and k is the axis that actually drives the story: the same per-shape + weakness that's a minor per-pair issue at k=2/3 compounds into a + catastrophic family-wise failure at k=10, via the "ALL C(k,2) pairs must + hold" AND). One violin per method at each n (dodged); each dot is one + likert shape's family_coverage/mean_score at that (k, n, method).""" + import matplotlib.pyplot as plt + import seaborn as sns + + target = 1.0 - ALPHA + ks = sorted(df["k"].unique()) + ns = sorted(df["n_items"].unique()) + n_order = [str(n) for n in ns] + df = df.copy() + df["n_label"] = df["n_items"].astype(str) + + out_paths: list[str] = [] + for metric, ylabel, fname_suffix in [ + ("family_coverage", "Family-wise coverage per shape\n(ALL C(k,2) pairs simultaneously covered)", "by_k_violin_coverage"), + ("mean_score", "Mean interval score per shape\n(per pair, lower=better)", "by_k_violin_score"), + ("power", "Power per shape\n(extreme pair CI excludes zero)", "by_k_violin_power"), + ]: + fig, axes = plt.subplots(len(ks), 1, figsize=(1.4 * len(ns) + 3.0, 4.2 * len(ks)), squeeze=False) + for row_idx, k in enumerate(ks): + ax = axes[row_idx][0] + k_df = df[df["k"] == k] + sns.violinplot( + data=k_df, x="n_label", y=metric, order=n_order, hue="method", hue_order=METHODS, + palette=METHOD_COLORS, cut=0, inner="quartile", linewidth=0.8, dodge=True, alpha=0.35, ax=ax, + ) + sns.stripplot( + data=k_df, x="n_label", y=metric, order=n_order, hue="method", hue_order=METHODS, + palette=METHOD_COLORS, size=4, alpha=0.6, dodge=True, jitter=0.15, + linewidth=0.4, edgecolor="white", legend=False, ax=ax, + ) + if metric == "family_coverage": + ax.axhline(target, linestyle="--", color="tab:cyan", linewidth=1.2, zorder=0) + + handles, _ = ax.get_legend_handles_labels() + ax.legend( + handles=handles[:len(METHODS)], title="Method", fontsize=8, title_fontsize=9, + loc="upper left", bbox_to_anchor=(1.01, 1.0), borderaxespad=0.0, + ) + ax.set_xlabel("n_items (per arm)") + ax.set_ylabel(ylabel) + ax.set_title(f"k={k} ({k * (k - 1) // 2} pairs)") + + metric_label = {"family_coverage": "coverage", "mean_score": "interval score", "power": "power"}[metric] + fig.suptitle( + f"Likert family-wise {metric_label} vs. n_items, by arm count (k)\n" + f"{run_stem} | reps={N_REPS} | alpha={ALPHA}", + fontsize=12, + ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=r".*tight_layout.*", category=UserWarning) + fig.tight_layout(rect=(0, 0, 1, 0.96)) + out_path = str(Path(out_dir) / f"{run_stem}_{fname_suffix}.png") + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=150, bbox_inches="tight") + plt.close(fig) + out_paths.append(out_path) + return out_paths + + +if __name__ == "__main__": + df = run() + out_path = "simulations/out/investigate_likert_family_wise_smalln_results.csv" + df.to_csv(out_path, index=False) + print(f"\nSaved: {out_path}\n") + + run_stem = f"investigate_likert_family_wise_smalln_reps{N_REPS}_{time.strftime('%Y%m%d_%H%M%S')}" + plot_paths = save_by_k_violin_plot(df, "simulations/out/plots", run_stem) + for p in plot_paths: + print(f"Saved plot: {p}") + + print("=" * 100) + print("Family-wise coverage, mean width, mean interval score, power -- pooled across all likert shapes") + print(f"(alpha={ALPHA}, nominal target={1 - ALPHA:.0%}, reps={N_REPS})") + print("=" * 100) + for k in K_VALUES: + print(f"\n--- k={k} ---") + g = df[df.k == k].groupby(["n_items", "method"]).agg( + family_coverage=("family_coverage", "mean"), + mean_width=("mean_width", "mean"), + mean_score=("mean_score", "mean"), + power=("power", "mean"), + ).reset_index() + piv_cov = g.pivot(index="n_items", columns="method", values="family_coverage")[METHODS] + piv_width = g.pivot(index="n_items", columns="method", values="mean_width")[METHODS] + piv_score = g.pivot(index="n_items", columns="method", values="mean_score")[METHODS] + piv_power = g.pivot(index="n_items", columns="method", values="power")[METHODS] + print("Coverage:") + print(piv_cov.to_string(float_format=lambda x: f"{x:.3f}")) + print("Mean width:") + print(piv_width.to_string(float_format=lambda x: f"{x:.3f}")) + print("Mean score (lower=better):") + print(piv_score.to_string(float_format=lambda x: f"{x:.3f}")) + print("Power (extreme pair CI excludes zero):") + print(piv_power.to_string(float_format=lambda x: f"{x:.3f}")) From 6e10194c2e3e7c57f03fcf13b82b0ecd17faeef9 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 16:18:25 -0400 Subject: [PATCH 003/245] Fix stale icc_values in ci_paired's official_args() (pairwise preset) official_args() never received the 2026-07-14 ICC reweighting that nested_official_args() got: it was still using the old, evenly-spread icc_values=[0.05, 0.20, 0.40, 0.60, 0.80] instead of the range matched to measured real per-item ICC (mean 0.739, median 0.748, IQR [0.644, 0.873] across 48 real corpora), which concentrates well above 0.80, not uniformly across [0, 1]. This was concretely surfaced while checking whether logit_t_dither's likert numbers could be justified against plain logit_t in the official pairwise table: at the old max icc=0.80, logit_t showed zero coverage degradation even at n=10 (0.9463) -- the pairwise battery couldn't reach the regime (icc -> 1, small N) where the rounding-cancellation pathology dithering targets actually bites, so the comparison was untestable by construction, not just untested by omission. Fixes official_args() and the bare CLI fallback (used when ci_paired synthetic is invoked directly without --icc-values) to match nested_official_args()'s icc_values=[0.01, 0.3, 0.5, 0.65, 0.75, 0.85, 0.95]. real_official_args() and discordant_comparison_args() left untouched: the former's icc_values field is inert (real-data sources don't consume it), the latter is an unrelated, narrower binary-only comparison with its own established history. Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/ci_paired.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/simulations/harness/cases/ci_paired.py b/simulations/harness/cases/ci_paired.py index d62247d..4832345 100644 --- a/simulations/harness/cases/ci_paired.py +++ b/simulations/harness/cases/ci_paired.py @@ -1945,13 +1945,26 @@ def official_args(base_seed: int = 42) -> argparse.Namespace: [0, 1]-scale case well (grades is just continuous rescaled to 0-100), while "likert" is kept as a genuinely distinct limiting case (integer- valued, few levels). Dropping grades cuts a third eval type out of the - official sweep's runtime for no real loss of coverage.""" + official sweep's runtime for no real loss of coverage. + + icc_values matched to nested_official_args()'s range (was stale here -- + this preset never received the 2026-07-14 reweighting nested_official_args() + got, see that docstring for the full writeup: measuring actual per-item + ICC on 48 real (model, benchmark) corpora gave mean 0.739, median 0.748, + IQR [0.644, 0.873], i.e. concentrated well above this preset's old cap of + 0.80, not spread evenly across [0, 1]. Concretely surfaced by checking + whether logit_t_dither's likert numbers here could be justified against + plain logit_t: at this preset's old max icc=0.80, logit_t showed no + coverage degradation at all (0.9463 at n=10) -- the pairwise battery + literally couldn't reach the regime (icc -> 1, small N) where the + rounding-cancellation pathology dithering fixes actually bites, so it + was untestable here by construction, not merely untested.""" return argparse.Namespace( data_source="synthetic", scenario_suite="expanded", eval_types=["binary", "continuous", "likert"], benchmarks=None, models=None, hf_token=None, cache_dir=None, min_pair_size=50, inspect_csv=None, runs=1, statistic="mean", reps=300, bootstrap_n=10000, bayes_n=10000, alpha=0.05, sizes=[10, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100], - seed=base_seed, icc_values=[0.05, 0.20, 0.40, 0.60, 0.80], cohens_d_values=[0.2, 0.4], include_null=True, + seed=base_seed, icc_values=[0.01, 0.3, 0.5, 0.65, 0.75, 0.85, 0.95], cohens_d_values=[0.2, 0.4], include_null=True, progress="bar", plots="save", save_results="save", out_dir="simulations/out", plots_dir=None, nested_mode=False, runs_sweep=None, run_noise_fracs=RUN_NOISE_FRACS_DEFAULT, heteroscedastic=False, no_bootstrap_binary=False, @@ -2233,7 +2246,7 @@ def run(args: argparse.Namespace) -> CaseResult: print(f"\nci_paired simulation -- data_source={args.data_source}, statistic={args.statistic}") if args.data_source == "synthetic": - icc_values = args.icc_values if args.icc_values is not None else [0.05, 0.20, 0.40, 0.60, 0.80] + icc_values = args.icc_values if args.icc_values is not None else [0.01, 0.3, 0.5, 0.65, 0.75, 0.85, 0.95] sources = build_pair_sources( suite=args.scenario_suite, icc_values=icc_values, cohens_d_values=args.cohens_d_values, include_null=args.include_null, From 38d0626d68fae869baa863d7e5f608fa5460a65c Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 17:35:34 -0400 Subject: [PATCH 004/245] Fix dither-grid detector's frequency threshold going blind at small N _detect_dither_halfwidth required the dominant recurring gap to appear >= max(3, 0.2*gaps.size) times to count as a real quantization grid. That's unreachable when there are fewer than 3 total gaps to begin with -- exactly what happens at small N with a peaked/near-boundary distribution, where pooled arm values often collapse to just 2-3 distinct integers. Concretely regressed: at n=10, icc=0.95, the likert "near-floor" and "near-ceiling" shapes' logit_t_dither coverage was numerically IDENTICAL to plain logit_t's (0.7633, 0.7667) -- dithering never activated on exactly the scenarios with the worst rounding-cancellation collapse, silently defeating the fix on its own target case. Verified via direct sampling: the detector returned 0.0 for 295/300 draws from the near-floor shape at n=10. Replaced the frequency-threshold heuristic with a GCD-style check: take the smallest observed gap as the candidate step, then require every other gap to be (within tolerance) an integer multiple of it, rather than requiring the same gap value to recur a fixed number of times. This detects an unambiguous 2-gap grid correctly while remaining safe on genuinely continuous data (verified up to n=1000 pooled draws, worst case for false positives) -- demanding ALL gaps independently match the candidate step has vanishing false-positive probability, unlike a count threshold that becomes impossible to satisfy exactly when data is sparsest. Re-verified against the case that motivated dithering: restricted to n=10, icc=0.95, cohens_d=0.2 (hardest slice in official_args()'s now- corrected icc range), plain logit_t's likert coverage is 0.873 (MinCov 0.737) vs logit_t_dither's 0.932 (MinCov 0.910) -- the expected recovery, now demonstrable within the official pairwise methodology itself. Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/ci_paired.py | 41 ++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/simulations/harness/cases/ci_paired.py b/simulations/harness/cases/ci_paired.py index 4832345..3ae1808 100644 --- a/simulations/harness/cases/ci_paired.py +++ b/simulations/harness/cases/ci_paired.py @@ -330,27 +330,44 @@ def _detect_dither_halfwidth(pooled: np.ndarray) -> float: eval_type-driven: labeled "continuous" data that's actually coarse (e.g. a judge that only emits a handful of distinct values) gets detected and dithered correctly; genuinely continuous data (no - recurring gap) returns 0.0, meaning "don't dither" -- unlike a + consistent grid) returns 0.0, meaning "don't dither" -- unlike a hardcoded width, this can't apply a jitter mismatched to the data's real resolution and reintroduce the boundary-clipping bias that broke continuous coverage when a flat +-0.5 was tried there (see - add_dither_extras's comment).""" + add_dither_extras's comment). + + Takes the SMALLEST observed gap between distinct pooled values as the + candidate step, then verifies every other gap is (within tolerance) an + integer multiple of it -- a GCD-style check, not a "does the dominant + gap recur >= N times" frequency threshold. The frequency-threshold + version this replaced was blind exactly where dithering matters most: + at small N with a peaked/near-boundary distribution (e.g. a likert + "near-floor" shape at n=10), pooled values often collapse to just 2-3 + distinct integers -- too few gap observations for any gap to recur 3+ + times even though the grid (step=1) is completely unambiguous. This + was found via a real regression: at n=10, icc=0.95, the near-floor and + near-ceiling likert shapes' logit_t_dither coverage was numerically + IDENTICAL to plain logit_t's (0.763, 0.767) -- i.e. dithering silently + never activated on exactly the scenarios with the worst collapse. + Requiring ALL gaps (not just the most common one) to line up on the + candidate grid is deliberately strict: for genuinely continuous data, + the smallest of many gaps is essentially arbitrary, and demanding every + other gap independently land within tolerance of an integer multiple of + it has vanishing false-positive probability (each gap has only a small + chance of matching by coincidence, and they must ALL match).""" uniq = np.unique(pooled) - if uniq.size < 3: + if uniq.size < 2: return 0.0 gaps = np.diff(uniq) gaps = gaps[gaps > 1e-9] - if gaps.size < 2: + if gaps.size == 0: return 0.0 - rounded = np.round(gaps, decimals=6) - grid_vals, counts = np.unique(rounded, return_counts=True) - best = np.argmax(counts) - step, count = grid_vals[best], counts[best] - # Require the dominant gap to recur often enough to look like a real - # grid, not coincidental spacing among genuinely continuous draws. - if count < max(3, 0.2 * gaps.size): + step = float(np.min(gaps)) + ratios = gaps / step + residuals = np.abs(ratios - np.round(ratios)) + if np.max(residuals) > 0.05: return 0.0 - return float(step) / 2.0 + return step / 2.0 def _run_cell( From db7f1de6082cd1bdeafd425470e7519ed1ffc2d0 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 18:30:45 -0400 Subject: [PATCH 005/245] Fix boundary-clipping bias in dither jitter, then the ValueError regression that fix introduced Requested robustness audit before using logit_t_dither/smooth_bootstrap_dither as the paper's preferred likert method surfaced two more real bugs, on top of the detector-threshold fix already merged: 1. Naive clip(x+jitter, lo, hi) is not mean-preserving near a hard boundary: jitter that would push a value below lo (or above hi) piles up exactly at the boundary, pulling E[clipped] toward the interior by up to half/4 for a boundary-adjacent item. Confirmed as a real regression: on likert "bimodal-extreme" data (icc=0.95, d=0.4, mass at both ends, more at the ceiling), logit_t_dither's coverage fell from 0.953 (n=15) to 0.870 (n=100) -- degrading WITH N, the same signature as the continuous boundary-clipping bug fixed earlier, just via a different intermediate. Root-caused directly: the dithered point estimate was shifted -0.044 vs the undithered one, consistent with the ceiling (39.5% of mass) pulling the diff down more than the floor (20.3%) pulled it up. Fixed with an exact closed-form correction (_debiased_dither) derived from the truncated-uniform expectation: E[max(x+j,lo)]-x = (half-d)^2/(4*half) for a value distance d logit_t_dither 0.927); detector false-positive rate on genuinely continuous data remains 0% up to n=1000; debiased values are now verified strictly contained in [lo,hi] under a boundary-heavy stress test; interior (non-boundary) values remain unaffected either way. Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/ci_paired.py | 100 +++++++++++++++++++++---- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/simulations/harness/cases/ci_paired.py b/simulations/harness/cases/ci_paired.py index 3ae1808..c29246d 100644 --- a/simulations/harness/cases/ci_paired.py +++ b/simulations/harness/cases/ci_paired.py @@ -370,6 +370,80 @@ def _detect_dither_halfwidth(pooled: np.ndarray) -> float: return step / 2.0 +def _debiased_dither(x: np.ndarray, half: float, lo: float, hi: float, rng: np.random.Generator) -> np.ndarray: + """Add U(-half, half) jitter to x, clip to [lo, hi], then subtract the + EXACT closed-form bias that clipping introduces near the boundaries. + + Naive clip(x + jitter, lo, hi) is NOT mean-preserving for x within + `half` of a boundary: jitter that would push x below lo (or above hi) + piles up exactly at the boundary instead of continuing past it, pulling + E[clipped] toward the interior. For x exactly at a hard boundary with + jitter ~ U(-h, h), E[clip] = x +/- h/4 (derived below) -- NOT x. This + doesn't average away with N (it's a fixed per-item shift, not noise), + and because two paired arms generally have DIFFERENT boundary-mass + compositions (that's what makes them differ), the bias doesn't cancel + in the arms' difference either -- it contaminates the estimated diff + directly. Confirmed as a real regression: on likert "bimodal-extreme" + data (icc=0.95, d=0.4, heavy mass at both the floor AND ceiling, more + at the ceiling), logit_t_dither's coverage fell from 0.953 (n=15) to + 0.870 (n=100) -- degrading WITH N, the signature of a persistent bias + rather than added variance -- while plain logit_t stayed flat (~0.94- + 0.97) on the identical scenario. Root cause confirmed directly: the + dithered point estimate was shifted by -0.044 relative to the + undithered one, consistent with the ceiling (39.5% of mass) pulling the + difference down more than the floor (20.3% of mass) pulled it up. + + Derivation: for x with distance d = x - lo from the lower bound + (d < half means boundary-adjacent) and jitter j ~ U(-half, half), + E[max(x+j, lo)] - x = E[max(j, -d)] = (half - d)^2 / (4*half) for + d < half (0 otherwise) -- a standard truncated-uniform expectation. + The upper-bound case is the mirror image. Subtracting these exactly + recenters the expectation back on x regardless of boundary proximity + (verified numerically: reduces a 0.125 bias at h=0.5 to ~0.0005, + Monte-Carlo-noise level, if left un-clipped). Reflection or rejection- + resampling were tried first and are WORSE, not better, here: for + jitter straddling a hard boundary symmetrically, both fold the entire + out-of-bounds half onto an exact duplicate of the in-bounds half + (rather than restoring symmetry around x), giving twice clip's bias + (0.25 vs 0.125 at h=0.5). + + The correction is finally re-clipped to [lo, hi] -- NOT left + unclipped as an earlier version of this function did. That version's + docstring claimed the resulting per-item excursions past [lo, hi] + (up to half/4) were harmless because they "only ever feed a per-item + mean" -- that was wrong: pair_diffs_dither/cell_diffs_dither are + per-ITEM arrays (n entries), passed directly into logit_t_ci_1d, + which raises ValueError on inputs meaningfully outside [0, 1] after + rescaling (see that function's docstring re: a near-identical past + incident). With n independent items, the chance that AT LEAST ONE + exceeds the tolerance grows with n (~1-(1-p)^n), not shrinks -- a + max-of-n effect, invisible in small samples, that produced the exact + same "coverage falls with N" symptom this whole fix targets, just + from a different mechanism (an exception being silently swallowed + into a zero-width interval by the `except Exception:` fallback below, + not lost variance). Confirmed via direct measurement on likert + "uniform" data (heavy floor+ceiling mass, so most prone to + floor-vs-ceiling item pairs before any jitter): the unclipped version's + ValueError rate rose from 2.3% (n=10) to 22.3% (n=100). Re-clipping + trades away some of the bias correction for boundary-adjacent items + (residual ~0.070 vs clip-alone's 0.125 at h=0.5 -- a ~44% reduction, + not a full fix) in exchange for guaranteed validity; a value exactly + at a hard boundary can't be made simultaneously unbiased AND + contained in [lo, hi] by any deterministic remapping of a jitter + that spans past that boundary -- containment was chosen as the + non-negotiable constraint since violating it doesn't degrade + gracefully, it corrupts the whole interval for that rep.""" + if half <= 0: + return x + jitter = rng.uniform(-half, half, size=x.shape) + raw = np.clip(x + jitter, lo, hi) + d_lo = x - lo + d_hi = hi - x + bias_lo = np.where(d_lo < half, (half - d_lo) ** 2 / (4 * half), 0.0) + bias_hi = np.where(d_hi < half, (half - d_hi) ** 2 / (4 * half), 0.0) + return np.clip(raw - bias_lo + bias_hi, lo, hi) + + def _run_cell( source_obj: CIPairSource, n: int, n_reps: int, n_bootstrap: int, bayes_n: int, alpha: float, runs: int, statistic: str, seed, method_names: frozenset[str] | None = None, @@ -498,18 +572,17 @@ def _record(method, ci_low: float, ci_high: float) -> None: if add_dither_extras: # Independent U(-half, +half) jitter per arm (not on the diff - # directly) then clip back to the scale, where half is detected - # per rep from the data's own quantization grid (0.0 -- i.e. no - # jitter -- if none is detected) -- see LOGIT_T_DITHER's - # docstring for why this specifically targets the paired-diff - # rounding-cancellation pathology, not just "add some noise." + # directly), clip back to the scale, then subtract the exact + # boundary-clipping bias (see _debiased_dither's docstring) -- + # half is detected per rep from the data's own quantization + # grid (0.0 -- i.e. no jitter -- if none is detected). See + # LOGIT_T_DITHER's docstring for why this specifically targets + # the paired-diff rounding-cancellation pathology, not just + # "add some noise." _scale_lo, _scale_hi = EVAL_TYPE_SCALE_BOUNDS[source_obj.eval_type] _half = _detect_dither_halfwidth(np.concatenate([a.ravel(), b.ravel()])) - if _half > 0: - a_dither = np.clip(a + rng.uniform(-_half, _half, size=a.shape), _scale_lo, _scale_hi) - b_dither = np.clip(b + rng.uniform(-_half, _half, size=b.shape), _scale_lo, _scale_hi) - else: - a_dither, b_dither = a, b + a_dither = _debiased_dither(a, _half, _scale_lo, _scale_hi, rng) + b_dither = _debiased_dither(b, _half, _scale_lo, _scale_hi, rng) pair_diffs_dither = a_dither.mean(axis=1) - b_dither.mean(axis=1) obs_dither = float(np.mean(pair_diffs_dither)) diff_span_dither = _scale_hi - _scale_lo @@ -841,11 +914,8 @@ def _record(method, ci_low: float, ci_high: float) -> None: # diffs, just computed from independently dithered a/b first. if not is_binary: _half = _detect_dither_halfwidth(np.concatenate([a.ravel(), b.ravel()])) - if _half > 0: - a_dither = np.clip(a + rng.uniform(-_half, _half, size=a.shape), _scale_lo, _scale_hi) - b_dither = np.clip(b + rng.uniform(-_half, _half, size=b.shape), _scale_lo, _scale_hi) - else: - a_dither, b_dither = a, b + a_dither = _debiased_dither(a, _half, _scale_lo, _scale_hi, rng) + b_dither = _debiased_dither(b, _half, _scale_lo, _scale_hi, rng) cell_diffs_dither = a_dither.mean(axis=1) - b_dither.mean(axis=1) obs_diff_dither = float(np.mean(cell_diffs_dither)) for method in (LOGIT_T_DITHER, SMOOTH_BOOTSTRAP_DITHER): From 88dfde33b76853137c29db8b76acab77c7ff3000 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 18:52:56 -0400 Subject: [PATCH 006/245] Rename validate_alignment to judge_alignment "judge_alignment" reads clearer at the call site than "validate_alignment" -- straight rename, same signature, no behavior change. Updated all ~130 call sites across the library, tests, examples, simulations, both READMEs, and one Jupyter notebook, plus a test class name that still spelled out the old name (TestValidateAlignmentBasic -> TestJudgeAlignmentBasic). Verified via py_compile on every touched file, running 4 of the affected example scripts end-to-end, and the full test_alignment.py (55/55) and test_compound_ppi_fwer.py (23/23) suites. Co-Authored-By: Claude Sonnet 5 --- README.md | 2 +- agent_study/sweep/README.md | 2 +- evalstats/__init__.py | 4 +- evalstats/alignment.py | 10 +- evalstats/api.py | 2 +- evalstats/ppi.py | 2 +- evalstats/tests/__init__.py | 6 +- examples/code_review_evalstats_demo.ipynb | 55 +------- examples/code_review_evalstats_demo.py | 2 +- examples/compare_alignment_ppi.py | 4 +- examples/compare_alignment_ppi_fwer.py | 4 +- examples/compare_alignment_subjects.py | 4 +- simulations/harness/cases/compare_e2e.py | 4 +- simulations/harness/cases/pvalues.py | 4 +- .../investigate_compound_ppi_fwer_power.py | 6 +- ...nvestigate_joint_bootstrap_fwer_highrep.py | 4 +- .../investigate_joint_bootstrap_power_tune.py | 10 +- ...stigate_joint_bootstrap_power_tune_grid.py | 10 +- ...tigate_joint_bootstrap_power_tune_grid2.py | 6 +- simulations/sim_type_i_calibration.py | 2 +- tests/test_alignment.py | 120 +++++++++--------- tests/test_compound_ppi_fwer.py | 48 +++---- 22 files changed, 132 insertions(+), 179 deletions(-) diff --git a/README.md b/README.md index 7d8f0e5..0fc86c7 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,7 @@ import evalstats as es evaldata = es.load_from(df) # Compute alignment between LLM and human judges -alignment = es.validate_alignment( +alignment = es.judge_alignment( evaldata, llm_metric="llm_score", human_groundtruth="human_score", diff --git a/agent_study/sweep/README.md b/agent_study/sweep/README.md index 197a6a6..4c7b3a2 100644 --- a/agent_study/sweep/README.md +++ b/agent_study/sweep/README.md @@ -42,7 +42,7 @@ presupposing in advance that it's important enough to fully cross. **Held fixed / out of scope for this sweep** (see the design discussion that produced this grid, in conversation history, for the reasoning): -- Judge-score-correction (PPI/`validate_alignment`) task type -- deferred to +- Judge-score-correction (PPI/`judge_alignment`) task type -- deferred to a separate, smaller sub-sweep with its own axes (judge reliability, human-label fraction), since those don't apply to `prompts`/`models`. - Correlation structure: paired throughout (`base_corr=1.0` -- the same diff --git a/evalstats/__init__.py b/evalstats/__init__.py index f2b7c0e..e368620 100644 --- a/evalstats/__init__.py +++ b/evalstats/__init__.py @@ -33,7 +33,7 @@ # "compare" name if it were imported before the submodule. from evalstats.loader import load_from, EvalResults, EvalLoadError from evalstats.api import compare, compare_models, compare_prompts, ComparisonResult -from evalstats.alignment import validate_alignment, AlignmentResult +from evalstats.alignment import judge_alignment, AlignmentResult from evalstats import ppi from evalstats import tests @@ -42,7 +42,7 @@ __all__ = [ # High-level spec API "load_from", - "validate_alignment", + "judge_alignment", "AlignmentResult", "ppi", "tests", diff --git a/evalstats/alignment.py b/evalstats/alignment.py index fe83413..cffd50e 100644 --- a/evalstats/alignment.py +++ b/evalstats/alignment.py @@ -1,6 +1,6 @@ """Judge alignment validation and MC-based uncertainty propagation. -Provides :func:`validate_alignment` and :class:`AlignmentResult` for +Provides :func:`judge_alignment` and :class:`AlignmentResult` for characterising how well an LLM judge aligns with human graders, and for propagating that uncertainty into downstream comparisons via Monte-Carlo imputation of latent human labels. @@ -22,7 +22,7 @@ class AlignmentResult: """Carries a fitted calibration model and alignment diagnostics. - Created by :func:`validate_alignment`. Pass it to + Created by :func:`judge_alignment`. Pass it to ``compare(alignment={metric_col: result})`` to widen confidence intervals to account for LLM-judge measurement uncertainty via Monte-Carlo imputation. @@ -593,7 +593,7 @@ def _build_bias_check( interpretation = ( "the judge tracks human relative ordering but disagrees on " "absolute scale — treat raw judge scores as biased; consider " - "using the Bayesian calibration model fit by validate_alignment " + "using the Bayesian calibration model fit by judge_alignment " "(e.g. via compare(alignment=...)) to correct for it before " "drawing conclusions from raw judge scores" ) @@ -978,10 +978,10 @@ def _check_slice_column( # ───────────────────────────────────────────────────────────────────────────── -# validate_alignment +# judge_alignment # ───────────────────────────────────────────────────────────────────────────── -def validate_alignment( +def judge_alignment( evaldata, *, llm_metric: str, diff --git a/evalstats/api.py b/evalstats/api.py index 705cfce..b854c5b 100644 --- a/evalstats/api.py +++ b/evalstats/api.py @@ -1464,7 +1464,7 @@ def _run_alignment_ppi( raise ValueError( f"PPI alignment requires at least 15 human-labeled items; " f"got n_lab={n_lab}. Expand the alignment set and re-run " - "validate_alignment()." + "judge_alignment()." ) if n_all < 50: raise ValueError( diff --git a/evalstats/ppi.py b/evalstats/ppi.py index 29cbe94..a328ae1 100644 --- a/evalstats/ppi.py +++ b/evalstats/ppi.py @@ -613,7 +613,7 @@ def resolve_arrays( group_col : str Column of group labels (factor / condition). alignment_result : AlignmentResult - From :func:`~evalstats.alignment.validate_alignment`. + From :func:`~evalstats.alignment.judge_alignment`. Its ``human_col`` attribute identifies the sparse human-label column. Returns diff --git a/evalstats/tests/__init__.py b/evalstats/tests/__init__.py index b660d50..61c6ef7 100644 --- a/evalstats/tests/__init__.py +++ b/evalstats/tests/__init__.py @@ -42,7 +42,7 @@ Alignment report ---------------- -When human labels are supplied, ``validate_alignment()`` is called +When human labels are supplied, ``judge_alignment()`` is called internally and its report is printed before the test result so alignment quality is always visible. """ @@ -481,7 +481,7 @@ def _run_alignment_report(llm_all: np.ndarray, human_sparse: np.ndarray): ------- AlignmentResult """ - from evalstats.alignment import validate_alignment + from evalstats.alignment import judge_alignment from evalstats.loader import _detect_score_type _LLM = "__llm__" @@ -493,7 +493,7 @@ def __init__(self): self._df = df self._score_types = {_LLM: _detect_score_type(pd.Series(llm_all))} - ar = validate_alignment(_EvalStub(), llm_metric=_LLM, human_groundtruth=_HUM) + ar = judge_alignment(_EvalStub(), llm_metric=_LLM, human_groundtruth=_HUM) ar.summary() return ar diff --git a/examples/code_review_evalstats_demo.ipynb b/examples/code_review_evalstats_demo.ipynb index 57543b6..278a56c 100644 --- a/examples/code_review_evalstats_demo.ipynb +++ b/examples/code_review_evalstats_demo.ipynb @@ -363,58 +363,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "2c3d1033", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Judge alignment report\n", - "──────────────────────────────────────────────────────────\n", - "Alignment set : 300 of 2000 items have human labels (15.0%)\n", - "\n", - "Representativeness diagnostics:\n", - " Score distribution: ✓ KS p=0.953\n", - " -> What this checks: Kolmogorov–Smirnov test comparing the labeled subset's score distribution to the full item pool's.\n", - " -> Why it was computed in this case: The calibration model and alignment metrics above are fit only on the labeled subset; if that subset isn't representative of the full item pool, statistical inference may not generalize to unlabeled items.\n", - " -> How to interpret this result: no evidence (p ≥ 0.05) that the score distribution differs between the labeled subset and the full pool — alignment estimates should generalize reasonably well\n", - "\n", - " 'model': ✓ χ² p=1.000\n", - " -> What this checks: Chi-square test comparing the distribution of 'model' between labeled and unlabeled items.\n", - " -> Why it was computed in this case: Checks whether the alignment set is representative across this categorical variable — important if judge accuracy might vary by subgroup (e.g. domain, difficulty, model).\n", - " -> How to interpret this result: no evidence (p ≥ 0.05) that 'model' differs between the labeled subset and the full pool — alignment estimates should generalize reasonably well\n", - "\n", - "Alignment metrics (score type: likert):\n", - " (labels are ordered categories, so metrics designed for ordinal data are used)\n", - "\n", - " Weighted Cohen's κ : 0.518 [0.431, 0.597]\n", - " -> What this metric is: Cohen's κ extended so that disagreements receive larger penalties as ratings become farther apart on the ordinal scale (Cohen, 1968).\n", - " -> Why it was computed in this case: Your judge produces ordered categorical (Likert) labels, so an ordinal-aware kappa is used instead of the unweighted version, which would penalize a near-miss (e.g. judge=4 vs human=5) as harshly as a large disagreement.\n", - " -> How to interpret this result: moderate agreement (Landis & Koch, 1977 benchmarks)\n", - " -> Example paper reporting: \"Weighted Cohen's κ = 0.52, 95% CI [0.43, 0.60] (n=300), indicating moderate agreement between the LLM judge and human raters, per the Landis & Koch (1977) benchmarks.\"\n", - "\n", - " Spearman r : 0.537 [0.440, 0.619]\n", - " -> What this metric is: Rank correlation between judge and human scores — checks whether higher judge scores correspond to higher human scores, without assuming the categories are equally spaced.\n", - " -> Why it was computed in this case: Reported alongside weighted κ to show whether the judge preserves relative ordering, which matters if judge scores are mainly used to rank or compare outputs.\n", - " -> How to interpret this result: large positive correlation (Cohen, 1988 conventions)\n", - " -> Example paper reporting: \"Spearman r = 0.54, 95% CI [0.44, 0.62] (n=300), a large positive correlation between the LLM judge and human scores (Cohen, 1988 conventions).\"\n", - "\n", - "──────────────────────────────────────────────────────────\n" - ] - } - ], - "source": [ - "alignment = es.validate_alignment(\n", - " evaldata,\n", - " llm_metric=\"review_score\",\n", - " human_groundtruth=\"expert_score\",\n", - ")\n", - "alignment.summary()\n", - "\n", - "# We need text in evalstats outputs that justifies explanation of why certain stats tests were used." - ] + "outputs": [], + "source": "alignment = es.judge_alignment(\n evaldata,\n llm_metric=\"review_score\",\n human_groundtruth=\"expert_score\",\n)\nalignment.summary()\n\n# We need text in evalstats outputs that justifies explanation of why certain stats tests were used." }, { "cell_type": "markdown", @@ -718,4 +671,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/examples/code_review_evalstats_demo.py b/examples/code_review_evalstats_demo.py index b823fa8..2688f5a 100644 --- a/examples/code_review_evalstats_demo.py +++ b/examples/code_review_evalstats_demo.py @@ -73,7 +73,7 @@ def main() -> None: # ── Step 3: validate the judge against the human labels we have ───────── _banner("STEP 3 — Validate the judge against the 30-item human gold set") - alignment = es.validate_alignment( + alignment = es.judge_alignment( evaldata, llm_metric="review_score", human_groundtruth="expert_score", diff --git a/examples/compare_alignment_ppi.py b/examples/compare_alignment_ppi.py index c85768f..3b992e7 100644 --- a/examples/compare_alignment_ppi.py +++ b/examples/compare_alignment_ppi.py @@ -7,7 +7,7 @@ model_A, causing it to dramatically overestimate model_A's win rate. Demonstrates: - 1. validate_alignment() — quantify LLM-vs-human agreement on the gold set. + 1. judge_alignment() — quantify LLM-vs-human agreement on the gold set. 2. compare(..., alignment=...) — PPI-corrected model comparison that uses the human labels to debias the LLM-only estimates. 3. es.ppi.correct() — apply PPI to a custom estimator (win-rate advantage @@ -111,7 +111,7 @@ print("STEP 1 — Validate LLM judge alignment") print("=" * 62) -ar = es.validate_alignment( +ar = es.judge_alignment( evaldata, llm_metric="llm_score", human_groundtruth="human_score", diff --git a/examples/compare_alignment_ppi_fwer.py b/examples/compare_alignment_ppi_fwer.py index 5d00acc..74d9396 100644 --- a/examples/compare_alignment_ppi_fwer.py +++ b/examples/compare_alignment_ppi_fwer.py @@ -4,7 +4,7 @@ Simulates a common real-world scenario: - 4 models are scored by an LLM judge on 150 items each (cheap, scalable). - A human annotator has labelled 60 of those items per model (expensive, - sparse) -- validate_alignment()/compare(alignment=...) use these to + sparse) -- judge_alignment()/compare(alignment=...) use these to debias the LLM-only estimates via Prediction-Powered Inference (PPI). - With 4 models there are C(4,2) = 6 pairwise comparisons, so a family-wise error rate (FWER) correction is also needed to avoid false positives @@ -124,7 +124,7 @@ print("STEP 1 — Validate LLM judge alignment") print("=" * 70) -ar = es.validate_alignment( +ar = es.judge_alignment( evaldata, llm_metric="llm_score", human_groundtruth="human_score", diff --git a/examples/compare_alignment_subjects.py b/examples/compare_alignment_subjects.py index 4f96278..b17a2bb 100644 --- a/examples/compare_alignment_subjects.py +++ b/examples/compare_alignment_subjects.py @@ -155,7 +155,7 @@ def _human(quality: int) -> float: print("=" * 62) print("STEP 1 — LLM judge alignment (between-subjects data)") print("=" * 62) -ar_bs = es.validate_alignment( +ar_bs = es.judge_alignment( evaldata_bs, llm_metric="llm_score", human_groundtruth="human_score", @@ -165,7 +165,7 @@ def _human(quality: int) -> float: print("=" * 62) print("STEP 1 — LLM judge alignment (within-subjects data)") print("=" * 62) -ar_ws = es.validate_alignment( +ar_ws = es.judge_alignment( evaldata_ws, llm_metric="llm_score", human_groundtruth="human_score", diff --git a/simulations/harness/cases/compare_e2e.py b/simulations/harness/cases/compare_e2e.py index c481fd2..3078962 100644 --- a/simulations/harness/cases/compare_e2e.py +++ b/simulations/harness/cases/compare_e2e.py @@ -70,7 +70,7 @@ import pandas as pd import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from evalstats.core.stats_utils import interval_score from ..latex_tables import booktabs_table, escape_latex @@ -524,7 +524,7 @@ def _run_cell( if ppi_frac is not None: with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="score", human_groundtruth="human_score") kwargs["alignment"] = {"score": ar} if score_range is not None: kwargs["score_range"] = score_range diff --git a/simulations/harness/cases/pvalues.py b/simulations/harness/cases/pvalues.py index 2d4beee..c836af4 100644 --- a/simulations/harness/cases/pvalues.py +++ b/simulations/harness/cases/pvalues.py @@ -3240,7 +3240,7 @@ def save_simultaneous_ci_violin_vs_n_plot(*, results: list[SimultaneousCIResult] # wrappers under judge bias/miscalibration, ported from # sim_type_i_calibration.py's _run_one. Calls evalstats.tests' internal PPI # functions directly (the same functions back the public es.tests.* API) to -# skip validate_alignment overhead, exactly as the legacy script does. +# skip judge_alignment overhead, exactly as the legacy script does. # --------------------------------------------------------------------------- @@ -6910,7 +6910,7 @@ def _kappa_band(x: float) -> str: """Landis & Koch (1977) benchmarks for kappa-type statistics -- same bands evalstats.alignment._interpret_kappa uses for the public alignment report, reused here so a bucket's qualitative label matches what a user - would see calling validate_alignment() on the same kind of judge.""" + would see calling judge_alignment() on the same kind of judge.""" if x < 0: return "poor" if x <= 0.20: diff --git a/simulations/investigate_compound_ppi_fwer_power.py b/simulations/investigate_compound_ppi_fwer_power.py index 671cb6c..5e1154e 100644 --- a/simulations/investigate_compound_ppi_fwer_power.py +++ b/simulations/investigate_compound_ppi_fwer_power.py @@ -15,7 +15,7 @@ sys.path.insert(0, "tests") import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from test_compound_ppi_fwer import _make_multiarm_binary, _rng N_REPS_POWER = 150 @@ -32,7 +32,7 @@ def measure_power(seed_base: int = 2000) -> float: ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -58,7 +58,7 @@ def measure_null_fwer(seed_base: int = 1000, n_reps: int = N_REPS_NULL) -> float ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, diff --git a/simulations/investigate_joint_bootstrap_fwer_highrep.py b/simulations/investigate_joint_bootstrap_fwer_highrep.py index 9ec5ae9..bb75c96 100644 --- a/simulations/investigate_joint_bootstrap_fwer_highrep.py +++ b/simulations/investigate_joint_bootstrap_fwer_highrep.py @@ -14,7 +14,7 @@ sys.path.insert(0, "tests") import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from test_compound_ppi_fwer import _make_multiarm_binary, _rng from simulations.investigate_joint_bootstrap_power_tune import _PowerTuneOverride @@ -44,7 +44,7 @@ def measure_null_fwer_only(n_entities: int, n_items: int, label_frac: float, pow ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, diff --git a/simulations/investigate_joint_bootstrap_power_tune.py b/simulations/investigate_joint_bootstrap_power_tune.py index 89dc61c..f0c56f8 100644 --- a/simulations/investigate_joint_bootstrap_power_tune.py +++ b/simulations/investigate_joint_bootstrap_power_tune.py @@ -22,7 +22,7 @@ import evalstats as es import evalstats.api as api_mod -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from test_compound_ppi_fwer import _make_multiarm_binary, _rng N_REPS_NULL = 200 @@ -64,7 +64,7 @@ def measure_romano_wolf(power_tune: bool, seed_base: int = 4000, effect_size: fl ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -88,7 +88,7 @@ def measure_romano_wolf_null_fwer(power_tune: bool, seed_base: int = 3000, n_rep ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -133,7 +133,7 @@ def measure_max_t(power_tune: bool, seed_base: int = 5000, effect_size: float = ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = _compare_with_max_t(evaldata, ar, seed_base + i) bundle = result._primary_bundle() assert bundle.pairwise.simultaneous_ci_method == "max_t" @@ -146,7 +146,7 @@ def measure_max_t(power_tune: bool, seed_base: int = 5000, effect_size: float = ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = _compare_with_max_t(evaldata, ar, seed_base + 10000 + i) p = result._primary_bundle().pairwise.get("M0", "M3").p_value if p is not None and p < alpha: diff --git a/simulations/investigate_joint_bootstrap_power_tune_grid.py b/simulations/investigate_joint_bootstrap_power_tune_grid.py index a09d00d..85102d8 100644 --- a/simulations/investigate_joint_bootstrap_power_tune_grid.py +++ b/simulations/investigate_joint_bootstrap_power_tune_grid.py @@ -17,7 +17,7 @@ sys.path.insert(0, "tests") import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from test_compound_ppi_fwer import _make_multiarm_binary, _rng from simulations.investigate_joint_bootstrap_power_tune import _PowerTuneOverride, _compare_with_max_t @@ -62,7 +62,7 @@ def measure_romano_wolf_condition(n_entities: int, n_items: int, label_frac: flo ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -81,7 +81,7 @@ def measure_romano_wolf_condition(n_entities: int, n_items: int, label_frac: flo ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -108,7 +108,7 @@ def measure_max_t_condition(n_entities: int, n_items: int, label_frac: float, po ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = _compare_with_max_t(evaldata, ar, seed_base + i) bundle = result._primary_bundle() assert bundle.pairwise.simultaneous_ci_method == "max_t" @@ -123,7 +123,7 @@ def measure_max_t_condition(n_entities: int, n_items: int, label_frac: float, po ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = _compare_with_max_t(evaldata, ar, seed_base + 20000 + i) bundle = result._primary_bundle() last_pair = last_pair or (f"M0", f"M{n_entities - 1}") diff --git a/simulations/investigate_joint_bootstrap_power_tune_grid2.py b/simulations/investigate_joint_bootstrap_power_tune_grid2.py index 4ea251f..f3b3ed8 100644 --- a/simulations/investigate_joint_bootstrap_power_tune_grid2.py +++ b/simulations/investigate_joint_bootstrap_power_tune_grid2.py @@ -15,7 +15,7 @@ sys.path.insert(0, "tests") import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment from test_compound_ppi_fwer import _make_multiarm_binary, _rng from simulations.investigate_joint_bootstrap_power_tune import _PowerTuneOverride @@ -50,7 +50,7 @@ def measure_condition(n_entities: int, n_items: int, label_frac: float, power_tu ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -69,7 +69,7 @@ def measure_condition(n_entities: int, n_items: int, label_frac: float, power_tu ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, diff --git a/simulations/sim_type_i_calibration.py b/simulations/sim_type_i_calibration.py index d2a27f3..efbb2e6 100644 --- a/simulations/sim_type_i_calibration.py +++ b/simulations/sim_type_i_calibration.py @@ -91,7 +91,7 @@ (_plot_power_results) instead of the Type I one. The internal PPI functions (_ppi_two_sample, _ppi_paired_arrays, etc.) are called -directly to skip the validate_alignment overhead (~360ms/call) — we are testing +directly to skip the judge_alignment overhead (~360ms/call) — we are testing statistical calibration, not the pipeline UX. Usage: diff --git a/tests/test_alignment.py b/tests/test_alignment.py index a0a5c09..a18e91b 100644 --- a/tests/test_alignment.py +++ b/tests/test_alignment.py @@ -1,4 +1,4 @@ -"""Tests for validate_alignment() and compare(alignment=...) PPI propagation.""" +"""Tests for judge_alignment() and compare(alignment=...) PPI propagation.""" from __future__ import annotations @@ -10,7 +10,7 @@ import evalstats as es from evalstats.config import GRADIENT_CI_ALPHAS -from evalstats.alignment import AlignmentResult, validate_alignment, _fit_calibration +from evalstats.alignment import AlignmentResult, judge_alignment, _fit_calibration from evalstats.api import ComparisonResult @@ -102,22 +102,22 @@ def _make_continuous_evaldata( # --------------------------------------------------------------------------- -# validate_alignment — basic contracts +# judge_alignment — basic contracts # --------------------------------------------------------------------------- -class TestValidateAlignmentBasic: +class TestJudgeAlignmentBasic: def test_returns_alignment_result(self): evaldata, metric = _make_binary_evaldata() with warnings.catch_warnings(): warnings.simplefilter("ignore") - result = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + result = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert isinstance(result, AlignmentResult) def test_stores_metadata(self): evaldata, metric = _make_binary_evaldata(n_items=60, n_labeled=30) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert ar.llm_metric == metric assert ar.human_col == "human_score" assert ar.score_type == "binary" @@ -127,35 +127,35 @@ def test_stores_metadata(self): def test_raises_missing_llm_column(self): evaldata, _ = _make_binary_evaldata() with pytest.raises(ValueError, match="llm_metric column"): - validate_alignment(evaldata, llm_metric="nonexistent", human_groundtruth="human_score") + judge_alignment(evaldata, llm_metric="nonexistent", human_groundtruth="human_score") def test_raises_missing_human_column(self): evaldata, metric = _make_binary_evaldata() with pytest.raises(ValueError, match="human_groundtruth column"): - validate_alignment(evaldata, llm_metric=metric, human_groundtruth="nonexistent") + judge_alignment(evaldata, llm_metric=metric, human_groundtruth="nonexistent") def test_raises_no_labels_at_all(self): evaldata, metric = _make_binary_evaldata() evaldata._df["human_score"] = np.nan # wipe all labels with pytest.raises(ValueError, match="No rows have human labels"): - validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") def test_warns_small_n_labeled(self): evaldata, metric = _make_binary_evaldata(n_labeled=15) with pytest.warns(UserWarning, match="fewer than ~30 labeled items"): - validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") def test_no_small_n_warning_above_threshold(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") small_n_warns = [w for w in caught if "fewer than ~30" in str(w.message)] assert len(small_n_warns) == 0 # --------------------------------------------------------------------------- -# validate_alignment — alignment metrics by score type +# judge_alignment — alignment metrics by score type # --------------------------------------------------------------------------- class TestAlignmentMetrics: @@ -163,7 +163,7 @@ def test_binary_has_agreement_and_kappa(self): evaldata, metric = _make_binary_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert "percent_agreement" in ar.alignment_metrics assert "cohens_kappa" in ar.alignment_metrics @@ -171,7 +171,7 @@ def test_binary_agreement_in_range(self): evaldata, metric = _make_binary_evaldata(n_labeled=50, agreement_rate=0.80) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") pa = ar.alignment_metrics["percent_agreement"]["estimate"] assert 0.0 <= pa <= 1.0 # With ~80% agreement rate we expect measured agreement between 0.5 and 1.0 @@ -181,7 +181,7 @@ def test_binary_ci_bounds_ordered(self): evaldata, metric = _make_binary_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") for entry in ar.alignment_metrics.values(): assert entry["ci_low"] <= entry["estimate"] <= entry["ci_high"] @@ -189,7 +189,7 @@ def test_likert_has_weighted_kappa_and_spearman(self): evaldata, metric = _make_likert_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert "weighted_kappa" in ar.alignment_metrics assert "spearman_r" in ar.alignment_metrics @@ -197,7 +197,7 @@ def test_continuous_has_pearson_and_spearman(self): evaldata, metric = _make_continuous_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert "pearson_r" in ar.alignment_metrics assert "spearman_r" in ar.alignment_metrics @@ -216,13 +216,13 @@ def test_perfect_agreement_kappa_near_one(self): human[labeled_idx] = df.loc[labeled_idx, "llm_score"].to_numpy() df["human_score"] = human evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") kappa = ar.alignment_metrics["cohens_kappa"]["estimate"] assert kappa >= 0.90 # --------------------------------------------------------------------------- -# validate_alignment — representativeness checks +# judge_alignment — representativeness checks # --------------------------------------------------------------------------- class TestRepresentativenessCheck: @@ -231,7 +231,7 @@ def test_representative_set_passes(self): evaldata, metric = _make_binary_evaldata(n_labeled=40, seed=7) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") assert "score_distribution" in ar.representativeness def test_skewed_alignment_set_warns(self): @@ -254,7 +254,7 @@ def test_skewed_alignment_set_warns(self): with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") repr_warns = [w for w in caught if "non-representative" in str(w.message).lower() or "representative" in str(w.message).lower()] assert len(repr_warns) >= 1 @@ -268,7 +268,7 @@ def test_slice_column_check_added_to_result(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") slice_keys = [k for k in ar.representativeness if k.startswith("slice_")] assert "slice_difficulty" in slice_keys @@ -282,7 +282,7 @@ def test_binary_output_is_zero_or_one(self): evaldata, metric = _make_binary_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") llm_scores = evaldata._df[metric].to_numpy(dtype=float) rng = _rng(10) imputed = ar._sample_imputed_scores(llm_scores, rng) @@ -293,7 +293,7 @@ def test_likert_output_in_category_set(self): evaldata, metric = _make_likert_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") llm_scores = evaldata._df[metric].to_numpy(dtype=float) rng = _rng(11) imputed = ar._sample_imputed_scores(llm_scores, rng) @@ -305,7 +305,7 @@ def test_continuous_output_is_float_array(self): evaldata, metric = _make_continuous_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") llm_scores = evaldata._df[metric].to_numpy(dtype=float) rng = _rng(12) imputed = ar._sample_imputed_scores(llm_scores, rng) @@ -317,7 +317,7 @@ def test_different_rng_states_give_different_draws(self): evaldata, metric = _make_binary_evaldata(n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") llm_scores = evaldata._df[metric].to_numpy(dtype=float) draw1 = ar._sample_imputed_scores(llm_scores, _rng(1)) draw2 = ar._sample_imputed_scores(llm_scores, _rng(2)) @@ -338,7 +338,7 @@ def test_perfect_calibration_produces_near_identical_scores(self): human[labeled_idx] = df.loc[labeled_idx, "llm_score"].to_numpy() df["human_score"] = human evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") llm_scores = evaldata._df["llm_score"].to_numpy(dtype=float) imputed = ar._sample_imputed_scores(llm_scores, _rng(99)) @@ -362,7 +362,7 @@ def test_cis_widen_under_misalignment_binary(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result_mc = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=30) result_base = es.compare(evaldata, factors="model", metric=metric) @@ -399,7 +399,7 @@ def _make_scenario(agreement_rate, seed): evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=50) @@ -438,7 +438,7 @@ def test_rubin_cis_converge_under_perfect_alignment(self): df["human_score"] = human evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result_mc = es.compare(evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=50) result_base = es.compare(evaldata, factors="model", metric="llm_score") @@ -458,7 +458,7 @@ def test_variance_components_populated(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) d = result.to_dict() @@ -491,7 +491,7 @@ def test_pairwise_cis_also_widen(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result_mc = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=30) result_base = es.compare(evaldata, factors="model", metric=metric) @@ -506,7 +506,7 @@ def test_alignment_works_with_likert(self): evaldata, metric = _make_likert_evaldata(n_items=60, n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) assert isinstance(result, ComparisonResult) @@ -518,7 +518,7 @@ def test_alignment_works_with_continuous(self): evaldata, metric = _make_continuous_evaldata(n_items=60, n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) assert isinstance(result, ComparisonResult) @@ -544,7 +544,7 @@ def test_alignment_works_with_path_c_arbitrary_factor(self): evaldata = es.load_from(df, col_map={"system": "model", "item": "item"}) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=20) assert isinstance(result, ComparisonResult) @@ -555,7 +555,7 @@ def test_wrong_alignment_key_warns(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") base = es.compare(evaldata, factors="model", metric=metric) with pytest.warns(UserWarning, match="no entry for metric column"): @@ -572,7 +572,7 @@ def test_alignment_not_dict_warns(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") with pytest.warns(UserWarning, match="must be a dict"): result = es.compare(evaldata, factors="model", metric=metric, alignment=ar, n_mc=20) @@ -596,7 +596,7 @@ def test_multimodel_alignment_warns_not_supported(self): evaldata = es.load_from(df, col_map={"model": "model", "prompt": "prompt", "item": "item"}) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.warns(UserWarning, match="not yet supported"): es.compare(evaldata, factors="model", metric="llm_score", @@ -615,7 +615,7 @@ def test_pairwise_pvalues_present_after_mc(self): evaldata, metric = _make_binary_evaldata(n_items=80, n_labeled=40, seed=61) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=25) @@ -630,7 +630,7 @@ def test_pvalues_in_valid_range(self): evaldata, metric = _make_binary_evaldata(n_items=80, n_labeled=40, seed=62) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=25) @@ -652,7 +652,7 @@ def test_ci_excludes_zero_implies_p_significant(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alpha=0.05, alignment={metric: ar}, n_mc=30, rng=np.random.default_rng(99)) @@ -670,7 +670,7 @@ def test_n_mc_small_succeeds(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=1) vc = result.to_dict()["variance_components"] @@ -682,7 +682,7 @@ def test_n_mc_zero_succeeds(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=0) vc = result.to_dict()["variance_components"] @@ -694,7 +694,7 @@ def test_pairwise_pvalues_consistent_across_directions(self): evaldata, metric = _make_binary_evaldata(n_items=80, n_labeled=40, seed=64) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=25) @@ -714,7 +714,7 @@ def test_correction_method_applied_to_pooled_pvalues(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") # Compare with different correction methods (via backend's native routing) # Note: correction is currently hardcoded in _run_alignment_mc, but we can @@ -730,7 +730,7 @@ def test_reproducibility_with_seeded_rng(self): evaldata, metric = _make_binary_evaldata(n_items=80, n_labeled=40, seed=66) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") rng1 = np.random.default_rng(42) result1 = es.compare(evaldata, factors="model", metric=metric, @@ -771,7 +771,7 @@ def _get_pvalues(agreement_rate, seed): evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=35) @@ -797,7 +797,7 @@ def test_pairwise_point_diff_and_ci_consistent(self): evaldata, metric = _make_binary_evaldata(n_items=80, n_labeled=40, seed=67) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=25) @@ -813,7 +813,7 @@ def test_pvalues_populated_with_likert_alignment(self): evaldata, metric = _make_likert_evaldata(n_items=80, n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) @@ -827,7 +827,7 @@ def test_pvalues_populated_with_continuous_alignment(self): evaldata, metric = _make_continuous_evaldata(n_items=80, n_labeled=40) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) @@ -841,7 +841,7 @@ def test_multi_ci_populated_after_mc(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=20) bundle = result._primary_bundle() @@ -861,7 +861,7 @@ def test_n_mc_parameter_controls_n_boot(self): evaldata, metric = _make_binary_evaldata(n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") for n_mc, expected_n_boot in [(10, 1000), (25, 1000), (2000, 2000)]: result = es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, n_mc=n_mc) @@ -898,7 +898,7 @@ def test_raises_when_n_lab_below_15(self): evaldata = _make_small_evaldata(n_items=40, n_labeled=10) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.raises(ValueError, match="15 human-labeled items"): es.compare(evaldata, factors="model", metric="llm_score", @@ -910,7 +910,7 @@ def test_raises_when_n_all_below_50(self): evaldata = _make_small_evaldata(n_items=20, n_labeled=15) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.raises(ValueError, match="50 items"): es.compare(evaldata, factors="model", metric="llm_score", @@ -923,7 +923,7 @@ def test_warns_when_n_lab_below_30(self): evaldata = _make_small_evaldata(n_items=60, n_labeled=20) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.warns(UserWarning, match="recommend ≥ 30"): es.compare(evaldata, factors="model", metric="llm_score", @@ -935,7 +935,7 @@ def test_warns_when_n_all_below_100(self): evaldata = _make_small_evaldata(n_items=30, n_labeled=30) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.warns(UserWarning, match="recommend ≥ 100"): es.compare(evaldata, factors="model", metric="llm_score", @@ -968,7 +968,7 @@ def test_raises_clear_error_when_entity_is_100pct_labeled(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with pytest.raises(ValueError, match="at least one unlabeled item"): es.compare(evaldata, factors="model", metric="llm_score", @@ -979,7 +979,7 @@ def test_no_size_warnings_above_thresholds(self): evaldata, metric = _make_binary_evaldata(n_items=60, n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}) @@ -1002,7 +1002,7 @@ def test_raises_when_method_has_no_ppi_correction(self): evaldata, metric = _make_binary_evaldata(n_items=60, n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") with pytest.raises(ValueError, match="no validated implementation"): es.compare(evaldata, factors="model", metric=metric, alignment={metric: ar}, @@ -1014,7 +1014,7 @@ def test_plain_bootstrap_method_is_ppi_corrected(self): evaldata, metric = _make_binary_evaldata(n_items=60, n_labeled=35) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric=metric, human_groundtruth="human_score") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") result = es.compare(evaldata, factors="model", metric=metric, diff --git a/tests/test_compound_ppi_fwer.py b/tests/test_compound_ppi_fwer.py index 4b52efc..021cf44 100644 --- a/tests/test_compound_ppi_fwer.py +++ b/tests/test_compound_ppi_fwer.py @@ -22,7 +22,7 @@ import pandas as pd import evalstats as es -from evalstats.alignment import validate_alignment +from evalstats.alignment import judge_alignment def _rng(seed: int = 0) -> np.random.Generator: @@ -150,7 +150,7 @@ def test_simultaneous_ci_flag_and_valid_output_survive_ppi(self): evaldata = _make_multiarm_binary(n_entities=3, seed=10) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -172,7 +172,7 @@ def test_bonferroni_pair_alpha_widens_ppi_cis_vs_non_simultaneous(self): evaldata = _make_multiarm_binary(n_entities=3, seed=11) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result_sim = es.compare( evaldata, factors="model", metric="llm_score", @@ -210,7 +210,7 @@ def test_correction_only_changes_pvalues_not_cis(self): evaldata = _make_multiarm_binary(n_entities=3, seed=12) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") # Both calls share a seeded rng: at this N (150/entity, binary), # simultaneous_ci=True now resolves to "boot" (joint bootstrap @@ -248,7 +248,7 @@ def test_default_auto_reaches_boot_at_n_ge_30(self): evaldata = _make_multiarm_continuous(n_entities=3, n_items=200, n_labeled=80, seed=13) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -267,7 +267,7 @@ def test_default_auto_reaches_sidak_below_n_threshold(self): evaldata = _make_multiarm_continuous(n_entities=3, n_items=25, n_labeled=15, seed=131) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -288,7 +288,7 @@ def test_method_bootstrap_t_uses_boot_not_max_t_under_auto(self): evaldata = _make_multiarm_binary(n_entities=3, n_items=150, n_labeled=60, seed=14) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=50, method="bootstrap_t", @@ -315,7 +315,7 @@ def test_prefer_kwarg_is_not_forwarded_through_compare(self): evaldata = _make_multiarm_binary(n_entities=3, n_items=150, n_labeled=60, seed=141) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") @@ -344,7 +344,7 @@ def test_explicit_max_t_reachable_via_private_function(self): evaldata = _make_multiarm_binary(n_entities=3, n_items=150, n_labeled=60, seed=142) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", method="bootstrap_t", simultaneous_ci=True, correction="shaffer", @@ -378,7 +378,7 @@ def test_max_t_silently_falls_back_to_bonferroni_when_overlap_insufficient(self) evaldata, _labels = _make_mixed_branch_binary(seed=15) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") @@ -402,7 +402,7 @@ def test_skipped_pair_stays_uncorrected_while_others_are_ppi_corrected(self): evaldata, labels = _make_mixed_branch_binary(seed=16) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -423,7 +423,7 @@ def test_two_arm_compound_does_not_widen_for_fwer(self): evaldata = _make_multiarm_binary(n_entities=2, seed=17) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result_sim = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, method="tango", @@ -456,7 +456,7 @@ def test_correction_auto_resolves_instead_of_crashing(self): evaldata = _make_multiarm_binary(n_entities=3, n_items=150, n_labeled=60, seed=143) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -484,7 +484,7 @@ def test_explicit_romano_wolf_produces_valid_pvalues(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=150, n_labeled=60, seed=200) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -503,7 +503,7 @@ def test_auto_resolves_to_romano_wolf_at_n_ge_30(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=100, n_labeled=40, seed=201) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -517,7 +517,7 @@ def test_auto_resolves_to_shaffer_below_n_30(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=25, n_labeled=15, seed=202) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -535,7 +535,7 @@ def test_romano_wolf_falls_back_to_shaffer_when_overlap_insufficient(self): evaldata, _labels = _make_mixed_branch_binary(seed=203) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -557,7 +557,7 @@ def test_wilcoxon_companion_pvalues_use_shaffer_not_romano_wolf(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=150, n_labeled=60, seed=204) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -577,7 +577,7 @@ def test_correction_method_field_reflects_actual_correction(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=150, n_labeled=60, seed=205) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result_none = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -609,7 +609,7 @@ def test_romano_wolf_reuses_joint_resample_shared_with_boot(self): evaldata = _make_multiarm_binary(n_entities=4, n_items=150, n_labeled=60, seed=206) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -652,7 +652,7 @@ def test_fwer_controlled_under_null_with_noisy_judge(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -712,7 +712,7 @@ def test_compound_correction_power_cost_is_bounded(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -764,7 +764,7 @@ def test_romano_wolf_fwer_controlled_under_null(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, @@ -803,7 +803,7 @@ def test_romano_wolf_power_not_worse_than_shaffer(self): ) with warnings.catch_warnings(): warnings.simplefilter("ignore") - ar = validate_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") + ar = judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") result_shaffer = es.compare( evaldata, factors="model", metric="llm_score", alignment={"llm_score": ar}, n_mc=30, From b6b47434f4cb653561decfa88a98edb1c5e3bf27 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 19:05:13 -0400 Subject: [PATCH 007/245] Fix NIG's prior variance being reused unscaled for ci_paired's wider diff rescale nig_ci_1d's default b0=0.0625 (prior mean of sigma^2, sigma~=0.25) is documented as calibrated for ci_single.py's rescale span [scale_lo, scale_hi] ("weak knowledge that scores live in [0, 1]"). ci_paired.py instead rescales paired diffs onto [-diff_span, diff_span] (needed so a zero diff maps to 0.5, NIG's own prior centre) -- twice as wide a span as ci_single's. Reusing b0=0.0625 unchanged there implies 2^2=4x the prior variance in real diff units (variance scales with the square of a linear rescale factor), producing persistent, substantial over-coverage that isn't a deliberate safety margin -- just an unpropagated rescale-span change, not a general flaw in NIG's prior. Confirmed via direct comparison against ci_single.py's own likert usage (same eval type, correct narrower rescale): already well-calibrated there (0.947/0.952/0.932 at n=10/30/100, no systematic over-coverage) -- isolating the bug to ci_paired.py's wider rescale specifically, not NIG generally. Fixed by passing b0=0.0625/4 to nig_ci_1d in both ci_paired.py call sites (flat-mode _run_cell and nested-mode _run_nested_pairwise_cell), via functools.partial -- restores NIG's effective prior to the same absolute variance ci_single.py already uses, not a new invented value. The functools.partial wrapping required switching the flat-mode dispatch's "which methods need rescaling" check from function identity (`fn is nig_ci_1d`) to method identity (`method is NIG`), since a partial-wrapped function is no longer identical to the original. Re-validated at reps=300 (flat) / reps=150 (nested) across the corrected icc range (0.01-0.95), continuous and likert: - Flat, likert: coverage 0.983 -> 0.952 at n=10, width 23% narrower; now narrower than logit_t's own width at every n. - Flat, continuous: coverage 0.983 -> 0.964 at n=10 (still a touch conservative but much closer), width matched to logit_t's. - Nested, likert: coverage 0.951, essentially identical to logit_t's 0.951, width narrower (0.585 vs 0.626). - Nested, continuous: coverage 0.957, width narrower than logit_t's (0.136 vs 0.143). No warnings or errors in any re-validation run. Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/ci_paired.py | 31 +++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/simulations/harness/cases/ci_paired.py b/simulations/harness/cases/ci_paired.py index c29246d..03acd63 100644 --- a/simulations/harness/cases/ci_paired.py +++ b/simulations/harness/cases/ci_paired.py @@ -53,6 +53,7 @@ import argparse import csv +import functools import io import itertools import multiprocessing as _mp @@ -321,6 +322,28 @@ def _pairwise_ci( return float(np.percentile(boot_stats, 100 * alpha / 2)), float(np.percentile(boot_stats, 100 * (1 - alpha / 2))) +_NIG_PAIRED_DIFF_B0 = 0.0625 / 4 +"""nig_ci_1d's default b0=0.0625 (prior mean of sigma^2, i.e. prior +sigma~=0.25) is calibrated for ci_single.py's own rescale span +[scale_lo, scale_hi] -- see that function's docstring: "weak knowledge +that scores live in [0, 1]". ci_paired.py instead rescales paired diffs +onto [-diff_span, diff_span] = [-(scale_hi-scale_lo), (scale_hi-scale_lo)] +(needed so a zero diff maps to 0.5, nig's own prior centre) -- TWICE as +wide a span as ci_single's own [scale_lo, scale_hi]. Reusing b0=0.0625 +unchanged there implies 2^2=4x the prior variance in real diff units +(variance scales with the square of a linear rescale factor) versus what +ci_single already uses for a raw score on the same eval type, causing +persistent, substantial over-coverage that isn't a deliberate safety +margin, just an unpropagated rescale-span change. Verified directly on +likert paired diffs: coverage 0.983 (n=10, default b0) vs 0.946 (n=10, +this correction) -- the corrected version is 23% NARROWER for the same +validity, and the same ~20-30% narrowing (with coverage moving from +badly over- to essentially exactly at nominal) holds at n=30, n=100, and +on continuous data too. This restores NIG's effective prior to match +ci_single.py's own calibration point; it is not a new invented value, +just correctly propagated through the wider diff rescale.""" + + def _detect_dither_halfwidth(pooled: np.ndarray) -> float: """Auto-detect a rounding/quantization grid step from pooled raw arm values (both arms, one rep) and return half that step -- the dither @@ -554,12 +577,13 @@ def _record(method, ci_low: float, ci_high: float) -> None: _scale_lo, _scale_hi = EVAL_TYPE_SCALE_BOUNDS[source_obj.eval_type] diff_span = _scale_hi - _scale_lo diff_lo, diff_hi = -diff_span, diff_span - _extra_fns = dict(zip(PAIRWISE_EXTRA_METHODS, (t_interval_ci_1d, logit_t_ci_1d, nig_ci_1d, el_ci_1d))) + _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) + _extra_fns = dict(zip(PAIRWISE_EXTRA_METHODS, (t_interval_ci_1d, logit_t_ci_1d, _nig_paired, el_ci_1d))) for method in active_pairwise_extras: fn = _extra_fns[method] _t0 = time.perf_counter() try: - if fn is nig_ci_1d or fn is logit_t_ci_1d: + if method is NIG or method is LOGIT_T: ci_low, ci_high = rescaled_ci(fn, pair_diffs, alpha, diff_lo, diff_hi) else: ci_low, ci_high = fn(pair_diffs, alpha) @@ -885,7 +909,8 @@ def _record(method, ci_low: float, ci_high: float) -> None: _scale_lo, _scale_hi = EVAL_TYPE_SCALE_BOUNDS[source_obj.eval_type] diff_span = _scale_hi - _scale_lo diff_lo, diff_hi = -diff_span, diff_span - for method, fn in zip((LOGIT_T, NIG, EL), (logit_t_ci_1d, nig_ci_1d, el_ci_1d)): + _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) + for method, fn in zip((LOGIT_T, NIG, EL), (logit_t_ci_1d, _nig_paired, el_ci_1d)): if not _want(method.name): continue _t0 = time.perf_counter() From 7a79229d8964bba70dbf5e9daebbe147ffed299b Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 19:17:57 -0400 Subject: [PATCH 008/245] Add quick primitives: mean_ci, summarize, stability, judge_debias_mean_ci For users who don't want compare()'s full comparative report -- just a trustworthy point estimate (or a few other common building blocks) as plain data to hand to their own plotting library. All reuse the exact auto method-selection/calibration machinery compare() uses internally (extracted into router.resolve_auto_robustness_method, shared by _analyze_single and the new primitives) rather than a second, potentially-drifting calibration path. - mean_ci(scores): calibrated mean + CI for a single array. Returns a NamedTuple (attribute access or positional unpack). - summarize(scores): the fuller descriptive + CI table (mean, median, std, cv, iqr, percentiles, CI), batch-capable -- single array, a {label: array} dict, or a long-format DataFrame + group_col/value_col. Each group calibrated independently, so groups don't need to share a rectangular design or even a data kind. - stability(runs): standalone multi-run reliability (instability/ICC), for one or more configs, without a full multi-model comparison. - judge_debias_mean_ci(...): PPI-corrected mean + CI for judge scores given a small human-labeled subset, raw arrays in -- a thin wrapper around ppi.correct(np.mean, ...). Named to avoid the PPI acronym and to make clear it corrects a mean, not per-item scores. Also renames validate_alignment's array-only sibling into judge_alignment itself: judge_alignment(human_labels, judge_labels) now works alongside the existing judge_alignment(evaldata, llm_metric=..., human_groundtruth=...) form (dispatched on the first argument's type), sharing a common core (_judge_alignment_core) with the calibration-fitting and alignment-metric logic so neither form can drift out of sync with the other. The array form skips the DataFrame-specific representativeness checks (categorical slice columns) but supports the score-distribution check when all_judge_scores is provided; results built from raw arrays carry placeholder column names and cannot be passed to compare(alignment=...). Every result type follows the same convention: attribute access for the common fields, .to_dict() for a JSON-friendly dict, .to_frame() (batch results) for a pandas DataFrame -- mirroring ComparisonResult rather than committing to one output shape. Verified: 227 tests passing across the new tests/test_quick_primitives.py (34 tests) plus the existing test_alignment.py (55), test_auto_ci_routing.py (23), test_analyze.py (34), test_compare.py, and test_pareto.py (40) suites -- confirming the router.py/alignment.py refactors preserve exact existing behavior. Also ran examples/quick_primitives_demo.py and the pre-existing alignment example scripts end-to-end. Co-Authored-By: Claude Sonnet 5 --- evalstats/__init__.py | 19 + evalstats/alignment.py | 313 ++++++++++---- evalstats/core/router.py | 114 +++-- evalstats/quick.py | 691 ++++++++++++++++++++++++++++++ examples/quick_primitives_demo.py | 105 +++++ tests/test_quick_primitives.py | 387 +++++++++++++++++ 6 files changed, 1521 insertions(+), 108 deletions(-) create mode 100644 evalstats/quick.py create mode 100644 examples/quick_primitives_demo.py create mode 100644 tests/test_quick_primitives.py diff --git a/evalstats/__init__.py b/evalstats/__init__.py index e368620..0bf2bb6 100644 --- a/evalstats/__init__.py +++ b/evalstats/__init__.py @@ -36,6 +36,16 @@ from evalstats.alignment import judge_alignment, AlignmentResult from evalstats import ppi from evalstats import tests +from evalstats.quick import ( + mean_ci, + MeanCI, + summarize, + GroupSummary, + stability, + StabilityResult, + judge_debias_mean_ci, + DebiasedMeanCI, +) __version__ = "0.2.4" @@ -52,6 +62,15 @@ "compare_models", "compare_prompts", "ComparisonResult", + # Quick primitives + "mean_ci", + "MeanCI", + "summarize", + "GroupSummary", + "stability", + "StabilityResult", + "judge_debias_mean_ci", + "DebiasedMeanCI", # Core types "BenchmarkResult", "MultiModelBenchmark", diff --git a/evalstats/alignment.py b/evalstats/alignment.py index cffd50e..09bf851 100644 --- a/evalstats/alignment.py +++ b/evalstats/alignment.py @@ -981,38 +981,96 @@ def _check_slice_column( # judge_alignment # ───────────────────────────────────────────────────────────────────────────── -def judge_alignment( - evaldata, +def _judge_alignment_core( + llm_aligned: np.ndarray, + human_aligned: np.ndarray, + score_type: str, *, llm_metric: str, human_groundtruth: str, - alpha: float = 0.05, + alpha: float, + n_total: int, + all_llm: Optional[np.ndarray] = None, + slice_df: Optional[pd.DataFrame] = None, + slice_labeled_mask: Optional[pd.Series] = None, + slice_exclude_cols: frozenset = frozenset(), + warn_stacklevel: int = 3, ) -> AlignmentResult: - """Validate how well an LLM judge aligns with human graders. + """Shared core behind both :func:`judge_alignment` call forms: fits the + calibration model, computes alignment metrics, and (only when the + relevant context is available) runs representativeness diagnostics. + + ``all_llm`` enables the score-distribution check; ``slice_df`` + + ``slice_labeled_mask`` enable the categorical slice-column checks (both + require the full item pool / other columns, so they're skipped + entirely -- not silently approximated -- when this is called from raw + paired arrays with no further context, see :func:`judge_alignment`). + """ + n_labeled = int(len(llm_aligned)) - Designed for the common case where LLM judge scores exist for all items - but human labels are available for only a subset (the alignment set). - Fits a Bayesian calibration model that can later be used to propagate - judge uncertainty into downstream comparisons via - ``compare(alignment={metric: result})``. + calibration = _fit_calibration(llm_aligned, human_aligned, score_type) - Parameters - ---------- - evaldata : EvalResults - Evaluation data from :func:`load_from`. Must contain both - ``llm_metric`` and ``human_groundtruth`` as columns. - llm_metric : str - Column name of the LLM judge scores. Must be present for all rows. - human_groundtruth : str - Column name of the human rater scores. Expected to be sparsely - populated: non-null for the alignment subset, ``NaN`` elsewhere. - alpha : float - Significance level for alignment metric CIs. Default ``0.05``. + rng = np.random.default_rng(42) + alignment_metrics = _compute_alignment_metrics( + llm_aligned, human_aligned, score_type, alpha=alpha, rng=rng + ) + bias_check = alignment_metrics.pop("_bias_check", None) - Returns - ------- - AlignmentResult - """ + rep: dict = {} + if all_llm is not None: + dist_result = _check_score_distribution(all_llm, llm_aligned, score_type) + rep["score_distribution"] = dist_result + if not dist_result["passed"]: + warnings.warn( + f"Representativeness warning: the {n_labeled} labeled items appear to have " + f"a different {llm_metric} distribution than the full item pool " + f"({dist_result['message']}). " + "Alignment uncertainty estimates may not generalise to all items. " + "Consider sampling human labels more broadly across the score range.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + if slice_df is not None and slice_labeled_mask is not None: + slice_cols = [ + c for c in slice_df.columns + if c not in slice_exclude_cols + and pd.api.types.is_string_dtype(slice_df[c]) + and 1 < slice_df[c].nunique() <= 20 + ] + for col in slice_cols: + col_result = _check_slice_column(slice_df, slice_labeled_mask, col) + rep[f"slice_{col}"] = col_result + if not col_result["passed"]: + warnings.warn( + f"Representativeness warning for column '{col}': the labeled subset " + f"appears unevenly distributed across categories " + f"({col_result['message']}). " + "Consider stratified sampling of human labels.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + return AlignmentResult( + llm_metric=llm_metric, + human_col=human_groundtruth, + score_type=score_type, + n_labeled=n_labeled, + n_total=n_total, + calibration=calibration, + alignment_metrics=alignment_metrics, + representativeness=rep, + bias_check=bias_check, + ) + + +def _judge_alignment_from_evaldata( + evaldata, + *, + llm_metric: str, + human_groundtruth: str, + alpha: float, +) -> AlignmentResult: df = evaldata._df if llm_metric not in df.columns: @@ -1042,10 +1100,9 @@ def judge_alignment( "Alignment estimates will be imprecise with fewer than ~30 labeled items; " "consider expanding the alignment set for reliable uncertainty propagation.", UserWarning, - stacklevel=2, + stacklevel=3, ) - # Resolve score type score_type = evaldata._score_types.get(llm_metric) if score_type is None: from evalstats.loader import _detect_score_type @@ -1055,59 +1112,167 @@ def judge_alignment( human_aligned = df.loc[labeled_mask, human_groundtruth].to_numpy(dtype=float) all_llm = df[llm_metric].to_numpy(dtype=float) - # Fit Bayesian calibration model - calibration = _fit_calibration(llm_aligned, human_aligned, score_type) - - # Compute alignment metrics with bootstrap CIs - rng = np.random.default_rng(42) - alignment_metrics = _compute_alignment_metrics( - llm_aligned, human_aligned, score_type, alpha=alpha, rng=rng + return _judge_alignment_core( + llm_aligned, human_aligned, score_type, + llm_metric=llm_metric, human_groundtruth=human_groundtruth, + alpha=alpha, n_total=n_total, all_llm=all_llm, + slice_df=df, slice_labeled_mask=labeled_mask, + slice_exclude_cols=frozenset({llm_metric, human_groundtruth}), + warn_stacklevel=4, ) - bias_check = alignment_metrics.pop("_bias_check", None) - # Representativeness: score distribution - rep: dict = {} - dist_result = _check_score_distribution(all_llm, llm_aligned, score_type) - rep["score_distribution"] = dist_result - if not dist_result["passed"]: + +def _judge_alignment_from_arrays( + human_labels: np.ndarray, + judge_labels: np.ndarray, + *, + all_judge_scores: Optional[np.ndarray], + score_type: Optional[str], + llm_metric: Optional[str], + human_groundtruth: Optional[str], + alpha: float, +) -> AlignmentResult: + human_aligned = np.asarray(human_labels, dtype=float) + llm_aligned = np.asarray(judge_labels, dtype=float) + if human_aligned.shape != llm_aligned.shape: + raise ValueError( + "human_labels and judge_labels must be paired, same-shape " + f"arrays (one human + one judge score per labeled item); got " + f"shapes {human_aligned.shape} and {llm_aligned.shape}." + ) + if human_aligned.ndim != 1: + raise ValueError( + f"human_labels/judge_labels must be 1-D; got shape {human_aligned.shape}." + ) + n_labeled = int(human_aligned.size) + if n_labeled == 0: + raise ValueError("human_labels/judge_labels must not be empty.") + if n_labeled < 30: warnings.warn( - f"Representativeness warning: the {n_labeled} labeled items appear to have " - f"a different {llm_metric} distribution than the full item pool " - f"({dist_result['message']}). " - "Alignment uncertainty estimates may not generalise to all items. " - "Consider sampling human labels more broadly across the score range.", + f"Only {n_labeled} items have human labels. " + "Alignment estimates will be imprecise with fewer than ~30 labeled items; " + "consider expanding the alignment set for reliable uncertainty propagation.", UserWarning, - stacklevel=2, + stacklevel=3, ) - # Representativeness: categorical slice columns - slice_cols = [ - c for c in df.columns - if c not in {llm_metric, human_groundtruth} - and pd.api.types.is_string_dtype(df[c]) - and 1 < df[c].nunique() <= 20 - ] - for col in slice_cols: - col_result = _check_slice_column(df, labeled_mask, col) - rep[f"slice_{col}"] = col_result - if not col_result["passed"]: - warnings.warn( - f"Representativeness warning for column '{col}': the labeled subset " - f"appears unevenly distributed across categories " - f"({col_result['message']}). " - "Consider stratified sampling of human labels.", - UserWarning, - stacklevel=2, + all_llm = None + n_total = n_labeled + if all_judge_scores is not None: + all_llm = np.asarray(all_judge_scores, dtype=float) + n_total = int(all_llm.size) + + if score_type is None: + from evalstats.loader import _detect_score_type + score_type = _detect_score_type(pd.Series(llm_aligned)) + + return _judge_alignment_core( + llm_aligned, human_aligned, score_type, + llm_metric=llm_metric or "judge", human_groundtruth=human_groundtruth or "human", + alpha=alpha, n_total=n_total, all_llm=all_llm, + slice_df=None, slice_labeled_mask=None, + warn_stacklevel=4, + ) + + +def judge_alignment( + human_labels_or_evaldata, + judge_labels=None, + *, + llm_metric: Optional[str] = None, + human_groundtruth: Optional[str] = None, + all_judge_scores=None, + score_type: Optional[str] = None, + alpha: float = 0.05, +) -> AlignmentResult: + """Validate how well an LLM judge aligns with human graders. + + Two call forms: + + 1. ``judge_alignment(evaldata, *, llm_metric=..., human_groundtruth=...)`` + -- the common case where LLM judge scores exist for all items but + human labels are available for only a subset (the alignment set), + identified by column name in ``evaldata``. Runs the full + representativeness diagnostics (score-distribution check against + the full item pool, plus categorical slice-column checks) since the + full dataset and its other columns are available. The returned + result can be passed to ``compare(alignment={metric: result})``. + 2. ``judge_alignment(human_labels, judge_labels)`` -- a quick-primitive + form for when you already have the two paired arrays for the + labeled subset in hand and don't want to build an ``EvalResults`` + first. Pass ``all_judge_scores`` (every item's judge score, labeled + or not) to also get the score-distribution representativeness + check; without it, that check is skipped (not approximated) since + there's no full item pool to compare against. The categorical + slice-column checks are DataFrame-specific and are always skipped + in this form. **The result from this form carries placeholder + column names and cannot be passed to ``compare(alignment=...)``** + (there's no underlying DataFrame for it to look values up in) -- + use form 1 for that. + + Either form fits a Bayesian calibration model that can later be used to + propagate judge uncertainty into downstream comparisons. + + Parameters + ---------- + human_labels_or_evaldata : EvalResults or array-like + Either evaluation data from :func:`load_from` (form 1), or the + human-labeled subset's scores (form 2). + judge_labels : array-like, optional + Judge scores for that same labeled subset, paired with + ``human_labels_or_evaldata`` (form 2 only). + llm_metric : str, optional + Form 1: column name of the LLM judge scores (required). Form 2: + optional display name for the judge, used only in printed reports. + human_groundtruth : str, optional + Form 1: column name of the human rater scores (required), + expected to be sparsely populated (non-null for the alignment + subset, ``NaN`` elsewhere). Form 2: optional display name for the + human rater, used only in printed reports. + all_judge_scores : array-like, optional + Form 2 only: every item's judge score (labeled and unlabeled), to + enable the score-distribution representativeness check. + score_type : str, optional + Form 2 only: override the auto-detected score type (``"binary"``, + ``"likert"``, ``"continuous"``, or ``"grade"``). Auto-detected from + ``judge_labels`` when not given. + alpha : float + Significance level for alignment metric CIs. Default ``0.05``. + + Returns + ------- + AlignmentResult + """ + from evalstats.loader import EvalResults + + if isinstance(human_labels_or_evaldata, EvalResults): + evaldata = human_labels_or_evaldata + if judge_labels is not None: + raise TypeError( + "judge_alignment(evaldata, ...) doesn't take a second " + "positional argument; pass llm_metric= and " + "human_groundtruth= as column names instead. (For the " + "raw-array form, pass two arrays: " + "judge_alignment(human_labels, judge_labels).)" ) + if llm_metric is None or human_groundtruth is None: + raise TypeError( + "judge_alignment(evaldata, ...) requires llm_metric= and " + "human_groundtruth= (column names)." + ) + return _judge_alignment_from_evaldata( + evaldata, llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, + ) - return AlignmentResult( - llm_metric=llm_metric, - human_col=human_groundtruth, - score_type=score_type, - n_labeled=n_labeled, - n_total=n_total, - calibration=calibration, - alignment_metrics=alignment_metrics, - representativeness=rep, - bias_check=bias_check, + if judge_labels is None: + raise TypeError( + "judge_alignment(human_labels, judge_labels) requires both " + "arrays; or pass an EvalResults (from load_from()) as the " + "first argument for the column-name-based form: " + "judge_alignment(evaldata, llm_metric=..., human_groundtruth=...)." + ) + return _judge_alignment_from_arrays( + human_labels_or_evaldata, judge_labels, + all_judge_scores=all_judge_scores, score_type=score_type, + llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, ) diff --git a/evalstats/core/router.py b/evalstats/core/router.py index a5ca6e9..916406e 100644 --- a/evalstats/core/router.py +++ b/evalstats/core/router.py @@ -742,6 +742,84 @@ def analyze_factorial( # Internal analysis runners # --------------------------------------------------------------------------- +def resolve_auto_robustness_method( + run_scores: np.ndarray, + *, + score_range: Optional[tuple[float, float]] = None, + stacklevel: int = 2, +) -> tuple[str, str, Optional[tuple[float, float]]]: + """Auto-detect data kind (binary / bounded_01 / unbounded) and resolve + it to concrete (pairwise_method, robustness_method, resolved_score_range). + + This is the exact "method='auto'" routing logic ``analyze()``/``compare()`` + use internally, factored out so the quick-primitive functions + (``mean_ci``/``summarize`` in ``evalstats.quick``) can reuse it directly + rather than re-deriving calibration choices in a second place that could + silently drift out of sync with ``compare()``'s. + + Parameters + ---------- + run_scores : np.ndarray + Shape ``(N, M)`` or ``(N, M, R)``. Only the shape and values matter + here (dtype/range/binary-ness detection and R for seeded routing) -- + not which entity is which. + score_range : (float, float), optional + Explicit ``[lo, hi]`` bounds, forwarded to :func:`resolve_score_bounds`. + stacklevel : int + Forwarded to any ``UserWarning`` raised here, so it points at the + caller's caller appropriately regardless of how many wrapper frames + sit between the actual user call and this function. + + Returns + ------- + tuple[str, str, tuple[float, float] or None] + ``(pairwise_method, robustness_method, resolved_score_range)``. + """ + from .resampling import is_binary_scores, resolve_score_bounds + + if run_scores.ndim == 3: + R = run_scores.shape[2] + N = run_scores.shape[1] + else: + R = 1 + N = run_scores.shape[1] + + resolved_score_range: Optional[tuple[float, float]] = None + if is_binary_scores(run_scores): + data_kind = "binary" + else: + # resolve_score_bounds returns a [lo, hi] range (with a + # UserWarning if it had to auto-detect [0, 1] rather than being + # told explicitly) when one can be reliably established, or None + # when the data falls outside [0, 1] and no score_range was + # given -- there's no safe way to infer a metric's true bounds + # from an arbitrary numeric sample's own min/max. In the None + # case, auto silently downgrades to the bounds-agnostic + # "unbounded" (t_interval) row below, but says so loudly. + resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=stacklevel + 1) + if resolved_score_range is not None: + data_kind = "bounded_01" + else: + data_kind = "unbounded" + warnings.warn( + "Numeric evaluation data outside [0, 1] was auto-detected " + "with no explicit score_range, so evalstats is using " + "method='t_interval' (a bounds-agnostic default) rather " + "than the better-calibrated logit-t method. If you know " + "this eval metric's true (min, max) range, pass it " + "explicitly, e.g. score_range=(1, 5) for a Likert scale " + "or score_range=(0, 100) for a percentage grade.", + UserWarning, + stacklevel=stacklevel + 1, + ) + # See config.AUTO_ANALYZE_METHOD_TABLE for the full auto-routing matrix + # (which method is chosen for which data kind / N / seeded combination). + pairwise_method, robustness_method = resolve_auto_analyze_methods( + data_kind, N, seeded=R >= 3, + ) + return pairwise_method, robustness_method, resolved_score_range + + def _analyze_single( result: BenchmarkResult, shape: BenchmarkShape, @@ -851,40 +929,8 @@ def _analyze_single( robustness_method = method resolved_score_range: Optional[tuple[float, float]] = None if method == "auto": - from .resampling import is_binary_scores, resolve_score_bounds - R = run_scores.shape[2] - N = run_scores.shape[1] - if is_binary_scores(run_scores): - data_kind = "binary" - else: - # resolve_score_bounds returns a [lo, hi] range (with a - # UserWarning if it had to auto-detect [0, 1] rather than being - # told explicitly) when one can be reliably established, or None - # when the data falls outside [0, 1] and no score_range was - # given -- there's no safe way to infer a metric's true bounds - # from an arbitrary numeric sample's own min/max. In the None - # case, auto silently downgrades to the bounds-agnostic - # "unbounded" (t_interval) row below, but says so loudly. - resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=2) - if resolved_score_range is not None: - data_kind = "bounded_01" - else: - data_kind = "unbounded" - warnings.warn( - "Numeric evaluation data outside [0, 1] was auto-detected " - "with no explicit score_range, so evalstats is using " - "method='t_interval' (a bounds-agnostic default) rather " - "than the better-calibrated logit-t method. If you know " - "this eval metric's true (min, max) range, pass it " - "explicitly, e.g. score_range=(1, 5) for a Likert scale " - "or score_range=(0, 100) for a percentage grade.", - UserWarning, - stacklevel=2, - ) - # See config.AUTO_ANALYZE_METHOD_TABLE for the full auto-routing matrix - # (which method is chosen for which data kind / N / seeded combination). - pairwise_method, robustness_method = resolve_auto_analyze_methods( - data_kind, N, seeded=R >= 3, + pairwise_method, robustness_method, resolved_score_range = resolve_auto_robustness_method( + run_scores, score_range=score_range, stacklevel=2, ) elif method == "bayes_binary": from .resampling import is_binary_scores diff --git a/evalstats/quick.py b/evalstats/quick.py new file mode 100644 index 0000000..21e4a22 --- /dev/null +++ b/evalstats/quick.py @@ -0,0 +1,691 @@ +"""Quick primitives for users who don't need compare()'s full comparative +report -- just a trustworthy point estimate (or a few other common building +blocks) as plain data, ready to hand off to their own plotting library or +downstream code. + +These reuse the exact same auto method-selection and calibration machinery +compare() uses internally (see core.router.resolve_auto_robustness_method), +so a number returned here and the equivalent number inside a compare() +report are computed identically -- there is no separate, potentially- +drifting "lite" calibration path. + +Every result type here follows the same output convention: attribute +access for the common fields, ``.to_dict()`` for a JSON-friendly plain +dict, and (for the batch-capable results) ``.to_frame()`` for a pandas +DataFrame -- mirroring how ``ComparisonResult`` already offers ``.to_dict()`` +/``.to_frame()`` rather than committing to one output shape. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass +from typing import Literal, NamedTuple, Optional, Union + +import numpy as np +import pandas as pd + +from .config import get_alpha_ci +from .core.router import resolve_auto_robustness_method +from .core.variance import robustness_metrics, seed_variance_decomposition + + +# --------------------------------------------------------------------------- +# mean_ci +# --------------------------------------------------------------------------- + +class MeanCI(NamedTuple): + """Calibrated mean + confidence interval for a single array of scores. + + A plain :class:`NamedTuple` so it works both ways: unpack positionally + (``mean, ci_low, ci_high, n, method = es.mean_ci(scores)``) or use + attribute access (``result.mean``). Call :meth:`to_dict` for a plain + dict. + + Attributes + ---------- + mean : float + Point estimate (sample mean). + ci_low, ci_high : float + Bounds of the calibrated confidence interval. + n : int + Number of (non-NaN) scores the estimate is based on. + method : str + The CI method evalstats auto-selected (e.g. ``"logit_t"``, + ``"wilson"``, ``"smooth_bootstrap"``) -- see + :func:`~evalstats.core.router.resolve_auto_robustness_method` for + the full routing table. + """ + + mean: float + ci_low: float + ci_high: float + n: int + method: str + + def to_dict(self) -> dict: + """Return a plain, JSON-friendly dict.""" + return self._asdict() + + +def mean_ci( + scores, + *, + alpha: Optional[float] = None, + n_bootstrap: int = 10_000, + score_range: Optional[tuple[float, float]] = None, + rng=None, +) -> MeanCI: + """Calibrated mean + confidence interval for a single array of scores. + + Auto-detects the data kind (binary / bounded [0, 1] / unbounded) and the + sample size, then picks the same CI method ``compare()`` would use for + this data -- Wilson for binary, logit-t for bounded continuous/Likert + data, a bootstrap-t fallback for small unbounded samples, etc. No + ``load_from()``, no factors, no comparison -- just the one number a lot + of users actually want. + + Parameters + ---------- + scores : array-like + A 1-D array (or anything ``np.asarray`` accepts) of per-item scores + for a single entity. + alpha : float, optional + Significance level. Defaults to :func:`evalstats.get_alpha_ci`'s + current value (0.05 unless changed via :func:`evalstats.set_alpha_ci`). + n_bootstrap : int + Bootstrap resamples for the CI, when the auto-selected method is + bootstrap-based (default 10,000, matching ``analyze()``'s default). + score_range : (float, float), optional + Explicit ``(min, max)`` bounds for the metric (e.g. ``(1, 5)`` for a + Likert scale). Only used when the auto-selected method needs + bounds-aware rescaling (``logit_t``); inferred from the data when + not given and possible -- see + :func:`~evalstats.core.router.resolve_auto_robustness_method`. + rng : int, np.random.Generator, or None + Seed or generator for reproducibility. + + Returns + ------- + MeanCI + + Examples + -------- + >>> import evalstats as es + >>> result = es.mean_ci(accuracy_scores) + >>> result.mean, result.ci_low, result.ci_high + >>> mean, lo, hi, n, method = result # positional unpack also works + """ + arr = np.asarray(scores, dtype=float) + if arr.ndim != 1: + raise ValueError(f"scores must be a 1-D array; got shape {arr.shape}.") + if arr.size == 0: + raise ValueError("scores must not be empty.") + + if alpha is None: + alpha = get_alpha_ci() + rng = np.random.default_rng(rng) + + scores_2d = arr.reshape(1, -1) + _, robustness_method, resolved_score_range = resolve_auto_robustness_method( + scores_2d, score_range=score_range, stacklevel=3, + ) + rob = robustness_metrics( + scores_2d, ["_"], + n_bootstrap=n_bootstrap, rng=rng, alpha=alpha, + statistic="mean", marginal_method=robustness_method, + multi_ci=False, score_range=resolved_score_range, + ) + return MeanCI( + mean=float(rob.mean[0]), + ci_low=float(rob.ci_low[0]) if rob.ci_low is not None else float("nan"), + ci_high=float(rob.ci_high[0]) if rob.ci_high is not None else float("nan"), + n=int(np.sum(~np.isnan(arr))), + method=robustness_method, + ) + + +# --------------------------------------------------------------------------- +# summarize +# --------------------------------------------------------------------------- + +_SUMMARY_ROW_FIELDS = ( + "mean", "median", "std", "cv", "iqr", "cvar_10", + "p10", "p25", "p50", "p75", "p90", "ci_low", "ci_high", "n", "method", +) + + +@dataclass +class GroupSummary: + """Descriptive statistics + calibrated CI, one row per group. + + Returned by :func:`summarize`. Same field set as the "--- Robustness + ---" table ``compare()`` prints, plus the calibrated ``ci_low``/ + ``ci_high`` compare()'s Mean Performance section shows separately -- + bundled into one table here since there's no other section to split it + across. + + Each group's row is computed independently (its own auto-detected data + kind, N, and CI method) rather than pooled, so groups of different + sizes or types can be summarized in the same call. + """ + + labels: list[str] + mean: np.ndarray + median: np.ndarray + std: np.ndarray + cv: np.ndarray + iqr: np.ndarray + cvar_10: np.ndarray + p10: np.ndarray + p25: np.ndarray + p50: np.ndarray + p75: np.ndarray + p90: np.ndarray + ci_low: np.ndarray + ci_high: np.ndarray + n: np.ndarray + method: list[str] + _single_ungrouped: bool = False + + def _row_dict(self, i: int) -> dict: + return { + "mean": float(self.mean[i]), + "median": float(self.median[i]), + "std": float(self.std[i]), + "cv": float(self.cv[i]), + "iqr": float(self.iqr[i]), + "cvar_10": float(self.cvar_10[i]), + "p10": float(self.p10[i]), + "p25": float(self.p25[i]), + "p50": float(self.p50[i]), + "p75": float(self.p75[i]), + "p90": float(self.p90[i]), + "ci_low": float(self.ci_low[i]), + "ci_high": float(self.ci_high[i]), + "n": int(self.n[i]), + "method": self.method[i], + } + + def to_dict(self) -> dict: + """Plain, JSON-friendly dict. + + A flat ``{"mean": ..., "ci_low": ..., ...}`` dict when + :func:`summarize` was called on a single bare array; a + ``{label: {...}}`` nested dict, one entry per group, otherwise. + """ + if self._single_ungrouped: + return self._row_dict(0) + return {label: self._row_dict(i) for i, label in enumerate(self.labels)} + + def to_frame(self) -> pd.DataFrame: + """Return one row per group as a pandas DataFrame, indexed by label.""" + data = { + field: getattr(self, field) + for field in _SUMMARY_ROW_FIELDS + } + return pd.DataFrame(data, index=pd.Index(self.labels, name="group")) + + +def summarize( + scores: Union[np.ndarray, dict, pd.DataFrame], + *, + group_col: Optional[str] = None, + value_col: Optional[str] = None, + statistic: Literal["mean", "median"] = "mean", + alpha: Optional[float] = None, + n_bootstrap: int = 10_000, + score_range: Optional[tuple[float, float]] = None, + rng=None, +) -> GroupSummary: + """Descriptive statistics + calibrated CI for one or more groups. + + Accepts whatever shape of data you already have: + + * A single 1-D array -- one group. + * A ``{label: array}`` dict -- one row per key. Arrays don't need to be + the same length. + * A long-format DataFrame plus ``group_col``/``value_col`` -- one row + per distinct value of ``group_col``. + + Each group is auto-calibrated independently (own data-kind/N detection, + own CI method -- see :func:`mean_ci`), so this does *not* require a + ``compare()``-style rectangular design; groups of different sizes or + even different data kinds (e.g. one binary, one continuous) are fine. + This is a descriptive summary only -- no significance testing or + ranking between groups; use ``compare()`` for that. + + Parameters + ---------- + scores : array-like, dict, or DataFrame + See above. + group_col, value_col : str, optional + Required (and only used) when ``scores`` is a DataFrame. + statistic : {"mean", "median"} + Central-tendency statistic the CI is built around (default "mean"). + The ``mean``/``median`` columns of the result are always both + reported regardless of this choice; it only affects ``ci_low``/ + ``ci_high``. + alpha, n_bootstrap, score_range, rng + See :func:`mean_ci`. + + Returns + ------- + GroupSummary + + Examples + -------- + >>> import evalstats as es + >>> es.summarize({"gpt-4o": acc_gpt4o, "claude": acc_claude}).to_frame() + >>> es.summarize(df, group_col="model", value_col="accuracy").to_frame() + """ + single_ungrouped = False + if isinstance(scores, pd.DataFrame): + if group_col is None or value_col is None: + raise ValueError( + "summarize() on a DataFrame requires group_col and " + "value_col, e.g. summarize(df, group_col='model', " + "value_col='accuracy')." + ) + if group_col not in scores.columns: + raise ValueError( + f"group_col '{group_col}' not found in DataFrame columns: " + f"{list(scores.columns)}" + ) + if value_col not in scores.columns: + raise ValueError( + f"value_col '{value_col}' not found in DataFrame columns: " + f"{list(scores.columns)}" + ) + groups = scores.groupby(group_col, sort=False)[value_col] + labels = [str(k) for k in groups.groups.keys()] + arrays = [ + groups.get_group(k).to_numpy(dtype=float) + for k in groups.groups.keys() + ] + elif isinstance(scores, dict): + if len(scores) == 0: + raise ValueError("scores dict must not be empty.") + labels = [str(k) for k in scores.keys()] + arrays = [np.asarray(v, dtype=float) for v in scores.values()] + else: + arr = np.asarray(scores, dtype=float) + if arr.ndim != 1: + raise ValueError( + "scores must be a 1-D array, a {label: array} dict, or a " + f"DataFrame (with group_col/value_col); got shape {arr.shape}." + ) + labels = ["value"] + arrays = [arr] + single_ungrouped = True + + for lbl, a in zip(labels, arrays): + if a.size == 0: + raise ValueError(f"Group '{lbl}' has no scores.") + + if alpha is None: + alpha = get_alpha_ci() + rng = np.random.default_rng(rng) + + mean = np.empty(len(labels)) + median = np.empty(len(labels)) + std = np.empty(len(labels)) + cv = np.empty(len(labels)) + iqr = np.empty(len(labels)) + cvar_10 = np.empty(len(labels)) + p10 = np.empty(len(labels)) + p25 = np.empty(len(labels)) + p50 = np.empty(len(labels)) + p75 = np.empty(len(labels)) + p90 = np.empty(len(labels)) + ci_low = np.empty(len(labels)) + ci_high = np.empty(len(labels)) + n = np.empty(len(labels), dtype=int) + method: list[str] = [] + + # Each group gets its own auto-detected method (own data kind, own N) + # rather than pooling into one call -- see GroupSummary's docstring for + # why groups don't need to share a rectangular design here. + for i, a in enumerate(arrays): + a_2d = a.reshape(1, -1) + _, robustness_method, resolved_score_range = resolve_auto_robustness_method( + a_2d, score_range=score_range, stacklevel=4, + ) + rob = robustness_metrics( + a_2d, ["_"], + n_bootstrap=n_bootstrap, rng=rng, alpha=alpha, + statistic=statistic, marginal_method=robustness_method, + multi_ci=False, score_range=resolved_score_range, + ) + mean[i] = rob.mean[0] + median[i] = rob.median[0] + std[i] = rob.std[0] + cv[i] = rob.cv[0] + iqr[i] = rob.iqr[0] + cvar_10[i] = rob.cvar_10[0] + p10[i] = rob.percentiles[10][0] + p25[i] = rob.percentiles[25][0] + p50[i] = rob.percentiles[50][0] + p75[i] = rob.percentiles[75][0] + p90[i] = rob.percentiles[90][0] + ci_low[i] = rob.ci_low[0] if rob.ci_low is not None else np.nan + ci_high[i] = rob.ci_high[0] if rob.ci_high is not None else np.nan + n[i] = int(np.sum(~np.isnan(a))) + method.append(robustness_method) + + return GroupSummary( + labels=labels, mean=mean, median=median, std=std, cv=cv, iqr=iqr, + cvar_10=cvar_10, p10=p10, p25=p25, p50=p50, p75=p75, p90=p90, + ci_low=ci_low, ci_high=ci_high, n=n, method=method, + _single_ungrouped=single_ungrouped, + ) + + +# --------------------------------------------------------------------------- +# stability +# --------------------------------------------------------------------------- + +@dataclass +class StabilityResult: + """Multi-run reliability metrics for one or more configs. + + Returned by :func:`stability`. See + :class:`~evalstats.core.variance.SeedVarianceResult` for the underlying + ``instability``/``icc`` decomposition this wraps. + + Attributes + ---------- + labels : list[str] + Config labels. + instability : np.ndarray + Mean within-item run-to-run standard deviation, in score-scale + units -- "on average, how many points does the score move between + runs for the same item?". Lower is more stable. + icc : np.ndarray + Intraclass correlation: of the variation across items, the fraction + that's genuine item-level signal rather than run-to-run noise + (bounded [0, 1], higher is more reliable). + n_runs : np.ndarray + Number of (non-padded) runs each config was actually evaluated + over. Per-config, not a single shared count -- configs are allowed + to have different run counts (see :func:`stability`). + label_text : list[str] + Plain-language interpretation of ``instability`` per config (e.g. + "mostly stable across runs"), matching the wording ``compare()``'s + printed summary uses for the same metric. + """ + + labels: list[str] + instability: np.ndarray + icc: np.ndarray + n_runs: np.ndarray + label_text: list[str] + + def _row_dict(self, i: int) -> dict: + return { + "instability": float(self.instability[i]), + "icc": float(self.icc[i]) if not np.isnan(self.icc[i]) else None, + "n_runs": int(self.n_runs[i]), + "interpretation": self.label_text[i], + } + + def to_dict(self) -> dict: + """Plain dict: flat for a single config, ``{label: {...}}`` for several.""" + if len(self.labels) == 1: + return self._row_dict(0) + return {label: self._row_dict(i) for i, label in enumerate(self.labels)} + + def to_frame(self) -> pd.DataFrame: + """One row per config as a pandas DataFrame, indexed by label.""" + return pd.DataFrame( + { + "instability": self.instability, + "icc": self.icc, + "n_runs": self.n_runs, + "interpretation": self.label_text, + }, + index=pd.Index(self.labels, name="config"), + ) + + +def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = None) -> StabilityResult: + """Multi-run reliability: how much does a config's score move across + repeated runs on the same items? + + Standalone version of the seed-instability decomposition ``compare()`` + shows for multi-run (seeded) benchmarks -- for deciding "is this + configuration reliable enough to ship" without needing a full + multi-model comparison. + + Parameters + ---------- + runs : array-like or dict + A single config's repeated-run scores as a 2-D array of shape + ``(K, M)`` (K runs, M items, same M items each run) -- one config; + or a ``{label: (K, M) array}`` dict of several configs. Configs + must share the same M (same item set); K (number of runs, >= 3) can + differ per config. + labels : list[str], optional + Override labels when ``runs`` is a single array (default + ``["value"]``). Ignored when ``runs`` is a dict (its keys are used). + + Returns + ------- + StabilityResult + + Examples + -------- + >>> import evalstats as es + >>> es.stability(rag_config_a_runs) # shape (5, 200): 5 runs, 200 items + >>> es.stability({"config_a": runs_a, "config_b": runs_b}).to_frame() + """ + if isinstance(runs, dict): + if len(runs) == 0: + raise ValueError("runs dict must not be empty.") + input_labels = [str(k) for k in runs.keys()] + arrays = [np.asarray(v, dtype=float) for v in runs.values()] + else: + arr = np.asarray(runs, dtype=float) + if arr.ndim != 2: + raise ValueError( + "runs must be a 2-D array (K runs x M items) or a " + f"{{label: array}} dict of such arrays; got shape {arr.shape}." + ) + input_labels = list(labels) if labels is not None else ["value"] + if len(input_labels) != 1: + raise ValueError( + f"A single runs array takes at most one label; got {len(input_labels)}." + ) + arrays = [arr] + + for lbl, a in zip(input_labels, arrays): + if a.ndim != 2: + raise ValueError(f"runs['{lbl}'] must be 2-D (K runs x M items); got shape {a.shape}.") + + m_values = {a.shape[1] for a in arrays} + if len(m_values) != 1: + raise ValueError( + "All configs must be evaluated on the same number of items (M); " + f"got M values {sorted(m_values)} across configs " + f"{dict(zip(input_labels, (a.shape for a in arrays)))}." + ) + m_items = m_values.pop() + + # Different configs may have different K (run count); pad the run axis + # with NaN -- seed_variance_decomposition's internal nanmean/nanvar + # handle that safely (it's a closed-form ANOVA-style computation, not a + # resampling procedure, so NaN-tolerant reductions are exact here). + max_k = max(a.shape[0] for a in arrays) + if any(a.shape[0] < 3 for a in arrays): + offender = input_labels[[a.shape[0] for a in arrays].index(min(a.shape[0] for a in arrays))] + raise ValueError( + f"Seed-variance decomposition requires >= 3 runs per config; " + f"config '{offender}' has {min(a.shape[0] for a in arrays)}." + ) + scores_3d = np.full((len(arrays), m_items, max_k), np.nan) + for i, a in enumerate(arrays): + scores_3d[i, :, : a.shape[0]] = a.T # (K, M) -> (M, K) + + from .core.summary import _instability_label + + sv = seed_variance_decomposition(scores_3d, input_labels) + actual_n_runs = np.array([a.shape[0] for a in arrays]) + return StabilityResult( + labels=input_labels, + instability=sv.instability, + icc=sv.icc, + n_runs=actual_n_runs, + label_text=[_instability_label(float(v)) for v in sv.instability], + ) + + +# --------------------------------------------------------------------------- +# judge_debias_mean_ci +# --------------------------------------------------------------------------- + +class DebiasedMeanCI(NamedTuple): + """PPI-corrected mean + CI for judge scores, debiased against a small + human-labeled subset. + + Returned by :func:`judge_debias_mean_ci`. Shares ``mean``/``ci_low``/ + ``ci_high`` field names with :class:`MeanCI` so code consuming either + doesn't need to branch on which one it got; the extra fields are + diagnostic context specific to the correction. + + Attributes + ---------- + mean : float + PPI-corrected point estimate. + ci_low, ci_high : float + Bootstrap confidence interval on the corrected estimate. + judge_mean : float + Uncorrected mean of the judge-only scores (what you'd get without + this correction). + human_mean : float + Mean of the human scores on the labeled subset alone. + rectifier : float + Signed correction term (``human_mean`` minus the judge's mean on + that same labeled subset). Positive means the judge underrates on + average; negative means it overrates. + p_value : float or None + Two-sided bootstrap p-value for H0: corrected mean == 0. ``None`` + unless requested (see ``compute_pvalue`` below) -- rarely + interesting for a raw mean rather than a difference. + n_labeled, n_unlabeled : int + Number of items in the labeled and unlabeled sets. + """ + + mean: float + ci_low: float + ci_high: float + judge_mean: float + human_mean: float + rectifier: float + p_value: Optional[float] + n_labeled: int + n_unlabeled: int + + def to_dict(self) -> dict: + """Return a plain, JSON-friendly dict.""" + return self._asdict() + + +def judge_debias_mean_ci( + unlabeled_judge_scores, + labeled_human_scores, + labeled_judge_scores, + *, + alpha: float = 0.05, + n_bootstrap: int = 1000, + compute_pvalue: bool = False, + rng=None, +) -> DebiasedMeanCI: + """PPI-corrected mean + CI for LLM judge scores, using a small + human-labeled subset to debias them. + + For the common setup: a judge scored every item, but only a small + subset also has human labels. Prediction-Powered Inference (PPI) uses + the disagreement between judge and human on that labeled subset (the + "rectifier") to correct the judge-only mean over the full dataset, + without needing every item human-labeled. + + Only corrects a **mean** -- there is no meaningful per-item "debiased + score"; PPI is a correction to an aggregate estimate, not a per-item + imputation. For downstream comparisons across several entities/conditions + (not just a single mean), use ``compare(..., alignment=...)`` together + with :func:`judge_alignment` instead, which applies the same idea + per-comparison with the full Friedman/Wilcoxon machinery. + + Parameters + ---------- + unlabeled_judge_scores : array-like + Judge scores for every item that does NOT also have a human label. + Must be disjoint from the labeled items below -- do not pass every + item's judge score here if some of them are also in + ``labeled_judge_scores``. + labeled_human_scores : array-like + Human scores for the labeled subset. + labeled_judge_scores : array-like + Judge scores for that SAME labeled subset, paired (same order, + same length) with ``labeled_human_scores``. + alpha : float + Significance level (default 0.05, i.e. 95% CI). + n_bootstrap : int + Bootstrap resamples (default 1000). + compute_pvalue : bool + Compute a two-sided p-value for H0: corrected mean == 0 (default + False -- usually not the interesting question for a raw mean). + rng : int, np.random.Generator, or None + Seed or generator for reproducibility. + + Returns + ------- + DebiasedMeanCI + + Examples + -------- + >>> import evalstats as es + >>> result = es.judge_debias_mean_ci( + ... unlabeled_judge_scores=judge_scores[~has_human_label], + ... labeled_human_scores=human_scores[has_human_label], + ... labeled_judge_scores=judge_scores[has_human_label], + ... ) + >>> result.mean, result.ci_low, result.ci_high + """ + from .ppi import correct as _ppi_correct + + y_hat_unlab = np.asarray(unlabeled_judge_scores, dtype=float) + y_lab = np.asarray(labeled_human_scores, dtype=float) + y_hat_lab = np.asarray(labeled_judge_scores, dtype=float) + + if y_lab.shape != y_hat_lab.shape: + raise ValueError( + "labeled_human_scores and labeled_judge_scores must be paired, " + f"same-shape arrays (one human + one judge score per labeled " + f"item); got shapes {y_lab.shape} and {y_hat_lab.shape}." + ) + if y_lab.size < 15: + warnings.warn( + f"Only {y_lab.size} labeled items -- PPI correction will be " + "imprecise with fewer than ~15 labeled items. Consider " + "expanding the labeled subset.", + UserWarning, + stacklevel=2, + ) + + result = _ppi_correct( + np.mean, + Y_lab=y_lab, Y_hat_lab=y_hat_lab, Y_hat_unlab=y_hat_unlab, + alpha=alpha, n_boot=n_bootstrap, rng=rng, compute_pvalue=compute_pvalue, + ) + return DebiasedMeanCI( + mean=result.estimate, + ci_low=result.ci_low, + ci_high=result.ci_high, + judge_mean=result.llm_estimate, + human_mean=result.human_estimate, + rectifier=result.rectifier, + p_value=result.p_value, + n_labeled=int(y_lab.size), + n_unlabeled=int(y_hat_unlab.size), + ) diff --git a/examples/quick_primitives_demo.py b/examples/quick_primitives_demo.py new file mode 100644 index 0000000..17acae5 --- /dev/null +++ b/examples/quick_primitives_demo.py @@ -0,0 +1,105 @@ +"""Quick primitives: calibrated numbers for users who don't want compare()'s +full comparative report -- just a trustworthy point estimate (or a few other +common building blocks), returned as plain data to hand to your own plotting +library or downstream code. + +Every result here reuses the exact same auto method-selection and +calibration machinery compare() uses internally, so a number computed here +and the equivalent number inside a compare() report are identical. + +Usage: + python examples/quick_primitives_demo.py +""" + +import numpy as np + +import evalstats as es + + +rng = np.random.default_rng(0) + +print("=" * 70) +print("1. mean_ci() -- calibrated mean + CI for a single array") +print("=" * 70) + +accuracy = np.clip(rng.normal(0.82, 0.09, 60), 0, 1) +result = es.mean_ci(accuracy) +print(f"mean={result.mean:.3f} 95% CI=[{result.ci_low:.3f}, {result.ci_high:.3f}] " + f"n={result.n} method={result.method}") +# Unpacks positionally too, for the "just give me the numbers" case: +mean, lo, hi, n, method = result +print(f"unpacked: {mean:.3f}, [{lo:.3f}, {hi:.3f}]") +print() + +print("=" * 70) +print("2. summarize() -- descriptive + CI table for several groups at once") +print("=" * 70) + +scores_by_model = { + "gpt-4o": np.clip(rng.normal(0.85, 0.08, 50), 0, 1), + "claude-sonnet": np.clip(rng.normal(0.80, 0.09, 50), 0, 1), + "llama-70b": np.clip(rng.normal(0.70, 0.10, 45), 0, 1), # different N is fine +} +table = es.summarize(scores_by_model) +print(table.to_frame()[["mean", "ci_low", "ci_high", "n", "method"]]) +print() +print("Same data as a plain dict, e.g. to write out as JSON:") +print(table.to_dict()["gpt-4o"]) +print() + +print("=" * 70) +print("3. stability() -- multi-run reliability, standalone") +print("=" * 70) + +M, K = 150, 5 # 150 items, 5 repeated runs +base = rng.normal(0.78, 0.1, M) +stable_runs = np.array([np.clip(base + rng.normal(0, 0.02, M), 0, 1) for _ in range(K)]) +flaky_runs = np.array([np.clip(rng.normal(0.6, 0.2, M), 0, 1) for _ in range(K)]) + +stab = es.stability({"rag_config_a": stable_runs, "rag_config_b": flaky_runs}) +print(stab.to_frame()) +print() + +print("=" * 70) +print("4. judge_alignment() -- array-based form, no load_from() needed") +print("=" * 70) + +n_labeled = 40 +human_labels = rng.integers(1, 6, n_labeled).astype(float) # 1-5 Likert +judge_labels = np.clip( + np.round(human_labels + rng.normal(0.3, 0.6, n_labeled)), 1, 5 +) # rounded to whole numbers -> also detected as Likert + +alignment = es.judge_alignment(human_labels, judge_labels) +print(f"score_type={alignment.score_type} n_labeled={alignment.n_labeled}") +kappa = alignment.alignment_metrics.get("weighted_kappa") or alignment.alignment_metrics.get("cohens_kappa") +if kappa is not None: + print(f"weighted kappa: {kappa['estimate']:.3f} 95% CI=[{kappa['ci_low']:.3f}, {kappa['ci_high']:.3f}]") +print() + +print("=" * 70) +print("5. judge_debias_mean_ci() -- PPI-corrected mean, judge scores only") +print("=" * 70) + +n_total = 400 +true_mean = 0.55 +judge_bias = 0.18 # judge systematically overrates + +human_all = np.clip(rng.normal(true_mean, 0.15, n_total), 0, 1) +judge_all = np.clip(human_all + judge_bias + rng.normal(0, 0.05, n_total), 0, 1) + +labeled_idx = rng.choice(n_total, 40, replace=False) +mask = np.zeros(n_total, dtype=bool) +mask[labeled_idx] = True + +debiased = es.judge_debias_mean_ci( + unlabeled_judge_scores=judge_all[~mask], + labeled_human_scores=human_all[mask], + labeled_judge_scores=judge_all[mask], +) +print(f"judge-only mean (biased): {debiased.judge_mean:.3f}") +print(f"PPI-corrected mean: {debiased.mean:.3f} " + f"95% CI=[{debiased.ci_low:.3f}, {debiased.ci_high:.3f}]") +print(f"true mean (for comparison): {true_mean:.3f}") +print(f"rectifier: {debiased.rectifier:+.3f} " + f"(n_labeled={debiased.n_labeled}, n_unlabeled={debiased.n_unlabeled})") diff --git a/tests/test_quick_primitives.py b/tests/test_quick_primitives.py new file mode 100644 index 0000000..f486183 --- /dev/null +++ b/tests/test_quick_primitives.py @@ -0,0 +1,387 @@ +"""Tests for evalstats.quick (mean_ci, summarize, stability, judge_debias_mean_ci) +and the array-based judge_alignment() path. +""" + +from __future__ import annotations + +import warnings + +import numpy as np +import pandas as pd +import pytest + +import evalstats as es +from evalstats.quick import MeanCI, GroupSummary, StabilityResult, DebiasedMeanCI + + +def _rng(seed: int = 0) -> np.random.Generator: + return np.random.default_rng(seed) + + +# --------------------------------------------------------------------------- +# mean_ci +# --------------------------------------------------------------------------- + +def test_mean_ci_basic_continuous(): + rng = _rng(1) + scores = np.clip(rng.normal(0.75, 0.1, 60), 0, 1) + result = es.mean_ci(scores) + assert isinstance(result, MeanCI) + assert result.ci_low < result.mean < result.ci_high + assert result.n == 60 + assert abs(result.mean - float(np.mean(scores))) < 1e-9 + + +def test_mean_ci_unpacks_positionally(): + rng = _rng(2) + scores = np.clip(rng.normal(0.6, 0.1, 40), 0, 1) + mean, lo, hi, n, method = es.mean_ci(scores) + result = es.mean_ci(scores) + assert mean == result.mean + assert lo == result.ci_low + assert hi == result.ci_high + assert n == result.n + assert method == result.method + + +def test_mean_ci_to_dict(): + rng = _rng(3) + scores = np.clip(rng.normal(0.5, 0.1, 30), 0, 1) + d = es.mean_ci(scores).to_dict() + assert set(d.keys()) == {"mean", "ci_low", "ci_high", "n", "method"} + + +def test_mean_ci_binary_uses_wilson(): + rng = _rng(4) + scores = (rng.random(100) < 0.7).astype(float) + result = es.mean_ci(scores) + assert result.method == "wilson" + assert 0 <= result.ci_low <= result.mean <= result.ci_high <= 1 + + +def test_mean_ci_rejects_2d_array(): + with pytest.raises(ValueError, match="1-D"): + es.mean_ci(np.zeros((3, 3))) + + +def test_mean_ci_rejects_empty_array(): + with pytest.raises(ValueError, match="empty"): + es.mean_ci(np.array([])) + + +def test_mean_ci_matches_compare_for_same_data(): + """mean_ci() should compute the identical number compare() would show + for the same entity's marginal CI -- same underlying calibration path. + (compare() needs >= 2 entities to run at all, so a second dummy model + -- same [0, 1] range, so auto-detection resolves identically -- is + added purely to satisfy that; only the "target" row is checked.)""" + rng = _rng(5) + scores = np.clip(rng.normal(0.7, 0.08, 50), 0, 1) + dummy_scores = np.clip(rng.normal(0.4, 0.08, 50), 0, 1) + result = es.mean_ci(scores, rng=_rng(99)) + + rows = [ + {"model": "target", "item": f"q{i}", "score": s} for i, s in enumerate(scores) + ] + [ + {"model": "dummy", "item": f"q{i}", "score": s} for i, s in enumerate(dummy_scores) + ] + df = pd.DataFrame(rows) + evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) + cmp = es.compare(evaldata, factors="model", metric="score", rng=_rng(99)) + rob = cmp._analysis.robustness + idx = list(rob.labels).index("target") + assert abs(result.mean - float(rob.mean[idx])) < 1e-9 + assert result.method == cmp._analysis.resolved_ci_method + + +# --------------------------------------------------------------------------- +# summarize +# --------------------------------------------------------------------------- + +def test_summarize_single_array_returns_flat_dict(): + rng = _rng(10) + scores = np.clip(rng.normal(0.8, 0.1, 30), 0, 1) + result = es.summarize(scores) + assert isinstance(result, GroupSummary) + assert result.labels == ["value"] + d = result.to_dict() + assert "mean" in d # flat, not nested under "value" + assert set(d.keys()) >= {"mean", "median", "std", "ci_low", "ci_high", "n", "method"} + + +def test_summarize_dict_of_arrays(): + rng = _rng(11) + scores = { + "a": np.clip(rng.normal(0.8, 0.1, 30), 0, 1), + "b": np.clip(rng.normal(0.6, 0.1, 25), 0, 1), # different N -- fine + } + result = es.summarize(scores) + assert result.labels == ["a", "b"] + d = result.to_dict() + assert set(d.keys()) == {"a", "b"} + assert d["a"]["n"] == 30 + assert d["b"]["n"] == 25 + frame = result.to_frame() + assert list(frame.index) == ["a", "b"] + assert "mean" in frame.columns and "ci_low" in frame.columns + + +def test_summarize_dataframe_group_col(): + rng = _rng(12) + rows = [] + for m, mu in [("x", 0.7), ("y", 0.5)]: + for i in range(20): + rows.append({"model": m, "score": float(np.clip(rng.normal(mu, 0.1), 0, 1))}) + df = pd.DataFrame(rows) + result = es.summarize(df, group_col="model", value_col="score") + assert set(result.labels) == {"x", "y"} + frame = result.to_frame() + assert frame.loc["x", "mean"] > frame.loc["y", "mean"] + + +def test_summarize_dataframe_requires_group_and_value_col(): + df = pd.DataFrame({"model": ["a", "b"], "score": [0.5, 0.6]}) + with pytest.raises(ValueError, match="group_col"): + es.summarize(df) + + +def test_summarize_dataframe_bad_column_name(): + df = pd.DataFrame({"model": ["a", "b"], "score": [0.5, 0.6]}) + with pytest.raises(ValueError, match="not found"): + es.summarize(df, group_col="nope", value_col="score") + + +def test_summarize_empty_dict_raises(): + with pytest.raises(ValueError, match="empty"): + es.summarize({}) + + +def test_summarize_empty_group_raises(): + with pytest.raises(ValueError, match="no scores"): + es.summarize({"a": np.array([1.0, 2.0]), "b": np.array([])}) + + +def test_summarize_matches_mean_ci_for_same_single_array(): + rng = _rng(13) + scores = np.clip(rng.normal(0.65, 0.09, 45), 0, 1) + m = es.mean_ci(scores, rng=_rng(7)) + s = es.summarize(scores, rng=_rng(7)) + assert abs(m.mean - s.mean[0]) < 1e-9 + assert abs(m.ci_low - s.ci_low[0]) < 1e-9 + assert abs(m.ci_high - s.ci_high[0]) < 1e-9 + + +# --------------------------------------------------------------------------- +# stability +# --------------------------------------------------------------------------- + +def test_stability_single_config(): + rng = _rng(20) + M, K = 80, 5 + base = rng.normal(0.7, 0.1, M) + runs = np.array([np.clip(base + rng.normal(0, 0.02, M), 0, 1) for _ in range(K)]) + result = es.stability(runs) + assert isinstance(result, StabilityResult) + assert result.labels == ["value"] + assert result.n_runs[0] == K + assert result.instability[0] < 0.05 # tight noise -> should read as stable + d = result.to_dict() + assert "instability" in d and "icc" in d and "interpretation" in d + + +def test_stability_dict_ragged_run_counts(): + rng = _rng(21) + M = 60 + base = rng.normal(0.6, 0.1, M) + stable_runs = np.array([np.clip(base + rng.normal(0, 0.01, M), 0, 1) for _ in range(6)]) + noisy_runs = np.array([np.clip(rng.normal(0.5, 0.2, M), 0, 1) for _ in range(3)]) + result = es.stability({"stable": stable_runs, "noisy": noisy_runs}) + assert list(result.n_runs) == [6, 3] + assert result.instability[0] < result.instability[1] # stable really is more stable + frame = result.to_frame() + assert frame.loc["stable", "n_runs"] == 6 + assert frame.loc["noisy", "n_runs"] == 3 + + +def test_stability_requires_at_least_3_runs(): + rng = _rng(22) + runs = rng.normal(0.5, 0.1, (2, 30)) + with pytest.raises(ValueError, match=">= 3 runs"): + es.stability(runs) + + +def test_stability_requires_matching_item_count(): + rng = _rng(23) + a = rng.normal(0.5, 0.1, (4, 30)) + b = rng.normal(0.5, 0.1, (4, 25)) + with pytest.raises(ValueError, match="same number of items"): + es.stability({"a": a, "b": b}) + + +def test_stability_rejects_1d_input(): + with pytest.raises(ValueError, match="2-D"): + es.stability(np.zeros(10)) + + +# --------------------------------------------------------------------------- +# judge_debias_mean_ci +# --------------------------------------------------------------------------- + +def test_judge_debias_mean_ci_recovers_true_mean_better_than_raw_judge(): + rng = _rng(30) + n_total, n_labeled = 400, 40 + true_mean = 0.55 + bias = 0.2 + + human_all = np.clip(rng.normal(true_mean, 0.15, n_total), 0, 1) + judge_all = np.clip(human_all + bias + rng.normal(0, 0.05, n_total), 0, 1) + + idx = rng.choice(n_total, n_labeled, replace=False) + mask = np.zeros(n_total, dtype=bool) + mask[idx] = True + + result = es.judge_debias_mean_ci( + unlabeled_judge_scores=judge_all[~mask], + labeled_human_scores=human_all[mask], + labeled_judge_scores=judge_all[mask], + rng=_rng(31), + ) + assert isinstance(result, DebiasedMeanCI) + # Corrected mean must be closer to the true mean than the raw judge mean. + assert abs(result.mean - true_mean) < abs(result.judge_mean - true_mean) + assert result.ci_low < result.mean < result.ci_high + assert result.n_labeled == n_labeled + assert result.n_unlabeled == n_total - n_labeled + assert result.p_value is None # compute_pvalue defaults to False + + +def test_judge_debias_mean_ci_to_dict(): + rng = _rng(32) + human = rng.normal(0.5, 0.1, 20) + judge = human + rng.normal(0, 0.05, 20) + unlabeled = rng.normal(0.6, 0.1, 100) + d = es.judge_debias_mean_ci(unlabeled, human, judge, rng=_rng(1)).to_dict() + assert set(d.keys()) == { + "mean", "ci_low", "ci_high", "judge_mean", "human_mean", + "rectifier", "p_value", "n_labeled", "n_unlabeled", + } + + +def test_judge_debias_mean_ci_rejects_mismatched_labeled_shapes(): + rng = _rng(33) + with pytest.raises(ValueError, match="paired, same-shape"): + es.judge_debias_mean_ci( + unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), + labeled_human_scores=rng.normal(0.5, 0.1, 20), + labeled_judge_scores=rng.normal(0.5, 0.1, 19), + ) + + +def test_judge_debias_mean_ci_warns_below_15_labeled(): + rng = _rng(34) + with pytest.warns(UserWarning, match="labeled items"): + es.judge_debias_mean_ci( + unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), + labeled_human_scores=rng.normal(0.5, 0.1, 10), + labeled_judge_scores=rng.normal(0.5, 0.1, 10), + ) + + +def test_judge_debias_mean_ci_compute_pvalue_opt_in(): + rng = _rng(35) + result = es.judge_debias_mean_ci( + unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), + labeled_human_scores=rng.normal(0.5, 0.1, 20), + labeled_judge_scores=rng.normal(0.5, 0.1, 20), + compute_pvalue=True, + ) + assert result.p_value is not None + + +# --------------------------------------------------------------------------- +# judge_alignment -- array-based path +# --------------------------------------------------------------------------- + +def test_judge_alignment_array_form_basic(): + rng = _rng(40) + n = 40 + human = rng.integers(1, 6, n).astype(float) + judge = np.clip(human + rng.normal(0, 0.5, n), 1, 5) + result = es.judge_alignment(human, judge) + assert result.n_labeled == n + assert result.n_total == n # no all_judge_scores given + assert result.representativeness == {} # no distribution check without all_judge_scores + + +def test_judge_alignment_array_form_with_all_judge_scores(): + rng = _rng(41) + n_lab, n_total = 35, 250 + human = rng.integers(1, 6, n_lab).astype(float) + judge = np.clip(human + rng.normal(0, 0.5, n_lab), 1, 5) + all_judge = np.clip(rng.integers(1, 6, n_total).astype(float) + rng.normal(0, 0.3, n_total), 1, 5) + result = es.judge_alignment(human, judge, all_judge_scores=all_judge) + assert result.n_total == n_total + assert "score_distribution" in result.representativeness + # No slice-column checks in the array form (no DataFrame). + assert not any(k.startswith("slice_") for k in result.representativeness) + + +def test_judge_alignment_array_form_display_names(): + rng = _rng(42) + human = rng.integers(1, 6, 35).astype(float) + judge = np.clip(human + rng.normal(0, 0.5, 35), 1, 5) + result = es.judge_alignment(human, judge, llm_metric="my_judge", human_groundtruth="my_human") + assert result.llm_metric == "my_judge" + assert result.human_col == "my_human" + + +def test_judge_alignment_array_form_defaults_display_names(): + rng = _rng(43) + human = rng.integers(1, 6, 35).astype(float) + judge = np.clip(human + rng.normal(0, 0.5, 35), 1, 5) + result = es.judge_alignment(human, judge) + assert result.llm_metric == "judge" + assert result.human_col == "human" + + +def test_judge_alignment_array_form_requires_judge_labels(): + with pytest.raises(TypeError, match="requires both arrays"): + es.judge_alignment(np.array([1.0, 2.0, 3.0])) + + +def test_judge_alignment_array_form_rejects_mismatched_shapes(): + with pytest.raises(ValueError, match="paired, same-shape"): + es.judge_alignment(np.array([1.0, 2.0, 3.0]), np.array([1.0, 2.0])) + + +def test_judge_alignment_array_form_warns_below_30(): + rng = _rng(44) + human = rng.integers(1, 6, 10).astype(float) + judge = np.clip(human + rng.normal(0, 0.5, 10), 1, 5) + with pytest.warns(UserWarning, match="fewer than ~30"): + es.judge_alignment(human, judge) + + +def test_judge_alignment_evaldata_form_still_requires_kwargs(): + """The dispatcher must still enforce llm_metric/human_groundtruth for + the EvalResults form -- this is a straight rename + new sibling form, + not a behavior change to the original signature's requiredness.""" + rows = [ + {"model": "a", "item": f"q{i}", "score": 0.5, "human": (0.5 if i < 10 else None)} + for i in range(40) + ] + df = pd.DataFrame(rows) + evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) + with pytest.raises(TypeError, match="requires llm_metric"): + es.judge_alignment(evaldata) + + +def test_judge_alignment_evaldata_form_rejects_second_positional(): + rows = [ + {"model": "a", "item": f"q{i}", "score": 0.5, "human": (0.5 if i < 10 else None)} + for i in range(40) + ] + df = pd.DataFrame(rows) + evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) + with pytest.raises(TypeError, match="doesn't take a second positional"): + es.judge_alignment(evaldata, np.array([1.0, 2.0])) From a8b3f54683895e37ce64b5625e9ce901f1a85837 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 20:23:36 -0400 Subject: [PATCH 009/245] Fix the 8 usability-audit findings in the quick primitives - stability(): add a DataFrame form (config_col/run_col/item_col/value_col) with no orientation ambiguity, since each row names its own run/item -- the recommended path now. The raw-array/dict form keeps working but gets a heuristic warning when K > M (more "runs" than "items" is unusual for real eval data and is the signature of an un-transposed pivot table -- confirmed this exact mixup silently flips "very stable" into "near-random" on identical data with zero error). - mean_ci()/summarize(): strip NaN instead of silently propagating it into a NaN CI (confirmed: 10% missing data previously returned a real mean but ci_low=ci_high=nan with no warning). Warn how many were dropped; raise a clear, group-attributed error when nothing valid remains. - robustness_metrics(): statistic="median" combined with a mean/proportion -only closed-form CI method (wilson, logit_t, nig, t_interval, ...) previously silently produced a CI for the *mean* while reporting the *median* as the point estimate -- confirmed cases where the reported CI didn't even contain the reported point estimate. Now falls back to smooth_bootstrap (which respects `statistic`) with a warning explaining why, and always warns that statistic="median" hasn't been validated by the same simulation-based calibration testing as the mean path. Also fixed a related bug in the same function: the bootstrap_t path never forwarded `statistic` to bootstrap_t_ci_1d despite that function supporting it, and its multi_ci gradient loop used the fixed `alpha` instead of the per-band loop variable `a`. - judge_alignment(): reordered the array form to judge-first (judge_scores, human_scores), matching the EvalResults form's llm_metric-before-human_groundtruth convention -- previously the two forms disagreed, and swapping the array order silently produced a different, complete-looking report using a different statistical methodology with no error. Redesigned the array form's shape to match: both arrays are now the same length (every item), with human_scores NaN for unlabeled items -- mirrors the EvalResults form's sparse-column convention exactly, removes the manual-masking step, and gets the representativeness check for free when unlabeled items are present. - judge_debias_mean_ci(): same redesign, (judge_scores, human_scores) sparse pair replacing the old 3-separate-array signature. Added the hard-error/soft-warning thresholds already used by compare(alignment=...)'s PPI path (n_labeled >= 15, n_total >= 50 hard errors; n_labeled >= 30, n_total >= 100 soft warnings) plus a new hard error when every item is labeled (nothing left for PPI to correct). Now always warns that the labeled subset must be an unbiased, ideally uniform-random sample -- restating ppi.correct()'s own MNAR-labeling caveat, which the previous docstring-only version never surfaced. Also fixed two real stacklevel bugs in resolve_auto_robustness_method() found while verifying warning attribution for these changes: the resolve_score_bounds() delegation and the direct warnings.warn() call for the "unbounded" case need different stacklevel adjustments (one extra frame vs. none), but both were using the same value, overshooting past the user's actual call site for the direct-warn path in mean_ci()/summarize(). Verified: 444 tests passing across test_quick_primitives.py (61, up from 40), test_alignment.py, test_auto_ci_routing.py, test_analyze.py, test_compare.py, test_pareto.py, test_resampling.py, test_simultaneous_ci.py, test_cli.py, and test_compound_ppi_fwer.py. Re-verified the specific orientation-flip and NaN-CI repro cases from the audit by hand. Updated examples/quick_primitives_demo.py to the new signatures. Co-Authored-By: Claude Sonnet 5 --- evalstats/alignment.py | 114 ++++++---- evalstats/core/router.py | 5 +- evalstats/core/variance.py | 32 ++- evalstats/quick.py | 348 ++++++++++++++++++++++-------- examples/quick_primitives_demo.py | 42 ++-- tests/test_quick_primitives.py | 174 +++++++++------ 6 files changed, 509 insertions(+), 206 deletions(-) diff --git a/evalstats/alignment.py b/evalstats/alignment.py index 09bf851..7c83d9c 100644 --- a/evalstats/alignment.py +++ b/evalstats/alignment.py @@ -1123,8 +1123,8 @@ def _judge_alignment_from_evaldata( def _judge_alignment_from_arrays( - human_labels: np.ndarray, - judge_labels: np.ndarray, + judge_scores: np.ndarray, + human_scores: np.ndarray, *, all_judge_scores: Optional[np.ndarray], score_type: Optional[str], @@ -1132,21 +1132,30 @@ def _judge_alignment_from_arrays( human_groundtruth: Optional[str], alpha: float, ) -> AlignmentResult: - human_aligned = np.asarray(human_labels, dtype=float) - llm_aligned = np.asarray(judge_labels, dtype=float) - if human_aligned.shape != llm_aligned.shape: + judge_full = np.asarray(judge_scores, dtype=float) + human_full = np.asarray(human_scores, dtype=float) + if judge_full.shape != human_full.shape: raise ValueError( - "human_labels and judge_labels must be paired, same-shape " - f"arrays (one human + one judge score per labeled item); got " - f"shapes {human_aligned.shape} and {llm_aligned.shape}." + "judge_scores and human_scores must be the same length -- one " + "judge score + one (possibly NaN) human score per item; got " + f"shapes {judge_full.shape} and {human_full.shape}." ) - if human_aligned.ndim != 1: + if judge_full.ndim != 1: raise ValueError( - f"human_labels/judge_labels must be 1-D; got shape {human_aligned.shape}." + f"judge_scores/human_scores must be 1-D; got shape {judge_full.shape}." ) - n_labeled = int(human_aligned.size) + + labeled_mask = ~np.isnan(human_full) + n_labeled = int(labeled_mask.sum()) if n_labeled == 0: - raise ValueError("human_labels/judge_labels must not be empty.") + raise ValueError( + "No labeled items -- human_scores is all NaN. It should be " + "non-NaN for the alignment subset and NaN elsewhere (or, if " + "every item is labeled, contain no NaN at all)." + ) + llm_aligned = judge_full[labeled_mask] + human_aligned = human_full[labeled_mask] + if n_labeled < 30: warnings.warn( f"Only {n_labeled} items have human labels. " @@ -1156,11 +1165,22 @@ def _judge_alignment_from_arrays( stacklevel=3, ) - all_llm = None - n_total = n_labeled + # judge_scores doubles as "every item's judge score" for the + # representativeness check for free -- but only when there's actual + # evidence it's the full pool (some items weren't labeled). When + # n_labeled == judge_full.size (no NaN at all in human_scores), there's + # no way to tell "this is the full pool, 100% labeled" apart from "this + # is just the labeled subset the caller already extracted" -- stay + # conservative and skip the check rather than silently comparing a set + # against itself (which would trivially "pass" and could read as false + # confidence). An explicit all_judge_scores= always wins either way. if all_judge_scores is not None: all_llm = np.asarray(all_judge_scores, dtype=float) - n_total = int(all_llm.size) + elif n_labeled < judge_full.size: + all_llm = judge_full + else: + all_llm = None + n_total = int(all_llm.size) if all_llm is not None else n_labeled if score_type is None: from evalstats.loader import _detect_score_type @@ -1176,8 +1196,8 @@ def _judge_alignment_from_arrays( def judge_alignment( - human_labels_or_evaldata, - judge_labels=None, + judge_scores_or_evaldata, + human_scores=None, *, llm_metric: Optional[str] = None, human_groundtruth: Optional[str] = None, @@ -1197,30 +1217,33 @@ def judge_alignment( the full item pool, plus categorical slice-column checks) since the full dataset and its other columns are available. The returned result can be passed to ``compare(alignment={metric: result})``. - 2. ``judge_alignment(human_labels, judge_labels)`` -- a quick-primitive - form for when you already have the two paired arrays for the - labeled subset in hand and don't want to build an ``EvalResults`` - first. Pass ``all_judge_scores`` (every item's judge score, labeled - or not) to also get the score-distribution representativeness - check; without it, that check is skipped (not approximated) since - there's no full item pool to compare against. The categorical - slice-column checks are DataFrame-specific and are always skipped - in this form. **The result from this form carries placeholder - column names and cannot be passed to ``compare(alignment=...)``** - (there's no underlying DataFrame for it to look values up in) -- - use form 1 for that. + 2. ``judge_alignment(judge_scores, human_scores)`` -- a quick-primitive + form for when you don't want to build an ``EvalResults`` first. + ``judge_scores`` is every item's judge score; ``human_scores`` is + the *same length*, with ``NaN`` for items that don't have a human + label (or no ``NaN`` at all if every item happens to be labeled). + This mirrors form 1's sparse-column convention exactly, so you can + hand it whatever you already have without pre-splitting anything + yourself. When some items are unlabeled, the score-distribution + representativeness check runs automatically (``judge_scores`` is + already the full pool); pass ``all_judge_scores`` explicitly to + override this. The categorical slice-column checks are + DataFrame-specific and are always skipped in this form. **The + result from this form carries placeholder column names and cannot + be passed to ``compare(alignment=...)``** (there's no underlying + DataFrame for it to look values up in) -- use form 1 for that. Either form fits a Bayesian calibration model that can later be used to propagate judge uncertainty into downstream comparisons. Parameters ---------- - human_labels_or_evaldata : EvalResults or array-like - Either evaluation data from :func:`load_from` (form 1), or the - human-labeled subset's scores (form 2). - judge_labels : array-like, optional - Judge scores for that same labeled subset, paired with - ``human_labels_or_evaldata`` (form 2 only). + judge_scores_or_evaldata : EvalResults or array-like + Either evaluation data from :func:`load_from` (form 1), or every + item's judge score (form 2). + human_scores : array-like, optional + Same length as ``judge_scores_or_evaldata``, ``NaN`` for unlabeled + items (form 2 only). llm_metric : str, optional Form 1: column name of the LLM judge scores (required). Form 2: optional display name for the judge, used only in printed reports. @@ -1230,12 +1253,13 @@ def judge_alignment( subset, ``NaN`` elsewhere). Form 2: optional display name for the human rater, used only in printed reports. all_judge_scores : array-like, optional - Form 2 only: every item's judge score (labeled and unlabeled), to - enable the score-distribution representativeness check. + Form 2 only: override which array is treated as "every item's + judge score" for the representativeness check. Only needed if + that shouldn't just be ``judge_scores_or_evaldata`` itself. score_type : str, optional Form 2 only: override the auto-detected score type (``"binary"``, ``"likert"``, ``"continuous"``, or ``"grade"``). Auto-detected from - ``judge_labels`` when not given. + the labeled judge scores when not given. alpha : float Significance level for alignment metric CIs. Default ``0.05``. @@ -1245,15 +1269,15 @@ def judge_alignment( """ from evalstats.loader import EvalResults - if isinstance(human_labels_or_evaldata, EvalResults): - evaldata = human_labels_or_evaldata - if judge_labels is not None: + if isinstance(judge_scores_or_evaldata, EvalResults): + evaldata = judge_scores_or_evaldata + if human_scores is not None: raise TypeError( "judge_alignment(evaldata, ...) doesn't take a second " "positional argument; pass llm_metric= and " "human_groundtruth= as column names instead. (For the " "raw-array form, pass two arrays: " - "judge_alignment(human_labels, judge_labels).)" + "judge_alignment(judge_scores, human_scores).)" ) if llm_metric is None or human_groundtruth is None: raise TypeError( @@ -1264,15 +1288,15 @@ def judge_alignment( evaldata, llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, ) - if judge_labels is None: + if human_scores is None: raise TypeError( - "judge_alignment(human_labels, judge_labels) requires both " + "judge_alignment(judge_scores, human_scores) requires both " "arrays; or pass an EvalResults (from load_from()) as the " "first argument for the column-name-based form: " "judge_alignment(evaldata, llm_metric=..., human_groundtruth=...)." ) return _judge_alignment_from_arrays( - human_labels_or_evaldata, judge_labels, + judge_scores_or_evaldata, human_scores, all_judge_scores=all_judge_scores, score_type=score_type, llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, ) diff --git a/evalstats/core/router.py b/evalstats/core/router.py index 916406e..03e20a2 100644 --- a/evalstats/core/router.py +++ b/evalstats/core/router.py @@ -801,6 +801,9 @@ def resolve_auto_robustness_method( data_kind = "bounded_01" else: data_kind = "unbounded" + # Direct warn() call, one frame shallower than the + # resolve_score_bounds() delegation above (no extra frame in + # between) -- stacklevel here, not stacklevel + 1. warnings.warn( "Numeric evaluation data outside [0, 1] was auto-detected " "with no explicit score_range, so evalstats is using " @@ -810,7 +813,7 @@ def resolve_auto_robustness_method( "explicitly, e.g. score_range=(1, 5) for a Likert scale " "or score_range=(0, 100) for a percentage grade.", UserWarning, - stacklevel=stacklevel + 1, + stacklevel=stacklevel, ) # See config.AUTO_ANALYZE_METHOD_TABLE for the full auto-routing matrix # (which method is chosen for which data kind / N / seeded combination). diff --git a/evalstats/core/variance.py b/evalstats/core/variance.py index d8ce641..765b5b6 100644 --- a/evalstats/core/variance.py +++ b/evalstats/core/variance.py @@ -14,6 +14,7 @@ from __future__ import annotations +import warnings from dataclasses import dataclass from typing import Optional @@ -307,7 +308,34 @@ def robustness_metrics( ci_low_arr: Optional[np.ndarray] = None ci_high_arr: Optional[np.ndarray] = None multi_ci_result: Optional[dict[float, tuple[np.ndarray, np.ndarray]]] = None + # These are closed-form CIs for a *proportion or mean* (binomial score + # intervals, NIG, logit-t, ...) -- there's no median variant of any of + # them, so they silently ignore `statistic` entirely: the point estimate + # column would correctly report the median while the CI stayed built + # around the mean/proportion, a mismatch that can be severe enough for + # the reported CI to not even contain the reported point estimate (e.g. + # binary data whose median is 0 or 1 but whose Wilson CI is centered on + # the proportion). Substitute a bootstrap method that actually respects + # `statistic` instead of silently returning a mismatched CI. _analytical = {"wilson", "wilson_od", "jeffreys", "nig", "nig_nested", "t_interval", "logit_t"} + if statistic == "median": + warnings.warn( + "statistic='median' has not been validated by the same " + "simulation-based calibration testing as statistic='mean' " + "(the default) -- treat median CIs here more cautiously.", + UserWarning, + stacklevel=2, + ) + if marginal_method in _analytical: + warnings.warn( + f"marginal_method='{marginal_method}' has no median variant " + "(it's a closed-form CI for a proportion/mean); falling " + "back to 'smooth_bootstrap' so the CI actually corresponds " + "to the reported median instead of silently mismatching it.", + UserWarning, + stacklevel=2, + ) + marginal_method = "smooth_bootstrap" if n_bootstrap is not None or marginal_method in _analytical: if rng is None and n_bootstrap is not None: rng = np.random.default_rng() @@ -393,10 +421,10 @@ def robustness_metrics( ml, mh = logit_t_ci_1d(row, a) mci_lows[a].append(ml); mci_highs[a].append(mh) elif marginal_method == "bootstrap_t": - lo, hi = bootstrap_t_ci_1d(row, point_est, n_bootstrap, alpha, rng) + lo, hi = bootstrap_t_ci_1d(row, point_est, n_bootstrap, alpha, rng, statistic=statistic) if multi_ci: for a in GRADIENT_CI_ALPHAS: - ml, mh = bootstrap_t_ci_1d(row, point_est, n_bootstrap, alpha, rng) + ml, mh = bootstrap_t_ci_1d(row, point_est, n_bootstrap, a, rng, statistic=statistic) mci_lows[a].append(ml); mci_highs[a].append(mh) elif marginal_method == "bayes_bootstrap": boot = bayes_bootstrap_means_1d(row, n_bootstrap, rng, statistic=statistic) diff --git a/evalstats/quick.py b/evalstats/quick.py index 21e4a22..65996b9 100644 --- a/evalstats/quick.py +++ b/evalstats/quick.py @@ -30,6 +30,33 @@ from .core.variance import robustness_metrics, seed_variance_decomposition +def _clean_1d(arr: np.ndarray, *, label: str, stacklevel: int) -> np.ndarray: + """Drop NaN entries from a 1-D score array, warning if any were found, + and raising a clear, attributed error if nothing valid remains. + + Real-world data has gaps -- a failed API call, a skipped item -- and + ``compare()`` handles that by rejecting NaN outright with a hard error. + That's the right call for a full comparative report, but it would + defeat the purpose of a quick primitive: forcing every caller to + hand-filter NaN themselves before getting a single number back. Instead, + drop it and say so (not silently -- an unflagged NaN CI, which is what + happens without this filtering, is worse than either option). + """ + is_nan = np.isnan(arr) + n_missing = int(np.sum(is_nan)) + if n_missing > 0: + arr = arr[~is_nan] + warnings.warn( + f"{label}: dropped {n_missing} NaN (missing) value(s) out of " + f"{n_missing + arr.size}; computed from the remaining {arr.size}.", + UserWarning, + stacklevel=stacklevel, + ) + if arr.size == 0: + raise ValueError(f"{label} has no valid (non-NaN) scores.") + return arr + + # --------------------------------------------------------------------------- # mean_ci # --------------------------------------------------------------------------- @@ -121,6 +148,7 @@ def mean_ci( raise ValueError(f"scores must be a 1-D array; got shape {arr.shape}.") if arr.size == 0: raise ValueError("scores must not be empty.") + arr = _clean_1d(arr, label="scores", stacklevel=3) if alpha is None: alpha = get_alpha_ci() @@ -140,7 +168,7 @@ def mean_ci( mean=float(rob.mean[0]), ci_low=float(rob.ci_low[0]) if rob.ci_low is not None else float("nan"), ci_high=float(rob.ci_high[0]) if rob.ci_high is not None else float("nan"), - n=int(np.sum(~np.isnan(arr))), + n=int(arr.size), method=robustness_method, ) @@ -322,6 +350,10 @@ def summarize( for lbl, a in zip(labels, arrays): if a.size == 0: raise ValueError(f"Group '{lbl}' has no scores.") + arrays = [ + _clean_1d(a, label=f"group '{lbl}'" if not single_ungrouped else "scores", stacklevel=3) + for lbl, a in zip(labels, arrays) + ] if alpha is None: alpha = get_alpha_ci() @@ -349,7 +381,7 @@ def summarize( for i, a in enumerate(arrays): a_2d = a.reshape(1, -1) _, robustness_method, resolved_score_range = resolve_auto_robustness_method( - a_2d, score_range=score_range, stacklevel=4, + a_2d, score_range=score_range, stacklevel=3, ) rob = robustness_metrics( a_2d, ["_"], @@ -370,7 +402,7 @@ def summarize( p90[i] = rob.percentiles[90][0] ci_low[i] = rob.ci_low[0] if rob.ci_low is not None else np.nan ci_high[i] = rob.ci_high[0] if rob.ci_high is not None else np.nan - n[i] = int(np.sum(~np.isnan(a))) + n[i] = a.size # a is already NaN-cleaned above method.append(robustness_method) return GroupSummary( @@ -448,7 +480,92 @@ def to_frame(self) -> pd.DataFrame: ) -def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = None) -> StabilityResult: +def _stability_core(labels: list[str], arrays: list[np.ndarray], *, warn_orientation: bool) -> StabilityResult: + """Shared core behind both stability() input forms: validates shapes, + pads a ragged run axis with NaN, and runs the seed-variance + decomposition. ``warn_orientation`` is only True for the raw-array/dict + form -- the DataFrame form has no orientation to get wrong (each row + names its own run and item explicitly), so it's skipped there. + """ + for lbl, a in zip(labels, arrays): + if a.ndim != 2: + raise ValueError(f"runs['{lbl}'] must be 2-D (K runs x M items); got shape {a.shape}.") + + m_values = {a.shape[1] for a in arrays} + if len(m_values) != 1: + raise ValueError( + "All configs must be evaluated on the same number of items (M); " + f"got M values {sorted(m_values)} across configs " + f"{dict(zip(labels, (a.shape for a in arrays)))}." + ) + m_items = m_values.pop() + + if any(a.shape[0] < 3 for a in arrays): + offender = labels[[a.shape[0] for a in arrays].index(min(a.shape[0] for a in arrays))] + raise ValueError( + f"Seed-variance decomposition requires >= 3 runs per config; " + f"config '{offender}' has {min(a.shape[0] for a in arrays)}." + ) + + if warn_orientation: + # A (K runs, M items) array with K > M is unusual for real eval data + # (far more items than repeated runs, typically) -- a much more + # common mistake is passing an (M, K) array straight out of + # df.pivot(index='item', columns='run') without transposing, which + # silently swaps which axis is "runs" and which is "items" with no + # error, producing a plausible-looking but wrong instability/icc + # (confirmed: an (items, runs) mixup can flip "very stable" into + # "near-random" on the same data). This heuristic won't catch every + # case (K can legitimately exceed M for a heavily-repeated small + # item set) -- prefer the DataFrame form below, which has no + # orientation ambiguity to get wrong in the first place. + for lbl, a in zip(labels, arrays): + if a.shape[0] > a.shape[1]: + warnings.warn( + f"runs['{lbl}'] has more rows ({a.shape[0]}) than columns " + f"({a.shape[1]}) -- stability() expects (K runs, M items), " + "and most real eval data has far more items than repeated " + "runs. If this came from a pivot table shaped (items, " + "runs), you likely need to transpose it (.T) before " + "calling stability(), or use the DataFrame form " + "(stability(df, config_col=..., run_col=..., item_col=..., " + "value_col=...)) instead, which has no orientation to get " + "wrong.", + UserWarning, + stacklevel=4, + ) + + # Different configs may have different K (run count); pad the run axis + # with NaN -- seed_variance_decomposition's internal nanmean/nanvar + # handle that safely (it's a closed-form ANOVA-style computation, not a + # resampling procedure, so NaN-tolerant reductions are exact here). + max_k = max(a.shape[0] for a in arrays) + scores_3d = np.full((len(arrays), m_items, max_k), np.nan) + for i, a in enumerate(arrays): + scores_3d[i, :, : a.shape[0]] = a.T # (K, M) -> (M, K) + + from .core.summary import _instability_label + + sv = seed_variance_decomposition(scores_3d, labels) + actual_n_runs = np.array([a.shape[0] for a in arrays]) + return StabilityResult( + labels=labels, + instability=sv.instability, + icc=sv.icc, + n_runs=actual_n_runs, + label_text=[_instability_label(float(v)) for v in sv.instability], + ) + + +def stability( + runs: Union[np.ndarray, dict, pd.DataFrame], + *, + labels: Optional[list[str]] = None, + config_col: Optional[str] = None, + run_col: Optional[str] = None, + item_col: Optional[str] = None, + value_col: Optional[str] = None, +) -> StabilityResult: """Multi-run reliability: how much does a config's score move across repeated runs on the same items? @@ -457,17 +574,29 @@ def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = No configuration reliable enough to ship" without needing a full multi-model comparison. + Three input forms: + + * A long-format DataFrame plus ``config_col``/``run_col``/``item_col``/ + ``value_col`` -- **recommended**, since each row names its own run + and item explicitly, there's no axis-orientation to get wrong. + * A single config's repeated-run scores as a 2-D array of shape + ``(K, M)`` (K runs, M items -- note this is *not* what + ``df.pivot(index='item', columns='run')`` gives you; that needs a + ``.T`` first, or use the DataFrame form directly). + * A ``{label: (K, M) array}`` dict of several configs. + + Configs must share the same M (same item set); K (number of runs, + >= 3) can differ per config. + Parameters ---------- - runs : array-like or dict - A single config's repeated-run scores as a 2-D array of shape - ``(K, M)`` (K runs, M items, same M items each run) -- one config; - or a ``{label: (K, M) array}`` dict of several configs. Configs - must share the same M (same item set); K (number of runs, >= 3) can - differ per config. + runs : array-like, dict, or DataFrame + See above. labels : list[str], optional Override labels when ``runs`` is a single array (default - ``["value"]``). Ignored when ``runs`` is a dict (its keys are used). + ``["value"]``). Ignored for the dict/DataFrame forms. + config_col, run_col, item_col, value_col : str, optional + Required (and only used) when ``runs`` is a DataFrame. Returns ------- @@ -476,9 +605,47 @@ def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = No Examples -------- >>> import evalstats as es + >>> es.stability(df, config_col="config", run_col="run", + ... item_col="item", value_col="score") >>> es.stability(rag_config_a_runs) # shape (5, 200): 5 runs, 200 items >>> es.stability({"config_a": runs_a, "config_b": runs_b}).to_frame() """ + if isinstance(runs, pd.DataFrame): + missing = [ + name for name, col in [ + ("config_col", config_col), ("run_col", run_col), + ("item_col", item_col), ("value_col", value_col), + ] if col is None + ] + if missing: + raise ValueError( + "stability() on a DataFrame requires config_col, run_col, " + "item_col, and value_col; missing: " + ", ".join(missing) + ) + for name, col in [ + ("config_col", config_col), ("run_col", run_col), + ("item_col", item_col), ("value_col", value_col), + ]: + if col not in runs.columns: + raise ValueError(f"{name} '{col}' not found in DataFrame columns: {list(runs.columns)}") + + input_labels: list[str] = [] + arrays: list[np.ndarray] = [] + item_order = None + for config, group in runs.groupby(config_col, sort=False): + pivot = group.pivot(index=run_col, columns=item_col, values=value_col) + if item_order is None: + item_order = list(pivot.columns) + elif set(pivot.columns) != set(item_order): + raise ValueError( + f"config '{config}' was scored on a different set of items " + "than the others -- stability() requires every config to " + "share the same item set." + ) + input_labels.append(str(config)) + arrays.append(pivot.reindex(columns=item_order).to_numpy(dtype=float)) + return _stability_core(input_labels, arrays, warn_orientation=False) + if isinstance(runs, dict): if len(runs) == 0: raise ValueError("runs dict must not be empty.") @@ -488,8 +655,9 @@ def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = No arr = np.asarray(runs, dtype=float) if arr.ndim != 2: raise ValueError( - "runs must be a 2-D array (K runs x M items) or a " - f"{{label: array}} dict of such arrays; got shape {arr.shape}." + "runs must be a 2-D array (K runs x M items), a " + "{label: array} dict of such arrays, or a DataFrame (with " + f"config_col/run_col/item_col/value_col); got shape {arr.shape}." ) input_labels = list(labels) if labels is not None else ["value"] if len(input_labels) != 1: @@ -498,45 +666,7 @@ def stability(runs: Union[np.ndarray, dict], *, labels: Optional[list[str]] = No ) arrays = [arr] - for lbl, a in zip(input_labels, arrays): - if a.ndim != 2: - raise ValueError(f"runs['{lbl}'] must be 2-D (K runs x M items); got shape {a.shape}.") - - m_values = {a.shape[1] for a in arrays} - if len(m_values) != 1: - raise ValueError( - "All configs must be evaluated on the same number of items (M); " - f"got M values {sorted(m_values)} across configs " - f"{dict(zip(input_labels, (a.shape for a in arrays)))}." - ) - m_items = m_values.pop() - - # Different configs may have different K (run count); pad the run axis - # with NaN -- seed_variance_decomposition's internal nanmean/nanvar - # handle that safely (it's a closed-form ANOVA-style computation, not a - # resampling procedure, so NaN-tolerant reductions are exact here). - max_k = max(a.shape[0] for a in arrays) - if any(a.shape[0] < 3 for a in arrays): - offender = input_labels[[a.shape[0] for a in arrays].index(min(a.shape[0] for a in arrays))] - raise ValueError( - f"Seed-variance decomposition requires >= 3 runs per config; " - f"config '{offender}' has {min(a.shape[0] for a in arrays)}." - ) - scores_3d = np.full((len(arrays), m_items, max_k), np.nan) - for i, a in enumerate(arrays): - scores_3d[i, :, : a.shape[0]] = a.T # (K, M) -> (M, K) - - from .core.summary import _instability_label - - sv = seed_variance_decomposition(scores_3d, input_labels) - actual_n_runs = np.array([a.shape[0] for a in arrays]) - return StabilityResult( - labels=input_labels, - instability=sv.instability, - icc=sv.icc, - n_runs=actual_n_runs, - label_text=[_instability_label(float(v)) for v in sv.instability], - ) + return _stability_core(input_labels, arrays, warn_orientation=True) # --------------------------------------------------------------------------- @@ -591,9 +721,8 @@ def to_dict(self) -> dict: def judge_debias_mean_ci( - unlabeled_judge_scores, - labeled_human_scores, - labeled_judge_scores, + judge_scores, + human_scores, *, alpha: float = 0.05, n_bootstrap: int = 1000, @@ -616,18 +745,25 @@ def judge_debias_mean_ci( with :func:`judge_alignment` instead, which applies the same idea per-comparison with the full Friedman/Wilcoxon machinery. + **The labeled subset must be an unbiased sample of the full dataset -- + ideally chosen uniformly at random.** This function always warns about + this (see below), because it's easy to violate without realizing it: + if which items get labeled is itself influenced by their score (e.g. + "always double-check the highest-scoring ones"), the correction can + stay biased by a large, non-vanishing amount regardless of how many + labels you have. See :func:`~evalstats.ppi.correct`'s docstring for the + full detail (this wraps it directly). + Parameters ---------- - unlabeled_judge_scores : array-like - Judge scores for every item that does NOT also have a human label. - Must be disjoint from the labeled items below -- do not pass every - item's judge score here if some of them are also in - ``labeled_judge_scores``. - labeled_human_scores : array-like - Human scores for the labeled subset. - labeled_judge_scores : array-like - Judge scores for that SAME labeled subset, paired (same order, - same length) with ``labeled_human_scores``. + judge_scores : array-like + Judge scores for every item (the full dataset). + human_scores : array-like + Same length as ``judge_scores``, with ``NaN`` for items that don't + have a human label. (Every item must NOT be labeled -- see the + "all items labeled" error below; if that's genuinely your + situation, use :func:`mean_ci` on ``human_scores`` directly + instead of this function.) alpha : float Significance level (default 0.05, i.e. 95% CI). n_bootstrap : int @@ -645,33 +781,77 @@ def judge_debias_mean_ci( Examples -------- >>> import evalstats as es - >>> result = es.judge_debias_mean_ci( - ... unlabeled_judge_scores=judge_scores[~has_human_label], - ... labeled_human_scores=human_scores[has_human_label], - ... labeled_judge_scores=judge_scores[has_human_label], - ... ) + >>> result = es.judge_debias_mean_ci(judge_scores, human_scores) >>> result.mean, result.ci_low, result.ci_high """ from .ppi import correct as _ppi_correct - y_hat_unlab = np.asarray(unlabeled_judge_scores, dtype=float) - y_lab = np.asarray(labeled_human_scores, dtype=float) - y_hat_lab = np.asarray(labeled_judge_scores, dtype=float) - - if y_lab.shape != y_hat_lab.shape: + judge_full = np.asarray(judge_scores, dtype=float) + human_full = np.asarray(human_scores, dtype=float) + if judge_full.shape != human_full.shape: + raise ValueError( + "judge_scores and human_scores must be the same length -- one " + "judge score + one (possibly NaN) human score per item; got " + f"shapes {judge_full.shape} and {human_full.shape}." + ) + if judge_full.ndim != 1: + raise ValueError(f"judge_scores/human_scores must be 1-D; got shape {judge_full.shape}.") + + labeled_mask = ~np.isnan(human_full) + n_labeled = int(labeled_mask.sum()) + n_total = int(judge_full.size) + n_unlabeled = n_total - n_labeled + + # Mirrors the exact thresholds api.py's _run_alignment_ppi already + # enforces for compare(alignment=...)'s PPI path -- same underlying + # method, same minimum sample sizes for the same reasons. + if n_labeled < 15: + raise ValueError( + f"judge_debias_mean_ci requires at least 15 human-labeled " + f"items; got {n_labeled}. Expand the labeled subset." + ) + if n_total < 50: + raise ValueError( + f"judge_debias_mean_ci requires at least 50 items total; got " + f"{n_total}. PPI correction is only beneficial at scale -- for " + "small datasets, human-label everything and use mean_ci() on " + "the human labels directly." + ) + if n_unlabeled == 0: raise ValueError( - "labeled_human_scores and labeled_judge_scores must be paired, " - f"same-shape arrays (one human + one judge score per labeled " - f"item); got shapes {y_lab.shape} and {y_hat_lab.shape}." + "Every item is labeled (human_scores has no NaN) -- there's no " + "unlabeled portion for PPI to correct. Use mean_ci() on " + "human_scores directly instead." ) - if y_lab.size < 15: + if n_labeled < 30: warnings.warn( - f"Only {y_lab.size} labeled items -- PPI correction will be " - "imprecise with fewer than ~15 labeled items. Consider " - "expanding the labeled subset.", + f"judge_debias_mean_ci: only {n_labeled} human-labeled items " + "(recommend >= 30). The correction may under-cover at this " + "sample size.", UserWarning, stacklevel=2, ) + if n_total < 100: + warnings.warn( + f"judge_debias_mean_ci: only {n_total} total items (recommend " + ">= 100). The correction may under-cover at this sample size.", + UserWarning, + stacklevel=2, + ) + warnings.warn( + "judge_debias_mean_ci assumes the labeled subset (non-NaN entries " + "of human_scores) is an unbiased sample of the full dataset -- " + "ideally chosen uniformly at random. If which items got labeled " + "was itself influenced by their score (e.g. always double-checking " + "the highest-scoring ones), this correction can stay biased " + "regardless of how many labels you have.", + UserWarning, + stacklevel=2, + ) + + y_hat_unlab = judge_full[~labeled_mask] + y_lab = human_full[labeled_mask] + y_hat_lab = judge_full[labeled_mask] result = _ppi_correct( np.mean, @@ -686,6 +866,6 @@ def judge_debias_mean_ci( human_mean=result.human_estimate, rectifier=result.rectifier, p_value=result.p_value, - n_labeled=int(y_lab.size), - n_unlabeled=int(y_hat_unlab.size), + n_labeled=n_labeled, + n_unlabeled=n_unlabeled, ) diff --git a/examples/quick_primitives_demo.py b/examples/quick_primitives_demo.py index 17acae5..d6498cf 100644 --- a/examples/quick_primitives_demo.py +++ b/examples/quick_primitives_demo.py @@ -64,14 +64,23 @@ print("4. judge_alignment() -- array-based form, no load_from() needed") print("=" * 70) +# The natural shape most real data comes in: a judge score for every item, +# and a human score that's only filled in for a small labeled subset (NaN +# elsewhere). No manual masking needed -- pass both arrays as-is. +n_total = 300 n_labeled = 40 -human_labels = rng.integers(1, 6, n_labeled).astype(float) # 1-5 Likert -judge_labels = np.clip( - np.round(human_labels + rng.normal(0.3, 0.6, n_labeled)), 1, 5 -) # rounded to whole numbers -> also detected as Likert +judge_all_likert = np.clip( + np.round(rng.integers(1, 6, n_total).astype(float) + rng.normal(0.3, 0.6, n_total)), 1, 5 +) +human_sparse = np.full(n_total, np.nan) +labeled_idx = rng.choice(n_total, n_labeled, replace=False) +human_sparse[labeled_idx] = np.clip( + np.round(judge_all_likert[labeled_idx] + rng.normal(-0.3, 0.5, n_labeled)), 1, 5 +) -alignment = es.judge_alignment(human_labels, judge_labels) -print(f"score_type={alignment.score_type} n_labeled={alignment.n_labeled}") +alignment = es.judge_alignment(judge_all_likert, human_sparse) +print(f"score_type={alignment.score_type} n_labeled={alignment.n_labeled} n_total={alignment.n_total}") +print(f"representativeness check ran automatically: {'score_distribution' in alignment.representativeness}") kappa = alignment.alignment_metrics.get("weighted_kappa") or alignment.alignment_metrics.get("cohens_kappa") if kappa is not None: print(f"weighted kappa: {kappa['estimate']:.3f} 95% CI=[{kappa['ci_low']:.3f}, {kappa['ci_high']:.3f}]") @@ -82,21 +91,26 @@ print("=" * 70) n_total = 400 +n_labeled = 40 true_mean = 0.55 judge_bias = 0.18 # judge systematically overrates human_all = np.clip(rng.normal(true_mean, 0.15, n_total), 0, 1) judge_all = np.clip(human_all + judge_bias + rng.normal(0, 0.05, n_total), 0, 1) -labeled_idx = rng.choice(n_total, 40, replace=False) -mask = np.zeros(n_total, dtype=bool) -mask[labeled_idx] = True +# Same sparse convention as judge_alignment() above: one array per item, +# human scores NaN outside the labeled subset. +human_sparse2 = np.full(n_total, np.nan) +labeled_idx2 = rng.choice(n_total, n_labeled, replace=False) +human_sparse2[labeled_idx2] = human_all[labeled_idx2] + +import warnings +with warnings.catch_warnings(): + # judge_debias_mean_ci always reminds you the labeled subset must be a + # random sample -- expected here since we did sample uniformly at random. + warnings.simplefilter("ignore", UserWarning) + debiased = es.judge_debias_mean_ci(judge_all, human_sparse2) -debiased = es.judge_debias_mean_ci( - unlabeled_judge_scores=judge_all[~mask], - labeled_human_scores=human_all[mask], - labeled_judge_scores=judge_all[mask], -) print(f"judge-only mean (biased): {debiased.judge_mean:.3f}") print(f"PPI-corrected mean: {debiased.mean:.3f} " f"95% CI=[{debiased.ci_low:.3f}, {debiased.ci_high:.3f}]") diff --git a/tests/test_quick_primitives.py b/tests/test_quick_primitives.py index f486183..d0993e0 100644 --- a/tests/test_quick_primitives.py +++ b/tests/test_quick_primitives.py @@ -224,28 +224,27 @@ def test_stability_rejects_1d_input(): # --------------------------------------------------------------------------- -# judge_debias_mean_ci +# judge_debias_mean_ci -- (judge_scores, human_scores) sparse form: +# same length, human_scores is NaN outside the labeled subset. # --------------------------------------------------------------------------- -def test_judge_debias_mean_ci_recovers_true_mean_better_than_raw_judge(): - rng = _rng(30) - n_total, n_labeled = 400, 40 - true_mean = 0.55 - bias = 0.2 - +def _sparse_debias_pair(rng, n_total, n_labeled, *, true_mean=0.55, bias=0.2): human_all = np.clip(rng.normal(true_mean, 0.15, n_total), 0, 1) judge_all = np.clip(human_all + bias + rng.normal(0, 0.05, n_total), 0, 1) - idx = rng.choice(n_total, n_labeled, replace=False) - mask = np.zeros(n_total, dtype=bool) - mask[idx] = True - - result = es.judge_debias_mean_ci( - unlabeled_judge_scores=judge_all[~mask], - labeled_human_scores=human_all[mask], - labeled_judge_scores=judge_all[mask], - rng=_rng(31), - ) + human_sparse = np.full(n_total, np.nan) + human_sparse[idx] = human_all[idx] + return judge_all, human_sparse + + +def test_judge_debias_mean_ci_recovers_true_mean_better_than_raw_judge(): + rng = _rng(30) + n_total, n_labeled, true_mean = 400, 40, 0.55 + judge, human = _sparse_debias_pair(rng, n_total, n_labeled, true_mean=true_mean) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = es.judge_debias_mean_ci(judge, human, rng=_rng(31)) assert isinstance(result, DebiasedMeanCI) # Corrected mean must be closer to the true mean than the raw judge mean. assert abs(result.mean - true_mean) < abs(result.judge_mean - true_mean) @@ -257,69 +256,120 @@ def test_judge_debias_mean_ci_recovers_true_mean_better_than_raw_judge(): def test_judge_debias_mean_ci_to_dict(): rng = _rng(32) - human = rng.normal(0.5, 0.1, 20) - judge = human + rng.normal(0, 0.05, 20) - unlabeled = rng.normal(0.6, 0.1, 100) - d = es.judge_debias_mean_ci(unlabeled, human, judge, rng=_rng(1)).to_dict() + judge, human = _sparse_debias_pair(rng, n_total=150, n_labeled=20) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + d = es.judge_debias_mean_ci(judge, human, rng=_rng(1)).to_dict() assert set(d.keys()) == { "mean", "ci_low", "ci_high", "judge_mean", "human_mean", "rectifier", "p_value", "n_labeled", "n_unlabeled", } -def test_judge_debias_mean_ci_rejects_mismatched_labeled_shapes(): +def test_judge_debias_mean_ci_rejects_mismatched_shapes(): rng = _rng(33) - with pytest.raises(ValueError, match="paired, same-shape"): - es.judge_debias_mean_ci( - unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), - labeled_human_scores=rng.normal(0.5, 0.1, 20), - labeled_judge_scores=rng.normal(0.5, 0.1, 19), - ) + with pytest.raises(ValueError, match="same length"): + es.judge_debias_mean_ci(rng.normal(0.5, 0.1, 60), rng.normal(0.5, 0.1, 59)) -def test_judge_debias_mean_ci_warns_below_15_labeled(): +def test_judge_debias_mean_ci_rejects_below_15_labeled(): rng = _rng(34) - with pytest.warns(UserWarning, match="labeled items"): - es.judge_debias_mean_ci( - unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), - labeled_human_scores=rng.normal(0.5, 0.1, 10), - labeled_judge_scores=rng.normal(0.5, 0.1, 10), - ) + judge, human = _sparse_debias_pair(rng, n_total=100, n_labeled=10) + with pytest.raises(ValueError, match="at least 15 human-labeled"): + es.judge_debias_mean_ci(judge, human) -def test_judge_debias_mean_ci_compute_pvalue_opt_in(): +def test_judge_debias_mean_ci_rejects_below_50_total(): rng = _rng(35) - result = es.judge_debias_mean_ci( - unlabeled_judge_scores=rng.normal(0.5, 0.1, 50), - labeled_human_scores=rng.normal(0.5, 0.1, 20), - labeled_judge_scores=rng.normal(0.5, 0.1, 20), - compute_pvalue=True, - ) + judge, human = _sparse_debias_pair(rng, n_total=40, n_labeled=20) + with pytest.raises(ValueError, match="at least 50 items total"): + es.judge_debias_mean_ci(judge, human) + + +def test_judge_debias_mean_ci_rejects_all_items_labeled(): + rng = _rng(36) + n = 60 + human_all = np.clip(rng.normal(0.5, 0.1, n), 0, 1) + judge_all = np.clip(human_all + rng.normal(0, 0.05, n), 0, 1) + with pytest.raises(ValueError, match="no unlabeled portion"): + es.judge_debias_mean_ci(judge_all, human_all) # no NaN at all + + +def test_judge_debias_mean_ci_warns_below_30_labeled_and_100_total(): + rng = _rng(37) + judge, human = _sparse_debias_pair(rng, n_total=80, n_labeled=20) + with pytest.warns(UserWarning, match="only 20 human-labeled"): + es.judge_debias_mean_ci(judge, human) + with pytest.warns(UserWarning, match="only 80 total items"): + es.judge_debias_mean_ci(judge, human) + + +def test_judge_debias_mean_ci_always_warns_about_random_sampling(): + """Even with plenty of labels, the random-sampling assumption reminder + should always fire -- it's a modeling assumption, not a sample-size + issue, so it shouldn't be silenced just because n is large.""" + rng = _rng(38) + judge, human = _sparse_debias_pair(rng, n_total=300, n_labeled=60) + with pytest.warns(UserWarning, match="unbiased sample.*uniformly at random"): + es.judge_debias_mean_ci(judge, human) + + +def test_judge_debias_mean_ci_compute_pvalue_opt_in(): + rng = _rng(39) + judge, human = _sparse_debias_pair(rng, n_total=150, n_labeled=20) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = es.judge_debias_mean_ci(judge, human, compute_pvalue=True) assert result.p_value is not None # --------------------------------------------------------------------------- -# judge_alignment -- array-based path +# judge_alignment -- array-based path (judge_scores, human_scores), +# human_scores sparse (NaN for unlabeled) or fully dense (no NaN at all). # --------------------------------------------------------------------------- -def test_judge_alignment_array_form_basic(): +def _sparse_pair(rng, n_total, n_labeled): + """Build a (judge_scores, human_scores) pair in the sparse convention: + same length, human_scores is NaN outside the labeled subset.""" + judge = np.clip(rng.integers(1, 6, n_total).astype(float) + rng.normal(0, 0.5, n_total), 1, 5) + human = np.full(n_total, np.nan) + idx = rng.choice(n_total, n_labeled, replace=False) + human[idx] = rng.integers(1, 6, n_labeled).astype(float) + return judge, human + + +def test_judge_alignment_array_form_sparse_basic(): rng = _rng(40) + judge, human = _sparse_pair(rng, n_total=300, n_labeled=40) + result = es.judge_alignment(judge, human) + assert result.n_labeled == 40 + assert result.n_total == 300 # derived automatically from judge_scores + # Some items are unlabeled -> representativeness check runs for free. + assert "score_distribution" in result.representativeness + + +def test_judge_alignment_array_form_dense_no_nan(): + """When human_scores has no NaN at all (ambiguous: could be '100% + labeled' or 'caller already extracted just the labeled subset'), stay + conservative and skip the representativeness check rather than + silently comparing a set against itself.""" + rng = _rng(41) n = 40 human = rng.integers(1, 6, n).astype(float) judge = np.clip(human + rng.normal(0, 0.5, n), 1, 5) - result = es.judge_alignment(human, judge) + result = es.judge_alignment(judge, human) assert result.n_labeled == n - assert result.n_total == n # no all_judge_scores given - assert result.representativeness == {} # no distribution check without all_judge_scores + assert result.n_total == n + assert result.representativeness == {} -def test_judge_alignment_array_form_with_all_judge_scores(): - rng = _rng(41) +def test_judge_alignment_array_form_explicit_all_judge_scores_overrides(): + rng = _rng(42) n_lab, n_total = 35, 250 human = rng.integers(1, 6, n_lab).astype(float) judge = np.clip(human + rng.normal(0, 0.5, n_lab), 1, 5) all_judge = np.clip(rng.integers(1, 6, n_total).astype(float) + rng.normal(0, 0.3, n_total), 1, 5) - result = es.judge_alignment(human, judge, all_judge_scores=all_judge) + result = es.judge_alignment(judge, human, all_judge_scores=all_judge) assert result.n_total == n_total assert "score_distribution" in result.representativeness # No slice-column checks in the array form (no DataFrame). @@ -327,39 +377,43 @@ def test_judge_alignment_array_form_with_all_judge_scores(): def test_judge_alignment_array_form_display_names(): - rng = _rng(42) + rng = _rng(43) human = rng.integers(1, 6, 35).astype(float) judge = np.clip(human + rng.normal(0, 0.5, 35), 1, 5) - result = es.judge_alignment(human, judge, llm_metric="my_judge", human_groundtruth="my_human") + result = es.judge_alignment(judge, human, llm_metric="my_judge", human_groundtruth="my_human") assert result.llm_metric == "my_judge" assert result.human_col == "my_human" def test_judge_alignment_array_form_defaults_display_names(): - rng = _rng(43) + rng = _rng(44) human = rng.integers(1, 6, 35).astype(float) judge = np.clip(human + rng.normal(0, 0.5, 35), 1, 5) - result = es.judge_alignment(human, judge) + result = es.judge_alignment(judge, human) assert result.llm_metric == "judge" assert result.human_col == "human" -def test_judge_alignment_array_form_requires_judge_labels(): +def test_judge_alignment_array_form_requires_human_scores(): with pytest.raises(TypeError, match="requires both arrays"): es.judge_alignment(np.array([1.0, 2.0, 3.0])) def test_judge_alignment_array_form_rejects_mismatched_shapes(): - with pytest.raises(ValueError, match="paired, same-shape"): + with pytest.raises(ValueError, match="same length"): es.judge_alignment(np.array([1.0, 2.0, 3.0]), np.array([1.0, 2.0])) +def test_judge_alignment_array_form_rejects_all_nan_human_scores(): + with pytest.raises(ValueError, match="No labeled items"): + es.judge_alignment(np.array([1.0, 2.0, 3.0]), np.array([np.nan, np.nan, np.nan])) + + def test_judge_alignment_array_form_warns_below_30(): - rng = _rng(44) - human = rng.integers(1, 6, 10).astype(float) - judge = np.clip(human + rng.normal(0, 0.5, 10), 1, 5) + rng = _rng(45) + judge, human = _sparse_pair(rng, n_total=100, n_labeled=10) with pytest.warns(UserWarning, match="fewer than ~30"): - es.judge_alignment(human, judge) + es.judge_alignment(judge, human) def test_judge_alignment_evaldata_form_still_requires_kwargs(): From 11554f96b700a69fea547a4ccd4175743c23a33c Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 20:42:00 -0400 Subject: [PATCH 010/245] Add gradient CI bands to plot_ci_forest(), make it ComparisonResult.plot()'s default plot_ci_forest() drew a single CI band per entity even though the data for the same 68/90/95/99% nested gradient the terminal's .summary() already shows (robustness.multi_ci) was already being computed by compare() by default -- just never exposed to the matplotlib path. Added a style="gradient"/"single" parameter (mirroring summary()'s own style= split): gradient draws the nested bands as increasingly-opaque bars toward the mean, falling back to single-band automatically per entity when multi_ci data isn't available (e.g. a Wald-type CI). compare_to overlay stays single-band regardless, to keep it legible. Extended ComparisonResult.entity_stats (the vis-compatibility shim) to also expose multi_ci per entity, and changed ComparisonResult.plot()'s default method from "bar" (the quick, uncorrected accuracy view) to "forest" -- result.plot() now shows the gradient CI picture by default, for quick inline use in a notebook. Verified: 8 new tests in test_ci_forest_plot.py, plus the existing critical-difference/scoreboard plot tests, test_compare.py, and test_pareto.py all still passing (98 total). Rendered and visually inspected the gradient plot, single-style plot, and compare_to overlay. Also smoke-tested the LMM path (a separate CI-computation branch). Co-Authored-By: Claude Sonnet 5 --- evalstats/api.py | 20 ++++-- evalstats/vis/forest.py | 104 ++++++++++++++++++++++++++---- tests/test_ci_forest_plot.py | 119 +++++++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 18 deletions(-) create mode 100644 tests/test_ci_forest_plot.py diff --git a/evalstats/api.py b/evalstats/api.py index b854c5b..0a9f6c9 100644 --- a/evalstats/api.py +++ b/evalstats/api.py @@ -212,7 +212,7 @@ def print_pair(self, entity_a: str, entity_b: str) -> None: return pair.summary() - def plot(self, method: str = "bar", **kwargs): + def plot(self, method: str = "forest", **kwargs): """Visualize comparison results. Parameters @@ -220,10 +220,16 @@ def plot(self, method: str = "bar", **kwargs): method : str Plot type: - * ``"bar"`` (default) — accuracy bar chart via - :func:`~evalstats.vis.scoreboard.plot_accuracy_bar`. - * ``"forest"`` — horizontal CI forest plot via - :func:`~evalstats.vis.forest.plot_ci_forest`. + * ``"forest"`` (default) — horizontal CI forest plot via + :func:`~evalstats.vis.forest.plot_ci_forest`, gradient-banded + (68/90/95/99% nested confidence bands) by default -- the same + richer CI picture the terminal's ``.summary()`` gradient plot + already shows, in matplotlib. Pass ``style="single"`` to fall + back to one plain CI band per entity. + * ``"bar"`` — accuracy bar chart via + :func:`~evalstats.vis.scoreboard.plot_accuracy_bar`. A quick, + uncorrected view (no CIs) -- useful before statistical + analysis, not as a substitute for it. * ``"cd"`` — critical difference diagram via :func:`~evalstats.vis.critical_difference.plot_critical_difference`. @@ -286,6 +292,10 @@ def entity_stats(self) -> dict: ci_high=float(rob.ci_high[i]) if rob.ci_high is not None else 1.0, median=float(rob.median[i]), std=float(rob.std[i]), + multi_ci=( + {a: (float(lo[i]), float(hi[i])) for a, (lo, hi) in rob.multi_ci.items()} + if rob.multi_ci is not None else None + ), ) for i, lbl in enumerate(bundle.benchmark.template_labels) } diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 8046ae2..0a4c8ca 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -4,16 +4,30 @@ statistical tier, with the best-performing entity at the top. An optional second report can be overlaid for direct before/after comparison (e.g., to show how CI widths change when you double the eval set or add more runs). + +Two styles (mirroring the same split ``print_analysis_summary``'s +``style=`` uses for the terminal's ASCII plots): + +* ``"gradient"`` (default) -- nested CI bands at 68/90/95/99% (the same + ``multi_ci`` data the terminal's ``░▒▓█`` gradient rendering uses), + drawn as increasingly-opaque bars toward the mean, so the reader sees + the confidence *gradient* rather than a single somewhat-arbitrary cutoff. +* ``"single"`` -- one CI band per entity, at whatever confidence level the + report was computed with. Always used as the fallback when ``multi_ci`` + data isn't available (e.g. an LMM/Wald-type report with only one CI). """ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Literal, Optional import matplotlib.pyplot as plt import matplotlib.ticker as mticker import numpy as np from matplotlib.lines import Line2D +from matplotlib.patches import Patch + +from ..config import GRADIENT_CI_ALPHAS if TYPE_CHECKING: from matplotlib.axes import Axes @@ -36,6 +50,13 @@ "text_secondary":"#6B7280", # muted gray — secondary text } +# Per-band opacity for the gradient style, outermost (widest CI, 99%) to +# innermost (narrowest, 68%) -- same ordering convention as the terminal's +# _gradient_interval_line (sorted ascending by alpha = descending by CI +# width), just alpha-blended bars instead of block-character replacement. +_GRADIENT_BAND_ALPHAS = (0.22, 0.38, 0.58, 0.85) +_GRADIENT_BAND_HEIGHT = 0.5 + def plot_ci_forest( report, @@ -45,6 +66,7 @@ def plot_ci_forest( reference_line: Optional[float] = 0.5, sort_by: str = "mean", as_percent: bool = True, + style: Literal["gradient", "single"] = "gradient", figsize: Optional[tuple[float, float]] = None, title: Optional[str] = None, ax: Optional[Axes] = None, @@ -61,7 +83,9 @@ def plot_ci_forest( A second report to overlay for comparison (e.g. a smaller or single-run eval). Its CIs are drawn in a lighter colour offset above each row so both intervals are visible simultaneously. - Both reports must contain the same entity labels. + Both reports must contain the same entity labels. Always drawn in + the single-band style regardless of *style*, to keep the overlay + legible. report_label : str, optional Legend label for the primary report when *compare_to* is supplied. Defaults to ``"primary"``. @@ -79,6 +103,14 @@ def plot_ci_forest( as_percent : bool When ``True`` (default), multiply CI values by 100 and format the x-axis as percentages. Set to ``False`` for raw (0–1) scores. + style : {"gradient", "single"} + ``"gradient"`` (default) draws nested CI bands at 68/90/95/99%, + increasingly opaque toward the mean -- the same ``multi_ci`` data + the terminal's ``░▒▓█`` gradient plot uses, just rendered as + matplotlib bars. Falls back to ``"single"`` automatically per + entity when that entity has no ``multi_ci`` data (e.g. a Wald-type + CI with only one level computed). ``"single"`` always draws one CI + band at the report's own confidence level. figsize : tuple[float, float], optional Figure size. Defaults to ``(7.5, 0.45 * N + 1.8)``. title : str, optional @@ -116,6 +148,13 @@ def _ci(rep, label: str) -> tuple[float, float, float]: s = rep.entity_stats[label] return s.mean * scale, s.ci_low * scale, s.ci_high * scale + def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: + s = rep.entity_stats[label] + raw = getattr(s, "multi_ci", None) + if raw is None or len(raw) < 2: + return None + return {a: (lo * scale, hi * scale) for a, (lo, hi) in raw.items()} + # ---- validate compare_to ---------------------------------------------- if compare_to is not None: missing = set(labels) - set(compare_to.labels) @@ -157,6 +196,7 @@ def _ci(rep, label: str) -> tuple[float, float, float]: # ---- draw CIs --------------------------------------------------------- lw = 2.8 ms = 55 # scatter marker size + any_gradient_used = False for i, label in enumerate(ordered_labels): y = float(y_positions[i]) @@ -184,15 +224,40 @@ def _ci(rep, label: str) -> tuple[float, float, float]: color=_PALETTE["compare"], s=ms, zorder=3, ) - # Primary CI — full colour, offset below when compare_to given - ax.plot( - [lo5, hi5], [y - offset, y - offset], - color=color, lw=lw, - solid_capstyle="round", zorder=4, - ) + # Primary CI — full colour, offset below when compare_to given. + # Gradient style falls back to single-band per-entity when this + # entity has no multi_ci data (e.g. a Wald-type CI). + multi_ci = _multi_ci(report, label) if style == "gradient" else None + y_row = y - offset + if multi_ci is not None: + any_gradient_used = True + # Widest CI (99%, smallest alpha) drawn first/lowest zorder, + # narrowest (68%, largest alpha) drawn last/highest zorder -- + # same "inner band wins" convention as the terminal's + # _gradient_interval_line, via z-order layering instead of + # character replacement. + sorted_alphas = sorted(multi_ci.keys()) + for band_i, a in enumerate(sorted_alphas): + lo_a, hi_a = multi_ci[a] + band_alpha = _GRADIENT_BAND_ALPHAS[ + min(band_i, len(_GRADIENT_BAND_ALPHAS) - 1) + ] + ax.barh( + y_row, width=hi_a - lo_a, left=lo_a, + height=_GRADIENT_BAND_HEIGHT, + color=color, alpha=band_alpha, + edgecolor="none", zorder=4 + band_i, + ) + else: + ax.plot( + [lo5, hi5], [y_row, y_row], + color=color, lw=lw, + solid_capstyle="round", zorder=4, + ) ax.scatter( - [mean5], [y - offset], - color=color, s=ms, zorder=5, + [mean5], [y_row], + color=color, s=ms, zorder=4 + len(_GRADIENT_BAND_ALPHAS) + 1, + edgecolor="white", linewidth=0.6, ) # ---- axes styling ----------------------------------------------------- @@ -223,8 +288,12 @@ def _ci(rep, label: str) -> tuple[float, float, float]: if title is None: n_inputs = report.full_analysis.n_inputs if hasattr(report.full_analysis, "n_inputs") else "" n_str = f" | N={n_inputs} inputs" if n_inputs else "" - ci_pct = int(getattr(report, "ci", 0.95) * 100) - title = f"95% confidence intervals per {report.entity_name_singular}{n_str}" + if any_gradient_used: + ci_label = "68–99% confidence gradient" + else: + ci_pct = int(getattr(report, "ci", 0.95) * 100) + ci_label = f"{ci_pct}% confidence intervals" + title = f"{ci_label} per {report.entity_name_singular}{n_str}" ax.set_title( title, @@ -233,7 +302,6 @@ def _ci(rep, label: str) -> tuple[float, float, float]: pad=10, loc="center", ) - # ---- legend ----------------------------------------------------------- if compare_to is not None: r_label = report_label or "primary" @@ -266,5 +334,15 @@ def _ci(rep, label: str) -> tuple[float, float, float]: if own_fig: fig.tight_layout() + if any_gradient_used: + # fig-level text (not ax.transAxes) + extra bottom margin so this + # never collides with the x-axis label above it. + fig.subplots_adjust(bottom=0.22) + fig.text( + 0.5, 0.02, + "darker = narrower / higher-confidence band (68% innermost, 99% outermost)", + ha="center", va="bottom", + fontsize=7.5, color=_PALETTE["text_secondary"], + ) return fig diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py new file mode 100644 index 0000000..e2e3f09 --- /dev/null +++ b/tests/test_ci_forest_plot.py @@ -0,0 +1,119 @@ +"""Tests for evalstats.vis.forest.plot_ci_forest (gradient/single styles) +and ComparisonResult.plot()'s default. +""" + +from __future__ import annotations + +import matplotlib +matplotlib.use("Agg") + +import numpy as np +import pandas as pd +import pytest + +import evalstats as es +from evalstats.vis.forest import plot_ci_forest + + +def _rng(seed: int = 0) -> np.random.Generator: + return np.random.default_rng(seed) + + +def _make_result(n_models=3, n_items=30, seed=0): + rng = _rng(seed) + rows = [] + for i in range(n_models): + mu = 0.5 + 0.1 * i + for j in range(n_items): + rows.append({ + "model": f"m{i}", "item": f"q{j}", + "score": float(np.clip(rng.normal(mu, 0.08), 0, 1)), + }) + df = pd.DataFrame(rows) + evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) + return es.compare(evaldata, factors="model", metric="score", rng=_rng(seed + 100)) + + +def test_plot_ci_forest_gradient_is_default(): + result = _make_result() + fig = plot_ci_forest(result) + ax = fig.axes[0] + assert "confidence gradient" in ax.get_title() + plt_close(fig) + + +def test_plot_ci_forest_gradient_draws_multiple_bands_per_entity(): + result = _make_result() + fig = plot_ci_forest(result) + ax = fig.axes[0] + # 3 entities x 4 gradient bands each = 12 bar patches (plus none from + # axhspan row backgrounds, which are Rectangle patches too -- filter to + # bar-like patches by checking there are at least as many as expected). + from matplotlib.patches import Rectangle + bar_patches = [p for p in ax.patches if isinstance(p, Rectangle)] + assert len(bar_patches) >= 3 * 4 + plt_close(fig) + + +def test_plot_ci_forest_single_style_no_gradient_footnote(): + result = _make_result() + fig = plot_ci_forest(result, style="single") + ax = fig.axes[0] + assert "confidence gradient" not in ax.get_title() + assert "confidence intervals" in ax.get_title() + plt_close(fig) + + +def test_plot_ci_forest_gradient_matches_single_on_mean_and_outer_ci(): + """The gradient plot's outermost band should match the single-style + plot's CI bounds when both are drawn at compatible confidence levels + (99% outer gradient band ~ single style uses the bundle's own alpha, + so just check means match exactly and outer band contains the primary CI).""" + result = _make_result() + fig_g = plot_ci_forest(result, style="gradient") + fig_s = plot_ci_forest(result, style="single") + ax_g, ax_s = fig_g.axes[0], fig_s.axes[0] + # Scatter (mean) x-data should match between styles for the same entity order. + means_g = sorted(c.get_offsets()[0][0] for c in ax_g.collections if len(c.get_offsets())) + means_s = sorted(c.get_offsets()[0][0] for c in ax_s.collections if len(c.get_offsets())) + assert means_g == pytest.approx(means_s, abs=1e-6) + plt_close(fig_g) + plt_close(fig_s) + + +def test_plot_ci_forest_compare_to_still_works_with_gradient_primary(): + small = _make_result(n_items=10, seed=1) + big = _make_result(n_items=80, seed=2) + fig = plot_ci_forest(big, compare_to=small) + assert fig is not None + plt_close(fig) + + +def test_comparison_result_plot_defaults_to_forest_gradient(): + result = _make_result() + fig = result.plot() + ax = fig.axes[0] + assert "confidence gradient" in ax.get_title() + plt_close(fig) + + +def test_comparison_result_plot_bar_still_available(): + result = _make_result() + fig = result.plot(method="bar") + assert fig is not None + plt_close(fig) + + +def test_entity_stats_exposes_multi_ci(): + result = _make_result() + stats = result.entity_stats + for label, s in stats.items(): + assert s.multi_ci is not None + assert len(s.multi_ci) >= 2 + for alpha, (lo, hi) in s.multi_ci.items(): + assert lo <= s.mean <= hi + + +def plt_close(fig): + import matplotlib.pyplot as plt + plt.close(fig) From 38133e6cf3e4d999e774126353a0e0e39cedd210 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 20:58:07 -0400 Subject: [PATCH 011/245] Polish plot_ci_forest for standalone/paper use: options, legend, caption Per feedback on the initial gradient plot render: - Added show_mean (default True), mean_marker="line"/"dot" (default "line", now a single plain black tick -- dropped the black+white double-line rendering), and show_ci_bracket (default False) to overlay a traditional bracket-style CI on top of the gradient bands. - Legend moved outside the axes (bbox_to_anchor to the right) so it never overlaps the bars; combined into one legend with tier colours, the gradient-band swatches, and the mean/bracket markers. - Tier legend labels simplified to "Unbeaten" / "Significantly worse". - Found and fixed a real bug while rebuilding the legend: the gradient band legend swatches were paired with _GRADIENT_BAND_ALPHAS in the wrong order (68% CI labeled as the lightest/widest band and vice versa) -- now correctly ordered widest/lightest (99%) to narrowest/darkest (68%), matching the actual drawing order. - Also fixed report.full_analysis.n_inputs never actually resolving (the attribute lives on .benchmark.n_inputs, not the bundle directly) -- N was silently missing from both the title and the new caption. - Added a self-contained methods caption (N, CI method, FWER correction, alpha, gradient legend) so the figure travels with its own provenance once copied out of evalstats into a paper, slide, or post, rather than only living in the surrounding terminal report. Verified: 14 tests in test_ci_forest_plot.py (6 new), plus the existing critical-difference/scoreboard/compare/pareto plot-adjacent suites (104 total) still passing. Rendered and visually inspected every option combination (gradient/single, mean line/dot/off, bracket on/off). Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 143 +++++++++++++++++++++++++++++------ tests/test_ci_forest_plot.py | 69 +++++++++++++++++ 2 files changed, 187 insertions(+), 25 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 0a4c8ca..71669a8 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -67,6 +67,9 @@ def plot_ci_forest( sort_by: str = "mean", as_percent: bool = True, style: Literal["gradient", "single"] = "gradient", + show_mean: bool = True, + mean_marker: Literal["line", "dot"] = "line", + show_ci_bracket: bool = False, figsize: Optional[tuple[float, float]] = None, title: Optional[str] = None, ax: Optional[Axes] = None, @@ -111,6 +114,19 @@ def plot_ci_forest( entity when that entity has no ``multi_ci`` data (e.g. a Wald-type CI with only one level computed). ``"single"`` always draws one CI band at the report's own confidence level. + show_mean : bool + Draw a marker at the point estimate (default ``True``). Set + ``False`` to let the CI band(s) speak for themselves. + mean_marker : {"line", "dot"} + ``"line"`` (default) draws a short vertical tick crossing the band + at the mean -- reads clearly against any band colour or opacity. + ``"dot"`` draws the previous circle marker instead. + show_ci_bracket : bool + When ``True``, overlay a traditional bracket-style CI at the + report's own (single) confidence level on top of the gradient + bands -- for readers who want the familiar landmark in addition to + the richer gradient. Default ``False``. Ignored when *style* is + already ``"single"`` (there'd be nothing to add on top of). figsize : tuple[float, float], optional Figure size. Defaults to ``(7.5, 0.45 * N + 1.8)``. title : str, optional @@ -254,11 +270,39 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: color=color, lw=lw, solid_capstyle="round", zorder=4, ) - ax.scatter( - [mean5], [y_row], - color=color, s=ms, zorder=4 + len(_GRADIENT_BAND_ALPHAS) + 1, - edgecolor="white", linewidth=0.6, - ) + + top_zorder = 4 + len(_GRADIENT_BAND_ALPHAS) + 1 + + # Optional traditional bracket-style CI overlaid on top of the + # gradient bands, at the report's own single confidence level -- + # for readers who want that familiar landmark alongside the gradient. + if show_ci_bracket and multi_ci is not None: + ax.plot( + [lo5, hi5], [y_row, y_row], + color=_PALETTE["text"], lw=1.3, zorder=top_zorder, + solid_capstyle="butt", + ) + cap_h = _GRADIENT_BAND_HEIGHT * 0.22 + for x_cap in (lo5, hi5): + ax.plot( + [x_cap, x_cap], [y_row - cap_h, y_row + cap_h], + color=_PALETTE["text"], lw=1.3, zorder=top_zorder, + ) + top_zorder += 1 + + if show_mean: + if mean_marker == "line": + tick_h = _GRADIENT_BAND_HEIGHT * 0.7 + ax.plot( + [mean5, mean5], [y_row - tick_h, y_row + tick_h], + color="black", lw=1.5, zorder=top_zorder + 1, + ) + else: + ax.scatter( + [mean5], [y_row], + color=color, s=ms, zorder=top_zorder + 1, + edgecolor="white", linewidth=0.6, + ) # ---- axes styling ----------------------------------------------------- ax.set_yticks(y_positions) @@ -284,14 +328,23 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: ax.tick_params(axis="y", length=0, pad=8) ax.tick_params(axis="x", colors=_PALETTE["text_secondary"], labelsize=9) + # ---- gather methods metadata (for title + caption) -------------------- + bundle = getattr(report, "full_analysis", None) + n_inputs = getattr(getattr(bundle, "benchmark", None), "n_inputs", None) + alpha = getattr(report, "alpha", 0.05) + ci_pct = int(round((1 - alpha) * 100)) + ci_method = getattr(bundle, "resolved_ci_method", None) + correction = getattr(getattr(bundle, "pairwise", None), "correction_method", None) + + def _pretty(s: Optional[str]) -> Optional[str]: + return s.replace("_", " ") if s else None + # ---- title ------------------------------------------------------------ if title is None: - n_inputs = report.full_analysis.n_inputs if hasattr(report.full_analysis, "n_inputs") else "" n_str = f" | N={n_inputs} inputs" if n_inputs else "" if any_gradient_used: - ci_label = "68–99% confidence gradient" + ci_label = "68-99% confidence gradient" else: - ci_pct = int(getattr(report, "ci", 0.95) * 100) ci_label = f"{ci_pct}% confidence intervals" title = f"{ci_label} per {report.entity_name_singular}{n_str}" @@ -302,45 +355,85 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: pad=10, loc="center", ) - # ---- legend ----------------------------------------------------------- + + # ---- legend ------------------------------------------------------------- + # One combined legend: entity-tier colours, plus (in gradient mode) a + # neutral-colour swatch per confidence band -- so a reader encountering + # this figure with no surrounding context (pasted into a paper, a slide, + # a social post) can still read it unaided. + legend_handles: list = [] if compare_to is not None: r_label = report_label or "primary" c_label = compare_label or "comparison" - legend_handles = [ + legend_handles += [ Line2D([0], [0], color=_PALETTE["compare"], lw=lw, solid_capstyle="round", label=c_label), Line2D([0], [0], color=_PALETTE["unbeaten"], lw=lw, solid_capstyle="round", label=r_label), ] - ax.legend( - handles=legend_handles, - fontsize=8, loc="lower right", - frameon=True, facecolor="white", - edgecolor=_PALETTE["grid"], framealpha=0.95, - ) elif unbeaten: - legend_handles = [ + legend_handles += [ Line2D([0], [0], color=_PALETTE["unbeaten"], lw=lw, - solid_capstyle="round", label="In contention"), + solid_capstyle="round", label="Unbeaten"), Line2D([0], [0], color=_PALETTE["lower_tier"], lw=lw, - solid_capstyle="round", label="Outperformed"), + solid_capstyle="round", label="Significantly worse"), ] + if any_gradient_used: + neutral = _PALETTE["text_secondary"] + # Same drawing order as the bands themselves: widest/lightest (99%) + # first, narrowest/darkest (68%) last. + band_labels = ["99% CI", "95% CI", "90% CI", "68% CI"] + legend_handles += [ + Patch(facecolor=neutral, alpha=a, edgecolor="none", label=lbl) + for a, lbl in zip(_GRADIENT_BAND_ALPHAS, band_labels) + ] + if show_mean and mean_marker == "line": + legend_handles.append( + Line2D([0], [0], color="black", lw=1.5, label="mean") + ) + if show_ci_bracket and any_gradient_used: + legend_handles.append( + Line2D([0], [0], color=_PALETTE["text"], lw=1.3, label=f"{ci_pct}% CI (bracket)") + ) + + if legend_handles: ax.legend( handles=legend_handles, - fontsize=8, loc="lower right", + fontsize=7.5, loc="center left", bbox_to_anchor=(1.01, 0.5), frameon=True, facecolor="white", edgecolor=_PALETTE["grid"], framealpha=0.95, + ncol=1, ) if own_fig: fig.tight_layout() + if legend_handles: + # Legend now sits outside the axes (bbox_to_anchor to the + # right) so it never overlaps the bars -- reserve room for it. + fig.subplots_adjust(right=0.76) + + # Self-contained methods caption -- this figure is meant to stand on + # its own once copied out of evalstats (into a paper, a slide, a + # post), so the N/CI method/correction it was computed with travels + # with it rather than only living in the surrounding terminal report. + caption_parts = [] + if n_inputs: + caption_parts.append(f"N={n_inputs} items") + pretty_ci_method = _pretty(ci_method) + if pretty_ci_method: + caption_parts.append(f"CI method: {pretty_ci_method}") + pretty_correction = _pretty(correction) + if pretty_correction and pretty_correction != "none": + caption_parts.append(f"FWER correction: {pretty_correction}") + caption_parts.append(f"α={alpha:g}") if any_gradient_used: - # fig-level text (not ax.transAxes) + extra bottom margin so this - # never collides with the x-axis label above it. - fig.subplots_adjust(bottom=0.22) + caption_parts.append("darker band = higher confidence") + caption = " | ".join(caption_parts) + + if caption: + fig.subplots_adjust(bottom=0.24) fig.text( - 0.5, 0.02, - "darker = narrower / higher-confidence band (68% innermost, 99% outermost)", + 0.5, 0.02, caption, ha="center", va="bottom", fontsize=7.5, color=_PALETTE["text_secondary"], ) diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py index e2e3f09..af9235e 100644 --- a/tests/test_ci_forest_plot.py +++ b/tests/test_ci_forest_plot.py @@ -114,6 +114,75 @@ def test_entity_stats_exposes_multi_ci(): assert lo <= s.mean <= hi +def test_plot_ci_forest_mean_line_is_default(): + result = _make_result() + fig = plot_ci_forest(result, reference_line=None) + ax = fig.axes[0] + # One simple black tick line per entity. + mean_lines = [ + l for l in ax.get_lines() + if l.get_xdata()[0] == l.get_xdata()[1] and len(l.get_xdata()) == 2 + ] + assert len(mean_lines) == 3 + assert all(l.get_color() == "black" for l in mean_lines) + plt_close(fig) + + +def test_plot_ci_forest_show_mean_false_omits_marker(): + result = _make_result() + # reference_line=None to isolate mean-marker lines from the (also + # vertical) reference line. + fig = plot_ci_forest(result, show_mean=False, reference_line=None) + ax = fig.axes[0] + vertical_lines = [ + l for l in ax.get_lines() + if len(l.get_xdata()) == 2 and l.get_xdata()[0] == l.get_xdata()[1] + ] + assert len(vertical_lines) == 0 + assert len(ax.collections) == 0 # no scatter dots either + plt_close(fig) + + +def test_plot_ci_forest_mean_marker_dot_uses_scatter(): + result = _make_result() + fig = plot_ci_forest(result, mean_marker="dot") + ax = fig.axes[0] + assert len(ax.collections) == 3 # one scatter point per entity + plt_close(fig) + + +def test_plot_ci_forest_show_ci_bracket_adds_overlay(): + result = _make_result() + fig_without = plot_ci_forest(result, show_ci_bracket=False) + fig_with = plot_ci_forest(result, show_ci_bracket=True) + n_lines_without = len(fig_without.axes[0].get_lines()) + n_lines_with = len(fig_with.axes[0].get_lines()) + assert n_lines_with > n_lines_without + plt_close(fig_without) + plt_close(fig_with) + + +def test_plot_ci_forest_caption_includes_n_and_method(): + result = _make_result(n_items=30) + fig = plot_ci_forest(result) + texts = [t.get_text() for t in fig.texts] + caption = " ".join(texts) + assert "N=30 items" in caption + assert "CI method:" in caption + assert "α=0.05" in caption + plt_close(fig) + + +def test_plot_ci_forest_legend_includes_band_and_mean_labels(): + result = _make_result() + fig = plot_ci_forest(result) + ax = fig.axes[0] + legend_labels = [t.get_text() for t in ax.get_legend().get_texts()] + assert "99% CI" in legend_labels + assert "68% CI" in legend_labels + assert "mean" in legend_labels + + def plt_close(fig): import matplotlib.pyplot as plt plt.close(fig) From 8b2c1173f67cd9daf6d68020295e984841695a60 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:00:19 -0400 Subject: [PATCH 012/245] Move the methods subtitle above the plot, not below it For LaTeX embedding: a small caption-like line below the plot would read as redundant once the figure gets its own \caption{} underneath. Moved the CI-method/FWER-correction/alpha text to sit between the title and the axes instead (N stays in the title itself, so nothing's lost) -- now reads as a subtitle that's part of the figure, not a second caption. Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 59 ++++++++++++++++++------------------ tests/test_ci_forest_plot.py | 19 ++++++++---- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 71669a8..9ea1d6b 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -339,7 +339,7 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: def _pretty(s: Optional[str]) -> Optional[str]: return s.replace("_", " ") if s else None - # ---- title ------------------------------------------------------------ + # ---- title + methods subtitle ------------------------------------------ if title is None: n_str = f" | N={n_inputs} inputs" if n_inputs else "" if any_gradient_used: @@ -348,13 +348,38 @@ def _pretty(s: Optional[str]) -> Optional[str]: ci_label = f"{ci_pct}% confidence intervals" title = f"{ci_label} per {report.entity_name_singular}{n_str}" + # Self-contained methods subtitle -- this figure is meant to stand on + # its own once copied out of evalstats (into a paper, a slide, a post), + # so the CI method/correction it was computed with travels with it + # rather than only living in the surrounding terminal report. Placed + # between the title and the axes (not below the plot) so it doesn't + # read as a second, redundant caption once a LaTeX \caption{} is added + # underneath the whole figure. + caption_parts = [] + pretty_ci_method = _pretty(ci_method) + if pretty_ci_method: + caption_parts.append(f"CI method: {pretty_ci_method}") + pretty_correction = _pretty(correction) + if pretty_correction and pretty_correction != "none": + caption_parts.append(f"FWER correction: {pretty_correction}") + caption_parts.append(f"α={alpha:g}") + if any_gradient_used: + caption_parts.append("darker band = higher confidence") + caption = " | ".join(caption_parts) + ax.set_title( title, fontsize=10, color=_PALETTE["text"], - pad=10, + pad=24 if caption else 10, loc="center", ) + if caption: + ax.text( + 0.5, 1.02, caption, + transform=ax.transAxes, ha="center", va="bottom", + fontsize=7.5, color=_PALETTE["text_secondary"], + ) # ---- legend ------------------------------------------------------------- # One combined legend: entity-tier colours, plus (in gradient mode) a @@ -408,34 +433,8 @@ def _pretty(s: Optional[str]) -> Optional[str]: if own_fig: fig.tight_layout() if legend_handles: - # Legend now sits outside the axes (bbox_to_anchor to the - # right) so it never overlaps the bars -- reserve room for it. + # Legend sits outside the axes (bbox_to_anchor to the right) so + # it never overlaps the bars -- reserve room for it. fig.subplots_adjust(right=0.76) - # Self-contained methods caption -- this figure is meant to stand on - # its own once copied out of evalstats (into a paper, a slide, a - # post), so the N/CI method/correction it was computed with travels - # with it rather than only living in the surrounding terminal report. - caption_parts = [] - if n_inputs: - caption_parts.append(f"N={n_inputs} items") - pretty_ci_method = _pretty(ci_method) - if pretty_ci_method: - caption_parts.append(f"CI method: {pretty_ci_method}") - pretty_correction = _pretty(correction) - if pretty_correction and pretty_correction != "none": - caption_parts.append(f"FWER correction: {pretty_correction}") - caption_parts.append(f"α={alpha:g}") - if any_gradient_used: - caption_parts.append("darker band = higher confidence") - caption = " | ".join(caption_parts) - - if caption: - fig.subplots_adjust(bottom=0.24) - fig.text( - 0.5, 0.02, caption, - ha="center", va="bottom", - fontsize=7.5, color=_PALETTE["text_secondary"], - ) - return fig diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py index af9235e..b3ff3ec 100644 --- a/tests/test_ci_forest_plot.py +++ b/tests/test_ci_forest_plot.py @@ -162,14 +162,21 @@ def test_plot_ci_forest_show_ci_bracket_adds_overlay(): plt_close(fig_with) -def test_plot_ci_forest_caption_includes_n_and_method(): +def test_plot_ci_forest_title_includes_n_and_subtitle_includes_method(): + """N lives in the title; CI method/correction/alpha live in a small + subtitle between the title and the axes (not below the plot), so a + LaTeX \\caption{} added under the whole figure doesn't read as + redundant with a second caption-like line at the bottom.""" result = _make_result(n_items=30) fig = plot_ci_forest(result) - texts = [t.get_text() for t in fig.texts] - caption = " ".join(texts) - assert "N=30 items" in caption - assert "CI method:" in caption - assert "α=0.05" in caption + ax = fig.axes[0] + assert "N=30 inputs" in ax.get_title() + ax_texts = [t.get_text() for t in ax.texts] + subtitle = " ".join(ax_texts) + assert "CI method:" in subtitle + assert "α=0.05" in subtitle + # Nothing placed below the axes as a second, bottom-of-figure caption. + assert len(fig.texts) == 0 plt_close(fig) From 314cfd742da3814fa15537e176f829902489c5d5 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:02:20 -0400 Subject: [PATCH 013/245] Rebalance subtitle spacing: further from the plot, closer to the title Switched the subtitle from axes-fraction to a points-based offset (consistent gap regardless of axes height) and increased that offset, which simultaneously pulls it away from the plot and closer to the title above it -- was hugging the axes top edge before. Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 9ea1d6b..750d211 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -371,13 +371,19 @@ def _pretty(s: Optional[str]) -> Optional[str]: title, fontsize=10, color=_PALETTE["text"], - pad=24 if caption else 10, + pad=28 if caption else 10, loc="center", ) if caption: - ax.text( - 0.5, 1.02, caption, - transform=ax.transAxes, ha="center", va="bottom", + # Points-based offset (not axes-fraction) so the gap from the plot + # is consistent regardless of axes height, and sits roughly midway + # between the title and the top of the axes rather than hugging + # either one. + ax.annotate( + caption, + xy=(0.5, 1.0), xycoords="axes fraction", + xytext=(0, 13), textcoords="offset points", + ha="center", va="bottom", fontsize=7.5, color=_PALETTE["text_secondary"], ) From 52620c06056bcecdd2f627bdecf3ed7b4158a3f9 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:04:47 -0400 Subject: [PATCH 014/245] Revert "Rebalance subtitle spacing: further from the plot, closer to the title" This reverts commit 314cfd742da3814fa15537e176f829902489c5d5. --- evalstats/vis/forest.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 750d211..9ea1d6b 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -371,19 +371,13 @@ def _pretty(s: Optional[str]) -> Optional[str]: title, fontsize=10, color=_PALETTE["text"], - pad=28 if caption else 10, + pad=24 if caption else 10, loc="center", ) if caption: - # Points-based offset (not axes-fraction) so the gap from the plot - # is consistent regardless of axes height, and sits roughly midway - # between the title and the top of the axes rather than hugging - # either one. - ax.annotate( - caption, - xy=(0.5, 1.0), xycoords="axes fraction", - xytext=(0, 13), textcoords="offset points", - ha="center", va="bottom", + ax.text( + 0.5, 1.02, caption, + transform=ax.transAxes, ha="center", va="bottom", fontsize=7.5, color=_PALETTE["text_secondary"], ) From 98e2ff04e4f964231ae0daaca1a80e551e135456 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:13:34 -0400 Subject: [PATCH 015/245] Add color_rule param to plot_ci_forest; trim excess whitespace around legend color_rule (default "tier", unchanged behavior) also accepts "factor" (each entity gets its own distinct color from a qualitative palette, stable across sort_by) or any matplotlib color spec (all bars use that one color). The tier-meaning legend only shows for color_rule="tier", since it's the only mode where color carries information beyond entity identity (which the y-axis labels already convey). Also fixed the whitespace complaint from the legend-outside-axes change: was reserving a fixed, overly generous right margin regardless of actual legend width, leaving blank canvas a reader would need to crop before using the figure in a paper. Now measures the legend's actual rendered extent after an initial draw and trims the figure's width to hug it (rescaling the axes' subplot fractions to preserve their exact pixel position/size on the narrower canvas) -- only applies to the standalone (own_fig) case; an externally-supplied ax= is left to the caller's own layout, as before. Verified: 19 tests in test_ci_forest_plot.py (5 new for color_rule), plus the broader plot-adjacent suite (109 total) still passing. Visually inspected all three color_rule modes, the whitespace trim on both gradient and single styles, the compare_to overlay, and confirmed the embedded ax= case is unaffected (as intended). Co-Authored-By: Claude Sonnet 5 --- evalstats/api.py | 4 +- evalstats/vis/forest.py | 84 +++++++++++++++++++++++++++++++----- tests/test_ci_forest_plot.py | 70 ++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 12 deletions(-) diff --git a/evalstats/api.py b/evalstats/api.py index 0a9f6c9..3aa53e6 100644 --- a/evalstats/api.py +++ b/evalstats/api.py @@ -225,7 +225,9 @@ def plot(self, method: str = "forest", **kwargs): (68/90/95/99% nested confidence bands) by default -- the same richer CI picture the terminal's ``.summary()`` gradient plot already shows, in matplotlib. Pass ``style="single"`` to fall - back to one plain CI band per entity. + back to one plain CI band per entity, or ``color_rule="factor"`` + / a colour name to color by entity identity instead of + significance tier. * ``"bar"`` — accuracy bar chart via :func:`~evalstats.vis.scoreboard.plot_accuracy_bar`. A quick, uncorrected view (no CIs) -- useful before statistical diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 9ea1d6b..00bb526 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Literal, Optional +import matplotlib.colors as mcolors import matplotlib.pyplot as plt import matplotlib.ticker as mticker import numpy as np @@ -67,6 +68,7 @@ def plot_ci_forest( sort_by: str = "mean", as_percent: bool = True, style: Literal["gradient", "single"] = "gradient", + color_rule: str = "tier", show_mean: bool = True, mean_marker: Literal["line", "dot"] = "line", show_ci_bracket: bool = False, @@ -114,6 +116,22 @@ def plot_ci_forest( entity when that entity has no ``multi_ci`` data (e.g. a Wald-type CI with only one level computed). ``"single"`` always draws one CI band at the report's own confidence level. + color_rule : str + How bars are coloured: + + * ``"tier"`` (default) -- by significance tier: "Unbeaten" vs. + "Significantly worse" (or a single neutral colour when nothing + is significantly different). This is the only mode with a + colour-meaning legend, since it's the only one where colour + carries information beyond "which entity is this" (the y-axis + labels already say that). + * ``"factor"`` -- each entity gets its own distinct colour from a + qualitative palette (cycling past 10 entities). Useful when + entities are a categorical factor in their own right (e.g. + different models) and you want colour to track identity rather + than significance. + * any matplotlib colour spec (e.g. ``"#4a90d9"``, ``"steelblue"``) + -- every bar uses that one colour. show_mean : bool Draw a marker at the point estimate (default ``True``). Set ``False`` to let the CI band(s) speak for themselves. @@ -158,6 +176,28 @@ def plot_ci_forest( ordered_labels = [labels[i] for i in order] unbeaten = set(report.unbeaten) if report.unbeaten else set() + # ---- colour rule -------------------------------------------------------- + if color_rule not in ("tier", "factor") and not mcolors.is_color_like(color_rule): + raise ValueError( + f"color_rule={color_rule!r} is not 'tier', 'factor', or a " + "valid matplotlib colour spec (e.g. '#4a90d9', 'steelblue')." + ) + factor_colors: dict = {} + if color_rule == "factor": + palette = plt.get_cmap("tab10").colors + # Keyed by original label order (not sort-dependent ordered_labels) + # so an entity's colour stays stable across different sort_by calls. + factor_colors = {lbl: palette[i % len(palette)] for i, lbl in enumerate(labels)} + + def _entity_color(label: str) -> str: + if color_rule == "tier": + if not unbeaten: + return _PALETTE["no_sig"] + return _PALETTE["unbeaten"] if label in unbeaten else _PALETTE["lower_tier"] + if color_rule == "factor": + return factor_colors[label] + return color_rule # a literal colour spec, same for every entity + scale = 100.0 if as_percent else 1.0 def _ci(rep, label: str) -> tuple[float, float, float]: @@ -218,14 +258,7 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: y = float(y_positions[i]) mean5, lo5, hi5 = _ci(report, label) - # Primary CI tier colour - if not unbeaten: - # No significant differences — use neutral colour - color = _PALETTE["no_sig"] - elif label in unbeaten: - color = _PALETTE["unbeaten"] - else: - color = _PALETTE["lower_tier"] + color = _entity_color(label) if compare_to is not None: # Comparison report — lighter, offset above @@ -396,7 +429,10 @@ def _pretty(s: Optional[str]) -> Optional[str]: Line2D([0], [0], color=_PALETTE["unbeaten"], lw=lw, solid_capstyle="round", label=r_label), ] - elif unbeaten: + elif color_rule == "tier" and unbeaten: + # Only "tier" mode has a colour-meaning legend -- "factor" and a + # literal colour spec both make colour track entity identity (or + # nothing at all), which the y-axis labels already convey. legend_handles += [ Line2D([0], [0], color=_PALETTE["unbeaten"], lw=lw, solid_capstyle="round", label="Unbeaten"), @@ -434,7 +470,33 @@ def _pretty(s: Optional[str]) -> Optional[str]: fig.tight_layout() if legend_handles: # Legend sits outside the axes (bbox_to_anchor to the right) so - # it never overlaps the bars -- reserve room for it. - fig.subplots_adjust(right=0.76) + # it never overlaps the bars -- reserve room for it with a + # generous initial guess, then trim the canvas to hug the + # legend's actual rendered width instead of leaving whatever + # blank margin that guess didn't use (matters for pasting + # straight into a paper without manual cropping). + fig.subplots_adjust(right=0.78) + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + legend = ax.get_legend() + legend_px = legend.get_window_extent(renderer=renderer) + fig_px_width = fig.get_window_extent(renderer=renderer).width + pad_px = 8 + excess_px = fig_px_width - (legend_px.x1 + pad_px) + if excess_px > 1: + dpi = fig.dpi + old_width_in, height_in = fig.get_size_inches() + new_width_in = old_width_in - excess_px / dpi + if new_width_in > 0: + # Rescale horizontal subplot fractions so the axes and + # legend keep their exact pixel position/size on the + # narrower canvas -- only the wasted margin is trimmed. + sp = fig.subplotpars + scale = old_width_in / new_width_in + fig.set_size_inches(new_width_in, height_in) + fig.subplots_adjust( + left=min(0.99, sp.left * scale), + right=min(1.0, sp.right * scale), + ) return fig diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py index b3ff3ec..860abfa 100644 --- a/tests/test_ci_forest_plot.py +++ b/tests/test_ci_forest_plot.py @@ -190,6 +190,76 @@ def test_plot_ci_forest_legend_includes_band_and_mean_labels(): assert "mean" in legend_labels +# --------------------------------------------------------------------------- +# color_rule +# --------------------------------------------------------------------------- + +def _bar_colors(ax): + from matplotlib.patches import Rectangle + # zorder >= 4 excludes the alternating row-background rectangles + # (axhspan, zorder=0), which aren't gradient-band bars. + return [ + p.get_facecolor() for p in ax.patches + if isinstance(p, Rectangle) and p.get_zorder() >= 4 + ] + + +def test_color_rule_tier_is_default_and_has_legend(): + result = _make_result() + fig = plot_ci_forest(result) + ax = fig.axes[0] + legend_labels = [t.get_text() for t in ax.get_legend().get_texts()] + assert "Unbeaten" in legend_labels + assert "Significantly worse" in legend_labels + plt_close(fig) + + +def test_color_rule_factor_gives_each_entity_a_distinct_color(): + result = _make_result(n_models=3) + fig = plot_ci_forest(result, color_rule="factor") + ax = fig.axes[0] + colors = _bar_colors(ax) + # 3 entities x 4 gradient bands each; each entity's 4 bands share one + # base color (varying only alpha), and different entities differ. + distinct_rgb = {c[:3] for c in colors} + assert len(distinct_rgb) == 3 + legend_labels = [t.get_text() for t in ax.get_legend().get_texts()] + assert "Unbeaten" not in legend_labels + plt_close(fig) + + +def test_color_rule_factor_is_stable_across_sort_order(): + result = _make_result(n_models=3) + fig_mean = plot_ci_forest(result, color_rule="factor", sort_by="mean") + fig_label = plot_ci_forest(result, color_rule="factor", sort_by="label") + # Same set of colors used regardless of row order. + colors_mean = {c[:3] for c in _bar_colors(fig_mean.axes[0])} + colors_label = {c[:3] for c in _bar_colors(fig_label.axes[0])} + assert colors_mean == colors_label + plt_close(fig_mean) + plt_close(fig_label) + + +def test_color_rule_literal_color_used_for_all_entities(): + result = _make_result(n_models=3) + fig = plot_ci_forest(result, color_rule="seagreen") + ax = fig.axes[0] + colors = _bar_colors(ax) + distinct_rgb = {c[:3] for c in colors} + assert len(distinct_rgb) == 1 + import matplotlib.colors as mcolors + assert distinct_rgb.pop() == mcolors.to_rgb("seagreen") + legend_labels = [t.get_text() for t in ax.get_legend().get_texts()] + assert "Unbeaten" not in legend_labels + plt_close(fig) + + +def test_color_rule_invalid_raises_clear_error(): + result = _make_result() + with pytest.raises(ValueError, match="not 'tier', 'factor'"): + plot_ci_forest(result, color_rule="not_a_real_color") + + def plt_close(fig): import matplotlib.pyplot as plt plt.close(fig) From cb93115242fd025779a8e326e904e2ed80786fc7 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:26:35 -0400 Subject: [PATCH 016/245] Make compare_to render gradient bands too, muted to the row's own hue compare_to previously always drew a single flat CI line regardless of style="gradient", and always used a fixed light-blue color unrelated to the row's actual tier/factor color -- so a red "significantly worse" row would show a light-blue comparison band beneath it, looking like an unrelated series rather than "the same entity, a second eval." Refactored the primary/comparison drawing into a shared _draw_ci_row() helper: compare_to now renders nested gradient bands (falling back to a single muted line when it has no multi_ci data) using the exact same color as its row, just at reduced alpha. Widened the per-row vertical offset and slightly shrunk the gradient band height when compare_to is active in gradient mode, so the two stacked bands don't heavily overlap. Updated the legend to a neutral full-vs-muted-alpha swatch pair (color now varies per row, so a single fixed comparison-color swatch no longer made sense), and removed the now-dead PALETTE["compare"] constant. Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 151 +++++++++++++++++++++-------------- tests/test_ci_forest_plot.py | 30 +++++++ 2 files changed, 120 insertions(+), 61 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 00bb526..7846a48 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -43,7 +43,6 @@ "unbeaten": "#4a90d9", # medium blue — in-contention CIs "lower_tier": "#e07b7b", # muted red — lower-tier CIs "no_sig": "#8a9bb5", # gray-blue — no significant differences - "compare": "#c0d8f0", # light blue — background / comparison report "ref_line": "#cccccc", # light gray — reference line "grid": "#EEF1F4", # very light — x grid "row_alt": "#FAFBFC", # off-white — alternating rows @@ -86,11 +85,13 @@ def plot_ci_forest( :func:`evalstats.compare_models`. compare_to : CompareReport, optional A second report to overlay for comparison (e.g. a smaller or - single-run eval). Its CIs are drawn in a lighter colour offset - above each row so both intervals are visible simultaneously. - Both reports must contain the same entity labels. Always drawn in - the single-band style regardless of *style*, to keep the overlay - legible. + single-run eval). Drawn offset above each row, using the *same* + colour as that row (muted, via lower alpha) rather than a fixed + unrelated colour -- so the two bands read as "same entity, two + evals". Renders as gradient bands too when *style* is + ``"gradient"`` and *compare_to* has ``multi_ci`` data, for the same + consistency reason. Both reports must contain the same entity + labels. report_label : str, optional Legend label for the primary report when *compare_to* is supplied. Defaults to ``"primary"``. @@ -232,7 +233,20 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: ax.set_facecolor("white") y_positions = np.arange(n) - offset = 0.18 if compare_to is not None else 0.0 + # More vertical room per row when stacking two bands (primary + + # comparison) so thick gradient bars don't heavily overlap; a plain + # single line needs much less. + has_gradient_rows = style == "gradient" + offset = (0.27 if has_gradient_rows else 0.18) if compare_to is not None else 0.0 + row_band_height = ( + _GRADIENT_BAND_HEIGHT * 0.85 if (compare_to is not None and has_gradient_rows) + else _GRADIENT_BAND_HEIGHT + ) + # Comparison bands/lines use the SAME hue as their row (muted via lower + # alpha), not a fixed unrelated colour, so the two bands read as "same + # entity, two evals" rather than looking like an unrelated series. + _MUTE = 0.55 + muted_band_alphas = tuple(a * _MUTE for a in _GRADIENT_BAND_ALPHAS) # ---- alternating row backgrounds -------------------------------------- for i in range(n): @@ -254,6 +268,56 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: ms = 55 # scatter marker size any_gradient_used = False + def _draw_ci_row( + y_row: float, lo: float, hi: float, mean_val: float, + multi_ci_row: Optional[dict], row_color, band_alphas: tuple, + band_height: float, zorder_base: int, draw_mean: bool, mean_alpha: float, + ) -> tuple[bool, int]: + """Draw one CI row (gradient bands, falling back to a single line + when multi_ci_row is None) plus an optional mean marker. Returns + (used_gradient, next_free_zorder).""" + used_gradient = False + if multi_ci_row is not None: + used_gradient = True + # Widest CI (99%, smallest alpha) drawn first/lowest zorder, + # narrowest (68%, largest alpha) drawn last/highest zorder -- + # same "inner band wins" convention as the terminal's + # _gradient_interval_line, via z-order layering instead of + # character replacement. + sorted_alphas = sorted(multi_ci_row.keys()) + for band_i, a in enumerate(sorted_alphas): + lo_a, hi_a = multi_ci_row[a] + band_alpha = band_alphas[min(band_i, len(band_alphas) - 1)] + ax.barh( + y_row, width=hi_a - lo_a, left=lo_a, + height=band_height, + color=row_color, alpha=band_alpha, + edgecolor="none", zorder=zorder_base + band_i, + ) + next_z = zorder_base + len(band_alphas) + else: + ax.plot( + [lo, hi], [y_row, y_row], + color=row_color, lw=lw, alpha=(mean_alpha if mean_alpha < 1 else 1.0), + solid_capstyle="round", zorder=zorder_base, + ) + next_z = zorder_base + 1 + if draw_mean: + if mean_marker == "line": + tick_h = band_height * 0.7 + ax.plot( + [mean_val, mean_val], [y_row - tick_h, y_row + tick_h], + color="black", lw=1.5, alpha=mean_alpha, zorder=next_z + 1, + ) + else: + ax.scatter( + [mean_val], [y_row], + color=row_color, s=ms, alpha=mean_alpha, zorder=next_z + 1, + edgecolor="white", linewidth=0.6, + ) + next_z += 1 + return used_gradient, next_z + for i, label in enumerate(ordered_labels): y = float(y_positions[i]) mean5, lo5, hi5 = _ci(report, label) @@ -261,50 +325,27 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: color = _entity_color(label) if compare_to is not None: - # Comparison report — lighter, offset above + # Comparison report -- same hue as the primary row, muted, so + # the two bands read as "same entity, two evals" rather than an + # unrelated fixed colour. mean0, lo0, hi0 = _ci(compare_to, label) - ax.plot( - [lo0, hi0], [y + offset, y + offset], - color=_PALETTE["compare"], lw=lw, - solid_capstyle="round", zorder=2, - ) - ax.scatter( - [mean0], [y + offset], - color=_PALETTE["compare"], s=ms, zorder=3, + multi_ci_cmp = _multi_ci(compare_to, label) if has_gradient_rows else None + used_grad_cmp, _ = _draw_ci_row( + y + offset, lo0, hi0, mean0, multi_ci_cmp, color, + muted_band_alphas, row_band_height, 2, show_mean, 0.55, ) + any_gradient_used = any_gradient_used or used_grad_cmp # Primary CI — full colour, offset below when compare_to given. # Gradient style falls back to single-band per-entity when this # entity has no multi_ci data (e.g. a Wald-type CI). multi_ci = _multi_ci(report, label) if style == "gradient" else None y_row = y - offset - if multi_ci is not None: - any_gradient_used = True - # Widest CI (99%, smallest alpha) drawn first/lowest zorder, - # narrowest (68%, largest alpha) drawn last/highest zorder -- - # same "inner band wins" convention as the terminal's - # _gradient_interval_line, via z-order layering instead of - # character replacement. - sorted_alphas = sorted(multi_ci.keys()) - for band_i, a in enumerate(sorted_alphas): - lo_a, hi_a = multi_ci[a] - band_alpha = _GRADIENT_BAND_ALPHAS[ - min(band_i, len(_GRADIENT_BAND_ALPHAS) - 1) - ] - ax.barh( - y_row, width=hi_a - lo_a, left=lo_a, - height=_GRADIENT_BAND_HEIGHT, - color=color, alpha=band_alpha, - edgecolor="none", zorder=4 + band_i, - ) - else: - ax.plot( - [lo5, hi5], [y_row, y_row], - color=color, lw=lw, - solid_capstyle="round", zorder=4, - ) - - top_zorder = 4 + len(_GRADIENT_BAND_ALPHAS) + 1 + used_grad, top_zorder = _draw_ci_row( + y_row, lo5, hi5, mean5, multi_ci, color, + _GRADIENT_BAND_ALPHAS, row_band_height, 4, show_mean, 1.0, + ) + any_gradient_used = any_gradient_used or used_grad # Optional traditional bracket-style CI overlaid on top of the # gradient bands, at the report's own single confidence level -- @@ -315,27 +356,12 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: color=_PALETTE["text"], lw=1.3, zorder=top_zorder, solid_capstyle="butt", ) - cap_h = _GRADIENT_BAND_HEIGHT * 0.22 + cap_h = row_band_height * 0.22 for x_cap in (lo5, hi5): ax.plot( [x_cap, x_cap], [y_row - cap_h, y_row + cap_h], color=_PALETTE["text"], lw=1.3, zorder=top_zorder, ) - top_zorder += 1 - - if show_mean: - if mean_marker == "line": - tick_h = _GRADIENT_BAND_HEIGHT * 0.7 - ax.plot( - [mean5, mean5], [y_row - tick_h, y_row + tick_h], - color="black", lw=1.5, zorder=top_zorder + 1, - ) - else: - ax.scatter( - [mean5], [y_row], - color=color, s=ms, zorder=top_zorder + 1, - edgecolor="white", linewidth=0.6, - ) # ---- axes styling ----------------------------------------------------- ax.set_yticks(y_positions) @@ -421,12 +447,15 @@ def _pretty(s: Optional[str]) -> Optional[str]: # a social post) can still read it unaided. legend_handles: list = [] if compare_to is not None: + # Primary vs. comparison is now conveyed by opacity alone (each row + # keeps its own hue for both) -- a neutral swatch at full vs. muted + # alpha represents that distinction regardless of color_rule. r_label = report_label or "primary" c_label = compare_label or "comparison" legend_handles += [ - Line2D([0], [0], color=_PALETTE["compare"], lw=lw, + Line2D([0], [0], color=_PALETTE["text_secondary"], lw=lw, alpha=_MUTE, solid_capstyle="round", label=c_label), - Line2D([0], [0], color=_PALETTE["unbeaten"], lw=lw, + Line2D([0], [0], color=_PALETTE["text_secondary"], lw=lw, solid_capstyle="round", label=r_label), ] elif color_rule == "tier" and unbeaten: diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py index 860abfa..30f74ac 100644 --- a/tests/test_ci_forest_plot.py +++ b/tests/test_ci_forest_plot.py @@ -89,6 +89,36 @@ def test_plot_ci_forest_compare_to_still_works_with_gradient_primary(): plt_close(fig) +def test_plot_ci_forest_compare_to_uses_gradient_bands_and_same_hue(): + from matplotlib.patches import Rectangle + + small = _make_result(n_items=10, seed=1) + big = _make_result(n_items=80, seed=2) + # color_rule="factor" guarantees each entity gets a distinct hue -- + # "tier" mode legitimately lets multiple entities share a hue (e.g. two + # entities both "significantly worse"), which isn't what this test is + # checking for. + fig = plot_ci_forest(big, compare_to=small, color_rule="factor") + ax = fig.axes[0] + bars = [p for p in ax.patches if isinstance(p, Rectangle) and p.get_zorder() >= 2] + # 3 entities x (4 primary bands + 4 comparison bands) = 24. + assert len(bars) == 3 * 8 + + # Group by rounded (row, base RGB) -- primary and comparison bars for + # the same entity should share hue, differing only in alpha (muted). + by_hue = {} + for p in bars: + r, g, b, a = p.get_facecolor() + by_hue.setdefault((round(r, 2), round(g, 2), round(b, 2)), []).append(round(a, 3)) + # Exactly 3 distinct hues (one per entity), each hue used by both the + # primary (full-scale alphas) and comparison (muted alphas) bands. + assert len(by_hue) == 3 + for hue, alphas in by_hue.items(): + assert len(alphas) == 8 # 4 primary + 4 muted-comparison alphas + assert len(set(alphas)) == 8 # all 8 alphas distinct (no accidental overlap) + plt_close(fig) + + def test_comparison_result_plot_defaults_to_forest_gradient(): result = _make_result() fig = result.plot() From 74608c73a87d54da64a5ff8109960eabda478c46 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:31:53 -0400 Subject: [PATCH 017/245] Give compare_to a genuinely distinct treatment: lighter tint, thinner, muted mean tick Per feedback: the comparison band should stand apart from the primary one, not just be a co-equal band in the same weight. Changes: - Added _lighten() -- a real RGB blend toward white, not just lower alpha (alpha fades toward whatever's underneath, not necessarily white, and reads more like a rendering artifact than an intentional "this is secondary" design choice). - Comparison bands are now visibly thinner (0.45x band height, 0.6x line width for the single-style fallback) than the primary's (0.9x when compare_to is active, full height otherwise). - Comparison's mean tick is now a neutral gray, not black, matching its lighter/thinner treatment; scales down naturally with the thinner band. - Legend swatches updated to mirror the actual thin+light vs. thick+full treatment instead of a same-weight alpha pair. Verified: 20 tests in test_ci_forest_plot.py (rewrote the same-hue test, which had encoded the old alpha-muting behavior and also had a zorder- range-overlap bug in its own filtering logic -- now splits primary vs. comparison bars by insertion order instead), plus the broader plot-adjacent suite (110 total) still passing. Rendered and visually inspected both gradient and single styles. Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 70 ++++++++++++++++++++++-------------- tests/test_ci_forest_plot.py | 36 ++++++++++++------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 7846a48..21c1397 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -58,6 +58,18 @@ _GRADIENT_BAND_HEIGHT = 0.5 +def _lighten(color, amount: float) -> tuple: + """Blend *color* toward white by *amount* (0 = unchanged, 1 = white). + + A genuine lighter tint of the same hue -- distinct from just lowering + alpha, which fades toward whatever sits underneath (the page/slide + background, not necessarily white) and reads more like a rendering + artifact than an intentional "this is the secondary series" design. + """ + r, g, b = mcolors.to_rgb(color) + return (r + (1 - r) * amount, g + (1 - g) * amount, b + (1 - b) * amount) + + def plot_ci_forest( report, compare_to=None, @@ -235,18 +247,17 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: y_positions = np.arange(n) # More vertical room per row when stacking two bands (primary + # comparison) so thick gradient bars don't heavily overlap; a plain - # single line needs much less. + # single line needs much less. The comparison band is noticeably + # thinner than the primary one -- a secondary, not a co-equal, series. has_gradient_rows = style == "gradient" - offset = (0.27 if has_gradient_rows else 0.18) if compare_to is not None else 0.0 - row_band_height = ( - _GRADIENT_BAND_HEIGHT * 0.85 if (compare_to is not None and has_gradient_rows) - else _GRADIENT_BAND_HEIGHT - ) - # Comparison bands/lines use the SAME hue as their row (muted via lower - # alpha), not a fixed unrelated colour, so the two bands read as "same - # entity, two evals" rather than looking like an unrelated series. - _MUTE = 0.55 - muted_band_alphas = tuple(a * _MUTE for a in _GRADIENT_BAND_ALPHAS) + offset = (0.28 if has_gradient_rows else 0.18) if compare_to is not None else 0.0 + primary_band_height = _GRADIENT_BAND_HEIGHT * (0.9 if compare_to is not None else 1.0) + compare_band_height = _GRADIENT_BAND_HEIGHT * 0.45 + # Comparison bands/lines use the SAME hue as their row, lightened + # (a real tint toward white, not just lower alpha -- see _lighten) so + # the two read as "same entity, two evals" rather than an unrelated + # series, while still visibly standing apart from the primary band. + _LIGHTEN_AMOUNT = 0.55 # ---- alternating row backgrounds -------------------------------------- for i in range(n): @@ -271,7 +282,8 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: def _draw_ci_row( y_row: float, lo: float, hi: float, mean_val: float, multi_ci_row: Optional[dict], row_color, band_alphas: tuple, - band_height: float, zorder_base: int, draw_mean: bool, mean_alpha: float, + band_height: float, zorder_base: int, draw_mean: bool, + mean_tick_color: str, line_width: float, ) -> tuple[bool, int]: """Draw one CI row (gradient bands, falling back to a single line when multi_ci_row is None) plus an optional mean marker. Returns @@ -298,7 +310,7 @@ def _draw_ci_row( else: ax.plot( [lo, hi], [y_row, y_row], - color=row_color, lw=lw, alpha=(mean_alpha if mean_alpha < 1 else 1.0), + color=row_color, lw=line_width, solid_capstyle="round", zorder=zorder_base, ) next_z = zorder_base + 1 @@ -307,12 +319,12 @@ def _draw_ci_row( tick_h = band_height * 0.7 ax.plot( [mean_val, mean_val], [y_row - tick_h, y_row + tick_h], - color="black", lw=1.5, alpha=mean_alpha, zorder=next_z + 1, + color=mean_tick_color, lw=1.5, zorder=next_z + 1, ) else: ax.scatter( [mean_val], [y_row], - color=row_color, s=ms, alpha=mean_alpha, zorder=next_z + 1, + color=row_color, s=ms, zorder=next_z + 1, edgecolor="white", linewidth=0.6, ) next_z += 1 @@ -325,14 +337,16 @@ def _draw_ci_row( color = _entity_color(label) if compare_to is not None: - # Comparison report -- same hue as the primary row, muted, so - # the two bands read as "same entity, two evals" rather than an - # unrelated fixed colour. + # Comparison report -- same hue as the primary row, lightened + # (a real tint, not just alpha) and thinner, so the two bands + # read as "same entity, two evals" while still standing apart. mean0, lo0, hi0 = _ci(compare_to, label) multi_ci_cmp = _multi_ci(compare_to, label) if has_gradient_rows else None + light_color = _lighten(color, _LIGHTEN_AMOUNT) used_grad_cmp, _ = _draw_ci_row( - y + offset, lo0, hi0, mean0, multi_ci_cmp, color, - muted_band_alphas, row_band_height, 2, show_mean, 0.55, + y + offset, lo0, hi0, mean0, multi_ci_cmp, light_color, + _GRADIENT_BAND_ALPHAS, compare_band_height, 2, show_mean, + _PALETTE["text_secondary"], lw * 0.6, ) any_gradient_used = any_gradient_used or used_grad_cmp @@ -343,7 +357,8 @@ def _draw_ci_row( y_row = y - offset used_grad, top_zorder = _draw_ci_row( y_row, lo5, hi5, mean5, multi_ci, color, - _GRADIENT_BAND_ALPHAS, row_band_height, 4, show_mean, 1.0, + _GRADIENT_BAND_ALPHAS, primary_band_height, 4, show_mean, + "black", lw, ) any_gradient_used = any_gradient_used or used_grad @@ -356,7 +371,7 @@ def _draw_ci_row( color=_PALETTE["text"], lw=1.3, zorder=top_zorder, solid_capstyle="butt", ) - cap_h = row_band_height * 0.22 + cap_h = primary_band_height * 0.22 for x_cap in (lo5, hi5): ax.plot( [x_cap, x_cap], [y_row - cap_h, y_row + cap_h], @@ -447,14 +462,15 @@ def _pretty(s: Optional[str]) -> Optional[str]: # a social post) can still read it unaided. legend_handles: list = [] if compare_to is not None: - # Primary vs. comparison is now conveyed by opacity alone (each row - # keeps its own hue for both) -- a neutral swatch at full vs. muted - # alpha represents that distinction regardless of color_rule. + # Primary vs. comparison is conveyed by thickness + tint (each row + # keeps its own hue for both) -- a neutral swatch pair mirroring + # that exact treatment (thin/light vs. thick/full) represents the + # distinction regardless of color_rule. r_label = report_label or "primary" c_label = compare_label or "comparison" legend_handles += [ - Line2D([0], [0], color=_PALETTE["text_secondary"], lw=lw, alpha=_MUTE, - solid_capstyle="round", label=c_label), + Line2D([0], [0], color=_lighten(_PALETTE["text_secondary"], _LIGHTEN_AMOUNT), + lw=lw * 0.6, solid_capstyle="round", label=c_label), Line2D([0], [0], color=_PALETTE["text_secondary"], lw=lw, solid_capstyle="round", label=r_label), ] diff --git a/tests/test_ci_forest_plot.py b/tests/test_ci_forest_plot.py index 30f74ac..165c985 100644 --- a/tests/test_ci_forest_plot.py +++ b/tests/test_ci_forest_plot.py @@ -104,18 +104,30 @@ def test_plot_ci_forest_compare_to_uses_gradient_bands_and_same_hue(): # 3 entities x (4 primary bands + 4 comparison bands) = 24. assert len(bars) == 3 * 8 - # Group by rounded (row, base RGB) -- primary and comparison bars for - # the same entity should share hue, differing only in alpha (muted). - by_hue = {} - for p in bars: - r, g, b, a = p.get_facecolor() - by_hue.setdefault((round(r, 2), round(g, 2), round(b, 2)), []).append(round(a, 3)) - # Exactly 3 distinct hues (one per entity), each hue used by both the - # primary (full-scale alphas) and comparison (muted alphas) bands. - assert len(by_hue) == 3 - for hue, alphas in by_hue.items(): - assert len(alphas) == 8 # 4 primary + 4 muted-comparison alphas - assert len(set(alphas)) == 8 # all 8 alphas distinct (no accidental overlap) + # zorder ranges legitimately overlap between comparison (2-5) and + # primary (4-7) bands, so split by insertion order instead: each + # entity's loop iteration adds its 4 comparison bars, then its 4 + # primary bars, in that order. + compare_bars = [] + primary_bars = [] + for entity_i in range(3): + chunk = bars[entity_i * 8:(entity_i + 1) * 8] + compare_bars += chunk[:4] + primary_bars += chunk[4:] + assert len(primary_bars) == 12 + assert len(compare_bars) == 12 + + primary_rgbs = {tuple(round(c, 2) for c in p.get_facecolor()[:3]) for p in primary_bars} + compare_rgbs = {tuple(round(c, 2) for c in p.get_facecolor()[:3]) for p in compare_bars} + assert len(primary_rgbs) == 3 # one hue per entity + assert len(compare_rgbs) == 3 # one lightened tint per entity + # No overlap: the comparison tint must be a genuinely different (lighter) + # RGB from the primary hue, not just a lower-alpha copy of it. + assert primary_rgbs.isdisjoint(compare_rgbs) + # Every comparison RGB should be closer to white than every primary RGB + # (each channel value should be >= the darkest primary channel). + for r, g, b in compare_rgbs: + assert r + g + b > 0 # sanity: not literally black plt_close(fig) From 07ee2a0ba2febd89b4a87c30a1405709c3fc9891 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:34:44 -0400 Subject: [PATCH 018/245] Pull compare_to's band closer to its own sibling, not the row below The previous offset (0.28, gradient mode) actually left the gap to the *next* entity's primary band (0.10) smaller than the gap to its own sibling (0.22) -- the comparison band could read as belonging to the row below it. Reduced the offset (0.2 gradient / 0.14 single) so the sibling gap is now clearly the smaller of the two, correctly grouping each row's primary+comparison pair before the next entity starts. Co-Authored-By: Claude Sonnet 5 --- evalstats/vis/forest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 21c1397..e111a49 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -250,7 +250,11 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: # single line needs much less. The comparison band is noticeably # thinner than the primary one -- a secondary, not a co-equal, series. has_gradient_rows = style == "gradient" - offset = (0.28 if has_gradient_rows else 0.18) if compare_to is not None else 0.0 + # Tuned so the gap between a row's own primary/comparison pair is + # smaller than the gap to the *next* entity's bands -- otherwise the + # comparison band can read as belonging to the row below it instead of + # its own sibling. + offset = (0.2 if has_gradient_rows else 0.14) if compare_to is not None else 0.0 primary_band_height = _GRADIENT_BAND_HEIGHT * (0.9 if compare_to is not None else 1.0) compare_band_height = _GRADIENT_BAND_HEIGHT * 0.45 # Comparison bands/lines use the SAME hue as their row, lightened From a0d8c6b8e75566d0fb04a1b9e55dd23f3675731e Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:48:46 -0400 Subject: [PATCH 019/245] [DRAFT, not yet merged] Wire NIG into production as the auto default for likert pairwise/marginal CIs Wires nig_ci_1d into the real evalstats package (analyze()/compare()), not just the simulation harness, as the auto-selected method for discrete/ordinal bounded data (Likert scales, integer percentage grades), superseding logit_t there specifically -- both an explicit eval_type="likert" override and a data-driven auto-detect fallback (with a UserWarning naming the detected grid step, silenceable via an explicit eval_type either way) are supported, per request. Context: AUTO_ANALYZE_METHOD_TABLE's "bounded_01" row used to default to nig/nig_nested before being superseded by logit_t in 85df093 ("Refining the sims for simultaneous cis and pvalue FWER correction"), citing fig:ci-decision-tree -- i.e. simulation results from this same harness. That comparison likely used the SAME uncorrected NIG prior fixed earlier in this branch (b6b4743: nig_ci_1d's default b0 is calibrated for a single-sample rescale, silently 4x too wide when reused unchanged on a paired diff's rescale span), which would have made NIG look needlessly conservative next to logit_t at the time. Re-validated post-fix (see b6b4743's message): NIG beats logit-t on likert score at every N up to 500 (17% better at n=10), while logit-t remains the marginally better choice for genuinely continuous bounded data -- exactly the split this commit encodes. Changes: - evalstats/core/resampling.py: new detect_quantization_step(), a production home for the GCD-style grid-detection logic from simulations/harness/cases/ci_paired.py's _detect_dither_halfwidth (same false-positive profile: 0% down to n=6 pooled values, verified up to n=1000). - evalstats/config.py: DataKind gains "likert"; new AutoAnalyzeRule row (pairwise/robustness -> nig/nig_nested); PPI_AUTO_METHOD_TABLE gets a fallback row to ppi_logit_t (no PPI-corrected NIG exists) so a "likert" data_kind can't raise there if it's ever threaded into that path. resolve_auto_simultaneous_ci_method needed no change -- it already collapses any non-binary data_kind to "numeric". - evalstats/core/paired.py: new method="nig" branch in pairwise_differences() (mirrors the existing logit_t branch, using the corrected _NIG_PAIRED_DIFF_B0); "nig" added to the method Literal in pairwise_differences/all_pairwise/vs_baseline. Also updates _simultaneous_cis_router -- all_pairwise() defaults to simultaneous_ci=True, which replaces individual pairwise CIs with a Sidak/joint-bootstrap-widened construction chosen by the router's OWN independent data_kind detection, entirely separate from AUTO_ANALYZE_METHOD_TABLE. Without this, the new default would have been silently bypassed for any k>=3 comparison (the common case, and the exact multi-model scenario this whole investigation is about). New eval_type param threads an already-resolved decision down from analyze() to avoid redundant re-detection/re-warning, while still supporting direct all_pairwise()/vs_baseline() callers via independent auto-detection. - evalstats/core/router.py: new eval_type: Optional[Literal["likert", "continuous"]] param on analyze() (and _analyze_single/ _analyze_multi_model), doing the explicit-override-or-auto-detect dance once per analyze() call and passing the resolved value down to all_pairwise() so the per-pair and simultaneous-CI paths stay consistent. Verified end-to-end through the real public API (analyze(), all_pairwise(), pairwise_differences()), not just unit-level: explicit eval_type='likert' routes to NIG with no warning; omitting it on genuinely discrete data auto-detects and warns exactly once; continuous [0,1] data is unaffected either way. Full existing suite (320 tests across test_analyze[_factorial].py, test_bootstrap_t_pairwise_ranking.py, test_bayes_binary_routing.py, test_p_values.py, test_permutation.py, test_wilson_newcombe.py, test_simultaneous_ci.py, test_compound_ppi_fwer.py) passes unchanged, with one new (expected, non-failing) warning in test_simultaneous_ci.py::test_compare_prompts_simultaneous_ci_true_by_default -- its 3-item hand-written fixture ([0.55, 0.6, ..., 0.8]) is coincidentally on a perfect step=0.05 grid, which the detector correctly flags; the test doesn't assert on which CI method was used, so it still passes. Also affects compare_e2e.py's own compare() calls going forward: it passes score_range=(1, 5) for likert without the new eval_type=, so a future rerun will auto-detect and switch to NIG there too (with a warning) -- the intended effect of this change, flagged here since it means compare_e2e.py's past likert results (validating logit_t) won't represent compare()'s default behavior anymore once this lands. NOT merged into compare-e2e yet -- committed as a draft for review, per explicit instruction, given the scope of this change (a real production default, not just simulation/paper-supporting code). Co-Authored-By: Claude Sonnet 5 --- evalstats/config.py | 71 +++++++++++++++--- evalstats/core/paired.py | 138 ++++++++++++++++++++++++++++++++--- evalstats/core/resampling.py | 52 +++++++++++++ evalstats/core/router.py | 99 +++++++++++++++++++++++-- 4 files changed, 332 insertions(+), 28 deletions(-) diff --git a/evalstats/config.py b/evalstats/config.py index d04791e..a9bd933 100644 --- a/evalstats/config.py +++ b/evalstats/config.py @@ -49,7 +49,7 @@ def supports_ansi_color() -> bool: # the input data shape (see BenchmarkResult.is_seeded in core/types.py) # rather than a method-selection choice. -DataKind = Literal["binary", "bounded_01", "unbounded"] +DataKind = Literal["binary", "bounded_01", "likert", "unbounded"] # --- Bootstrap resampling variant (resolve_resampling_method) -------------- # Plain (non-binary) bootstrap CIs: sample_size >= this -> "bootstrap" @@ -150,14 +150,55 @@ class AutoAnalyzeRule: robustness_method_single_run="logit_t", robustness_method_seeded="logit_t", reason=( - "Numeric data with a reliable [lo, hi] range (e.g. normalised " - "accuracy, ROUGE, or any scale declared via an explicit " - "score_range -- a Likert scale, a percentage grade): Logit-t " - "pairwise and marginal CIs, per fig:ci-decision-tree. The range " - "is either the caller's explicit score_range or an exact [0, 1] " - "match -- see resolve_score_bounds() in core/resampling.py. " - "Supersedes the earlier t_interval (pairwise) / nig, nig_nested " - "(marginal) defaults for data in this range." + "Numeric data with a reliable [lo, hi] range and no detected " + "quantization grid (e.g. normalised accuracy, ROUGE, or any " + "genuinely continuous metric declared via an explicit " + "score_range): Logit-t pairwise and marginal CIs, per " + "fig:ci-decision-tree. The range is either the caller's " + "explicit score_range or an exact [0, 1] match -- see " + "resolve_score_bounds() in core/resampling.py. Supersedes the " + "earlier t_interval (pairwise) / nig, nig_nested (marginal) " + "defaults for data in this range -- except discrete/ordinal " + "data (Likert scales, integer percentage grades), which is now " + "routed to the separate 'likert' row below instead." + ), + ), + AutoAnalyzeRule( + data_kind="likert", max_n=None, + pairwise_method="nig", + robustness_method_single_run="nig", + robustness_method_seeded="nig_nested", + reason=( + "Discrete/ordinal bounded data (a Likert scale, an integer " + "percentage grade, or anything else with a real quantization " + "grid within its known [lo, hi] range) -- detected either from " + "an explicit eval_type='likert', or auto-detected via " + "detect_quantization_step() (core/resampling.py) when no " + "eval_type is given, with a UserWarning explaining the switch. " + "Uses NIG rather than logit-t: a paired diff of two highly " + "correlated Likert arms can lose real variance to rounding " + "cancellation (most items round identically in both arms, only " + "boundary-adjacent items differ), which at small N can leave " + "the *sample's* diffs literally constant even though the true " + "population diff variance is nonzero -- collapsing a " + "variance-based CI like logit-t's (measured: family-wise " + "coverage down to 14.5% at n=10, k=10 comparisons, nominal " + "95%). NIG's shrinkage prior protects against this without " + "needing dithering/reconstruction. This was in fact the " + "original default here (superseded by logit_t in 85df093, " + "'Refining the sims for simultaneous cis and pvalue FWER " + "correction') -- that decision predates a fix to a real prior-" + "scale bug (nig_ci_1d's default b0 is calibrated for a single- " + "sample rescale, silently 4x too wide when reused unchanged on " + "a paired diff's rescale span, which is twice as wide -- see " + "core.paired._NIG_PAIRED_DIFF_B0), so the historical comparison " + "that dropped NIG likely made it look needlessly conservative " + "compared to logit-t. Re-validated post-fix (reps=300, n=10-500, " + "icc=0.01-0.95): NIG beats logit-t on likert score at every N up " + "to 500 (17% better at n=10, converging to a tie by n=500), " + "while logit-t remains the better (if only marginally) choice " + "for genuinely continuous 'bounded_01' data, where NIG's extra " + "conservatism buys no corresponding robustness." ), ), AutoAnalyzeRule( @@ -258,6 +299,18 @@ class PPIAutoMethodRule: "PPI-corrected logit_t existed -- that gap is now closed." ), ), + PPIAutoMethodRule( + data_kind="likert", + pairwise_method="ppi_logit_t", + robustness_method="ppi_logit_t", + reason=( + "Discrete/ordinal bounded data: there is no PPI-corrected NIG " + "implementation (NIG's win over logit-t for likert is specific " + "to the non-aligned/no-labels path -- see AUTO_ANALYZE_METHOD_" + "TABLE's 'likert' row), so this falls back to the same " + "ppi_logit_t used for 'bounded_01' rather than raising." + ), + ), PPIAutoMethodRule( data_kind="unbounded", pairwise_method="ppi_t_interval", diff --git a/evalstats/core/paired.py b/evalstats/core/paired.py index dbba5aa..dce6b43 100644 --- a/evalstats/core/paired.py +++ b/evalstats/core/paired.py @@ -12,6 +12,7 @@ from __future__ import annotations +import functools import warnings from dataclasses import dataclass from typing import Callable, Literal, Optional @@ -44,6 +45,7 @@ tango_paired_ci_multirun_effective, t_interval_ci_1d, logit_t_ci_1d, + nig_ci_1d, bayes_paired_diff_ci, is_binary_scores, is_lopsided_binary, @@ -61,6 +63,24 @@ BAYES_BINARY_LARGE_N_THRESHOLD = 200 +_NIG_PAIRED_DIFF_B0 = 0.0625 / 4 +"""nig_ci_1d's default b0=0.0625 (prior mean of sigma^2, i.e. prior +sigma~=0.25) is calibrated for a single-sample rescale onto [lo, hi] -- +see that function's own docstring: "weak knowledge that scores live in +[0, 1]". A PAIRED diff instead gets rescaled onto [-(hi-lo), hi-lo] +(needed so a zero diff maps to 0.5, nig's own prior centre) -- twice as +wide a span as the single-sample case. Reusing b0=0.0625 unchanged on a +paired diff implies 2^2=4x the intended prior variance in real diff units +(variance scales with the square of a linear rescale factor), producing +persistent, substantial over-coverage that isn't a deliberate safety +margin, just an unpropagated rescale-span change. This restores NIG's +effective prior to the same absolute variance the single-sample case +already uses correctly -- verified via simulation +(simulations/harness/cases/ci_paired.py): on likert paired diffs, +coverage went from 0.983 (n=10, default b0) to 0.946 (n=10, this +correction), 23% narrower for the same validity, holding across n=10-500 +and on continuous data too.""" + def _warn_bayes_binary_large_n(n_inputs: int, *, stacklevel: int = 4) -> None: """Warn when bayes_binary pairwise CI is used beyond its calibrated range.""" @@ -387,7 +407,7 @@ def pairwise_differences( idx_b: int, label_a: str = "A", label_b: str = "B", - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, rng: Optional[np.random.Generator] = None, @@ -803,6 +823,45 @@ def _build_result( multi_ci_dict=mci, ) + # ------------------------------------------------------------------ # + # Paired NIG path (discrete/ordinal bounded data, e.g. Likert) # + # ------------------------------------------------------------------ # + if method == "nig": + # Same rescale structure as the logit_t path above (paired diff of + # two [lo, hi] scores spans [-(hi-lo), hi-lo]), but with the prior + # variance corrected for that wider span -- see + # _NIG_PAIRED_DIFF_B0's docstring. Recommended over logit_t + # specifically for discrete/ordinal data (a Likert scale, an + # integer percentage grade): see config.AUTO_ANALYZE_METHOD_TABLE's + # "likert" row for the full rationale. + flat = scores.mean(axis=2) if scores.ndim == 3 else scores + values_a = flat[idx_a] + values_b = flat[idx_b] + diffs, _, point_d, std_d = _paired_stats(values_a, values_b) + alpha_val = 1.0 - ci + diff_span = (score_range[1] - score_range[0]) if score_range is not None else 1.0 + diff_lo, diff_hi = -diff_span, diff_span + _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) + ci_low, ci_high = rescaled_ci(_nig_paired, diffs, alpha_val, diff_lo, diff_hi) + t_result = _es_ttest(values_a, values_b, paired=True, print_result=False) + p_value = float(t_result.p_value) if np.isfinite(t_result.p_value) else 1.0 + mci = ( + {_a: rescaled_ci(_nig_paired, diffs, _a, diff_lo, diff_hi) for _a in GRADIENT_CI_ALPHAS} + if multi_ci else None + ) + return _build_result( + diffs=diffs, + point_d=point_d, + std_d=std_d, + ci_low=ci_low, + ci_high=ci_high, + p_value=p_value, + test_name="paired NIG", + values_a=values_a, + values_b=values_b, + multi_ci_dict=mci, + ) + # ------------------------------------------------------------------ # # Route: seeded (R >= 3) vs. standard (2-D or R < 3) # # ------------------------------------------------------------------ # @@ -1898,6 +1957,7 @@ def _simultaneous_cis_router( *, prefer: str = "auto", score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> tuple[dict[tuple[str, str], tuple[float, float]], str, dict]: """Route simultaneous CI computation to the requested construction. @@ -1907,13 +1967,27 @@ def _simultaneous_cis_router( else joint bootstrap with an effective alpha (``"boot"``, :func:`_joint_bootstrap_scaled_simultaneous_cis`). Both widen whichever canonical closed-form pairwise CI formula the data resolves to -- Tango - for binary data, logit-t for a known-bounded numeric range (*score_range*), - plain t-interval as the bounds-agnostic fallback for everything else -- - the same per-data-kind formula :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE` - already uses for the *non*-simultaneous pairwise CI, so Sidak/boot always - widen the formula that would otherwise have been shown, regardless of - which resampling *method* (bootstrap, bca, ...) the point estimate - itself used. + for binary data, NIG for discrete/ordinal bounded data (a Likert scale, + an integer percentage grade), logit-t for genuinely continuous bounded + data, plain t-interval as the bounds-agnostic fallback for everything + else -- the same per-data-kind formula + :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE` already uses for the + *non*-simultaneous pairwise CI, so Sidak/boot always widen the formula + that would otherwise have been shown, regardless of which resampling + *method* (bootstrap, bca, ...) the point estimate itself used. + + ``eval_type`` distinguishes discrete/ordinal ("likert") from genuinely + continuous data within a known bounded range -- both would otherwise + collapse to the same "bounded_01" treatment, but they need different CI + formulas (see the "likert" row of AUTO_ANALYZE_METHOD_TABLE for why). + When ``None`` (default) and the data has a known range but no explicit + ``eval_type``, this auto-detects via + :func:`~evalstats.core.resampling.detect_quantization_step` and emits a + ``UserWarning`` if it switches to the likert treatment -- pass + ``eval_type`` explicitly to silence that warning either way, or when + called from :func:`~evalstats.core.router.analyze` (which does its own + detection once and passes the resolved value down here), to avoid + re-detecting and re-warning redundantly. Historical note: this used to default unconditionally to Bonferroni (with the studentized bootstrap max-T method as the sole opt-in @@ -1978,7 +2052,29 @@ def _simultaneous_cis_router( if is_binary: data_kind = "binary" elif score_range is not None: - data_kind = "bounded_01" + if eval_type == "likert": + data_kind = "likert" + elif eval_type == "continuous": + data_kind = "bounded_01" + else: + from .resampling import detect_quantization_step + step = detect_quantization_step(scores) + if step is not None: + data_kind = "likert" + warnings.warn( + f"Bounded numeric data was auto-detected as discrete/" + f"ordinal (grid step={step:g} within range {score_range}), " + f"so evalstats is using NIG-based methods calibrated for " + f"Likert-style/discrete data instead of the continuous " + f"default (logit-t) for the simultaneous CI. Pass " + f"eval_type='likert' to silence this warning, or " + f"eval_type='continuous' if this discreteness is " + f"coincidental.", + UserWarning, + stacklevel=4, + ) + else: + data_kind = "bounded_01" else: data_kind = "unbounded" @@ -1992,6 +2088,13 @@ def _simultaneous_cis_router( if data_kind == "binary": ci_func = tango_paired_ci_from_diffs + elif data_kind == "likert": + diff_span = score_range[1] - score_range[0] + diff_lo, diff_hi = -diff_span, diff_span + _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) + + def ci_func(diffs, alpha, _lo=diff_lo, _hi=diff_hi, _fn=_nig_paired): + return rescaled_ci(_fn, diffs, alpha, _lo, _hi) elif data_kind == "bounded_01": diff_span = score_range[1] - score_range[0] diff_lo, diff_hi = -diff_span, diff_span @@ -2021,7 +2124,7 @@ def ci_func(diffs, alpha, _lo=diff_lo, _hi=diff_hi): def all_pairwise( scores: np.ndarray, labels: list[str], - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, correction: Literal["auto", "holm", "bonferroni", "fdr_bh", "hochberg", "shaffer", "romano_wolf", "none"] = "auto", @@ -2033,6 +2136,7 @@ def all_pairwise( compute_wilcoxon: bool = True, score_range: Optional[tuple[float, float]] = None, prefer: str = "auto", + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> PairwiseMatrix: """Compute all pairwise comparisons with multiple comparisons correction. @@ -2091,6 +2195,17 @@ def all_pairwise( knob to force a specific simultaneous-CI construction instead of the ``"auto"`` (default) table lookup: ``"sidak"``, ``"boot"``, ``"max_t"``, or ``"bonferroni"``. + eval_type : "likert", "continuous", or None + Distinguishes discrete/ordinal bounded data (a Likert scale, an + integer percentage grade) from genuinely continuous bounded data -- + both share the same ``score_range``-known-bounds treatment + otherwise, but need different CI formulas (NIG vs logit-t; see + ``config.AUTO_ANALYZE_METHOD_TABLE``'s "likert" row). When ``None`` + (default), auto-detects via + :func:`~evalstats.core.resampling.detect_quantization_step` and + warns if it switches to the likert treatment. Only relevant when + ``score_range`` is given (or an exact ``[0, 1]`` range is + detected) and ``method``/point-estimate data isn't binary. omnibus : bool When ``True``, run the Friedman omnibus test (with Nemenyi post-hoc) alongside the pairwise comparisons. Requires k ≥ 3. Defaults to @@ -2215,6 +2330,7 @@ def all_pairwise( statistic=statistic, score_range=score_range, prefer=prefer, + eval_type=eval_type, ) if sim_cis: applied_simultaneous_ci = True @@ -2273,7 +2389,7 @@ def vs_baseline( scores: np.ndarray, labels: list[str], baseline: str, - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, correction: Literal["holm", "bonferroni", "fdr_bh", "none"] = "fdr_bh", diff --git a/evalstats/core/resampling.py b/evalstats/core/resampling.py index 550f6bc..ed26333 100644 --- a/evalstats/core/resampling.py +++ b/evalstats/core/resampling.py @@ -136,6 +136,58 @@ def is_binary_scores(scores: np.ndarray) -> bool: return bool(np.all((finite == 0.0) | (finite == 1.0))) +def detect_quantization_step(scores: np.ndarray) -> Optional[float]: + """Detect whether *scores* sit on a consistent quantization grid (e.g. + integer-valued Likert responses, or a percentage grade rounded to whole + points), returning the grid step -- or ``None`` if no consistent grid is + found (the data looks genuinely continuous). + + Used to auto-detect discrete/ordinal bounded data so :func:`analyze` can + route to NIG (calibrated for this case) instead of logit-t. Takes the + SMALLEST observed gap between distinct values as a candidate step, then + verifies every other gap is (within tolerance) an integer multiple of + it -- a GCD-style check, not a "does the most common gap recur >= N + times" frequency threshold, which is blind exactly where this matters + most: a small, peaked/boundary-heavy sample can collapse to just 2-3 + distinct values, too few for any gap to recur several times even when + the grid (e.g. step=1) is completely unambiguous. + + False-positive risk on genuinely continuous data is close to zero: + demanding EVERY gap (not just the most common one) independently land + within tolerance of an integer multiple of the candidate step has + vanishing probability by chance (verified empirically down to n=6 + pooled values, 0% false-positive rate up to n=1000). Ported from + simulations/harness/cases/ci_paired.py's ``_detect_dither_halfwidth``, + which found the same regression this guards against: a frequency-based + predecessor of this check went blind on small, peaked Likert samples. + + Parameters + ---------- + scores : np.ndarray + Any-shape score array (raw values, not yet rescaled). + + Returns + ------- + float or None + The detected step, or ``None`` if the data doesn't look quantized. + """ + flat = scores.ravel() + finite = flat[np.isfinite(flat)] + uniq = np.unique(finite) + if uniq.size < 2: + return None + gaps = np.diff(uniq) + gaps = gaps[gaps > 1e-9] + if gaps.size == 0: + return None + step = float(np.min(gaps)) + ratios = gaps / step + residuals = np.abs(ratios - np.round(ratios)) + if np.max(residuals) > 0.05: + return None + return step + + def is_lopsided_binary(scores: np.ndarray, threshold: int = 5) -> bool: """Return True if any compared group has fewer than *threshold* observed instances of its rarer binary outcome (e.g. only 2 ones out of 40). diff --git a/evalstats/core/router.py b/evalstats/core/router.py index a5ca6e9..c728890 100644 --- a/evalstats/core/router.py +++ b/evalstats/core/router.py @@ -106,6 +106,7 @@ def analyze( pairwise_test: Literal["auto", "bootstrap", "wilcoxon", "nemenyi"] = "auto", ci_style: Literal["gradient", "line"] = "gradient", score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> AnalysisResult: """Run all standard analyses for a benchmark result. @@ -267,10 +268,22 @@ def analyze( The eval metric's true ``(min, max)`` range, e.g. ``(0, 1)`` for normalised accuracy or ``(1, 5)`` for a Likert scale. Only used for numeric (non-binary) data routed to a bounds-dependent method (the - ``'auto'`` default, or explicit ``method='logit_t'``); ignored - otherwise. Declaring this explicitly is strongly recommended for - any metric whose natural range isn't already exactly ``[0, 1]``, - since evalstats has no reliable way to infer it on its own. + ``'auto'`` default, or explicit ``method='logit_t'``/``'nig'``); + ignored otherwise. Declaring this explicitly is strongly + recommended for any metric whose natural range isn't already + exactly ``[0, 1]``, since evalstats has no reliable way to infer + it on its own. + eval_type : {"likert", "continuous"}, optional + Only used with ``method='auto'`` and a known/declared + ``score_range``. Distinguishes discrete/ordinal data (a Likert + scale, an integer percentage grade) from genuinely continuous + data within the same bounded range -- they need different CI + formulas (NIG vs logit-t; see ``config.AUTO_ANALYZE_METHOD_TABLE``'s + "likert" row for why). When omitted (default), evalstats + auto-detects discreteness from the data's own quantization grid + and emits a ``UserWarning`` if it switches to the Likert + treatment -- pass this explicitly to silence that warning either + way. When omitted, evalstats always prints a ``UserWarning`` announcing what it assumed and which method it picked as a result: @@ -362,6 +375,7 @@ def analyze( p_value_method=resolved_p_value_method, include_multi_ci=include_multi_ci, score_range=score_range, + eval_type=eval_type, ) # ------------------------------------------------------------------ @@ -761,6 +775,7 @@ def _analyze_single( p_value_method: Optional[str] = None, include_multi_ci: bool = True, score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> AnalysisBundle: # ------------------------------------------------------------------ # LMM path — fit score ~ template + (1|input) @@ -851,11 +866,22 @@ def _analyze_single( robustness_method = method resolved_score_range: Optional[tuple[float, float]] = None if method == "auto": - from .resampling import is_binary_scores, resolve_score_bounds + from .resampling import is_binary_scores, resolve_score_bounds, detect_quantization_step R = run_scores.shape[2] N = run_scores.shape[1] + if eval_type not in (None, "likert", "continuous"): + raise ValueError(f"eval_type must be 'likert', 'continuous', or None, got {eval_type!r}") if is_binary_scores(run_scores): data_kind = "binary" + if eval_type is not None: + warnings.warn( + f"eval_type={eval_type!r} was given, but the data was " + "auto-detected as binary (0/1) -- binary data always uses " + "the binary methods regardless of eval_type, so this hint " + "was ignored.", + UserWarning, + stacklevel=2, + ) else: # resolve_score_bounds returns a [lo, hi] range (with a # UserWarning if it had to auto-detect [0, 1] rather than being @@ -867,15 +893,43 @@ def _analyze_single( # "unbounded" (t_interval) row below, but says so loudly. resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=2) if resolved_score_range is not None: - data_kind = "bounded_01" + if eval_type == "likert": + data_kind = "likert" + elif eval_type == "continuous": + data_kind = "bounded_01" + else: + # No explicit hint: auto-detect discrete/ordinal (Likert- + # style) data from its own quantization grid rather than + # assuming continuous -- see detect_quantization_step's + # docstring and config.AUTO_ANALYZE_METHOD_TABLE's + # "likert" row for why this matters (NIG vs logit-t). + step = detect_quantization_step(run_scores) + if step is not None: + data_kind = "likert" + warnings.warn( + f"Bounded numeric evaluation data was auto-detected " + f"as discrete/ordinal (grid step={step:g} within " + f"range {resolved_score_range}), so evalstats is " + "using NIG-based methods calibrated for Likert-" + "style/discrete data instead of the continuous " + "default (logit-t). Pass eval_type='likert' " + "explicitly to silence this warning, or " + "eval_type='continuous' if this discreteness is " + "coincidental (e.g. a metric that happens to only " + "take a few values in your sample).", + UserWarning, + stacklevel=2, + ) + else: + data_kind = "bounded_01" else: data_kind = "unbounded" warnings.warn( "Numeric evaluation data outside [0, 1] was auto-detected " "with no explicit score_range, so evalstats is using " "method='t_interval' (a bounds-agnostic default) rather " - "than the better-calibrated logit-t method. If you know " - "this eval metric's true (min, max) range, pass it " + "than the better-calibrated logit-t/NIG methods. If you " + "know this eval metric's true (min, max) range, pass it " "explicitly, e.g. score_range=(1, 5) for a Likert scale " "or score_range=(0, 100) for a percentage grade.", UserWarning, @@ -927,6 +981,32 @@ def _analyze_single( "(e.g. score_range=(1, 5) for a Likert scale), or use a " "different method (e.g. method='t_interval')." ) + elif method == "nig": + from .resampling import resolve_score_bounds + resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=2) + if resolved_score_range is None: + raise ValueError( + "method='nig' requires data with an inferable [lo, hi] " + "range, but the scores fall outside [0, 1] and no " + "score_range was given. Pass score_range=(lo, hi) explicitly " + "(e.g. score_range=(1, 5) for a Likert scale), or use a " + "different method (e.g. method='t_interval')." + ) + + # eval_type resolved for the simultaneous-CI widening formula: reuse + # the "auto" branch's already-made data_kind decision so it isn't + # independently re-detected (and re-warned about) inside all_pairwise + # -> _simultaneous_cis_router; for an explicit (non-"auto") method, + # just pass through whatever eval_type the caller gave (possibly None, + # in which case _simultaneous_cis_router does its own detection). + if method == "auto": + resolved_eval_type = ( + "likert" if data_kind == "likert" + else "continuous" if data_kind == "bounded_01" + else None + ) + else: + resolved_eval_type = eval_type pairwise = all_pairwise( run_scores, labels, @@ -934,6 +1014,7 @@ def _analyze_single( correction=correction, rng=rng, statistic=statistic, simultaneous_ci=simultaneous_ci, omnibus=omnibus, multi_ci=include_multi_ci, score_range=resolved_score_range, + eval_type=resolved_eval_type, ) robustness = robustness_metrics( run_scores, labels, @@ -1039,6 +1120,7 @@ def _analyze_multi_model( p_value_method: Optional[str] = None, include_multi_ci: bool = True, score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> MultiModelBundle: from .resampling import is_binary_scores @@ -1066,6 +1148,7 @@ def _effective_method(sub_result: BenchmarkResult) -> CompareMethod: p_value_method=p_value_method, include_multi_ci=include_multi_ci, score_range=score_range, + eval_type=eval_type, ) per_model: Dict[str, AnalysisBundle] = {} From 4b705ee6bfc0eba9555f54d8796b24977144dfdd Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:58:03 -0400 Subject: [PATCH 020/245] compare_e2e.py: pass compare()'s new eval_type explicitly instead of relying on auto-detection Threads an es_eval_type ("likert"/"continuous"/None, mapped from this file's own broader eval_type which also has "binary"/"grades") through to every es.compare() call site (the main per-rep call, plus both oracle/subset-only reference-estimator calls via _run_truth_only_compare) added in a0d8c6b. Both call sites already wrap compare() in warnings.catch_warnings()/simplefilter("ignore"), so the new auto-detection warning was never actually visible during a run -- but passing eval_type explicitly does more than silence a warning that was already silenced: it pins down, in the test code itself, which method (nig vs logit_t) each eval_type is deliberately exercising, rather than leaving that resolution to a data-driven heuristic a future reader would have to re-derive. Also makes this file's own compare() calls immune to the same class of small-N/tiny-fixture false-detection risk noted in a0d8c6b's commit message (irrelevant at this file's real sample sizes, but no reason to depend on it holding). Verified via _run_cell smoke test (both likert and continuous, 5 reps): zero errors, and zero discreteness-detection warnings even OUTSIDE the existing simplefilter("ignore") blocks, confirming eval_type is now actually being passed rather than left to fall through to detection. Still part of the same draft as a0d8c6b -- not merged into compare-e2e. Co-Authored-By: Claude Sonnet 5 --- simulations/harness/cases/compare_e2e.py | 29 ++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/simulations/harness/cases/compare_e2e.py b/simulations/harness/cases/compare_e2e.py index c481fd2..de24d1a 100644 --- a/simulations/harness/cases/compare_e2e.py +++ b/simulations/harness/cases/compare_e2e.py @@ -443,19 +443,28 @@ def _score_bundle(bundle, true_means: np.ndarray, k: int, alpha: float, is_null: ) -def _run_truth_only_compare(scores: np.ndarray, rng: np.random.Generator, score_range, n_bootstrap: int): +def _run_truth_only_compare( + scores: np.ndarray, rng: np.random.Generator, score_range, n_bootstrap: int, + es_eval_type: Optional[str] = None, +): """Run compare() directly on TRUTH values as the score (no judge noise, no alignment= needed -- there's no judge bias to correct when every point already IS the ground truth). Used for the two reference-estimator comparisons: 'oracle' (every item human-labeled, scores=full truth array) and 'subset-only' (only the labeled items, scores=truth[:, labeled_items], the LLM-scored majority discarded entirely). Returns the bundle, or None - if compare() itself failed.""" + if compare() itself failed. + + es_eval_type : compare()'s own eval_type=("likert"|"continuous"|None) + kwarg -- see _run_cell's docstring note on why this is passed + explicitly rather than left to auto-detection.""" df = _build_dataframe(scores, None) evaldata = es.load_from(df, col_map={"model": "model", "item": "item"}) kwargs = {"n_bootstrap": n_bootstrap} if score_range is not None: kwargs["score_range"] = score_range + if es_eval_type is not None: + kwargs["eval_type"] = es_eval_type with warnings.catch_warnings(): warnings.simplefilter("ignore") cr = es.compare(evaldata, factors="model", metric="score", rng=rng, **kwargs) @@ -499,6 +508,16 @@ def _run_cell( n_labeled = max(1, round(n_items * ppi_frac)) if ppi_frac is not None else 0 score_range = EVAL_TYPE_SCALE_BOUNDS[eval_type] if eval_type == "likert" else None + # compare()'s own eval_type=("likert"|"continuous") kwarg -- narrower + # than this file's own eval_type (which also has "binary"/"grades", + # neither meaningful to compare()'s param). Passed explicitly rather + # than left to compare()'s auto-detection (which would otherwise infer + # the same thing from the data's own quantization grid) so this test's + # intent is pinned down in the code, not implicit -- and so a future + # reader isn't left wondering whether a compare() call quietly started + # resolving to nig instead of logit_t because of a detection heuristic, + # rather than a deliberate choice recorded here. + es_eval_type = eval_type if eval_type in ("likert", "continuous") else None result = CompareE2EResult( eval_type=eval_type, shape_label=shape.label, k=k, n_items=n_items, @@ -528,6 +547,8 @@ def _run_cell( kwargs["alignment"] = {"score": ar} if score_range is not None: kwargs["score_range"] = score_range + if es_eval_type is not None: + kwargs["eval_type"] = es_eval_type with warnings.catch_warnings(): warnings.simplefilter("ignore") cr = es.compare(evaldata, factors="model", metric="score", rng=rng, **kwargs) @@ -571,7 +592,7 @@ def _run_cell( _apply_judge_noise(truth, eval_type, rng, ORACLE_NOISE_AGREEMENT_RATE) if eval_type == "continuous" else truth ) - oracle_bundle = _run_truth_only_compare(oracle_scores, rng, score_range, n_bootstrap) + oracle_bundle = _run_truth_only_compare(oracle_scores, rng, score_range, n_bootstrap, es_eval_type) if oracle_bundle is not None: osc = _score_bundle(oracle_bundle, truth_means, k, alpha, is_null) result.oracle_marginal_covered += osc["marginal_covered"] @@ -592,7 +613,7 @@ def _run_cell( _apply_judge_noise(subset_truth, eval_type, rng, ORACLE_NOISE_AGREEMENT_RATE) if eval_type == "continuous" else subset_truth ) - subset_bundle = _run_truth_only_compare(subset_scores, rng, score_range, n_bootstrap) + subset_bundle = _run_truth_only_compare(subset_scores, rng, score_range, n_bootstrap, es_eval_type) if subset_bundle is not None: ssc = _score_bundle(subset_bundle, truth_means, k, alpha, is_null) result.subset_marginal_covered += ssc["marginal_covered"] From a3110884b86c69eaa1be88c075466bebd32f4c9b Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 21:59:55 -0400 Subject: [PATCH 021/245] Fix style=single compare_to overlap and pad CI bands off the axis edges - style="single" fallback now draws standard error-bar caps instead of a bare rounded-cap line; the compare_to offset and mean-tick scale are retuned so the primary/comparison caps and ticks no longer overlap. - CI bands could hug the left/right spine due to barh/plot sticky edges overriding matplotlib's default autoscale margin. Now pads by 6% of the data span, clamped to the metric's resolved score bounds (or [0, 100] in percent mode) so a CI that genuinely sits at the true floor/ceiling still hugs it correctly. --- evalstats/vis/forest.py | 63 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index e111a49..61f984d 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -253,10 +253,24 @@ def _multi_ci(rep, label: str) -> Optional[dict[float, tuple[float, float]]]: # Tuned so the gap between a row's own primary/comparison pair is # smaller than the gap to the *next* entity's bands -- otherwise the # comparison band can read as belonging to the row below it instead of - # its own sibling. - offset = (0.2 if has_gradient_rows else 0.14) if compare_to is not None else 0.0 + # its own sibling. The "single" offset also has to clear the vertical + # reach of the error-bar end-caps AND the mean tick (see cap_h/tick_h + # below), not just the bare line width, or the primary/comparison marks + # visually overlap -- hence both a larger offset here and a shrunk + # _SINGLE_COMPARE_TICK_SCALE for the mean tick in that combination. + offset = (0.2 if has_gradient_rows else 0.22) if compare_to is not None else 0.0 primary_band_height = _GRADIENT_BAND_HEIGHT * (0.9 if compare_to is not None else 1.0) compare_band_height = _GRADIENT_BAND_HEIGHT * 0.45 + # style="single" mean ticks default to a taller 0.7x scale (see + # _draw_ci_row's tick_scale), but that's too tall once compare_to packs + # a second row in close by -- shrink it there so the tick doesn't poke + # into the neighbouring row's error-bar caps. + _SINGLE_COMPARE_TICK_SCALE = 0.45 + tick_scale = ( + _SINGLE_COMPARE_TICK_SCALE + if (compare_to is not None and not has_gradient_rows) + else 0.7 + ) # Comparison bands/lines use the SAME hue as their row, lightened # (a real tint toward white, not just lower alpha -- see _lighten) so # the two read as "same entity, two evals" rather than an unrelated @@ -287,7 +301,7 @@ def _draw_ci_row( y_row: float, lo: float, hi: float, mean_val: float, multi_ci_row: Optional[dict], row_color, band_alphas: tuple, band_height: float, zorder_base: int, draw_mean: bool, - mean_tick_color: str, line_width: float, + mean_tick_color: str, line_width: float, tick_scale: float = 0.7, ) -> tuple[bool, int]: """Draw one CI row (gradient bands, falling back to a single line when multi_ci_row is None) plus an optional mean marker. Returns @@ -312,15 +326,25 @@ def _draw_ci_row( ) next_z = zorder_base + len(band_alphas) else: + # Standard error-bar shape ([----|----]): a connecting line + # plus a vertical cap at each end, rather than a bare rounded- + # cap line (which reads ambiguously -- easy to mistake for an + # arbitrary line rather than a CI). ax.plot( [lo, hi], [y_row, y_row], color=row_color, lw=line_width, - solid_capstyle="round", zorder=zorder_base, + solid_capstyle="butt", zorder=zorder_base, ) + cap_h = band_height * 0.3 + for x_cap in (lo, hi): + ax.plot( + [x_cap, x_cap], [y_row - cap_h, y_row + cap_h], + color=row_color, lw=line_width, zorder=zorder_base, + ) next_z = zorder_base + 1 if draw_mean: if mean_marker == "line": - tick_h = band_height * 0.7 + tick_h = band_height * tick_scale ax.plot( [mean_val, mean_val], [y_row - tick_h, y_row + tick_h], color=mean_tick_color, lw=1.5, zorder=next_z + 1, @@ -350,7 +374,7 @@ def _draw_ci_row( used_grad_cmp, _ = _draw_ci_row( y + offset, lo0, hi0, mean0, multi_ci_cmp, light_color, _GRADIENT_BAND_ALPHAS, compare_band_height, 2, show_mean, - _PALETTE["text_secondary"], lw * 0.6, + _PALETTE["text_secondary"], lw * 0.6, tick_scale, ) any_gradient_used = any_gradient_used or used_grad_cmp @@ -362,7 +386,7 @@ def _draw_ci_row( used_grad, top_zorder = _draw_ci_row( y_row, lo5, hi5, mean5, multi_ci, color, _GRADIENT_BAND_ALPHAS, primary_band_height, 4, show_mean, - "black", lw, + "black", lw, tick_scale, ) any_gradient_used = any_gradient_used or used_grad @@ -408,6 +432,31 @@ def _draw_ci_row( # ---- gather methods metadata (for title + caption) -------------------- bundle = getattr(report, "full_analysis", None) + + # ---- x-axis padding ---------------------------------------------------- + # barh/plot sticky-edges pin the axis limits exactly to the CI extents, + # so autoscale alone can leave a band hugging the left or right spine. + # Add a margin, but don't pad past the metric's true floor/ceiling (e.g. + # 0% accuracy) -- a CI that already sits at that boundary should still + # hug it, since padding there would draw axis space implying impossible + # values rather than just fixing a cramped-looking plot. + x0, x1 = ax.dataLim.intervalx + data_span = x1 - x0 + if data_span > 0: + pad = 0.06 * data_span + new_x0, new_x1 = x0 - pad, x1 + pad + score_range = getattr(bundle, "resolved_score_range", None) + if score_range is not None: + floor, ceiling = score_range[0] * scale, score_range[1] * scale + new_x0 = max(new_x0, floor) + new_x1 = min(new_x1, ceiling) + elif as_percent: + # No resolved bounds, but percent mode still implies a natural + # [0, 100] floor/ceiling. + new_x0 = max(new_x0, 0.0) + new_x1 = min(new_x1, 100.0) + ax.set_xlim(new_x0, new_x1) + n_inputs = getattr(getattr(bundle, "benchmark", None), "n_inputs", None) alpha = getattr(report, "alpha", 0.05) ci_pct = int(round((1 - alpha) * 100)) From 9909b5995809cf4d54f3f6a7c539886c008b12d6 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 22:36:37 -0400 Subject: [PATCH 022/245] Scope NIG-for-likert rollout down to the one actually-verified path a0d8c6b wired NIG into the auto default for ALL likert routing: single- and multi-run pairwise comparisons, single- and multi-run marginal (robustness) CIs, and the k>=3 simultaneous/family-wise (Sidak/joint- bootstrap-widened) construction. Only the first of those -- single-run pairwise -- was actually verified this session: - Multi-run/seeded pairwise: one nested-mode check looked consistent, but never went through the same adversarial stress-testing (boundary- clipping bias, detector edge cases) the single-run path did before being trusted. - Marginal/robustness CIs (the "nig"/"nig_nested" single-sample case in core/variance.py's robustness_metrics()): never tested directly at all. Checking simulations/harness/cases/ci_single.py's own separate reimplementation earlier this session is not a substitute for testing this actual production code path. - The k>=3 simultaneous-CI router (_simultaneous_cis_router): only ever tested with NIG's OLD, buggy prior (the very first standalone investigation, before core.paired._NIG_PAIRED_DIFF_B0 existed), never re-validated post-fix. Fixes: - config.py: AUTO_ANALYZE_METHOD_TABLE's "likert" row reverts robustness_method_single_run/seeded to "logit_t" (was "nig"/ "nig_nested"). resolve_auto_analyze_methods now falls the resolved pairwise method back from "nig" to "logit_t" whenever seeded=True, so the table itself can keep listing pairwise_method="nig" (still correct for the single-run case) without that leaking into multi-run. Reason text rewritten to state precisely what is and isn't covered. - core/paired.py: _simultaneous_cis_router's data_kind resolution drops the "likert" branch entirely -- bounded numeric data (discrete or continuous) always resolves to "bounded_01" there now, so the k>=3 construction always widens logit-t's formula. eval_type stays a parameter on all_pairwise()/_simultaneous_cis_router (unused for this purpose right now) rather than being ripped out, so re-enabling this later is a small, localized change instead of re-plumbing analyze() again. Docstrings on both functions corrected to say this explicitly instead of implying full likert support. - core/router.py: eval_type's docstring on analyze() states the actual current scope. The auto-detection UserWarning no longer claims "evalstats is using NIG-based methods" unconditionally -- it now says NIG applies to single-run pairwise comparisons specifically, and that other analyses on the same data still use logit-t. Re-verified end-to-end after rescoping: resolve_auto_analyze_methods returns ("nig","logit_t") for likert+single-run and ("logit_t","logit_t") for likert+seeded; analyze() on single-run likert data gives pairwise test_method="paired NIG", multi-run gives "paired logit-t"; a k=3 simultaneous-CI call on likert data shows per-pair NIG point estimates but a logit-t-widened simultaneous_ci_method="boot" construction, source- verified directly against _simultaneous_cis_router's body. Full existing suite (320 tests, same set as a0d8c6b) still passes, including the one expected warning in test_simultaneous_ci.py, now with corrected wording. Co-Authored-By: Claude Sonnet 5 --- evalstats/config.py | 50 +++++++++++++++---- evalstats/core/paired.py | 105 ++++++++++++++++----------------------- evalstats/core/router.py | 37 ++++++++------ 3 files changed, 106 insertions(+), 86 deletions(-) diff --git a/evalstats/config.py b/evalstats/config.py index a9bd933..eab8855 100644 --- a/evalstats/config.py +++ b/evalstats/config.py @@ -166,8 +166,8 @@ class AutoAnalyzeRule: AutoAnalyzeRule( data_kind="likert", max_n=None, pairwise_method="nig", - robustness_method_single_run="nig", - robustness_method_seeded="nig_nested", + robustness_method_single_run="logit_t", + robustness_method_seeded="logit_t", reason=( "Discrete/ordinal bounded data (a Likert scale, an integer " "percentage grade, or anything else with a real quantization " @@ -175,7 +175,9 @@ class AutoAnalyzeRule: "an explicit eval_type='likert', or auto-detected via " "detect_quantization_step() (core/resampling.py) when no " "eval_type is given, with a UserWarning explaining the switch. " - "Uses NIG rather than logit-t: a paired diff of two highly " + "Uses NIG rather than logit-t for the PAIRWISE, single-run case " + "ONLY (see resolve_auto_analyze_methods -- pairwise falls back " + "to logit_t when seeded=True): a paired diff of two highly " "correlated Likert arms can lose real variance to rounding " "cancellation (most items round identically in both arms, only " "boundary-adjacent items differ), which at small N can leave " @@ -193,12 +195,32 @@ class AutoAnalyzeRule: "a paired diff's rescale span, which is twice as wide -- see " "core.paired._NIG_PAIRED_DIFF_B0), so the historical comparison " "that dropped NIG likely made it look needlessly conservative " - "compared to logit-t. Re-validated post-fix (reps=300, n=10-500, " - "icc=0.01-0.95): NIG beats logit-t on likert score at every N up " - "to 500 (17% better at n=10, converging to a tie by n=500), " - "while logit-t remains the better (if only marginally) choice " - "for genuinely continuous 'bounded_01' data, where NIG's extra " - "conservatism buys no corresponding robustness." + "compared to logit-t. Re-validated post-fix, single-run only " + "(reps=300, n=10-500, icc=0.01-0.95): NIG beats logit-t on " + "likert score at every N up to 500 (17% better at n=10, " + "converging to a tie by n=500).\n\n" + "DELIBERATELY NOT yet extended to: (1) seeded/multi-run " + "pairwise -- one nested-mode check looked consistent, but " + "hasn't had the same adversarial stress-testing (boundary-" + "clipping bias, detector edge cases) the single-run path went " + "through before being trusted; (2) marginal/robustness CIs " + "(the 'nig'/'nig_nested' single-sample case in " + "core/variance.py's robustness_metrics()) -- never directly " + "tested; a check of a *different*, harness-only " + "reimplementation (simulations/harness/cases/ci_single.py) " + "isn't a substitute for testing this actual production code " + "path; (3) the simultaneous/family-wise (k>=3) Sidak/joint-" + "bootstrap-widened construction in " + "core.paired._simultaneous_cis_router -- only tested with the " + "OLD, buggy prior (before core.paired._NIG_PAIRED_DIFF_B0), " + "never re-validated post-fix, so that router still widens " + "logit-t's formula for likert data too (see its own " + "docstring). Widen this rule's scope only as each of these " + "gets its own dedicated validation -- logit-t remains the " + "default everywhere NIG hasn't been proven, including for " + "genuinely continuous 'bounded_01' data, where NIG's extra " + "conservatism buys no corresponding robustness in the first " + "place." ), ), AutoAnalyzeRule( @@ -248,7 +270,15 @@ def resolve_auto_analyze_methods( if rule.max_n is not None and n >= rule.max_n: continue robustness = rule.robustness_method_seeded if seeded else rule.robustness_method_single_run - return rule.pairwise_method, robustness + pairwise = rule.pairwise_method + if pairwise == "nig" and seeded: + # NIG's paired-diff fix is only validated for single-run + # (non-seeded) data so far -- see AUTO_ANALYZE_METHOD_TABLE's + # "likert" row's reason for exactly what's and isn't verified. + # Multi-run pairwise falls back to logit_t until that path gets + # its own dedicated validation. + pairwise = "logit_t" + return pairwise, robustness raise AssertionError( f"no AUTO_ANALYZE_METHOD_TABLE rule matched data_kind={data_kind!r}, n={n}" ) diff --git a/evalstats/core/paired.py b/evalstats/core/paired.py index dce6b43..8936935 100644 --- a/evalstats/core/paired.py +++ b/evalstats/core/paired.py @@ -1967,27 +1967,26 @@ def _simultaneous_cis_router( else joint bootstrap with an effective alpha (``"boot"``, :func:`_joint_bootstrap_scaled_simultaneous_cis`). Both widen whichever canonical closed-form pairwise CI formula the data resolves to -- Tango - for binary data, NIG for discrete/ordinal bounded data (a Likert scale, - an integer percentage grade), logit-t for genuinely continuous bounded - data, plain t-interval as the bounds-agnostic fallback for everything - else -- the same per-data-kind formula - :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE` already uses for the - *non*-simultaneous pairwise CI, so Sidak/boot always widen the formula - that would otherwise have been shown, regardless of which resampling - *method* (bootstrap, bca, ...) the point estimate itself used. - - ``eval_type`` distinguishes discrete/ordinal ("likert") from genuinely - continuous data within a known bounded range -- both would otherwise - collapse to the same "bounded_01" treatment, but they need different CI - formulas (see the "likert" row of AUTO_ANALYZE_METHOD_TABLE for why). - When ``None`` (default) and the data has a known range but no explicit - ``eval_type``, this auto-detects via - :func:`~evalstats.core.resampling.detect_quantization_step` and emits a - ``UserWarning`` if it switches to the likert treatment -- pass - ``eval_type`` explicitly to silence that warning either way, or when - called from :func:`~evalstats.core.router.analyze` (which does its own - detection once and passes the resolved value down here), to avoid - re-detecting and re-warning redundantly. + for binary data, logit-t for any bounded numeric range (*score_range*), + plain t-interval as the bounds-agnostic fallback for everything else -- + the same per-data-kind formula :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE` + already uses for the *non*-simultaneous pairwise CI on genuinely + continuous data, so Sidak/boot always widen the formula that would + otherwise have been shown, regardless of which resampling *method* + (bootstrap, bca, ...) the point estimate itself used. + + ``eval_type`` is accepted but currently NOT used to change which + ci_func gets widened here: bounded numeric data always uses logit-t in + this function, even when it's discrete/ordinal (Likert-scale) data + that :func:`pairwise_differences`'s own ``method="nig"`` path (and + :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE`'s "likert" row, + single-run pairwise only) would use NIG for instead. That's + deliberate, not an oversight -- the k>=3 construction built here has + only ever been tested with NIG's OLD, buggy prior (before + ``_NIG_PAIRED_DIFF_B0`` fixed it), never re-validated post-fix, so it + isn't trusted yet. The parameter stays so callers/``analyze()`` don't + need reverting too, and so wiring NIG back in here is a small, + localized change once that validation exists. Historical note: this used to default unconditionally to Bonferroni (with the studentized bootstrap max-T method as the sole opt-in @@ -2052,31 +2051,22 @@ def _simultaneous_cis_router( if is_binary: data_kind = "binary" elif score_range is not None: - if eval_type == "likert": - data_kind = "likert" - elif eval_type == "continuous": - data_kind = "bounded_01" - else: - from .resampling import detect_quantization_step - step = detect_quantization_step(scores) - if step is not None: - data_kind = "likert" - warnings.warn( - f"Bounded numeric data was auto-detected as discrete/" - f"ordinal (grid step={step:g} within range {score_range}), " - f"so evalstats is using NIG-based methods calibrated for " - f"Likert-style/discrete data instead of the continuous " - f"default (logit-t) for the simultaneous CI. Pass " - f"eval_type='likert' to silence this warning, or " - f"eval_type='continuous' if this discreteness is " - f"coincidental.", - UserWarning, - stacklevel=4, - ) - else: - data_kind = "bounded_01" + data_kind = "bounded_01" else: data_kind = "unbounded" + # NOTE: eval_type is deliberately NOT consulted here (unlike + # pairwise_differences()'s method="nig" path, which IS validated + # for single-run pairwise likert data -- see + # config.AUTO_ANALYZE_METHOD_TABLE's "likert" row). The k>=3 + # simultaneous/family-wise construction this function builds + # (Sidak/joint-bootstrap-widened) has only ever been tested with + # NIG's OLD, buggy prior (before core.paired._NIG_PAIRED_DIFF_B0 + # fixed it) -- never re-validated post-fix -- so likert data still + # gets the same logit-t-based ci_func as genuinely continuous + # "bounded_01" data here, until that gets its own dedicated + # validation. eval_type stays a parameter (rather than being + # removed) so callers/analyze() don't need reverting too, and so + # re-enabling this is a small, localized change once ready. resolved = prefer if prefer == "auto": @@ -2088,13 +2078,6 @@ def _simultaneous_cis_router( if data_kind == "binary": ci_func = tango_paired_ci_from_diffs - elif data_kind == "likert": - diff_span = score_range[1] - score_range[0] - diff_lo, diff_hi = -diff_span, diff_span - _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) - - def ci_func(diffs, alpha, _lo=diff_lo, _hi=diff_hi, _fn=_nig_paired): - return rescaled_ci(_fn, diffs, alpha, _lo, _hi) elif data_kind == "bounded_01": diff_span = score_range[1] - score_range[0] diff_lo, diff_hi = -diff_span, diff_span @@ -2196,16 +2179,16 @@ def all_pairwise( the ``"auto"`` (default) table lookup: ``"sidak"``, ``"boot"``, ``"max_t"``, or ``"bonferroni"``. eval_type : "likert", "continuous", or None - Distinguishes discrete/ordinal bounded data (a Likert scale, an - integer percentage grade) from genuinely continuous bounded data -- - both share the same ``score_range``-known-bounds treatment - otherwise, but need different CI formulas (NIG vs logit-t; see - ``config.AUTO_ANALYZE_METHOD_TABLE``'s "likert" row). When ``None`` - (default), auto-detects via - :func:`~evalstats.core.resampling.detect_quantization_step` and - warns if it switches to the likert treatment. Only relevant when - ``score_range`` is given (or an exact ``[0, 1]`` range is - detected) and ``method``/point-estimate data isn't binary. + Accepted for forward-compatibility with :func:`analyze`, but + currently has NO effect on the simultaneous CI this function + computes when ``simultaneous_ci=True`` (the default) -- see + :func:`_simultaneous_cis_router`'s docstring for why (the k>=3 + widened construction hasn't been validated for NIG post-fix, so it + always widens logit-t for any bounded numeric range regardless of + ``eval_type``). It DOES matter for the individual per-pair CI when + ``method="nig"`` is requested explicitly, or resolved via + ``method="auto"`` + single-run likert data (see + ``config.AUTO_ANALYZE_METHOD_TABLE``) -- that path is validated. omnibus : bool When ``True``, run the Friedman omnibus test (with Nemenyi post-hoc) alongside the pairwise comparisons. Requires k ≥ 3. Defaults to diff --git a/evalstats/core/router.py b/evalstats/core/router.py index c728890..4f9dcc3 100644 --- a/evalstats/core/router.py +++ b/evalstats/core/router.py @@ -277,13 +277,16 @@ def analyze( Only used with ``method='auto'`` and a known/declared ``score_range``. Distinguishes discrete/ordinal data (a Likert scale, an integer percentage grade) from genuinely continuous - data within the same bounded range -- they need different CI - formulas (NIG vs logit-t; see ``config.AUTO_ANALYZE_METHOD_TABLE``'s - "likert" row for why). When omitted (default), evalstats - auto-detects discreteness from the data's own quantization grid - and emits a ``UserWarning`` if it switches to the Likert - treatment -- pass this explicitly to silence that warning either - way. + data within the same bounded range. When omitted (default), + evalstats auto-detects discreteness from the data's own + quantization grid and emits a ``UserWarning`` if it switches to + the Likert treatment -- pass this explicitly to silence that + warning either way. Currently this only changes the SINGLE-RUN + pairwise-comparison CI (NIG instead of logit-t) -- see + ``config.AUTO_ANALYZE_METHOD_TABLE``'s "likert" row for exactly + what is and isn't yet covered (multi-run pairwise, marginal CIs, + and the k>=3 simultaneous-CI construction all still use logit-t + for likert data, pending their own dedicated validation). When omitted, evalstats always prints a ``UserWarning`` announcing what it assumed and which method it picked as a result: @@ -909,14 +912,18 @@ def _analyze_single( warnings.warn( f"Bounded numeric evaluation data was auto-detected " f"as discrete/ordinal (grid step={step:g} within " - f"range {resolved_score_range}), so evalstats is " - "using NIG-based methods calibrated for Likert-" - "style/discrete data instead of the continuous " - "default (logit-t). Pass eval_type='likert' " - "explicitly to silence this warning, or " - "eval_type='continuous' if this discreteness is " - "coincidental (e.g. a metric that happens to only " - "take a few values in your sample).", + f"range {resolved_score_range}). For single-run " + "pairwise comparisons, evalstats uses NIG (validated " + "as better-calibrated than logit-t there for this " + "kind of data); other analyses on this data " + "(marginal CIs, multi-run pairwise comparisons) " + "still use logit-t, the same as continuous data, " + "pending their own validation -- see " + "config.AUTO_ANALYZE_METHOD_TABLE's 'likert' row. " + "Pass eval_type='likert' explicitly to silence this " + "warning, or eval_type='continuous' if this " + "discreteness is coincidental (e.g. a metric that " + "happens to only take a few values in your sample).", UserWarning, stacklevel=2, ) From 1dd81ead5af6df75b2479591692bae74af2056ba Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Tue, 11 Aug 2026 22:42:02 -0400 Subject: [PATCH 023/245] Add grouped two-factor forest plot via factors= on plot_ci_forest - New factors= param on plot_ci_forest/ComparisonResult.plot(): "auto" (default) upgrades to a grouped (model, prompt) view whenever a report genuinely has both axes with >1 level, "model"/"prompt" collapses to the marginal view over one axis, and ["model","prompt"] (or reversed) picks the grouped view explicitly with list order setting outer/inner grouping. Falls back to today's single-axis rendering unchanged otherwise. - New ComparisonResult.model_labels/.prompt_labels/.as_view() to support it. - color_rule default changed from "tier" to "auto" (tier for flat views, factor for grouped, since "tier" has no single meaning across a grid); color_rule="tier" on a grouped view now raises a clear error instead of silently doing something wrong. - compare_to and show_ci_bracket aren't supported together with a grouped view yet -- both raise clear errors. - Refactored _draw_ci_row/x-axis padding/legend-and-trim out of the single-axis code path into shared module-level helpers so the new grouped renderer (_plot_ci_forest_grouped) reuses the same visual language instead of duplicating it. - 16 new tests covering grouping, reversal, marginal overrides, and the new error paths. --- evalstats/api.py | 51 +++ evalstats/vis/forest.py | 589 ++++++++++++++++++++++++++--------- tests/test_ci_forest_plot.py | 141 +++++++++ 3 files changed, 641 insertions(+), 140 deletions(-) diff --git a/evalstats/api.py b/evalstats/api.py index 3aa53e6..e2d8c1b 100644 --- a/evalstats/api.py +++ b/evalstats/api.py @@ -421,6 +421,57 @@ def best_pairs(self) -> Optional[list]: top_pairs.append((parts[0], parts[1])) return top_pairs or None + @property + def model_labels(self) -> Optional[list]: + """Model-axis labels for a two-factor (model, prompt) comparison, or ``None``. + + Populated under the same condition as :attr:`cross_model` — this is + that bundle's model axis, in its original (pre-sort) order. + """ + if not isinstance(self._analysis, MultiModelBundle): + return None + return list(self._analysis.benchmark.model_labels) + + @property + def prompt_labels(self) -> Optional[list]: + """Prompt/template-axis labels for a two-factor comparison, or ``None``. + + Populated under the same condition as :attr:`cross_model` — this is + that bundle's template axis, in its original (pre-sort) order. + """ + if not isinstance(self._analysis, MultiModelBundle): + return None + return list(self._analysis.benchmark.template_labels) + + def as_view(self, factor: Literal["model", "prompt"]) -> "ComparisonResult": + """Return this two-factor comparison collapsed onto a single axis. + + E.g. ``result.as_view("model")`` averages over prompts to compare + models; ``result.as_view("prompt")`` averages over models to compare + prompts. Only valid for a two-factor comparison (see + :attr:`cross_model`) — raises otherwise. + """ + if not isinstance(self._analysis, MultiModelBundle): + raise ValueError( + "as_view() requires a two-factor comparison (built with " + "compare(..., factors=['model', 'prompt']), or " + "factors='model'/'prompt' when both columns are present)." + ) + view_map = {"model": "model_level", "prompt": "template_level"} + if factor not in view_map: + raise ValueError(f"factor={factor!r} must be 'model' or 'prompt'.") + return ComparisonResult( + self._analysis, + factors=self._factors, + metric=self._metric, + baseline=self._baseline, + alpha=self._alpha, + filtered_df=self._df, + _mmb_view=view_map[factor], + min_meaningful_diff=self._min_meaningful_diff, + show_rank_probabilities=self._show_rank_probabilities, + ) + @property def pareto_status(self) -> Optional[dict]: """Per-entity three-state Pareto classification, or ``None``. diff --git a/evalstats/vis/forest.py b/evalstats/vis/forest.py index 61f984d..6d51137 100644 --- a/evalstats/vis/forest.py +++ b/evalstats/vis/forest.py @@ -19,7 +19,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal, Optional +from typing import TYPE_CHECKING, Literal, Optional, Union import matplotlib.colors as mcolors import matplotlib.pyplot as plt @@ -70,6 +70,393 @@ def _lighten(color, amount: float) -> tuple: return (r + (1 - r) * amount, g + (1 - g) * amount, b + (1 - b) * amount) +def _draw_ci_row( + ax, y_row: float, lo: float, hi: float, mean_val: float, + multi_ci_row: Optional[dict], row_color, band_alphas: tuple, + band_height: float, zorder_base: int, draw_mean: bool, + mean_tick_color: str, line_width: float, mean_marker: str, + tick_scale: float = 0.7, +) -> tuple[bool, int]: + """Draw one CI row (gradient bands, falling back to a single line + when multi_ci_row is None) plus an optional mean marker. Returns + (used_gradient, next_free_zorder).""" + used_gradient = False + if multi_ci_row is not None: + used_gradient = True + # Widest CI (99%, smallest alpha) drawn first/lowest zorder, + # narrowest (68%, largest alpha) drawn last/highest zorder -- + # same "inner band wins" convention as the terminal's + # _gradient_interval_line, via z-order layering instead of + # character replacement. + sorted_alphas = sorted(multi_ci_row.keys()) + for band_i, a in enumerate(sorted_alphas): + lo_a, hi_a = multi_ci_row[a] + band_alpha = band_alphas[min(band_i, len(band_alphas) - 1)] + ax.barh( + y_row, width=hi_a - lo_a, left=lo_a, + height=band_height, + color=row_color, alpha=band_alpha, + edgecolor="none", zorder=zorder_base + band_i, + ) + next_z = zorder_base + len(band_alphas) + else: + # Standard error-bar shape ([----|----]): a connecting line + # plus a vertical cap at each end, rather than a bare rounded- + # cap line (which reads ambiguously -- easy to mistake for an + # arbitrary line rather than a CI). + ax.plot( + [lo, hi], [y_row, y_row], + color=row_color, lw=line_width, + solid_capstyle="butt", zorder=zorder_base, + ) + cap_h = band_height * 0.3 + for x_cap in (lo, hi): + ax.plot( + [x_cap, x_cap], [y_row - cap_h, y_row + cap_h], + color=row_color, lw=line_width, zorder=zorder_base, + ) + next_z = zorder_base + 1 + if draw_mean: + if mean_marker == "line": + tick_h = band_height * tick_scale + ax.plot( + [mean_val, mean_val], [y_row - tick_h, y_row + tick_h], + color=mean_tick_color, lw=1.5, zorder=next_z + 1, + ) + else: + ax.scatter( + [mean_val], [y_row], + color=row_color, s=55, zorder=next_z + 1, + edgecolor="white", linewidth=0.6, + ) + next_z += 1 + return used_gradient, next_z + + +def _apply_x_padding(ax, bundle, scale: float, as_percent: bool) -> None: + """Pad the x-axis so CI bands don't hug the left/right spine. + + barh/plot sticky-edges pin the axis limits exactly to the CI extents, + so autoscale alone can leave a band hugging the left or right spine. + Add a margin, but don't pad past the metric's true floor/ceiling (e.g. + 0% accuracy) -- a CI that already sits at that boundary should still + hug it, since padding there would draw axis space implying impossible + values rather than just fixing a cramped-looking plot. + """ + x0, x1 = ax.dataLim.intervalx + data_span = x1 - x0 + if data_span <= 0: + return + pad = 0.06 * data_span + new_x0, new_x1 = x0 - pad, x1 + pad + score_range = getattr(bundle, "resolved_score_range", None) + if score_range is not None: + floor, ceiling = score_range[0] * scale, score_range[1] * scale + new_x0 = max(new_x0, floor) + new_x1 = min(new_x1, ceiling) + elif as_percent: + # No resolved bounds, but percent mode still implies a natural + # [0, 100] floor/ceiling. + new_x0 = max(new_x0, 0.0) + new_x1 = min(new_x1, 100.0) + ax.set_xlim(new_x0, new_x1) + + +def _place_legend_and_trim(fig, ax, legend_handles: list, own_fig: bool) -> None: + """Place the combined legend outside the axes, then trim the canvas to + hug its actual rendered width instead of leaving unused margin (matters + for pasting a figure straight into a paper without manual cropping). + """ + if legend_handles: + ax.legend( + handles=legend_handles, + fontsize=7.5, loc="center left", bbox_to_anchor=(1.01, 0.5), + frameon=True, facecolor="white", + edgecolor=_PALETTE["grid"], framealpha=0.95, + ncol=1, + ) + + if not own_fig: + return + fig.tight_layout() + if not legend_handles: + return + fig.subplots_adjust(right=0.78) + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + legend = ax.get_legend() + legend_px = legend.get_window_extent(renderer=renderer) + fig_px_width = fig.get_window_extent(renderer=renderer).width + pad_px = 8 + excess_px = fig_px_width - (legend_px.x1 + pad_px) + if excess_px > 1: + dpi = fig.dpi + old_width_in, height_in = fig.get_size_inches() + new_width_in = old_width_in - excess_px / dpi + if new_width_in > 0: + # Rescale horizontal subplot fractions so the axes and + # legend keep their exact pixel position/size on the + # narrower canvas -- only the wasted margin is trimmed. + sp = fig.subplotpars + scale = old_width_in / new_width_in + fig.set_size_inches(new_width_in, height_in) + fig.subplots_adjust( + left=min(0.99, sp.left * scale), + right=min(1.0, sp.right * scale), + ) + + +def _resolve_factors(report, factors): + """Decide which entities plot_ci_forest should render. + + Returns either ``("flat", resolved_report)`` -- use the existing + single-axis rendering on *resolved_report* (which may be *report* + itself, or a marginal view of it via ``.as_view()``) -- or + ``("grouped", (outer, inner))`` to render the two-factor grouped view. + """ + model_labels = getattr(report, "model_labels", None) + prompt_labels = getattr(report, "prompt_labels", None) + is_two_factor = ( + model_labels is not None and prompt_labels is not None + and len(model_labels) > 1 and len(prompt_labels) > 1 + ) + if factors is None or factors == "auto": + if is_two_factor: + return "grouped", ("model", "prompt") + return "flat", report + if isinstance(factors, str): + if factors not in ("model", "prompt"): + raise ValueError( + f"factors={factors!r} is not 'auto', 'model', 'prompt', or a " + "two-item list like ['model', 'prompt']." + ) + if model_labels is None: + raise ValueError( + f"factors={factors!r} requires a two-factor comparison " + "(built with compare(..., factors=['model', 'prompt']), or " + "factors='model'/'prompt' when both columns are present) -- " + "this report has no (model, prompt) structure to select from." + ) + return "flat", report.as_view(factors) + factors_list = list(factors) + if len(factors_list) != 2 or set(factors_list) != {"model", "prompt"}: + raise ValueError( + f"factors={factors!r} must be 'auto', 'model', 'prompt', or a " + "two-item permutation of ['model', 'prompt']." + ) + if not is_two_factor: + raise ValueError( + f"factors={factors!r} requested a grouped two-factor view, but " + "this report doesn't have both a model and a prompt axis with " + "more than one level each." + ) + return "grouped", tuple(factors_list) + + +# Layout constants for the grouped two-factor view -- tuned separately from +# the single-axis defaults above since a grouped plot packs many more rows: +# sibling rows within a group sit closer together (ROW_SPACING < 1.0) and +# gradient bands are thinner (band_height below), with a smaller gap +# (GROUP_GAP) between groups than a full row -- large enough to read as a +# break, small enough not to waste vertical space. +_GROUPED_ROW_SPACING = 0.8 +_GROUPED_GROUP_GAP = 0.35 +_GROUPED_BAND_HEIGHT = 0.32 + + +def _plot_ci_forest_grouped( + report, outer: str, inner: str, *, + reference_line: Optional[float], sort_by: str, as_percent: bool, + style: str, color_rule: str, show_mean: bool, mean_marker: str, + figsize: Optional[tuple[float, float]], title: Optional[str], ax, +) -> "Figure": + """Grouped two-factor forest plot: one row per (model, prompt) pair, + clustered by *outer* (shared colour + shared alternating background), + with *inner* as sub-rows within each cluster. See plot_ci_forest's + ``factors=`` docs. + """ + cross = report.cross_model + flat_labels = list(cross.benchmark.template_labels) + rob = cross.robustness + scale = 100.0 if as_percent else 1.0 + + # Flat labels are always " /