diff --git a/.gitignore b/.gitignore index a706a39..f4c281e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ simulations/out/* !simulations/out/judge_bias_privacy_judge_items.csv !simulations/out/judge_bias_iclr_metareview.csv !simulations/out/judge_bias_iclr_metareview_items.csv +!simulations/out/results_why_ppi_shrink_1_over_0.md +!simulations/out/appstore_scenario_reviews.csv +!simulations/out/appstore_scenario_judge_scores.csv .agent-study-venv-full/ .agent-study-venv-baseline/ .agent-study-venv-runner/ @@ -38,4 +41,9 @@ simulations/harness/methods_table.tex simulations/harness/revise_latex_tables.py simulations/PPI_TESTBED_REVIEW.md coefs.csv -examples/.cache/ \ No newline at end of file +examples/.cache/ +lit_review/api_key.txt +simulations/papers + +# Simulation and LaTeX scratch logs (drift.log, texput.log, ...) +*.log diff --git a/README.md b/README.md index 7d8f0e5..1502f4d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,24 @@ # evalstats -Rigorous statistical analysis for LLM evaluations, from model and prompt comparisons to statistical tests resilient to LLM judge bias, including in small sample data regimes. +[![PyPI](https://img.shields.io/pypi/v/evalstats)](https://pypi.org/project/evalstats/) +[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) +[![Python](https://img.shields.io/badge/python-3.9%2B-blue)](pyproject.toml) + +Rigorous statistical analysis for LLM evaluations — from model and prompt comparisons to statistical tests resilient to LLM judge bias, including in small-sample data regimes. `evalstats` helps you answer questions like: - - Is Prompt A actually better than Prompt B, or just slightly luckier on this dataset? - - Does Model A beat Model B, or only under a specific prompt phrasing? - - How sensitive is model performance to prompt wording? - - Are my performance differences large enough to be meaningful, or just noise? - - How stable are scores across runs, evaluators, or inputs? - - Can I trust my LLM-judge scores, or do they need correcting against human labels first? +- Is Prompt A actually better than Prompt B, or just slightly luckier on this dataset? +- Does Model A beat Model B, or only under a specific prompt phrasing? +- Are my performance differences large enough to be meaningful, or just noise? +- How stable are scores across runs, evaluators, or inputs? +- Can I trust my LLM-judge scores, or do they need correcting against human labels first? -You give `evalstats` your benchmark data, and it runs statistically appropriate analyses that quantify uncertainty and provide confidence bounds on your claims. It does this in two main ways: +You give `evalstats` your benchmark data, and it runs statistically appropriate analyses that quantify uncertainty and provide confidence bounds on your claims, in two main ways: -- **Comparisons**: Comparing models, prompts, or both at once (or any other thing you're comparing, like agent harnesses), and get 95% confidence intervals, pairwise significance tests, and multi-run sensitivity analyses. `evalstats` guides you toward best practices and choose well-calibrated methods and procedures by default, backed by simulations, and was built specifically to fill the gap of statistical knowledge for small-sample size datasets N<100; it will output stats as long as there are at least 15 samples. See [Statistics](#statistics). -- **PPI-corrected inference**: using a small set of human labels to correct bias in noisy LLM-judge scores, so your means, confidence intervals, and hypothesis tests p-values are accurately calibrated in the face of LLM judge bias. This builds on prediction-powered inference (PPI). See [PPI-Corrected Inference](#ppi-corrected-inference-means-cis-and-tests). +- **Comparisons**: compare models, prompts, or both at once, and get 95% confidence intervals, pairwise significance tests, and multi-run sensitivity analyses. `evalstats` picks well-calibrated methods by default, backed by simulations, and was built specifically for small-sample datasets (N<100) — it will output stats down to 15 samples. See [Recommended Methods](#recommended-methods). +- **PPI-corrected inference**: use a small set of human labels to correct bias in noisy LLM-judge scores, so your means, confidence intervals, and hypothesis-test p-values stay calibrated in the face of LLM judge bias. See [PPI-Corrected Inference](#ppi-corrected-inference-means-cis-and-tests). -In particular, scientists can use our PPI-corrected statistical tests to analyze data for **mixed human-AI subject studies**, where some observations are human-labeled and the rest are graded by an LLM judge. Use `evalstats.tests` directly for LLM-judge-bias-corrected versions of: +Scientists can use our PPI-corrected statistical tests for **mixed human-AI subject studies**, where some observations are human-labeled and the rest are graded by an LLM judge. `evalstats.tests` gives you LLM-judge-bias-corrected versions of: - t-test (`ttest`, independent or paired; Welch's by default, or Student's equal-variance via `equal_var=True`) - Mann–Whitney U (`mannwhitney`) @@ -24,101 +27,94 @@ In particular, scientists can use our PPI-corrected statistical tests to analyze - Friedman test (`friedman`, repeated-measures rank-based) - Kruskal-Wallis (`kruskalwallis`, independent-groups rank-based) -As long as the items for human labeling were sampled at random from the full dataset, p-values will stay calibrated even when the LLM judge is biased or miscalibrated. These corrections are validated via extensive Monte Carlo simulations (see `simulations/harness`). To the best of our knowledge, `evalstats` provides the only known implementations of PPI-corrected rank-based nonparametric tests like Wilcoxon. - -As well, +As long as items for human labeling were sampled at random from the full dataset, p-values stay calibrated even when the LLM judge is biased or miscalibrated — validated via extensive Monte Carlo simulations (see [`simulations/harness`](simulations/harness)). To the best of our knowledge, `evalstats` provides the only known implementations of PPI-corrected rank-based nonparametric tests like Wilcoxon. > [!IMPORTANT] -> We are actively building out this project, both the website/guide and the package. -> Aside from the package itself, there is a "learning" guide in `website/` which I am building out and will return to after writing up the simulations. This will include simulation- and research-backed examples of statistics for LLM evals, as well as example code (which will, obviously, use `evalstats`, but the lessons hold regardless of implementation). -> If there's something you'd like to see, or guidance on a specific topic, let us know -> by raising an Issue. - -## Sample output - -Running `es.compare(evaldata, factors="prompt")` and then `result.summary()` prints a full statistical report to the terminal, including confidence interval line plots, pairwise comparisons between prompt templates, and per-input stability across runs (how stable the model is across multiple runs for the same input). Below is example excerpt from an analysis of a 4-template sentiment-classification benchmark (GPT-4.1-nano, 27 inputs, 3 runs, 3 evaluators): +> We are actively building out this project. A paper with the full methodology and simulation-backed validation behind every default is forthcoming — see [Recommended Methods](#recommended-methods) and [Citation](#citation). In the meantime, the [Stats for LLM Evals guide](https://statsforevals.com/) covers the same material in web form. If there's something you'd like to see, let us know by raising an Issue. + +## Contents + +- [Installation](#installation) +- [Quick start](#quick-start) +- [See it in action](#see-it-in-action) +- [Recommended Methods](#recommended-methods) +- [Python API](#python-api) +- [PPI-Corrected Inference](#ppi-corrected-inference-means-cis-and-tests) +- [CLI Reference](#cli-reference) +- [Examples](#examples) +- [Mixed effects models (LMM)](#mixed-effects-models-lmm) +- [Reproducibility: Monte Carlo simulations](#reproducibility-monte-carlo-simulations) +- [Motivation](#motivation) +- [Development and Contributions](#development-and-contributions) +- [Citation](#citation) +- [License](#license) + +## Installation -![Example terminal output](docs/example-output.png) - -From this output, we can see that Minimal and Instructive are the most promising candidates, but it is statistically unclear which is better. We also see that Chain-of-thought gives the least consistent outputs across multiple runs for the same inputs, compared to the other methods. +```bash +pip install evalstats +``` -In the most recent version of `evalstats`, there's also helpful colors to help you see -this information. For instance, comparing models and prompts at the same time, -`evalstats` shows a 4-way tie between four combinations of model-prompt: +For Excel (`.xlsx`) input support: `pip install "evalstats[xlsx]"`. For every optional extra (including mixed-effects/LMM support): `pip install "evalstats[all]"`. -![Example terminal output with colors](docs/terminal-output-example.jpg) +## Quick start -You can also plot within notebook environments (although this feature is being actively built out over time and the least developed at the moment). The `plot_point_estimates` function produces a chart showing each template's absolute mean score with marginal confidence intervals: +From the command line, `evalstats` can read a CSV or Excel file directly and print a statistical summary: -![Mean advantage plot](docs/mean_advantage.png) +```bash +evalstats analyze results.csv +``` -## Statistics +The input file should have a prompt/template column, an item/input column, and a score column (model, run, and evaluator columns are optional) — see the [column alias table](#python-api) below, or run `evalstats analyze --help` for the full option and alias list. -The specific statistical tests that `evalstats.compare()` runs (via the lower-level `analyze()` engine underneath it) are: +From Python, the main entry point is `load_from()` + `compare()`: -- **All pairwise prompt comparisons (paired by input)** via `all_pairwise(...)`: - - Computes mean or median difference (mean by default), bootstrapped 95% confidence interval, and p-value for every prompt template pair. - - Comparison method defaults to `method="auto"`: - - **Smoothed bootstrap with a Gaussian KDE** (`method="smooth_bootstrap"`) in situations of non-binary data. It has been verified in our simulations that for eval-type data and small sample sizes especially, smoothed is superior to the other bootstrap methods considered (percentile, BCa, Bayesian). - - **Bayesian pairwise from [`bayes_evals`](https://github.com/sambowyer/bayes_evals/tree/main) and McNemar's test**: Default methods for binary scores (0 or 1 only). Our simulations showed Bayesian pairwise was superior to bootstrap at small N. Note that Bayesian methods should technically be called credible intervals, but they estimate the confidence interval very closely. - - Multiple-comparisons correction for p-values (defaults to **Benjamini–Hochberg (fdr_bh)**). - - Also reports Wilcoxon signed-rank test p-value, in case you need it for people familiar with that test, although p-values from bootstrapped CIs are more robust +```python +import pandas as pd +import evalstats as es -- **Bootstrap rank distribution** via `bootstrap_ranks(...)`: - - Estimates each prompt template’s `P(best)` and expected rank among the full list of prompt templates. +df = pd.read_csv("results.csv") # columns: prompt, item, score (model optional) -- **Point estimates** via `robustness_metrics(...)`: - - Descriptive stats like mean, median, std, CV, IQR, CVaR-10, and key percentiles. - - Marginal confidence intervals on absolute means/medians. +evaldata = es.load_from(df) +evaldata.summary() # inspect detected structure/column assignments before analyzing -If your benchmark includes repeated runs (`R >= 3`), bootstrap-based analyses above use a **two-level nested bootstrap** (resample inputs, then runs within each input) so run-to-run stochasticity is propagated into CIs and rankings. In that case, `analyze()` also returns a seed/input variance decomposition via `seed_variance_decomposition(...)`. +result = es.compare(evaldata, factors="prompt") +result.summary() # full terminal report: CIs, pairwise tests, rank probabilities +``` -If you set `method="lmm"`, `analyze()` switches to a mixed-effects path (`score ~ template + (1|input)`) with Wald CIs and parametric rank distributions. By default this uses `statsmodels` (pure Python, no additional setup required); pass `backend="pymer4"` to use R's lme4/emmeans instead (requires a separate R installation — see below). **Mixed effects model support is more experimental at the moment.** +See [Python API](#python-api) for the full data-format and `compare()` reference. -## Installation and Quick start CLI +## See it in action -```bash -pip install evalstats -``` +Running `es.compare(evaldata, factors="prompt")` then `result.summary()` prints a full statistical report to the terminal — confidence interval line plots, pairwise comparisons, and per-input stability across runs. Below: a 4-template sentiment-classification benchmark (GPT-4.1-nano, 27 inputs, 3 runs, 3 evaluators). -For Excel (`.xlsx`) input support: +![Example terminal output](docs/example-output.png) -```bash -pip install "evalstats[xlsx]" -``` +From this we can see Minimal and Instructive are the most promising candidates, but it's statistically unclear which is better — and Chain-of-thought gives the least consistent outputs across runs. -For all optional extras (including mixed-effects/LMM support): +Comparing models and prompts at once, `evalstats` colors in a 4-way tie between four model-prompt combinations: -```bash -pip install "evalstats[all]" -``` +![Example terminal output with colors](docs/terminal-output-example.jpg) -From the command line, `evalstats` can read a CSV or Excel file directly and print a statistical summary: +You can also plot within notebook environments. `plot_point_estimates` shows each template's absolute mean score with marginal confidence intervals: -```bash -evalstats analyze results.csv -``` +![Mean advantage plot](docs/mean_advantage.png) -The input file should have a prompt/template column, an item/input column, and a score column (model, run, and evaluator columns are optional) — see the column alias table in the [Python API](#python-api) section below for recognized names. Run `evalstats analyze --help` for the full list of options and supported column aliases. +And LLMs are stochastic at temperature > 0 — the "noise plot" visualizes (in)stability across runs for the same input: -For more complex statistical analysis with mixed effects models, use `method="lmm"`. The default `statsmodels` backend works out of the box; for the optional R-based backend, see below. +![Per-input noise across runs](docs/per-input-noise.png) -## Python API +## Recommended Methods -The main entry point is `load_from()` + `compare()`: parse your data once into an `EvalResults` object, then run comparisons against it. +`evalstats.compare()` defaults to `method="auto"`, which picks a well-calibrated statistical method based on your data's estimand, data type, and sample size. These defaults come from an extensive Monte Carlo simulation study across eval data types, sample sizes, and comparison setups, cross-checked against real LLM eval data — summarized in the two decision trees below. **Boxed methods are the default; gray notes give the multi-run variant and conservative alternatives.** -```python -import pandas as pd -import evalstats as es +![Decision tree for selecting a 95% confidence interval method](docs/decision-tree-ci.png) -df = pd.read_csv("results.csv") # columns: prompt, item, score (model optional) +![Decision tree for selecting a p-value method or FWER correction](docs/decision-tree-pvalue.png) -evaldata = es.load_from(df) -evaldata.summary() # inspect detected structure/column assignments before analyzing +A paper with the full methodology, simulation results, and justification behind each recommendation is forthcoming (see [Citation](#citation)). Until then, see the [Which Method?](https://statsforevals.com/which-method.html) page on the `evalstats` site for the web version of these trees, and [`simulations/harness`](simulations/harness) to reproduce the underlying simulations yourself. -result = es.compare(evaldata, factors="prompt") -result.summary() # full terminal report: CIs, pairwise tests, rank probabilities -``` +## Python API `evalstats` expects **long-format** data: one row per (item, score) observation, plus whichever axis you want to compare — `model`, `prompt`, or both — and optionally `run` for repeated runs. Only `item` and `score` are strictly required; you need at least one of `model`/`prompt` too, whichever you pass to `compare(factors=...)`. `load_from()` auto-detects each column's role by matching its name (case-insensitively) against this table: @@ -139,22 +135,19 @@ For example, a minimal CSV comparing prompts: | Minimal | q2 | 0.75 | | Instructive | q2 | 0.88 | -If your columns don't match any of the aliases above, remap them explicitly with `col_map`: - -```python -evaldata = es.load_from(df, col_map={"llm": "model", "variant": "prompt", "q_id": "item"}) -``` +If your columns don't match any alias above, remap them explicitly: `es.load_from(df, col_map={"llm": "model", "variant": "prompt", "q_id": "item"})`. `compare()` also handles: - **Comparing models**: `factors="model"` - **Factorial designs** (model × prompt): `factors=["model", "prompt"]` (routes to an LMM backend) - **Filtering**: any keyword matching a column name acts as a row filter, e.g. `es.compare(evaldata, factors="model", split="test")` -- **PPI-corrected inference** for noisy LLM-judge scores against a smaller human-labeled subset — see [PPI-Corrected Inference](#ppi-corrected-inference-means-cis-and-tests) below +- **PPI-corrected inference** for noisy LLM-judge scores — see [PPI-Corrected Inference](#ppi-corrected-inference-means-cis-and-tests) -The returned `result` is a `ComparisonResult`. Besides `.summary()`, it has `.to_frame()` / `.to_dict()` for programmatic access, `.plot(method="bar" | "forest" | "cd")` for charts, and `.disagreements()` to surface the items entities disagree on most. +The returned `result` is a `ComparisonResult`. Besides `.summary()`, it has `.to_frame()` / `.to_dict()` for programmatic access, `.plot(method="forest" | "bar" | "cd" | "pareto")` for charts, and `.disagreements()` to surface the items entities disagree on most. -### Advanced: raw score arrays (low-level engine) +
+Advanced: raw score arrays (low-level engine) `compare()` is a wrapper around a lower-level engine, `analyze()`, which operates directly on `BenchmarkResult` / `MultiModelBenchmark` objects (numpy score arrays) rather than a DataFrame. Reach for this path only if you already have scores as arrays and don't want to build a DataFrame first — most use cases should use `compare()` above. @@ -169,11 +162,9 @@ your_scores = [ [0.85, 0.82, 0.80], [0.79, 0.76, 0.74], ] -n_templates = 4 -n_inputs = 3 +n_templates, n_inputs = 4, 3 # scores shape: (n_templates, n_inputs, n_runs, n_evaluators) -# For a single evaluator and single run, shape is (N, M, 1, 1) scores = np.array(your_scores).reshape(n_templates, n_inputs, 1, 1) result = estats.BenchmarkResult( @@ -186,19 +177,14 @@ analysis = estats.analyze(result, reference="grand_mean", n_bootstrap=5_000) analysis.summary() # same terminal report as ComparisonResult.summary() ``` -If you want this lower-level path from a DataFrame (e.g. to inspect the raw `BenchmarkResult` object, or to fine-tune `strict_complete_design`), use `from_dataframe()` instead of `load_from()`. It returns the array-based `BenchmarkResult` / `MultiModelBenchmark` that `analyze()` expects, plus an optional `DataLoadReport` — a data-quality log of coercions/repairs made while parsing (not a statistical report): +If you want this lower-level path from a DataFrame (e.g. to inspect the raw `BenchmarkResult` object, or fine-tune `strict_complete_design`), use `from_dataframe()` instead of `load_from()`. It returns the array-based `BenchmarkResult` / `MultiModelBenchmark` plus an optional `DataLoadReport` — a data-quality log of coercions/repairs made while parsing: ```python import evalstats as estats benchmark, load_report = estats.from_dataframe( - df, - format="auto", # auto / wide / long - repair=True, # average duplicate cells + fill partial run slots - strict_complete_design=True, # set False to keep NaNs - return_report=True, + df, format="auto", repair=True, strict_complete_design=True, return_report=True, ) - for line in load_report.to_lines(): print(line) @@ -206,226 +192,119 @@ analysis = estats.analyze(benchmark) analysis.summary() ``` -To visualize absolute prompt performance directly from a `BenchmarkResult`, bypassing `analyze()` (use `result.plot()` above instead if you're on the `compare()` path): +To visualize absolute prompt performance directly from a `BenchmarkResult`, bypassing `analyze()`: ```python fig = estats.plot_point_estimates(result) fig.savefig("mean_performance.png", dpi=150, bbox_inches="tight") ``` -## PPI-Corrected Inference (Means, CIs, and Tests) - -`evalstats` supports PPI-corrected inference for means, confidence intervals, and common statistical tests. +
-PPI (Prediction-Powered Inference) lets you use lots of cheap LLM -judgments plus a smaller set of human labels to correct measurement error from the LLM -judge. This gives you corrected estimates and uncertainty that better reflect what you -would have gotten from a fully human-labeled study (Angelopoulos et al., 2023). +## PPI-Corrected Inference (Means, CIs, and Tests) -Most PPI correction methods use PPIBoot (bootstrap variant of PPI; Zrnic, 2024). -Implemented corrections have been battle-tested via simulations (see `simulations/sim_type_i_calibration.py`). +PPI (Prediction-Powered Inference) lets you use lots of cheap LLM judgments plus a smaller set of human labels to correct measurement error from the LLM judge, giving you corrected estimates and uncertainty that better reflect what you'd have gotten from a fully human-labeled study (Angelopoulos et al., 2023). Most corrections use PPIBoot (bootstrap variant of PPI; Zrnic, 2024), battle-tested via simulations (see [`simulations/sim_type_i_calibration.py`](simulations/sim_type_i_calibration.py)). -> **Important: which items get a human label must be chosen uniformly at -> random.** PPI correction assumes the labeled subset is representative of -> the full dataset. If your labeling process instead targets specific items -> — e.g. "always double-check the borderline or highest-scoring responses," -> a common real-world review habit — that's missing-not-at-random (MNAR) -> selection on the outcome, and PPI correction can stay badly miscalibrated -> **no matter how many items you label**. This isn't ordinary small-sample -> noise that more labels fixes; it was confirmed in simulation to persist -> from 15 up through 300 labeled items out of 400. See -> `evalstats.ppi.correct`'s docstring for the full analysis. If you can't -> guarantee random labeling, treat any PPI-corrected result here with -> caution regardless of the reported CI/p-value. +> [!IMPORTANT] +> **Which items get a human label must be chosen uniformly at random.** PPI correction assumes the labeled subset is representative of the full dataset. If your labeling process instead targets specific items — e.g. "always double-check the borderline or highest-scoring responses" — that's missing-not-at-random (MNAR) selection on the outcome, and PPI correction can stay badly miscalibrated **no matter how many items you label**. This isn't ordinary small-sample noise that more labels fixes; confirmed in simulation to persist from 15 up through 300 labeled items out of 400. See `evalstats.ppi.correct`'s docstring for the full analysis, and use `evalstats label` (below) to draw a compliant random sample. -### Example: Comparing models with corrected LLM judge evals via `compare(..., alignment=...)` +### Comparing models with corrected LLM judge evals via `compare(..., alignment=...)` ```python import evalstats as es -# Dataframe columns include: -# model item llm_score human_score (NaN for unlabeled rows) +# Dataframe columns include: model item llm_score human_score (NaN for unlabeled rows) evaldata = es.load_from(df) -# Compute alignment between LLM and human judges -alignment = es.validate_alignment( - evaldata, - llm_metric="llm_score", - human_groundtruth="human_score", -) +alignment = es.judge_alignment(evaldata, llm_metric="llm_score", human_groundtruth="human_score") -# Compare models, using PPI to correct for bias/misalignment with human graders result = es.compare( - evaldata, - factors="model", - metric="llm_score", - alignment={"llm_score": alignment}, + evaldata, factors="model", metric="llm_score", alignment={"llm_score": alignment}, ) - result.summary() ``` -### Example: T-test PPI-correction via `evalstats.tests.ttest` - -Use this for a t-test of mean differences between two groups (or two paired -conditions when `paired=True`). +### T-test PPI-correction via `evalstats.tests.ttest` ```python import evalstats as es res = es.tests.ttest( - a=llm_a, - b=llm_b, + a=llm_a, b=llm_b, a_lab=human_a, # same length as llm_a, NaN where unlabeled b_lab=human_b, # same length as llm_b, NaN where unlabeled - paired=False, - print_result=False, + paired=False, print_result=False, ) - print(res.p_value, res.corrected_p_value, res.corrected_ci) ``` -### Example: Mann-Whitney U test PPI-correction via `evalstats.tests.mannwhitney` +
+More PPI-corrected tests: Mann-Whitney U, Wilcoxon, one-way ANOVA -Use this for a Mann-Whitney U test, a nonparametric two-group comparison based -on relative ranks rather than assuming normally distributed scores. +**Mann-Whitney U** (`evalstats.tests.mannwhitney`) — nonparametric two-group comparison based on relative ranks rather than assuming normally distributed scores: ```python -import evalstats as es - -res = es.tests.mannwhitney( - x=llm_x, - y=llm_y, - x_lab=human_x, - y_lab=human_y, - print_result=False, -) - +res = es.tests.mannwhitney(x=llm_x, y=llm_y, x_lab=human_x, y_lab=human_y, print_result=False) print(res.p_value, res.corrected_p_value, res.corrected_ci) ``` -### Example: Wilcoxon signed-ranks test PPI-correction via `evalstats.tests.wilcoxon` (paired) - -Use this for a Wilcoxon signed-rank test, a nonparametric paired test for -matched observations (before/after, A/B on the same items, etc.). +**Wilcoxon signed-rank** (`evalstats.tests.wilcoxon`) — nonparametric paired test for matched observations (before/after, A/B on the same items, etc.): ```python -import evalstats as es - -res = es.tests.wilcoxon( - x=llm_before, - y=llm_after, - x_lab=human_before, - y_lab=human_after, - print_result=False, -) - +res = es.tests.wilcoxon(x=llm_before, y=llm_after, x_lab=human_before, y_lab=human_after, print_result=False) print(res.p_value, res.corrected_p_value, res.corrected_ci) ``` -### Example: One-way ANOVA PPI-correction via `evalstats.tests.anova_oneway` - -Use this for one-way ANOVA when comparing more than two groups, with -`repeated=True` for repeated-measures (same subjects across conditions). +**One-way ANOVA** (`evalstats.tests.anova_oneway`) — more than two groups; pass `repeated=True` for repeated-measures (same subjects across conditions): ```python -import evalstats as es - res = es.tests.anova_oneway( - llm_g1, - llm_g2, - llm_g3, - groups_lab=[human_g1, human_g2, human_g3], - repeated=False, - print_result=False, + llm_g1, llm_g2, llm_g3, + groups_lab=[human_g1, human_g2, human_g3], repeated=False, print_result=False, ) - print(res.p_value, res.corrected_p_value, res.corrected_ci) ``` -## Motivation - -Most eval tools in the LLM evaluation space don't help users perform _any_ statistical tests, let alone showcase variances in performance between prompts or models. They instead present bar charts of average performance. Developers then glance at the bar chart and decide that "prompt/model A is better than B." But was it really? - -Relying purely on bar charts and averages can very, very easily lead to erroneous conclusions—B might actually be more robust than A, or B performs well on an important subset of data, or there's not enough data to conclude one way or the other. - -Why do people do evals this way? Well, they don't have the time, tools, or knowledge on how to do it better—frequently, they don't even know there's a better way. - -`evalstats` aims to rectify this with simple, powerful defaults—just throw us your data and we'll run the stats and plot the results for you. Upstream applications, like LLM observability platforms, could take `evalstats` results and plot them in their own front-ends. Prompt optimization tools could also use `evalstats` to decide, e.g., when to cull a candidate prompt and how to present results to users. - -## Examples - -### Is one prompt "better" than others? Quantify uncertainty - -When you have scores for multiple prompt templates across a set of inputs, `evalstats` computes bootstrapped 95% confidence intervals and pairwise significance tests so you can see not just which prompt scored highest on average, but how certain you can be about that ranking. It plots these to the terminal so you can check at a glance: +
-![Comparing across prompts output](docs/compare-prompts-output.png) - -### Comparing across models while accounting for prompt sensitivity - -A common failure mode in LLM benchmarking, both in academic papers and practitioner evaluations, is testing each model with a single prompt template and reporting the resulting scores as if they reflect stable model capabilities. In reality, model rankings can flip under semantically equivalent paraphrases of the same instruction. A benchmark result that says "Model A beats Model B" may be an artifact of prompt phrasing, not a meaningful capability difference. - -Here, we can see the difference between OpenAI's `gpt-4.1-nano` and MistralAI's `ministral-8b-2512` on a small sentiment classification benchmark, quantified by bootstrapped 95% confidence intervals: - -![Comparing across models output](docs/compare-models-output.png) - -In this run, multiple prompt template variations were considered, making this result more robust than trying a single prompt and calling it a day. - -### How stable is the performance across runs? - -LLMs are stochastic at temperature>0. Will the performance stay similar, even upon multiple runs for the same inputs? `evalstats` offers a helpful "noise plot" which visualizes (in)stability across runs: - -![Per-input noise across runs](docs/per-input-noise.png) - -## Running Example Scripts - -We provide multiple standalone example scripts that rig up a simple benchmark, collect LLM responses, and run analyses over them. From the repository root, run any example script directly: +## CLI Reference ```bash -python examples/synthetic_mean_advantage.py +evalstats analyze results.csv # full statistical report from a CSV/XLSX file +evalstats label results.csv # draw a random, MCAR-compliant sample of items for human labeling ``` -Additional examples: +`evalstats label` picks a uniformly random sample of items per condition (respecting the PPI sample-size floors: 15 minimum, 30 recommended) and writes a CSV/XLSX with a `human_` column ready for grading — the safe way to build the labeled subset `alignment=`/`*_lab` needs above. Run `evalstats label --help` for the full option list. -```bash -# OpenAI sentiment benchmark (single run) -python examples/sentiment.py - -# Multi-run variant (captures run-to-run variability) -python examples/sentiment_multirun.py +## Examples -# Multi-model comparison across prompt templates -python examples/compare_models_multirun.py +`examples/` has 25+ runnable, self-contained scripts covering common workflows — synthetic and OpenAI-backed benchmarks, multi-run comparisons, PPI-corrected judge alignment, factorial designs, and reliability/robustness demos. From the repository root: -# Manual API call walkthrough -python examples/sentiment_manual_api_calls.py +```bash +python examples/synthetic_mean_advantage.py # no API key needed +python examples/sentiment.py # OpenAI sentiment benchmark +python examples/sentiment_multirun.py # captures run-to-run variability +python examples/compare_models_multirun.py # multi-model comparison across prompts +python examples/compare_alignment_ppi.py # PPI-corrected judge comparison ``` -OpenAI-powered examples require `OPENAI_API_KEY` set in your environment. But, you can easily swap out the model calls to whatever model you prefer. +OpenAI-powered examples require `OPENAI_API_KEY` set in your environment, but the model calls are easy to swap for whichever provider you prefer. ## Mixed effects models (LMM) > [!IMPORTANT] -> Mixed effects analysis is experimental, and currently offers only the advantage of gracefully -> dealing with missing data. In the future, we plan to add factor decomposition across multiple inputs. -> We recommend only using `method="lmm"` if you need robustness to missing data (`NaN`). Keep -> in mind that missing data must be reasonably random (i.e., like sampling from a larger distribution). - -`evalstats` supports mixed-effects models (`score ~ template + (1|input)`) for: -- Missing data in inputs (some score cells are `NaN`) -- Factor decomposition when multiple input factors are present +> Mixed effects analysis is experimental, currently offering only graceful handling of missing data (assumed reasonably random). Use `method="lmm"` if you need robustness to missing (`NaN`) cells; factor decomposition across multiple input factors is planned. -### Default backend: statsmodels (pure Python) - -No extra setup required — `statsmodels` is included in the standard `pip install evalstats`. Simply pass `method="lmm"`: +`evalstats` supports mixed-effects models (`score ~ template + (1|input)`) for missing data and multi-factor decomposition. The default backend is pure-Python `statsmodels` — no extra setup required: ```python analysis = estats.analyze(result, method="lmm") ``` -`evalstats` fits the model with REML, computes Wald CIs via the delta method, and estimates rank distributions by parametric simulation. +This fits the model with REML, computes Wald CIs via the delta method, and estimates rank distributions by parametric simulation. -### Optional backend: pymer4 (requires R) +
+Optional backend: pymer4 (requires R) For Satterthwaite degrees of freedom and `emmeans`-based pairwise contrasts (R's gold standard for mixed models), pass `backend="pymer4"`: @@ -433,39 +312,25 @@ For Satterthwaite degrees of freedom and `emmeans`-based pairwise contrasts (R's analysis = estats.analyze(result, method="lmm", backend="pymer4") ``` -This requires a working R installation with the following packages: +This requires a working R installation with: ```r -install.packages(c( - "lme4", - "emmeans", - "tibble", - "broom", - "broom.mixed", - "lmerTest", - "report", - "car" -)) +install.packages(c("lme4", "emmeans", "tibble", "broom", "broom.mixed", "lmerTest", "report", "car")) ``` -Then install the Python LMM extra: +Then `pip install "evalstats[lmm]"`. If your environment needs manual dependency pinning, this is the tested equivalent: ```bash -pip install "evalstats[lmm]" +pip install "pymer4>=0.9" great_tables joblib rpy2 polars scikit-learn formulae pyarrow ``` -> [!NOTE] -> If your environment needs manual dependency pinning, this is the tested equivalent: -> -> ```bash -> pip install "pymer4>=0.9" great_tables joblib rpy2 polars scikit-learn formulae pyarrow -> ``` - Installation details may differ on your system. +
+ ## Reproducibility: Monte Carlo simulations -Claims in this README like "verified in our simulations" are backed by a runnable simulation harness in `simulations/harness/` of this package. We engineered these simulations so that you can run these yourself. For instance: +Claims in this README and on the [`evalstats` site](https://statsforevals.com/) like "verified in our simulations" are backed by a runnable harness in [`simulations/harness/`](simulations/harness): ```bash python -m simulations.harness.cli --list-cases @@ -474,21 +339,42 @@ python -m simulations.harness.cli ci_single --reps 50 --sizes 10 20 python -m simulations.harness.cli pvalues --mode ppi --tests ttest wilcoxon anova_rep ``` -`--official-tests will bring up a CLI with options to run specific tests. Each runs each case's canonical, full-scale preset and writes results plus a `manifest.json` (args, output paths, key metrics, pass/fail) to `simulations/out/official_/`. See [`simulations/harness/README.md`](simulations/harness/README.md) for the full case list, scenario library, and verification methodology against the original standalone scripts. Note that *each* simulation can take *very long* to run; even on a MacBook Pro with an M4 Max chip and 64GB RAM, with computation paralellized across 16 CPU cores, it often takes many hours. -- `ci_single` / `ci_paired` — coverage and width of confidence interval methods (bootstrap, smoothed bootstrap, Bayesian, Wilson, etc.) across synthetic distributions and real benchmark data (OpenEval, Inspect AI). -- `pvalues --mode pairwise` / `--mode multiarm` — Type-I error and power for pairwise and multi-arm comparisons, including multiple-comparisons correction strategies. +`--official-tests` brings up a CLI to run each case's canonical, full-scale preset, writing results plus a `manifest.json` (args, output paths, key metrics, pass/fail) to `simulations/out/official_/`. See [`simulations/harness/README.md`](simulations/harness/README.md) for the full case list and scenario library. Note that each simulation can take a long time to run — even parallelized across many cores, official-scale runs can take hours. + +- `ci_single` / `ci_paired` — coverage and width of confidence interval methods across synthetic distributions and real benchmark data (OpenEval, Inspect AI). +- `pvalues --mode pairwise` / `--mode multiarm` — Type-I error and power for pairwise and multi-arm comparisons, including FWER correction strategies. - `pvalues --mode ppi` — Type-I error calibration and power for every PPI-corrected test in `evalstats.tests`, swept across judge-bias severity, label fraction, and MNAR-labeling scenarios. +## Motivation + +Most eval tools in the LLM evaluation space don't help users perform *any* statistical tests — they present bar charts of average performance, and developers glance at the chart and decide "prompt/model A is better than B." But was it really? Relying on bar charts and averages alone can easily lead to erroneous conclusions: B might be more robust than A, or perform better on an important data subset, or there might not be enough data to conclude either way. + +People do evals this way because they don't have the time, tools, or statistical knowledge to do better — often they don't even know there's a better way. `evalstats` aims to rectify this with simple, powerful defaults: throw us your data, and we'll run the stats and plot the results for you. ## Development and Contributions For package build, release validation, and maintainer workflows, see [DEVELOPMENT.md](DEVELOPMENT.md). -We welcome contributions, especially refinements to our statistical methods. If you're proposing a new correction, CI method, or a fix to an existing one, we encourage battle-testing it against the [simulation harness](#reproducibility-monte-carlo-simulations) first. Please add or extend a scenario and confirm your change holds up on Type-I error and power, not just on the case that motivated it, before opening a PR. The `evalstats` repository already offers a rigorous, expansive synthetic suite that generally has held up against real data. +We welcome contributions, especially refinements to our statistical methods. If you're proposing a new correction, CI method, or a fix to an existing one, battle-test it against the [simulation harness](#reproducibility-monte-carlo-simulations) first — add or extend a scenario and confirm your change holds up on Type-I error and power, not just on the case that motivated it, before opening a PR. + +## Citation + +`evalstats` doesn't have a paper yet — one covering the full simulation-backed method validation is forthcoming. Until then, please cite the GitHub repository: + +```bibtex +@software{arawjo_evalstats, + author = {Arawjo, Ian}, + title = {evalstats: Statistically Sound Analysis for LLM Evaluations}, + url = {https://github.com/ianarawjo/evalstats}, + year = {2026} +} +``` + +or in prose: Ian Arawjo, *evalstats* (GitHub: [ianarawjo/evalstats](https://github.com/ianarawjo/evalstats)). ## License This repository uses two licenses: - **`evalstats` package** (everything outside `website/`) — [MIT](LICENSE). -- **Stats for Evals Website** (everything in `website/`) — [CC BY-NC-ND 4.0](website/LICENSE). You may share it with attribution non-commercially, but commercial use and derivative works are not permitted. +- **Stats for Evals Website** (everything in `website/`) — [CC BY-NC-ND 4.0](website/LICENSE). You may share it with attribution non-commercially, but commercial use and derivative works are not permitted. diff --git a/RESUME_ppi_fixes.md b/RESUME_ppi_fixes.md new file mode 100644 index 0000000..ffad179 --- /dev/null +++ b/RESUME_ppi_fixes.md @@ -0,0 +1,154 @@ +# PPI fixes + compare_e2e scenario rework — 2026-08-21/22 + +Verification is COMPLETE; this file is kept as the validation record. + +Originally written as a handoff when a session closed mid-verification. All +outstanding items below have since been run and are recorded inline. + +## What changed + +**1. `evalstats/api.py` — relative floor on the joint bootstrap's SE** +New module constant `_JOINT_BOOT_SE_REL_FLOOR = 0.20` and one line in +`_ppi_bootstrap_t_joint_stats`: + + boot_se = np.maximum(boot_se, _JOINT_BOOT_SE_REL_FLOOR * obs_se[None, :]) + +Set the constant to `0.0` to get the old behaviour back exactly. + +Fixes: a near-degenerate pair's bootstrap SE could collapse (the old guard +was absolute, `1e-12`, so it could not see a small-but-nonzero collapse), +sending that pair's `|T|` to 60–2000. Both consumers of this ONE joint +resample reduce it with a MAX over pairs — Romano-Wolf's step-down +suffix-max, and `_M_b_from_T` for max-T / "boot" CI widening — so one bad +pair poisons the whole family. Measured: a degenerate pair with |T|max=66 +drove an UNRELATED pair's Romano-Wolf p from ~0 to 0.363 while that pair's +CI still excluded 0 by a wide margin. + +**2. `evalstats/api.py` + `core/bundles.py` + `core/router.py` — likert PPI routing** +PPI's `method="auto"` re-derived `data_kind` locally with only +binary/bounded_01/unbounded branches, ignoring `score_range` and +`eval_type`. Likert (e.g. 1–5) fell through to `unbounded` and silently took +`ppi_t_interval`, making `PPI_AUTO_METHOD_TABLE`'s `likert -> ppi_logit_t` +row unreachable. Now the router's single decision is recorded as +`AnalysisBundle.resolved_data_kind` and reused; the old local test remains +as the fallback for non-`auto` callers. (`data_kind` also had to be +initialized to `None` in router.py — it was only bound inside the +`method == "auto"` branch.) + +## Verification status + +DONE, before touching api.py: + - binding rates: 12–19% on degenerate cells, **0.0000%** on 8 + non-degenerate conditions at every c up to 0.50 + - FWER on the non-degenerate DGP: identical for c in {0,.1,.2,.3,.5} + - power on degenerate cells: 0.665 -> 1.000, FWER unchanged + - symptom: contradictory reps 7/20 -> 0/20; romano_wolf p 0.371 -> 0.001 + - `tests/test_auto_ci_routing.py`, `test_bayes_binary_routing.py`: 78 pass + (routing fix only — run BEFORE the boot_se floor was added) + +DONE after resuming — shipped-vs-prefix validation (validate3 / 400 reps, +`simulations/out/joint_bootstrap_se_floor/val3.log`). FWER **identical to 4 +decimals in all 10 conditions**; power on the degenerate cells: + continuous k=3 N=100 0.6425 -> 0.9975 + continuous k=3 N=200 0.6825 -> 1.0000 + continuous k=5 N=100 0.3850 -> 1.0000 + likert (all N) unchanged, as predicted + all 4 non-degenerate unchanged to 4 decimals +So the floor recovers power exactly where the bootstrap had broken down and +changes literally nothing anywhere else. + +DONE: regression tests — **167 passed** (test_compound_ppi_fwer.py incl. +TestRomanoWolfCalibration, test_ppi_ci_methods.py, test_simultaneous_ci.py, +test_auto_ci_routing.py), 23 min. + +DONE: null-binding FWER test — the hole in the earlier evidence. Ordinary +nulls have uniq(d_true)==1 so the floor is inert and "FWER unchanged" was +trivial. Built a null where it DOES bind (near-identical arms: shared base, +each arm perturbs a small random subset by ±delta with mean-zero signs). +**With binding up to 12% under a genuine null, FWER is unchanged** (largest +move +0.0025 at 0.28% binding = 0.23 MC SE). See +`simulations/investigate_joint_bootstrap_se_floor_nullbind.py` and +`simulations/out/joint_bootstrap_se_floor/val6.log`. + +REMAINING: + 3. End-to-end: + `.venv/bin/python -m simulations.harness.cli compare_e2e --reps 60 --eval-types likert continuous --k-values 3 --sizes 50 100 200 --ppi-fracs none 0.20 0.40 --plots off --save-results off --progress off` + Pass = continuous PPI power clears its subset-only `ref.pwr` floor + (was 20 points BELOW it), and likert `fam.cov` falls from 99.5%. + 4. Not reviewed: the max-T CI path and other consumers of this same joint + resample may want the same treatment. + +## Caveat on likert + +Its low-N conservatism is MOSTLY LEGITIMATE, not this bug. `alpha_eff` comes +from `M_b`'s p95 (3.07 -> 0.00214, matching the observed 0.002116), and a +max-|T| p95 of 3.07 at n_lab=20 is ordinary bootstrap-t small-sample +inflation; the degenerate tail sits in the p99/max. Likert binds only ~1% at +c=0.2 vs continuous's 18%. Expect a small move, not a large one. + +UNVERIFIED residual suspect, do not act on without checking: the bootstrap-t +critical value is converted to an effective alpha through a NORMAL quantile +in `_ppi_alpha_eff_from_M_b`, then fed to a t-based CI formula — possibly +double-counting small-sample inflation. + +## Also disproven this session + +The hypothesis that likert's conservatism came from `logit_t` hitting its +Clopper-Pearson fallback (`degenerate_sample_ci`): instrumented every +module-level reference, **0 calls** in every likert and continuous cell at +N=20/50, PPI and non-PPI. Instrument verified against a forced constant +sample, which does trigger it. + +## Files + +Untracked helpers (safe to delete): + simulations/investigate_joint_bootstrap_se_floor_*.py (validation harness) + simulations/out/joint_bootstrap_se_floor/ (pre-fix logs + api.py backup) + RESUME_ppi_fixes.md (this file) + + +--- + +## Final status (2026-08-22) + +All verification complete: + +- **167 regression tests** pass (test_compound_ppi_fwer incl. + TestRomanoWolfCalibration, test_ppi_ci_methods, test_simultaneous_ci, + test_auto_ci_routing), plus 115 routing/dispatch and 58 unpaired. +- **Shipped-vs-prefix, 400 reps**: FWER identical to 4dp in all 10 conditions; + power on degenerate cells 0.6425 -> 0.9975 / 0.3850 -> 1.0000. +- **Null-binding FWER test**: with the floor binding up to 12% of replicates + under a genuine null, FWER is unchanged (largest move +0.0025 = 0.23 MC SE). +- **compare_e2e end-to-end**: Type-I nominal on both paths; PPI power above + the human-subset floor on all three eval types. + +## Two findings that were NOT library bugs + +1. **"PPI below the human-subset floor"** -- a k-mismatch in the PLOTTING + code, not in evalstats. oracle/subset rates exist only at + REFERENCE_ESTIMATOR_K=3 while PPI's power_rate exists at every k, so + pooling PPI over k=2+3 against a k=3-only reference compared a 1-step + effect against a 2-step one. At continuous N=250: PPI(k=2+3)=0.840, + PPI(k=3)=0.962, subset(k=3)=0.905. PPI was winning throughout. +2. **Romano-Wolf "missing" on the non-PPI path** -- not missing. Both paths + share AUTO_PVALUE_CORRECTION_METHOD_TABLE (Shaffer <30, Romano-Wolf >=30); + they differ only because the subset arm legitimately sees fewer items. + +## Romano-Wolf vs Shaffer under PPI (the open question) + +Same cells, only the correction varied. Romano-Wolf matches or edges Shaffer +everywhere and is slightly LESS conservative (Type-I 0.017 vs 0.008 at +k=3/N=250), consistent with the existing simulation findings. Differences are +inside MC noise at 80 reps (SE ~0.028), so the defensible claim is +"indistinguishable, no evidence of a power cost" -- not "Romano-Wolf wins". + +## Still open + +- `nlab=30, N=250` Type-I: a small fixed label budget with a large unlabelled + pool is where the rectifier is estimated from fewest labels. Every look so + far has been at rep counts where any drift is inside noise. Worth a + dedicated high-rep check. +- `_pooled_k_group_lambda` still carries the uncentered-pooling defect its + two-group sibling had (fixed in 836f811), deliberately unfixed pending + Type-I/coverage validation. diff --git a/agent_study/sweep/README.md b/agent_study/sweep/README.md index 197a6a6..4c7b3a2 100644 --- a/agent_study/sweep/README.md +++ b/agent_study/sweep/README.md @@ -42,7 +42,7 @@ presupposing in advance that it's important enough to fully cross. **Held fixed / out of scope for this sweep** (see the design discussion that produced this grid, in conversation history, for the reasoning): -- Judge-score-correction (PPI/`validate_alignment`) task type -- deferred to +- Judge-score-correction (PPI/`judge_alignment`) task type -- deferred to a separate, smaller sub-sweep with its own axes (judge reliability, human-label fraction), since those don't apply to `prompts`/`models`. - Correlation structure: paired throughout (`base_corr=1.0` -- the same diff --git a/docs/decision-tree-ci.png b/docs/decision-tree-ci.png new file mode 100644 index 0000000..e5c3989 Binary files /dev/null and b/docs/decision-tree-ci.png differ diff --git a/docs/decision-tree-pvalue.png b/docs/decision-tree-pvalue.png new file mode 100644 index 0000000..ad38dca Binary files /dev/null and b/docs/decision-tree-pvalue.png differ diff --git a/evalstats/__init__.py b/evalstats/__init__.py index f2b7c0e..42dfc3b 100644 --- a/evalstats/__init__.py +++ b/evalstats/__init__.py @@ -33,16 +33,28 @@ # "compare" name if it were imported before the submodule. from evalstats.loader import load_from, EvalResults, EvalLoadError from evalstats.api import compare, compare_models, compare_prompts, ComparisonResult -from evalstats.alignment import validate_alignment, AlignmentResult +from evalstats.alignment import judge_alignment, AlignmentResult from evalstats import ppi from evalstats import tests +from evalstats.quick import ( + mean_ci, + MeanCI, + summarize, + GroupSummary, + stability, + StabilityResult, + tradeoff, + TradeoffResult, + judge_debias_mean_ci, + DebiasedMeanCI, +) __version__ = "0.2.4" __all__ = [ # High-level spec API "load_from", - "validate_alignment", + "judge_alignment", "AlignmentResult", "ppi", "tests", @@ -52,6 +64,17 @@ "compare_models", "compare_prompts", "ComparisonResult", + # Quick primitives + "mean_ci", + "MeanCI", + "summarize", + "GroupSummary", + "stability", + "StabilityResult", + "tradeoff", + "TradeoffResult", + "judge_debias_mean_ci", + "DebiasedMeanCI", # Core types "BenchmarkResult", "MultiModelBenchmark", diff --git a/evalstats/alignment.py b/evalstats/alignment.py index fe83413..b3b5361 100644 --- a/evalstats/alignment.py +++ b/evalstats/alignment.py @@ -1,18 +1,20 @@ """Judge alignment validation and MC-based uncertainty propagation. -Provides :func:`validate_alignment` and :class:`AlignmentResult` for +Provides :func:`judge_alignment` and :class:`AlignmentResult` for characterising how well an LLM judge aligns with human graders, and for propagating that uncertainty into downstream comparisons via Monte-Carlo imputation of latent human labels. """ from __future__ import annotations +import math import warnings +from itertools import combinations from typing import Optional import numpy as np import pandas as pd -from scipy.stats import ks_2samp, chi2_contingency, pearsonr, spearmanr +from scipy.stats import ks_2samp, chi2_contingency, pearsonr, spearmanr, norm # ───────────────────────────────────────────────────────────────────────────── @@ -22,7 +24,7 @@ class AlignmentResult: """Carries a fitted calibration model and alignment diagnostics. - Created by :func:`validate_alignment`. Pass it to + Created by :func:`judge_alignment`. Pass it to ``compare(alignment={metric_col: result})`` to widen confidence intervals to account for LLM-judge measurement uncertainty via Monte-Carlo imputation. @@ -39,16 +41,38 @@ class AlignmentResult: Number of items with human labels (alignment set size). n_total : int Total number of items in the dataset. + selection : str + How the labeled subset was chosen, as declared by the caller via + :func:`judge_alignment`'s ``selection=`` -- ``"random"``, + ``"stratified"``, ``"manual"``, or ``"unknown"`` (the default when + not specified). Every correction :func:`judge_alignment` / + ``compare(alignment=...)`` applies assumes the labeled subset is a + random sample of the full item pool (MCAR, "missing completely at + random"); anything other than ``"random"`` means that assumption + is either known-violated or unconfirmed, and a warning is raised + at call time -- see :attr:`representativeness` and + :meth:`summary` for the diagnostics that check for this in practice. alignment_metrics : dict Point estimates and bootstrap CIs for each alignment metric. representativeness : dict - Representativeness check results (distribution and slice columns). + Representativeness check results (distribution, slice columns, and + label-position contiguity). bias_check : dict or None For likert/continuous/grade score types, compares the correlation-type metric (weighted κ or Pearson r) against ICC(2,1) to flag whether the judge is systematically biased in absolute scale despite tracking human relative ordering. ``None`` for binary score types, where ICC isn't computed. + test : str or None + The test named via ``test=``, if any -- see :func:`judge_alignment`. + For a single condition, only ``"mean_estimate"`` is valid (no + comparison to linearize against). + test_metric : dict or None + Set iff ``test`` was given: the correlation entry (same shape as + ``alignment_metrics``' entries, with ``multiplier``/``n_eff`` added) + that governs ``test``'s PPI variance reduction. For + ``test="mean_estimate"`` this is identical to ``alignment_metrics + ["pearson_r"]``. :attr:`n_eff`/:attr:`multiplier` read from here. """ def __init__( @@ -63,6 +87,9 @@ def __init__( alignment_metrics: dict, representativeness: dict, bias_check: Optional[dict] = None, + selection: str = "unknown", + test: Optional[str] = None, + test_metric: Optional[dict] = None, ) -> None: self.llm_metric = llm_metric self.human_col = human_col @@ -73,6 +100,32 @@ def __init__( self.alignment_metrics = alignment_metrics self.representativeness = representativeness self.bias_check = bias_check + self.selection = selection + self.test = test + self.test_metric = test_metric + + @property + def n_eff(self) -> float: + """Effective human-label sample size for the ``test=`` you + specified -- see :func:`judge_alignment`'s ``test=`` docs. Raises + if you didn't pass ``test=``.""" + return self._require_test()["n_eff"] + + @property + def multiplier(self) -> float: + """Label-efficiency savings multiplier for the ``test=`` you + specified. Raises if you didn't pass ``test=``.""" + return self._require_test()["multiplier"] + + def _require_test(self) -> dict: + if self.test_metric is None: + raise ValueError( + "No test= was given to judge_alignment(), so there's no single " + "n_eff/multiplier answer -- inspect .alignment_metrics directly " + "(raw Pearson/Spearman r, not test-specific), or re-call with " + "test='mean_estimate'." + ) + return self.test_metric # ── sampling ───────────────────────────────────────────────────────────── @@ -165,30 +218,41 @@ def _header(self) -> None: f"Alignment set : {self.n_labeled} of {self.n_total} items " f"have human labels ({pct:.1f}%)" ) + sel_icon = "✓" if self.selection == "random" else "⚠ " + print(f"Label selection: {sel_icon} {self.selection}") + print( + "Note: corrections below assume the labeled subset is a random " + "sample of the full item pool (MCAR) — see 'Representativeness'." + ) print() def _summary_simple(self) -> None: self._header() - if self.bias_check is not None: + # This check compares a correlation against ICC(2,1), so the only thing + # it can see is a systematic shift or compression of the judge's raw + # scores. PPI absorbs that either way (verified: a judge compressed to + # 0.55x with a +0.9 offset understates a true +0.50 effect as +0.27 raw, + # and the corrected estimate recovers +0.51), so the result never changes + # whether to correct. Only the FAILING branch is printed here, because it + # says something about the judge worth knowing; a passing result is not + # evidence that correction can be skipped -- the bias PPI is most needed + # for errs in different directions across conditions, which is invisible + # to any pooled statistic including this one. Both branches stay in + # summary(verbose=True), where the surrounding text supplies that context. + if self.bias_check is not None and not self.bias_check["passed"]: bc = self.bias_check - if not bc["passed"]: - print( - f"⚠ Possible judge bias: {bc['corr_label']} = " - f"{bc['corr_estimate']:.2f} but ICC(2,1) = {bc['icc_estimate']:.2f} " - "— the judge ranks items like humans do, but its raw scores " - "look shifted or compressed relative to human scores." - ) - print( - " Treat raw judge scores with caution; consider " - "recalibrating (compare(alignment=...)) before using them " - "directly. Run .summary(verbose=True) for the full check." - ) - else: - print( - f"✓ No sign of judge bias: {bc['corr_label']} and ICC(2,1) " - "roughly agree." - ) + print( + f"⚠ Possible judge scale bias: {bc['corr_label']} = " + f"{bc['corr_estimate']:.2f} but ICC(2,1) = {bc['icc_estimate']:.2f} " + "— the judge ranks items like humans do, but its raw scores " + "look shifted or compressed relative to human scores." + ) + print( + " This affects raw judge scores only; a PPI-corrected " + "comparison (compare(alignment=...)) already absorbs it. " + "Run .summary(verbose=True) for the full check." + ) print() rep = self.representativeness @@ -198,8 +262,7 @@ def _summary_simple(self) -> None: if rep_failed: print("⚠ Representativeness: the labeled sample may not be representative") for key, val in rep_failed: - name = "score distribution" if key == "score_distribution" else key[len("slice_"):] - print(f" - {name}: {val['message']}") + print(f" - {_rep_check_display_name(key)}: {val['message']}") else: print("✓ Representativeness: labeled items look like the full item pool") print() @@ -250,8 +313,11 @@ def _print_check(title: str, val: dict) -> None: dist = rep.get("score_distribution") if dist: _print_check("Score distribution", dist) + contiguity = rep.get("label_contiguity") + if contiguity: + _print_check("Label position contiguity", contiguity) for key, val in rep.items(): - if key == "score_distribution": + if key in ("score_distribution", "label_contiguity"): continue if key.startswith("slice_"): col = key[len("slice_"):] @@ -593,7 +659,7 @@ def _build_bias_check( interpretation = ( "the judge tracks human relative ordering but disagrees on " "absolute scale — treat raw judge scores as biased; consider " - "using the Bayesian calibration model fit by validate_alignment " + "using the Bayesian calibration model fit by judge_alignment " "(e.g. via compare(alignment=...)) to correct for it before " "drawing conclusions from raw judge scores" ) @@ -625,9 +691,25 @@ def _compute_alignment_metrics( *, alpha: float = 0.05, rng: np.random.Generator, + ci: bool = True, ) -> dict: metrics: dict = {} + # ci=False skips every bootstrap CI on the alignment metrics and reports + # NaN bounds, keeping only the (deterministic, closed-form) point + # estimates. Each CI is 2000 resamples of its metric, so this is the bulk + # of judge_alignment()'s cost. Intended for callers that consume only the + # estimates -- notably compare(alignment=...), whose PPI correction reads + # the point estimates alone -- and for large simulation sweeps. + if ci: + _ci2, _cigap = _bootstrap_ci_2, _bootstrap_ci_gap + else: + def _ci2(fn, a, b, **_kw): + return float(fn(a, b)), float("nan"), float("nan") + + def _cigap(fn_corr, fn_icc, a, b, **_kw): + return float(fn_corr(a, b)) - float(fn_icc(a, b)), float("nan"), float("nan") + if score_type == "binary": def agree(a, b): return float(np.mean(a == b)) @@ -640,7 +722,7 @@ def kappa(a, b): ) return (p_o - p_e) / (1.0 - p_e) if p_e < 1.0 else 1.0 - est, lo, hi = _bootstrap_ci_2(agree, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(agree, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_pct_agreement(est, lo, hi, len(llm), "Percent agreement") metrics["percent_agreement"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -657,7 +739,7 @@ def kappa(a, b): "interpretation": interp, "example": example, } - est, lo, hi = _bootstrap_ci_2(kappa, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(kappa, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_kappa(est, lo, hi, len(llm), "Cohen's κ") metrics["cohens_kappa"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -677,6 +759,59 @@ def kappa(a, b): "example": example, } + def pe(a, b): + r, _ = pearsonr(a, b) + return float(r) + + def sp(a, b): + r, _ = spearmanr(a, b) + return float(r) + + est, lo, hi = _ci2(pe, llm, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Pearson r") + metrics["pearson_r"] = { + "estimate": est, "ci_low": lo, "ci_high": hi, + "label": "Pearson r", + "band": band, + "what": ( + "Linear correlation coefficient between judge and human labels -- " + "for two binary (0/1) variables this is the phi coefficient, " + "algebraically equivalent to Cohen's κ's numerator rescaled by " + "the marginal proportions." + ), + "why": ( + "Reported alongside Cohen's κ/percent agreement because a " + "PPI-corrected hypothesis test's variance reduction is governed " + "by this correlation (or its rank-based counterpart below), not " + "by κ -- see the label-efficiency guidance in the package docs " + "for which one your test needs." + ), + "interpretation": interp, + "example": example, + } + est, lo, hi = _ci2(sp, llm, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Spearman r") + metrics["spearman_r"] = { + "estimate": est, "ci_low": lo, "ci_high": hi, + "label": "Spearman r", + "band": band, + "what": ( + "Rank correlation between judge and human labels -- for two " + "binary (0/1) variables this is numerically identical to " + "Pearson r above (rank-transforming a two-valued variable is " + "just an increasing affine rescaling of it, which Pearson r is " + "invariant to)." + ), + "why": ( + "Reported for consistency with the continuous/likert score " + "types, and because rank-based hypothesis tests (e.g. " + "Mann-Whitney) predict their PPI variance reduction from this " + "correlation, not Pearson's." + ), + "interpretation": interp, + "example": example, + } + elif score_type == "likert": cats = sorted(set(llm.tolist()) | set(human.tolist())) k = len(cats) @@ -699,8 +834,12 @@ def sp(a, b): r, _ = spearmanr(a, b) return float(r) + def pe(a, b): + r, _ = pearsonr(a, b) + return float(r) + if k >= 2: - est, lo, hi = _bootstrap_ci_2(wk, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(wk, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_kappa(est, lo, hi, len(llm), "Weighted Cohen's κ") metrics["weighted_kappa"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -719,7 +858,28 @@ def sp(a, b): "interpretation": interp, "example": example, } - est, lo, hi = _bootstrap_ci_2(sp, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(pe, llm, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Pearson r") + metrics["pearson_r"] = { + "estimate": est, "ci_low": lo, "ci_high": hi, + "label": "Pearson r", + "band": band, + "what": ( + "Linear correlation coefficient between judge and human scores, " + "treating the Likert categories as equally-spaced numeric values." + ), + "why": ( + "Reported alongside weighted κ/Spearman r because a PPI-corrected " + "parametric or mean-based test (e.g. a $t$-test on Likert scores " + "treated as numeric) draws its variance reduction from this " + "correlation, not from weighted κ or Spearman's rank-based one -- " + "see the label-efficiency guidance in the package docs for which " + "one your test needs." + ), + "interpretation": interp, + "example": example, + } + est, lo, hi = _ci2(sp, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Spearman r") metrics["spearman_r"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -740,7 +900,7 @@ def sp(a, b): } if k >= 2: - icc_est, icc_lo, icc_hi = _bootstrap_ci_2(_icc_21, llm, human, alpha=alpha, rng=rng) + icc_est, icc_lo, icc_hi = _ci2(_icc_21, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_icc(icc_est, icc_lo, icc_hi, len(llm), "ICC(2,1)") metrics["icc_21"] = { "estimate": icc_est, "ci_low": icc_lo, "ci_high": icc_hi, @@ -763,7 +923,7 @@ def sp(a, b): "example": example, } - gap_est, gap_lo, gap_hi = _bootstrap_ci_gap(wk, _icc_21, llm, human, alpha=alpha, rng=rng) + gap_est, gap_lo, gap_hi = _cigap(wk, _icc_21, llm, human, alpha=alpha, rng=rng) metrics["_bias_check"] = _build_bias_check( "Weighted Cohen's κ", metrics["weighted_kappa"]["estimate"], icc_est, gap_est, gap_lo, gap_hi, @@ -778,7 +938,7 @@ def sp(a, b): r, _ = spearmanr(a, b) return float(r) - est, lo, hi = _bootstrap_ci_2(pe, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(pe, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Pearson r") metrics["pearson_r"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -792,7 +952,7 @@ def sp(a, b): "interpretation": interp, "example": example, } - est, lo, hi = _bootstrap_ci_2(sp, llm, human, alpha=alpha, rng=rng) + est, lo, hi = _ci2(sp, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_corr(est, lo, hi, len(llm), "Spearman r") metrics["spearman_r"] = { "estimate": est, "ci_low": lo, "ci_high": hi, @@ -808,7 +968,7 @@ def sp(a, b): "example": example, } - icc_est, icc_lo, icc_hi = _bootstrap_ci_2(_icc_21, llm, human, alpha=alpha, rng=rng) + icc_est, icc_lo, icc_hi = _ci2(_icc_21, llm, human, alpha=alpha, rng=rng) band, interp, example = _interpret_icc(icc_est, icc_lo, icc_hi, len(llm), "ICC(2,1)") metrics["icc_21"] = { "estimate": icc_est, "ci_low": icc_lo, "ci_high": icc_hi, @@ -831,7 +991,7 @@ def sp(a, b): "example": example, } - gap_est, gap_lo, gap_hi = _bootstrap_ci_gap(pe, _icc_21, llm, human, alpha=alpha, rng=rng) + gap_est, gap_lo, gap_hi = _cigap(pe, _icc_21, llm, human, alpha=alpha, rng=rng) metrics["_bias_check"] = _build_bias_check( "Pearson r", metrics["pearson_r"]["estimate"], icc_est, gap_est, gap_lo, gap_hi, @@ -844,27 +1004,58 @@ def sp(a, b): # Representativeness checks # ───────────────────────────────────────────────────────────────────────────── -# Why representativeness is checked at all — shared across the score-distribution -# and slice-column checks, since both exist to answer the same question. +# Why representativeness is checked at all — shared across the score-distribution, +# slice-column, and label-contiguity checks, since all three exist to answer the +# same question. Named explicitly (not just "representative") because the +# developer needs the actual causal mechanism to avoid it next time: the natural +# QA instinct is to hand-label the items you're *unsure about* (borderline +# scores, ones the judge seemed shaky on) -- which is exactly the kind of +# selection that breaks this assumption. _REPRESENTATIVENESS_WHY = ( "The calibration model and alignment metrics above are fit only on the " - "labeled subset; if that subset isn't representative of the full item pool, " - "statistical inference may not generalize to " - "unlabeled items." + "labeled subset, and assume it's a random sample of the full item pool " + "(\"missing completely at random\", MCAR, in the statistics literature) -- " + "not, for example, the items you were most unsure about, or the " + "lowest-scoring ones. If that assumption doesn't hold, statistical " + "inference may not generalize to unlabeled items." ) +# Significance threshold for every representativeness "passed" verdict +# (score distribution, slice columns, label contiguity). Deliberately lower +# than the conventional 0.05: these are diagnostic tripwires meant to catch +# real MNAR violations, not confirmatory hypothesis tests -- and a +# well-calibrated test fires on true-null (genuinely random) data at +# whatever rate this is set to, so 0.05 means real random samples get +# flagged 1-in-20 times. 0.02 trades a bit of detection power for fewer +# false alarms crying wolf on real random data. Alignment-metric CIs +# (Pearson r, kappa, etc.) use their own, separate alpha= (default 0.05) +# and are unaffected by this constant. +_REP_ALPHA = 0.02 + + +def _rep_check_display_name(key: str) -> str: + """Human-readable label for a representativeness-check dict key, for + the short/simple summary (:meth:`AlignmentResult._summary_simple`).""" + if key == "score_distribution": + return "score distribution" + if key == "label_contiguity": + return "label position" + if key.startswith("slice_"): + return key[len("slice_"):] + return key + def _interpret_representativeness(passed: bool, subject: str) -> str: if passed: return ( - f"no evidence (p ≥ 0.05) that {subject} differs between the labeled " - "subset and the full pool — alignment estimates should generalize " - "reasonably well" + f"no evidence (p ≥ {_REP_ALPHA:g}) that {subject} differs between the " + "labeled subset and the full pool — alignment estimates should " + "generalize reasonably well" ) return ( f"{subject} differs between the labeled subset and the full pool " - "(p < 0.05) — treat alignment estimates as potentially biased for " - "unlabeled items; consider expanding or re-sampling the alignment set" + f"(p < {_REP_ALPHA:g}) — treat alignment estimates as potentially biased " + "for unlabeled items; consider expanding or re-sampling the alignment set" ) @@ -872,7 +1063,16 @@ def _check_score_distribution( all_scores: np.ndarray, labeled_scores: np.ndarray, score_type: str, + unlabeled_scores: Optional[np.ndarray] = None, ) -> dict: + """``unlabeled_scores``, when available, is used as the KS-test comparison + target instead of ``all_scores``. ``all_scores`` includes the labeled + subset by construction, so comparing against it (rather than the + unlabeled complement) dilutes any real divergence -- the more of the + pool is labeled, the more the sample resembles the thing it's being + compared to. The binary branch is unaffected: it already derives + unlabeled counts by subtraction, which is exact regardless. + """ if score_type == "binary": what = ( "Chi-square test comparing the labeled subset's 0/1 score distribution " @@ -900,14 +1100,17 @@ def _check_score_distribution( p = float(p) except ValueError: p = 1.0 - passed = p >= 0.05 + passed = p >= _REP_ALPHA msg = f"χ² p={p:.3f}" if not passed: msg += " — labeled 0/1 distribution differs from unlabeled pool" else: + compare_target = unlabeled_scores if unlabeled_scores is not None and len(unlabeled_scores) > 0 else all_scores what = ( "Kolmogorov–Smirnov test comparing the labeled subset's score " - "distribution to the full item pool's." + "distribution to " + + ("the unlabeled complement's." if unlabeled_scores is not None and len(unlabeled_scores) > 0 + else "the full item pool's (unlabeled-only comparison unavailable in this call form).") ) if len(np.unique(labeled_scores)) < 2: return { @@ -919,9 +1122,9 @@ def _check_score_distribution( "this test" ), } - _, p = ks_2samp(labeled_scores, all_scores) + _, p = ks_2samp(labeled_scores, compare_target) p = float(p) - passed = p >= 0.05 + passed = p >= _REP_ALPHA msg = f"KS p={p:.3f}" if not passed: msg += " — labeled subset appears non-representative of full score range" @@ -966,7 +1169,7 @@ def _check_slice_column( p = float(p) except ValueError: p = 1.0 - passed = p >= 0.05 + passed = p >= _REP_ALPHA msg = f"χ² p={p:.3f}" if not passed: msg += " — labeled subset is over/under-represented in some categories" @@ -977,42 +1180,460 @@ def _check_slice_column( } +def _check_slice_column_numeric( + df: pd.DataFrame, + labeled_mask: pd.Series, + col: str, +) -> dict: + """KS-test analogue of :func:`_check_slice_column` for numeric covariates + (e.g. difficulty, length, latency) -- these are never string dtype, so + the chi-square categorical check above silently skips them entirely. + """ + what = ( + f"Kolmogorov–Smirnov test comparing the distribution of numeric " + f"column {col!r} between labeled and unlabeled items." + ) + why = ( + "Checks whether the alignment set is representative across this " + "numeric covariate — important if judge accuracy might vary with it " + "(e.g. difficulty, length, latency). Categorical (string) columns are " + "checked with a chi-square test instead; this covers the numeric " + "columns that check silently skips." + ) + labeled = df.loc[labeled_mask, col].dropna().to_numpy(dtype=float) + unlabeled = df.loc[~labeled_mask, col].dropna().to_numpy(dtype=float) + if len(unlabeled) == 0: + return { + "passed": True, "message": "no unlabeled items", "p_value": None, + "what": what, "why": why, + "interpretation": ( + "not applicable — there are no unlabeled items to compare against" + ), + } + if len(np.unique(labeled)) < 2: + return { + "passed": True, "message": "insufficient labeled variation to test", "p_value": None, + "what": what, "why": why, + "interpretation": ( + "not applicable — the labeled values don't vary enough to run " + "this test" + ), + } + _, p = ks_2samp(labeled, unlabeled) + p = float(p) + passed = p >= _REP_ALPHA + msg = f"KS p={p:.3f}" + if not passed: + msg += " — labeled subset differs from unlabeled pool on this covariate" + return { + "passed": passed, "message": msg, "p_value": p, + "what": what, "why": why, + "interpretation": _interpret_representativeness(passed, f"{col!r}"), + } + + +def _apply_family_correction(results: dict[str, dict], method: str = "holm") -> dict[str, dict]: + """Apply a family-wise multiple-testing correction across a set of + representativeness checks (one per covariate), keyed by name. + + Without this, testing many slice columns inflates the chance of at least + one spurious "not representative" flag well above the nominal alpha -- + e.g. ~63% with 20 unrelated covariates under a true null at alpha=0.05, + empirically (worse at looser alpha, better at the stricter _REP_ALPHA + this module actually uses). Entries with no ``p_value`` (not-applicable + checks) pass through untouched and aren't counted in the correction + family. Only annotates the message for entries whose *raw* p was below + ``_REP_ALPHA`` (Holm-adjusted p is never smaller than the raw p, so a + raw-passing entry always still passes -- nothing to say there). + """ + testable = [k for k, v in results.items() if v.get("p_value") is not None] + if len(testable) <= 1: + return results + from evalstats.core.stats_utils import correct_pvalues + raw_p = np.array([results[k]["p_value"] for k in testable]) + adj_p = correct_pvalues(raw_p, method=method) + out = dict(results) + for k, p_adj in zip(testable, adj_p): + p_adj = float(p_adj) + res = dict(out[k]) + raw_p_k = res["p_value"] + passed = bool(p_adj >= _REP_ALPHA) + res["p_value_adjusted"] = p_adj + res["passed"] = passed + if raw_p_k < _REP_ALPHA: + if passed: + res["message"] += ( + f" — no longer significant after Holm correction across " + f"{len(testable)} covariates (adjusted p={p_adj:.3f})" + ) + else: + res["message"] += ( + f" — still significant after Holm correction across " + f"{len(testable)} covariates (adjusted p={p_adj:.3f})" + ) + res["interpretation"] = _interpret_representativeness(passed, "this covariate") + out[k] = res + return out + + +def _safe_comb(n: int, k: int) -> int: + if k < 0 or n < 0 or k > n: + return 0 + return math.comb(n, k) + + +def _count_runs(mask: np.ndarray) -> int: + """Number of maximal contiguous same-value stretches in a boolean sequence.""" + if len(mask) == 0: + return 0 + return int(1 + np.sum(mask[1:] != mask[:-1])) + + +def _runs_test_pvalue(n1: int, n2: int, r_obs: int) -> float: + """Two-sided Wald–Wolfowitz runs-test p-value for ``r_obs`` runs among + ``n1`` items of one kind and ``n2`` of another, arranged uniformly at + random. Flags both too few runs (clustering, e.g. a contiguous block or + a couple of blocks) and too many runs (suspicious regularity, e.g. every + Kth position). + + Uses the exact distribution (summed directly, cheap for realistic + dataset sizes) below ``n1 + n2 <= 4000``; falls back to the standard + normal approximation with continuity correction above that, since the + exact pmf's binomial-coefficient terms grow expensive to sum one-by-one + at that scale while the normal approximation is already excellent there. + """ + n = n1 + n2 + if n1 == 0 or n2 == 0: + return 1.0 + if n <= 4000: + total = math.comb(n, n1) + + def pmf(r: int) -> float: + if r % 2 == 0: + k = r // 2 + return 2 * _safe_comb(n1 - 1, k - 1) * _safe_comb(n2 - 1, k - 1) / total + k = (r - 1) // 2 + return ( + _safe_comb(n1 - 1, k) * _safe_comb(n2 - 1, k - 1) + + _safe_comb(n1 - 1, k - 1) * _safe_comb(n2 - 1, k) + ) / total + + p_le = sum(pmf(r) for r in range(2, r_obs + 1)) + p_ge = sum(pmf(r) for r in range(r_obs, n + 1)) + return float(min(1.0, 2 * min(p_le, p_ge))) + + mu = 1.0 + 2.0 * n1 * n2 / n + var = (2.0 * n1 * n2 * (2.0 * n1 * n2 - n1 - n2)) / (n**2 * (n - 1)) + if var <= 0: + return 1.0 + sd = math.sqrt(var) + cc = 0.5 if r_obs < mu else -0.5 + z = (r_obs - mu + cc) / sd + return float(2 * norm.sf(abs(z))) + + +def _check_label_contiguity(n_total: int, labeled_mask: np.ndarray) -> dict: + """Runs test on where the labeled items sit in the dataset. + + Unlike the distribution-based checks above, this doesn't look at scores + at all -- it only looks at *where in the dataset* the labeled items sit. + A single contiguous block (e.g. "the first N" or "the last N" items) is + the most common way evalstats has seen this assumption broken in + practice, but it's just the most extreme case of a broader failure mode: + labeled items clustered into a small number of blocks (e.g. first-N- + plus-last-N), or laid out with suspicious regularity (e.g. every Kth + row). The Wald-Wolfowitz runs test catches all of these by comparing the + observed number of contiguous same-label runs against what genuine + uniform-random sampling would produce -- even in the case where such a + selection happens to produce a labeled subset whose score distribution + passes the other checks by chance. + """ + what = ( + "Runs test on the labeled/unlabeled sequence: checks whether the " + "labeled items form too few contiguous blocks (clustering, e.g. " + "rows 0-14, or first-15-plus-last-15) or too many (suspicious " + "regularity, e.g. every 10th row) to be a uniformly random subset." + ) + why = ( + "The most common way evalstats has seen this assumption broken in " + "practice isn't a subtle score-distribution skew -- it's literally " + "labeling \"the first N\" or \"the last N\" items, often just because " + "that's what a spreadsheet or a `.head()` call hands you first. A " + "runs test catches that pattern and its variants (e.g. a couple of " + "blocks, or artificially regular spacing) in one check, rather than " + "only the single-contiguous-block special case." + ) + n_labeled = int(labeled_mask.sum()) + n_unlabeled = n_total - n_labeled + if n_labeled < 2 or n_unlabeled < 2: + return { + "passed": True, "message": "not applicable", "p_value": None, + "what": what, "why": why, + "interpretation": ( + "not applicable -- fewer than 2 labeled or 2 unlabeled items, " + "so there's no position pattern to check" + ), + } + mask = labeled_mask.astype(bool) + r_obs = _count_runs(mask) + p = _runs_test_pvalue(n_labeled, n_unlabeled, r_obs) + mu = 1.0 + 2.0 * n_labeled * n_unlabeled / n_total + passed = p >= _REP_ALPHA + + positions = np.flatnonzero(mask) + span = int(positions.max() - positions.min() + 1) + is_single_block = span == n_labeled and r_obs <= 2 + + if not passed: + if is_single_block: + start, end = int(positions.min()), int(positions.max()) + msg = ( + f"the {n_labeled} labeled items are exactly rows {start}-{end} " + f"of {n_total} -- a single contiguous block ({r_obs} run(s) vs. " + f"~{mu:.0f} expected under random selection, p={p:.2e})" + ) + elif r_obs < mu: + msg = ( + f"labeled item positions form only {r_obs} contiguous run(s), " + f"vs. ~{mu:.0f} expected under random selection (p={p:.2e}) -- " + "looks like a small number of blocks (e.g. first-N-plus-" + "last-N) rather than a scattered random sample" + ) + else: + msg = ( + f"labeled item positions form {r_obs} runs, far more than " + f"the ~{mu:.0f} expected under random selection (p={p:.2e}) " + "-- looks like an artificially regular pattern (e.g. every " + "Kth row) rather than genuine random sampling" + ) + else: + msg = ( + f"labeled item positions look scattered ({r_obs} runs, " + f"~{mu:.0f} expected under random selection, p={p:.2f})" + ) + + if passed: + interpretation = ( + "the labeled items' positions don't form a suspicious clustered " + "or artificially regular pattern -- doesn't confirm random " + "selection, but rules out the most common non-random patterns" + ) + else: + interpretation = ( + "the labeled items' positions are essentially impossible from " + "real random sampling -- treat alignment estimates as unreliable " + "for unlabeled items unless this was deliberate (e.g. the " + "dataset itself was already shuffled before labeling); consider " + "re-sampling the alignment set uniformly at random instead" + ) + return { + "passed": passed, "message": msg, "p_value": p, + "what": what, "why": why, + "interpretation": interpretation, + } + + # ───────────────────────────────────────────────────────────────────────────── -# validate_alignment +# judge_alignment # ───────────────────────────────────────────────────────────────────────────── -def validate_alignment( - evaldata, +_VALID_SELECTIONS = ("random", "stratified", "manual", "unknown") + + +def _judge_alignment_core( + llm_aligned: np.ndarray, + human_aligned: np.ndarray, + score_type: str, *, llm_metric: str, human_groundtruth: str, - alpha: float = 0.05, + alpha: float, + n_total: int, + ci: bool = True, + all_llm: Optional[np.ndarray] = None, + slice_df: Optional[pd.DataFrame] = None, + slice_labeled_mask: Optional[pd.Series] = None, + slice_exclude_cols: frozenset = frozenset(), + labeled_mask: Optional[np.ndarray] = None, + selection: str = "unknown", + test: Optional[str] = None, + warn_stacklevel: int = 3, ) -> AlignmentResult: - """Validate how well an LLM judge aligns with human graders. + """Shared core behind both :func:`judge_alignment` call forms: fits the + calibration model, computes alignment metrics, and (only when the + relevant context is available) runs representativeness diagnostics. + + ``all_llm`` enables the score-distribution check; ``slice_df`` + + ``slice_labeled_mask`` enable the categorical slice-column checks; a + non-``None`` ``labeled_mask`` (positions of labeled items within the + ``n_total``-length item pool, in dataset row order) enables the + label-contiguity check. All three require the full item pool / other + columns, so they're skipped entirely -- not silently approximated -- + when this is called from raw paired arrays with no further context, + see :func:`judge_alignment`. + """ + if selection not in _VALID_SELECTIONS: + raise ValueError( + f"selection={selection!r} -- must be one of {_VALID_SELECTIONS}." + ) + n_labeled = int(len(llm_aligned)) - Designed for the common case where LLM judge scores exist for all items - but human labels are available for only a subset (the alignment set). - Fits a Bayesian calibration model that can later be used to propagate - judge uncertainty into downstream comparisons via - ``compare(alignment={metric: result})``. + calibration = _fit_calibration(llm_aligned, human_aligned, score_type) - Parameters - ---------- - evaldata : EvalResults - Evaluation data from :func:`load_from`. Must contain both - ``llm_metric`` and ``human_groundtruth`` as columns. - llm_metric : str - Column name of the LLM judge scores. Must be present for all rows. - human_groundtruth : str - Column name of the human rater scores. Expected to be sparsely - populated: non-null for the alignment subset, ``NaN`` elsewhere. - alpha : float - Significance level for alignment metric CIs. Default ``0.05``. + rng = np.random.default_rng(42) + alignment_metrics = _compute_alignment_metrics( + llm_aligned, human_aligned, score_type, alpha=alpha, rng=rng, ci=ci + ) + bias_check = alignment_metrics.pop("_bias_check", None) - Returns - ------- - AlignmentResult - """ + rep: dict = {} + if all_llm is not None: + unlabeled_llm = None + if labeled_mask is not None and len(labeled_mask) == len(all_llm): + unlabeled_llm = all_llm[~labeled_mask.astype(bool)] + dist_result = _check_score_distribution( + all_llm, llm_aligned, score_type, unlabeled_scores=unlabeled_llm + ) + rep["score_distribution"] = dist_result + if not dist_result["passed"]: + warnings.warn( + f"Representativeness warning: the {n_labeled} labeled items appear to have " + f"a different {llm_metric} distribution than the full item pool " + f"({dist_result['message']}). " + "Alignment uncertainty estimates may not generalise to all items. " + "Consider sampling human labels more broadly across the score range.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + if slice_df is not None and slice_labeled_mask is not None: + cat_cols = [ + c for c in slice_df.columns + if c not in slice_exclude_cols + and pd.api.types.is_string_dtype(slice_df[c]) + and 1 < slice_df[c].nunique() <= 20 + ] + num_cols = [ + c for c in slice_df.columns + if c not in slice_exclude_cols + and pd.api.types.is_numeric_dtype(slice_df[c]) + and not pd.api.types.is_bool_dtype(slice_df[c]) + and slice_df[c].nunique() > 1 + ] + slice_results: dict = {} + for col in cat_cols: + slice_results[col] = _check_slice_column(slice_df, slice_labeled_mask, col) + for col in num_cols: + slice_results[col] = _check_slice_column_numeric(slice_df, slice_labeled_mask, col) + + # Correct across the whole covariate family jointly (not per-column) -- + # testing many slice columns otherwise inflates the false-alarm rate + # well above the nominal 5% (empirically ~63% at 20 columns). + slice_results = _apply_family_correction(slice_results, method="holm") + + for col, col_result in slice_results.items(): + rep[f"slice_{col}"] = col_result + if not col_result["passed"]: + warnings.warn( + f"Representativeness warning for column '{col}': the labeled subset " + f"appears unevenly distributed across categories " + f"({col_result['message']}). " + "Consider stratified sampling of human labels.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + if labeled_mask is not None: + contiguity_result = _check_label_contiguity(n_total, labeled_mask) + rep["label_contiguity"] = contiguity_result + if not contiguity_result["passed"]: + warnings.warn( + f"Representativeness warning: {contiguity_result['message']}. " + "This looks like 'the first N' or 'the last N' items were " + "labeled rather than a random sample. Consider re-sampling " + "the alignment set uniformly at random.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + if selection == "unknown": + warnings.warn( + "judge_alignment() was not told how the labeled subset was " + "selected (selection=). Every correction it and " + "compare(alignment=...) apply assumes the labeled items are a " + "random sample of the full item pool -- pass selection='random' " + "to confirm that's the case, or selection='manual'/'stratified' " + "if not, so this is a deliberate acknowledgment rather than an " + "unexamined default.", + UserWarning, + stacklevel=warn_stacklevel, + ) + elif selection == "manual": + warnings.warn( + "selection='manual': the labeled subset was NOT randomly " + "sampled. PPI/alignment correction assumes random sampling " + "(MCAR) to be valid -- with a manually-chosen subset, the " + "corrected estimates and CIs compare()/judge_alignment() report " + "may be miscalibrated, not just imprecise. Treat them as " + "informal unless the alignment set is re-sampled at random.", + UserWarning, + stacklevel=warn_stacklevel, + ) + elif selection == "stratified": + warnings.warn( + "selection='stratified': evalstats' current correction doesn't " + "account for stratification weights, so this is only valid if " + "each stratum was itself sampled uniformly at random and the " + "strata are otherwise ignorable for the metric being judged. " + "If items were hand-picked within strata, treat corrected " + "estimates as potentially biased, same as selection='manual'.", + UserWarning, + stacklevel=warn_stacklevel, + ) + + for key in ("pearson_r", "spearman_r"): + if key in alignment_metrics: + mult, n_eff = _n_eff(alignment_metrics[key]["estimate"], n_labeled, n_total) + alignment_metrics[key]["multiplier"] = mult + alignment_metrics[key]["n_eff"] = n_eff + + test_metric = None + if test is not None: + if test != "mean_estimate": + raise ValueError( + f"test={test!r} needs a comparison (2+ conditions) -- pass a " + "{name: (judge_scores, human_scores)} dict instead of plain " + "arrays, or use test='mean_estimate' for a single-condition " + "estimate (no comparison)." + ) + test_metric = dict(alignment_metrics["pearson_r"]) + test_metric["label"] = "mean_estimate rho" + + return AlignmentResult( + llm_metric=llm_metric, + human_col=human_groundtruth, + score_type=score_type, + n_labeled=n_labeled, + n_total=n_total, + calibration=calibration, + alignment_metrics=alignment_metrics, + representativeness=rep, + bias_check=bias_check, + selection=selection, + test=test, + test_metric=test_metric, + ) + + +def _judge_alignment_from_evaldata( + evaldata, + *, + llm_metric: str, + human_groundtruth: str, + alpha: float, + selection: str = "unknown", + ci: bool = True, +) -> AlignmentResult: df = evaldata._df if llm_metric not in df.columns: @@ -1042,10 +1663,9 @@ def validate_alignment( "Alignment estimates will be imprecise with fewer than ~30 labeled items; " "consider expanding the alignment set for reliable uncertainty propagation.", UserWarning, - stacklevel=2, + stacklevel=3, ) - # Resolve score type score_type = evaldata._score_types.get(llm_metric) if score_type is None: from evalstats.loader import _detect_score_type @@ -1055,59 +1675,1101 @@ def validate_alignment( human_aligned = df.loc[labeled_mask, human_groundtruth].to_numpy(dtype=float) all_llm = df[llm_metric].to_numpy(dtype=float) - # Fit Bayesian calibration model - calibration = _fit_calibration(llm_aligned, human_aligned, score_type) + # Structural role columns (model/item/run) are row/group identifiers, not + # domain covariates -- an "item" column is frequently just a sequential + # index (or unique per row), which the numeric-covariate check would + # otherwise happily test, redundantly rediscovering (in a noisier form) + # exactly what the position-based label-contiguity check already covers. + structural_cols = { + c for c in (evaldata._col.get("model"), evaldata._col.get("item"), evaldata._col.get("run")) + if c is not None + } - # Compute alignment metrics with bootstrap CIs - rng = np.random.default_rng(42) - alignment_metrics = _compute_alignment_metrics( - llm_aligned, human_aligned, score_type, alpha=alpha, rng=rng + return _judge_alignment_core( + llm_aligned, human_aligned, score_type, + llm_metric=llm_metric, human_groundtruth=human_groundtruth, + alpha=alpha, n_total=n_total, all_llm=all_llm, + slice_df=df, slice_labeled_mask=labeled_mask, + slice_exclude_cols=frozenset({llm_metric, human_groundtruth}) | structural_cols, + labeled_mask=labeled_mask.to_numpy(), selection=selection, ci=ci, + warn_stacklevel=4, ) - bias_check = alignment_metrics.pop("_bias_check", None) - # Representativeness: score distribution - rep: dict = {} - dist_result = _check_score_distribution(all_llm, llm_aligned, score_type) - rep["score_distribution"] = dist_result - if not dist_result["passed"]: + +def _judge_alignment_from_arrays( + judge_scores: np.ndarray, + human_scores: np.ndarray, + *, + all_judge_scores: Optional[np.ndarray], + score_type: Optional[str], + llm_metric: Optional[str], + human_groundtruth: Optional[str], + alpha: float, + selection: str = "unknown", + test: Optional[str] = None, + ci: bool = True, +) -> AlignmentResult: + judge_full = np.asarray(judge_scores, dtype=float) + human_full = np.asarray(human_scores, dtype=float) + if judge_full.shape != human_full.shape: + raise ValueError( + "judge_scores and human_scores must be the same length -- one " + "judge score + one (possibly NaN) human score per item; got " + f"shapes {judge_full.shape} and {human_full.shape}." + ) + if judge_full.ndim != 1: + raise ValueError( + f"judge_scores/human_scores must be 1-D; got shape {judge_full.shape}." + ) + + labeled_mask = ~np.isnan(human_full) + n_labeled = int(labeled_mask.sum()) + if n_labeled == 0: + raise ValueError( + "No labeled items -- human_scores is all NaN. It should be " + "non-NaN for the alignment subset and NaN elsewhere (or, if " + "every item is labeled, contain no NaN at all)." + ) + llm_aligned = judge_full[labeled_mask] + human_aligned = human_full[labeled_mask] + + if n_labeled < 30: warnings.warn( - f"Representativeness warning: the {n_labeled} labeled items appear to have " - f"a different {llm_metric} distribution than the full item pool " - f"({dist_result['message']}). " - "Alignment uncertainty estimates may not generalise to all items. " - "Consider sampling human labels more broadly across the score range.", + f"Only {n_labeled} items have human labels. " + "Alignment estimates will be imprecise with fewer than ~30 labeled items; " + "consider expanding the alignment set for reliable uncertainty propagation.", UserWarning, - stacklevel=2, + stacklevel=3, ) - # Representativeness: categorical slice columns - slice_cols = [ - c for c in df.columns - if c not in {llm_metric, human_groundtruth} - and pd.api.types.is_string_dtype(df[c]) - and 1 < df[c].nunique() <= 20 - ] - for col in slice_cols: - col_result = _check_slice_column(df, labeled_mask, col) - rep[f"slice_{col}"] = col_result - if not col_result["passed"]: - warnings.warn( - f"Representativeness warning for column '{col}': the labeled subset " - f"appears unevenly distributed across categories " - f"({col_result['message']}). " - "Consider stratified sampling of human labels.", - UserWarning, - stacklevel=2, + # judge_scores doubles as "every item's judge score" for the + # representativeness check for free -- but only when there's actual + # evidence it's the full pool (some items weren't labeled). When + # n_labeled == judge_full.size (no NaN at all in human_scores), there's + # no way to tell "this is the full pool, 100% labeled" apart from "this + # is just the labeled subset the caller already extracted" -- stay + # conservative and skip the check rather than silently comparing a set + # against itself (which would trivially "pass" and could read as false + # confidence). An explicit all_judge_scores= always wins either way. + # Position-based (label-contiguity) check needs labeled_mask to actually + # index into all_llm -- only true when all_llm *is* judge_full itself. + # An explicit all_judge_scores= has no known positional correspondence + # to judge_scores/human_scores, so the check is skipped rather than + # guessed at. + position_mask = None + if all_judge_scores is not None: + all_llm = np.asarray(all_judge_scores, dtype=float) + elif n_labeled < judge_full.size: + all_llm = judge_full + position_mask = labeled_mask + else: + all_llm = None + n_total = int(all_llm.size) if all_llm is not None else n_labeled + + if score_type is None: + from evalstats.loader import _detect_score_type + score_type = _detect_score_type(pd.Series(llm_aligned)) + + return _judge_alignment_core( + llm_aligned, human_aligned, score_type, + llm_metric=llm_metric or "judge", human_groundtruth=human_groundtruth or "human", + alpha=alpha, n_total=n_total, all_llm=all_llm, + slice_df=None, slice_labeled_mask=None, + labeled_mask=position_mask, selection=selection, test=test, ci=ci, + warn_stacklevel=4, + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Multi-condition (pairwise) alignment -- within-subjects and between-subjects +# comparisons across 2+ named conditions, and the label-efficiency numbers a +# PPI-corrected hypothesis test's savings depend on. +# ───────────────────────────────────────────────────────────────────────────── + +_VALID_DESIGNS = ("within", "between") + +# Which correlation governs each evalstats.tests function's PPI variance +# reduction, precisely -- NOT a fixed "Pearson for mean tests, Spearman for +# rank tests" recipe (an earlier version of this table used that split; it's +# WRONG for rank tests). Every test's rho is actually a Pearson correlation +# on a test-specific LINEARIZATION of the raw values -- identity for +# mean-type tests (whose influence function psi(y)=y-mu is already linear, +# hence exactly effect-size-invariant), but a genuine transform for rank-type +# tests, whose named/raw-Spearman recipe DRIFTS with effect size (confirmed +# via Monte Carlo: -13% to -38% at d=2 for mwu/kruskal/wilcoxon, -89% for +# friedman at higher effects -- see notes/omnibus_label_efficiency.html and +# the git history around commits 8460a16/eca96d8/23ffbc5). See +# _linearize_for_test for the dispatch and each _linearize_* function for +# the actual recipe + validation provenance. +# +# design: the design each test implies, or None if the caller must say +# ("within"/"between" both valid, e.g. ttest/anova_oneway paired vs +# independent). min_k/max_k: condition-count bounds (None = unbounded). +_TEST_STRUCTURE = { + "ttest": {"design": None, "min_k": 2, "max_k": 2}, + "wilcoxon": {"design": "within", "min_k": 2, "max_k": 2}, + "mannwhitney": {"design": "between", "min_k": 2, "max_k": 2}, + "anova_oneway": {"design": None, "min_k": 2, "max_k": None}, + "kruskalwallis": {"design": "between", "min_k": 2, "max_k": None}, + "friedman": {"design": "within", "min_k": 2, "max_k": None}, + "mean_estimate": {"design": None, "min_k": 1, "max_k": 1}, +} + + +def _linearize_mean(conditions: dict, design: str) -> tuple[np.ndarray, np.ndarray]: + """Identity linearization for mean-type tests (ttest, anova_oneway) -- + Pearson r on (possibly centered/differenced) raw scores IS the governing + correlation, since a mean's influence function psi(y)=y-mu is linear and + hence exactly effect-size-invariant; no rank/placement transform needed. + Generalizes the 2-condition pairwise recipe to k conditions: + + design="within": a plain paired difference at k=2 (same as + _condition_pair_arrays); DOUBLY centered (each participant's own mean + AND each condition's mean removed) at k>2 -- row-centering alone leaks + the condition effect into the correlation (the shared between-condition + mean judge and humans both track contributes no cross-participant + variance, but a row-only-centered recipe still credits it), confirmed to + be exactly what makes repeated-measures ANOVA's recipe effect-invariant + in notes/omnibus_label_efficiency.html's Method 3 (flat at rho^2=0.646 + for d=0..1.0 there; row-centering alone climbs 0.640->0.862). + + design="between": each condition centered on its own mean, then pooled + (concatenated) -- the within-group pooled correlation validated for + anova_oneway in the same note's Method 1; reduces to the existing + 2-condition recipe at k=2. + """ + names = list(conditions.keys()) + arrs = {n: (np.asarray(j, dtype=float), np.asarray(h, dtype=float)) for n, (j, h) in conditions.items()} + + if design == "within": + lengths = {len(j) for j, h in arrs.values()} + if len(lengths) != 1: + raise ValueError( + "design='within' requires every condition to have the same " + "length (same items/participants in the same order)." + ) + judge_mat = np.column_stack([arrs[n][0] for n in names]) + human_mat = np.column_stack([arrs[n][1] for n in names]) + overlap = ~np.isnan(human_mat).any(axis=1) + judge_mat, human_mat = judge_mat[overlap], human_mat[overlap] + if len(names) == 2: + judge = judge_mat[:, 0] - judge_mat[:, 1] + human = human_mat[:, 0] - human_mat[:, 1] + else: + def double_center(m: np.ndarray) -> np.ndarray: + return m - m.mean(axis=1, keepdims=True) - m.mean(axis=0, keepdims=True) + m.mean() + judge = double_center(judge_mat).ravel() + human = double_center(human_mat).ravel() + else: + judge_parts, human_parts = [], [] + for n in names: + j, h = arrs[n] + mask = ~np.isnan(h) + jj, hh = j[mask], h[mask] + if len(jj) == 0: + continue + judge_parts.append(jj - jj.mean()) + human_parts.append(hh - hh.mean()) + judge = np.concatenate(judge_parts) if judge_parts else np.array([]) + human = np.concatenate(human_parts) if human_parts else np.array([]) + return judge, human + + +def _linearize_wilcoxon(conditions: dict) -> tuple[np.ndarray, np.ndarray]: + """Hajek-projection linearization for Wilcoxon signed-rank (paired, + exactly 2 conditions): the judge-side and human-side paired differences + are each mapped through ``evalstats.ppi._walsh_theta_h1_components``, + the per-item empirical Hajek projection ``h1(d) = P(D > -d)`` (mid-ranks + for ties) of the Walsh/Hodges-Lehmann estimand ``wilcoxon()`` actually + uses. That is the SAME production function + ``_analytic_walsh_theta_correct``/``_walsh_theta_analytic_variance`` + already build their variance estimates from, so the correlation + reported here is taken against the very quantity the correction's own + variance is computed on, rather than a re-derived lookalike. + + Note this is deliberately NOT ``sign(d) * (2*F_{|D|}(|d|) - 1)``: that + expands to ``4*F_D(d) - sign(d) - 2``, which is not affine in + ``F_D(d)`` (the ``sign`` term survives) and is non-monotonic in ``d``, + returning about -1 just above zero and about +1 just below it. An + earlier version of this function used exactly that (borrowed from the + since-removed ``hajek_experimental`` path) and measured about 0.6x the + directly-measured ``Var(classical)/Var(PPI)``. + + Replaces the raw-Spearman-of-differences recipe, which drifts -25% by + d=2 (notes/omnibus_label_efficiency.html).""" + from evalstats.ppi import _walsh_theta_h1_components + + names = list(conditions.keys()) + if len(names) != 2: + raise ValueError(f"wilcoxon needs exactly 2 conditions, got {len(names)}.") + (ja, ha), (jb, hb) = conditions[names[0]], conditions[names[1]] + ja, ha = np.asarray(ja, dtype=float), np.asarray(ha, dtype=float) + jb, hb = np.asarray(jb, dtype=float), np.asarray(hb, dtype=float) + if not (len(ja) == len(ha) == len(jb) == len(hb)): + raise ValueError("wilcoxon requires both conditions to have the same length (same items in the same order).") + mask = ~np.isnan(ha) & ~np.isnan(hb) + judge = _walsh_theta_h1_components(ja[mask] - jb[mask]) + human = _walsh_theta_h1_components(ha[mask] - hb[mask]) + return judge, human + + +def _linearize_mannwhitney(conditions: dict) -> tuple[np.ndarray, np.ndarray]: + """Empirical placement-value linearization for Mann-Whitney/Wilcoxon + rank-sum (independent groups, exactly 2 conditions), the influence + function of the ``theta = P(X > Y)`` estimand ``mannwhitney()`` uses. + + For item ``x_i`` in group A the score is ``F_Y(x_i)``, its mid-rank + placement within group B; for item ``y_j`` in group B it is + ``P(X > y_j) = 1 - F_X(y_j)``. Built on the same searchsorted mid-rank + construction already used and tested in + ``evalstats.tests._p_x_gt_y_midrank`` for the point estimate itself, + extracted PER ITEM instead of summed to one ``P(X > Y)`` number. + + Both halves are then centered on their OWN mean before pooling. Two + reasons, and skipping either one was a real measured bug: + + 1. Sign, not negation. An earlier version scored group B as + ``-F_X(y_j)`` rather than ``1 - F_X(y_j)``. Both have the same + spread, but their MEANS differ by 1 (``theta - 1`` vs ``theta``), + so pooling put the two halves a constant ~1.0 apart on both the + judge and human side -- a lockstep offset that both sides share and + that Pearson therefore scores as agreement. Measured effect: rho^2 + inflated to ~0.92-0.98 and the predicted multiplier roughly 2x the + directly-measured ``Var(classical)/Var(PPI)``. + 2. Per-group centering. Even with the correct sign, the pooled + correlation must be the WITHIN-group one: any between-group + difference in mean placement is shared by judge and humans and + would again be counted as agreement. This is the same failure mode + -- and the same fix -- as the uncentered pooling corrected in + ``_pooled_two_group_lambda``, and as ``_linearize_mean``'s + "between" branch, which centers each condition before pooling for + exactly this reason. + + The raw-Spearman recipe this replaces drifts -13% by d=2 + (notes/omnibus_label_efficiency.html). + + Coarse-scale caveat (likert): placement values take only ~k distinct + levels on a k-point scale, so the influence function loses most of its + spread and MWU/kruskal run conservative. Paired rank tests + (wilcoxon/friedman) are unaffected -- they score differences, not + cross-group comparisons. See api._ppi_pairwise's mannwhitney branch.""" + names = list(conditions.keys()) + if len(names) != 2: + raise ValueError(f"mannwhitney needs exactly 2 conditions, got {len(names)}.") + (ja, ha), (jb, hb) = conditions[names[0]], conditions[names[1]] + ja, ha = np.asarray(ja, dtype=float), np.asarray(ha, dtype=float) + jb, hb = np.asarray(jb, dtype=float), np.asarray(hb, dtype=float) + mask_a, mask_b = ~np.isnan(ha), ~np.isnan(hb) + ja, ha = ja[mask_a], ha[mask_a] + jb, hb = jb[mask_b], hb[mask_b] + + def placement(x: np.ndarray, y: np.ndarray) -> np.ndarray: + if len(y) == 0: + return np.zeros_like(x) + y_sorted = np.sort(y) + n_lt = np.searchsorted(y_sorted, x, side="left") + n_le = np.searchsorted(y_sorted, x, side="right") + return (n_lt + 0.5 * (n_le - n_lt)) / len(y) + + def _pool(a_scores: np.ndarray, b_scores: np.ndarray) -> np.ndarray: + if len(a_scores) == 0 or len(b_scores) == 0: + return np.array([]) + return np.concatenate([a_scores - a_scores.mean(), b_scores - b_scores.mean()]) + + judge = _pool(placement(ja, jb), 1.0 - placement(jb, ja)) + human = _pool(placement(ha, hb), 1.0 - placement(hb, ha)) + return judge, human + + +def _linearize_kruskal(conditions: dict) -> tuple[np.ndarray, np.ndarray]: + """Spearman of within-condition-CENTERED, pooled values -- the + validated recipe for Kruskal-Wallis (notes/omnibus_label_efficiency.html + Method 2): each condition's judge/human values centered on that + condition's own mean (removing the between-condition location signal, + exactly like _linearize_mean's "between" branch), THEN pooled + (concatenated) across conditions, THEN rank-transformed as one combined + array -- NOT ranked within each condition separately first. Spearman + correlation of the pooled-then-globally-ranked residuals is, by + definition, Pearson correlation of their ranks; that's what's returned + here for the caller to correlate. (Ranking within each condition + separately before pooling -- an earlier, wrong version of this + function -- discards the very between-condition-relative information + centering-then-pooling is supposed to preserve, and empirically came + out suspiciously perfectly flat across effect sizes, unlike the note's + documented "mild drift" -- a sign it wasn't computing the validated + recipe.) + + VALIDATED against the note's published figures: with its fixed judge + (within-condition rho^2 = 0.64) this returns 0.6151 where the note + reports 0.620 for the same recipe. + + INHERITED CAVEAT, documented in the note and not fixed here: this + recipe is effect-invariant (flat at 0.6151 for d=0.5 and d=1.0) while + the TRUE implied rho^2 falls (0.606 -> 0.578 over that range), so it + runs mildly optimistic -- about 8% on N_eff at d=1.0 -- and more so + further out. Treat Kruskal-Wallis's number as a ceiling rather than a + point estimate when a large effect is expected. This is the same + rank-drift phenomenon that hits Friedman much harder, in mild form; + unlike Friedman (see :func:`_linearize_friedman`), no + doubly-centred/plug-in replacement for it has been validated.""" + from scipy.stats import rankdata + + judge_parts, human_parts = [], [] + for j, h in conditions.values(): + j, h = np.asarray(j, dtype=float), np.asarray(h, dtype=float) + mask = ~np.isnan(h) + jj, hh = j[mask], h[mask] + if len(jj) == 0: + continue + judge_parts.append(jj - jj.mean()) + human_parts.append(hh - hh.mean()) + judge_pooled = np.concatenate(judge_parts) if judge_parts else np.array([]) + human_pooled = np.concatenate(human_parts) if human_parts else np.array([]) + return rankdata(judge_pooled), rankdata(human_pooled) + + +def _linearize_friedman(conditions: dict) -> tuple[np.ndarray, np.ndarray]: + """Doubly-centered within-subject ranks -- the validated recipe for + Friedman (notes/omnibus_label_efficiency.html Method 4): rank each + participant's k conditions (row-wise) for judge and humans alike, + subtract each condition's (column) mean rank, correlate the pooled + residuals. Emphatically NOT the average per-participant Spearman (an + earlier, wrong version of this function's design) -- that recipe moves + in the OPPOSITE direction from the truth as effect size grows (rises + while the truth falls), reaching +89% N_eff overstatement at k=5, d=1.0 + in the note's measurements. The row-wise rank transform substitutes for + row-centering (ranks are already row-normalized by construction); only + the column (condition) mean needs explicit removal. + + VALIDATED against the note's published figures: with its fixed judge + (within-condition rho^2 = 0.64, k=3) this returns 0.4106 / 0.3953 / + 0.3628 at d = 0.0 / 0.5 / 1.0, against the note's 0.409 / 0.394 / + 0.356 for the same recipe -- within 0.007 throughout, and correctly + FALLING with effect size, tracking the note's implied 0.422 / 0.388 / + 0.348 rather than rising the way the naive average per-participant + Spearman does.""" + from scipy.stats import rankdata + + names = list(conditions.keys()) + arrs = {n: (np.asarray(j, dtype=float), np.asarray(h, dtype=float)) for n, (j, h) in conditions.items()} + lengths = {len(j) for j, h in arrs.values()} + if len(lengths) != 1: + raise ValueError( + "friedman requires every condition to have the same length " + "(same items/participants in the same order)." + ) + judge_mat = np.column_stack([arrs[n][0] for n in names]) + human_mat = np.column_stack([arrs[n][1] for n in names]) + overlap = ~np.isnan(human_mat).any(axis=1) + judge_mat, human_mat = judge_mat[overlap], human_mat[overlap] + judge_ranks = rankdata(judge_mat, axis=1, method="average") + human_ranks = rankdata(human_mat, axis=1, method="average") + judge = (judge_ranks - judge_ranks.mean(axis=0, keepdims=True)).ravel() + human = (human_ranks - human_ranks.mean(axis=0, keepdims=True)).ravel() + return judge, human + + +def _linearize_for_test( + conditions: dict, *, test: str, design: Optional[str], +) -> tuple[np.ndarray, np.ndarray, str]: + """Dispatch to the right _linearize_* function for `test`, validating + condition count and design against _TEST_STRUCTURE first. Returns + (judge_linearized, human_linearized, resolved_design).""" + if test not in _TEST_STRUCTURE: + raise ValueError(f"Unrecognized test={test!r}. Known: {sorted(_TEST_STRUCTURE)}.") + spec = _TEST_STRUCTURE[test] + k = len(conditions) + if k < spec["min_k"] or (spec["max_k"] is not None and k > spec["max_k"]): + bound = f"exactly {spec['min_k']}" if spec["min_k"] == spec["max_k"] else f"at least {spec['min_k']}" + raise ValueError(f"test={test!r} needs {bound} condition(s), got {k}.") + + implied = spec["design"] + if implied is not None: + if design is not None and design != implied: + raise ValueError(f"test={test!r} is always design={implied!r}; design={design!r} conflicts.") + design = implied + elif design is None and test != "mean_estimate": + raise ValueError(f"test={test!r} needs an explicit design= ('within' or 'between').") + + if test in ("ttest", "anova_oneway"): + judge, human = _linearize_mean(conditions, design) + elif test == "wilcoxon": + judge, human = _linearize_wilcoxon(conditions) + elif test == "mannwhitney": + judge, human = _linearize_mannwhitney(conditions) + elif test == "kruskalwallis": + judge, human = _linearize_kruskal(conditions) + elif test == "friedman": + judge, human = _linearize_friedman(conditions) + else: # mean_estimate + (j, h), = conditions.values() + j, h = np.asarray(j, dtype=float), np.asarray(h, dtype=float) + mask = ~np.isnan(h) + judge, human = j[mask], h[mask] + return judge, human, design + + +def _pearson_spearman_metrics( + judge: np.ndarray, human: np.ndarray, *, alpha: float, rng: np.random.Generator, + pearson_label: str, spearman_label: str, what_suffix: str = "", +) -> dict: + """Pearson r and Spearman r (point estimate + bootstrap CI) between two + already-prepared 1-D arrays -- the shared low-level computation behind + the multi-condition pairwise path below, so there is exactly one place + this math lives. Callers are responsible for whatever + differencing/pooling/masking the two arrays need before calling this + (see :func:`_condition_pair_arrays`). + """ + n = len(judge) + + def pe(a, b): + r, _ = pearsonr(a, b) + return float(r) + + def sp(a, b): + r, _ = spearmanr(a, b) + return float(r) + + est, lo, hi = _bootstrap_ci_2(pe, judge, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, n, pearson_label) + pearson_entry = { + "estimate": est, "ci_low": lo, "ci_high": hi, "label": pearson_label, "band": band, "n": n, + "what": f"Linear correlation coefficient between judge and human values{what_suffix}.", + "why": ( + "Governs the label-efficiency multiplier for parametric/mean-based " + "tests (t-test, ANOVA, mean estimation) -- see judge_alignment()'s test=." + ), + "interpretation": interp, "example": example, + } + est, lo, hi = _bootstrap_ci_2(sp, judge, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, n, spearman_label) + spearman_entry = { + "estimate": est, "ci_low": lo, "ci_high": hi, "label": spearman_label, "band": band, "n": n, + "what": f"Rank correlation coefficient between judge and human values{what_suffix}.", + "why": ( + "Governs the label-efficiency multiplier for rank-based tests " + "(Mann-Whitney, Wilcoxon, Friedman) -- see judge_alignment()'s test=." + ), + "interpretation": interp, "example": example, + } + return {"pearson_r": pearson_entry, "spearman_r": spearman_entry} + + +def _condition_pair_arrays( + judge_a, human_a, judge_b, human_b, *, design: str, label_a: str, label_b: str, +) -> tuple[np.ndarray, np.ndarray]: + """Reduce one pair of conditions to the two 1-D arrays whose correlation + actually governs that pair's PPI variance reduction, per `design`: + + "within" (paired/repeated-measures): the estimand is a function of the + per-item DIFFERENCE between conditions, so the governing correlation is + between the two conditions' differences -- Corr(judge_a - judge_b, + human_a - human_b) -- not between either condition's raw scores. Uses + only items labeled in BOTH conditions (the overlap). + + "between" (independent groups): the estimand spans both groups, so the + governing correlation is the WITHIN-GROUP pooled one -- each condition's + judge/human values centered on their own mean, then concatenated. Plain + pooling without per-group centering would inflate/deflate the + correlation by the between-group difference itself, which isn't part of + what the control variate is being credited for. + """ + judge_a = np.asarray(judge_a, dtype=float) + human_a = np.asarray(human_a, dtype=float) + judge_b = np.asarray(judge_b, dtype=float) + human_b = np.asarray(human_b, dtype=float) + + if design == "within": + if not (len(judge_a) == len(human_a) == len(judge_b) == len(human_b)): + raise ValueError( + f"design='within' requires {label_a!r} and {label_b!r} to have the " + "same length (same items in the same order) -- pass NaN in the " + "human array for items without a label, not a shorter array." + ) + mask = ~np.isnan(human_a) & ~np.isnan(human_b) + judge = judge_a[mask] - judge_b[mask] + human = human_a[mask] - human_b[mask] + else: + mask_a = ~np.isnan(human_a) + mask_b = ~np.isnan(human_b) + ja, ha = judge_a[mask_a], human_a[mask_a] + jb, hb = judge_b[mask_b], human_b[mask_b] + judge = np.concatenate([ja - ja.mean(), jb - jb.mean()]) if len(ja) and len(jb) else np.array([]) + human = np.concatenate([ha - ha.mean(), hb - hb.mean()]) if len(ha) and len(hb) else np.array([]) + + if len(judge) < 3: + raise ValueError( + f"Not enough overlapping labeled items between {label_a!r} and " + f"{label_b!r} (n={len(judge)}) to compute a correlation." + ) + return judge, human + + +def _single_metric( + judge: np.ndarray, human: np.ndarray, *, alpha: float, rng: np.random.Generator, + label: str, what: str = "", why: str = "", +) -> dict: + """One Pearson-r metric dict (point estimate + bootstrap CI) from two + already-linearized 1-D arrays -- the shared low-level computation + behind every _linearize_* function's reported rho. Always Pearson: once + the test-specific linearization has been applied (identity for + mean-type tests, Hajek/placement/rank-based for the others), Pearson r + of the two linearized arrays IS the governing correlation -- see + _TEST_STRUCTURE's docstring.""" + n = len(judge) + + def pe(a, b): + r, _ = pearsonr(a, b) + return float(r) + + est, lo, hi = _bootstrap_ci_2(pe, judge, human, alpha=alpha, rng=rng) + band, interp, example = _interpret_corr(est, lo, hi, n, label) + return { + "estimate": est, "ci_low": lo, "ci_high": hi, "label": label, "band": band, "n": n, + "what": what, "why": why, "interpretation": interp, "example": example, + } + + +class PairwiseAlignmentResult: + """Judge-human correlation for every pair among 2+ named conditions. + + Returned by :func:`judge_alignment` when called with a dict of named + conditions instead of a single (judge, human) array pair. Answers "how + well does my judge track human labels for the comparison I'm about to + run" across every pairwise comparison, rather than a single item-level + number -- see :attr:`pairwise_metrics`. + + Attributes + ---------- + conditions : list[str] + Condition names, in input order. + design : {"within", "between"} + Whether each pair's correlation was computed on within-subject + differences or between-subjects pooled values -- see + :func:`_condition_pair_arrays`. + pairwise_metrics : dict[tuple[str, str], dict] + ``(condition_a, condition_b) -> {"pearson_r": {...}, "spearman_r": {...}}``, + one entry per unordered pair -- the RAW correlations, for + comparability to prior work. NOT necessarily the correlation that + governs your test's PPI variance reduction for rank-based tests + (see :attr:`test`/:attr:`test_pairwise_metrics`/:attr:`omnibus_metric`). + test : str or None + The test named via ``test=``, if any -- see :func:`judge_alignment`. + test_pairwise_metrics : dict[tuple[str, str], dict] or None + Only set when ``test`` needs exactly 2 conditions (ttest, wilcoxon, + mannwhitney): that test's CORRECT, test-specific linearized rho for + every pair -- e.g. the Hajek-projection correlation for wilcoxon, + not raw Spearman. This is the number to use for planning/reporting + a pairwise (e.g. post-hoc) run of that test between two conditions. + omnibus_metric : dict or None + Only set when ``test`` can span 2+ conditions at once + (anova_oneway, kruskalwallis, friedman): that test's own validated + whole-design rho, computed once across ALL conditions together + (not decomposable into pairs) -- see the relevant + ``_linearize_*`` function's docstring for the recipe and its + Monte-Carlo validation in notes/omnibus_label_efficiency.html. + condition_counts : dict[str, tuple[int, int]] + ``name -> (n_labeled, n_total)`` for each condition, as passed -- + for reporting "N_lab and N per condition/measure" alongside the + correlations above. Not used in any correlation/N_eff computation + itself (those use the labeled OVERLAP / pooled totals actually + involved in each specific pair or the whole design). + selection : str + How the labeled subset was chosen -- see :func:`judge_alignment`'s + ``selection=``. Same MCAR-assumption warning as the single- + condition form when left at ``"unknown"``. + + Every metric dict in ``pairwise_metrics``/``test_pairwise_metrics``/ + ``omnibus_metric`` also carries ``multiplier``/``n_eff`` (the + label-efficiency savings implied by that correlation, at the N/N_lab + actually spanned by that specific correlation -- summed across the + conditions it pools, for a "between" pair/omnibus, or a single + condition's N for a "within" one, where every condition shares the + same items by construction). These are the ORACLE bound -- the + efficiency available at the variance-minimizing lambda, validated to + within 1.7% against direct oracle-lambda simulation. A particular + corrected test may realize less, since ``evalstats.tests`` sometimes + trades efficiency for calibration when choosing lambda; see + :func:`_attach_savings`'s docstring for the measured size of that gap + for ``wilcoxon(power_tune=True)``. + + Notes + ----- + With 3+ conditions, ``pairwise_metrics``/``test_pairwise_metrics`` are + NOT statistically independent of each other across pairs (e.g. "post vs + pre" and "post vs mid" both involve the "post" condition's data) -- + fine to report each pair's own number, but don't average them across + pairs as if they were independent samples. + """ + + def __init__( + self, *, conditions: list, design: str, pairwise_metrics: dict, + condition_counts: dict, selection: str = "unknown", + test: Optional[str] = None, test_pairwise_metrics: Optional[dict] = None, + omnibus_metric: Optional[dict] = None, + ) -> None: + self.conditions = conditions + self.design = design + self.pairwise_metrics = pairwise_metrics + self.condition_counts = condition_counts + self.selection = selection + self.test = test + self.test_pairwise_metrics = test_pairwise_metrics + self.omnibus_metric = omnibus_metric + + @staticmethod + def _print_metric(d: dict) -> None: + print( + f" {d['label']:<28} {d['estimate']:+.3f} " + f"95% CI [{d['ci_low']:+.3f}, {d['ci_high']:+.3f}] (n={d['n']})" + ) + if "n_eff" in d: + print(f" multiplier = {d['multiplier']:.2f}x N_eff = {d['n_eff']:.0f} (N={d['N']})") + + def summary(self) -> None: + """Print one line per pair per metric.""" + print("Pairwise judge alignment report") + print("─" * 58) + print(f"Conditions : {', '.join(self.conditions)}") + print(f"Design : {self.design}-subjects") + for name in self.conditions: + n_lab, n_tot = self.condition_counts[name] + print(f" {name}: N_lab={n_lab}, N={n_tot} ({100.0*n_lab/n_tot:.1f}% labeled)") + if len(self.conditions) > 2: + print( + "Note: pairwise correlations below are not independent of " + "each other (they share conditions) -- see class docstring." ) + print() + if self.omnibus_metric is not None: + print(f"test={self.test!r} (whole-design, all {len(self.conditions)} conditions):") + self._print_metric(self.omnibus_metric) + print() + for (a, b), metrics in self.pairwise_metrics.items(): + print(f"{a} vs {b}:") + for entry in metrics.values(): + self._print_metric(entry) + if self.test_pairwise_metrics is not None: + self._print_metric(self.test_pairwise_metrics[(a, b)]) + print() - return AlignmentResult( - llm_metric=llm_metric, - human_col=human_groundtruth, - score_type=score_type, - n_labeled=n_labeled, - n_total=n_total, - calibration=calibration, - alignment_metrics=alignment_metrics, - representativeness=rep, - bias_check=bias_check, + +def _pair_total_n(conditions: dict, names: list, design: str) -> int: + """Total (labeled+unlabeled) item count spanned by a correlation over + `names`' conditions: one condition's length for design="within" (every + condition shares the same items by construction, enforced elsewhere), + or the sum across conditions for design="between" (independent groups, + pooled).""" + if design == "within": + return len(np.asarray(conditions[names[0]][0])) + return sum(len(np.asarray(conditions[n][0])) for n in names) + + +def _attach_savings(metric: dict, N: int) -> dict: + """Attach multiplier/n_eff to a correlation metric dict, using `N` + (see :func:`_pair_total_n`) as the savings formula's total item count. + + VALIDATED. With ``lam*`` the variance-minimizing PPI++ weight, the + algebra gives ``Var_min = (V_h/n_lab) * [1 - rho^2 * (n_unlab/N)]``, + i.e. exactly ``multiplier = 1/(1 - rho^2*(1 - n_lab/N))`` with ``rho`` + the correlation of the two sides' INFLUENCE FUNCTIONS. Confirmed + numerically against a direct oracle-lambda simulation (no bootstrap, + N=1000, n_lab=200, 4000 reps): predicted/oracle multiplier ratios were + 0.975 / 1.009 / 1.017 for wilcoxon and 0.987 / 1.008 / 0.993 for + mannwhitney at d = 0 / 0.3-0.5 / 1.0 -- within 1.7% and stable across + effect sizes. + + Note the ``N`` passed here cannot itself be a source of error: the + multiplier depends on ``N`` and ``n_lab`` only through their RATIO, and + pooling equal-sized groups preserves that ratio exactly (60/300 = + 120/600). An earlier revision of this docstring blamed a measured + discrepancy on "N-aggregation"; that was wrong on both counts -- see + :func:`_linearize_mannwhitney` and :func:`_linearize_wilcoxon` for the + two real (now-fixed) bugs, which were in the linearizations. + + ONE STANDING CAVEAT, and it is about the library's lambda, not this + formula: ``multiplier``/``n_eff`` are the ORACLE bound -- what is + achievable at the variance-minimizing lambda. ``evalstats.tests`` + deliberately does not always use that lambda. In particular + ``wilcoxon(power_tune=True)`` evaluates the human term's variance under + H0 (sign-flip null variance) rather than plug-in, a deliberate + calibration trade documented in + ``evalstats.ppi._analytic_walsh_theta_correct``. That lambda is + intentionally sub-optimal, and drifts further from optimal as the true + effect moves away from H0 -- measured at n_lab=60, the realized + multiplier fell to ~1/1.17 of this bound at d=0 and ~1/1.74 at d=0.3. + So report these as "the efficiency this judge makes available", not as + a guarantee of what a particular corrected test will realize.""" + mult, n_eff = _n_eff(metric["estimate"], metric["n"], N) + metric["N"] = N + metric["multiplier"] = mult + metric["n_eff"] = n_eff + return metric + + +def _judge_alignment_pairwise( + conditions: dict, *, design: Optional[str], alpha: float, + test: Optional[str] = None, selection: str = "unknown", warn_stacklevel: int = 3, +) -> PairwiseAlignmentResult: + if len(conditions) < 2: + raise ValueError( + "judge_alignment(conditions_dict) needs at least 2 conditions. " + "For a single condition, call judge_alignment(judge_scores, human_scores) " + "with plain arrays instead." + ) + if selection not in _VALID_SELECTIONS: + raise ValueError(f"selection={selection!r} -- must be one of {_VALID_SELECTIONS}.") + + resolved_design = design + if test is not None: + if test not in _TEST_STRUCTURE or _TEST_STRUCTURE[test]["min_k"] < 2: + multi_cond_tests = sorted(t for t, s in _TEST_STRUCTURE.items() if s["min_k"] >= 2) + raise ValueError(f"Unrecognized or single-condition-only test={test!r}. Known: {multi_cond_tests}.") + implied = _TEST_STRUCTURE[test]["design"] + if implied is not None: + if design is not None and design != implied: + raise ValueError(f"test={test!r} is always design={implied!r}; design={design!r} conflicts.") + resolved_design = implied + + if resolved_design not in _VALID_DESIGNS: + raise ValueError( + f"design={resolved_design!r} -- with 2+ conditions, pass design='within' " + "(paired/repeated-measures: the same items/participants in every " + "condition) or design='between' (independent groups) explicitly, " + "or a test= that implies one. This can't be inferred from the data " + "alone -- two conditions look the same positionally whether they're " + "a paired comparison or two independent groups, and each needs " + "different math." + ) + + if selection == "unknown": + warnings.warn( + "judge_alignment() was not told how the labeled subset was " + "selected (selection=). Every correction it and " + "compare(alignment=...) apply assumes the labeled items are a " + "random sample of the full item pool -- pass selection='random' " + "to confirm that's the case, or selection='manual'/'stratified' " + "if not, so this is a deliberate acknowledgment rather than an " + "unexamined default.", + UserWarning, stacklevel=warn_stacklevel, + ) + elif selection == "manual": + warnings.warn( + "selection='manual': the labeled subset was NOT randomly " + "sampled. PPI/alignment correction assumes random sampling " + "(MCAR) to be valid -- with a manually-chosen subset, the " + "corrected estimates and CIs compare()/judge_alignment() report " + "may be miscalibrated, not just imprecise. Treat them as " + "informal unless the alignment set is re-sampled at random.", + UserWarning, stacklevel=warn_stacklevel, + ) + elif selection == "stratified": + warnings.warn( + "selection='stratified': evalstats' current correction doesn't " + "account for stratification weights, so this is only valid if " + "each stratum was itself sampled uniformly at random and the " + "strata are otherwise ignorable for the metric being judged. " + "If items were hand-picked within strata, treat corrected " + "estimates as potentially biased, same as selection='manual'.", + UserWarning, stacklevel=warn_stacklevel, + ) + + names = list(conditions.keys()) + condition_counts = {} + for n in names: + j, h = conditions[n] + h = np.asarray(h, dtype=float) + condition_counts[n] = (int((~np.isnan(h)).sum()), len(h)) + + rng = np.random.default_rng(42) + pairwise_metrics = {} + for name_a, name_b in combinations(names, 2): + judge_a, human_a = conditions[name_a] + judge_b, human_b = conditions[name_b] + judge, human = _condition_pair_arrays( + judge_a, human_a, judge_b, human_b, design=resolved_design, label_a=name_a, label_b=name_b, + ) + pair_n = _pair_total_n(conditions, [name_a, name_b], resolved_design) + metrics = _pearson_spearman_metrics( + judge, human, alpha=alpha, rng=rng, + pearson_label="Pearson r", spearman_label="Spearman r", + what_suffix=( + f" between {name_a} and {name_b}'s differences" if resolved_design == "within" + else f" between {name_a} and {name_b}, within-group centered" + ), + ) + for m in metrics.values(): + _attach_savings(m, pair_n) + pairwise_metrics[(name_a, name_b)] = metrics + + test_pairwise_metrics = None + omnibus_metric = None + if test is not None: + spec = _TEST_STRUCTURE[test] + if spec["max_k"] == 2: + test_pairwise_metrics = {} + for name_a, name_b in combinations(names, 2): + pair = {name_a: conditions[name_a], name_b: conditions[name_b]} + jl, hl, _ = _linearize_for_test(pair, test=test, design=resolved_design) + pair_n = _pair_total_n(conditions, [name_a, name_b], resolved_design) + m = _single_metric( + jl, hl, alpha=alpha, rng=rng, label=f"{test} rho (test-correct)", + why=f"The correlation that actually governs {test}'s PPI variance reduction, not raw Pearson/Spearman.", + ) + test_pairwise_metrics[(name_a, name_b)] = _attach_savings(m, pair_n) + else: + jl, hl, _ = _linearize_for_test(conditions, test=test, design=resolved_design) + whole_n = _pair_total_n(conditions, names, resolved_design) + m = _single_metric( + jl, hl, alpha=alpha, rng=rng, label=f"{test} rho (whole-design)", + why=f"{test}'s own validated correlation across all conditions at once -- not decomposable into pairs.", + ) + omnibus_metric = _attach_savings(m, whole_n) + + return PairwiseAlignmentResult( + conditions=names, design=resolved_design, pairwise_metrics=pairwise_metrics, + condition_counts=condition_counts, selection=selection, + test=test, test_pairwise_metrics=test_pairwise_metrics, omnibus_metric=omnibus_metric, + ) + + +def judge_alignment( + judge_scores_or_evaldata, + human_scores=None, + *, + llm_metric: Optional[str] = None, + human_groundtruth: Optional[str] = None, + all_judge_scores=None, + score_type: Optional[str] = None, + design: Optional[str] = None, + test: Optional[str] = None, + alpha: float = 0.05, + selection: str = "unknown", + ci: bool = True, +) -> "AlignmentResult | PairwiseAlignmentResult": + """Validate how well an LLM judge aligns with human graders. + + Parameters + ---------- + ci : bool, default True + Whether to bootstrap confidence intervals for the alignment metrics. + ``ci=False`` returns the point estimates with NaN bounds and skips + roughly 2000 resamples per metric -- the bulk of this function's + runtime. Use it when only the estimates are consumed (for example + building the ``alignment=`` argument to :func:`compare`, whose PPI + correction reads point estimates only), or in large sweeps. + + Three call forms: + + 1. ``judge_alignment(evaldata, *, llm_metric=..., human_groundtruth=...)`` + -- the common case where LLM judge scores exist for all items but + human labels are available for only a subset (the alignment set), + identified by column name in ``evaldata``. Runs the full + representativeness diagnostics (score-distribution check against + the full item pool, plus categorical slice-column checks) since the + full dataset and its other columns are available. The returned + result can be passed to ``compare(alignment={metric: result})``. + 2. ``judge_alignment(judge_scores, human_scores)`` -- a quick-primitive + form for when you don't want to build an ``EvalResults`` first. + ``judge_scores`` is every item's judge score; ``human_scores`` is + the *same length*, with ``NaN`` for items that don't have a human + label (or no ``NaN`` at all if every item happens to be labeled). + This mirrors form 1's sparse-column convention exactly, so you can + hand it whatever you already have without pre-splitting anything + yourself. When some items are unlabeled, the score-distribution + representativeness check runs automatically (``judge_scores`` is + already the full pool); pass ``all_judge_scores`` explicitly to + override this. The categorical slice-column checks are + DataFrame-specific and are always skipped in this form. **The + result from this form carries placeholder column names and cannot + be passed to ``compare(alignment=...)``** (there's no underlying + DataFrame for it to look values up in) -- use form 1 for that. + 3. ``judge_alignment({"pre": (judge_a, human_a), "post": (judge_b, human_b)}, + design="within")`` -- for a comparison you're about to run across 2+ + named conditions (arms of a study, timepoints, whatever your design + calls them), rather than a single item-level check. Each dict value + is a ``(judge_scores, human_scores)`` pair in form 2's shape (same + length, ``NaN`` for unlabeled items). Returns a + :class:`PairwiseAlignmentResult` with raw Pearson r and Spearman r + for every pair of conditions (for comparability to prior work), + PLUS -- when you pass ``test=`` -- the correlation that actually + governs THAT test's PPI variance reduction, which for every + rank-based test is NOT the same as raw Spearman (raw Spearman + drifts with effect size; see ``test=`` below). ``design`` is + required unless ``test=`` implies one (e.g. ``test="wilcoxon"`` + always implies ``"within"``) and can't otherwise be inferred from + the data: "within" (paired/repeated-measures -- the same + items/participants in every condition) or "between" (independent + groups) -- two conditions look identical positionally either way, + but need different math. Every correlation reported also carries + ``multiplier``/``n_eff`` -- see :attr:`PairwiseAlignmentResult + .pairwise_metrics` -- so there's no separate function to call for + "how many effective human labels do I have." + + Every form fits a Bayesian calibration model that can later be used to + propagate judge uncertainty into downstream comparisons (forms 1-2 + only; form 3 has no single calibration model to fit, since it spans 2+ + conditions -- use form 1 or 2 per-condition first if you need that). + + Parameters + ---------- + judge_scores_or_evaldata : EvalResults, array-like, or dict + Evaluation data from :func:`load_from` (form 1), every item's judge + score (form 2), or a ``{name: (judge_scores, human_scores)}`` dict + of 2+ named conditions (form 3). + human_scores : array-like, optional + Same length as ``judge_scores_or_evaldata``, ``NaN`` for unlabeled + items (form 2 only). + design : {"within", "between"}, optional + Form 3 only. Required there unless ``test=`` implies a design. See + form 3 above. + test : str, optional + One of ``"ttest"``, ``"wilcoxon"``, ``"mannwhitney"`` (exactly 2 + conditions), ``"anova_oneway"``, ``"kruskalwallis"``, ``"friedman"`` + (2+ conditions, form 3 only), or ``"mean_estimate"`` (form 1/2 + only -- a single condition, no comparison, e.g. a one-sample + mean/proportion estimate). When given, computes the correlation + that actually governs that test's PPI variance reduction, and + unlocks :attr:`AlignmentResult.n_eff`/:attr:`.multiplier` (forms + 1-2) -- for a 2-condition-only test in form 3, one number per pair + (:attr:`PairwiseAlignmentResult.test_pairwise_metrics`, also the + number to use for planning pairwise post-hoc tests with 3+ + conditions); for a test that spans the whole design, one number + across all conditions at once + (:attr:`PairwiseAlignmentResult.omnibus_metric`) -- e.g. with 3+ + conditions and ``test="friedman"``, you get BOTH Friedman's own + whole-design number AND the raw pairwise breakdown. See + :data:`_TEST_STRUCTURE` and each ``_linearize_*`` function for the + recipe/validation behind each test. + llm_metric : str, optional + Form 1: column name of the LLM judge scores (required). Form 2: + optional display name for the judge, used only in printed reports. + human_groundtruth : str, optional + Form 1: column name of the human rater scores (required), + expected to be sparsely populated (non-null for the alignment + subset, ``NaN`` elsewhere). Form 2: optional display name for the + human rater, used only in printed reports. + all_judge_scores : array-like, optional + Form 2 only: override which array is treated as "every item's + judge score" for the representativeness check. Only needed if + that shouldn't just be ``judge_scores_or_evaldata`` itself. + score_type : str, optional + Form 2 only: override the auto-detected score type (``"binary"``, + ``"likert"``, ``"continuous"``, or ``"grade"``). Auto-detected from + the labeled judge scores when not given. + alpha : float + Significance level for alignment metric CIs. Default ``0.05``. + selection : {"random", "stratified", "manual", "unknown"} + How the labeled subset was chosen. Every correction this function + (and ``compare(alignment=...)``) applies assumes the labeled items + are a random sample of the full item pool ("missing completely at + random", MCAR) -- pass ``"random"`` to confirm that's the case. + Anything else (including the ``"unknown"`` default) raises a + ``UserWarning`` explaining the risk, since the natural QA instinct + -- hand-labeling the items you're least sure about -- is exactly + the kind of selection that breaks this assumption. ``"manual"`` + and ``"stratified"`` are for acknowledging a known-non-random + selection explicitly rather than leaving it unexamined. + + Notes + ----- + ``alignment_metrics`` (forms 1-2) and ``pairwise_metrics`` (form 3) + always include both raw ``pearson_r`` and ``spearman_r`` (alongside + score-type-specific metrics: percent agreement/Cohen's κ for binary, + weighted κ/ICC(2,1) for likert, ICC(2,1) for continuous/grade) -- + reported for comparability to prior work. These are NOT simply "Pearson + for mean-based tests, Spearman for rank-based tests": that split is the + naive recipe, and for every rank-based test (Mann-Whitney, Wilcoxon, + Kruskal-Wallis, Friedman) it drifts with effect size -- confirmed via + Monte Carlo to overstate ``n_eff`` by -13% to +89% depending on the + test and effect size (see :data:`_TEST_STRUCTURE`'s docstring and + notes/omnibus_label_efficiency.html). The correlation that actually + governs a given test's PPI variance reduction is a Pearson correlation + on a TEST-SPECIFIC linearization of the raw values (identity for + mean-type tests, a genuine transform -- Hájek projection, empirical + placements, or centered ranks -- for rank-type ones); pass ``test=`` to + get it, rather than reading ``pearson_r``/``spearman_r`` directly for a + rank-based test. + + If you're comparing 2+ conditions (a study arm, a timepoint, anything + with its own judge/human scores), use form 3 above rather than form + 1/2 called once per condition -- it computes the right correlation + automatically for both within-subjects (paired-difference) and + between-subjects (pooled, within-group-centered) designs, and covers + 3+ conditions via every pairwise comparison (plus, with ``test=``, an + omnibus test's own whole-design number where applicable). + + Returns + ------- + AlignmentResult or PairwiseAlignmentResult + ``PairwiseAlignmentResult`` for form 3 (dict input); ``AlignmentResult`` + for forms 1-2. + """ + if isinstance(judge_scores_or_evaldata, dict): + if human_scores is not None: + raise TypeError( + "judge_alignment(conditions_dict, ...) doesn't take a second " + "positional argument -- each dict value is already a " + "(judge_scores, human_scores) pair." + ) + return _judge_alignment_pairwise( + judge_scores_or_evaldata, design=design, alpha=alpha, test=test, selection=selection, + ) + + from evalstats.loader import EvalResults + + if isinstance(judge_scores_or_evaldata, EvalResults): + evaldata = judge_scores_or_evaldata + if human_scores is not None: + raise TypeError( + "judge_alignment(evaldata, ...) doesn't take a second " + "positional argument; pass llm_metric= and " + "human_groundtruth= as column names instead. (For the " + "raw-array form, pass two arrays: " + "judge_alignment(judge_scores, human_scores).)" + ) + if llm_metric is None or human_groundtruth is None: + raise TypeError( + "judge_alignment(evaldata, ...) requires llm_metric= and " + "human_groundtruth= (column names)." + ) + return _judge_alignment_from_evaldata( + evaldata, llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, + selection=selection, ci=ci, + ) + + if human_scores is None: + raise TypeError( + "judge_alignment(judge_scores, human_scores) requires both " + "arrays; or pass an EvalResults (from load_from()) as the " + "first argument for the column-name-based form: " + "judge_alignment(evaldata, llm_metric=..., human_groundtruth=...)." + ) + return _judge_alignment_from_arrays( + judge_scores_or_evaldata, human_scores, + all_judge_scores=all_judge_scores, score_type=score_type, + llm_metric=llm_metric, human_groundtruth=human_groundtruth, alpha=alpha, + selection=selection, test=test, ci=ci, ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Label-efficiency savings formula, shared by AlignmentResult.n_eff/ +# .multiplier and PairwiseAlignmentResult's per-metric multiplier/n_eff. +# ───────────────────────────────────────────────────────────────────────────── + +def _n_eff(r: float, n_lab: int, N: int) -> tuple[float, float]: + """(multiplier, N_eff) from the control-variate savings formula + ``1 / (1 - rho^2 * (1 - N_lab/N))`` -- see + simulations/harness/cases/pvalues.py's ``_ppi_predicted_savings`` for + the derivation and validation (R^2=0.9968 against measured variance + ratios over a 48-cell grid at 3000 reps/cell).""" + if not np.isfinite(r) or N <= 0: + return float("nan"), float("nan") + rho2 = float(np.clip(r, -1.0, 1.0)) ** 2 + k = max(0.0, 1.0 - float(n_lab) / float(N)) + denom = 1.0 - rho2 * k + mult = 1.0 / denom if denom > 1e-9 else float("inf") + return mult, n_lab * mult + + diff --git a/evalstats/api.py b/evalstats/api.py index 705cfce..24a79e1 100644 --- a/evalstats/api.py +++ b/evalstats/api.py @@ -22,6 +22,8 @@ from evalstats.config import get_alpha_ci, GRADIENT_CI_ALPHAS from evalstats.core.router import analyze, analyze_factorial, _analyze_single_lightweight from evalstats.core.bundles import AnalysisBundle, MultiModelBundle, AnalysisResult +from evalstats.core.design import detect_paired +from evalstats.core.unpaired import compare_unpaired, GroupComparisonResult from evalstats.core.stats_utils import correct_pvalues from evalstats.core.summary import ( print_analysis_summary, @@ -68,7 +70,7 @@ def __init__( self._mmb_view = _mmb_view # which MultiModelBundle view is primary self._min_meaningful_diff = min_meaningful_diff self._variance_components: Optional[dict] = None # set by MC alignment loop - self._pareto: Optional[dict] = None # set by _run_pareto_if_needed when secondary= is passed + self._pareto: Optional[dict] = None # set by _run_pareto_if_needed when secondary_metric= is passed # Bootstrap P(Best)/E[Rank] output is opt-in, not opt-out: it reads as # a confident, almost authoritative verdict (e.g. "63.6% probability # of being best") even when the underlying CIs overlap heavily and the @@ -212,7 +214,7 @@ def print_pair(self, entity_a: str, entity_b: str) -> None: return pair.summary() - def plot(self, method: str = "bar", **kwargs): + def plot(self, method: str = "forest", **kwargs): """Visualize comparison results. Parameters @@ -220,12 +222,27 @@ def plot(self, method: str = "bar", **kwargs): method : str Plot type: - * ``"bar"`` (default) — accuracy bar chart via - :func:`~evalstats.vis.scoreboard.plot_accuracy_bar`. - * ``"forest"`` — horizontal CI forest plot via - :func:`~evalstats.vis.forest.plot_ci_forest`. + * ``"forest"`` (default) — horizontal CI forest plot via + :func:`~evalstats.vis.forest.plot_ci_forest`, gradient-banded + (68/90/95/99% nested confidence bands) by default -- the same + richer CI picture the terminal's ``.summary()`` gradient plot + already shows, in matplotlib. Pass ``style="single"`` to fall + back to one plain CI band per entity, or ``color_rule="factor"`` + / a colour name to color by entity identity instead of + significance tier. + * ``"bar"`` — accuracy bar chart via + :func:`~evalstats.vis.scoreboard.plot_accuracy_bar`. A quick, + uncorrected view (no CIs) -- useful before statistical + analysis, not as a substitute for it. * ``"cd"`` — critical difference diagram via :func:`~evalstats.vis.critical_difference.plot_critical_difference`. + * ``"pareto"`` — uncertainty-aware trade-off scatter via + :func:`~evalstats.vis.pareto.plot_pareto_tradeoff`. Only + available when ``compare(..., secondary_metric=...)`` was passed; + raises otherwise. One bootstrap point cloud per entity plus a + percentile band over per-replicate Pareto frontiers, colored + by calibrated status (frontier / dominated / ambiguous) -- + see :attr:`pareto_status`. **kwargs Forwarded to the underlying plot function. @@ -243,10 +260,19 @@ def plot(self, method: str = "bar", **kwargs): elif method == "cd": from evalstats.vis.critical_difference import plot_critical_difference return plot_critical_difference(self, **kwargs) + elif method == "pareto": + if self._pareto is None: + raise ValueError( + "method='pareto' requires compare(..., secondary_metric=...) " + "to have been passed -- no Pareto analysis was run for " + "this result." + ) + from evalstats.vis.pareto import plot_pareto_tradeoff + return plot_pareto_tradeoff(self._pareto, metric=self._metric, **kwargs) else: raise ValueError( f"Unknown plot method: {method!r}. " - "Expected 'bar', 'forest', or 'cd'." + "Expected 'bar', 'forest', 'cd', or 'pareto'." ) def report(self, format: str = "markdown") -> str: @@ -286,6 +312,10 @@ def entity_stats(self) -> dict: ci_high=float(rob.ci_high[i]) if rob.ci_high is not None else 1.0, median=float(rob.median[i]), std=float(rob.std[i]), + multi_ci=( + {a: (float(lo[i]), float(hi[i])) for a, (lo, hi) in rob.multi_ci.items()} + if rob.multi_ci is not None else None + ), ) for i, lbl in enumerate(bundle.benchmark.template_labels) } @@ -394,7 +424,7 @@ def best_pairs(self) -> Optional[list]: if not isinstance(self._analysis, MultiModelBundle): return None cross = self._analysis.cross_model - labels = list(cross.rank_dist.labels) + labels = list(cross.labels) if len(labels) < 2: return None means = cross.robustness.mean @@ -409,11 +439,62 @@ def best_pairs(self) -> Optional[list]: top_pairs.append((parts[0], parts[1])) return top_pairs or None + @property + def model_labels(self) -> Optional[list]: + """Model-axis labels for a two-factor (model, prompt) comparison, or ``None``. + + Populated under the same condition as :attr:`cross_model` — this is + that bundle's model axis, in its original (pre-sort) order. + """ + if not isinstance(self._analysis, MultiModelBundle): + return None + return list(self._analysis.benchmark.model_labels) + + @property + def prompt_labels(self) -> Optional[list]: + """Prompt/template-axis labels for a two-factor comparison, or ``None``. + + Populated under the same condition as :attr:`cross_model` — this is + that bundle's template axis, in its original (pre-sort) order. + """ + if not isinstance(self._analysis, MultiModelBundle): + return None + return list(self._analysis.benchmark.template_labels) + + def as_view(self, factor: Literal["model", "prompt"]) -> "ComparisonResult": + """Return this two-factor comparison collapsed onto a single axis. + + E.g. ``result.as_view("model")`` averages over prompts to compare + models; ``result.as_view("prompt")`` averages over models to compare + prompts. Only valid for a two-factor comparison (see + :attr:`cross_model`) — raises otherwise. + """ + if not isinstance(self._analysis, MultiModelBundle): + raise ValueError( + "as_view() requires a two-factor comparison (built with " + "compare(..., factors=['model', 'prompt']), or " + "factors='model'/'prompt' when both columns are present)." + ) + view_map = {"model": "model_level", "prompt": "template_level"} + if factor not in view_map: + raise ValueError(f"factor={factor!r} must be 'model' or 'prompt'.") + return ComparisonResult( + self._analysis, + factors=self._factors, + metric=self._metric, + baseline=self._baseline, + alpha=self._alpha, + filtered_df=self._df, + _mmb_view=view_map[factor], + min_meaningful_diff=self._min_meaningful_diff, + show_rank_probabilities=self._show_rank_probabilities, + ) + @property def pareto_status(self) -> Optional[dict]: """Per-entity three-state Pareto classification, or ``None``. - Populated only when ``compare(..., secondary=...)`` was passed. + Populated only when ``compare(..., secondary_metric=...)`` was passed. Keys are entity labels, values are :class:`~evalstats.core.pareto.ParetoStatus` (``.status`` is one of ``"frontier"``, ``"dominated"``, ``"ambiguous"`` -- see that class's @@ -430,7 +511,7 @@ def pareto_status(self) -> Optional[dict]: def pareto_frontier_probability(self) -> Optional[dict]: """Per-entity ``P(entity is Pareto-optimal)``, or ``None``. - Populated only when ``compare(..., secondary=...)`` was passed. Keys + Populated only when ``compare(..., secondary_metric=...)`` was passed. Keys are entity labels, values are the fraction of joint bootstrap replicates in which that entity was non-dominated -- a continuous probability, not the calibrated three-state label @@ -845,11 +926,12 @@ def _bridge_to_io( # PPI alignment correction # ───────────────────────────────────────────────────────────────────────────── -_PPI_PAIRWISE_SUPPORTED = ("tango", "t_interval", "bootstrap", "wilcoxon", "mannwhitney", "bootstrap_t", "bayes_bootstrap", "ppi_t_interval", "ppi_logit_t") +_PPI_PAIRWISE_SUPPORTED = ("bonett_price", "mj_floor", "t_interval", "bootstrap", "wilcoxon", "mannwhitney", "bootstrap_t", "bayes_bootstrap", "ppi_t_interval", "ppi_logit_t") _PPI_ROBUSTNESS_SUPPORTED = ("wilson", "bootstrap", "bootstrap_t", "ppi_t_interval", "ppi_logit_t") -def _ppi_pairwise_dispatch(method: str, a, b, a_lab, b_lab, alpha: float, n_boot: int, rng): +def _ppi_pairwise_dispatch(method: str, a, b, a_lab, b_lab, alpha: float, n_boot: int, rng, + score_range: Optional[tuple[float, float]] = None): """Dispatch to the PPI-corrected pairwise implementation of *method*. Only methods with a validated PPI-corrected counterpart (see @@ -864,12 +946,15 @@ def _ppi_pairwise_dispatch(method: str, a, b, a_lab, b_lab, alpha: float, n_boot silently collide with that existing mapping. """ from evalstats.tests import ( - _ppi_paired_tango, _ppi_paired_bootstrap_t, _ppi_paired_bayes_bootstrap, + _ppi_paired_mj_floor, _ppi_paired_bonett_price, _ppi_paired_bootstrap_t, + _ppi_paired_bayes_bootstrap, _ppi_paired_arrays, _ppi_two_sample, _p_x_gt_y_midrank, _ppi_paired_t_interval, _ppi_paired_logit_t, ) - if method == "tango": - return _ppi_paired_tango(a, b, a_lab, b_lab, alpha) + if method == "bonett_price": + return _ppi_paired_bonett_price(a, b, a_lab, b_lab, alpha) + if method == "mj_floor": + return _ppi_paired_mj_floor(a, b, a_lab, b_lab, alpha) if method == "bootstrap_t": return _ppi_paired_bootstrap_t(a, b, a_lab, b_lab, alpha, n_boot, rng) if method == "bayes_bootstrap": @@ -880,7 +965,14 @@ def _ppi_pairwise_dispatch(method: str, a, b, a_lab, b_lab, alpha: float, n_boot # lo/hi default (0.0, 1.0): this dispatch path has no score_range # concept (see _run_alignment_ppi's is_bounded_01_scores check -- # "bounded_01" always means raw scores are literally in [0, 1] here). - return _ppi_paired_logit_t(a, b, a_lab, b_lab, alpha) + # lo/hi MUST come from the resolved score_range, not the (0, 1) + # default: logit-t is scale-dependent, and this method is reachable + # for any bounded numeric scale (likert 1-5, grades 0-100), not just + # [0, 1]. Passing the wrong bounds returns a CI on the [0, 1] scale + # while the estimand lives on the real one -- 0% coverage, not a + # subtle miscalibration. + _lo, _hi = score_range if score_range is not None else (0.0, 1.0) + return _ppi_paired_logit_t(a, b, a_lab, b_lab, alpha, lo=_lo, hi=_hi) if method in ("t_interval", "bootstrap"): return _ppi_paired_arrays(a, b, a_lab, b_lab, np.mean, alpha, n_boot, rng, rectifier_func=np.mean) if method == "wilcoxon": @@ -897,6 +989,12 @@ def _ppi_pairwise_dispatch(method: str, a, b, a_lab, b_lab, alpha: float, n_boot # production stays aligned with what's actually been exercised by # that sanctioned pipeline until "ridge" gets its own official # pass. + # KNOWN, NOT FIXED (2026-08-24): conservative on likert. Mid-rank + # placement over a 5-level scale collapses the influence function + # (~22% tied pairs), giving corrected Type-I ~0.011 vs ~0.04 for the + # parametric tests. Unlike wilcoxon, this path never reaches + # correct()'s smoothed-bootstrap jitter (ppi._tie_jitter_scale) -- + # the likeliest fix, but unvalidated here. Errs safe (under-rejects). return _ppi_two_sample(a, b, a_lab, b_lab, lambda xa, ya: _p_x_gt_y_midrank(xa, ya) - 0.5, alpha, n_boot, rng) raise ValueError( f"PPI alignment correction has no validated implementation for pairwise " @@ -922,6 +1020,13 @@ def _ppi_pairwise_unpaired_fallback(a, b, a_lab, b_lab, alpha: float, n_boot: in return _ppi_two_sample(a, b, a_lab, b_lab, lambda ya, yb: float(ya.mean() - yb.mean()), alpha, n_boot, rng) +_JOINT_BOOT_SE_REL_FLOOR = 0.20 +"""Relative floor on a bootstrap replicate's SE, as a fraction of the +observed SE, inside :func:`_ppi_bootstrap_t_joint_stats`. See the comment at +its use site for the failure mode this prevents and how 0.20 was calibrated. +Set to 0.0 to reproduce the pre-fix behaviour exactly.""" + + def _ppi_bootstrap_t_joint_stats( scores_2d: np.ndarray, lab_matrix: np.ndarray, @@ -959,7 +1064,7 @@ def _ppi_bootstrap_t_joint_stats( estimate/variance use the same closed-form variance-minimizing lambda* :func:`evalstats.ppi._analytic_mean_point_se` derives for ``ppi_t_interval``/``ppi_logit_t``/Tango (since evalstats.tests. - _ppi_paired_tango's own power-tuning flip), instead of the fixed + _ppi_paired_mj_floor's own power-tuning flip), instead of the fixed lambda=1 rectifier -- generalized here to a per-pair lambda*, computed once per pair (closed-form, not re-estimated per bootstrap replicate, avoiding the "double dipping" undercoverage a same-draw lambda/CI @@ -1001,7 +1106,9 @@ def _ppi_bootstrap_t_joint_stats( full matrix (Romano-Wolf's step-down) use it directly), and ``t_obs`` has shape ``(k,)``. """ - from evalstats.ppi import _POWER_TUNE_SHRINKAGE_C + from evalstats.ppi import ( + _adaptive_shrink_lambda, _analytic_mean_lambda_replicates, _lambda_var_inflation, + ) k = len(pair_keys) @@ -1029,6 +1136,7 @@ def _ppi_bootstrap_t_joint_stats( point_ests = np.empty(k) obs_se = np.empty(k) lam = np.ones(k) + lambda_extra_var = np.zeros(k) # per-pair r_term**2 * Var(lambda_hat), held fixed across replicates like lam for p_idx, (ea, eb) in enumerate(pair_keys): ia, ib = entity_idx[ea], entity_idx[eb] d_all = scores_2d[ia] - scores_2d[ib] @@ -1062,17 +1170,40 @@ def _ppi_bootstrap_t_joint_stats( float(np.cov(d_lab_true, d_lab_llm, ddof=1)[0, 1]) / n_lab if n_lab > 1 else 0.0 ) - lam_p = 1.0 + lam_p_raw = 1.0 denom = var_unlab_n + var_hat_lab_n if denom > 1e-12: - lam_p = min(max(cov_lab_n / denom, 0.0), 1.0) - lam_p = 1.0 - (1.0 - lam_p) * n_lab / (n_lab + _POWER_TUNE_SHRINKAGE_C) + lam_p_raw = min(max(cov_lab_n / denom, 0.0), 1.0) + # Adaptive shrinkage (see evalstats.ppi._adaptive_shrink_lambda's + # docstring for the shared rationale) -- was previously fixed + # toward a target of 1 regardless of what the data supported, + # unlike every other power_tune site in this codebase. Falls back + # to target=1 when d_lab_true is near-degenerate, same guard + # evalstats.ppi._analytic_mean_point_se uses. + raw_var_lab_true = float(np.var(d_lab_true, ddof=1)) if n_lab > 1 else 0.0 + raw_var_lab_llm = float(np.var(d_lab_llm, ddof=1)) if n_lab > 1 else 0.0 + if n_lab <= 1 or raw_var_lab_true < raw_var_lab_llm * 1e-6: + lam_p_replicates = None + else: + lam_p_replicates = _analytic_mean_lambda_replicates(d_lab_true, d_lab_llm, var_unlab_n, n_lab) + lam_p = _adaptive_shrink_lambda(lam_p_raw, lam_p_replicates, n_lab) lam[p_idx] = lam_p point_ests[p_idx] = f_lab + lam_p * (f_unlab - f_hat_lab) var_estimate = max( var_lab_n + lam_p * lam_p * (var_unlab_n + var_hat_lab_n) - 2.0 * lam_p * cov_lab_n, 0.0, ) + # Precompute the extra variance ONCE per pair, from the OBSERVED + # (fixed) r_term -- not re-derived per bootstrap replicate below -- + # matching how lam itself is held fixed across every replicate for + # this pair. An earlier version used each replicate's own resampled + # r_term_b instead, making the injected variance data-dependent + # within the bootstrap itself; a paired high-rep recheck confirmed + # that was the cause of a real FWER regression in one tested + # condition (fixed by holding r_term fixed here) -- see + # simulations/out/results_why_ppi_shrink_1_over_0.md Addendum 20/21. + lambda_extra_var[p_idx] = _lambda_var_inflation(f_unlab - f_hat_lab, lam_p_replicates) + var_estimate += lambda_extra_var[p_idx] obs_se[p_idx] = np.sqrt(var_estimate) boot_theta = np.empty((n_boot, k)) @@ -1116,10 +1247,56 @@ def _ppi_bootstrap_t_joint_stats( var_b = np.maximum( var_lab_b + lam_col * lam_col * (var_unlab_b + var_hat_lab_b) - 2.0 * lam_col * cov_lab_b, 0.0, ) + # Same fixed per-pair extra variance as obs_se above, broadcast + # across every replicate (not re-derived per replicate) -- see the + # comment there. + var_b = var_b + lambda_extra_var[:, np.newaxis] boot_theta[start:stop] = theta_b.T boot_se[start:stop] = np.sqrt(var_b).T start = stop + # Floor each replicate's SE relative to the OBSERVED one before + # studentizing. T is a studentized statistic, so the meaningful scale + # for "this replicate's SE is degenerate" is obs_se, not an absolute + # constant -- and the previous guard was absolute (1e-12), which cannot + # catch a boot_se that is small-but-nonzero. + # + # Why it matters: when a pair is near-degenerate (a paired difference + # with almost no item-level spread -- e.g. two nearly identical arms, or + # a generator that shifts every item by the same constant), a resample + # can draw an almost-constant vector, collapsing boot_se far below + # obs_se and sending |T| to 60-2000. Because BOTH consumers of this + # joint resample reduce it with a MAX over pairs + # (_ppi_romano_wolf_pvalues_from_joint_stats' step-down suffix-max, and + # _M_b_from_T for max-T/"boot" CI widening), one such pair poisons every + # other pair in the family: measured on a k=3 cell, a degenerate pair + # with |T|max=66 drove the UNRELATED extreme pair's Romano-Wolf p from + # ~0 to 0.363 while its own CI still excluded 0 by a wide margin -- the + # p-value and the CI contradicting each other inside one bundle. + # + # Calibration of the 0.20 coefficient: under regularity boot_se/obs_se + # concentrates at 1 with sd ~ 1/sqrt(2*n_lab) (~0.11 at n_lab=40), so + # 0.20 sits many sd below anything a well-behaved resample produces. + # Measured binding rates (fraction of replicates floored): 0.0000% on + # every non-degenerate condition tested (k=3/5, N=100-400, + # n_lab=20-160, judge rho=0.80-0.99, including the small-n_lab + + # excellent-judge corner where boot_se is most variable), versus + # 12-19% on the degenerate cells this exists for. It is inert where the + # bootstrap is healthy and only engages where it has broken down. + # Validated for FWER, not just power. Note the ordinary nulls are + # UNINFORMATIVE for that: under them the paired truth difference is + # exactly constant (uniq==1), so boot_se never collapses and the floor + # is inert -- Type-I is then identical for trivial reasons. The real + # test is a null where the floor DOES bind, i.e. the near-identical-arms + # case above: arms sharing a base, each perturbing a small random subset + # of items by +/- delta with mean-zero signs, so the null holds exactly + # while d_true has several distinct values and tiny variance. With + # binding up to 12% of replicates under that null, FWER is unchanged + # (largest move +0.0025 at 0.28% binding, 0.23 MC SE, on 400 reps). + # Power on the degenerate alternative recovers 0.6425 -> 0.9975 (k=3 + # N=100), 0.3850 -> 1.0000 (k=5) with FWER byte-identical. + # See simulations/investigate_joint_bootstrap_se_floor_*.py. + boot_se = np.maximum(boot_se, _JOINT_BOOT_SE_REL_FLOOR * obs_se[np.newaxis, :]) se_boot_safe = np.where(boot_se > 1e-12, boot_se, 1.0) T = (boot_theta - point_ests[np.newaxis, :]) / se_boot_safe # (n_boot, k) @@ -1266,7 +1443,8 @@ def _ppi_alpha_eff_from_M_b(M_b: np.ndarray, ci: float) -> float: return min(max(alpha_eff, 1e-9), 1.0 - 1e-9) -def _ppi_robustness_dispatch(method: str, a, a_lab, alpha: float, n_boot: int, rng): +def _ppi_robustness_dispatch(method: str, a, a_lab, alpha: float, n_boot: int, rng, + score_range: Optional[tuple[float, float]] = None): """Dispatch to the PPI-corrected single-sample implementation of *method*.""" from evalstats.tests import ( _ppi_single_wilson, _ppi_single_bootstrap_t, _ppi_single_t_interval, _ppi_single_logit_t, @@ -1278,8 +1456,10 @@ def _ppi_robustness_dispatch(method: str, a, a_lab, alpha: float, n_boot: int, r if method == "ppi_t_interval": return _ppi_single_t_interval(a, a_lab, alpha) if method == "ppi_logit_t": - # lo/hi default (0.0, 1.0) -- see _ppi_pairwise_dispatch's matching note. - return _ppi_single_logit_t(a, a_lab, alpha) + # lo/hi from the resolved score_range -- see _ppi_pairwise_dispatch's + # matching note for why the (0, 1) default is wrong here. + _lo, _hi = score_range if score_range is not None else (0.0, 1.0) + return _ppi_single_logit_t(a, a_lab, alpha, lo=_lo, hi=_hi) if method == "bootstrap": from evalstats.ppi import correct as _ppi_correct mask = ~np.isnan(a_lab) @@ -1392,8 +1572,8 @@ def _run_alignment_ppi( For ``method="auto"`` the PPI-specific auto table (``evalstats.config.resolve_ppi_auto_methods``) picks a method validated for PPI use, which need not match the non-aligned auto default for the - same data (e.g. binary data defaults to ``bayes_binary``/``tango`` - depending on N without alignment, but always ``tango`` once PPI + same data (e.g. binary data defaults to ``bayes_binary``/``mj_floor`` + depending on N without alignment, but always ``mj_floor`` once PPI correction is in play, since ``bayes_binary`` has no PPI-corrected form). When the user passes an explicit ``method=``, that exact method's PPI-corrected counterpart is used, and a clear ``ValueError`` is raised if @@ -1464,7 +1644,7 @@ def _run_alignment_ppi( raise ValueError( f"PPI alignment requires at least 15 human-labeled items; " f"got n_lab={n_lab}. Expand the alignment set and re-run " - "validate_alignment()." + "judge_alignment()." ) if n_all < 50: raise ValueError( @@ -1514,13 +1694,27 @@ def _run_alignment_ppi( from evalstats.core.resampling import is_binary_scores, is_bounded_01_scores from evalstats.config import resolve_ppi_auto_methods - if is_binary_scores(scores_2d): - data_kind = "binary" - elif is_bounded_01_scores(scores_2d): - data_kind = "bounded_01" - else: - data_kind = "unbounded" + # Reuse the ONE data-kind decision method="auto"'s router already made + # (recorded on the bundle) rather than re-deriving it here. The previous + # local re-derivation was a binary/bounded_01/unbounded test with no + # "likert" branch that consulted neither score_range nor eval_type, so + # Likert data on e.g. a 1-5 scale fell through to "unbounded" and + # silently took ppi_t_interval -- making PPI_AUTO_METHOD_TABLE's + # "likert" row (ppi_logit_t) unreachable in every case it exists for. + # The local test remains as the fallback for non-"auto" callers, where + # the router records no resolution. + data_kind = getattr(bundle, "resolved_data_kind", None) + if data_kind is None: + if is_binary_scores(scores_2d): + data_kind = "binary" + elif is_bounded_01_scores(scores_2d): + data_kind = "bounded_01" + else: + data_kind = "unbounded" + # Bounds for the scale-dependent dispatches (ppi_logit_t). Prefer the + # router's resolved range; fall back to (0, 1) only when it recorded none. + ppi_score_range = getattr(bundle, "resolved_score_range", None) if method == "auto": pairwise_method, robustness_method = resolve_ppi_auto_methods(data_kind) else: @@ -1542,13 +1736,13 @@ def _run_alignment_ppi( arr = scores_2d[i, valid] lab_arr = lab_matrix[i, valid] - res = _ppi_robustness_dispatch(robustness_method, arr, lab_arr, alpha, n_boot, rng) + res = _ppi_robustness_dispatch(robustness_method, arr, lab_arr, alpha, n_boot, rng, ppi_score_range) final_means[i] = res.estimate final_ci_low[i] = res.ci_low final_ci_high[i] = res.ci_high entity_rectifier[e] = res.rectifier for a in GRADIENT_CI_ALPHAS: - g = _ppi_robustness_dispatch(robustness_method, arr, lab_arr, a, n_boot, rng) + g = _ppi_robustness_dispatch(robustness_method, arr, lab_arr, a, n_boot, rng, ppi_score_range) multi_ci_lo[a][i] = g.ci_low multi_ci_hi[a][i] = g.ci_high @@ -1765,7 +1959,8 @@ def _pair_alpha_for(level_alpha: float) -> float: continue dispatch = lambda a_, n_boot_, rng_: _ppi_pairwise_dispatch( - pairwise_method, a_arr, b_arr, a_lab_arr, b_lab_arr, a_, n_boot_, rng_ + pairwise_method, a_arr, b_arr, a_lab_arr, b_lab_arr, a_, n_boot_, rng_, + ppi_score_range, ) elif branch == "fallback": # Not enough items are labeled for *both* entities to run the @@ -1881,9 +2076,14 @@ def _pair_alpha_for(level_alpha: float) -> float: # bundle.rank_dist was built from the raw, uncorrected LLM scores and does # not reflect the correction above — without this, P(Best)/E[Rank] would # silently stay frozen at pre-correction values even as means/CIs shift. - from evalstats.core.ranking import ppi_bootstrap_ranks - bundle.rank_dist = ppi_bootstrap_ranks(scores_2d, lab_matrix, labels, n_boot, rng) + from evalstats.core.ranking import LazyRankDistribution, ppi_bootstrap_ranks + bundle.rank_dist = LazyRankDistribution( + labels, n_boot, + lambda _rng: ppi_bootstrap_ranks(scores_2d, lab_matrix, labels, n_boot, _rng), + rng=rng, + ) bundle.ppi_applied = True + bundle.alignment_result = alignment_result # ── Override _analysis in-place ─────────────────────────────────────────── bundle.robustness.mean = final_means @@ -2035,7 +2235,7 @@ def _run_judge_alignment_if_needed( def _run_pareto_if_needed( cr: "ComparisonResult", *, - secondary, + secondary_metric, df: pd.DataFrame, factor_col: str, item_col: str, @@ -2044,36 +2244,36 @@ def _run_pareto_if_needed( rng, ) -> None: """Run uncertainty-aware Pareto-front analysis and store it on *cr*, if - ``secondary=`` was passed. + ``secondary_metric=`` was passed. Mirrors ``_run_judge_alignment_if_needed``'s validate-and-dispatch shape: - warns and no-ops on a malformed ``secondary=``, and is only supported for + warns and no-ops on a malformed ``secondary_metric=``, and is only supported for a single-factor result (a plain ``AnalysisBundle`` -- multi-model and factorial results are not yet supported, same restriction as ``alignment=``). """ - if secondary is None: + if secondary_metric is None: return - if not isinstance(secondary, dict): + if not isinstance(secondary_metric, dict): warnings.warn( - "secondary= must be a dict mapping a metric column name to " - "'min' or 'max', e.g. secondary={'latency_ms': 'min'}. " - "secondary= will be ignored.", + "secondary_metric= must be a dict mapping a metric column name to " + "'min' or 'max', e.g. secondary_metric={'latency_ms': 'min'}. " + "secondary_metric= will be ignored.", UserWarning, stacklevel=4, ) return - if len(secondary) != 1: + if len(secondary_metric) != 1: raise NotImplementedError( - f"secondary= currently supports exactly one secondary metric " - f"(bivariate Pareto fronts only); got {len(secondary)}: " - f"{list(secondary.keys())}. N-way Pareto fronts are not yet " + f"secondary_metric= currently supports exactly one secondary metric " + f"(bivariate Pareto fronts only); got {len(secondary_metric)}: " + f"{list(secondary_metric.keys())}. N-way Pareto fronts are not yet " "implemented." ) - (secondary_col, direction), = secondary.items() + (secondary_col, direction), = secondary_metric.items() if direction not in ("min", "max"): raise ValueError( - f"secondary={{'{secondary_col}': {direction!r}}} -- direction " + f"secondary_metric={{'{secondary_col}': {direction!r}}} -- direction " "must be 'min' or 'max'." ) if secondary_col not in df.columns: @@ -2088,8 +2288,8 @@ def _run_pareto_if_needed( # would never actually catch the multi-model case. Must check # cr._analysis itself, same as _run_judge_alignment_if_needed does. warnings.warn( - "Pareto-front analysis (secondary=) is not yet supported for " - "multi-model or factorial analyses. secondary= will be ignored " + "Pareto-front analysis (secondary_metric=) is not yet supported for " + "multi-model or factorial analyses. secondary_metric= will be ignored " "for this comparison.", UserWarning, stacklevel=4, @@ -2098,9 +2298,9 @@ def _run_pareto_if_needed( bundle = cr._primary_bundle() if bundle.benchmark.is_seeded: raise ValueError( - "Pareto-front analysis (secondary=) does not yet support seeded " + "Pareto-front analysis (secondary_metric=) does not yet support seeded " "benchmarks (R >= 3 repeated runs). Aggregate runs to a single " - "score per (template, input) cell before passing secondary=." + "score per (template, input) cell before passing secondary_metric=." ) from evalstats.core.pareto import pareto_bootstrap, classify_pareto_status, orient_higher_is_better @@ -2136,7 +2336,7 @@ def _run_pareto_if_needed( if np.any(np.isnan(scores_secondary)): n_missing = int(np.sum(np.isnan(scores_secondary))) raise ValueError( - f"secondary='{secondary_col}' has {n_missing} missing (entity, item) " + f"secondary_metric='{secondary_col}' has {n_missing} missing (entity, item) " f"cell(s) out of {n_entities * n_items} -- Pareto-front analysis " "currently requires a complete design (every entity scored on " "every item for the secondary metric too)." @@ -2148,6 +2348,11 @@ def _run_pareto_if_needed( result = pareto_bootstrap( scores_primary, scores_secondary_oriented, labels, n_bootstrap=n_boot, rng=rng_gen, + # Retained for plot_pareto_tradeoff()'s bootstrap point cloud, so it + # draws from the exact same replicates the calibrated status/P(Pareto- + # optimal) numbers come from, rather than a second independent + # bootstrap. Cheap: O(N x n_bootstrap) floats, not O(N^2). + return_replicates=True, ) statuses = classify_pareto_status(result, alpha=alpha) @@ -2198,17 +2403,18 @@ def compare( baseline: Optional[str] = None, block: Union[str, list[str], Literal["auto"]] = "auto", slices=None, # deferred - secondary: Optional[dict[str, Literal["min", "max"]]] = None, + secondary_metric: Optional[dict[str, Literal["min", "max"]]] = None, alignment=None, n_mc: int = 200, min_meaningful_diff=None, # deferred alpha: Optional[float] = None, - p_values: bool = False, - omnibus: bool = False, + p_values: Optional[bool] = None, + omnibus: Optional[bool] = None, pairwise_test: Literal["auto", "bootstrap", "wilcoxon", "nemenyi"] = "auto", show_rank_probabilities: bool = False, + design: Literal["auto", "paired", "unpaired"] = "auto", **kwargs: Any, -) -> ComparisonResult: +) -> Union[ComparisonResult, GroupComparisonResult]: """Compare entities along one or more factor axes. Parameters @@ -2232,22 +2438,27 @@ def compare( block : str, list[str], or "auto" Blocking variable(s) — typically ``"item"`` or ``"input"``. ``"auto"`` (default) uses the item column detected by ``load_from``. - secondary : dict[str, {"min", "max"}], optional + secondary_metric : dict[str, {"min", "max"}], optional Run an uncertainty-aware Pareto-front analysis against a second - metric, e.g. ``secondary={"latency_ms": "min"}`` to find the + metric, e.g. ``secondary_metric={"latency_ms": "min"}`` to find the accuracy/latency frontier (``"min"`` for a cost-like metric where lower is better, ``"max"`` for a benefit-like one). Currently - supports exactly one secondary metric, a complete design (every - entity scored on every item for it too), and a single-factor result + supports exactly one secondary metric and a single-factor result (not yet supported for multi-model/factorial comparisons or seeded - R>=3 benchmarks). Both metrics are resampled *jointly* (a shared - per-item bootstrap draw, not two independent marginal bootstraps) - so that correlation between them (e.g. harder items being both - slower and less accurate) is preserved rather than dropped, and a - marginally-better point estimate on both axes isn't reported as a - confident "dominates" call when the data can't actually support it. - See :attr:`ComparisonResult.pareto_status` / - :attr:`ComparisonResult.pareto_frontier_probability`. + R>=3 benchmarks). On the paired path (default), also requires a + complete design (every entity scored on every item for the + secondary metric too) and resamples both metrics *jointly* via a + shared per-item bootstrap draw (not two independent marginal + bootstraps) so correlation between them (e.g. harder items being + both slower and less accurate) is preserved rather than dropped — + a marginally-better point estimate on both axes isn't reported as + a confident "dominates" call when the data can't actually support + it. On the unpaired path (``design="unpaired"``), the same idea + applies at row granularity instead — see ``design=``'s docstring + for exactly how. See :attr:`ComparisonResult.pareto_status` / + :attr:`ComparisonResult.pareto_frontier_probability` (also exposed + identically on :class:`~evalstats.core.unpaired.GroupComparisonResult` + for the unpaired path). alpha : float, optional Significance level / CI width: ``alpha=0.05`` → 95 % CIs. When ``None`` (default), uses the global value set by @@ -2284,6 +2495,60 @@ def compare( than opt-out. Ranking is still computed either way; this only controls whether it's surfaced. Can be overridden per-call via the same-named argument on ``.summary()``/``.to_dict()``/``.to_frame()``. + design : {"auto", "paired", "unpaired"} + Experimental design for single-factor comparisons (``factors`` names + one column, and no factorial/multi-model second axis applies). + ``"auto"`` (default) checks whether items are shared across the + compared groups: when they are (the normal within-subjects case — + every entity scored on the same items), analysis proceeds exactly + as before. When items are disjoint per group (a between-subjects + design — e.g. independent user cohorts, one per condition), a + ``ValueError`` is raised rather than silently forcing a paired + analysis onto unpaired data, since ``compare()``'s default engine + assumes paired items. Pass ``design="unpaired"`` to explicitly run + the between-subjects path instead: a per-group descriptive summary + plus all-pairs comparisons (Kruskal-Wallis omnibus / Mann-Whitney U + post-hoc for continuous, likert, and grade metrics; one-way ANOVA / + Welch's t-test for binary metrics), Bonferroni-corrected CIs and + Holm-corrected p-values, PPI-corrected when ``alignment=`` is + passed. Between-subjects data commonly has no natural item/reviewer + id at all (e.g. just group + rating) — ``load_from()`` still + requires *some* item column to build ``evaldata`` in the first + place, so add a throwaway one first if needed, e.g. + ``df["item"] = range(len(df))``, before calling ``load_from()``. + Returns a :class:`~evalstats.core.unpaired.GroupComparisonResult` + instead of :class:`ComparisonResult` — see its ``.summary()``, + ``.to_dict()``, ``.to_frame()``, ``.groups_to_frame()``. Pass + ``design="paired"`` to force the existing paired analysis even on + data that looks between-subjects (matches pre-``design=`` behavior). + Not supported for factorial (2+ factor) comparisons; for + ``method="lmm"``/``"factorial_lmm"``, which already tolerate + unbalanced/disjoint designs natively via random effects; for any + other explicit ``method=``/``backend=`` override (the between- + subjects engine's CI construction isn't a pluggable-method + surface); or together with multi-run (seeded) data. + ``secondary_metric=`` (Pareto-front analysis) IS supported here — + unlike the paired path's shared-item-index joint bootstrap (every + entity resampled at the same item positions), the between-subjects + version resamples each group's own rows independently (there's no + shared item pool across disjoint groups to preserve correlation + through), still preserving each row's own primary/secondary + pairing. Populates + :attr:`~evalstats.core.unpaired.GroupComparisonResult.pareto_status`/ + ``pareto_frontier_probability`` exactly like the paired path's own + attributes. ``score_range=`` is honored (passed through to the + per-group marginal CI's auto-method resolution, same as the paired + path). ``n_mc=`` has no effect — the equivalent knob is + ``n_bootstrap=``. ``p_values=`` and ``omnibus=`` are honored, but + with unpaired-specific *defaults of True* (not ``compare()``'s own + ``False``) — leave them unset to get this path's normal, always- + shown report; pass ``p_values=False`` to hide the pairwise table's + p-value column (the underlying values stay in ``.to_dict()``/ + ``.to_frame()``), or ``omnibus=False`` to skip running the omnibus + test entirely at 3+ groups. ``baseline=``, ``pairwise_test=``, and + ``show_rank_probabilities=`` still have no effect on this path — + it always reports all-pairs comparisons (no baseline-relative + view) and has no rank-probability view. **kwargs Two uses: @@ -2429,6 +2694,26 @@ def compare( not is_model_comparison and not is_prompt_comparison) is_factorial = len(factors_list) >= 2 + # Reject NaN/missing values in factor column(s) early with a clear, + # correctly-attributed error -- otherwise a NaN factor value silently + # becomes its own group and only surfaces later as a confusing "scores + # contain N NaN cells" error that blames the metric column instead. + for _f in factors_list: + _resolved_factor_col = ( + model_col if (_f == "model" and model_col and model_col in df.columns) else + prompt_col if (_f in {"prompt", "template"} and prompt_col and prompt_col in df.columns) else + _f if _f in df.columns else None + ) + if _resolved_factor_col is not None: + _n_na_factor = int(df[_resolved_factor_col].isna().sum()) + if _n_na_factor > 0: + raise ValueError( + f"factor column {_resolved_factor_col!r} contains {_n_na_factor} " + "missing (NaN) value(s). Every row must have a value for the " + "factor being compared -- drop or fill these rows before " + "calling compare()." + ) + # Also handle the case where factor is neither "model" nor "prompt" but names # a canonical-alias column directly (e.g. user mapped "llm" → "model", then # passes factors="model" which now IS model_col). @@ -2437,6 +2722,95 @@ def compare( if factor_col_name in df.columns: is_canonical_col = True + # ── design detection / routing (paired vs. unpaired) ───────────────────── + # Scoped to "pure" single-factor cases only -- i.e. whichever of paths A/B/C + # would run below, and only when that path's own implicit multi-model second + # axis (block_col) is absent, since the multi-model/factorial machinery is + # out of scope here. Factorial calls and method="lmm"/"factorial_lmm" are + # exempt entirely: LMM already tolerates incomplete/disjoint designs via + # random effects, and no currently-passing non-LMM call can be affected by + # this new check, because the bootstrap path already hard-crashes on + # genuinely unpaired data (has_missing) -- so paired-path behavior for every + # existing call is unchanged. + _design_backend = engine_kwargs.get("method") or engine_kwargs.get("backend") + _design_exempt = is_factorial or _design_backend in {"lmm", "factorial_lmm"} + + if _design_exempt: + if design == "unpaired": + raise ValueError( + 'design="unpaired" is not supported for factorial (2+ factor) ' + 'comparisons or for method="lmm"/"factorial_lmm", which already ' + "handle unbalanced/disjoint designs natively via random effects." + ) + else: + if is_model_comparison: + _design_factor_col = model_col + _design_is_pure_single_factor = not (prompt_col and prompt_col in df.columns) + elif is_prompt_comparison: + _design_factor_col = prompt_col + _design_is_pure_single_factor = not (model_col and model_col in df.columns) + elif is_canonical_col or (not is_factorial and factors_list[0] in df.columns): + _design_factor_col = factors_list[0] + _design_is_pure_single_factor = True + else: + _design_factor_col = None + _design_is_pure_single_factor = False + + if _design_is_pure_single_factor and _design_factor_col: + if design == "unpaired" and run_col and run_col in df.columns and df[run_col].nunique() > 1: + raise ValueError( + f'design="unpaired" does not yet support multi-run (seeded) data ' + f"-- column {run_col!r} has more than one run per item. Treating " + "each run as its own row would silently inflate the effective " + "sample size and break the independence assumption the between-" + "subjects tests rely on (same scoping precedent as PPI alignment's " + "own seeded-benchmark refusal). Aggregate runs to a single score " + f"per item first, e.g. df.groupby([{_design_factor_col!r}, " + f"{item_col!r}])[{metric_col!r}].mean().reset_index()." + ) + if design == "unpaired" and _design_backend not in (None, "auto"): + raise ValueError( + f'method={_design_backend!r} is not supported together with ' + 'design="unpaired" -- the between-subjects engine\'s CI ' + "construction (Bonferroni/Holm pairwise, Kruskal-Wallis/ANOVA " + "omnibus) isn't a pluggable-method surface the way the paired " + 'path is. Drop method= for this comparison. score_range= is ' + "still honored." + ) + if design == "unpaired": + # Unlike the paired path, this narrower report defaults both + # to True (an unpaired-specific default, not compare()'s own + # False) -- unset (None, meaning the caller didn't pass + # either) preserves the always-shown behavior this path was + # built and battle-tested with; an explicit True/False is + # honored as a real suppress/show toggle. + _up_p_values = True if engine_kwargs.get("p_values") is None else bool(engine_kwargs.get("p_values")) + _up_omnibus = True if engine_kwargs.get("omnibus") is None else bool(engine_kwargs.get("omnibus")) + return compare_unpaired( + df, factor_col=_design_factor_col, metric_col=metric_col, + item_col=item_col, alignment=alignment, alpha=alpha, + n_boot=engine_kwargs.get("n_bootstrap", 2000), + rng=engine_kwargs.get("rng"), + score_range=engine_kwargs.get("score_range"), + p_values=_up_p_values, omnibus=_up_omnibus, + secondary_metric=secondary_metric, + ) + if design == "auto" and not detect_paired(df, _design_factor_col, item_col): + raise ValueError( + f"Data for factor {_design_factor_col!r} looks between-subjects " + "(items are not shared across the compared groups), but " + "compare()'s default analysis assumes within-subjects (paired) " + 'data. Pass design="unpaired" to run the between-subjects ' + 'comparison instead, or design="paired" to force the existing ' + "paired analysis anyway." + ) + elif design == "unpaired": + raise ValueError( + 'design="unpaired" is not supported for this comparison (it ' + "implies a multi-model/multi-template second axis, which is out " + "of scope for the between-subjects path)." + ) + # ── path A: model comparison ────────────────────────────────────────────── if is_model_comparison: factor_col_name = model_col @@ -2491,7 +2865,7 @@ def compare( df=df, factor_col=factor_col_name, item_col=item_col, run_col=run_col, ) _run_pareto_if_needed( - cr, secondary=secondary, df=df, factor_col=factor_col_name, + cr, secondary_metric=secondary_metric, df=df, factor_col=factor_col_name, item_col=item_col, alpha=alpha, n_boot=max(n_mc, 1000), rng=engine_kwargs.get("rng"), ) @@ -2545,7 +2919,7 @@ def compare( df=df, factor_col=factor_col_name, item_col=item_col, run_col=run_col, ) _run_pareto_if_needed( - cr, secondary=secondary, df=df, factor_col=factor_col_name, + cr, secondary_metric=secondary_metric, df=df, factor_col=factor_col_name, item_col=item_col, alpha=alpha, n_boot=max(n_mc, 1000), rng=engine_kwargs.get("rng"), ) @@ -2582,7 +2956,7 @@ def compare( df=df, factor_col=factor_col_name, item_col=item_col, run_col=run_col, ) _run_pareto_if_needed( - cr, secondary=secondary, df=df, factor_col=factor_col_name, + cr, secondary_metric=secondary_metric, df=df, factor_col=factor_col_name, item_col=item_col, alpha=alpha, n_boot=max(n_mc, 1000), rng=engine_kwargs.get("rng"), ) diff --git a/evalstats/cli.py b/evalstats/cli.py index 6b41b4a..8f397e9 100644 --- a/evalstats/cli.py +++ b/evalstats/cli.py @@ -11,6 +11,9 @@ evalstats analyze data.xlsx --sheet "Results" evalstats analyze data.csv --ci 0.90 --n-bootstrap 5000 evalstats analyze data.csv --evaluator-mode per_evaluator + + evalstats label data.csv --metric llm_score + evalstats label data.csv --metric llm_score --n-lab 20 --interactive """ from __future__ import annotations @@ -39,6 +42,8 @@ def main() -> None: args = parser.parse_args() if args.command == "analyze": _cmd_analyze(args) + elif args.command == "label": + _cmd_label(args) else: parser.print_help() sys.exit(1) @@ -109,6 +114,110 @@ def main() -> None: run → seed, repeat, run_id, trial """ +_LABEL_DESCRIPTION = """\ +Picks which rows of your data need a human grade -- a genuinely random +sample, not "the ones I happened to eyeball" -- and, if you want, lets you +grade them right here in the terminal. + +WHY THIS EXISTS: if you're using an LLM to score/judge your data and later +want to statistically correct for the judge's mistakes (judge_alignment() +and compare()'s PPI correction), the human-labeled subset has to be a +random sample of the full dataset. Hand-picking "the items I wasn't sure +about" -- the natural instinct -- breaks that assumption and silently +biases the correction. This command does the random part for you. + +CONCRETE SCENARIOS -- what your spreadsheet can look like: + + 1) Comparing several prompts/models on the SAME questions (the common + case -- one row per (condition, item), item ids repeat across every + condition): + + model, item, llm_score + baseline, 0, 0.8 + baseline, 1, 0.6 + cot, 0, 0.9 + cot, 1, 0.7 + ... + + -> picks --n-lab items ONCE and reuses them across every condition + (15 items selected = 15 x n_conditions rows marked), since the same + item ids repeat across models/prompts here. This is what the paper + example (factor='model') looks like. + + 2) A between-subjects study -- each participant/item appears under only + ONE condition, so there's nothing to share across conditions: + + condition, participant, helpfulness + control, p001, 3 + treatment, p002, 5 + ... + + -> samples --n-lab participants independently WITHIN each condition + instead (this needs --factor condition --item-col participant, + since those column names aren't auto-detected -- see FILE FORMAT + below). + + 3) You don't have LLM judge scores yet -- you just want to sample and + hand-label some ground truth first. Common for HCI researchers + collecting labels before a judge model even exists. Omit --metric + entirely, and declare what kind of grade you'll give with + --score-type (there's no judge column to guess it from): + + model, item, response_text + gpt-4o, 0, "..." + claude-3, 0, "..." + ... + + -> samples items the same way, creates one generic human_label + column instead of one per metric. + + 4) Several judge metrics on the same content (e.g. accuracy AND + fluency) -- pass --metric more than once; one round of grading + covers every metric on the same sampled items: + + model, item, accuracy, fluency + gpt-4o, 0, 0.8, 4.2 + ... + + -> --metric accuracy fluency shares one sampled item set, but each + metric gets its own human_ column. + +Re-running on an already-marked file is safe -- it tops up any condition +still short of --n-lab without disturbing prior selections or labels +already filled in, so --interactive sessions can be stopped and resumed +freely. +""" + +_LABEL_EPILOG = """\ +FILE FORMAT +----------- + +Deliberately looser than `analyze`'s: any CSV/XLSX with a numeric metric +column works (or no metric column at all -- see scenario 3 above). No +'run' column, no duplicate-row restriction, and this also accepts +between-subjects data with no shared item id across conditions -- it +doesn't need a full BenchmarkResult, just enough structure to sample from. + +Column auto-detection (case-insensitive), same aliases load_from() uses, +except metric columns -- those are always given explicitly via --metric +(never auto-detected), and score types -- those are auto-detected from an +existing --metric column, or declared via --score-type when there isn't one: + + item column : item, input, example, id, input_label + factor column : model, model_label, model_name, + prompt, template, prompt_template + +Both are optional and fall back gracefully when not found or not given: + no item column -> each row is treated as its own item (forces + independent per-condition sampling -- there's no + shared identity to reuse across conditions) + no factor column -> every row is treated as one group + +--factor/--item-col override auto-detection; use them when your columns +don't match the aliases above (as in scenario 2), or when the confirmation +prompt shows the wrong design. +""" + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -195,13 +304,15 @@ def _build_parser() -> argparse.ArgumentParser: "wilson", "newcombe", "tango", + "mj_floor", + "bonett_price", ], default="auto", metavar="METHOD", help=( "Inference method (default: auto). Use 'lmm' for mixed-effects modeling; " "binary-only modes include 'bayes_binary', 'wilson', 'newcombe', " - "and 'tango'." + "'mj_floor' (floored May & Johnson) and 'tango' (the exact Tango score interval)." ), ) analyze.add_argument( @@ -223,9 +334,13 @@ def _build_parser() -> argparse.ArgumentParser: ) analyze.add_argument( "--correction", - choices=["holm", "bonferroni", "fdr_bh", "none"], - default="fdr_bh", - help="Multiple-comparisons p-value correction (default: fdr_bh).", + choices=["auto", "holm", "bonferroni", "fdr_bh", "hochberg", "shaffer", "romano_wolf", "none"], + default="auto", + help=( + "Multiple-comparisons p-value correction (default: auto, matching " + "analyze()'s own default -- resolves to 'shaffer' or 'romano_wolf' " + "depending on N and data shape, never 'fdr_bh')." + ), ) analyze.add_argument( "--reference", @@ -345,6 +460,124 @@ def _build_parser() -> argparse.ArgumentParser: ".json (structured analysis), and .png (robustness interval plot)." ), ) + + label = sub.add_parser( + "label", + help="Randomly sample items for human labeling (for judge_alignment()/PPI).", + description=_LABEL_DESCRIPTION, + epilog=_LABEL_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + label.add_argument( + "file", + type=Path, + help=( + "Path to a CSV or XLSX file. Looser than 'analyze': any numeric metric " + "column works, no item/factor column is required -- see FILE FORMAT below." + ), + ) + label.add_argument( + "--metric", + nargs="*", + default=[], + metavar="COL", + help=( + "LLM-judge score column(s) to validate against human labels. Optional -- " + "omit entirely to sample/label ground truth before you have any judge " + "column at all (see scenario 3 above); pass --score-type in that case, " + "since there's no judge score to detect it from. Multiple metrics share " + "one sampled item set (one round of grading covers every metric on the " + "same content) but each gets its own human_ column." + ), + ) + label.add_argument( + "--score-type", + nargs="+", + default=None, + choices=["binary", "likert", "continuous", "grade"], + metavar="TYPE", + help=( + "Declare the grading scale for --interactive, instead of auto-detecting " + "it from an existing --metric column. Required when --metric is omitted " + "(nothing to auto-detect from); optional otherwise, to override a wrong " + "guess. Pass one value to apply to every metric, or one per --metric in " + "the same order. binary=0/1, likert=small integer (e.g. 1-5), " + "grade=0-100, continuous=any number." + ), + ) + label.add_argument( + "--factor", + default=None, + metavar="COL", + help=( + "Condition/factor column (e.g. 'model' or 'prompt'). Auto-detected via " + "the same column aliases load_from() uses when omitted." + ), + ) + label.add_argument( + "--item-col", + default=None, + metavar="COL", + help="Item/input identifier column. Auto-detected when omitted.", + ) + label.add_argument( + "--n-lab", + type=int, + default=15, + metavar="INT", + help="Target number of labeled items per condition (default: 15).", + ) + label.add_argument( + "--seed", + type=int, + default=None, + metavar="INT", + help=( + "Random seed for sampling. If omitted, one is generated and printed -- " + "record it for reproducibility." + ), + ) + label.add_argument( + "--human-prefix", + default="human_", + metavar="STR", + help="Prefix for the created human-label column(s) (default: 'human_').", + ) + label.add_argument( + "--sheet", + default="0", + metavar="SHEET", + help="Sheet name or 0-based index for XLSX files (default: 0).", + ) + label.add_argument( + "--out", + default=None, + metavar="PATH", + help=( + "Output file path. Defaults to '_for_labeling' next to the " + "input file, so the original is never silently overwritten." + ), + ) + label.add_argument( + "--interactive", + action="store_true", + help=( + "Grade the sampled items right here in the terminal after marking them. " + "Never shows the LLM judge's own score for the metric being graded, to " + "avoid anchoring the human rater on it. Saves after every answer; 'q' " + "quits and saves, 's' skips an item. Re-running with --interactive " + "resumes on whatever's still ungraded." + ), + ) + label.add_argument( + "-y", "--yes", + action="store_true", + help=( + "Skip the 'does this match your experimental design?' confirmation " + "prompt (auto-detected item/factor columns and paired/unpaired design). " + "For scripted use; interactive terminal runs should leave this off." + ), + ) return parser @@ -492,6 +725,146 @@ def _cmd_analyze(args: argparse.Namespace) -> None: ) +# --------------------------------------------------------------------------- +# label command +# --------------------------------------------------------------------------- + +def _cmd_label(args: argparse.Namespace) -> None: + from evalstats.labeling import ( + detect_design, describe_design, sample_for_labeling, run_interactive_labeling, + MARKER_COL, GENERIC_LABEL_KEY, VALID_SCORE_TYPES, + ) + + metrics = list(getattr(args, "metric", None) or []) + score_type_keys = metrics if metrics else [GENERIC_LABEL_KEY] + + # Argument-level validation first -- fails before touching the file at all. + score_type_overrides: dict[str, str] = {} + if args.score_type: + if len(args.score_type) == 1: + score_type_overrides = {k: args.score_type[0] for k in score_type_keys} + elif len(args.score_type) == len(score_type_keys): + score_type_overrides = dict(zip(score_type_keys, args.score_type)) + else: + _die( + f"--score-type expects 1 value (applied to all) or " + f"{len(score_type_keys)} (one per --metric, in order); " + f"got {len(args.score_type)}." + ) + if ( + getattr(args, "interactive", False) + and not metrics + and GENERIC_LABEL_KEY not in score_type_overrides + ): + _die( + "--interactive with no --metric needs --score-type -- there's no judge " + f"column to auto-detect it from. Pass one of {VALID_SCORE_TYPES}." + ) + + path = args.file.expanduser().resolve() + if not path.exists(): + _die(f"file not found: {path}") + + print(f"Loading {path.name} ...", flush=True) + sheet = _parse_sheet(getattr(args, "sheet", "0")) + try: + df = _load_file(path, sheet=sheet) + except ImportError as exc: + _die( + f"{exc}\n" + "Install openpyxl for XLSX support: pip install openpyxl\n" + "Or install with the xlsx extra: pip install evalstats[xlsx]" + ) + except Exception as exc: + _die(f"could not read file: {exc}") + + print(f" {len(df)} rows × {len(df.columns)} columns: {list(df.columns)}") + print() + + try: + design = detect_design( + df, + metrics=metrics, + factor=args.factor, + item_col=args.item_col, + ) + except ValueError as exc: + _die(str(exc)) + + print(describe_design(design)) + print() + + if not getattr(args, "yes", False): + reply = input("Does this match your experimental design? [Y/n]: ").strip().lower() + if reply not in ("", "y", "yes"): + _die( + "Aborted -- override auto-detection with --factor/--item-col " + "if it got something wrong, then re-run." + ) + + try: + marked_df, info = sample_for_labeling( + design["df"], + metrics=metrics, + factor=design["factor_col"], + item_col=design["item_col"], + n_lab=args.n_lab, + seed=args.seed, + human_col_prefix=args.human_prefix, + ) + except ValueError as exc: + _die(str(exc)) + + out_path = ( + Path(args.out).expanduser().resolve() + if args.out + else path.with_name(f"{path.stem}_for_labeling{path.suffix}") + ) + + def _save(d: pd.DataFrame) -> None: + _write_table(d, out_path) + + _save(marked_df) + + print() + print(f"Random seed used: {info['seed']} (record this for reproducibility)") + print(f"Marker column: '{MARKER_COL}' Human label column(s): {list(info['human_cols'].values())}") + print("Coverage (labeled/marked so far vs. target):") + for lvl, n in info["coverage"].items(): + print(f" {lvl}: {n}/{args.n_lab}") + print(f"Wrote: {out_path}") + + if getattr(args, "interactive", False): + try: + run_interactive_labeling( + marked_df, info, save_fn=_save, score_type_overrides=score_type_overrides + ) + except ValueError as exc: + _die(str(exc)) + print(f"Wrote: {out_path}") + else: + human_cols = list(info["human_cols"].values()) + print() + print( + "Hand this file to your labeler (fill in the human_* column(s) for " + "marked rows), or re-run with --interactive to grade it here." + ) + if metrics: + print( + "Once labeled, call judge_alignment(evaldata, llm_metric=..., " + f"human_groundtruth={human_cols[0]!r}, selection='random')." + ) + else: + print( + "No --metric given -- this samples ground-truth labels ahead of time. " + "Once you also have LLM judge scores for this data, merge them in and call" + ) + print( + f"judge_alignment(evaldata, llm_metric=..., " + f"human_groundtruth={human_cols[0]!r}, selection='random')." + ) + + # --------------------------------------------------------------------------- # File loading # --------------------------------------------------------------------------- @@ -517,6 +890,18 @@ def _load_file(path: Path, sheet: Union[int, str] = 0) -> pd.DataFrame: ) +def _write_table(df: pd.DataFrame, path: Path) -> None: + suffix = path.suffix.lower() + if suffix == ".csv": + df.to_csv(path, index=False) + elif suffix in (".xlsx", ".xls"): + df.to_excel(path, index=False) + else: + raise ValueError( + f"Unsupported output file type '{suffix}'. Use .csv, .xlsx, or .xls." + ) + + def _die(msg: str) -> None: sys.stdout.flush() print(f"evalstats error: {msg}", file=sys.stderr) diff --git a/evalstats/config.py b/evalstats/config.py index d04791e..e0e4a7e 100644 --- a/evalstats/config.py +++ b/evalstats/config.py @@ -12,7 +12,9 @@ _alpha: float = 0.05 # Alpha levels used to build the gradient CI bands in terminal plots. -# Ordered narrowest→widest: 90%, 95%, 99%, 99.9% CI. +# Ordered widest→narrowest by CI: alpha 0.32, 0.10, 0.05, 0.01 give the +# 68%, 90%, 95%, and 99% intervals. The terminal legend prints them as +# [99%/95%/90%/68%]; keep this list and that legend in step. GRADIENT_CI_ALPHAS: tuple[float, ...] = (0.32, 0.10, 0.05, 0.01) @@ -49,7 +51,7 @@ def supports_ansi_color() -> bool: # the input data shape (see BenchmarkResult.is_seeded in core/types.py) # rather than a method-selection choice. -DataKind = Literal["binary", "bounded_01", "unbounded"] +DataKind = Literal["binary", "bounded_01", "likert", "unbounded"] # --- Bootstrap resampling variant (resolve_resampling_method) -------------- # Plain (non-binary) bootstrap CIs: sample_size >= this -> "bootstrap" @@ -95,11 +97,12 @@ class AutoAnalyzeRule: # on marginal_method, so plain "wilson" / "logit_t" applied to that # already-collapsed array *is* the flat/run-means variant. Same story for # "Logit-t on run mean differences" in all_pairwise()'s "logit_t" path. And -# "ER-Tango" is not a separate method name either -- pairwise_method="tango" -# internally detects R >= 3 seeded runs and switches to the effective-N -# multirun variant (tango_paired_ci_multirun_effective), which *is* ER-Tango. -# (tango_paired_ci_multirun_moments is a related but distinct variant -- -# still available in core/resampling.py, but not what "tango" routes to.) +# the binary pairwise multirun variant is not a separate user-facing method +# name -- pairwise_method="bonett_price" internally detects R >= 3 seeded runs +# and switches to the clustered multirun variant +# (bonett_price_paired_ci_multirun_cluster). mj_floor and its own multirun +# variants remain available in core/resampling.py and callable by name, but +# are no longer what "auto" routes to for binary data. # # "bounded_01" (the data_kind label) no longer means the data is literally # valued in [0, 1] -- it means router.py._analyze_single could establish a @@ -119,45 +122,108 @@ class AutoAnalyzeRule: # recurring source of confusion between the package and the harness. AUTO_ANALYZE_METHOD_TABLE: tuple[AutoAnalyzeRule, ...] = ( AutoAnalyzeRule( - data_kind="binary", max_n=50, - pairwise_method="bayes_binary", + data_kind="binary", max_n=None, + pairwise_method="bonett_price", robustness_method_single_run="wilson", robustness_method_seeded="wilson", reason=( - "Real-data simulations show Tango under-covers in " - "dominated/jointly-sparse pairs at small N, regardless of run " - "count, so small-N binary data uses the Bayesian paired model " - "(citet{dontusetheclt}) instead. Cutoff N=50, per " - "fig:ci-decision-tree. Marginal CIs use plain Wilson regardless " - "of seeding ('Wilson flat' in the figure) -- NIG-nested was " - "tried and dropped in favor of this simpler, better-calibrated " - "choice." + "Binary data at every N: Bonett & Price (2012) adjusted-Wald " + "pairwise, Wilson-flat marginal. This is a SINGLE row where there " + "used to be two (Bayesian paired below N=50, mj_floor above) -- " + "bonett_price is best-calibrated across the whole range, so the " + "decision tree loses the split rather than just swapping a name. " + "Its Laplace adjustment (two pseudo-items) keeps the interval " + "well-behaved in the dominated/jointly-sparse pairs where the " + "score-interval form under-covers, which is what motivated the " + "old small-N branch in the first place. Seeded (R >= 3) data " + "dispatches to the clustered multirun variant automatically -- " + "see core/paired.py's bonett_price branch. Marginal CIs use plain " + "Wilson regardless of seeding ('Wilson flat' in " + "fig:ci-decision-tree)." ), ), AutoAnalyzeRule( - data_kind="binary", max_n=None, - pairwise_method="tango", - robustness_method_single_run="wilson", - robustness_method_seeded="wilson", + data_kind="bounded_01", max_n=None, + pairwise_method="logit_t", + robustness_method_single_run="logit_t", + robustness_method_seeded="logit_t", reason=( - "N >= 50 binary data: Tango pairwise (ER-Tango via the " - "effective-N multirun variant when seeded), Wilson-flat marginal." + "Numeric data with a reliable [lo, hi] range and no detected " + "quantization grid (e.g. normalised accuracy, ROUGE, or any " + "genuinely continuous metric declared via an explicit " + "score_range): Logit-t pairwise and marginal CIs, per " + "fig:ci-decision-tree. The range is either the caller's " + "explicit score_range or an exact [0, 1] match -- see " + "resolve_score_bounds() in core/resampling.py. Supersedes the " + "earlier t_interval (pairwise) / nig, nig_nested (marginal) " + "defaults for data in this range -- except discrete/ordinal " + "data (Likert scales, integer percentage grades), which is now " + "routed to the separate 'likert' row below instead." ), ), AutoAnalyzeRule( - data_kind="bounded_01", max_n=None, - pairwise_method="logit_t", + data_kind="likert", max_n=None, + pairwise_method="nig", robustness_method_single_run="logit_t", robustness_method_seeded="logit_t", reason=( - "Numeric data with a reliable [lo, hi] range (e.g. normalised " - "accuracy, ROUGE, or any scale declared via an explicit " - "score_range -- a Likert scale, a percentage grade): Logit-t " - "pairwise and marginal CIs, per fig:ci-decision-tree. The range " - "is either the caller's explicit score_range or an exact [0, 1] " - "match -- see resolve_score_bounds() in core/resampling.py. " - "Supersedes the earlier t_interval (pairwise) / nig, nig_nested " - "(marginal) defaults for data in this range." + "Discrete/ordinal bounded data (a Likert scale, an integer " + "percentage grade, or anything else with a real quantization " + "grid within its known [lo, hi] range) -- detected either from " + "an explicit eval_type='likert', or auto-detected via " + "detect_quantization_step() (core/resampling.py) when no " + "eval_type is given, with a UserWarning explaining the switch. " + "Uses NIG rather than logit-t for the PAIRWISE case -- both " + "single-run AND seeded/multi-run (unlike every other row here, " + "this one does not vary pairwise_method by seeded=): a paired " + "diff of two highly correlated Likert arms can lose real " + "variance to rounding cancellation (most items round " + "identically in both arms, only boundary-adjacent items " + "differ), which at small N can leave the *sample's* diffs " + "literally constant even though the true population diff " + "variance is nonzero -- collapsing a variance-based CI like " + "logit-t's (measured: family-wise coverage down to 14.5% at " + "n=10, k=10 comparisons, nominal 95%; reproduced again in a " + "full compare_e2e overnight sweep after NIG had been scoped " + "down to single-run-only -- fam.cov 10-26% at n=15, k=10, " + "confirming the k>=3 simultaneous-CI router's own logit-t " + "fallback carries the exact same failure mode regardless of " + "run count, since NIG's paired-diff computation is identical " + "for single- and multi-run data -- see the simulation harness's " + "simulations/harness/cases/ci_paired.py:_run_nested_pairwise_cell: " + "both reduce to the same cell-mean diffs before any CI is " + "built, R=5 just averages them first). " + "NIG's shrinkage prior protects against this without needing " + "dithering/reconstruction. This was in fact the original " + "default here (superseded by logit_t in 85df093, 'Refining the " + "sims for simultaneous cis and pvalue FWER correction') -- " + "that decision predates a fix to a real prior-scale bug " + "(nig_ci_1d's default b0 is calibrated for a single-sample " + "rescale, silently 4x too wide when reused unchanged on a " + "paired diff's rescale span, which is twice as wide -- see " + "core.paired._NIG_PAIRED_DIFF_B0), so the historical comparison " + "that dropped NIG likely made it look needlessly conservative " + "compared to logit-t. Validated post-fix, single-run " + "(reps=300, n=10-500, icc=0.01-0.95): NIG beats logit-t on " + "likert score at every N up to 500 (17% better at n=10, " + "converging to a tie by n=500); nested/multi-run (R=5, " + "reps=300): coverage nearly ties logit-t (both well-calibrated " + "by R=5, since averaging over runs smooths out the same " + "rounding-cancellation quantization that hurts logit-t at " + "single-run), but NIG still wins meaningfully on width/score " + "(~5-8% better interval score); the k>=3 simultaneous/family-" + "wise construction (core.paired._simultaneous_cis_router) now " + "also widens NIG instead of logit-t for likert data -- see " + "that function's docstring for the same fix.\n\n" + "Still NOT extended to marginal/robustness CIs (the " + "'nig'/'nig_nested' single-sample case in core/variance.py's " + "robustness_metrics()) -- never directly tested; a check of a " + "*different*, harness-only reimplementation " + "(simulations/harness/cases/ci_single.py) isn't a substitute " + "for testing this actual production code path. logit-t remains " + "the default there, and for genuinely continuous 'bounded_01' " + "data everywhere, where NIG's extra conservatism buys no " + "corresponding robustness in the first place." ), ), AutoAnalyzeRule( @@ -207,7 +273,8 @@ def resolve_auto_analyze_methods( if rule.max_n is not None and n >= rule.max_n: continue robustness = rule.robustness_method_seeded if seeded else rule.robustness_method_single_run - return rule.pairwise_method, robustness + pairwise = rule.pairwise_method + return pairwise, robustness raise AssertionError( f"no AUTO_ANALYZE_METHOD_TABLE rule matched data_kind={data_kind!r}, n={n}" ) @@ -236,12 +303,22 @@ class PPIAutoMethodRule: PPI_AUTO_METHOD_TABLE: tuple[PPIAutoMethodRule, ...] = ( PPIAutoMethodRule( data_kind="binary", - pairwise_method="tango", + pairwise_method="bonett_price", robustness_method="wilson", reason=( - "Binary data: Tango (pairwise) and Wilson (marginal) both have " - "closed-form PPI-corrected forms via an effective-n substitution " - "(see evalstats.tests._ppi_paired_tango / _ppi_single_wilson)." + "Binary data: bonett_price (pairwise) and Wilson (marginal) both " + "have closed-form PPI-corrected forms via an effective-n " + "substitution (see evalstats.tests._ppi_paired_bonett_price / " + "_ppi_single_wilson). Bonett-Price's Laplace adjustment keeps the " + "interval well-behaved when the labeled subset carries little " + "discordance information, where the score-interval form collapses " + "toward zero width. " + "Wilson matches the non-aligned default's own marginal choice " + "(AUTO_ANALYZE_METHOD_TABLE's marginal is 'wilson' at every N). " + "Pairwise is bonett_price even below the non-aligned default's " + "N<50 cutoff for bayes_binary -- a forced deviation, not a choice: " + "bayes_binary has no PPI-corrected form, so bonett_price is used " + "at every N under PPI alignment rather than raising below N=50." ), ), PPIAutoMethodRule( @@ -258,6 +335,18 @@ class PPIAutoMethodRule: "PPI-corrected logit_t existed -- that gap is now closed." ), ), + PPIAutoMethodRule( + data_kind="likert", + pairwise_method="ppi_logit_t", + robustness_method="ppi_logit_t", + reason=( + "Discrete/ordinal bounded data: there is no PPI-corrected NIG " + "implementation (NIG's win over logit-t for likert is specific " + "to the non-aligned/no-labels path -- see AUTO_ANALYZE_METHOD_" + "TABLE's 'likert' row), so this falls back to the same " + "ppi_logit_t used for 'bounded_01' rather than raising." + ), + ), PPIAutoMethodRule( data_kind="unbounded", pairwise_method="ppi_t_interval", @@ -327,33 +416,46 @@ class AutoSimultaneousCIRule: AUTO_SIMULTANEOUS_CI_METHOD_TABLE: tuple[AutoSimultaneousCIRule, ...] = ( AutoSimultaneousCIRule( - data_kind="binary", max_n=50, + data_kind="binary", max_n=None, method="sidak", reason=( - "Binary data, N < 50: Sidak's closed-form, independence-based " - "adjustment stays well-calibrated and avoids the joint " - "bootstrap's small-N instability." + "Binary data, every N: Sidak. See the numeric rule below -- the " + "reasoning is not data-kind specific, and binary is where the " + "joint bootstrap failed hardest (worst-case family coverage 0.50 " + "at n=15 and 0.74 at n=30 on the expanded scenario suite, against " + "Sidak's 0.94)." ), ), AutoSimultaneousCIRule( - data_kind="binary", max_n=None, - method="boot", + data_kind="numeric", max_n=None, + method="sidak", reason=( - "Binary data, N >= 50: joint bootstrap with an effective alpha " - "(_joint_bootstrap_scaled_simultaneous_cis) accounts for " - "correlation between comparisons that Sidak cannot." + "Numeric data, every N: Sidak, and it is now the only rule -- the " + "small-N/large-N split this table used to encode is gone.\n\n" + "Sidak was the only construction whose WORST-CASE family coverage " + "held across the expanded scenario suite (min 0.913-0.943 for " + "every eval type and N). The joint bootstrap ('boot') is better " + "centred on average and 3-5%% narrower, but its worst case " + "collapses: 0.50 on sparse/lopsided binary at n=15, and it " + "under-covers Likert at every N (0.943 pooled, degrading with k) " + "because its alpha_eff step converts a bootstrap critical value " + "through the NORMAL cdf while the Likert pairwise formula (NIG) " + "is a t interval at df=2*a_n.\n\n" + "The width Sidak gives up is small and bounded. Tukey's " + "studentized range is the optimal equal-width procedure for " + "all-pairwise comparisons, and it beats Sidak by only 1.8-3.0%% " + "-- a bound that holds here because the shared-arm contrast " + "correlation really is 0.5 (measured 0.498-0.500 across the real " + "eval corpora), which is the structure that bound assumes. Tukey " + "itself needs normality/homoscedasticity (and sphericity in the " + "repeated-measures form that applies to paired evals), which " + "binary and Likert data violate. So Sidak sits within ~3%% of the " + "achievable optimum while making no distributional assumption at " + "all.\n\n" + "'boot'/'boot_cal'/'max_t'/'bonferroni' all remain reachable via " + "an explicit prefer= argument for anyone who wants them." ), ), - AutoSimultaneousCIRule( - data_kind="numeric", max_n=30, - method="sidak", - reason="Numeric data, N < 30: Sidak.", - ), - AutoSimultaneousCIRule( - data_kind="numeric", max_n=None, - method="boot", - reason="Numeric data, N >= 30: joint bootstrap with effective alpha.", - ), ) @@ -439,6 +541,121 @@ def resolve_auto_pvalue_correction_method(n: int, *, lopsided_binary: bool = Fal raise AssertionError(f"no AUTO_PVALUE_CORRECTION_METHOD_TABLE rule matched n={n}") +# --------------------------------------------------------------------------- +# Between-subjects (unpaired) design routing -- compare(design="unpaired") +# --------------------------------------------------------------------------- +# Separate from AUTO_ANALYZE_METHOD_TABLE above (which is paired-only): that +# table's data_kind taxonomy ("binary"/"bounded_01"/"likert"/"unbounded") is +# also different from the one used here ("binary"/"continuous"/"likert"/ +# "grade", matching evalstats.loader._detect_score_type -- kept local rather +# than imported to avoid coupling this low-level module to the loader, same +# reasoning as DataKind above being declared locally rather than imported). +# +# Deliberately just two rows, decided after extensive discussion, not derived +# mechanically from AUTO_ANALYZE_METHOD_TABLE's finer-grained routing: +# +# binary -> anova_oneway (omnibus) + ttest (pairwise, Welch's). The +# textbook-correct test for comparing proportions is chi-square/Fisher's +# exact, but neither has PPI correction machinery in this codebase, and +# deriving one would be new, unvalidated statistical work. Treating a 0/1 +# outcome as a numeric mean and reusing the already-validated ttest/ +# anova_oneway PPI paths (the "linear probability model" approach) gives +# Δp (proportion difference) with a CI -- the effect size a reader expects +# for a binary outcome -- using entirely existing, validated machinery. +# Known, accepted limitation: t-intervals on binary/bounded data can +# produce out-of-[0,1]/[-1,1] CIs at extreme proportions or small N (why +# the *paired* path uses mj_floor instead of a generic t-interval for binary +# data -- there is no between-subjects Tango equivalent today). A +# deliberate patch, not a clean solution. +# +# continuous / likert / grade -> kruskalwallis (omnibus + θ_ab pairwise +# post-hoc) + mannwhitney (the k=2 special case -- Kruskal-Wallis reduces +# to Mann-Whitney at k=2). Reports a stochastic-dominance probability +# θ=P(a>b), not a mean difference -- less immediately interpretable for +# continuous data than Δmean would be, but this is the only validated +# multi-group (k>=3) pairwise mechanism in the codebase for any score +# type; a Tukey-HSD-style joint mean-difference post-hoc for continuous +# data does not exist and would itself be new, unvalidated work. +# "grade" is treated as "continuous" here (closest existing behavior) -- +# flagged as an assumption needing real-data validation, not a settled +# choice (see PLAN §5). +UnpairedScoreType = Literal["binary", "continuous", "likert", "grade"] +UnpairedFamily = Literal["binary_proportion", "rank_based"] + + +@dataclass(frozen=True) +class AutoUnpairedRule: + """One row of the ``compare(design="unpaired")`` routing table.""" + score_type: UnpairedScoreType + family: UnpairedFamily + omnibus_method: str # "anova_oneway" or "kruskalwallis" + pairwise_method: str # "ttest" or "mannwhitney" + reason: str + + +AUTO_UNPAIRED_METHOD_TABLE: tuple[AutoUnpairedRule, ...] = ( + AutoUnpairedRule( + score_type="binary", family="binary_proportion", + omnibus_method="anova_oneway", pairwise_method="ttest", + reason=( + "No PPI-corrected chi-square/Fisher's-exact exists in this " + "codebase; treating the 0/1 outcome as a numeric mean and " + "reusing the validated anova_oneway/ttest PPI paths reports " + "the proportion difference a reader expects for a binary " + "outcome, using entirely existing machinery. Known limitation: " + "t-intervals on proportions can misbehave at extreme values or " + "small N." + ), + ), + AutoUnpairedRule( + score_type="continuous", family="rank_based", + omnibus_method="kruskalwallis", pairwise_method="mannwhitney", + reason=( + "Kruskal-Wallis's θ_ab pairwise post-hoc is the only validated " + "k>=3 pairwise mechanism in this codebase for any score type; " + "a PPI-corrected Tukey-HSD-style mean-difference post-hoc does " + "not exist and would be new, unvalidated statistical work." + ), + ), + AutoUnpairedRule( + score_type="likert", family="rank_based", + omnibus_method="kruskalwallis", pairwise_method="mannwhitney", + reason="Ordinal data -- rank-based tests are the standard HCI convention.", + ), + AutoUnpairedRule( + score_type="grade", family="rank_based", + omnibus_method="kruskalwallis", pairwise_method="mannwhitney", + reason=( + "Treated as continuous for this table (closest existing " + "behavior) -- unvalidated assumption, see PLAN §5." + ), + ), +) + + +def resolve_auto_unpaired_methods(score_type: str) -> tuple[UnpairedFamily, str, str]: + """Resolve ``compare(design="unpaired")``'s routing to + ``(family, omnibus_method, pairwise_method)`` -- see + :data:`AUTO_UNPAIRED_METHOD_TABLE`. + + ``family`` is returned directly (not re-derived from ``pairwise_method`` + by the caller) so the table stays the actual source of truth for which + engine runs -- editing a row here changes behavior, rather than editing + ``omnibus_method``/``pairwise_method`` silently doing nothing because + some other call site re-derives family from a hardcoded string check. + + The *k=2* special case (``mannwhitney``/``ttest`` used directly, no + omnibus test needed since there's only one comparison) is handled by + the caller (``evalstats.core.unpaired``), not this table. + """ + for rule in AUTO_UNPAIRED_METHOD_TABLE: + if rule.score_type == score_type: + return rule.family, rule.omnibus_method, rule.pairwise_method + raise AssertionError( + f"no AUTO_UNPAIRED_METHOD_TABLE rule matched score_type={score_type!r}" + ) + + def set_alpha_ci(alpha: float) -> None: """Set the default significance level used across all CI analyses. diff --git a/evalstats/core/bundles.py b/evalstats/core/bundles.py index cccfea2..9ddf087 100644 --- a/evalstats/core/bundles.py +++ b/evalstats/core/bundles.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from .mixed_effects import LMMInfo, FactorialLMMInfo + from ..alignment import AlignmentResult # --------------------------------------------------------------------------- @@ -93,12 +94,32 @@ class AnalysisBundle: True when ``compare(..., alignment=...)`` overrode this bundle's robustness/pairwise/rank_dist with a Prediction-Powered Inference correction (see ``evalstats.api._run_alignment_ppi``). + alignment_result : AlignmentResult or None + The :class:`~evalstats.alignment.AlignmentResult` the correction + above was computed from -- set together with ``ppi_applied``, + ``None`` otherwise. Lets the summary printer show the full + alignment/representativeness report inline instead of just the + boolean flag. resolved_score_range : tuple[float, float] or None The ``(lo, hi)`` bounds actually used to rescale data for ``resolved_method='logit_t'`` / ``resolved_ci_method='logit_t'`` (user-declared via ``score_range``, or auto-detected/approximated — see ``analyze()``'s ``score_range`` parameter). ``None`` when logit-t wasn't used. + resolved_data_kind : str or None + The data kind (``"binary"``/``"bounded_01"``/``"likert"``/ + ``"unbounded"``) the ``method="auto"`` router actually resolved for + this data -- see ``evalstats.core.router.resolve_auto_robustness_method``. + Recorded so downstream consumers reuse that ONE decision instead of + re-deriving it from the scores. ``evalstats.api._run_alignment_ppi`` + does exactly that when routing PPI's own ``method="auto"``: it + previously re-derived the kind with a binary/bounded_01/unbounded + test of its own, which had no ``"likert"`` branch and consulted + neither ``score_range`` nor ``eval_type``, so Likert data on e.g. a + 1-5 scale fell through to ``"unbounded"`` and silently took + ``ppi_t_interval`` -- leaving ``PPI_AUTO_METHOD_TABLE``'s ``likert`` + row (``ppi_logit_t``) unreachable. ``None`` for non-``auto`` methods + and the LMM paths, where no such resolution happens. """ benchmark: BenchmarkResult @@ -112,8 +133,23 @@ class AnalysisBundle: resolved_method: Optional[str] = None resolved_ci_method: Optional[str] = None resolved_score_range: Optional[tuple[float, float]] = None + resolved_data_kind: Optional[str] = None p_value_method: Optional[str] = None ppi_applied: bool = False + alignment_result: Optional["AlignmentResult"] = None + + @property + def labels(self) -> list[str]: + """Canonical entity labels for this bundle. + + The single source of truth is ``benchmark.template_labels`` -- the + same list ``core.router`` feeds to every downstream construction. + Read this rather than ``rank_dist.labels``: the rank distribution is + opt-in work (see ``core.ranking.LazyRankDistribution``), so treating + it as the label registry both inverts the dependency and can force a + bootstrap nobody asked for. + """ + return list(self.benchmark.template_labels) def summary(self, **kwargs) -> None: """Print the console summary for this bundle. diff --git a/evalstats/core/design.py b/evalstats/core/design.py new file mode 100644 index 0000000..93be04a --- /dev/null +++ b/evalstats/core/design.py @@ -0,0 +1,37 @@ +"""Paired-vs-unpaired experimental design detection. + +Kept as a standalone leaf module (zero dependencies beyond pandas/numpy) so +both ``evalstats.labeling`` (the CLI sampling helper) and ``evalstats.api`` +(``compare()``'s design auto-detection) can share exactly one implementation +without either importing the other -- same rationale as ``core/bundles.py``'s +own split, just for this one function instead of a family of dataclasses. +""" +from __future__ import annotations + +from typing import Optional + +import pandas as pd + + +def detect_paired(df: pd.DataFrame, factor_col: Optional[str], item_col: str) -> bool: + """True when item ids are (largely) shared across every factor level -- + the structure evalstats' loader assumes when building an item-aligned + score matrix, and the common case for prompt/model comparisons. False + when item pools are substantially disjoint per level (e.g. a genuinely + between-subjects design with no shared item id). + + Uses a 90% overlap-with-the-full-item-universe threshold per level + rather than requiring exact equality, since a few missing/dropped rows + per condition shouldn't flip the detected design. + """ + if factor_col is None or factor_col not in df.columns: + return True + levels = df[factor_col].dropna().unique() + if len(levels) <= 1: + return True + item_sets = [set(df.loc[df[factor_col] == lvl, item_col].dropna()) for lvl in levels] + universe = set.union(*item_sets) if item_sets else set() + if not universe: + return True + overlaps = [len(s) / len(universe) for s in item_sets] + return min(overlaps) >= 0.9 diff --git a/evalstats/core/paired.py b/evalstats/core/paired.py index dbba5aa..87dd2c5 100644 --- a/evalstats/core/paired.py +++ b/evalstats/core/paired.py @@ -12,6 +12,7 @@ from __future__ import annotations +import functools import warnings from dataclasses import dataclass from typing import Callable, Literal, Optional @@ -24,10 +25,13 @@ wilcoxon as _es_wilcoxon, friedman as _es_friedman, _mcnemar_p, + _mcnemar_midp_p, _paired_sign_test_p, _paired_signflip_pvalue, ) from .resampling import ( + _logit_t_alpha_crit_batch, + _nig_alpha_crit_batch, bca_interval_1d, bayes_bootstrap_means_1d, bayes_bootstrap_diffs_nested, @@ -38,13 +42,22 @@ bootstrap_t_ci_1d, bootstrap_t_ci_nested, resolve_resampling_method, - newcombe_paired_ci, - tango_paired_ci, - tango_paired_ci_from_diffs, - tango_paired_ci_multirun_effective, + newcombe_mover_paired_ci, + mj_floor_paired_ci, + tango_scc_paired_ci, + bonett_price_paired_ci_from_diffs, + mj_floor_paired_ci_from_diffs, + mj_floor_paired_ci_multirun_effective, + mj_floor_paired_ci_multirun_cluster, + bonett_price_paired_ci, + bonett_price_paired_ci_multirun_cluster, + bonett_price_paired_ci_multirun_shrunk, t_interval_ci_1d, logit_t_ci_1d, + nig_ci_1d, bayes_paired_diff_ci, + binary_routing_applies, + degenerate_sample_ci, is_binary_scores, is_lopsided_binary, _stat, @@ -61,6 +74,24 @@ BAYES_BINARY_LARGE_N_THRESHOLD = 200 +_NIG_PAIRED_DIFF_B0 = 0.0625 / 4 +"""nig_ci_1d's default b0=0.0625 (prior mean of sigma^2, i.e. prior +sigma~=0.25) is calibrated for a single-sample rescale onto [lo, hi] -- +see that function's own docstring: "weak knowledge that scores live in +[0, 1]". A PAIRED diff instead gets rescaled onto [-(hi-lo), hi-lo] +(needed so a zero diff maps to 0.5, nig's own prior centre) -- twice as +wide a span as the single-sample case. Reusing b0=0.0625 unchanged on a +paired diff implies 2^2=4x the intended prior variance in real diff units +(variance scales with the square of a linear rescale factor), producing +persistent, substantial over-coverage that isn't a deliberate safety +margin, just an unpropagated rescale-span change. This restores NIG's +effective prior to the same absolute variance the single-sample case +already uses correctly -- verified via simulation +(simulations/harness/cases/ci_paired.py): on likert paired diffs, +coverage went from 0.983 (n=10, default b0) to 0.946 (n=10, this +correction), 23% narrower for the same validity, holding across n=10-500 +and on continuous data too.""" + def _warn_bayes_binary_large_n(n_inputs: int, *, stacklevel: int = 4) -> None: """Warn when bayes_binary pairwise CI is used beyond its calibrated range.""" @@ -381,13 +412,61 @@ def point_diff_matrix(self) -> np.ndarray: return mat +def _paired_t_pvalue( + values_a: np.ndarray, values_b: np.ndarray, diffs: np.ndarray, +) -> float: + """Paired t-test p-value, with an exact sign-test floor on zero-variance + differences. + + Delegates to :func:`evalstats.tests.ttest`'s uncorrected paired path so + the scipy call has a single implementation, then guards the one input + where that p-value is not just imprecise but degenerate: a **constant + non-zero** difference vector (every ``a_i - b_i`` identical, e.g. arm A + scores 0.9 on every item and arm B scores 0.8). There ``s = 0``, so + ``t = d/(s/sqrt(M))`` diverges and scipy returns exactly ``0.0`` -- + certainty that the effect is non-zero, obtained from a sample that + contains no variance estimate at all. + + That number is indefensible on its own terms, and it also sits badly + next to the companion interval, which on this same input is now the + deliberately wide :func:`~evalstats.core.resampling.degenerate_sample_ci` + bound (see :func:`_bonferroni_simultaneous_cis`) and can straddle 0. The + two are not actually in conflict -- they answer different questions: an + all-same-sign difference vector *does* rule out a null symmetric about + 0, while still leaving the *mean* unbounded away from 0, because the + unobserved tail mass the CI has to allow for could sit anywhere in the + metric's range. But that reading only survives if the p-value is a real + number from a stated test rather than a divide-by-zero artifact. + + The replacement is :func:`~evalstats.tests._paired_sign_test_p`, the + exact two-sided sign test, which on M identical non-zero differences is + ``binomtest(M, M, 0.5) = 2 * 0.5**M``. This is the strongest claim the + data supports without a variance estimate -- it uses only the signs, + which is all a zero-spread sample actually pins down -- and it is not a + new convention here: the binary/Tango and ``sign_test`` paths already + report exactly this number on the same input (2**-29 at M=30), so this + makes the continuous paths agree with them instead of reporting 0. + + Applied as ``max()`` rather than a straight substitution, so it can only + ever widen the p-value, and only on the degenerate branch -- a genuinely + tiny t-test p-value from data that *does* have spread is left alone. + """ + t_result = _es_ttest(values_a, values_b, paired=True, print_result=False) + p_value = float(t_result.p_value) if np.isfinite(t_result.p_value) else 1.0 + if len(diffs) >= 1 and float(np.ptp(diffs)) == 0.0: + # Zero-variance differences. (_paired_sign_test_p itself returns 1.0 + # for the all-zero case, which is the right answer there too.) + return max(p_value, _paired_sign_test_p(diffs)) + return p_value + + def pairwise_differences( scores: np.ndarray, idx_a: int, idx_b: int, label_a: str = "A", label_b: str = "B", - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "mj_floor", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, rng: Optional[np.random.Generator] = None, @@ -414,9 +493,11 @@ def pairwise_differences( ``'bayes_bootstrap'`` (Bayesian bootstrap), ``'smooth_bootstrap'`` (smoothed bootstrap via Gaussian KDE), ``'bootstrap_t'`` (studentized bootstrap-t CI), ``'newcombe'`` for paired - binary (0/1) data using Newcombe CI + exact McNemar p-value, - ``'tango'`` for paired binary (0/1) data using Tango score CI + - exact McNemar p-value, or + binary (0/1) data using Newcombe CI + McNemar mid-p p-value, + ``'mj_floor'`` for paired binary (0/1) data using the floored + May & Johnson (1997) score CI + McNemar mid-p p-value, + ``'tango'`` for paired binary (0/1) data using the exact Tango + (1998) score CI + McNemar mid-p p-value (single-run only), or ``'bayes_binary'`` for paired binary (0/1) data using the Dirichlet-multinomial Bayesian model (Bowyer et al. 2025). Requires binary data; raises ValueError otherwise. @@ -636,9 +717,9 @@ def _build_result( values_b = flat[idx_b] diffs, _, point_d, std_d = _paired_stats(values_a, values_b) alpha_val = 1.0 - ci - ci_low, ci_high = newcombe_paired_ci(values_a, values_b, alpha_val) - p_value = _mcnemar_p(values_a, values_b) - mci = {_a: newcombe_paired_ci(values_a, values_b, _a) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None + ci_low, ci_high = newcombe_mover_paired_ci(values_a, values_b, alpha_val) + p_value = _mcnemar_midp_p(values_a, values_b) + mci = {_a: newcombe_mover_paired_ci(values_a, values_b, _a) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None return _build_result( diffs=diffs, point_d=point_d, @@ -646,16 +727,16 @@ def _build_result( ci_low=ci_low, ci_high=ci_high, p_value=p_value, - test_name="newcombe (mcnemar p-value)", + test_name="newcombe (mcnemar_midp p-value)", values_a=values_a, values_b=values_b, multi_ci_dict=mci, ) - if method == "tango": + if method == "mj_floor": multirun = scores.ndim == 3 and scores.shape[2] > 1 if multirun: - # Multi-run: use the effective-N variant (tango_multirun_effective, + # Multi-run: use the effective-N variant (mj_floor_er, # "ER-Tango" in the paper's decision tree / appendix), which estimates # an effective number of runs to account for within-item correlation # and reduces exactly to the standard Tango CI when n_runs == 1. @@ -671,15 +752,108 @@ def _build_result( alpha_val = 1.0 - ci if multirun: - ci_low, ci_high = tango_paired_ci_multirun_effective(values_a_full, values_b_full, alpha_val) + # Cluster (plain item-level variance), NOT the effective-runs variant: + # its Kish R_eff term cancels exactly when the max() does not clamp and + # inflates variance up to 2.8x when it does, so it was inert in the + # high-ICC regime real eval data occupies and conservative elsewhere. + ci_low, ci_high = mj_floor_paired_ci_multirun_cluster(values_a_full, values_b_full, alpha_val) if multi_ci: - mci = {_a: tango_paired_ci_multirun_effective(values_a_full, values_b_full, _a) for _a in GRADIENT_CI_ALPHAS} + mci = {_a: mj_floor_paired_ci_multirun_cluster(values_a_full, values_b_full, _a) for _a in GRADIENT_CI_ALPHAS} else: mci = None else: - ci_low, ci_high = tango_paired_ci(values_a, values_b, alpha_val) - mci = {_a: tango_paired_ci(values_a, values_b, _a) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None - p_value = _mcnemar_p(values_a, values_b) + ci_low, ci_high = mj_floor_paired_ci(values_a, values_b, alpha_val) + mci = {_a: mj_floor_paired_ci(values_a, values_b, _a) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None + p_value = _mcnemar_midp_p(values_a, values_b) + return _build_result( + diffs=diffs, + point_d=point_d, + std_d=std_d, + ci_low=ci_low, + ci_high=ci_high, + p_value=p_value, + test_name="mj_floor cluster" if multirun else "mj_floor", + values_a=values_a, + values_b=values_b, + multi_ci_dict=mci, + ) + + # ------------------------------------------------------------------ # + # Bonett-Price path for paired binary (0/1) data # + # ------------------------------------------------------------------ # + if method == "bonett_price": + multirun = scores.ndim == 3 and scores.shape[2] >= 3 + _flat_check = scores.mean(axis=2) if scores.ndim == 3 else scores + if not is_binary_scores(scores if multirun else _flat_check): + raise ValueError( + "method='bonett_price' requires binary (0/1) data, but the scores " + "array contains non-binary values. Use is_binary_scores() to check " + "before calling, or choose a different method." + ) + if multirun: + values_a_full = scores[idx_a] + values_b_full = scores[idx_b] + values_a = values_a_full[:, 0] + values_b = values_b_full[:, 0] + else: + flat = scores.mean(axis=2) if scores.ndim == 3 else scores + values_a = flat[idx_a] + values_b = flat[idx_b] + diffs, _, point_d, std_d = _paired_stats(values_a, values_b) + alpha_val = 1.0 - ci + if multirun: + # Multi-run default is the Laplace-shrunk-magnitude variant: the + # +/-1 pseudo-items of the _cluster form are the largest possible + # item-level discordance, which is right at R=1 but several times + # heavier than a real discordant item once runs are averaged. + # _shrunk reduces to bonett_price_paired_ci at R=1 bit-for-bit. + ci_low, ci_high = bonett_price_paired_ci_multirun_shrunk( + values_a_full, values_b_full, alpha_val + ) + mci = ({_a: bonett_price_paired_ci_multirun_shrunk(values_a_full, values_b_full, _a) + for _a in GRADIENT_CI_ALPHAS} if multi_ci else None) + else: + ci_low, ci_high = bonett_price_paired_ci(values_a, values_b, alpha_val) + mci = ({_a: bonett_price_paired_ci(values_a, values_b, _a) + for _a in GRADIENT_CI_ALPHAS} if multi_ci else None) + p_value = _mcnemar_midp_p(values_a, values_b) + return _build_result( + diffs=diffs, + point_d=point_d, + std_d=std_d, + ci_low=ci_low, + ci_high=ci_high, + p_value=p_value, + test_name="bonett_price shrunk" if multirun else "bonett_price", + values_a=values_a, + values_b=values_b, + multi_ci_dict=mci, + ) + + if method == "tango": + # The genuine Tango (1998) asymptotic score interval, obtained in + # closed form via Chang et al. (2024)'s quartic with the continuity + # correction set to zero. Validated against the published limits in + # Fagerland, Lydersen & Laake (2014), Table V. + # + # NOTE: before 2026-08-24 this name dispatched to what is now + # ``mj_floor`` -- a May & Johnson construction that is NOT Tango's + # interval. See mj_floor_paired_ci's docstring. + if scores.ndim == 3 and scores.shape[2] > 1: + raise NotImplementedError( + "method='tango' (the exact Tango score interval) has no " + "multi-run form. Use method='mj_floor' for multi-run paired " + "binary data, which dispatches to the effective-runs variant." + ) + flat = scores.mean(axis=2) if scores.ndim == 3 else scores + values_a = flat[idx_a] + values_b = flat[idx_b] + diffs, _, point_d, std_d = _paired_stats(values_a, values_b) + alpha_val = 1.0 - ci + ci_low, ci_high = tango_scc_paired_ci(values_a, values_b, alpha_val, c=0.0) + mci = ({_a: tango_scc_paired_ci(values_a, values_b, _a, c=0.0) + for _a in GRADIENT_CI_ALPHAS} if multi_ci else None) + p_value = _mcnemar_midp_p(values_a, values_b) return _build_result( diffs=diffs, point_d=point_d, @@ -687,7 +861,7 @@ def _build_result( ci_low=ci_low, ci_high=ci_high, p_value=p_value, - test_name="tango", + test_name="tango score (exact)", values_a=values_a, values_b=values_b, multi_ci_dict=mci, @@ -745,10 +919,7 @@ def _build_result( diffs, _, point_d, std_d = _paired_stats(values_a, values_b) alpha_val = 1.0 - ci ci_low, ci_high = t_interval_ci_1d(diffs, alpha_val) - # Delegates to evalstats.tests.ttest (uncorrected paired path) so the - # scipy call has a single implementation. - t_result = _es_ttest(values_a, values_b, paired=True, print_result=False) - p_value = float(t_result.p_value) if np.isfinite(t_result.p_value) else 1.0 + p_value = _paired_t_pvalue(values_a, values_b, diffs) mci = {_a: t_interval_ci_1d(diffs, _a) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None return _build_result( diffs=diffs, @@ -784,8 +955,7 @@ def _build_result( diff_span = (score_range[1] - score_range[0]) if score_range is not None else 1.0 diff_lo, diff_hi = -diff_span, diff_span ci_low, ci_high = rescaled_ci(logit_t_ci_1d, diffs, alpha_val, diff_lo, diff_hi) - t_result = _es_ttest(values_a, values_b, paired=True, print_result=False) - p_value = float(t_result.p_value) if np.isfinite(t_result.p_value) else 1.0 + p_value = _paired_t_pvalue(values_a, values_b, diffs) mci = ( {_a: rescaled_ci(logit_t_ci_1d, diffs, _a, diff_lo, diff_hi) for _a in GRADIENT_CI_ALPHAS} if multi_ci else None @@ -803,6 +973,44 @@ def _build_result( multi_ci_dict=mci, ) + # ------------------------------------------------------------------ # + # Paired NIG path (discrete/ordinal bounded data, e.g. Likert) # + # ------------------------------------------------------------------ # + if method == "nig": + # Same rescale structure as the logit_t path above (paired diff of + # two [lo, hi] scores spans [-(hi-lo), hi-lo]), but with the prior + # variance corrected for that wider span -- see + # _NIG_PAIRED_DIFF_B0's docstring. Recommended over logit_t + # specifically for discrete/ordinal data (a Likert scale, an + # integer percentage grade): see config.AUTO_ANALYZE_METHOD_TABLE's + # "likert" row for the full rationale. + flat = scores.mean(axis=2) if scores.ndim == 3 else scores + values_a = flat[idx_a] + values_b = flat[idx_b] + diffs, _, point_d, std_d = _paired_stats(values_a, values_b) + alpha_val = 1.0 - ci + diff_span = (score_range[1] - score_range[0]) if score_range is not None else 1.0 + diff_lo, diff_hi = -diff_span, diff_span + _nig_paired = functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0) + ci_low, ci_high = rescaled_ci(_nig_paired, diffs, alpha_val, diff_lo, diff_hi) + p_value = _paired_t_pvalue(values_a, values_b, diffs) + mci = ( + {_a: rescaled_ci(_nig_paired, diffs, _a, diff_lo, diff_hi) for _a in GRADIENT_CI_ALPHAS} + if multi_ci else None + ) + return _build_result( + diffs=diffs, + point_d=point_d, + std_d=std_d, + ci_low=ci_low, + ci_high=ci_high, + p_value=p_value, + test_name="paired NIG", + values_a=values_a, + values_b=values_b, + multi_ci_dict=mci, + ) + # ------------------------------------------------------------------ # # Route: seeded (R >= 3) vs. standard (2-D or R < 3) # # ------------------------------------------------------------------ # @@ -1279,7 +1487,7 @@ def _max_stat_simultaneous_cis( ``'bayes_bootstrap'``, ``'smooth_bootstrap'``, ``'bootstrap_t'``, ``'auto'`` (treated as ``'smooth_bootstrap'``), ``'permutation'``, ``'sign_test'``. Methods that do not use bootstrap resampling - for CIs (``'newcombe'``, ``'tango'``, ``'bayes_binary'``, + for CIs (``'newcombe'``, ``'mj_floor'``, ``'tango'``, ``'bayes_binary'``, ``'lmm'``) are not supported; an empty dict is returned for these. Ignored when *precomputed_boot_stats* is supplied. ci : float @@ -1559,10 +1767,84 @@ def _batch_resample( return _apply_max_t_cis(boot_stats, point_ests, pairs, ci) +def _degenerate_pair_ci( + point_diff: float, + M: int, + alpha: float, + diff_bounds: Optional[tuple[float, float]], +) -> tuple[float, float]: + """CI for a pair whose paired differences carry no variance information. + + Covers both degenerate inputs :func:`_bonferroni_simultaneous_cis` can + hit: ``M < 2`` (a single paired observation, or none) and a constant + difference vector (``se`` numerically 0). In both, every variance-driven + construction -- the t-interval, the delta method, any resampling scheme + -- collapses to the zero-width interval ``(point_diff, point_diff)``, + which asserts the effect is *exactly* ``point_diff`` with certainty and + covers the truth with probability 0 unless the difference really is a + point mass. See :func:`~evalstats.core.resampling.degenerate_sample_ci` + for the bound used instead and why it is the honest answer. + + Marginal coverage on a paired DGP that reaches this branch often + (difference = +0.1 with probability p, -0.5 otherwise, so the truth is + *near* the atom but not at it; 4000 reps, diff bounds [-1, 1], + nominal 95%):: + + n p P(degenerate) coverage before coverage after + 10 0.90 0.36 0.641 0.999 + 10 0.99 0.90 0.099 1.000 + 20 0.97 0.55 0.453 1.000 + 30 0.90 0.05 0.951 0.998 + 30 0.99 0.73 0.267 1.000 + + -- i.e. the old branch missed on *every* degenerate rep (the truth is + never exactly the atom), so coverage tracked ``1 - P(degenerate)`` and + collapsed as the branch fired more often. The new bound is conservative + rather than nominal (~100%, the price + :func:`~evalstats.core.resampling.degenerate_sample_ci` documents), and + is only ever paid on samples that would otherwise have been reported + with false certainty. + + *diff_bounds* is the support of a single paired difference, ``(-(hi-lo), + hi-lo)`` for a metric ranging over ``[lo, hi]`` -- not the metric's own + bounds. The router resolves it per data kind (see + :func:`_simultaneous_cis_router`). + + When *diff_bounds* is ``None`` -- the unbounded data kind, i.e. no + ``score_range`` and non-binary scores -- the result is + ``(-inf, +inf)``. That is not a punt: for a distribution with unbounded + support, no finite confidence interval for the mean has guaranteed + coverage over all distributions (Bahadur-Savage), and a zero-variance + sample is exactly the case where nothing else is left to lean on. An + infinite interval says "this tells you nothing about the mean", which is + true; the zero-width one said the opposite. Callers who want a finite + answer here should pass ``score_range`` -- the interval then narrows to + the ``degenerate_sample_ci`` bound, whose width is roughly + ``ln(2/alpha) * 2*(hi-lo)/M``. Emits a ``UserWarning`` saying so, since + an ``inf`` bound appearing in a report deserves an explanation. + """ + if diff_bounds is None: + warnings.warn( + "Simultaneous CI: a pair's per-input differences have zero " + "variance (all identical) and the data has no known bounds " + "(non-binary scores, no score_range given), so its mean cannot " + "be bounded at any confidence level and the interval is " + "reported as (-inf, +inf). Pass score_range=(min, max) to get " + "the finite conservative interval instead.", + UserWarning, + stacklevel=3, + ) + return (float("-inf"), float("inf")) + lo, hi = float(diff_bounds[0]), float(diff_bounds[1]) + value = float(min(max(point_diff, lo), hi)) if np.isfinite(point_diff) else lo + return degenerate_sample_ci(value, M, alpha, lo, hi) + + def _bonferroni_simultaneous_cis( results: dict[tuple[str, str], "PairedDiffResult"], pairs: list[tuple[str, str]], ci: float, + diff_bounds: Optional[tuple[float, float]] = None, ) -> dict[tuple[str, str], tuple[float, float]]: """Bonferroni-corrected simultaneous CIs via per-pair paired t-intervals. @@ -1571,7 +1853,27 @@ def _bonferroni_simultaneous_cis( ``per_input_diffs`` already stored in each :class:`PairedDiffResult`. This makes the result independent of the original CI method, so it works as a universal fallback for non-bootstrap methods such as - ``'newcombe'``, ``'tango'``, and ``'bayes_binary'``. + ``'newcombe'``, ``'mj_floor'``, ``'tango'``, and ``'bayes_binary'``. + + It is also the *only* construction that runs for a **single pair** + (k=1): :func:`_simultaneous_cis_router` gates Sidak/boot on + ``len(pairs) > 1``, so a two-arm comparison lands here unconditionally. + That makes the degenerate branches below load-bearing for the most + common shape of comparison there is, not just an edge case in a large + family -- they used to return ``(point_diff, point_diff)``, so + ``compare()`` on two arms with a constant offset (arm A ≡ 0.9, arm B ≡ + 0.8) reported a zero-width CI at exactly the point estimate, and it + *overrode* the underlying method's own correct interval on the same + result object. They now delegate to :func:`_degenerate_pair_ci`. + + Parameters + ---------- + diff_bounds : tuple[float, float], optional + Support of a single paired difference, ``(-(hi-lo), hi-lo)`` for a + metric over ``[lo, hi]``. Used *only* on the degenerate branches; + the ordinary t-interval path ignores it. ``None`` (the default) + means no bounds are known -- see :func:`_degenerate_pair_ci` for + what that yields and why. Returns ------- @@ -1593,11 +1895,15 @@ def _bonferroni_simultaneous_cis( diffs = r.per_input_diffs M = len(diffs) if M < 2: - sim_cis[pair] = (float(r.point_diff), float(r.point_diff)) + sim_cis[pair] = _degenerate_pair_ci( + float(r.point_diff), M, alpha_adj, diff_bounds, + ) continue se = float(np.std(diffs, ddof=1)) / np.sqrt(M) if se < 1e-12: - sim_cis[pair] = (float(r.point_diff), float(r.point_diff)) + sim_cis[pair] = _degenerate_pair_ci( + float(r.point_diff), M, alpha_adj, diff_bounds, + ) continue t_crit = float(_scipy_stats.t.ppf(1.0 - alpha_adj / 2.0, df=M - 1)) half = t_crit * se @@ -1618,8 +1924,8 @@ def _sidak_simultaneous_cis( This is agnostic to which CI construction it widens: *ci_func* is any callable ``(diffs, alpha) -> (ci_low, ci_high)`` -- e.g. - :func:`~evalstats.core.resampling.tango_paired_ci_from_diffs` for binary - paired data, but equally ``newcombe_paired_ci``, ``t_interval_ci_1d``, or + :func:`~evalstats.core.resampling.mj_floor_paired_ci_from_diffs` for binary + paired data, but equally ``newcombe_mover_paired_ci``, ``t_interval_ci_1d``, or any other closed-form interval that accepts a significance level. Each pair's CI is *ci_func* evaluated at the Sidak-adjusted per- @@ -1711,6 +2017,194 @@ def _joint_bootstrap_critical_value( return float(np.quantile(M_b, ci)) +#: Resample cap for _calibrated_joint_critical_value -- see its use there. +_CALIBRATED_JOINT_MAX_RESAMPLES = 1500 + + +def _scipy_stats_norm_ppf(a: float) -> float: + """z such that 2*(1-Phi(z)) == a -- the inverse of the alpha_eff step in + _calibrated_joint_simultaneous_cis, so an exactly-calibrated alpha survives + the round trip through that conversion unchanged.""" + from scipy import stats as _st + return float(_st.norm.ppf(a / 2.0)) + + +def _calibrated_joint_critical_value( + scores: np.ndarray, + pairs: list[tuple[str, str]], + labels: list[str], + ci: float, + n_bootstrap: int, + rng: "np.random.Generator", + ci_func: "Callable[[np.ndarray, float], tuple[float, float]]", + *, + statistic: Literal["mean", "median"] = "mean", + alpha_ref: float = 0.05, +) -> Optional[float]: + """Joint critical value studentized by *ci_func's own* centre and scale. + + :func:`_joint_bootstrap_critical_value` standardizes each replicate by the + BOOTSTRAP standard error of the point estimate, then + :func:`_joint_bootstrap_scaled_simultaneous_cis` converts the resulting + *c* to ``alpha_eff = 2(1-Phi(c))`` and evaluates ``ci_func`` there. That + composition is only exact when ``ci_func(., a)`` has coverage exactly + ``1-a``. When the formula is marginally conservative -- Bonett-Price's + Laplace adjustment measures ``1-a+delta`` with delta up to +4.3pp at + n=10, decaying to ~+0.2pp by n=100 -- the simultaneous interval inherits + that conservatism on top of the multiplicity widening. + + This variant removes that assumption by reading the centre and scale off + ``ci_func`` itself on every replicate:: + + lo, hi = ci_func(resampled_diffs_r, alpha_ref) + m = (lo + hi) / 2 # the formula's own centre + s = (hi - lo) / (2 z_{alpha_ref/2}) # the formula's own scale + z_r = |theta_r - m| / s + + so the returned quantile of ``max_r z_r`` is calibrated against the + construction's actual finite-sample behaviour, including any centre shift + (Bonett-Price shrinks the point estimate by n/(n+2), which the bootstrap-SE + route ignores entirely). Stays method-agnostic: ``ci_func`` is only ever + called as ``(diffs, alpha)``. + + Costs ``n_bootstrap * k`` calls to *ci_func* -- linear, not the nested + ``B^2`` a naive recalibration would need, because for an interval whose + half-width is proportional to the normal quantile the level at which a + replicate just covers has a closed form. + + Returns ``None`` when there are no pairs or every pair is degenerate. + """ + from scipy import stats as _scipy_stats + + k = len(pairs) + if k == 0: + return None + + z_ref = float(_scipy_stats.norm.ppf(1.0 - alpha_ref / 2.0)) + label_to_idx = {label: idx for idx, label in enumerate(labels)} + flat = scores.mean(axis=2) if scores.ndim == 3 else scores # (N, M) + pair_indices = [(label_to_idx[a], label_to_idx[b]) for (a, b) in pairs] + diffs_mat = np.stack([flat[i] - flat[j] for (i, j) in pair_indices], axis=0) # (k, M) + M = diffs_mat.shape[1] + point_ests = diffs_mat.mean(axis=1) if statistic == "mean" else np.median(diffs_mat, axis=1) + + # Same degeneracy rule as _joint_bootstrap_critical_value: a pair with no + # spread contributes an unbounded standardized deviation and would other- + # wise dominate every replicate's max. + spread = np.ptp(diffs_mat, axis=1) + valid = spread > 1e-12 + if not np.any(valid): + return None + + # The calibration only needs a (1-alpha) quantile of a max over k pairs, + # which stabilizes well before the resample count `boot` uses for its SE. + # Capped because this loop costs n_cal * k calls into ci_func (Python-level, + # since ci_func is an arbitrary callable), vs `boot`'s fully vectorized + # resample -- uncapped at n_bootstrap=5000 it runs ~100x slower than boot + # for no measurable gain in the quantile. + n_cal = int(min(n_bootstrap, _CALIBRATED_JOINT_MAX_RESAMPLES)) + input_idx = rng.integers(0, M, size=(n_cal, M)) + + # Fast path: a formula may publish `centre_scale_batch`, evaluating its own + # centre and scale over a whole (n_cal, M) resample matrix in one numpy + # call instead of n_cal scalar calls per pair. Two wins, not one: + # - ~485x faster on the inner loop (this is otherwise 1500 * k Python + # calls; at k=20 that is 285k of them); + # - EXACT, where the fallback is not. The fallback recovers scale as + # (hi - lo) / (2 z_ref), which understates it whenever the interval + # clipped at its bounds -- exactly the sparse small-n case this + # calibration exists for. + # EXACT path: a formula may publish `alpha_crit_batch`, giving the level at + # which each resample's interval just covers a target. That is the quantity + # this calibration actually wants, and it needs no reference-distribution + # assumption of ours: the formula answers in its own parameterization + # (Bonett-Price normal on the difference scale, NIG t at df=2*a_n, logit-t t + # at df=n-1 on the LOGIT scale). Joint coverage at a' is P(a' <= min_r + # alpha_crit), so alpha* is the alpha-quantile of those per-replicate minima. + # The centre/scale route below cannot express the logit-t case at all -- it + # assumes symmetry on the difference scale -- so it is an approximation + # there, not just a slower path. + acrit = getattr(ci_func, "alpha_crit_batch", None) + if acrit is not None: + a_min = np.ones(n_cal, dtype=float) + for r in range(k): + if not valid[r]: + continue + a_r = np.asarray(acrit(diffs_mat[r][input_idx], float(point_ests[r])), dtype=float) + np.minimum(a_min, a_r, out=a_min) + alpha_star = float(np.quantile(a_min, 1.0 - ci)) + alpha_star = min(max(alpha_star, 1e-12), 1.0 - 1e-12) + return -float(_scipy_stats_norm_ppf(alpha_star)) + + batch = getattr(ci_func, "centre_scale_batch", None) + if batch is not None: + z_max = np.zeros(n_cal) + for r in range(k): + if not valid[r]: + continue + centre, scale = batch(diffs_mat[r][input_idx], alpha_ref) + centre = np.asarray(centre, dtype=float) + scale = np.asarray(scale, dtype=float) + ok = np.isfinite(centre) & np.isfinite(scale) & (scale > 1e-12) + if not np.any(ok): + continue + z_r = np.zeros(n_cal) + z_r[ok] = np.abs(point_ests[r] - centre[ok]) / scale[ok] + np.maximum(z_max, z_r, out=z_max) + else: + z_max = np.empty(n_cal) + for b in range(n_cal): + idx = input_idx[b] + worst = 0.0 + for r in range(k): + if not valid[r]: + continue + lo, hi = ci_func(diffs_mat[r][idx], alpha_ref) + if not (np.isfinite(lo) and np.isfinite(hi)): + continue + scale = (hi - lo) / (2.0 * z_ref) + if not np.isfinite(scale) or scale <= 1e-12: + continue + centre = 0.5 * (lo + hi) + worst = max(worst, abs(point_ests[r] - centre) / scale) + z_max[b] = worst + if not np.any(z_max > 0.0): + return None + return float(np.quantile(z_max, ci)) + + +def _calibrated_joint_simultaneous_cis( + scores: np.ndarray, + results: dict[tuple[str, str], "PairedDiffResult"], + pairs: list[tuple[str, str]], + labels: list[str], + ci: float, + n_bootstrap: int, + rng: "np.random.Generator", + ci_func: "Callable[[np.ndarray, float], tuple[float, float]]", + *, + statistic: Literal["mean", "median"] = "mean", +) -> dict[tuple[str, str], tuple[float, float]]: + """``boot``, but with the joint level calibrated against *ci_func's* own + finite-sample behaviour rather than the nominal normal quantile -- see + :func:`_calibrated_joint_critical_value`. Same output contract as + :func:`_joint_bootstrap_scaled_simultaneous_cis`. + """ + from scipy import stats as _scipy_stats + + if not pairs: + return {} + c = _calibrated_joint_critical_value( + scores=scores, pairs=pairs, labels=labels, ci=ci, n_bootstrap=n_bootstrap, + rng=rng, ci_func=ci_func, statistic=statistic, + ) + if c is None: + return {} + alpha_eff = float(2.0 * (1.0 - _scipy_stats.norm.cdf(c))) + alpha_eff = min(max(alpha_eff, 1e-9), 1.0 - 1e-9) + return {pair: ci_func(results[pair].per_input_diffs, alpha_eff) for pair in pairs} + + def _joint_bootstrap_scaled_simultaneous_cis( scores: np.ndarray, results: dict[tuple[str, str], "PairedDiffResult"], @@ -1735,7 +2229,7 @@ def _joint_bootstrap_scaled_simultaneous_cis( k(k-1)/2 pairs per replicate, and uses the resulting ``(1-alpha)``-quantile critical value *c* in place of the marginal normal quantile ``z_{alpha/2}`` inside *ci_func* (most closed-form score - intervals -- e.g. ``tango_paired_ci_from_diffs`` -- derive ``z`` from + intervals -- e.g. ``mj_floor_paired_ci_from_diffs`` -- derive ``z`` from ``alpha`` internally, so translating *c* back to an equivalent ``alpha_eff = 2*(1 - Phi(c))`` and evaluating *ci_func* at that level is equivalent to substituting *c* for *z* directly). This keeps the @@ -1885,6 +2379,72 @@ def romano_wolf_stepdown_pvalues( } +def canonical_pairwise_ci_func(data_kind: str, diff_bounds, method: Optional[str] = None): + """The alpha-parameterized per-pair CI formula evalstats reports for + *data_kind*, as a ``(diffs, alpha) -> (lo, hi)`` callable. + + SINGLE SOURCE OF TRUTH for "which interval does a pairwise difference + get?". The simultaneous-CI constructions (Sidak, joint-bootstrap + scaling) must widen the SAME formula the non-simultaneous pairwise path + would otherwise show, or the two disagree about what a comparison's + interval even is. Simulation harnesses must call this rather than + re-listing the formulas, which is how they drift: cases/pvalues.py's + ``_canonical_ci_func`` had Likert on logit-t after Likert gained its own + NIG row, and binary on mj_floor after binary moved to Bonett-Price, so + the published simultaneous-CI numbers were measured on formulas the + library no longer used for those data kinds. + + Returns ``None`` for "unbounded", whose construction needs a degenerate + -sample fallback the caller supplies (there are no bounds to fall back + on -- see the router's else-branch). + """ + # PREFER the already-resolved pairwise method. The main path resolves + # method="auto" ONCE (router.analyze -> config.resolve_auto_analyze_methods) + # and hands the concrete name down; keying off it here means the + # simultaneous CI widens the very interval the pairwise row reports, + # instead of a second, independently-derived opinion about this data. + # data_kind is only the fallback, for resampling methods (bootstrap, bca, + # ...) that have no closed form to widen. + _bounded = None + if diff_bounds is not None: + _lo_b, _hi_b = diff_bounds + + def _bounded(fn): + def ci_func(diffs, alpha, _lo=_lo_b, _hi=_hi_b, _f=fn): + return rescaled_ci(_f, diffs, alpha, _lo, _hi) + return ci_func + + def _attach(f, provider): + try: + f.alpha_crit_batch = provider + except AttributeError: + pass + return f + + if method == "bonett_price": + return bonett_price_paired_ci_from_diffs + if method in ("mj_floor", "tango"): + return mj_floor_paired_ci_from_diffs + if method == "nig" and _bounded is not None: + return _attach(_bounded(functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0)), + functools.partial(_nig_alpha_crit_batch, b0=_NIG_PAIRED_DIFF_B0, + lo=_lo_b, hi=_hi_b)) + if method == "logit_t" and _bounded is not None: + return _attach(_bounded(logit_t_ci_1d), + functools.partial(_logit_t_alpha_crit_batch, lo=_lo_b, hi=_hi_b)) + + if data_kind == "binary": + return bonett_price_paired_ci_from_diffs + if data_kind in ("bounded_01", "likert") and _bounded is not None: + if data_kind == "likert": + return _attach(_bounded(functools.partial(nig_ci_1d, b0=_NIG_PAIRED_DIFF_B0)), + functools.partial(_nig_alpha_crit_batch, b0=_NIG_PAIRED_DIFF_B0, + lo=_lo_b, hi=_hi_b)) + return _attach(_bounded(logit_t_ci_1d), + functools.partial(_logit_t_alpha_crit_batch, lo=_lo_b, hi=_hi_b)) + return None + + def _simultaneous_cis_router( scores: np.ndarray, results: dict[tuple[str, str], "PairedDiffResult"], @@ -1898,6 +2458,7 @@ def _simultaneous_cis_router( *, prefer: str = "auto", score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> tuple[dict[tuple[str, str], tuple[float, float]], str, dict]: """Route simultaneous CI computation to the requested construction. @@ -1907,13 +2468,33 @@ def _simultaneous_cis_router( else joint bootstrap with an effective alpha (``"boot"``, :func:`_joint_bootstrap_scaled_simultaneous_cis`). Both widen whichever canonical closed-form pairwise CI formula the data resolves to -- Tango - for binary data, logit-t for a known-bounded numeric range (*score_range*), + for binary data, logit-t for any bounded numeric range (*score_range*), plain t-interval as the bounds-agnostic fallback for everything else -- the same per-data-kind formula :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE` - already uses for the *non*-simultaneous pairwise CI, so Sidak/boot always - widen the formula that would otherwise have been shown, regardless of - which resampling *method* (bootstrap, bca, ...) the point estimate - itself used. + already uses for the *non*-simultaneous pairwise CI on genuinely + continuous data, so Sidak/boot always widen the formula that would + otherwise have been shown, regardless of which resampling *method* + (bootstrap, bca, ...) the point estimate itself used. + + ``eval_type="likert"`` (or ``method="nig"`` passed directly) widens + NIG instead of logit-t for bounded numeric data here, matching + :func:`pairwise_differences`'s own ``method="nig"`` path and + :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE`'s "likert" row. + This used to be logit-t-only regardless of ``eval_type`` (the k>=3 + construction had only ever been tested with NIG's OLD, buggy prior, + before ``_NIG_PAIRED_DIFF_B0`` fixed it) -- a real compare_e2e + overnight sweep surfaced exactly the failure that scoping predicted: + family-wise coverage for likert data collapsing to 10-26% at n=15, + k=10 (vs. 93-99% for continuous at the same n, k), because Sidak's + shrinking per-pair alpha_adj as k grows drives logit-t straight into + the same paired-diff rounding-cancellation failure mode NIG was built + to fix in the first place -- this router just hadn't been made to use + the fix. See :data:`~evalstats.config.AUTO_ANALYZE_METHOD_TABLE`'s + "likert" row for the validation numbers (single-run and nested/multi- + run alike -- unlike the pairwise *method* table, this router doesn't + vary by seeded= at all, since Sidak/boot here widen whatever + ``results[pair].per_input_diffs`` already is, computed identically + for single- and multi-run data upstream). Historical note: this used to default unconditionally to Bonferroni (with the studentized bootstrap max-T method as the sole opt-in @@ -1946,6 +2527,43 @@ def _simultaneous_cis_router( *max_t_pvalues* maps each pair to its max-T p-value only when *method_used* is ``'max_t'``; empty dict otherwise. """ + # Resolve the data kind once, up front, rather than inside the Sidak/boot + # branch: every route can end at the Bonferroni fallback (max-T returning + # empty, a joint bootstrap that degenerates, prefer="bonferroni", or a + # single pair, which skips Sidak/boot by construction), and that fallback + # needs the diff bounds too -- they are what lets it produce a real + # interval instead of a zero-width one on a constant difference vector. + # See _degenerate_pair_ci. + # + # Same explicit-score_range-wins rule as the main router uses -- see + # resampling.binary_routing_applies. The warning is emitted there, at + # the routing decision, not repeated per pair. + is_binary = binary_routing_applies(scores, score_range) + if is_binary: + data_kind = "binary" + elif score_range is not None: + # eval_type="likert" (explicit or auto-resolved upstream via + # detect_quantization_step()) or an explicit method="nig" call + # both route to the NIG-widened branch below -- see this + # function's docstring for why (validated fix for logit-t's + # paired-diff rounding-cancellation failure mode, which gets + # worse, not better, as Sidak's alpha_adj shrinks with k). + data_kind = "likert" if (eval_type == "likert" or method == "nig") else "bounded_01" + else: + data_kind = "unbounded" + + # Support of a single paired difference: two scores in [lo, hi] differ by + # at most hi-lo in either direction, so the diff spans [-(hi-lo), hi-lo] + # -- the same widened span the logit_t/NIG paths rescale onto. Binary + # data is [0, 1] whether or not a score_range was passed. + if data_kind == "binary": + diff_bounds: Optional[tuple[float, float]] = (-1.0, 1.0) + elif score_range is not None: + _span = float(score_range[1]) - float(score_range[0]) + diff_bounds = (-_span, _span) + else: + diff_bounds = None + if prefer == "max_t" and method in _SIMULTANEOUS_CI_BOOTSTRAP_METHODS: cis, max_t_pvalues = _max_stat_simultaneous_cis( scores=scores, @@ -1960,7 +2578,7 @@ def _simultaneous_cis_router( if cis: return cis, "max_t", max_t_pvalues - elif prefer in ("auto", "sidak", "boot") and len(pairs) > 1: + elif prefer in ("auto", "sidak", "boot", "boot_cal") and len(pairs) > 1: # fig:fwer-decision-tree's Sidak/boot branch is explicitly scoped to # "Family of comparisons (k>=3)" -- with a single pair (k=2, one # comparison), there's no family to control FWER across, and @@ -1974,14 +2592,6 @@ def _simultaneous_cis_router( # outlier-contaminated median), so route k=1 through the plain # Bonferroni fallback below rather than attempting the k>=3-only # constructions at all. - is_binary = is_binary_scores(scores) - if is_binary: - data_kind = "binary" - elif score_range is not None: - data_kind = "bounded_01" - else: - data_kind = "unbounded" - resolved = prefer if prefer == "auto": n_items = scores.shape[1] @@ -1990,21 +2600,38 @@ def _simultaneous_cis_router( data_kind, n_items, lopsided_binary=lopsided, ) - if data_kind == "binary": - ci_func = tango_paired_ci_from_diffs - elif data_kind == "bounded_01": - diff_span = score_range[1] - score_range[0] - diff_lo, diff_hi = -diff_span, diff_span - - def ci_func(diffs, alpha, _lo=diff_lo, _hi=diff_hi): - return rescaled_ci(logit_t_ci_1d, diffs, alpha, _lo, _hi) - else: - ci_func = t_interval_ci_1d + ci_func = canonical_pairwise_ci_func(data_kind, diff_bounds, method) + if ci_func is None: + # Unbounded: t_interval_ci_1d still returns (mean, mean) on a + # constant difference vector -- the marginal contract that + # degenerate_sample_ci deliberately left alone, since with no + # bounds there is nothing for it to fall back on. Left as-is, + # that reintroduces exactly the zero-width interval this router's + # Bonferroni fallback now refuses to emit, on any family where + # sidak/boot succeed (k>=3 with at least one non-degenerate pair + # to carry the joint bootstrap). Route the degenerate case + # through the same _degenerate_pair_ci the fallback uses so both + # branches give one answer, without changing t_interval_ci_1d + # itself or any marginal path that depends on it. + def ci_func(diffs, alpha): + M = len(diffs) + if M < 2 or float(np.ptp(diffs)) == 0.0: + return _degenerate_pair_ci( + float(np.mean(diffs)) if M else 0.0, M, alpha, None, + ) + return t_interval_ci_1d(diffs, alpha) if resolved == "sidak": cis = _sidak_simultaneous_cis(results=results, pairs=pairs, ci=ci, ci_func=ci_func) if cis: return cis, "sidak", {} + elif resolved == "boot_cal": + cis = _calibrated_joint_simultaneous_cis( + scores=scores, results=results, pairs=pairs, labels=labels, + ci=ci, n_bootstrap=n_bootstrap, rng=rng, ci_func=ci_func, statistic=statistic, + ) + if cis: + return cis, "boot_cal", {} elif resolved == "boot": cis = _joint_bootstrap_scaled_simultaneous_cis( scores=scores, results=results, pairs=pairs, labels=labels, @@ -2013,15 +2640,19 @@ def ci_func(diffs, alpha, _lo=diff_lo, _hi=diff_hi): if cis: return cis, "boot", {} - # Fallback (and prefer="bonferroni"): Bonferroni t-intervals work for any method. - cis = _bonferroni_simultaneous_cis(results=results, pairs=pairs, ci=ci) + # Fallback (and prefer="bonferroni"): Bonferroni t-intervals work for any + # method. diff_bounds only affects its zero-variance branch, where it is + # the difference between a real interval and a zero-width one. + cis = _bonferroni_simultaneous_cis( + results=results, pairs=pairs, ci=ci, diff_bounds=diff_bounds, + ) return cis, "bonferroni", {} def all_pairwise( scores: np.ndarray, labels: list[str], - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "mj_floor", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, correction: Literal["auto", "holm", "bonferroni", "fdr_bh", "hochberg", "shaffer", "romano_wolf", "none"] = "auto", @@ -2033,6 +2664,7 @@ def all_pairwise( compute_wilcoxon: bool = True, score_range: Optional[tuple[float, float]] = None, prefer: str = "auto", + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> PairwiseMatrix: """Compute all pairwise comparisons with multiple comparisons correction. @@ -2091,6 +2723,16 @@ def all_pairwise( knob to force a specific simultaneous-CI construction instead of the ``"auto"`` (default) table lookup: ``"sidak"``, ``"boot"``, ``"max_t"``, or ``"bonferroni"``. + eval_type : "likert", "continuous", or None + ``"likert"`` (explicit, or resolved via ``method="auto"``'s own + quantization auto-detection) routes BOTH the per-pair CI (via + :func:`pairwise_differences`'s ``method="nig"`` path) AND the + ``simultaneous_ci=True`` (default) k>=3 Sidak/joint-bootstrap- + widened construction (:func:`_simultaneous_cis_router`) through + NIG instead of logit-t -- validated for single-run and nested/ + multi-run pairwise data alike (see + ``config.AUTO_ANALYZE_METHOD_TABLE``'s "likert" row). Continuous + (or unspecified) bounded numeric data keeps logit-t throughout. omnibus : bool When ``True``, run the Friedman omnibus test (with Nemenyi post-hoc) alongside the pairwise comparisons. Requires k ≥ 3. Defaults to @@ -2215,6 +2857,7 @@ def all_pairwise( statistic=statistic, score_range=score_range, prefer=prefer, + eval_type=eval_type, ) if sim_cis: applied_simultaneous_ci = True @@ -2273,7 +2916,7 @@ def vs_baseline( scores: np.ndarray, labels: list[str], baseline: str, - method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t"] = "auto", + method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto", "newcombe", "mj_floor", "tango", "bayes_binary", "permutation", "sign_test", "t_interval", "logit_t", "nig"] = "auto", ci: float = 0.95, n_bootstrap: int = 10_000, correction: Literal["holm", "bonferroni", "fdr_bh", "none"] = "fdr_bh", diff --git a/evalstats/core/pareto.py b/evalstats/core/pareto.py index 39f6d4e..8e0fd4d 100644 --- a/evalstats/core/pareto.py +++ b/evalstats/core/pareto.py @@ -1,6 +1,6 @@ """Uncertainty-aware Pareto-front analysis for a primary metric + secondary metric(s). -Backs ``compare(..., secondary=...)``. Unlike a naive Pareto front on point +Backs ``compare(..., secondary_metric=...)``. Unlike a naive Pareto front on point estimates -- which lets a marginally-lower cost or marginally-higher accuracy count as "dominates" even when the underlying data can't distinguish the two entities -- this module resamples both metrics jointly (same shared per-item @@ -65,6 +65,14 @@ class ParetoBootstrapResult: good, strictly better on at least one). Diagonal is always 0. n_bootstrap : int Number of bootstrap replicates the tallies above were computed from. + replicate_primary, replicate_secondary : np.ndarray, optional + Shape ``(N, n_bootstrap)``. Every replicate's per-entity (primary, + secondary) point value -- the raw joint draws the tallies above are + computed from. ``None`` unless ``pareto_bootstrap(..., + return_replicates=True)`` was requested; skipped by default since + no consumer of the calibrated dominance/CI path needs them, only + visualizations that want the joint (correlated) uncertainty shape + rather than independent per-metric marginal CIs. """ labels: list[str] @@ -73,6 +81,8 @@ class ParetoBootstrapResult: p_frontier: np.ndarray p_dominated_by: np.ndarray n_bootstrap: int + replicate_primary: Optional[np.ndarray] = None + replicate_secondary: Optional[np.ndarray] = None def orient_higher_is_better(scores: np.ndarray, direction: Literal["max", "min"]) -> np.ndarray: @@ -99,6 +109,7 @@ def pareto_bootstrap( *, statistic: Literal["mean", "median"] = "mean", batch_size: int = 256, + return_replicates: bool = False, ) -> ParetoBootstrapResult: """Joint bootstrap Pareto-dominance tallies for two per-item-aligned metrics. @@ -129,6 +140,12 @@ def pareto_bootstrap( ``(N, N, batch)`` pairwise-dominance array (mirrors the chunking pattern used elsewhere in this codebase, e.g. :func:`~evalstats.core.paired.romano_wolf_stepdown_pvalues`). + return_replicates : bool + When ``True``, also retain every replicate's per-entity (primary, + secondary) point value on the result (see + :attr:`ParetoBootstrapResult.replicate_primary`/ + ``replicate_secondary``) -- ``(N, n_bootstrap)`` each, so only worth + enabling for visualization, not the default calibrated path. Returns ------- @@ -153,6 +170,8 @@ def _stat(a: np.ndarray, axis: int) -> np.ndarray: frontier_count = np.zeros(N) dominated_count = np.zeros((N, N)) # [i, j] += 1 when j dominates i, this replicate eye = np.eye(N, dtype=bool) + replicate_primary = np.empty((N, n_bootstrap)) if return_replicates else None + replicate_secondary = np.empty((N, n_bootstrap)) if return_replicates else None start = 0 while start < n_bootstrap: @@ -165,6 +184,10 @@ def _stat(a: np.ndarray, axis: int) -> np.ndarray: bm1 = _stat(r1, axis=2) # (N, b) bm2 = _stat(r2, axis=2) # (N, b) + if return_replicates: + replicate_primary[:, start:stop] = bm1 + replicate_secondary[:, start:stop] = bm2 + m1_i, m1_j = bm1[:, None, :], bm1[None, :, :] # (N,1,b), (1,N,b) m2_i, m2_j = bm2[:, None, :], bm2[None, :, :] ge_both = (m1_j >= m1_i) & (m2_j >= m2_i) @@ -185,6 +208,111 @@ def _stat(a: np.ndarray, axis: int) -> np.ndarray: p_frontier=frontier_count / n_bootstrap, p_dominated_by=dominated_count / n_bootstrap, n_bootstrap=n_bootstrap, + replicate_primary=replicate_primary, + replicate_secondary=replicate_secondary, + ) + + +def pareto_bootstrap_unpaired( + groups_primary: list[np.ndarray], + groups_secondary: list[np.ndarray], + labels: list[str], + n_bootstrap: int, + rng: np.random.Generator, + *, + statistic: Literal["mean", "median"] = "mean", + return_replicates: bool = False, +) -> ParetoBootstrapResult: + """Joint bootstrap Pareto-dominance tallies for two metrics measured on + disjoint per-group items (between-subjects data) -- the counterpart to + :func:`pareto_bootstrap` for ``compare(design="unpaired", secondary_metric=...)``. + + :func:`pareto_bootstrap` draws one *shared* per-item resample index and + applies it to every entity, because every entity is scored on the same + items (paired by column position) -- that shared draw is what preserves + correlation between the two metrics for a given item across entities. + Between-subjects groups have no such shared item pool: each group's rows + are its own, independent items. The correlation that still needs + preserving is narrower but simpler -- item i's primary and secondary + values are the same row (the same reviewer/response), so each group's + row-index resample is drawn *independently*, not shared across groups. + Groups may also have different sizes (unbalanced), which the paired + function's fixed-``M``-for-all-entities design can't express at all. + + Parameters + ---------- + groups_primary, groups_secondary : list[np.ndarray] + One array per group/entity, row-aligned within each group (index i + of ``groups_primary[g]`` and ``groups_secondary[g]`` must be the + same underlying row) -- but groups may differ in length from each + other. Both already oriented so higher is better (see + :func:`orient_higher_is_better`). + labels : list[str] + Entity labels, length N (same order as the two group lists). + n_bootstrap, rng, statistic, return_replicates + Same meaning as :func:`pareto_bootstrap`. + + Returns + ------- + ParetoBootstrapResult + Identical shape/semantics to :func:`pareto_bootstrap`'s output, so + :func:`classify_pareto_status` (and any other consumer) works + unchanged regardless of which bootstrap produced it. + """ + N = len(labels) + if len(groups_primary) != N or len(groups_secondary) != N: + raise ValueError( + f"groups_primary/groups_secondary must have one entry per label " + f"(N={N}); got {len(groups_primary)} and {len(groups_secondary)}." + ) + for g_idx, (gp, gs) in enumerate(zip(groups_primary, groups_secondary)): + if len(gp) != len(gs): + raise ValueError( + f"group {labels[g_idx]!r}: primary/secondary length mismatch " + f"({len(gp)} vs {len(gs)}) -- both metrics must be row-aligned " + "within each group." + ) + if len(gp) == 0: + raise ValueError(f"group {labels[g_idx]!r} has no rows.") + + def _stat(a: np.ndarray, axis: int) -> np.ndarray: + return a.mean(axis=axis) if statistic == "mean" else np.median(a, axis=axis) + + point_primary = np.array([_stat(g, axis=0) for g in groups_primary]) + point_secondary = np.array([_stat(g, axis=0) for g in groups_secondary]) + + # Each group's resample is independent and small (m rows x n_bootstrap + # draws), unlike the paired function's shared (N, N, batch) x M dominance + # array -- N is typically a handful of groups, so no batching needed here. + bm1 = np.empty((N, n_bootstrap)) + bm2 = np.empty((N, n_bootstrap)) + for g_idx, (gp, gs) in enumerate(zip(groups_primary, groups_secondary)): + m = len(gp) + idx = rng.integers(0, m, size=(n_bootstrap, m)) + bm1[g_idx] = _stat(gp[idx], axis=1) + bm2[g_idx] = _stat(gs[idx], axis=1) + + eye = np.eye(N, dtype=bool) + m1_i, m1_j = bm1[:, None, :], bm1[None, :, :] + m2_i, m2_j = bm2[:, None, :], bm2[None, :, :] + ge_both = (m1_j >= m1_i) & (m2_j >= m2_i) + gt_either = (m1_j > m1_i) | (m2_j > m2_i) + j_dominates_i = ge_both & gt_either + j_dominates_i &= ~eye[:, :, None] + + dominated_count = j_dominates_i.sum(axis=2) + i_is_dominated = j_dominates_i.any(axis=1) + frontier_count = (~i_is_dominated).sum(axis=1) + + return ParetoBootstrapResult( + labels=list(labels), + point_primary=point_primary, + point_secondary=point_secondary, + p_frontier=frontier_count / n_bootstrap, + p_dominated_by=dominated_count / n_bootstrap, + n_bootstrap=n_bootstrap, + replicate_primary=bm1 if return_replicates else None, + replicate_secondary=bm2 if return_replicates else None, ) diff --git a/evalstats/core/ranking.py b/evalstats/core/ranking.py index 8620b0a..6f26661 100644 --- a/evalstats/core/ranking.py +++ b/evalstats/core/ranking.py @@ -76,6 +76,71 @@ class RankDistribution: n_bootstrap: int +class LazyRankDistribution(RankDistribution): + """A ``RankDistribution`` whose bootstrap is deferred until the rank + arrays are actually read. + + ``labels`` and ``n_bootstrap`` answer immediately -- a lot of code + (``core/summary.py`` especially) reads ``rank_dist.labels`` purely as the + canonical label list and must not pay for a rank bootstrap to get it. + Reading ``rank_probs``/``expected_ranks``/``p_best`` runs the bootstrap + once and caches it, so P(Best)/E[Rank] cost is paid only by callers that + actually want those numbers -- matching the opt-in story + ``ResultReport._show_rank_probabilities`` already tells for the *output*. + + The generator state is snapshotted at construction rather than the live + ``rng`` being held, so a deferred bootstrap draws exactly what an eager + one would have drawn. Note the parent ``rng`` is NOT advanced when the + ranks go uncomputed, so downstream draws differ from the old + always-compute behaviour; the rank distribution itself is unchanged. + """ + + def __init__(self, labels, n_bootstrap, compute, rng=None): + self.labels = list(labels) + self.n_bootstrap = n_bootstrap + self._compute = compute + self._resolved: Optional[RankDistribution] = None + self._state = None + if rng is not None: + try: + self._state = (type(rng.bit_generator), rng.bit_generator.state) + except Exception: + self._state = None + + def _resolve(self) -> RankDistribution: + if self._resolved is None: + rng = None + if self._state is not None: + bg_type, state = self._state + rng = np.random.Generator(bg_type()) + rng.bit_generator.state = state + self._resolved = self._compute(rng) + return self._resolved + + @property + def computed(self) -> bool: + """True once the bootstrap has actually run.""" + return self._resolved is not None + + @property + def rank_probs(self) -> np.ndarray: + return self._resolve().rank_probs + + @property + def expected_ranks(self) -> np.ndarray: + return self._resolve().expected_ranks + + @property + def p_best(self) -> np.ndarray: + return self._resolve().p_best + + def __repr__(self) -> str: + if self._resolved is None: + return (f"LazyRankDistribution(labels={self.labels!r}, " + f"n_bootstrap={self.n_bootstrap}, computed=False)") + return repr(self._resolved) + + def bootstrap_ranks( scores: np.ndarray, labels: list[str], diff --git a/evalstats/core/resampling.py b/evalstats/core/resampling.py index 550f6bc..1f33d2f 100644 --- a/evalstats/core/resampling.py +++ b/evalstats/core/resampling.py @@ -136,6 +136,126 @@ def is_binary_scores(scores: np.ndarray) -> bool: return bool(np.all((finite == 0.0) | (finite == 1.0))) +def binary_routing_applies( + scores: np.ndarray, + score_range: Optional[tuple[float, float]] = None, + *, + stacklevel: int = 3, +) -> bool: + """Whether all-{0, 1} *scores* should route to the binary CI methods. + + :func:`is_binary_scores` answers a question about the *sample*: are these + values all 0 or 1? Routing needs the question about the *population*: is + this metric Bernoulli? Those come apart whenever the caller has declared a + ``score_range`` wider than [0, 1] -- a 1-5 Likert scale where every + response landed on the floor, or a 0-100 grade where this particular + sample happens to contain only 0s and 1s. Treating those as Bernoulli + repeats the mistake :func:`degenerate_sample_ci` exists to avoid: reading + the sample's observed support as the population's support, when the + caller has explicitly said the metric ranges wider. The consequences are + concrete -- an all-floor 1-5 Likert sample used to get Wilson's + ``[0.886, 1.0]``, whose lower bound sits *below* the scale's minimum and + which opens downward from data that can only go up. + + So an explicitly passed ``score_range`` wins: it is a direct statement + about the metric, while binary detection is an inference from the values + that happened to be sampled. A ``score_range`` of exactly (0, 1) agrees + with the detection and changes nothing, and passing nothing at all leaves + auto-detection fully in charge -- the overwhelmingly common case, which + behaves exactly as before. + + Parameters + ---------- + scores : np.ndarray + Any-shape score array. + score_range : (float, float), optional + The metric's declared bounds, or None if the caller didn't say. + stacklevel : int + Passed through to ``warnings.warn`` so the override is reported at + the user's own call site. + + Returns + ------- + bool + True to use the binary methods (Wilson/Newcombe/mj_floor), False to fall + through to the bounds-aware continuous/Likert routing. + """ + if not is_binary_scores(scores): + return False + if score_range is None: + return True + lo, hi = float(score_range[0]), float(score_range[1]) + if lo == 0.0 and hi == 1.0: + return True + warnings.warn( + f"All scores are 0 or 1, which would normally auto-detect as binary " + f"data, but score_range={score_range} was given explicitly -- so this " + "is being treated as bounded numeric data on that scale, not as a " + "Bernoulli metric. The explicit range wins because a sample " + "containing only 0s and 1s doesn't establish that the metric can't " + "take other values (e.g. every response landing on a Likert scale's " + "floor, or a 0-100 grade where this sample happened to score only 0 " + "or 1); the binary methods would treat that unseen headroom as " + "impossible. Drop score_range (or pass score_range=(0, 1)) if the " + "metric really is binary.", + UserWarning, + stacklevel=stacklevel, + ) + return False + + +def detect_quantization_step(scores: np.ndarray) -> Optional[float]: + """Detect whether *scores* sit on a consistent quantization grid (e.g. + integer-valued Likert responses, or a percentage grade rounded to whole + points), returning the grid step -- or ``None`` if no consistent grid is + found (the data looks genuinely continuous). + + Used to auto-detect discrete/ordinal bounded data so :func:`analyze` can + route to NIG (calibrated for this case) instead of logit-t. Takes the + SMALLEST observed gap between distinct values as a candidate step, then + verifies every other gap is (within tolerance) an integer multiple of + it -- a GCD-style check, not a "does the most common gap recur >= N + times" frequency threshold, which is blind exactly where this matters + most: a small, peaked/boundary-heavy sample can collapse to just 2-3 + distinct values, too few for any gap to recur several times even when + the grid (e.g. step=1) is completely unambiguous. + + False-positive risk on genuinely continuous data is close to zero: + demanding EVERY gap (not just the most common one) independently land + within tolerance of an integer multiple of the candidate step has + vanishing probability by chance (verified empirically down to n=6 + pooled values, 0% false-positive rate up to n=1000). Ported from + simulations/harness/cases/ci_paired.py's ``_detect_dither_halfwidth``, + which found the same regression this guards against: a frequency-based + predecessor of this check went blind on small, peaked Likert samples. + + Parameters + ---------- + scores : np.ndarray + Any-shape score array (raw values, not yet rescaled). + + Returns + ------- + float or None + The detected step, or ``None`` if the data doesn't look quantized. + """ + flat = scores.ravel() + finite = flat[np.isfinite(flat)] + uniq = np.unique(finite) + if uniq.size < 2: + return None + gaps = np.diff(uniq) + gaps = gaps[gaps > 1e-9] + if gaps.size == 0: + return None + step = float(np.min(gaps)) + ratios = gaps / step + residuals = np.abs(ratios - np.round(ratios)) + if np.max(residuals) > 0.05: + return None + return step + + def is_lopsided_binary(scores: np.ndarray, threshold: int = 5) -> bool: """Return True if any compared group has fewer than *threshold* observed instances of its rarer binary outcome (e.g. only 2 ones out of 40). @@ -373,12 +493,12 @@ def clopper_pearson_ci(successes: int, n: int, alpha: float) -> tuple[float, flo return (lo, hi) -def tango_paired_ci_flat( +def mj_floor_paired_ci_flat( values_a: np.ndarray, values_b: np.ndarray, alpha: float, ) -> tuple[float, float]: - """Tango score CI treating multi-run data as a single-run flat baseline. + """Score CI treating multi-run data as a single-run flat baseline. When ``values_a`` / ``values_b`` are 2-D arrays of shape ``(N, R)``, only the **first run** (column 0) is used. This is the honest @@ -388,7 +508,7 @@ def tango_paired_ci_flat( this function avoids that by keeping the input as the unit of analysis. If 1-D arrays are passed the call is forwarded directly to - :func:`tango_paired_ci` unchanged. + :func:`mj_floor_paired_ci` unchanged. Parameters ---------- @@ -407,27 +527,40 @@ def tango_paired_ci_flat( a = a[:, 0] if b.ndim == 2: b = b[:, 0] - return tango_paired_ci(a, b, alpha) + return mj_floor_paired_ci(a, b, alpha) -def tango_paired_ci_mean( +def mj_floor_paired_ci_mean( values_a: np.ndarray, values_b: np.ndarray, alpha: float, ) -> tuple[float, float]: - """Heuristic Tango CI using per-item run means for multi-run inputs. + """DO NOT USE for multi-run coverage. Kept for reference only. + + Thresholding the run mean at 0.5 changes the estimand: it targets + E[1{mean_a >= 0.5}] - E[1{mean_b >= 0.5}], a majority-vote difference, + not the run-and-item-averaged difference E[a] - E[b] the other paired + methods estimate. Measured consequence on multi-run data: mean coverage + .843, MinCov 0.000, 1550 of 4536 cells below .90, and it gets WORSE with + more runs (.877 at R=2 down to .803 at R=20). The same pathology holds + for the Newcombe and Bonett-Price analogues, so it is the reduction that + is broken, not the interval. Use + :func:`bonett_price_paired_ci_multirun_cluster` for multi-run data. + + Original description follows. +Heuristic score CI using per-item run means for multi-run inputs. When ``values_a`` / ``values_b`` are 2-D arrays of shape ``(N, R)``, each item is first reduced to its run mean (shape ``(N,)``), then - :func:`tango_paired_ci` is applied. + :func:`mj_floor_paired_ci` is applied. - This is intentionally a pragmatic variant, not a strict Tango score - interval derivation: :func:`tango_paired_ci` was derived for paired + This is intentionally a pragmatic variant, not a strict score + interval derivation: :func:`mj_floor_paired_ci` was derived for paired Bernoulli observations, while run means live in ``[0, 1]`` and are - thresholded at 0.5 inside :func:`tango_paired_ci`. + thresholded at 0.5 inside :func:`mj_floor_paired_ci`. If 1-D arrays are passed the call is forwarded directly to - :func:`tango_paired_ci` unchanged. + :func:`mj_floor_paired_ci` unchanged. Parameters ---------- @@ -446,7 +579,73 @@ def tango_paired_ci_mean( a = np.mean(a, axis=1) if b.ndim == 2: b = np.mean(b, axis=1) - return tango_paired_ci(a, b, alpha) + return mj_floor_paired_ci(a, b, alpha) + + +def bonett_price_paired_ci_flat( + values_a: np.ndarray, + values_b: np.ndarray, + alpha: float = 0.05, +) -> tuple[float, float]: + """Bonett-Price CI treating multi-run data as a single-run flat baseline. + + The Bonett-Price counterpart of :func:`mj_floor_paired_ci_flat`, and the + honest single-run reference the multi-run variants have to beat: when + ``values_a`` / ``values_b`` are 2-D ``(N, R)`` arrays only the **first + run** (column 0) is used, i.e. exactly the data you would have had if + each input were run once. Flattening all ``N*R`` observations into one + long vector of "independent" pairs instead would inflate ``n`` to ``N*R`` + while the real information stays at the item scale, and under-cover badly. + + If 1-D arrays are passed the call is forwarded to + :func:`bonett_price_paired_ci` unchanged. + """ + a = np.asarray(values_a) + b = np.asarray(values_b) + if a.ndim == 2: + a = a[:, 0] + if b.ndim == 2: + b = b[:, 0] + return bonett_price_paired_ci(a, b, alpha) + + +def bonett_price_paired_ci_mean( + values_a: np.ndarray, + values_b: np.ndarray, + alpha: float = 0.05, +) -> tuple[float, float]: + """Heuristic Bonett-Price CI using per-item run means for multi-run inputs. + + The Bonett-Price counterpart of :func:`mj_floor_paired_ci_mean`: each item + is reduced to its run mean (shape ``(N,)``) and then thresholded at 0.5 + inside :func:`bonett_price_paired_ci`, i.e. every item is scored by + majority vote across its runs. + + Deliberately a pragmatic baseline, not a derivation. Two things are wrong + with it and both are worth stating, because they are the reason the + ``multirun`` variants below exist: + + 1. It changes the ESTIMAND. Thresholding the run means estimates + ``p(majority-vote A = 1) - p(majority-vote B = 1)``, not the + run-and-item-averaged ``p(A=1) - p(B=1)`` that the multi-run variants + (and the harness's ``true_diff``) target. Majority voting is a + different, less noisy system than the one being evaluated, so the two + estimands only coincide when the per-item run distributions are + symmetric about the threshold. + 2. It discards the within-item run spread entirely, so a knife-edge item + (half its runs 1, half 0) is recorded with the same confidence as a + deterministic one. + + If 1-D arrays are passed the call is forwarded to + :func:`bonett_price_paired_ci` unchanged. + """ + a = np.asarray(values_a) + b = np.asarray(values_b) + if a.ndim == 2: + a = np.mean(a, axis=1) + if b.ndim == 2: + b = np.mean(b, axis=1) + return bonett_price_paired_ci(a, b, alpha) def clopper_pearson_ci_1d(values: np.ndarray, alpha: float) -> tuple[float, float]: @@ -573,6 +772,11 @@ def beta_ci_1d( vals = np.asarray(values, dtype=float) x_bar = float(np.mean(vals)) s2 = float(np.var(vals, ddof=1)) + if n > 1 and float(np.ptp(vals)) == 0.0: + # Constant sample: the MOM fit is undefined and the t-interval this + # used to fall back to is itself zero-width here. Same binomial + # worst-case bound logit_t_ci_1d uses -- see degenerate_sample_ci. + return degenerate_sample_ci(float(vals[0]), n, alpha) if s2 <= 0.0 or not np.isfinite(s2) or x_bar <= 0.0 or x_bar >= 1.0: return t_interval_ci_1d(vals, alpha) # Method-of-moments: concentration κ = a+b from mean and variance @@ -596,6 +800,76 @@ def beta_ci_1d( rather than genuinely bad data -- see logit_t_ci_1d's docstring.""" +def degenerate_sample_ci( + value: float, n: int, alpha: float, lo: float = 0.0, hi: float = 1.0, +) -> tuple[float, float]: + """Conservative CI for E[X] when all *n* observed values are identical. + + A zero-variance sample carries no information about spread, so every + variance-driven interval (the delta method, the t-interval, any + resampling scheme) degenerates to zero width and covers the truth with + probability 0 whenever the population isn't genuinely a point mass. That + isn't a rounding artifact -- it's the honest answer to the wrong + question. The right question is what the sample *does* pin down. + + Treat "X == value" as a Bernoulli success. Observing n successes out of n + gives the exact (Clopper-Pearson) lower confidence bound + + p = P(X = value) >= (alpha/2) ** (1/n) + + and the remaining 1-p of the mass is unconstrained within the metric's + known bounds [lo, hi]. So + + E[X] in [p*value + (1-p)*lo, p*value + (1-p)*hi] + + covers E[X] for *any* configuration of that unseen mass whenever the + bound on p holds: the interval's endpoints are attained exactly at the + worst cases (all remaining mass at lo, or all at hi), and both endpoints + move monotonically in p (the gap between the truth and the endpoint is + (p - p_lo)*(value - lo) >= 0 at the bottom and (p - p_lo)*(value - hi) + <= 0 at the top). And when the bound fails -- true p < p_lo -- a + degenerate sample only arises with probability p**n < alpha/2 in the + first place, so the branch contributes at most alpha/2 to the overall + miss rate. + + For all-successes binary data (value == hi == 1, lo == 0) this reduces to + exactly the two-sided Clopper-Pearson interval [(alpha/2)**(1/n), 1], + which is the answer the binary methods already give -- so the bounded + continuous path and the binary path agree at the boundary instead of + disagreeing by the full width of the interval. + + The price is conservatism: width is (1-p)*(hi-lo), roughly + ln(2/alpha)*(hi-lo)/n -- about 0.15 at n=25 on a [0,1] metric, shrinking + like 1/n. That is the correct price for a sample that shows no spread at + all, and it is only ever paid on samples that would otherwise have been + reported with false certainty. + + Parameters + ---------- + value : float + The single value every observation took, assumed within [lo, hi]. + n : int + Number of observations (>= 1). + alpha : float + Significance level (1 - confidence level). + lo, hi : float + The metric's known bounds. Callers working on a rescaled [0, 1] axis + (see ``stats_utils.rescaled_ci``) should leave these at the default + and let the wrapper map the result back. + + Returns + ------- + (ci_low, ci_high) : tuple[float, float] + Interval clamped to [lo, hi]. + """ + if n < 1: + return (lo, hi) + p = float(alpha / 2.0) ** (1.0 / n) + ci_low = p * value + (1.0 - p) * lo + ci_high = p * value + (1.0 - p) * hi + return (max(lo, ci_low), min(hi, ci_high)) + + def logit_t_ci_1d(values: np.ndarray, alpha: float, order: int = 1) -> tuple[float, float]: """Logit-transform t-interval (delta method) for [0, 1]-bounded data. @@ -673,6 +947,22 @@ def logit_t_ci_1d(values: np.ndarray, alpha: float, order: int = 1) -> tuple[flo in either -- the noisy third-moment estimate at small n cancels out any theoretical gain -- so it isn't offered as an option. + A **zero-variance sample** (every value identical, which includes the + all-0s and all-1s boundary cases) is handed to + :func:`degenerate_sample_ci` rather than being reported as the zero-width + interval the delta method implies. This matters on saturated metrics: on + a one-inflated DGP at 95% inflation, 36% of n=20 samples come out + constant, and before this fallback existed those reps dragged marginal + coverage from a nominal 95% down to 60% -- while coverage *conditional* + on a non-degenerate sample stayed at 94.6%. The transform was never the + problem; the zero-width branch was, and it is shared with + ``t_interval``/``beta``/the bootstrap methods rather than being specific + to logit-t. See ``simulations/harness/scenarios/synthetic.py``'s + ``cont-{zero,one}-inflated-extreme`` shapes, added so the suite actually + reaches the regime where this fires (the pre-existing 70%-inflation + shapes produce a constant sample <3% of the time even at n=10, which is + why routine sweeps gave a clean bill of health here). + Parameters ---------- values : np.ndarray @@ -707,7 +997,15 @@ def logit_t_ci_1d(values: np.ndarray, alpha: float, order: int = 1) -> tuple[flo vals = np.clip(vals, 0.0, 1.0) x_bar = float(np.mean(vals)) se = float(np.std(vals, ddof=1)) / np.sqrt(n) + if float(np.ptp(vals)) == 0.0: + # Zero-variance sample (all values identical -- including the all-0s + # and all-1s boundary cases). The delta method has nothing to + # propagate here and would report a zero-width interval; hand off to + # the binomial worst-case bound instead. See degenerate_sample_ci. + return degenerate_sample_ci(float(vals[0]), n, alpha) if se <= 0.0 or not np.isfinite(se) or x_bar <= 0.0 or x_bar >= 1.0: + # Only reachable now for non-finite input (NaN/inf), since within + # [0, 1] both x_bar == 0 and x_bar == 1 imply a constant sample. return (x_bar, x_bar) # Delta method: SE of logit(x̄) ≈ SE(x̄) / (x̄(1−x̄)) logit_mean = float(np.log(x_bar / (1.0 - x_bar))) @@ -1539,79 +1837,393 @@ def wilson_nested_bb( return _wilson_neff(p_hat, n_eff, alpha) -def newcombe_paired_ci( - values_a: np.ndarray, - values_b: np.ndarray, - alpha: float, +def _paired_binary_cells(values_a, values_b, fname: str) -> tuple[int, int, int, int]: + """Return the 2x2 paired-binary cell counts (n11, n10, n01, n00). + + Shared input handling for the paired binary interval methods. Values are + thresholded at 0.5 (accommodates float representations). + """ + values_a = np.asarray(values_a) + values_b = np.asarray(values_b) + if values_a.ndim != 1 or values_b.ndim != 1: + raise ValueError(f"{fname} expects 1-D input arrays.") + if values_a.shape != values_b.shape: + raise ValueError(f"{fname} expects arrays with equal shape.") + a_bin = (values_a >= 0.5).astype(int) + b_bin = (values_b >= 0.5).astype(int) + return ( + int(np.sum((a_bin == 1) & (b_bin == 1))), + int(np.sum((a_bin == 1) & (b_bin == 0))), + int(np.sum((a_bin == 0) & (b_bin == 1))), + int(np.sum((a_bin == 0) & (b_bin == 0))), + ) + + +def bonett_price_paired_ci( + values_a: np.ndarray, values_b: np.ndarray, alpha: float = 0.05, ) -> tuple[float, float]: - """Newcombe score CI for the paired binary difference p(A=1) − p(B=1). + """Bonett-Price Laplace-adjusted Wald CI for the paired binary difference. - Uses the discordant-pairs formulation (Newcombe 1998, *Stat Med*). - Let n10 = number of inputs where A=1, B=0, and n01 = A=0, B=1. - A Wilson score interval is computed for theta = n10 / (n10 + n01) - (proportion of discordant pairs where A wins), then transformed to - the difference scale:: + Fagerland, Lydersen & Laake (2014) eq. (16) -- their *prime* recommendation + for a CI on the difference between paired proportions, on the grounds that + it is conservative, performs very well, and is trivial to compute. - d_low = (m / n) * (2 * theta_low − 1) - d_high = (m / n) * (2 * theta_high − 1) + Applies a Laplace (add-one) adjustment to the discordant cells before + forming a Wald interval:: - where m = n10 + n01 is the number of discordant pairs and n is the - total number of paired inputs. + p12 = (n10 + 1) / (n + 2), p21 = (n01 + 1) / (n + 2) + (p12 - p21) +/- z * sqrt[ (p12 + p21 - (p12 - p21)^2) / (n + 2) ] - Returns (0.0, 0.0) when m == 0 (no discordant pairs, perfect agreement). + Unlike the plain Wald interval it never produces a zero-width interval, + since the add-one adjustment keeps the variance term strictly positive. + Limits are truncated to [-1, 1]. - Parameters - ---------- - values_a, values_b : np.ndarray - 1-D arrays of equal length. Values are thresholded at 0.5 to - determine binary membership (accommodates float representations). - alpha : float - Significance level (1 − confidence level). + Reproduces Fagerland et al.'s Table V to the three decimals published. Returns ------- (ci_low, ci_high) : tuple[float, float] - CI on p(A=1) − p(B=1). - - Raises - ------ - ValueError - If inputs are not 1-D arrays of equal length. + CI on p(A=1) - p(B=1). """ - values_a = np.asarray(values_a) - values_b = np.asarray(values_b) - if values_a.ndim != 1 or values_b.ndim != 1: - raise ValueError("newcombe_paired_ci expects 1-D input arrays.") - if values_a.shape != values_b.shape: - raise ValueError("newcombe_paired_ci expects arrays with equal shape.") - - n = len(values_a) + _, n10, n01, _ = _paired_binary_cells(values_a, values_b, "bonett_price_paired_ci") + n = len(np.asarray(values_a)) if n <= 0: return (0.0, 0.0) - a_bin = (values_a >= 0.5).astype(int) - b_bin = (values_b >= 0.5).astype(int) - n10 = int(np.sum((a_bin == 1) & (b_bin == 0))) - n01 = int(np.sum((a_bin == 0) & (b_bin == 1))) - m = n10 + n01 - if m == 0: + z = float(stats.norm.ppf(1.0 - alpha / 2.0)) + p12 = (n10 + 1.0) / (n + 2.0) + p21 = (n01 + 1.0) / (n + 2.0) + diff = p12 - p21 + se = float(np.sqrt(max(p12 + p21 - diff * diff, 0.0) / (n + 2.0))) + return ( + float(np.clip(diff - z * se, -1.0, 1.0)), + float(np.clip(diff + z * se, -1.0, 1.0)), + ) + + + +def newcombe_mover_paired_ci( + values_a: np.ndarray, values_b: np.ndarray, alpha: float = 0.05, +) -> tuple[float, float]: + """Newcombe square-and-add (MOVER Wilson score) CI for the paired difference. + + Newcombe (1998) method 10, as presented in Fagerland, Lydersen & Laake + (2014) eqs. (19)-(22) -- one of their three recommended intervals. + + This is the "square-and-add"/MOVER construction: separate Wilson score + intervals are computed for the two *marginal* proportions p(A=1) and + p(B=1), then combined with a correlation correction:: + + L = d - sqrt[ (pA - l1)^2 + (u2 - pB)^2 - 2*phi*(pA - l1)*(u2 - pB) ] + U = d + sqrt[ (pB - l2)^2 + (u1 - pA)^2 - 2*phi*(pB - l2)*(u1 - pA) ] + + where phi is estimated from A = n11*n00 - n10*n01 as (A - n/2)/sqrt(...) + if A > n/2, 0 if 0 <= A <= n/2, and A/sqrt(...) if A < 0; phi is set to 0 + when any marginal sum is zero. + + This is the only Newcombe interval in evalstats. An earlier + discordant-pairs formulation (a Wilson interval on n10/(n10+n01) + rescaled to the difference scale) was removed on 2026-08-24: it is a + different, poorly-covering method that is NOT the one Fagerland et al. + recommend under the name "Newcombe". + + Reproduces Fagerland et al.'s Table V to the three decimals published. + + Returns + ------- + (ci_low, ci_high) : tuple[float, float] + CI on p(A=1) - p(B=1). + """ + n11, n10, n01, n00 = _paired_binary_cells( + values_a, values_b, "newcombe_mover_paired_ci" + ) + n = n11 + n10 + n01 + n00 + if n <= 0: return (0.0, 0.0) - theta_low, theta_high = wilson_ci(n10, m, alpha) - scale = m / n + n_a = n11 + n10 # successes for A (row margin) + n_b = n11 + n01 # successes for B (column margin) + l1, u1 = wilson_ci(n_a, n, alpha) + l2, u2 = wilson_ci(n_b, n, alpha) + p_a = n_a / n + p_b = n_b / n + + margins = (n_a, n - n_a, n_b, n - n_b) + if any(mg == 0 for mg in margins): + phi = 0.0 + else: + det = n11 * n00 - n10 * n01 + denom = float(np.sqrt(float(n_a) * (n - n_a) * n_b * (n - n_b))) + if det > n / 2.0: + phi = (det - n / 2.0) / denom + elif det < 0.0: + phi = det / denom + else: + phi = 0.0 + + d = p_a - p_b + lo_term = (p_a - l1) ** 2 + (u2 - p_b) ** 2 - 2.0 * phi * (p_a - l1) * (u2 - p_b) + hi_term = (p_b - l2) ** 2 + (u1 - p_a) ** 2 - 2.0 * phi * (p_b - l2) * (u1 - p_a) return ( - float(scale * (2.0 * theta_low - 1.0)), - float(scale * (2.0 * theta_high - 1.0)), + float(np.clip(d - np.sqrt(max(lo_term, 0.0)), -1.0, 1.0)), + float(np.clip(d + np.sqrt(max(hi_term, 0.0)), -1.0, 1.0)), + ) + + +def _clustered_paired_cells(values_a, values_b, fname): + """Per-item 2x2 cell counts (a_k, b_k, c_k, d_k) for (n_items, n_runs) data. + + Maps the clustered matched-pair layout of Yang, Sun & Hardin (2012) onto an + eval sweep: the ITEM is the cluster and each RUN is a unit within it, so + cluster sizes are equal (n_k = R for every k). That equality matters -- + Eliasziw & Donner's n_c and Yang's differ for unequal clusters but both + collapse to exactly R here, so the estimator is unambiguous for our design. + """ + va = np.asarray(values_a) + vb = np.asarray(values_b) + if va.shape != vb.shape: + raise ValueError(f"{fname} expects arrays with equal shape (n_items, n_runs).") + if va.ndim != 2: + raise ValueError(f"{fname} expects 2-D arrays (n_items, n_runs).") + a_bin = (va >= 0.5).astype(int) + b_bin = (vb >= 0.5).astype(int) + a_k = np.sum((a_bin == 1) & (b_bin == 1), axis=1).astype(float) + b_k = np.sum((a_bin == 1) & (b_bin == 0), axis=1).astype(float) + c_k = np.sum((a_bin == 0) & (b_bin == 1), axis=1).astype(float) + d_k = np.sum((a_bin == 0) & (b_bin == 0), axis=1).astype(float) + return a_k, b_k, c_k, d_k + + +def _eliasziw_inflation_factor(a_k, b_k, c_k, d_k): + """Variance inflation factor 1 + (n_c - 1) * rho_hat. + + Eliasziw & Donner (1991), as presented in Yang, Sun & Hardin (2012) sec 2.1: + rho_tilde comes from an ANOVA decomposition into between- and within-cluster + mean squares, then rho_hat rescales it using the discordant probabilities. + With equal cluster sizes n_c = R exactly. + + Returns 1.0 (no inflation) for most degenerate cases, following the paper's + Remark 1: if rho falls outside [-1, 1] or cannot be computed because only + one type of discordant pair is present, the factor is set to 1. + + ONE DELIBERATE DEVIATION from Remark 1. When the ANOVA denominator is + exactly 0 -- both mean squares vanish, so rho is literally 0/0 and the data + carry NO information about within-cluster correlation -- this returns + ``n_c`` (equivalently rho_hat = 1, full clustering) rather than 1. + + Remark 1's fallback of 1 asserts INDEPENDENCE of all ``N = K * n_c`` + units, which manufactures precision from absent data. That is harmless at + the cluster sizes Yang et al. study (their example averages 2.4 units per + cluster) but severe at the cluster sizes multi-run evals produce: on an + all-concordant table with K = 15 items and n_c = 20 runs, the fallback of 1 + gives a 95% interval of width 0.025 -- claiming +/-1.3% precision from data + containing no disagreements at all -- and the width shrinks further as runs + are added. Returning ``n_c`` instead makes the interval reduce to the + single-run score on ``K`` items, which is the correct answer when nothing + discordant was observed, and makes its width invariant to the number of + runs (verified: 0.40777 at n_c = 1, 3 and 20). + + This fires ONLY on the exact 0/0 branch; wherever rho is estimable the + factor is bit-identical to Remark 1's. Measured effect on a 54-cell sweep: + MinCov .6976 -> .9160 and cells below .93 coverage 6 -> 1, with cells where + rho is estimable unchanged to the digit. + """ + n_k = a_k + b_k + c_k + d_k + K = len(n_k) + N = float(n_k.sum()) + if K < 2 or N <= 0: + return 1.0 + n_bar = N / K + n_c = float((n_k ** 2).sum() / N) + p = np.array([a_k.sum(), b_k.sum(), c_k.sum(), d_k.sum()], dtype=float) / N + cells = np.stack([a_k, b_k, c_k, d_k], axis=1) + expect = np.outer(n_k, p) + with np.errstate(divide="ignore", invalid="ignore"): + bms = float((((cells - expect) ** 2).sum(axis=1) / n_k).sum() / K) + wms_num = (cells * (n_k[:, None] - cells)).sum(axis=1) / n_k + if n_bar <= 1.0: + return 1.0 + wms = float(wms_num.sum() / (K * (n_bar - 1.0))) + n_0 = n_bar - float(((n_k - n_bar) ** 2).sum()) / (K * (K - 1) * n_bar) + denom = bms + (n_0 - 1.0) * wms + if not np.isfinite(denom) or denom == 0.0: + # rho is 0/0 -- no information about clustering. Assume full + # clustering rather than independence; see the docstring's deviation + # note. rho_hat = 1 gives factor = 1 + (n_c - 1) * 1 = n_c. + return float(n_c) if np.isfinite(n_c) and n_c > 0 else 1.0 + rho_tilde = (bms - wms) / denom + if not np.isfinite(rho_tilde) or rho_tilde <= 0.0: + return 1.0 + q = (1.0 - rho_tilde) / rho_tilde + rho_hat = 1.0 / (1.0 + p[1] * q + p[2] * q) + if not np.isfinite(rho_hat) or not (-1.0 <= rho_hat <= 1.0): + return 1.0 + factor = 1.0 + (n_c - 1.0) * rho_hat + return float(factor) if np.isfinite(factor) and factor > 0 else 1.0 + + +def clustered_score_paired_ci( + values_a: np.ndarray, values_b: np.ndarray, alpha: float = 0.05, +) -> tuple[float, float]: + """Yang, Sun & Hardin (2012) score CI for clustered matched-pair binary data. + + Their X^2_Score: Tango's score statistic with the variance multiplied by the + Eliasziw-Donner inflation factor, inverted by solving the same quartic used + for the unclustered case. Concretely it is :func:`tango_scc_paired_ci` with + ``z^2`` replaced by ``z^2 * (1 + (n_c - 1) * rho_hat)``, which is why no new + solver is needed. + + Validated against Yang et al.'s published worked example (their Table II, + PET/SPECT data): reproduces the reported CI (-0.03829, 0.29140) exactly, and + reduces exactly to ``tango_scc_paired_ci(..., c=0)`` when the inflation + factor is 1. + + Unlike our earlier effective-runs correction, the design effect here + multiplies a variance built from POOLED run-level counts, which genuinely + understates uncertainty under clustering -- the level at which the + correction is meant to act. + """ + a_k, b_k, c_k, d_k = _clustered_paired_cells( + values_a, values_b, "clustered_score_paired_ci" ) + n_total = float((a_k + b_k + c_k + d_k).sum()) + if n_total <= 0: + return (0.0, 0.0) + z2 = float(stats.norm.ppf(1.0 - alpha / 2.0)) ** 2 + z2_eff = z2 * _eliasziw_inflation_factor(a_k, b_k, c_k, d_k) + b_tot, c_tot = float(b_k.sum()), float(c_k.sum()) + upper = _tango_scc_real_roots_in_range( + _tango_scc_quartic_coeffs(b_tot, c_tot, n_total, z2_eff, 0.0) + ) + d_hat = (b_tot - c_tot) / n_total + hi = float(upper[-1]) if len(upper) else d_hat + lo = float(upper[0]) if len(upper) else d_hat + lo, hi = (max(-1.0, min(lo, hi)), min(1.0, max(lo, hi))) + if c_tot == 0.0 and b_tot == n_total: + hi = 1.0 + elif b_tot == 0.0 and c_tot == n_total: + lo = -1.0 + return (lo, hi) -def tango_paired_ci( +def modified_obuchowski_paired_ci( + values_a: np.ndarray, values_b: np.ndarray, alpha: float = 0.05, +) -> tuple[float, float]: + """Yang et al. (2010) modified-Obuchowski CI for clustered matched-pair data. + + As given in Yang, Sun & Hardin (2012) sec 2.4:: + + (1/N) sum(b_k - c_k) +/- z * (1/N) * sqrt( + K / (2 (K-1)) * sum[ ((b_k-c_k) - mean_k(b-c))^2 + + ((b_k-c_k) - (n_k/N) sum(b-c))^2 ] ) + + Cluster-level and assumption-free about the within-cluster correlation + structure: no ICC is estimated at all. Yang et al. (2012) recommend this + over Obuchowski's and Durkalski's variants on power grounds for larger + numbers of clusters. The underlying X^2_MO statistic in this module's + tests reproduces the reference R implementation (clust.bin.pair) exactly. + """ + a_k, b_k, c_k, d_k = _clustered_paired_cells( + values_a, values_b, "modified_obuchowski_paired_ci" + ) + n_k = a_k + b_k + c_k + d_k + K = len(n_k) + N = float(n_k.sum()) + if N <= 0: + return (0.0, 0.0) + s_k = b_k - c_k + s_tot = float(s_k.sum()) + d_hat = s_tot / N + if K < 2: + return (max(-1.0, d_hat), min(1.0, d_hat)) + z = float(stats.norm.ppf(1.0 - alpha / 2.0)) + term = ((s_k - s_tot / K) ** 2 + (s_k - (n_k / N) * s_tot) ** 2).sum() + var = (K / (2.0 * (K - 1.0))) * float(term) + radius = z * np.sqrt(max(var, 0.0)) / N + return (max(-1.0, d_hat - radius), min(1.0, d_hat + radius)) + + +def _mj_discordance_floor(discordance_rate: float, floor: float = 0.25) -> float: + """Floored discordance term for the May & Johnson score interval. + + The closed-form solution of the score inversion carries an additive + ``z^2 * S_hat`` inside the discriminant, where ``S_hat`` is the observed + discordance rate (n10+n01)/n. Left unfloored that term vanishes when no + pairs disagree, collapsing the interval to zero width -- the degeneracy + Tango's 2000 letter to the editor (Statist. Med. 19(1):133-139) + criticised in the Quesenberry-Hurst / May & Johnson construction, along + with its anticonservatism at low discordance. + + Flooring ``S_hat`` at 1/4 removes that failure while never SHRINKING the + score interval's variance term, so the result is never narrower than the + published interval. Measured effect (see + simulations/papers/pairwise_binary_rerun_plan.md): at n=15 with 10% + discordance, unfloored May & Johnson covers 0.719 at its worst over the + true difference against a nominal 0.95 (0.787 at delta=0.04), while the + floored interval stays at or above 0.987. NOTE the coverage gap is + invisible exactly at delta=0, where the degenerate zero-width interval + still "contains" a true difference of zero. + On real eval corpora the floor lifts worst-case coverage + from 0.721 to 0.789 at n=10 (single-run) and 0.902 to 0.925 at n=50 + (multi-run), while leaving low-asymmetry corpora untouched. + """ + return max(float(discordance_rate), floor) + + +def mj_floor_paired_ci( values_a: np.ndarray, values_b: np.ndarray, alpha: float, + floor: float = 0.25, ) -> tuple[float, float]: - """Tango score CI for the paired binary difference p(A=1) - p(B=1). - - Implements the large-sample score interval proposed by Tango (1998) for - matched-pairs binary data. Let: + """Closed-form paired-binary CI for p(A=1) - p(B=1). + + This is NOT Tango (1998)'s own interval, despite the name it carries + throughout the codebase and paper. Tango's interval inverts a score test + through the constrained MLE and is solved iteratively (secant method), + which is why it is not used as a fast default here. + + What this actually computes is a Wilson-regularized Quesenberry-Hurst-style + interval. With m = n10 + n01 and d = n10 - n01:: + + d / (n + z^2) +/- z/(n + z^2) * sqrt( m - d^2/n + z^2/4 ) + + That is the centre of May & Johnson (1997), "Confidence intervals for + differences in correlated binary proportions" (Statistics in Medicine + 16(18):2127-2136), equation 11 -- their adaptation of Quesenberry-Hurst -- + with their variance term ``z^2 * m / n`` replaced by the constant + ``z^2 / 4`` from Wilson's one-sample score interval. Writing S_hat = m/n + for the observed discordance rate, their additive term is ``z^2 * S_hat`` + and ours is ``z^2 / 4``, so they coincide exactly at S_hat = 1/4. Below + that this interval is the WIDER of the two (conservative), above it the + narrower. Paired eval comparisons sit well below 1/4 -- competing models + agree on most items -- so the substitution is conservative in the regime + it is used in. + + Note ``1/4`` is not a max-variance bound here: Var(A_i - B_i) at delta=0 + is S in [0, 1], so its maximum is 1, not 1/4. The constant amounts to + imputing a fixed 25% discordance rate in a term of order z^2, which only + matters when m is small. + + That substitution is deliberate: the published Quesenberry-Hurst form + collapses to a ZERO-WIDTH interval when no pairs disagree (m = 0). Tango's + 2000 letter to the editor (Statist. Med. 19(1):133-139) criticised exactly + that degeneracy, and the anticonservatism, of Quesenberry-Hurst. Sparse + discordance is common in small eval sets, so the constant is what makes + this usable here. + + Note also that inverting the score test for this variance function returns + May & Johnson's interval exactly -- so this is NOT a score interval; it + freezes the variance at the observed d_hat rather than solving at the + hypothesised delta. + + Consequence worth knowing: like the other closed-form members of this + family, this runs somewhat NARROWER than Tango's exact score interval -- + by ~0.005-0.018 in absolute width at n=100-200, and it stays finite at + zero discordance where May-Johnson gives width 0. If you want the exact + interval in closed form, call :func:`tango_scc_paired_ci` with ``c=0.0``; + that implements Chang et al. (2024)'s quartic solution and agrees with a + direct numerical inversion of the score equation to ~5e-4. + + Let: * ``n10`` be the count of pairs with ``A=1, B=0`` * ``n01`` be the count of pairs with ``A=0, B=1`` @@ -1628,9 +2240,8 @@ def tango_paired_ci( + z^2 / (4 n^2) ) - This is a score-type interval for the paired risk difference; unlike - :func:`newcombe_paired_ci`, it remains non-degenerate even when there are - no discordant pairs. + This is a score-type interval for the paired risk difference; it + remains non-degenerate even when there are no discordant pairs. Parameters ---------- @@ -1652,19 +2263,161 @@ def tango_paired_ci( values_a = np.asarray(values_a) values_b = np.asarray(values_b) if values_a.ndim != 1 or values_b.ndim != 1: - raise ValueError("tango_paired_ci expects 1-D input arrays.") + raise ValueError("mj_floor_paired_ci expects 1-D input arrays.") if values_a.shape != values_b.shape: - raise ValueError("tango_paired_ci expects arrays with equal shape.") + raise ValueError("mj_floor_paired_ci expects arrays with equal shape.") a_bin = (values_a >= 0.5).astype(int) b_bin = (values_b >= 0.5).astype(int) - return tango_paired_ci_from_diffs(a_bin - b_bin, alpha) + return mj_floor_paired_ci_from_diffs(a_bin - b_bin, alpha, floor) + + +def bonett_price_paired_ci_from_diffs(diffs: np.ndarray, alpha: float = 0.05) -> tuple[float, float]: + """:func:`bonett_price_paired_ci` from a-minus-b diffs. + The Bonett-Price interval depends on the raw pairs only through the two + discordant counts and n (see that function: p12/p21 are built from n10, + n01 and n), and ``diffs`` in ``{-1, 0, 1}`` determines all three -- so + rebuilding a representative pair of binary arrays and delegating gives + bit-identical output while keeping ONE copy of the formula. -def tango_paired_ci_from_diffs(diffs: np.ndarray, alpha: float) -> tuple[float, float]: - """Tango score CI for the paired binary difference, from a-minus-b diffs. + Exists so simultaneous-CI constructions (Sidak, joint-bootstrap scaling) + can widen the SAME interval the non-simultaneous pairwise path reports + for binary data, reusing each comparison's stored ``per_input_diffs``. + Before this, the simultaneous path had no diffs-based Bonett-Price to + call and widened ``mj_floor`` instead, so the simultaneous and pairwise + CIs for binary data were built from two different formulas. + """ + d = np.asarray(diffs).ravel() + values_a = (d == 1).astype(float) + values_b = (d == -1).astype(float) + return bonett_price_paired_ci(values_a, values_b, alpha) + + +def _bonett_price_centre_scale_batch(diffs_2d: np.ndarray, alpha: float = 0.05): + """Vectorized (centre, scale) for :func:`bonett_price_paired_ci_from_diffs` + over a WHOLE matrix of resampled difference vectors at once. + + ``diffs_2d`` is ``(B, M)`` -- B resamples of the same pair's per-item + differences. Returns ``(centre, scale)``, each shape ``(B,)``, such that + the interval at level *a* is ``centre +/- z_{a/2} * scale`` (before the + formula's clip to [-1, 1]). + + Exists for :func:`~evalstats.core.paired._calibrated_joint_critical_value`, + whose calibration needs the construction's own centre and scale on every + resample. Calling the scalar formula B times per pair is ~485x slower and, + worse, recovers the scale as ``(hi - lo) / (2 z)`` -- which is WRONG + whenever the interval clipped at +/-1, precisely the sparse small-n case + the calibration is for. This computes both analytically from the counts, + so it is exact and never sees the clip. + + Bonett-Price depends on the data only through ``(n10, n01, n)``, so the + whole batch reduces to two count reductions along the item axis. The + ``alpha`` argument is accepted (and ignored) because centre and scale are + alpha-free for this Wald-form interval -- the signature matches the + protocol so other formulas can supply an alpha-dependent version. + """ + d = np.asarray(diffs_2d) + n = d.shape[1] + if n == 0: + z = np.zeros(d.shape[0]) + return z, z + n10 = (d == 1).sum(axis=1) + n01 = (d == -1).sum(axis=1) + p12 = (n10 + 1.0) / (n + 2.0) + p21 = (n01 + 1.0) / (n + 2.0) + centre = p12 - p21 + scale = np.sqrt(np.maximum(p12 + p21 - centre * centre, 0.0) / (n + 2.0)) + return centre, scale + + +def _alpha_crit_symmetric(target, centre, scale, sf): + """Level at which a symmetric interval ``centre +/- q(alpha/2)*scale`` just + covers *target*, given the reference distribution's survival function *sf*. + + Covering means ``|target-centre| <= q(alpha/2)*scale``; since ``q`` falls as + alpha rises, the crossing level is ``2*sf(|target-centre|/scale)``. + """ + centre = np.asarray(centre, dtype=float) + scale = np.asarray(scale, dtype=float) + out = np.ones(centre.shape, dtype=float) + ok = np.isfinite(centre) & np.isfinite(scale) & (scale > 1e-12) + if np.any(ok): + z = np.abs(target - centre[ok]) / scale[ok] + out[ok] = np.clip(2.0 * sf(z), 1e-12, 1.0) + return out + + +def _bonett_price_alpha_crit_batch(diffs_2d, target): + """alpha_crit for Bonett-Price -- symmetric on the difference scale, normal + reference (see :func:`bonett_price_paired_ci_from_diffs`).""" + centre, scale = _bonett_price_centre_scale_batch(diffs_2d) + return _alpha_crit_symmetric(target, centre, scale, stats.norm.sf) + + +def _logit_t_alpha_crit_batch(values_2d, target, lo=0.0, hi=1.0): + """alpha_crit for :func:`logit_t_ci_1d` (optionally through + :func:`~evalstats.core.stats_utils.rescaled_ci` bounds *lo*/*hi*). + + logit-t is symmetric on the LOGIT scale with a t reference, not on the + value scale -- so the crossing level is computed there and the target is + mapped through the same transform. Degenerate rows (zero-variance + resamples, which the scalar path hands to ``degenerate_sample_ci``) return + 1.0, i.e. they never bind the joint minimum, matching that path's skip. + """ + span = float(hi - lo) + v = (np.asarray(values_2d, dtype=float) - lo) / span + y = (float(target) - lo) / span + n = v.shape[1] + out = np.ones(v.shape[0], dtype=float) + if n <= 1: + return out + x_bar = v.mean(axis=1) + se = v.std(axis=1, ddof=1) / np.sqrt(n) + ok = (np.ptp(v, axis=1) > 0.0) & (se > 0.0) & np.isfinite(se) & (x_bar > 0.0) & (x_bar < 1.0) + ok &= (y > 0.0) & (y < 1.0) + if not np.any(ok): + return out + g = np.log(x_bar[ok] / (1.0 - x_bar[ok])) + se_g = se[ok] / (x_bar[ok] * (1.0 - x_bar[ok])) + z = np.abs(np.log(y / (1.0 - y)) - g) / se_g + out[ok] = np.clip(2.0 * stats.t.sf(z, df=n - 1), 1e-12, 1.0) + return out + + +def _nig_alpha_crit_batch(values_2d, target, b0=0.0625, m0=0.5, k0=1.0, a0=2.0, lo=0.0, hi=1.0): + """alpha_crit for :func:`nig_ci_1d` -- symmetric on the (rescaled) value + scale with a t reference at ``df = 2*a_n``. Mirrors that function's + posterior update exactly; see it for the parameterization.""" + span = float(hi - lo) + v = (np.asarray(values_2d, dtype=float) - lo) / span + y = (float(target) - lo) / span + n = v.shape[1] + out = np.ones(v.shape[0], dtype=float) + if n <= 0: + return out + xbar = v.mean(axis=1) + ss = ((v - xbar[:, None]) ** 2).sum(axis=1) + kn = k0 + n + mn = (k0 * m0 + n * xbar) / kn + an = a0 + n / 2.0 + bn = b0 + 0.5 * ss + (k0 * n * (xbar - m0) ** 2) / (2.0 * kn) + scale = np.sqrt(np.maximum(bn / (an * kn), 0.0)) + return _alpha_crit_symmetric(y, mn, scale, lambda z: stats.t.sf(z, df=2.0 * an)) + + +#: Optional fast path consumed by +#: evalstats.core.paired._calibrated_joint_critical_value. Formulas without +#: one fall back to per-resample scalar calls there. +bonett_price_paired_ci_from_diffs.centre_scale_batch = _bonett_price_centre_scale_batch +bonett_price_paired_ci_from_diffs.alpha_crit_batch = _bonett_price_alpha_crit_batch - Same closed-form score interval as :func:`tango_paired_ci`, but takes + +def mj_floor_paired_ci_from_diffs(diffs: np.ndarray, alpha: float, floor: float = 0.25) -> tuple[float, float]: + """Floored May & Johnson score CI for the paired binary difference, + from a-minus-b diffs. + + Same closed-form score interval as :func:`mj_floor_paired_ci`, but takes the already-computed per-pair difference ``a_bin - b_bin`` (values in ``{-1, 0, 1}``) directly instead of the two raw ``values_a``/``values_b`` arrays. Concordant pairs (``diff == 0``, whether both 1 or both 0) don't @@ -1699,10 +2452,11 @@ def tango_paired_ci_from_diffs(diffs: np.ndarray, alpha: float) -> tuple[float, z2 = z * z denom = 1.0 + z2 / n + s_hat = _mj_discordance_floor((n10 + n01) / n, floor) radicand = ( (n10 + n01) / (n * n) - ((n10 - n01) ** 2) / (n**3) - + z2 / (4.0 * n * n) + + z2 * s_hat / (n * n) ) radius = (z / denom) * float(np.sqrt(max(radicand, 0.0))) center = d_hat / denom @@ -1738,6 +2492,23 @@ def _tango_scc_real_roots_in_range(coeffs: list[float]) -> np.ndarray: return np.sort(roots[(roots >= -1.0 - 1e-9) & (roots <= 1.0 + 1e-9)]) +def mj_unfloored_paired_ci( + values_a: np.ndarray, + values_b: np.ndarray, + alpha: float, +) -> tuple[float, float]: + """May & Johnson (1997) eq. 11 as published, with NO discordance floor. + + Provided as the comparison baseline that shows why the floor exists: this + is the literal published interval, which degenerates to zero width when + no pairs disagree and under-covers badly at low discordance (worst-case + 0.719 against a nominal 0.95 at n=15, S=0.10, over the true difference). + Use :func:`mj_floor_paired_ci` in + practice. + """ + return mj_floor_paired_ci(values_a, values_b, alpha, floor=0.0) + + def tango_scc_paired_ci( values_a: np.ndarray, values_b: np.ndarray, @@ -1770,7 +2541,7 @@ def tango_scc_paired_ci( ``c=0.125`` is the paper's recommended "SCC-S" (small-correction) variant -- found in their simulations to best balance coverage and width against the plain (uncorrected) Tango score interval - (:func:`tango_paired_ci`, a separate, simpler large-sample approximation + (:func:`mj_floor_paired_ci`, a separate, simpler large-sample approximation that does not use this quartic's constrained-MLE derivation). ``c=0.25`` and ``c=0.5`` are their "SCC-M"/"SCC-L" variants. @@ -1819,16 +2590,37 @@ def tango_scc_paired_ci( ci_high = float(upper_roots[-1]) if len(upper_roots) else d_hat ci_low = float(lower_roots[0]) if len(lower_roots) else d_hat - return (max(-1.0, min(ci_low, ci_high)), min(1.0, max(ci_low, ci_high))) + ci_low, ci_high = (max(-1.0, min(ci_low, ci_high)), min(1.0, max(ci_low, ci_high))) + + # Yang, Sun & Hardin (2012) Remark 1. When every pair is discordant in + # the same direction the score statistic is 0/0 at delta = +/-1, so the + # quartic loses the corresponding root and the interval comes back + # EXCLUDING the point estimate d_hat = +/-1. Tango's interval is defined + # to take the boundary there. Without this the interval is also + # asymmetric under swapping a and b, since the root is recovered in one + # orientation but not the other. Matches Fagerland et al.'s reference + # implementation (R package contingencytables). + if n21 == 0.0 and n12 == N: + ci_high = 1.0 + elif n12 == 0.0 and n21 == N: + ci_low = -1.0 + return (ci_low, ci_high) -def tango_paired_ci_multirun_cluster( + +def mj_floor_paired_ci_multirun_cluster( values_a: np.ndarray, values_b: np.ndarray, alpha: float, ) -> tuple[float, float]: """Cluster-robust Tango CI for paired binary difference. + The additive discordance term in the discriminant is floored at 1/4 (see + :func:`_mj_discordance_floor`); this is the multi-run analogue of the + single-run floor in :func:`mj_floor_paired_ci`, using the mean per-item + discordance mass as S_hat. NOTE: this method is NOT Tango's interval + despite the name it carried before 2026-08-24. + Treats each item as the unit of analysis. Uses the variance of per-item paired differences directly, avoiding fragile within/between decomposition. @@ -1836,7 +2628,7 @@ def tango_paired_ci_multirun_cluster( This is the most robust multirun extension: runs are treated as internal noise already reflected in delta_i. - Reduces exactly to tango_paired_ci when n_runs == 1. + Reduces exactly to mj_floor_paired_ci when n_runs == 1. """ values_a = np.asarray(values_a) values_b = np.asarray(values_b) @@ -1851,7 +2643,7 @@ def tango_paired_ci_multirun_cluster( return (0.0, 0.0) if n_runs == 1: - return tango_paired_ci(values_a[:, 0], values_b[:, 0], alpha) + return mj_floor_paired_ci(values_a[:, 0], values_b[:, 0], alpha) # --- binarize --- a_bin = (values_a >= 0.5).astype(int) @@ -1878,7 +2670,8 @@ def tango_paired_ci_multirun_cluster( denom = 1.0 + z2 / n_items # --- variance (cluster-robust) --- - radicand = var_delta / n_items + z2 / (4.0 * n_items * n_items) + s_hat = _mj_discordance_floor(float(np.mean(d10_i + d01_i))) + radicand = var_delta / n_items + z2 * s_hat / (n_items * n_items) radius = (z / denom) * float(np.sqrt(max(radicand, 0.0))) center = d_hat / denom @@ -1889,15 +2682,21 @@ def tango_paired_ci_multirun_cluster( return (lo, hi) -def tango_paired_ci_multirun_effective( +def mj_floor_paired_ci_multirun_effective( values_a: np.ndarray, values_b: np.ndarray, alpha: float, ) -> tuple[float, float]: - """Correlation-aware multirun Tango CI using effective sample size. + """Correlation-aware multi-run score CI using effective sample size. - This is "ER-Tango" in the paper's CI decision-tree figure and appendix: - the multi-run pairwise-binary method for N >= 50 (``method='tango'`` + The additive discordance term in the discriminant is floored at 1/4 (see + :func:`_mj_discordance_floor`); this is the multi-run analogue of the + single-run floor in :func:`mj_floor_paired_ci`, using the mean per-item + discordance mass as S_hat. NOTE: this method is NOT Tango's interval + despite the name it carried before 2026-08-24. + + This is ``mj_floor_er`` in the paper's CI decision-tree figure and appendix (called "ER-Tango" before the 2026-08-24 rename): + the multi-run pairwise-binary method for N >= 50 (``method='mj_floor'`` dispatches here automatically when R >= 3 seeded runs are present -- see :func:`pairwise_differences`). @@ -1919,7 +2718,7 @@ def tango_paired_ci_multirun_effective( return (0.0, 0.0) if n_runs == 1: - return tango_paired_ci(values_a[:, 0], values_b[:, 0], alpha) + return mj_floor_paired_ci(values_a[:, 0], values_b[:, 0], alpha) # --- binarize --- a_bin = (values_a >= 0.5).astype(int) @@ -1963,7 +2762,8 @@ def tango_paired_ci_multirun_effective( z2 = z * z denom = 1.0 + z2 / n_items - radicand = between + within + z2 / (4.0 * n_items * n_items) + s_hat = _mj_discordance_floor(float(np.mean(u_i))) + radicand = between + within + z2 * s_hat / (n_items * n_items) radius = (z / denom) * float(np.sqrt(max(radicand, 0.0))) center = d_hat / denom @@ -1974,15 +2774,21 @@ def tango_paired_ci_multirun_effective( return (lo, hi) -def tango_paired_ci_multirun_moments( +def mj_floor_paired_ci_multirun_moments( values_a: np.ndarray, values_b: np.ndarray, alpha: float, ) -> tuple[float, float]: """Multi-run Tango-style CI using a cluster moments decomposition. - Not the method ``pairwise_differences(method='tango')`` dispatches to - for multi-run data -- that's :func:`tango_paired_ci_multirun_effective` + The additive discordance term in the discriminant is floored at 1/4 (see + :func:`_mj_discordance_floor`); this is the multi-run analogue of the + single-run floor in :func:`mj_floor_paired_ci`, using the mean per-item + discordance mass as S_hat. NOTE: this method is NOT Tango's interval + despite the name it carried before 2026-08-24. + + Not the method ``pairwise_differences(method='mj_floor')`` dispatches to + for multi-run data -- that's :func:`mj_floor_paired_ci_multirun_effective` ("ER-Tango" in the paper). This variant remains available as an alternative/comparison point (see ``simulations/harness``), not as a routed default. @@ -1995,7 +2801,7 @@ def tango_paired_ci_multirun_moments( where ``delta_i`` is the per-item mean paired difference across runs and ``u_i`` is the per-item discordance mass. It remains score-shrunk using - Tango's denominator and reverts exactly to :func:`tango_paired_ci` when + the same denominator and reverts exactly to :func:`mj_floor_paired_ci` when ``n_runs == 1``. Parameters @@ -2025,7 +2831,7 @@ def tango_paired_ci_multirun_moments( # Exact reduction to the original paired Tango interval for single-run data. if n_runs == 1: - return tango_paired_ci(values_a[:, 0], values_b[:, 0], alpha) + return mj_floor_paired_ci(values_a[:, 0], values_b[:, 0], alpha) a_bin = (values_a >= 0.5).astype(int) b_bin = (values_b >= 0.5).astype(int) @@ -2056,7 +2862,8 @@ def tango_paired_ci_multirun_moments( z2 = z * z denom = 1.0 + z2 / n_items - radicand = between + within + z2 / (4.0 * n_items * n_items) + s_hat = _mj_discordance_floor(float(np.mean(u_i))) + radicand = between + within + z2 * s_hat / (n_items * n_items) radius = (z / denom) * float(np.sqrt(max(radicand, 0.0))) center = d_hat / denom @@ -2066,6 +2873,288 @@ def tango_paired_ci_multirun_moments( return (lo, hi) +# --------------------------------------------------------------------------- +# Multi-run Bonett-Price +# +# THE DERIVATION, once, so the three variants below can just cite it. +# +# Write D_i = A_i - B_i in {-1, 0, +1} for the single-run per-item difference. +# Then sum_i D_i = n10 - n01 and sum_i D_i^2 = n10 + n01 (squaring a value in +# {-1,0,1} is the same as taking its absolute value), so the published +# Bonett & Price (2012) limits +# +# p12 = (n10 + 1)/(n + 2), p21 = (n01 + 1)/(n + 2) +# (p12 - p21) +/- z * sqrt[ (p12 + p21 - (p12 - p21)^2) / (n + 2) ] +# +# can be rewritten with no reference to the 2x2 table at all: +# +# p12 - p21 = (sum_i D_i) / (n + 2) +# p12 + p21 = (sum_i D_i^2 + 2) / (n + 2) +# variance term = mean(D^2) - mean(D)^2, both means over n + 2 +# +# i.e. BONETT-PRICE IS THE PLAIN WALD INTERVAL ON THE MEAN OF D, COMPUTED ON +# THE SAMPLE AUGMENTED BY TWO PSEUDO-ITEMS, ONE WITH D = +1 AND ONE WITH +# D = -1 -- with the ddof=0 plug-in variance and the divisor n + 2 used +# consistently for the mean, the variance and the standard error. (Verified +# numerically against :func:`bonett_price_paired_ci` to 2e-16 over a grid of +# n and alpha; see tests/test_bonett_price_multirun.py.) The Laplace +# adjustment is the two pseudo-items: they cancel in the numerator of the +# point estimate, shrinking it toward 0 by n/(n+2), and they contribute 2 to +# the second moment, which is what keeps the variance term strictly positive +# when no pairs disagree. +# +# That reading is what makes the multi-run generalisation obvious, and it +# settles the question the "+1/+2" naturally raises -- whether the +# pseudo-counts should be scaled by the number of runs R. They should NOT. +# The pseudo-observations are ITEMS, not runs: two extra items, each of which +# happens to be perfectly concordant across all R of its own runs (delta = +1 +# and -1, so within-item variance 0). Scaling them to R pseudo-runs +# (equivalently, using pseudo-items with delta = +-1/R) makes the whole +# regularisation vanish as R grows, so at zero observed discordance the +# interval would collapse toward zero width -- exactly the degeneracy the +# Laplace adjustment exists to prevent. Item-level heterogeneity is bounded +# by N, not by N*R: more runs per item tell you nothing about items you never +# sampled. That variant was implemented and measured, and it does fail this +# way; the numbers are in tests/test_bonett_price_multirun.py +# (``test_per_run_laplace_scaling_degenerates``), kept as a regression guard +# so nobody re-derives it. +# +# So, for (N, R) data, work at the item scale throughout: +# +# delta_i = mean_r (A_ir - B_ir) in [-1, 1] per-item mean difference +# u_i = mean_r |A_ir - B_ir| in [0, 1] per-item discordance mass +# w_i = u_i - delta_i^2 >= 0 per-item within-run variance +# +# augment with delta = +1 and delta = -1 (both with u = 1, hence w = 0), and +# the three augmented moments are +# +# delta~ = (sum_i delta_i) / (N + 2) +# m2~ = (sum_i delta_i^2 + 2) / (N + 2) +# V~ = m2~ - delta~^2 item-level total variance +# w~ = (sum_i w_i) / (N + 2) mean within-item variance +# +# with the interval delta~ +/- z * sqrt(V~ / (N + 2)). At R = 1 every +# delta_i is in {-1,0,1} so delta_i^2 = u_i, giving m2~ = p12 + p21 and +# w_i = 0 -- every variant below reduces to :func:`bonett_price_paired_ci` +# EXACTLY, not just asymptotically, and no special-casing of R == 1 is +# needed anywhere. +# +# WHY NO EXPLICIT BETWEEN-RUN CORRELATION TERM. V~ is already the right +# quantity. Items are the sampling unit and are iid; whatever correlation +# the R runs of item i have with each other only affects Var(delta_i), and +# Var(delta_i) is exactly what the item-level spread measures. Concretely, +# under the usual one-way random-effects decomposition +# +# Var(delta_i) = sigma_B^2 + sigma_W^2/R * (1 + (R-1)*rho) +# +# the design-effect factor is *inside* the quantity being estimated, so +# estimating Var(delta_i) directly needs no rho at all. This is not a hand +# wave: applying Kish's R_eff = R/(1 + (R-1)*rho) the way it is meant to be +# applied -- pool all N*R runs for a run-level variance B~ = u~ - delta~^2, +# estimate the run-level ICC rho = 1 - sigma_W^2/B~ with the UNBIASED within +# estimate sigma_W^2 = R/(R-1) * w~, then inflate by the design effect -- +# gives B~ * (1 + (R-1)*rho) / R == V~ IDENTICALLY, to machine precision and +# for every input, not merely in expectation. The design-effect correction +# and the item-level variance are the same estimator written two ways. (Also +# verified in tests/test_bonett_price_multirun.py.) +# +# So the three variants below differ ONLY in a floor applied to V~, mirroring +# what mj_floor's three multi-run variants turn out to differ by (their +# ``max(var - w/R', 0) + w/R'`` construction is algebraically +# ``max(var, w/R')``): +# +# cluster V~ no floor -- the derivation as-is +# moments max(V~, w~/R) floor at the within-item term alone +# effective max(V~, w~/R_eff) same, with Kish's R_eff <= R +# +# --------------------------------------------------------------------------- + + +def _bp_item_moments( + values_a: np.ndarray, values_b: np.ndarray, fname: str +) -> tuple[np.ndarray, np.ndarray]: + """Per-item ``(delta_i, u_i)`` from ``(n_items, n_runs)`` binary matrices. + + ``delta_i`` is the per-item mean of ``A_ir - B_ir`` and ``u_i`` the + per-item mean of ``|A_ir - B_ir|`` (the discordance mass). Values are + thresholded at 0.5. See the derivation block above. + """ + a = np.asarray(values_a) + b = np.asarray(values_b) + if a.shape != b.shape: + raise ValueError(f"{fname} expects arrays with equal shape (n_items, n_runs).") + if a.ndim != 2: + raise ValueError(f"{fname} expects 2-D arrays (n_items, n_runs).") + if a.shape[1] < 1: + # Caught explicitly: the per-item means below would be all-NaN and the + # w~/R floors would divide by zero, so an item with no runs at all + # would surface as a ZeroDivisionError from deep inside the variance + # rather than as the input error it is. + raise ValueError(f"{fname} expects at least one run per item.") + d = (a >= 0.5).astype(np.int8) - (b >= 0.5).astype(np.int8) + return np.mean(d, axis=1, dtype=float), np.mean(np.abs(d), axis=1, dtype=float) + + +def _bonett_price_augmented_interval( + delta_i: np.ndarray, alpha: float, var_floor: float = 0.0, + pseudo_m2: float = 1.0, +) -> tuple[float, float]: + """Wald interval on ``mean(delta_i)`` over the ``+/-1``-augmented item sample. + + The shared core of every Bonett-Price variant in this module, single-run + included: see the derivation block above for why the two pseudo-items are + the Laplace adjustment. ``var_floor`` is a lower bound on the augmented + item-level variance ``V~``, used by the ``moments`` and ``effective`` + variants; leave it at 0 for the plain (``cluster``) interval. + """ + n = int(np.asarray(delta_i).shape[0]) + if n <= 0: + return (0.0, 0.0) + delta_i = np.asarray(delta_i, dtype=float) + n_aug = n + 2.0 + delta_t = float(np.sum(delta_i)) / n_aug # pseudo-items cancel: +1 - 1 = 0 + m2_t = (float(np.sum(delta_i * delta_i)) + 2.0 * pseudo_m2) / n_aug # pseudo-items add m2 each + var_t = max(m2_t - delta_t * delta_t, float(var_floor), 0.0) + z = float(stats.norm.ppf(1.0 - alpha / 2.0)) + se = float(np.sqrt(var_t / n_aug)) + return ( + float(np.clip(delta_t - z * se, -1.0, 1.0)), + float(np.clip(delta_t + z * se, -1.0, 1.0)), + ) + + +def bonett_price_paired_ci_multirun_cluster( + values_a: np.ndarray, + values_b: np.ndarray, + alpha: float = 0.05, +) -> tuple[float, float]: + """Multi-run Bonett-Price CI, item-clustered -- the derivation with no floor. + + The Bonett-Price counterpart of + :func:`mj_floor_paired_ci_multirun_cluster`, and the most defensible of + the three: it is the single-run interval's own construction carried over + unchanged, with the item as the unit of analysis and no extra modelling. + + delta~ = (sum_i delta_i) / (N + 2) + V~ = (sum_i delta_i^2 + 2) / (N + 2) - delta~^2 + CI = delta~ +/- z * sqrt( V~ / (N + 2) ) + + where ``delta_i = mean_r (A_ir - B_ir)``. See the derivation block above + the private helpers in this module for the full argument, in particular + for why the ``+1/+2`` pseudo-counts stay on the ITEM scale and why no + between-run correlation term is needed (``V~`` already contains it, and + a correctly-specified Kish design effect provably reduces to ``V~``). + + Reduces to :func:`bonett_price_paired_ci` EXACTLY at ``n_runs == 1``, by + construction rather than by a special case: at R = 1 each ``delta_i`` is + in ``{-1, 0, 1}``, so ``sum_i delta_i = n10 - n01`` and + ``sum_i delta_i^2 = n10 + n01``. + + Parameters + ---------- + values_a, values_b : np.ndarray + Arrays of shape ``(n_items, n_runs)``, thresholded at 0.5. Runs are + assumed paired across A and B. + alpha : float + Significance level (1 - confidence level). + + Returns + ------- + (ci_low, ci_high) : tuple[float, float] + CI on p(A=1) - p(B=1), clamped to [-1, 1]. + """ + delta_i, _ = _bp_item_moments( + values_a, values_b, "bonett_price_paired_ci_multirun_cluster" + ) + return _bonett_price_augmented_interval(delta_i, alpha) + + +def bonett_price_paired_ci_multirun_shrunk( + values_a: np.ndarray, + values_b: np.ndarray, + alpha: float = 0.05, +) -> tuple[float, float]: + """Multi-run Bonett-Price with the pseudo-item MAGNITUDE Laplace-shrunk. + + :func:`bonett_price_paired_ci_multirun_cluster` pins its two pseudo-items at + ``delta = +/-1``, the largest possible item-level discordance. At ``R = 1`` + that is exactly right -- every discordant item has ``delta_i^2 = 1``, so a + pseudo-item IS one more typical discordant item, which is what a Laplace + pseudo-count means. At ``R > 1`` it stops being right: a discordant item's + ``delta_i^2`` shrinks toward its squared per-item rate, while the pseudo-mass + stays at 2, so the pseudo-items become several times heavier than any real + item and the floor progressively swallows the variance. + + The fix applies Bonett-Price's own device a second time -- once to the + discordance RATE (which the ``n + 2`` denominator already does) and once to + the discordance MAGNITUDE, shrinking it toward the ``R = 1`` reference of 1 + with the same weight of two pseudo-items:: + + m2 = (sum_i delta_i^2 + 2) / (sum_i u_i + 2), u_i = mean_r |A_ir - B_ir| + + and each pseudo-item then carries ``delta^2 = m2`` (still ``+/-sqrt(m2)``, so + they cancel in the mean and the centre is unchanged). + + ``sum_i u_i`` is the EFFECTIVE number of fully-discordant items: an item + discordant on 1 of 20 runs contributes 0.05, not 1. Using a plain count of + discordant items instead fails badly when items flip sign across runs -- + many items are then discordant on a few runs each, the count is large while + the mass is small, and ``m2`` collapses to ~0.2, removing the very + correction it exists to supply (measured MinCov .8268 vs .9296 on a + 300-cell sweep). + + Two properties make the construction easy to state. First, since + ``delta_i^2 <= |delta_i| <= u_i`` for every item, ``sum delta_i^2 <= + sum u_i`` and therefore ``m2`` lies in ``(0, 1]``, with ``m2 = 1`` exactly + when every discordant item is fully sign-consistent across its runs (which + includes ``R = 1``). The pseudo-mass can never exceed Bonett-Price's own, + and can never vanish. Second, ``m2`` IS a shrinkage estimator:: + + m2 = w * (sum delta_i^2 / sum u_i) + (1 - w) * 1, + w = sum u_i / (sum u_i + 2) + + (an identity, verified to 2.2e-16). The data term is the observed mean + squared magnitude per unit of discordance mass, the prior is the ``R = 1`` + value of 1, and the weight is "effective discordant items against two + pseudo-items". That is also the diagnosis of an earlier variant that used + the data term ALONE (``w = 1``, no prior): it undercovered badly wherever + discordance was sparse, because the quantity it shrinks toward zero has + nothing holding it up. + + Properties, all verified to machine precision: + + * ``R = 1``: ``u_i = |D_i|`` and ``delta_i^2 = |D_i|``, so + ``sum delta_i^2 = sum u_i`` and ``m2 = 1`` EXACTLY -- this reduces to + :func:`bonett_price_paired_ci` bit-for-bit, with no special case. + * Zero discordance: ``m2 = 1``, matching the ``+/-1`` construction, which was + already correct there. + * Replication invariance: both sums depend only on the per-item values, so + ``R`` identical copies of one run leave the interval unchanged. + * Antisymmetry under swapping the two arms. + + Note on measured worst-case coverage: a MinCov taken over many cells at + modest reps is biased low by selection. The three cells below .93 in the + 1536-cell sweep at reps=500 all returned .948-.954 when rerun at + reps=20000, i.e. they were 1-2 MC standard errors low, not failures. + + Calibration on a 300-cell sweep (5 discordance shapes x n in 20..100 x R in + 2..20 x five run-consistency mixtures): MinCov .9296 with one cell below + .93, mean coverage .9744, at 94% of the ``+/-1`` construction's width. The + ``+/-1`` form remains better on worst case (MinCov .9444, no cells below + .93) and is still the shipped default; this variant is closer to nominal on + average and narrower. + """ + delta_i, u_i = _bp_item_moments( + values_a, values_b, "bonett_price_paired_ci_multirun_shrunk" + ) + if delta_i.shape[0] == 0: + return (0.0, 0.0) + sq_sum = float(np.sum(delta_i * delta_i)) + u_sum = float(np.sum(u_i)) + m2 = (sq_sum + 2.0) / (u_sum + 2.0) + return _bonett_price_augmented_interval(delta_i, alpha, pseudo_m2=m2) + + def resolve_resampling_method( method: Literal["bootstrap", "bca", "bayes_bootstrap", "smooth_bootstrap", "bootstrap_t", "auto"], sample_size: int, diff --git a/evalstats/core/router.py b/evalstats/core/router.py index a5ca6e9..12d781d 100644 --- a/evalstats/core/router.py +++ b/evalstats/core/router.py @@ -30,7 +30,7 @@ AnalysisResult, ) from .paired import all_pairwise -from .ranking import bootstrap_ranks +from .ranking import LazyRankDistribution, bootstrap_ranks from .variance import robustness_metrics, seed_variance_decomposition from ..config import get_alpha_ci, resolve_auto_analyze_methods @@ -106,6 +106,7 @@ def analyze( pairwise_test: Literal["auto", "bootstrap", "wilcoxon", "nemenyi"] = "auto", ci_style: Literal["gradient", "line"] = "gradient", score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> AnalysisResult: """Run all standard analyses for a benchmark result. @@ -175,14 +176,20 @@ def analyze( The backend is controlled by the ``backend`` parameter. * ``'wilson'`` — Binary-only frequentist mode. Uses Wilson score intervals for point-advantage CIs and Newcombe score intervals - (+ exact McNemar p-values) for pairwise comparisons. + (+ McNemar mid-p p-values) for pairwise comparisons. * ``'newcombe'`` — Binary-only frequentist mode. Alias of ``'wilson'`` routing in ``analyze()``: pairwise comparisons use - Newcombe score intervals (+ exact McNemar p-values), while + Newcombe score intervals (+ McNemar mid-p p-values), while point-advantage CIs use Wilson score intervals. + * ``'mj_floor'`` — Binary-only frequentist mode. Pairwise + comparisons use the floored May & Johnson (1997) score interval + (+ McNemar mid-p p-values), while point-advantage CIs use Wilson + score intervals. This is what ``'auto'`` selects for binary + pairwise comparisons. * ``'tango'`` — Binary-only frequentist mode. Pairwise comparisons - use Tango score intervals (+ exact McNemar p-values), while - point-advantage CIs use Wilson score intervals. + use the exact Tango (1998) score interval (+ McNemar mid-p + p-values), while point-advantage CIs use Wilson score intervals. + Single-run data only; it has no multi-run form. backend : str LMM fitting backend (only used when ``method='lmm'``): ``'statsmodels'`` (default, pure Python, no R required) or @@ -267,10 +274,25 @@ def analyze( The eval metric's true ``(min, max)`` range, e.g. ``(0, 1)`` for normalised accuracy or ``(1, 5)`` for a Likert scale. Only used for numeric (non-binary) data routed to a bounds-dependent method (the - ``'auto'`` default, or explicit ``method='logit_t'``); ignored - otherwise. Declaring this explicitly is strongly recommended for - any metric whose natural range isn't already exactly ``[0, 1]``, - since evalstats has no reliable way to infer it on its own. + ``'auto'`` default, or explicit ``method='logit_t'``/``'nig'``); + ignored otherwise. Declaring this explicitly is strongly + recommended for any metric whose natural range isn't already + exactly ``[0, 1]``, since evalstats has no reliable way to infer + it on its own. + eval_type : {"likert", "continuous"}, optional + Only used with ``method='auto'`` and a known/declared + ``score_range``. Distinguishes discrete/ordinal data (a Likert + scale, an integer percentage grade) from genuinely continuous + data within the same bounded range. When omitted (default), + evalstats auto-detects discreteness from the data's own + quantization grid and emits a ``UserWarning`` if it switches to + the Likert treatment -- pass this explicitly to silence that + warning either way. This changes every pairwise-comparison CI + (NIG instead of logit-t) -- single-run, seeded/multi-run, and the + k>=3 simultaneous-CI construction alike -- see + ``config.AUTO_ANALYZE_METHOD_TABLE``'s "likert" row for the + validation. Marginal/robustness CIs still use logit-t for likert + data, pending their own dedicated validation. When omitted, evalstats always prints a ``UserWarning`` announcing what it assumed and which method it picked as a result: @@ -334,7 +356,7 @@ def analyze( include_multi_ci = ci_style == "gradient" - if method not in {"lmm", "bayes_bootstrap", "smooth_bootstrap", "auto", "bayes_binary", "wilson", "newcombe", "tango", "permutation", "sign_test", "t_interval", "logit_t"} and result.n_inputs < 15: + if method not in {"lmm", "bayes_bootstrap", "smooth_bootstrap", "auto", "bayes_binary", "wilson", "mj_floor", "newcombe", "tango", "permutation", "sign_test", "t_interval", "logit_t"} and result.n_inputs < 15: warnings.warn( f"Only M={result.n_inputs} benchmark input(s) detected. " "Bootstrap confidence intervals are unreliable with fewer than ~15 inputs. " @@ -362,6 +384,7 @@ def analyze( p_value_method=resolved_p_value_method, include_multi_ci=include_multi_ci, score_range=score_range, + eval_type=eval_type, ) # ------------------------------------------------------------------ @@ -742,6 +765,142 @@ def analyze_factorial( # Internal analysis runners # --------------------------------------------------------------------------- +def resolve_auto_robustness_method( + run_scores: np.ndarray, + *, + score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, + stacklevel: int = 2, +) -> tuple[str, str, Optional[tuple[float, float]], str]: + """Auto-detect data kind (binary / likert / bounded_01 / unbounded) and + resolve it to concrete (pairwise_method, robustness_method, + resolved_score_range, data_kind). + + This is the exact "method='auto'" routing logic ``analyze()``/``compare()`` + use internally, factored out so the quick-primitive functions + (``mean_ci``/``summarize`` in ``evalstats.quick``) can reuse it directly + rather than re-deriving calibration choices in a second place that could + silently drift out of sync with ``compare()``'s. + + Parameters + ---------- + run_scores : np.ndarray + Shape ``(N, M)`` or ``(N, M, R)``. Only the shape and values matter + here (dtype/range/binary-ness detection and R for seeded routing) -- + not which entity is which. + score_range : (float, float), optional + Explicit ``[lo, hi]`` bounds, forwarded to :func:`resolve_score_bounds`. + eval_type : {"likert", "continuous"}, optional + Hint disambiguating discrete/ordinal (Likert-style) data from + continuous bounded data when both look the same from the raw + values alone. When omitted, discrete/ordinal data is auto-detected + from its own quantization grid (see :func:`detect_quantization_step`). + Ignored (with a warning) for binary data, which always uses the + binary methods regardless of this hint. + stacklevel : int + Forwarded to any ``UserWarning`` raised here, so it points at the + caller's caller appropriately regardless of how many wrapper frames + sit between the actual user call and this function. + + Returns + ------- + tuple[str, str, tuple[float, float] or None, str] + ``(pairwise_method, robustness_method, resolved_score_range, data_kind)``. + """ + from .resampling import binary_routing_applies, resolve_score_bounds, detect_quantization_step + + if run_scores.ndim == 3: + R = run_scores.shape[2] + N = run_scores.shape[1] + else: + R = 1 + N = run_scores.shape[1] + + if eval_type not in (None, "likert", "continuous"): + raise ValueError(f"eval_type must be 'likert', 'continuous', or None, got {eval_type!r}") + + resolved_score_range: Optional[tuple[float, float]] = None + # An explicitly passed score_range wider than [0, 1] overrides binary + # auto-detection (and says so) -- see binary_routing_applies. + if binary_routing_applies(run_scores, score_range, stacklevel=stacklevel + 1): + data_kind = "binary" + if eval_type is not None: + warnings.warn( + f"eval_type={eval_type!r} was given, but the data was " + "auto-detected as binary (0/1) -- binary data always uses " + "the binary methods regardless of eval_type, so this hint " + "was ignored.", + UserWarning, + stacklevel=stacklevel, + ) + else: + # resolve_score_bounds returns a [lo, hi] range (with a + # UserWarning if it had to auto-detect [0, 1] rather than being + # told explicitly) when one can be reliably established, or None + # when the data falls outside [0, 1] and no score_range was + # given -- there's no safe way to infer a metric's true bounds + # from an arbitrary numeric sample's own min/max. In the None + # case, auto silently downgrades to the bounds-agnostic + # "unbounded" (t_interval) row below, but says so loudly. + resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=stacklevel + 1) + if resolved_score_range is not None: + if eval_type == "likert": + data_kind = "likert" + elif eval_type == "continuous": + data_kind = "bounded_01" + else: + # No explicit hint: auto-detect discrete/ordinal (Likert- + # style) data from its own quantization grid rather than + # assuming continuous -- see detect_quantization_step's + # docstring and config.AUTO_ANALYZE_METHOD_TABLE's + # "likert" row for why this matters (NIG vs logit-t). + step = detect_quantization_step(run_scores) + if step is not None: + data_kind = "likert" + warnings.warn( + f"Bounded numeric evaluation data was auto-detected " + f"as discrete/ordinal (grid step={step:g} within " + f"range {resolved_score_range}). For pairwise " + "comparisons (single-run and multi-run alike), " + "evalstats uses NIG (validated as better-calibrated " + "than logit-t there for this kind of data); " + "marginal/robustness CIs on this data still use " + "logit-t, the same as continuous data, pending " + "their own validation -- see " + "config.AUTO_ANALYZE_METHOD_TABLE's 'likert' row. " + "Pass eval_type='likert' explicitly to silence this " + "warning, or eval_type='continuous' if this " + "discreteness is coincidental (e.g. a metric that " + "happens to only take a few values in your sample).", + UserWarning, + stacklevel=stacklevel, + ) + else: + data_kind = "bounded_01" + else: + data_kind = "unbounded" + # Direct warn() call, one frame shallower than the + # resolve_score_bounds() delegation above (no extra frame in + # between) -- stacklevel here, not stacklevel + 1. + warnings.warn( + "Numeric evaluation data outside [0, 1] was auto-detected " + "with no explicit score_range, so evalstats is using " + "method='t_interval' (a bounds-agnostic default) rather " + "than the better-calibrated logit-t/NIG methods. If you " + "know this eval metric's true (min, max) range, pass it " + "explicitly, e.g. score_range=(1, 5) for a Likert scale " + "or score_range=(0, 100) for a percentage grade.", + UserWarning, + stacklevel=stacklevel, + ) + # See config.AUTO_ANALYZE_METHOD_TABLE for the full auto-routing matrix + # (which method is chosen for which data kind / N / seeded combination). + pairwise_method, robustness_method = resolve_auto_analyze_methods( + data_kind, N, seeded=R >= 3, + ) + return pairwise_method, robustness_method, resolved_score_range, data_kind + + def _analyze_single( result: BenchmarkResult, shape: BenchmarkShape, @@ -761,6 +920,7 @@ def _analyze_single( p_value_method: Optional[str] = None, include_multi_ci: bool = True, score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> AnalysisBundle: # ------------------------------------------------------------------ # LMM path — fit score ~ template + (1|input) @@ -850,41 +1010,12 @@ def _analyze_single( pairwise_method = method robustness_method = method resolved_score_range: Optional[tuple[float, float]] = None + # Only the "auto" branch resolves a data kind; stays None otherwise so + # the bundle records "no resolution happened" rather than a guess. + data_kind: Optional[str] = None if method == "auto": - from .resampling import is_binary_scores, resolve_score_bounds - R = run_scores.shape[2] - N = run_scores.shape[1] - if is_binary_scores(run_scores): - data_kind = "binary" - else: - # resolve_score_bounds returns a [lo, hi] range (with a - # UserWarning if it had to auto-detect [0, 1] rather than being - # told explicitly) when one can be reliably established, or None - # when the data falls outside [0, 1] and no score_range was - # given -- there's no safe way to infer a metric's true bounds - # from an arbitrary numeric sample's own min/max. In the None - # case, auto silently downgrades to the bounds-agnostic - # "unbounded" (t_interval) row below, but says so loudly. - resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=2) - if resolved_score_range is not None: - data_kind = "bounded_01" - else: - data_kind = "unbounded" - warnings.warn( - "Numeric evaluation data outside [0, 1] was auto-detected " - "with no explicit score_range, so evalstats is using " - "method='t_interval' (a bounds-agnostic default) rather " - "than the better-calibrated logit-t method. If you know " - "this eval metric's true (min, max) range, pass it " - "explicitly, e.g. score_range=(1, 5) for a Likert scale " - "or score_range=(0, 100) for a percentage grade.", - UserWarning, - stacklevel=2, - ) - # See config.AUTO_ANALYZE_METHOD_TABLE for the full auto-routing matrix - # (which method is chosen for which data kind / N / seeded combination). - pairwise_method, robustness_method = resolve_auto_analyze_methods( - data_kind, N, seeded=R >= 3, + pairwise_method, robustness_method, resolved_score_range, data_kind = resolve_auto_robustness_method( + run_scores, score_range=score_range, eval_type=eval_type, stacklevel=2, ) elif method == "bayes_binary": from .resampling import is_binary_scores @@ -897,7 +1028,7 @@ def _analyze_single( # Single-sample marginal CIs use Wilson; pairwise uses the Bayesian model. pairwise_method = "bayes_binary" robustness_method = "wilson" - elif method in {"wilson", "newcombe", "tango"}: + elif method in {"wilson", "newcombe", "tango", "mj_floor", "bonett_price"}: from .resampling import is_binary_scores if not is_binary_scores(run_scores): raise ValueError( @@ -905,11 +1036,11 @@ def _analyze_single( "scores array contains non-binary values. Use is_binary_scores() " "to check before calling, or choose a different method." ) - if method == "tango": - pairwise_method = "tango" + if method in ("tango", "mj_floor", "bonett_price"): + pairwise_method = method else: # In analyze(), explicit frequentist binary methods route to: - # - pairwise Newcombe + exact McNemar p-values + # - pairwise Newcombe + McNemar mid-p p-values # - single-sample marginal Wilson score CIs pairwise_method = "newcombe" robustness_method = "wilson" @@ -927,6 +1058,32 @@ def _analyze_single( "(e.g. score_range=(1, 5) for a Likert scale), or use a " "different method (e.g. method='t_interval')." ) + elif method == "nig": + from .resampling import resolve_score_bounds + resolved_score_range = resolve_score_bounds(run_scores, score_range, stacklevel=2) + if resolved_score_range is None: + raise ValueError( + "method='nig' requires data with an inferable [lo, hi] " + "range, but the scores fall outside [0, 1] and no " + "score_range was given. Pass score_range=(lo, hi) explicitly " + "(e.g. score_range=(1, 5) for a Likert scale), or use a " + "different method (e.g. method='t_interval')." + ) + + # eval_type resolved for the simultaneous-CI widening formula: reuse + # the "auto" branch's already-made data_kind decision so it isn't + # independently re-detected (and re-warned about) inside all_pairwise + # -> _simultaneous_cis_router; for an explicit (non-"auto") method, + # just pass through whatever eval_type the caller gave (possibly None, + # in which case _simultaneous_cis_router does its own detection). + if method == "auto": + resolved_eval_type = ( + "likert" if data_kind == "likert" + else "continuous" if data_kind == "bounded_01" + else None + ) + else: + resolved_eval_type = eval_type pairwise = all_pairwise( run_scores, labels, @@ -934,6 +1091,7 @@ def _analyze_single( correction=correction, rng=rng, statistic=statistic, simultaneous_ci=simultaneous_ci, omnibus=omnibus, multi_ci=include_multi_ci, score_range=resolved_score_range, + eval_type=resolved_eval_type, ) robustness = robustness_metrics( run_scores, labels, @@ -946,9 +1104,16 @@ def _analyze_single( multi_ci=include_multi_ci, score_range=resolved_score_range, ) - rank_dist = bootstrap_ranks( - run_scores, labels, - n_bootstrap=n_bootstrap, rng=rng, statistic=statistic, + # Deferred: nothing here computes the rank bootstrap unless a caller + # actually reads rank_probs/expected_ranks/p_best. See + # LazyRankDistribution -- .labels/.n_bootstrap stay free. + rank_dist = LazyRankDistribution( + labels, n_bootstrap, + lambda _rng: bootstrap_ranks( + run_scores, labels, + n_bootstrap=n_bootstrap, rng=_rng, statistic=statistic, + ), + rng=rng, ) seed_var = None @@ -965,6 +1130,7 @@ def _analyze_single( resolved_method=pairwise_method, resolved_ci_method=robustness_method, resolved_score_range=resolved_score_range, + resolved_data_kind=data_kind, p_value_method=p_value_method, ) @@ -1039,10 +1205,11 @@ def _analyze_multi_model( p_value_method: Optional[str] = None, include_multi_ci: bool = True, score_range: Optional[tuple[float, float]] = None, + eval_type: Optional[Literal["likert", "continuous"]] = None, ) -> MultiModelBundle: from .resampling import is_binary_scores - fallback_binary_methods = {"wilson", "newcombe", "tango"} + fallback_binary_methods = {"wilson", "newcombe", "tango", "mj_floor", "bonett_price"} def _effective_method(sub_result: BenchmarkResult) -> CompareMethod: """Fallback only for frequentist binary methods on auxiliary non-binary views.""" @@ -1066,6 +1233,7 @@ def _effective_method(sub_result: BenchmarkResult) -> CompareMethod: p_value_method=p_value_method, include_multi_ci=include_multi_ci, score_range=score_range, + eval_type=eval_type, ) per_model: Dict[str, AnalysisBundle] = {} diff --git a/evalstats/core/summary.py b/evalstats/core/summary.py index da750cc..f87ca31 100644 --- a/evalstats/core/summary.py +++ b/evalstats/core/summary.py @@ -37,6 +37,7 @@ _BRIGHT_YELLOW = "\033[93m" if _ANSI else "" _BRIGHT_CYAN = "\033[96m" if _ANSI else "" _BRIGHT_RED = "\033[91m" if _ANSI else "" +_BRIGHT_MAGENTA = "\033[95m" if _ANSI else "" # "pink" -- reserved for the PPI/MCAR reminder, nothing else def _p_best_color(p: float) -> str: @@ -91,10 +92,11 @@ def _uses_wilson_ci(bundle: "AnalysisBundle") -> bool: def _pairwise_p_value_label(test_method: str) -> str: """Return a human-readable p-value method label for pairwise summaries.""" method = test_method.lower() - if "tango" in method: - return "McNemar" - if "newcombe" in method: - return "McNemar exact" + # All three binary paired CI methods report the same p-value, McNemar's + # mid-p (see core.paired). Fagerland et al. (2014) sec. 9.1 recommend it + # over the exact conditional test, which is markedly conservative. + if "mj_floor" in method or "tango" in method or "newcombe" in method: + return "McNemar mid-p" if "sign test" in method: return "paired sign test" if "wilcoxon" in method: @@ -113,7 +115,8 @@ def _pairwise_display_pvalue(pair: PairedDiffResult) -> tuple[float, str]: """ method = pair.test_method.lower() is_exact_path = ( - "tango" in method + "mj_floor" in method + or "tango" in method or "newcombe" in method or "mcnemar" in method or "sign test" in method @@ -464,9 +467,10 @@ def _fmt(v: float) -> str: multi_ci=pair.multi_ci, ) ci_legend = _legend_ci_label(style, ci_pct, pair.multi_ci is not None) + mean_marker = _mean_marker_legend(style, pair.statistic) print( f" axis: [{axis_low:+.3f}, {axis_high:+.3f}] " - f"(· ±1σ spread, {ci_legend}, ● {pair.statistic}, │ zero)" + f"(· ±1σ spread, {ci_legend}{mean_marker}, │ zero)" ) print(f" {b} (<0) {line} (>0) {a}") print() @@ -610,6 +614,23 @@ def _consistency_color(icc: float) -> str: # Multi-model summary # --------------------------------------------------------------------------- +def _display_order(bundle) -> "np.ndarray": + """Indices giving a stable, readable display order: descending mean, + label as tiebreak. + + These orderings used to read ``rank_dist.expected_ranks``/``p_best``, + which forced the (opt-in) rank bootstrap purely to decide row order -- + see ``core.ranking.LazyRankDistribution``. Mean order is free, already + what the leaderboard sorts by elsewhere, and deterministic. + """ + means = np.asarray(bundle.robustness.mean, dtype=float) + labels = list(bundle.labels) + return np.array( + sorted(range(len(labels)), key=lambda i: (-means[i], labels[i])), + dtype=int, + ) + + def _print_multi_model_summary( bundle: MultiModelBundle, *, @@ -634,6 +655,12 @@ def _print_multi_model_summary( print(f"{_BOLD}Best pair by mean:{_RESET} model='{_BRIGHT_GREEN}{best_model}{_RESET}' template='{_BRIGHT_GREEN}{best_template}{_RESET}'") print() + # MultiModelBenchmark requires >= 2 models, so this section (comparing + # across models) always has something to show. The per-template section + # right below it doesn't have the same guarantee -- a single implicit + # template is common (e.g. a plain model-only comparison) -- so it, and + # the equally-degenerate per-model breakdown loop further down, are + # skipped when there's nothing to compare there. _print_loud_section("Model-level comparison (across all prompts):") _print_bundle_summary( bundle.model_level, @@ -646,23 +673,23 @@ def _print_multi_model_summary( min_meaningful_diff=min_meaningful_diff, show_rank_probabilities=show_rank_probabilities, ) - print() - _print_loud_section("Cross-model per-template comparison (models collapsed):") - _print_bundle_summary( - bundle.template_level, - top_pairwise=top_pairwise, - line_width=line_width, - item_singular="template", - item_plural="templates", - pairwise_sort=pairwise_sort, - style=style, - min_meaningful_diff=min_meaningful_diff, - show_rank_probabilities=show_rank_probabilities, - ) - best_idx = int(np.argmax(bundle.template_level.robustness.mean)) - best_template = bundle.template_level.benchmark.template_labels[best_idx] - + + if bundle.benchmark.n_templates > 1: + _print_loud_section("Cross-model per-template comparison (models collapsed):") + _print_bundle_summary( + bundle.template_level, + top_pairwise=top_pairwise, + line_width=line_width, + item_singular="template", + item_plural="templates", + pairwise_sort=pairwise_sort, + style=style, + min_meaningful_diff=min_meaningful_diff, + show_rank_probabilities=show_rank_probabilities, + ) + print() + # Instability across runs across models instability_rows = _collect_cross_model_seed_instability_rows(bundle) if instability_rows: @@ -674,38 +701,41 @@ def _print_multi_model_summary( f"(instability={instability:.4f}, {_instability_label(instability)})" ) - for model_label, model_bundle in bundle.per_model.items(): - print() - _print_loud_section(f"Per-Model Summary: {model_label}") - _print_bundle_summary( - model_bundle, - top_pairwise=top_pairwise, - line_width=line_width, - pairwise_sort=pairwise_sort, - style=style, - guidance=False, - show_rank_probabilities=show_rank_probabilities, - ) + if bundle.benchmark.n_templates > 1: + for model_label, model_bundle in bundle.per_model.items(): + print() + _print_loud_section(f"Per-Model Summary: {model_label}") + _print_bundle_summary( + model_bundle, + top_pairwise=top_pairwise, + line_width=line_width, + pairwise_sort=pairwise_sort, + style=style, + guidance=False, + show_rank_probabilities=show_rank_probabilities, + ) print() _print_loud_section("Cross-Model Ranking (all model/template pairs)") _print_model_template_matrix(bundle) - # Shared by the (optional) P(Best) block below and the unconditional - # "Mean Performance" listing further down -- computed once here so the - # latter doesn't depend on show_rank_probabilities being True. - p_best = bundle.cross_model.rank_dist.p_best - expected_ranks = bundle.cross_model.rank_dist.expected_ranks - rank_labels = bundle.cross_model.rank_dist.labels + # The unconditional "Mean Performance" listing orders by mean, so it + # needs nothing from the rank distribution. P(Best)/E[Rank] are read + # only inside the show_rank_probabilities block below, which keeps the + # rank bootstrap genuinely opt-in. + rank_labels = bundle.cross_model.labels rank_pairs = [_split_model_template_label(label) for label in rank_labels] rank_bar_width = 14 n_ranked_items = len(rank_labels) model_col_width = min(24, max(len(model) for model, _ in rank_pairs) + 2) template_col_width = min(24, max(len(template) for _, template in rank_pairs) + 2) - top_indices = np.argsort(-p_best) + top_indices = _display_order(bundle.cross_model) n_show = len(top_indices) if show_rank_probabilities: + p_best = bundle.cross_model.rank_dist.p_best + expected_ranks = bundle.cross_model.rank_dist.expected_ranks + pbest_indices = np.argsort(-p_best) _print_subsection(f"--- Rank Probabilities: All {n_show} by P(Best) ({_rank_method_label(bundle.cross_model)}) ---") print( f" {'Model':<{model_col_width}s} " @@ -713,7 +743,7 @@ def _print_multi_model_summary( f"{'P(Best)':>9s} {'':<{rank_bar_width}s} " f"{'E[Rank]':>9s} {'':<{rank_bar_width}s}" ) - for idx in top_indices[:n_show]: + for idx in pbest_indices[:n_show]: model_label, template_label = rank_pairs[idx] model_label = _truncate_label(model_label, model_col_width) template_label = _truncate_label(template_label, template_col_width) @@ -759,9 +789,10 @@ def _print_multi_model_summary( print() _print_subsection(f"--- {stat_label} Performance: All {n_show} (marginal CIs) ---") _ci_legend_mm = _legend_ci_label(style, int(round((1 - get_alpha_ci()) * 100)), cross_rob.multi_ci is not None) + _mean_marker_mm = _mean_marker_legend(style, stat_label.lower()) print( f" axis: [{ma_low:.3f}, {ma_high:.3f}] " - f"(· ±1σ, {_ci_legend_mm}, ● {stat_label.lower()}, │ {ref_label_str})" + f"(· ±1σ, {_ci_legend_mm}{_mean_marker_mm}, │ {ref_label_str})" ) print( f" {'Model':<{model_col_width}s} " @@ -824,7 +855,7 @@ def _print_model_template_matrix(bundle: MultiModelBundle) -> None: # Labels are formatted as "model / template" by get_flat_result(). cell_mean: dict[tuple[str, str], float] = {} for label, m in zip( - cross.rank_dist.labels, + cross.labels, cross.robustness.mean, ): parts = label.split(" / ", 1) @@ -841,7 +872,7 @@ def _print_model_template_matrix(bundle: MultiModelBundle) -> None: # estimate should not read as a decisive winner when the pairwise CIs # show it isn't distinguishable from its neighbors; that would defeat # the point of reporting calibrated intervals in the first place. - cross_labels_all = list(cross.rank_dist.labels) + cross_labels_all = list(cross.labels) cross_means_all = cross.robustness.mean sort_idx = list(np.argsort(-cross_means_all)) labels_sorted = [cross_labels_all[i] for i in sort_idx] @@ -907,7 +938,7 @@ def _fmt_cell(mdl: str, t: str) -> str: def _print_cross_model_executive_summary(bundle: MultiModelBundle) -> None: """Print executive leaderboard for cross-model (model/template) pairs.""" cross = bundle.cross_model - labels = list(cross.rank_dist.labels) + labels = list(cross.labels) n = len(labels) if n < 2: return @@ -936,7 +967,7 @@ def _print_cross_model_executive_summary(bundle: MultiModelBundle) -> None: global_cell_max = float(sv.per_cell_seed_std.max()) if has_stability else 0.0 _print_subsection("--- Executive Summary (Cross-model pair leaderboard) ---") - _cross_ci_header = "Wilson CI" if _uses_wilson_ci(cross) else "CI" + _cross_ci_header = "Wilson-flat CI" if _uses_wilson_ci(cross) else "CI" header = ( f" {'Model':<{model_w}s}" f" {'Template':<{template_w}s}" @@ -1010,81 +1041,42 @@ def _print_cross_model_executive_summary(bundle: MultiModelBundle) -> None: # Single-model bundle summary # --------------------------------------------------------------------------- -def _print_pairwise_section( +def _prepare_paired_pairwise_rows( bundle: "AnalysisBundle", *, - top_pairwise: int = None, - line_width: int, - sort: bool = True, - p_value_method: Optional[str] = None, - pairwise_sort: Literal["grouped", "significance"] = "grouped", - style: Literal["line", "gradient"] = "gradient", -) -> None: - """Print the pairwise comparisons block for an AnalysisBundle. - - Extracted so it can be reused by both the full bundle summary and the - focused CompareReport summary without duplicating code. - - Parameters - ---------- - p_value_method : str or None - Which p-value column to show. ``'auto'`` (default) picks the method - most commensurate with the CI: bootstrap p-value for bootstrap CI - paths, exact-test p-value for newcombe/fisher/sign paths, and - Wilcoxon signed-rank for LMM/other paths. Explicit choices: - ``'boot'`` (result.p_value), ``'wsr'`` (Wilcoxon signed-rank), - ``'nem'`` (Nemenyi post-hoc). Pass ``None`` to suppress p-values. - pairwise_sort : {"grouped", "significance"} - Sorting strategy for pairwise rows. ``"grouped"`` keeps a stable - left-item grouping, while ``"significance"`` sorts by p-value then - absolute effect size. + p_value_method: Optional[str], + sort: bool, + pairwise_sort: Literal["grouped", "significance"], +) -> tuple[Optional[list[dict]], dict]: + """Normalize an AnalysisBundle's pairwise results into the common row + shape :func:`_print_pairwise_section` renders, plus metadata describing + which optional columns/sections apply. + + Extracted from the pre-unification body of ``_print_pairwise_section`` + verbatim (same swap/canonicalization/sort/p-value-method-resolution + logic) so paired-path behavior is unchanged -- only repackaged so the + row-rendering core can be shared with the unpaired path via + :func:`_prepare_unpaired_pairwise_rows`. Returns ``(None, {})`` when + there's exactly one entity (nothing to compare), matching the old + early-return. """ pair_item_col_width = 24 - pair_stat_col_width = 8 - pair_ci_col_width = 9 - pair_sigma_col_width = 8 - # Determine statistic label from the first result (all share the same statistic). first_result = next(iter(bundle.pairwise.results.values()), None) + if first_result is None: + return None, {} pair_stat_label = first_result.statistic.capitalize() if first_result else "Mean" - # Detect CI method family for auto p-value selection. - is_newcombe_pairwise = ( - first_result is not None - and "newcombe" in first_result.test_method.lower() - ) - is_sign_pairwise = ( - first_result is not None - and "sign test" in first_result.test_method.lower() - ) - is_bootstrap_path = ( - first_result is not None - and "bootstrap" in first_result.test_method.lower() - ) - - # Whether simultaneous max-T CIs were used (affects p-value source for bootstrap paths). + is_newcombe_pairwise = "newcombe" in first_result.test_method.lower() + is_sign_pairwise = "sign test" in first_result.test_method.lower() + is_bootstrap_path = "bootstrap" in first_result.test_method.lower() using_max_t = bundle.pairwise.simultaneous_ci_method == "max_t" - - # Whether Romano-Wolf step-down was the FWER correction that actually - # fired for this bundle (only known post-hoc, once all_pairwise() has - # run -- see _resolve_p_value_method's docstring). Guarded on - # len(results) > 1 because all_pairwise() still resolves and reports - # correction_method="romano_wolf" for a single-pair (k=2) bundle even - # though nothing is actually corrected there (no family to correct - # across) -- Wilcoxon must stay the default at k=2 regardless of N. is_romano_wolf_active = ( bundle.pairwise.correction_method == "romano_wolf" and len(bundle.pairwise.results) > 1 ) - # Resolve the effective p-value source and column header. if p_value_method == "auto": - # Wilcoxon signed-ranks is the default pairwise test for any k >= 2 - # (fig:fwer-decision-tree's standard workflow), *except* when - # Romano-Wolf step-down is the resolved correction -- it has no - # Wilcoxon-compatible joint construction (see - # romano_wolf_stepdown_pvalues's docstring) and produces its own - # mean-based bootstrap-t p-value instead, which is what's shown here. if is_romano_wolf_active: eff_p_source, p_col_header = "boot", "p (RW)" else: @@ -1099,30 +1091,17 @@ def _print_pairwise_section( else: # None eff_p_source, p_col_header = None, None - pair_p_col_width = max(10, len(p_col_header)) if p_col_header else 0 - - _pairwise_header_method = first_result.test_method corr = bundle.pairwise.correction_method - if eff_p_source is not None and corr and corr != "none": - _pairwise_header_method += f" ({corr}-corrected p-values)" sim_ci_method = bundle.pairwise.simultaneous_ci_method - if sim_ci_method == "max_t": - _pairwise_header_method += " (simultaneous CIs computed with max-T)" - elif sim_ci_method == "bonferroni": - _pairwise_header_method += " (simultaneous CIs computed with Bonferroni)" - _print_subsection(f"--- Pairwise Comparisons ({_pairwise_header_method}) ---") + _pretty_ci_method = first_result.test_method[0].upper() + first_result.test_method[1:] pair_results = list(bundle.pairwise.results.values()) # Canonical left/right ordering based on expected-rank order keeps rows # readable by preventing arbitrary A/B flips between adjacent rows. + _labels_for_order = list(bundle.labels) rank_order = { - label: idx - for idx, (_, label) in enumerate( - sorted( - zip(bundle.rank_dist.expected_ranks, bundle.rank_dist.labels), - key=lambda item: (float(item[0]), item[1]), - ) - ) + _labels_for_order[i]: idx + for idx, i in enumerate(_display_order(bundle)) } if pair_results: @@ -1131,7 +1110,7 @@ def _print_pairwise_section( ) pair_item_col_width = min(30, max(12, max_label_len + 2)) - normalized_rows = [] + rows = [] for result in pair_results: a = result.template_a b = result.template_b @@ -1164,9 +1143,21 @@ def _print_pairwise_section( right_pos = pos_b swapped_multi_ci = result.multi_ci + if eff_p_source in {"max_t", "boot"}: + display_p = result.p_value + elif eff_p_source == "wsr": + display_p = result.wilcoxon_p + elif eff_p_source == "nem": + display_p = ( + bundle.pairwise.friedman.get_nemenyi_p(str(left_item), str(right_item)) + if bundle.pairwise.friedman is not None else None + ) + else: + display_p = None + # binary_confusion is symmetric in n11/n00; n10/n01 swap with direction # but for the bar we only need n_split = n10+n01, which is invariant. - normalized_rows.append( + rows.append( { "left": left_item, "right": right_item, @@ -1176,9 +1167,9 @@ def _print_pairwise_section( "ci_low": ci_low, "ci_high": ci_high, "std_diff": float(result.std_diff), - "rank_biserial": rank_biserial, - "p_value": result.p_value, - "wilcoxon_p": result.wilcoxon_p, + "es_value": rank_biserial, + "p_value": result.p_value, # sort key -- always the bootstrap p, regardless of eff_p_source + "display_p": display_p, "agreement_mcc": result.agreement_mcc, "binary_confusion": result.binary_confusion, "multi_ci": swapped_multi_ci, @@ -1190,8 +1181,8 @@ def _print_pairwise_section( if sort: if pairwise_sort == "grouped": - normalized_rows = sorted( - normalized_rows, + rows = sorted( + rows, key=lambda row: ( row["left_pos"], row["right_pos"], @@ -1200,8 +1191,8 @@ def _print_pairwise_section( ), ) else: - normalized_rows = sorted( - normalized_rows, + rows = sorted( + rows, key=lambda row: ( row["p_value"], -abs(row["point_diff"]), @@ -1209,14 +1200,10 @@ def _print_pairwise_section( row["right_pos"], ), ) - # By default, print all pairs unless top_pairwise is set. - if top_pairwise is None: - max_pairs = len(normalized_rows) - else: - max_pairs = max(0, min(top_pairwise, len(normalized_rows))) - # Friedman omnibus line (printed before the interval plot when pairs exist). - if max_pairs > 0 and bundle.pairwise.friedman is not None: + def _friedman_line() -> None: + if bundle.pairwise.friedman is None: + return fr = bundle.pairwise.friedman fr_p_str = _format_p_value(fr.p_value) fr_p_color = _BRIGHT_GREEN if fr.p_value <= 0.05 else _YELLOW @@ -1229,6 +1216,290 @@ def _print_pairwise_section( if fr.p_value > 0.05: print(f" {_YELLOW}[!] Friedman p > 0.05: no significant omnibus effect — treat pairwise results with caution.{_RESET}") + def _footer(_rows: list[dict], _max_pairs: int) -> None: + print(f"{_DIM} ES = Effect Size (r_rb) = rank biserial correlation (small≈0.1, medium≈0.3, large≈0.5){_RESET}") + + # Short p-value-method name (no correction detail -- that's stated + # separately on the FWER-corrections line below) for the explicit + # methods summary. The fuller descriptive line further down (with + # correction detail folded in) still prints too. + p_value_method_label = None + if eff_p_source in {"max_t", "boot"}: + if is_romano_wolf_active and eff_p_source == "boot": + p_value_method_label = "Romano-Wolf step-down" + elif is_newcombe_pairwise: + p_value_method_label = "McNemar mid-p test" + elif is_sign_pairwise: + p_value_method_label = "Paired sign test" + elif eff_p_source == "max_t": + p_value_method_label = "Max-T bootstrap" + else: + p_value_method_label = "Bootstrap" + elif eff_p_source == "wsr": + p_value_method_label = "Wilcoxon signed-rank" + elif eff_p_source == "nem": + p_value_method_label = "Nemenyi post-hoc" + + # Explicit methods summary, directly above the p-value-method detail + # line -- mirrors the matplotlib forest plot's subtitle, stated + # plainly rather than nested into the section header. Two lines: + # (1) CI method / p-value method / alpha, (2) simultaneous-CI method + # and the FWER correction applied to p-values, broken out separately + # since they can use different correction methods. Dimmed along with + # the rest of this footnote block (ES=, p-value detail, stars:) -- + # methods detail, not part of the data itself. + _line1 = [f"CI method: {_pretty_ci_method}"] + if p_value_method_label: + _line1.append(f"p-value method: {p_value_method_label}") + _line1.append(f"α={get_alpha_ci():g}") + print(f"{_DIM} {' | '.join(_line1)}{_RESET}") + + _line2 = [f"Simultaneous CI method: {_pretty_simultaneous_ci(sim_ci_method)}"] + if p_value_method_label: + _line2.append(f"FWER correction for p-values: {_pretty_correction(corr)}") + print(f"{_DIM} {' | '.join(_line2)}{_RESET}") + + if eff_p_source in {"max_t", "boot"}: + if is_romano_wolf_active and eff_p_source == "boot": + print(f"{_DIM} {p_col_header} = Romano-Wolf step-down (FWER-controlled){_RESET}") + elif is_newcombe_pairwise: + print(f"{_DIM} {p_col_header} = McNemar mid-p test (two-sided, uncorrected){_RESET}") + elif is_sign_pairwise: + print(f"{_DIM} {p_col_header} = paired sign test (two-sided exact, ties dropped, uncorrected){_RESET}") + elif eff_p_source == "max_t": + print(f"{_DIM} {p_col_header} = max-T bootstrap p-value (FWER-controlled, commensurate with simultaneous CIs){_RESET}") + else: + print(f"{_DIM} {p_col_header} = bootstrap p-value ({bundle.pairwise.correction_method}-corrected){_RESET}") + elif eff_p_source == "wsr": + ppi_note = ", PPI-corrected" if getattr(bundle, "ppi_applied", False) else "" + print(f"{_DIM} {p_col_header} = Wilcoxon signed-rank ({bundle.pairwise.correction_method}-corrected{ppi_note}){_RESET}") + elif eff_p_source == "nem": + print(f"{_DIM} {p_col_header} = Nemenyi post-hoc (Friedman-based, FWER-controlled){_RESET}") + if eff_p_source is not None: + print(f"{_DIM} stars: * p<0.01, ** p<0.001, *** p<0.0001{_RESET}") + print() + _cd_labels = list(bundle.labels) + labels_sorted = [_cd_labels[i] for i in _display_order(bundle)] + _print_critical_difference_groups( + bundle.pairwise, + labels_sorted=labels_sorted, + p_source="bootstrap", + ) + + meta = { + "section_header": f"--- Pairwise Comparisons ({_pretty_ci_method} CIs) ---", + "pair_stat_label": pair_stat_label, + "pair_item_col_width": pair_item_col_width, + "effect_label": "Left - Right", + "es_label": "ES", + "p_col_header": p_col_header, + "friedman_line_fn": _friedman_line, + "footer_fn": _footer, + } + return rows, meta + + +_FAMILY_DISPLAY_UNPAIRED = { + "binary_proportion": "proportion difference (Δp)", + "rank_based": "stochastic dominance (θ = P(a>b))", +} +_ESTIMAND_LABEL_UNPAIRED = {"mean_diff": "Δ", "dominance": "θ"} + + +def _prepare_unpaired_pairwise_rows( + result: "GroupComparisonResult", + *, + sort: bool, + pairwise_sort: Literal["grouped", "significance"], +) -> tuple[list[dict], dict]: + """Normalize a GroupComparisonResult's pairwise results into the same + row shape :func:`_prepare_paired_pairwise_rows` produces. + + The between-subjects engine always uses exactly one FWER scheme + (Bonferroni CI + Holm p) -- no Wilcoxon/Romano-Wolf/Newcombe/sign-test/ + max-T/Nemenyi method-family detection needed, and no ranking bootstrap + to derive an alternate canonical left/right order from (natural + factor-level order, i.e. ``result.labels``, is already canonical). + ``point_diff``/``ci_low``/``ci_high`` are shifted by each pair's + ``null_value`` (0.5 for the rank-based dominance family, 0.0 for the + binary mean-difference family) so the shared axis/bar-rendering math in + :func:`_print_pairwise_section` -- which assumes a signed quantity + centered at zero, same convention the paired path's own "Left - Right" + difference already has -- works identically for both estimand kinds. + """ + show_p = result.show_p_values + n_pairs = len(result.pairwise) + null_value = result.pairwise[0].null_value if result.pairwise else 0.0 + estimand = result.pairwise[0].estimand if result.pairwise else "mean_diff" + est_symbol = _ESTIMAND_LABEL_UNPAIRED.get(estimand, "Δ") + # A shifted dominance probability (null=0.5) is a deviation, not the raw + # estimand -- label it "Δθ" so the column header doesn't silently claim + # to show raw θ. A mean/proportion difference (null=0.0) is unaffected + # by the shift, so its existing "Δ" label already describes it exactly. + pair_stat_label = f"Δ{est_symbol}" if null_value != 0.0 else est_symbol + + # For the rank-based (dominance) family, Δθ alone doesn't say how far + # apart the groups are on the metric's own scale -- e.g. a 1-5 Likert + # score. Surface each pair's raw mean difference too (point estimate + # only, no separate CI -- same convention the paired path's own "ES" + # rank-biserial column uses), reusing the marginal means already + # computed for the "Mean Performance" section above this table. + mean_by_label = {g.label: g.mean for g in result.groups} + show_mean_diff = result.pairwise and result.pairwise[0].estimand == "dominance" + + label_index = {lbl: i for i, lbl in enumerate(result.labels)} + rows = [] + for p in result.pairwise: + row = { + "left": p.label_a, + "right": p.label_b, + "left_pos": label_index.get(p.label_a, 0), + "right_pos": label_index.get(p.label_b, 0), + "point_diff": p.point_estimate - null_value, + "ci_low": p.ci_low - null_value, + "ci_high": p.ci_high - null_value, + "std_diff": 0.0, # no ±1σ "spread" concept for a pairwise estimate itself + "p_value": p.p_value, + "display_p": p.p_value if show_p else None, + "multi_ci": None, + } + if show_mean_diff: + row["es_value"] = mean_by_label[p.label_a] - mean_by_label[p.label_b] + rows.append(row) + + if pairwise_sort not in {"grouped", "significance"}: + raise ValueError("pairwise_sort must be 'grouped' or 'significance'.") + if sort: + if pairwise_sort == "grouped": + rows = sorted( + rows, + key=lambda row: (row["left_pos"], row["right_pos"], row["p_value"], -abs(row["point_diff"])), + ) + else: + rows = sorted( + rows, + key=lambda row: (row["p_value"], -abs(row["point_diff"]), row["left_pos"], row["right_pos"]), + ) + + def _footer(_rows: list[dict], _max_pairs: int) -> None: + if n_pairs > 1 and show_p: + print( + f" {_DIM}Verdict reflects the {result.ci_correction}-corrected CI; p is " + f"independently {result.pvalue_correction}-corrected -- the two can rarely " + f"disagree right at the boundary, since they're different (both valid) " + f"FWER corrections.{_RESET}" + ) + # Critical-difference rank bands, shared with the paired path's own + # (see _prepare_paired_pairwise_rows) -- reuses + # _critical_difference_groups/_print_critical_difference_groups + # unmodified via _GroupDiffResultsAsPairwiseMatrix, an adapter whose + # .simultaneous_ci_method sentinel routes it through the same + # CI-exclusion significance check GroupDiffResult.significant + # already uses (this engine has no p-value-threshold alternative + # the way the paired path's Wilcoxon/Nemenyi paths do). Sorted by + # mean descending -- there's no ranking bootstrap here, so this is + # the same "best first" order the executive summary below uses. + from evalstats.core.unpaired import _GroupDiffResultsAsPairwiseMatrix + labels_sorted = [g.label for g in sorted(result.groups, key=lambda g: -g.mean)] + print() + _print_critical_difference_groups( + _GroupDiffResultsAsPairwiseMatrix(result.pairwise), + labels_sorted=labels_sorted, + alpha=result.alpha, + p_source="bootstrap", + ) + + label_width = min(24, max(8, max((len(g) for g in result.labels), default=8))) + correction_note = "" + if n_pairs > 1: + correction_note = f", {result.ci_correction} CI" + ( + f"/{result.pvalue_correction} p (family of {n_pairs})" if show_p else "" + ) + meta = { + "section_header": f"--- Pairwise Comparisons ({_FAMILY_DISPLAY_UNPAIRED[result.family]}{correction_note}) ---", + "pair_stat_label": pair_stat_label, + "pair_item_col_width": label_width, + "effect_label": "Left - Right", + # Only the dominance family gets a secondary raw-mean-difference + # column -- the binary/mean_diff family's primary column already + # *is* the raw difference (Δp), so a second copy would be redundant. + "es_label": "Δmean" if show_mean_diff else None, + "p_col_header": "p" if show_p else None, + "friedman_line_fn": None, + "footer_fn": _footer, + } + return rows, meta + + +def _print_pairwise_section( + bundle_or_result, + *, + top_pairwise: int = None, + line_width: int, + sort: bool = True, + p_value_method: Optional[str] = None, + pairwise_sort: Literal["grouped", "significance"] = "grouped", + style: Literal["line", "gradient"] = "gradient", +) -> None: + """Print the pairwise comparisons block for either a paired + ``AnalysisBundle`` or an unpaired ``GroupComparisonResult``. + + Shared by the paired path's full bundle summary and the unpaired path's + between-subjects summary (``print_group_comparison_summary``, also in + this module) -- the axis/legend/header/row-rendering core is identical + machinery either way (through ``_choose_interval_line``), + so one function renders both instead of two independently-drifting + implementations. What genuinely differs between the two designs (six + CI/p-value method families + Friedman/Nemenyi + critical-difference + rank bands for paired; one fixed Bonferroni-CI/Holm-p scheme for + unpaired, no ranking bootstrap) is resolved up front by + :func:`_prepare_paired_pairwise_rows`/:func:`_prepare_unpaired_pairwise_rows` + into the common row + metadata shape this function actually renders. + The Behavioral Agreement subsection (McNemar-style pass/fail bars) is + NOT handled here -- see :func:`_print_behavioral_agreement_section`, + paired-only, since ``agreement_mcc``/``binary_confusion`` need the same + item scored by both entities, which has no between-subjects equivalent. + + Parameters + ---------- + p_value_method : str or None + Paired-only. Which p-value column to show. See the pre-unification + docstring text preserved in :func:`_prepare_paired_pairwise_rows` + for the full method-selection semantics. Ignored for unpaired data + (that path's p-value display is controlled by + ``GroupComparisonResult.show_p_values`` instead). + pairwise_sort : {"grouped", "significance"} + Sorting strategy for pairwise rows. ``"grouped"`` keeps a stable + left-item grouping, while ``"significance"`` sorts by p-value then + absolute effect size. + """ + if isinstance(bundle_or_result, AnalysisBundle): + rows, meta = _prepare_paired_pairwise_rows( + bundle_or_result, p_value_method=p_value_method, sort=sort, pairwise_sort=pairwise_sort, + ) + if rows is None: + return + else: + rows, meta = _prepare_unpaired_pairwise_rows( + bundle_or_result, sort=sort, pairwise_sort=pairwise_sort, + ) + + _print_subsection(meta["section_header"]) + + if top_pairwise is None: + max_pairs = len(rows) + else: + max_pairs = max(0, min(top_pairwise, len(rows))) + + if max_pairs > 0 and meta["friedman_line_fn"] is not None: + meta["friedman_line_fn"]() + + pair_item_col_width = meta["pair_item_col_width"] + pair_stat_label = meta["pair_stat_label"] + es_label = meta["es_label"] + p_col_header = meta["p_col_header"] + pair_p_col_width = max(10, len(p_col_header)) if p_col_header else 0 + if max_pairs > 0: pair_max_abs = max( 1e-12, @@ -1240,194 +1511,206 @@ def _print_pairwise_section( abs(float(row["point_diff"] - row["std_diff"])), abs(float(row["point_diff"] + row["std_diff"])), ) - for row in normalized_rows[:max_pairs] + for row in rows[:max_pairs] ), ) pair_low = -pair_max_abs pair_high = pair_max_abs + # Clamp the shared axis ONCE for the whole block, not per row. A single + # unbounded pair (paired._degenerate_pair_ci on zero-variance + # differences with no declared score_range) makes pair_max_abs + # infinite, and letting each row fall back to its own finite window + # would silently put the rows on different scales -- two intervals of + # identical width drawn at different lengths -- while the legend still + # advertises one axis. The whole point of a shared axis is that bars + # are comparable down the column, so the finite rows must keep sharing + # it and the legend must report the axis actually drawn. + if not (np.isfinite(pair_low) and np.isfinite(pair_high)): + _cands: list[float] = [] + for row in rows[:max_pairs]: + for key in ("point_diff", "ci_low", "ci_high"): + _cands.append(float(row[key])) + _cands.append(float(row["point_diff"] - row["std_diff"])) + _cands.append(float(row["point_diff"] + row["std_diff"])) + _cands.append(0.0) # the zero reference is always drawn + pair_low, pair_high = _finite_axis(pair_low, pair_high, tuple(_cands)) # gradient mode always produces a gradient (synthesized when multi_ci is absent) _any_multi_ci = (style == "gradient") or any( - row.get("multi_ci") is not None for row in normalized_rows[:max_pairs] + row.get("multi_ci") is not None for row in rows[:max_pairs] ) _pair_ci_pct = int(round((1 - get_alpha_ci()) * 100)) _pair_ci_legend = _legend_ci_label(style, _pair_ci_pct, _any_multi_ci) + _pair_mean_marker = _mean_marker_legend(style, pair_stat_label.lower()) print( - f" legend: (· ±1σ, {_pair_ci_legend}, ● {pair_stat_label.lower()}, │ zero) " + f" legend: (· ±1σ, {_pair_ci_legend}{_pair_mean_marker}, │ zero) " f"axis: [{pair_low:+.3f}, {pair_high:+.3f}] " - "effect: Left - Right" + f"effect: {meta['effect_label']}" ) header = ( f" {'Left':<{pair_item_col_width}s} {'Right':<{pair_item_col_width}s} " f"{'Interval Plot':<{line_width}s} " - f"{pair_stat_label:>{pair_stat_col_width}s} " - f"{'CI Low':>{pair_ci_col_width}s} {'CI High':>{pair_ci_col_width}s} " - f"{'ES':>{pair_sigma_col_width}s}" + f"{pair_stat_label:>8s} " + f"{'CI Low':>9s} {'CI High':>9s}" ) + if es_label: + header += f" {es_label:>8s}" if p_col_header: header += f" {p_col_header:>{pair_p_col_width}s}" print(header) - for row_data in normalized_rows[:max_pairs]: - line = _choose_interval_line( - mean=float(row_data["point_diff"]), - ci_low=float(row_data["ci_low"]), - ci_high=float(row_data["ci_high"]), - spread_low=float(row_data["point_diff"] - row_data["std_diff"]), - spread_high=float(row_data["point_diff"] + row_data["std_diff"]), - axis_low=pair_low, - axis_high=pair_high, - width=line_width, - style=style, - multi_ci=row_data.get("multi_ci"), - ) - left_label = _truncate_label(str(row_data["left"]), pair_item_col_width) - right_label = _truncate_label(str(row_data["right"]), pair_item_col_width) - d_val = float(row_data["rank_biserial"]) - d_str = f"{d_val:>{pair_sigma_col_width}.3f}" - row = ( - f" {left_label:<{pair_item_col_width}s} " - f"{right_label:<{pair_item_col_width}s} " - f"{line:<{line_width}s} " - f"{float(row_data['point_diff']):+{pair_stat_col_width}.4f} " - f"{float(row_data['ci_low']):+{pair_ci_col_width}.4f} " - f"{float(row_data['ci_high']):+{pair_ci_col_width}.4f} " - f"{d_str}" - ) - if eff_p_source in {"max_t", "boot"}: - p_val = row_data["p_value"] - elif eff_p_source == "wsr": - p_val = row_data["wilcoxon_p"] - elif eff_p_source == "nem": - p_val = ( - bundle.pairwise.friedman.get_nemenyi_p(str(row_data["left"]), str(row_data["right"])) - if bundle.pairwise.friedman is not None else None + for row_data in rows[:max_pairs]: + line = _choose_interval_line( + mean=float(row_data["point_diff"]), + ci_low=float(row_data["ci_low"]), + ci_high=float(row_data["ci_high"]), + spread_low=float(row_data["point_diff"] - row_data["std_diff"]), + spread_high=float(row_data["point_diff"] + row_data["std_diff"]), + axis_low=pair_low, + axis_high=pair_high, + width=line_width, + style=style, + multi_ci=row_data.get("multi_ci"), ) - else: - p_val = None - if eff_p_source is not None: - row += f" {_format_p_value(p_val):>{pair_p_col_width}s}" - print(row) + left_label = _truncate_label(str(row_data["left"]), pair_item_col_width) + right_label = _truncate_label(str(row_data["right"]), pair_item_col_width) + row_str = ( + f" {left_label:<{pair_item_col_width}s} " + f"{right_label:<{pair_item_col_width}s} " + f"{line:<{line_width}s} " + f"{float(row_data['point_diff']):+8.4f} " + f"{float(row_data['ci_low']):+9.4f} " + f"{float(row_data['ci_high']):+9.4f}" + ) + if es_label: + row_str += f" {float(row_data['es_value']):>8.3f}" + if p_col_header: + row_str += f" {_format_p_value(row_data.get('display_p')):>{pair_p_col_width}s}" + print(row_str) if max_pairs == 0: print(" (no pairwise comparisons)") - elif max_pairs > 0: - print(f"{_DIM} ES = Effect Size (r_rb) = rank biserial correlation (small≈0.1, medium≈0.3, large≈0.5){_RESET}") - if eff_p_source in {"max_t", "boot"}: - if is_romano_wolf_active and eff_p_source == "boot": - print(f" {p_col_header} = Romano-Wolf bootstrap step-down (FWER-controlled; no Wilcoxon-compatible joint form exists, see romano_wolf_stepdown_pvalues)") - elif is_newcombe_pairwise: - print(f" {p_col_header} = McNemar exact test (two-sided, uncorrected)") - elif is_sign_pairwise: - print(f" {p_col_header} = paired sign test (two-sided exact, ties dropped, uncorrected)") - elif eff_p_source == "max_t": - print(f" {p_col_header} = max-T bootstrap p-value (FWER-controlled, commensurate with simultaneous CIs)") - else: - print(f" {p_col_header} = bootstrap p-value ({bundle.pairwise.correction_method}-corrected)") - elif eff_p_source == "wsr": - ppi_note = ", PPI-corrected" if getattr(bundle, "ppi_applied", False) else "" - print(f" {p_col_header} = Wilcoxon signed-rank ({bundle.pairwise.correction_method}-corrected{ppi_note})") - elif eff_p_source == "nem": - print(f" {p_col_header} = Nemenyi post-hoc (Friedman-based, FWER-controlled)") - if eff_p_source is not None: - print(" stars: * p<0.01, ** p<0.001, *** p<0.0001") - print() - labels_sorted = [ - label - for _, label in sorted( - zip(bundle.rank_dist.expected_ranks, bundle.rank_dist.labels), - key=lambda item: (float(item[0]), item[1]), - ) - ] - _print_critical_difference_groups( - bundle.pairwise, - labels_sorted=labels_sorted, - p_source="bootstrap", - ) + else: + meta["footer_fn"](rows, max_pairs) + + +def _print_behavioral_agreement_section( + bundle: "AnalysisBundle", + *, + top_pairwise: int = None, + sort: bool = True, + pairwise_sort: Literal["grouped", "significance"] = "grouped", +) -> None: + """Print the Pass/Fail Agreement (McNemar-style) subsection for binary + paired data. Paired-only, by design: ``agreement_mcc``/``binary_confusion`` + require the *same item* scored by both entities to know whether they got + it right or wrong together, which has no between-subjects equivalent + (disjoint groups have no shared items at all). + """ + rows, meta = _prepare_paired_pairwise_rows( + bundle, p_value_method=None, sort=sort, pairwise_sort=pairwise_sort, + ) + if rows is None: + return + if top_pairwise is None: + max_pairs = len(rows) + else: + max_pairs = max(0, min(top_pairwise, len(rows))) - # --- Behavioral Agreement subsection (binary data only) --- agr_rows = [ - r for r in normalized_rows[:max_pairs] + r for r in rows[:max_pairs] if r.get("agreement_mcc") is not None and r.get("binary_confusion") is not None ] agr_rows.sort(key=lambda row: float(row["agreement_mcc"]), reverse=True) - if agr_rows: - bar_width = 20 - mcc_col_width = 6 - strength_col_width = 14 - agr_item_col_width = pair_item_col_width - - _print_subsection("\n--- Pass/Fail Agreement ---") - print(f" Are pairs getting the same items right and wrong?") - print(f" \u2588 both pass \u2591 both fail {_BRIGHT_RED}\u2592{_RESET} disagree " - f"(MCC: 1=identical, 0=random, \u22121=opposite)") - print() + if not agr_rows: + return - agr_header = ( - f" {'Left':<{agr_item_col_width}s} {'Right':<{agr_item_col_width}s}" - f" {'Plot':<{bar_width+2}s} {'MCC':>{mcc_col_width}s}" - f" {'Agreement':<{strength_col_width}s} Interpretation" + bar_width = 20 + mcc_col_width = 6 + strength_col_width = 14 + agr_item_col_width = meta["pair_item_col_width"] + + _print_subsection("\n--- Pass/Fail Agreement ---") + print(f" Are pairs getting the same items right and wrong?") + print(f" █ both pass ░ both fail {_BRIGHT_RED}▒{_RESET} disagree " + f"(MCC: 1=identical, 0=random, −1=opposite)") + print() + + agr_header = ( + f" {'Left':<{agr_item_col_width}s} {'Right':<{agr_item_col_width}s}" + f" {'Plot':<{bar_width+2}s} {'MCC':>{mcc_col_width}s}" + f" {'Agreement':<{strength_col_width}s} Interpretation" + ) + print(agr_header) + + for row in agr_rows: + n11, n10, n01, n00 = row["binary_confusion"] + bar = _agreement_bar(n11, n10, n01, n00, width=bar_width) + mcc = row["agreement_mcc"] + left_label = _truncate_label(str(row["left"]), agr_item_col_width) + right_label = _truncate_label(str(row["right"]), agr_item_col_width) + print( + f" {left_label:<{agr_item_col_width}s} {right_label:<{agr_item_col_width}s}" + f" [{bar}] {mcc:>+{mcc_col_width}.3f}" + f" {_mcc_strength(mcc):<{strength_col_width}s} {_mcc_interpretation(mcc)}" ) - print(agr_header) - - for row in agr_rows: - n11, n10, n01, n00 = row["binary_confusion"] - bar = _agreement_bar(n11, n10, n01, n00, width=bar_width) - mcc = row["agreement_mcc"] - left_label = _truncate_label(str(row["left"]), agr_item_col_width) - right_label = _truncate_label(str(row["right"]), agr_item_col_width) - print( - f" {left_label:<{agr_item_col_width}s} {right_label:<{agr_item_col_width}s}" - f" [{bar}] {mcc:>+{mcc_col_width}.3f}" - f" {_mcc_strength(mcc):<{strength_col_width}s} {_mcc_interpretation(mcc)}" - ) - print() + print() def _print_mean_advantage( - bundle: "AnalysisBundle", *, + labels: list[str], + mean: np.ndarray, + std: np.ndarray, + ci_low: np.ndarray, + ci_high: np.ndarray, + multi_ci_per_entity: list, + resolved_ci_method: str, item_singular: str = "template", line_width: int, template_col_width: int = 24, style: Literal["line", "gradient"] = "gradient", ) -> None: - """Print the absolute performance interval-plot table for an AnalysisBundle. + """Print the absolute performance interval-plot table for a set of entities. Shows each entity's absolute mean with marginal bootstrap CIs (single-sample, independent per entity) and intrinsic spread bands. A reference line marks - the grand mean (or the specified reference entity) for visual comparison. + the grand mean of the passed entities for visual comparison. + + Shared by the paired path (``_print_bundle_summary``, entities all scored + on the same items -- a ``RobustnessResult``) and the unpaired path + (``print_group_comparison_summary``, also in this module, disjoint + items per group -- a ``list[GroupStat]``) -- both reduce to the same "N + entities, each with a mean/CI/spread" shape by the time they call this, + so one function renders both instead of two independently-drifting + per-entity loops. ``multi_ci_per_entity`` takes an already-per-entity- + sliced list (``{alpha: (lo, hi)}`` or ``None`` per entity) rather than a + combined dict-of-arrays, so callers with either shape (a + ``RobustnessResult.multi_ci`` sliced via ``_rob_multi_ci_at``, or a + ``GroupStat.multi_ci`` list that's already this shape) both fit without + conversion inside this function. """ item_singular_title = item_singular.capitalize() stat_label = "Mean" - rob = bundle.robustness - ref_val = float(np.mean(rob.mean)) - - # Per-entity absolute values. - abs_means = np.array([float(rob.mean[i]) for i in range(len(rob.labels))]) - - abs_ci_lows = rob.ci_low - abs_ci_highs = rob.ci_high + mean = np.asarray(mean, dtype=float) + std = np.asarray(std, dtype=float) + ci_low = np.asarray(ci_low, dtype=float) + ci_high = np.asarray(ci_high, dtype=float) + ref_val = float(np.mean(mean)) # ±1σ spread around the absolute mean. - abs_sigma_lows = abs_means - rob.std - abs_sigma_highs = abs_means + rob.std + sigma_lows = mean - std + sigma_highs = mean + std # Axis bounds: cover means, CIs, and ±1σ spread. - all_vals = np.concatenate([ - abs_means, - abs_ci_lows, - abs_ci_highs, - abs_sigma_lows, - abs_sigma_highs, - ]) + all_vals = np.concatenate([mean, ci_low, ci_high, sigma_lows, sigma_highs]) val_range = float(np.max(all_vals) - np.min(all_vals)) pad = max(val_range * 0.05, 1e-4) ma_low = float(np.min(all_vals)) - pad ma_high = float(np.max(all_vals)) + pad - _ci_method = (bundle.resolved_ci_method or "").lower() - if _uses_wilson_ci(bundle): - ci_note = "Wilson CIs" + _ci_method = (resolved_ci_method or "").lower() + if _ci_method in {"wilson", "newcombe", "bayes_binary"}: + ci_note = "Wilson-flat CIs" elif _ci_method == "nig": ci_note = "marginal NIG CIs" elif _ci_method == "logit_t": @@ -1441,39 +1724,66 @@ def _print_mean_advantage( _print_subsection(f"--- {stat_label} Performance ({ci_note}) ---") ref_label = "grand mean" ci_pct = int(round((1 - get_alpha_ci()) * 100)) - _ci_legend_ma = _legend_ci_label(style, ci_pct, rob.multi_ci is not None) + _any_multi_ci = any(m is not None for m in multi_ci_per_entity) + _ci_legend_ma = _legend_ci_label(style, ci_pct, _any_multi_ci) + _mean_marker_ma = _mean_marker_legend(style, stat_label.lower()) print( f" axis: [{ma_low:.3f}, {ma_high:.3f}]" - f" (· ±1σ, {_ci_legend_ma}, ● {stat_label.lower()}, │ {ref_label})" + f" (· ±1σ, {_ci_legend_ma}{_mean_marker_ma}, │ {ref_label})" ) print( f" {item_singular_title:<{template_col_width}s} {'Interval Plot':<{line_width}s} {stat_label:>8s} " f"{'CI Low':>9s} {'CI High':>9s}" ) - for i, label in enumerate(rob.labels): + for i, label in enumerate(labels): template_label = _truncate_label(label, template_col_width) line = _choose_interval_line( - mean=abs_means[i], - ci_low=float(abs_ci_lows[i]), - ci_high=float(abs_ci_highs[i]), - spread_low=float(abs_sigma_lows[i]), - spread_high=float(abs_sigma_highs[i]), + mean=float(mean[i]), + ci_low=float(ci_low[i]), + ci_high=float(ci_high[i]), + spread_low=float(sigma_lows[i]), + spread_high=float(sigma_highs[i]), axis_low=ma_low, axis_high=ma_high, width=line_width, reference=ref_val, style=style, - multi_ci=_rob_multi_ci_at(rob.multi_ci, i), + multi_ci=multi_ci_per_entity[i], ) print( f" {template_label:<{template_col_width}s} " f"{line:<{line_width}s} " - f"{abs_means[i]:>7.3f} " - f"{float(abs_ci_lows[i]):>8.3f} " - f"{float(abs_ci_highs[i]):>8.3f}" + f"{float(mean[i]):>7.3f} " + f"{float(ci_low[i]):>8.3f} " + f"{float(ci_high[i]):>8.3f}" ) +def _print_ppi_banner(alignment_result) -> None: + """Print the standard "PPI-CORRECTED" banner + inline alignment report. + + Shared by the paired path's ``_print_bundle_summary`` and the unpaired + path's ``print_group_comparison_summary`` (also in this module) -- + previously two copy-pasted, near-identical blocks; unified so a banner + text/formatting change only needs to happen once. + """ + banner = "═" * 58 + print(f"{_BOLD}{_BRIGHT_MAGENTA}{banner}{_RESET}") + print( + f"{_BOLD}{_BRIGHT_MAGENTA}PPI-CORRECTED — every estimate below relies on the " + f"alignment report printed here first.{_RESET}" + ) + print(f"{_BOLD}{_BRIGHT_MAGENTA}{banner}{_RESET}") + if alignment_result is not None: + alignment_result.summary() + else: + print( + f"{_BOLD}{_BRIGHT_MAGENTA}(Alignment report unavailable -- run " + f"judge_alignment(...).summary() directly.){_RESET}" + ) + print() + + def _print_bundle_summary( bundle: AnalysisBundle, *, @@ -1492,7 +1802,9 @@ def _print_bundle_summary( ) -> None: if p_value_method is _UNSET: p_value_method = bundle.p_value_method - template_col_width = 24 + template_col_width = min( + 24, max(len(item_singular), max(len(l) for l in bundle.robustness.labels)) + 2 + ) print(f"Shape: {bundle.shape}") n_runs = bundle.benchmark.n_runs @@ -1505,7 +1817,10 @@ def _print_bundle_summary( ) print() - _print_subsection("--- Robustness ---") + if getattr(bundle, "ppi_applied", False): + _print_ppi_banner(bundle.alignment_result) + + _print_subsection("--- Descriptive Statistics ---") _rob_df = bundle.robustness.summary_table() _rob_df.index.name = item_singular print(_rob_df.to_string()) @@ -1538,7 +1853,16 @@ def _print_bundle_summary( print() _print_mean_advantage( - bundle, + labels=bundle.robustness.labels, + mean=bundle.robustness.mean, + std=bundle.robustness.std, + ci_low=bundle.robustness.ci_low, + ci_high=bundle.robustness.ci_high, + multi_ci_per_entity=[ + _rob_multi_ci_at(bundle.robustness.multi_ci, i) + for i in range(len(bundle.robustness.labels)) + ], + resolved_ci_method=bundle.resolved_ci_method, item_singular=item_singular, line_width=line_width, template_col_width=template_col_width, @@ -1554,6 +1878,11 @@ def _print_bundle_summary( pairwise_sort=pairwise_sort, style=style, ) + _print_behavioral_agreement_section( + bundle, + top_pairwise=top_pairwise, + pairwise_sort=pairwise_sort, + ) # Seed variance section (only when seeded data is present). if bundle.seed_variance is not None: @@ -1651,24 +1980,29 @@ def _print_seed_variance( f" key: ▁–█ = per-input noise " f"(globally scaled; █ = {global_cell_max:.4f})" ) - num_w = 10 + # Wide enough for "instability" (11 chars) -- with num_w=10 that header + # overflowed its own field by 1 char and dragged every header after it + # out of alignment with the data rows below. + num_w = len("instability") consistency_w = 18 + verdicts = [_instability_label(float(v)) for v in sv.instability] + verdict_w = max(len("Verdict"), max(len(v) for v in verdicts)) print( f" {item_singular.capitalize():<{template_col_width}s} " f"{'Per-input noise':<{strip_width}s} " - f"{'seed_std':>{num_w}s} " + f"{'run_std':>{num_w}s} " f"{'input_std':>{num_w}s} " f"{'total_std':>{num_w}s} " f"{'instability':>{num_w}s} " f"{'Consistency (ICC)':<{consistency_w}s} " - f"Verdict" + f"{'Verdict':<{verdict_w}s}" ) for i, label in enumerate(sv.labels): strip = _seed_noise_strip( sv.per_cell_seed_std[i], global_cell_max, max_width=strip_width ) instability = float(sv.instability[i]) - verdict = _instability_label(instability) + verdict = verdicts[i] verdict_color = _instability_color(instability) icc = float(sv.icc[i]) icc_str = "—" if np.isnan(icc) else f"{icc:.2f} ({_consistency_label(icc)})" @@ -1681,17 +2015,15 @@ def _print_seed_variance( f"{np.sqrt(sv.total_var[i]):>{num_w}.4f} " f"{instability:>{num_w}.4f} " f"{icc_color}{icc_str:<{consistency_w}s}{_RESET} " - f"{verdict_color}{verdict}{_RESET}" + f"{verdict_color}{verdict:<{verdict_w}s}{_RESET}" ) print( - f"{_DIM} instability = mean std across repeated runs, in score units " - f"(how many points a score typically moves between runs){_RESET}" + f"{_DIM} instability = how many points a score typically moves " + f"between repeated runs{_RESET}" ) print( - f"{_DIM} Consistency (ICC) = intraclass correlation = input_var / " - f"(input_var + seed_var) — share of total variation that's real " - f"input-level signal rather than run noise (bands: <0.50 poor, " - f"0.50-0.75 moderate, 0.75-0.90 good, >0.90 excellent; Koo & Li 2016){_RESET}" + f"{_DIM} Consistency (ICC) = how much of the difference between " + f"inputs is real signal, rather than run-to-run noise{_RESET}" ) print() @@ -1897,9 +2229,10 @@ def _print_factorial_lmm_summary( level_w = min(28, max(len("Level"), max(len(str(v)) for v in mm_sorted["level"]) + 2)) _ci_legend_mm = _legend_ci_label(style, ci_pct, style == "gradient") + _mean_marker_lmm = _mean_marker_legend(style, "mean") print( f" axis: [{axis_low:+.3f}, {axis_high:+.3f}] " - f"(· ±SE, {_ci_legend_mm}, ● mean, │ factor mean)" + f"(· ±SE, {_ci_legend_mm}{_mean_marker_lmm}, │ factor mean)" ) print( f" {'Level':<{level_w}s} {'Interval Plot':<{line_width}s} " @@ -2096,12 +2429,17 @@ def _gradient_interval_line( ) -> str: """Render a one-line gradient CI plot using Unicode block characters. - Opacity mapping (outermost → innermost): - beyond 99.9% CI → ' ' (invisible) - 99% – 99.9% CI → '░' (10 % opacity) - 95% – 99% CI → '▒' (medium) - 90% – 95% CI → '▓' (high) - inside 90% CI → '█' (fully opaque) + Opacity mapping (outermost → innermost), for the default + ``GRADIENT_CI_ALPHAS`` of (0.32, 0.10, 0.05, 0.01): + beyond 99% CI → ' ' (invisible) + 95% – 99% CI → '░' (10 % opacity) + 90% – 95% CI → '▒' (medium) + 68% – 90% CI → '▓' (high) + inside 68% CI → '█' (fully opaque) + + Bands are paired to ``sorted(multi_ci)`` positionally, so a caller passing a + different alpha ladder gets the same outermost-to-innermost shading at + whatever levels it supplied. The ±1σ spread dots ('·') appear only where they peek beyond all CI bands. Falls back to ``_ascii_interval_line`` when fewer than 2 CI levels are present. @@ -2148,7 +2486,12 @@ def to_idx(x: float) -> int: chars[i] = char ref_idx = to_idx(reference) - mean_idx = to_idx(mean) + # No marker is drawn at the mean, deliberately. A point marker invites the + # reader to treat one value as the answer and the interval around it as + # decoration, which is the reading gradient plots exist to avoid + # (Correll & Gleicher, "Error bars considered harmful"). `mean` is still a + # parameter because _ascii_interval_line, the <2-band fallback above, does + # mark it. chars[ref_idx] = "│" # The reference line can obscure the tail of a CI band when the tail @@ -2188,6 +2531,36 @@ def _synth_multi_ci_from_se( } +def _finite_axis( + axis_low: float, + axis_high: float, + candidates: tuple[float, ...], +) -> tuple[float, float]: + """Return a finite (axis_low, axis_high) for the ASCII interval plots. + + Substitutes the spread of whatever finite values the row does carry when a + supplied bound is non-finite, and falls back to a unit window around 0 when + nothing finite is available. Purely a drawing concern -- the printed + numeric bounds are untouched. + """ + lo, hi = float(axis_low), float(axis_high) + if np.isfinite(lo) and np.isfinite(hi) and hi > lo: + return lo, hi + finite = [float(c) for c in candidates if c is not None and np.isfinite(float(c))] + if np.isfinite(lo): + finite.append(lo) + if np.isfinite(hi): + finite.append(hi) + if not finite: + return -1.0, 1.0 + f_lo, f_hi = min(finite), max(finite) + if f_hi <= f_lo: + pad = max(abs(f_lo), 1.0) * 0.5 + return f_lo - pad, f_hi + pad + pad = (f_hi - f_lo) * 0.15 + return f_lo - pad, f_hi + pad + + def _choose_interval_line( *, mean: float, @@ -2203,7 +2576,31 @@ def _choose_interval_line( multi_ci: Optional[dict[float, tuple[float, float]]] = None, ) -> str: """Dispatch to gradient or line renderer based on style and data availability.""" + # A non-finite bound is a real result, not a bug: paired._degenerate_pair_ci + # reports (-inf, +inf) for a zero-variance difference on unbounded data, + # where no finite interval has guaranteed coverage. Both renderers map a + # value onto an axis via (x - axis_low) / (axis_high - axis_low), which is + # NaN when the axis itself is infinite, so clamp the drawn axis to the + # finite information in the row. The interval still *prints* as -inf/inf in + # the CI Low/CI High columns -- this only bounds the little ASCII plot, and + # an interval that runs off both ends of it is the correct picture. + axis_low, axis_high = _finite_axis( + axis_low, axis_high, + candidates=(mean, ci_low, ci_high, spread_low, spread_high, reference), + ) effective_multi_ci = multi_ci + if not (np.isfinite(ci_low) and np.isfinite(ci_high)): + # The primary interval is unbounded, but the gradient bands handed in + # can still be zero-width: paired.py builds its `mci` dict per alpha + # from the method's own CI function, and t_interval_ci_1d keeps its + # (mean, mean) contract on a zero-variance sample. The renderer + # normally trusts multi_ci over the primary CI, which here would draw + # a single opaque block at the point estimate -- the exact picture of + # false certainty this branch exists to retract -- immediately beside + # a printed -inf/+inf. Drop the bands and draw the primary interval, + # so the plot says what the numbers say. Whether that marginal + # contract should itself move is a separate, statistical question. + effective_multi_ci = None if style == "gradient" and effective_multi_ci is None: # Synthesize gradient from the primary CI via normal z-scaling. # Appropriate for Wald-type CIs (LMM); a reasonable approximation elsewhere. @@ -2237,6 +2634,40 @@ def _rob_multi_ci_at( return {a: (float(lo[idx]), float(hi[idx])) for a, (lo, hi) in rob_multi_ci.items()} +_CORRECTION_DISPLAY_NAMES = { + "romano_wolf": "Romano-Wolf", + "fdr_bh": "FDR (BH)", + "bonferroni": "Bonferroni", + "holm": "Holm", + "hochberg": "Hochberg", + "shaffer": "Shaffer", + "max_t": "max-T", + "none": "none", +} + + +def _pretty_correction(code: Optional[str]) -> str: + """Human-readable name for a FWER correction method code.""" + if not code: + return "none" + return _CORRECTION_DISPLAY_NAMES.get(code, code.replace("_", " ").title()) + + +_SIMULTANEOUS_CI_DISPLAY_NAMES = { + "max_t": "max-T", + "sidak": "Šidák", + "boot": "Joint bootstrap", + "bonferroni": "Bonferroni", +} + + +def _pretty_simultaneous_ci(code: Optional[str]) -> str: + """Human-readable name for a simultaneous-CI method code.""" + if not code: + return "none" + return _SIMULTANEOUS_CI_DISPLAY_NAMES.get(code, code.replace("_", " ").title()) + + def _legend_ci_label(style: str, ci_pct: int, multi_ci_available: bool) -> str: """Return the CI portion of a legend string for the given style.""" if style == "gradient" and multi_ci_available: @@ -2248,6 +2679,21 @@ def _legend_ci_label(style: str, ci_pct: int, multi_ci_available: bool) -> str: return f"─ {ci_pct}% CI" +def _mean_marker_legend(style: str, label: str) -> str: + """Return the ', ●