From db8697f187b6703d464014c9f4f43543c8084d38 Mon Sep 17 00:00:00 2001 From: Dima Molodenskiy Date: Wed, 10 Jun 2026 09:44:08 +0200 Subject: [PATCH] Calibrate percentile sliders on positives only; drop slider poly-line Rebased onto main 1.1.0 (which shipped the validation-report/meta-score feature). Applies three corrections on top of the current report: - meta_score.py: BENCHMARK_QUANTILES recomputed on POSITIVE (interacting) benchmark pairs only (3,878 AF2/AF3 positive rows) instead of all 7,756 rows (half non-interacting database-negatives). A prediction is now ranked against the distribution of real interfaces, not a decoy-padded population: e.g. interface_ipSAE=0.5 maps to the 49th percentile (was 75th); interface_LIS=0.30 to 46th (was 72nd). Per-feature AUROC is unchanged (monotonic transform); production interface_meta_score AUROC on the balanced benchmark is 0.878. - report.py: removed the black poly-line connecting per-group slider markers (each metric still shows its black percentile marker). _row_meta_score now recomputes the meta score from the current calibration first, falling back to a precomputed interface_meta_score column only when raw features are absent, so the Meta marker stays consistent with the recalibrated sliders. Wording updated to state the scale is built from interacting (positive) pairs. - scripts/freeze_metascore_quantiles.py: reproduces the deciles from any benchmark CSV (--label-filter positive|negative|all; "all" reproduces the prior all-rows scale bit-for-bit). Uses csv+numpy only (no pandas, which is not an AlphaJudge dependency). Tests: test_meta_score.py + test_report.py pass (9/9). Co-Authored-By: Claude Opus 4.8 --- scripts/freeze_metascore_quantiles.py | 101 +++++++++++++ src/alphajudge/meta_score.py | 209 +++++++++++++------------- src/alphajudge/report.py | 53 +++---- 3 files changed, 228 insertions(+), 135 deletions(-) create mode 100644 scripts/freeze_metascore_quantiles.py diff --git a/scripts/freeze_metascore_quantiles.py b/scripts/freeze_metascore_quantiles.py new file mode 100644 index 00000000..e973704f --- /dev/null +++ b/scripts/freeze_metascore_quantiles.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Freeze the AlphaJudge metascore calibration deciles from the benchmark. + +The percentile sliders in AlphaJudge reports map each raw interface descriptor +onto a frozen percentile scale (``BENCHMARK_QUANTILES`` in +``alphajudge.meta_score``). Those deciles must describe the distribution of +*real, interacting* complexes, so a new prediction is ranked against the +population of true interfaces rather than against a benchmark that is half +non-interacting decoys. This script therefore calibrates on **positive +(interacting) pairs only** by default; the database-negative re-pairings are +excluded. + +For each metascore feature the value is oriented "higher is better" using +``FEATURE_DIRECTIONS`` (PAE and solvation energy are sign-flipped), NaNs are +dropped, and ``numpy.quantile`` is evaluated at ``CALIBRATION_LEVELS`` +(deciles 0.0..1.0). The printed dictionary can be pasted into +``src/alphajudge/meta_score.py``. + +Only ``numpy`` and the standard library are used (no pandas), so the script +runs in a stock AlphaJudge install. + +Usage: + python scripts/freeze_metascore_quantiles.py \ + --input-csv .../benchmark_best....csv \ + [--label-filter positive] # use "all" to reproduce the legacy scale +""" + +from __future__ import annotations + +import argparse +import csv +import math +from pathlib import Path + +import numpy as np + +from alphajudge.meta_score import ( + BENCHMARK_QUANTILES, + CALIBRATION_LEVELS, + FEATURE_DIRECTIONS, +) + +DEFAULT_BENCHMARK_CSV = Path( + "/scratch/dima/benchmark_26/final_sync_20260523_225722/staged_benchmark/" + "benchmark_best.final_sync_20260523_225722_force_recompute_nointerfacefix.csv" +) + + +def _safe_float(value) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return float("nan") + return parsed if math.isfinite(parsed) else float("nan") + + +def feature_deciles(rows: list[dict[str, str]], feature: str, direction: float) -> tuple[np.ndarray, int]: + oriented = np.asarray([_safe_float(row.get(feature)) for row in rows], dtype=float) * direction + oriented = oriented[np.isfinite(oriented)] + return np.quantile(oriented, list(CALIBRATION_LEVELS)), len(oriented) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-csv", default=str(DEFAULT_BENCHMARK_CSV)) + parser.add_argument( + "--label-filter", + default="positive", + choices=("positive", "negative", "all"), + help="Subset of rows to calibrate on (default: positive interacting pairs).", + ) + args = parser.parse_args() + + with Path(args.input_csv).open(newline="") as handle: + rows = list(csv.DictReader(handle)) + if not rows: + raise SystemExit(f"no rows found in {args.input_csv}") + + if args.label_filter != "all": + rows = [r for r in rows if str(r.get("label", "")).strip().lower() == args.label_filter] + if not rows: + raise SystemExit(f"no rows with label '{args.label_filter}' in {args.input_csv}") + + header = set(rows[0]) + print(f"# input: {args.input_csv}") + print(f"# label-filter: {args.label_filter} rows: {len(rows)}") + print("BENCHMARK_QUANTILES = {") + for feature in BENCHMARK_QUANTILES: + if feature not in header: + raise SystemExit(f"missing column in CSV: {feature}") + quantiles, n_finite = feature_deciles(rows, feature, FEATURE_DIRECTIONS[feature]) + print(f' "{feature}": ( # n_finite={n_finite}') + for value in quantiles: + print(f" {float(value)!r},") + print(" ),") + print("}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/alphajudge/meta_score.py b/src/alphajudge/meta_score.py index 628544af..658b7c26 100644 --- a/src/alphajudge/meta_score.py +++ b/src/alphajudge/meta_score.py @@ -47,113 +47,118 @@ ) # Frozen deciles from the benchmark_26 final synchronized best-interface run -# (7,756 AF2/AF3 positive/negative rows; final_sync_20260523_225722, after -# the missing pair-matched predictions were back-filled). Values are already -# oriented so larger is better; e.g. PAE and solvation energy are stored -# after sign flip. +# (final_sync_20260523_225722, after the missing pair-matched predictions were +# back-filled). Calibrated on POSITIVE (interacting) pairs ONLY: 3,878 AF2/AF3 +# positive rows out of the 7,756-row balanced table. The database-negative +# re-pairings are deliberately excluded so a new prediction is ranked against +# the distribution of real interfaces, not against a 50% non-interacting decoy +# population. Regenerate with +# `python scripts/freeze_metascore_quantiles.py --label-filter positive`. +# Values are already oriented so larger is better; e.g. PAE and solvation +# energy are stored after sign flip. BENCHMARK_QUANTILES = { "interface_LIS": ( 0.0, 0.0, - 0.0, - 0.0, - 0.0, - 0.04087949643906165, - 0.13930053033306905, - 0.2748901434822449, - 0.3884159952769791, - 0.5095176486875836, + 0.04060645514906991, + 0.16033887103863478, + 0.2576658136092226, + 0.32442128815185134, + 0.3851385587805126, + 0.44345538655101113, + 0.5088408754804322, + 0.5774309078720992, 0.7683597309793258, ), "interface_ipSAE": ( 0.0, 0.0, - 0.0, - 0.0, - 0.0, - 0.01145996318417455, - 0.04849651380227124, - 0.39191523664041816, - 0.631408256291724, - 0.7685336920803372, + 0.010959105568327996, + 0.11185110880195845, + 0.3555340463896866, + 0.5186285859409205, + 0.6285148344492705, + 0.702100751375003, + 0.7684464485394504, + 0.8257356696917769, 0.955598788837354, ), "interface_pDockQ2": ( 0.0, - 0.0088894923938337, - 0.0093154249757626, - 0.0097853519732417, - 0.0105167871915631, - 0.0121137962077202, - 0.01700470113788611, - 0.044079106104955995, - 0.147310180270119, - 0.4604512911372348, + 0.009453880310358351, + 0.0111891126550556, + 0.016886274027138628, + 0.03397041297897107, + 0.07344648145371385, + 0.14133308419757337, + 0.2696698397779966, + 0.45400387748598986, + 0.6963144657009852, 0.950422615923692, ), "iptm": ( - 0.04, - 0.15, - 0.18, - 0.21, - 0.2422413975000381, - 0.2993520200252533, - 0.38812878727913014, - 0.51, - 0.66, - 0.8086847960948944, + 0.08, + 0.2, + 0.271808648109436, + 0.37, + 0.4707051336765289, + 0.5603788185119629, + 0.6500772428512572, + 0.73, + 0.8022430896759034, + 0.87, 0.9710875749588012, ), "confidence_score": ( -99.73, - 0.24, - 0.279434748489446, - 0.32, - 0.3648142071999158, - 0.42, - 0.5, - 0.6002808797233267, - 0.7262544258927405, - 0.84, - 1.17, + 0.29009815345375634, + 0.38, + 0.4718669364580657, + 0.5582932754510956, + 0.6378477968232499, + 0.7118574504742852, + 0.78, + 0.8399975446618876, + 0.8912135719685363, + 1.16, ), "average_interface_pae": ( - -31.466666666666665, - -28.001159075666123, - -26.179307692307695, - -24.495731538992413, - -22.704047619047614, - -20.205151515151503, - -17.110826882477134, - -11.98133340586016, - -7.214102564102556, - -3.7483788429752054, + -31.39791666666667, + -25.547096774193545, + -21.96980016550708, + -17.9543, + -13.584722222222224, + -10.055666561613616, + -7.456405228758169, + -5.489171195652174, + -3.7889943074003773, + -2.4152747252747253, -1.0446969696969706, ), "pDockQ/mpDockQ": ( 0.0, - 0.04342955378732375, - 0.0744205498951194, - 0.11156935841275703, - 0.154663210767691, - 0.20778370916876399, - 0.28249571167245185, - 0.3758437409692674, - 0.4920927853075837, - 0.6177077320794855, + 0.08588056107981762, + 0.16023056468016583, + 0.24182412751855897, + 0.3247870448351307, + 0.4041803181799995, + 0.48010708308928685, + 0.5508260817138925, + 0.6163167605164049, + 0.6713316493073628, 0.7403745371795283, ), "interface_sc": ( -0.0909274325112387, - 0.2727699720080689, - 0.3812050398280308, - 0.4213154557548237, - 0.4488086853592169, - 0.47065393187337334, - 0.4938328663057683, - 0.5188898407198262, - 0.54885285443392, - 0.5914616385120754, + 0.3767070462297797, + 0.43997561327658047, + 0.46854106915590277, + 0.4903169925194296, + 0.5111733070545363, + 0.5327125306473834, + 0.5566747944324164, + 0.5840561089134118, + 0.6193315076367524, 0.744024124091351, ), # Deciles for interface_hb replace interface_area in METASCORE: H-bond @@ -163,41 +168,41 @@ # benchmark). "interface_hb": ( 0.0, - 2.0, 4.0, 6.0, - 8.0, - 10.0, - 12.0, - 15.0, + 9.0, + 11.0, + 13.0, + 16.0, 20.0, - 28.0, + 25.0, + 34.0, 129.0, ), "interface_area": ( 0.0, - 542.1321074137181, - 813.3952069762071, - 1038.3245889227087, - 1289.939806795484, - 1559.6427719907642, - 1897.2281626043305, - 2361.926012313381, - 2928.3984815923395, - 3943.2489991553953, - 19027.209490273777, + 811.1528034047047, + 1107.604424428278, + 1420.6931573271686, + 1716.9149871911516, + 2073.670645394412, + 2488.5877792892425, + 2942.3373065313444, + 3529.63873252237, + 4669.550388777939, + 17039.462876150043, ), "interface_solv_en": ( -26.14067293563187, - 0.03920834959131975, - 2.8962665146694064, - 5.443213873630101, - 8.057221500635805, - 10.930657812354951, - 14.625751361301866, - 19.243051691872445, - 26.38771636631324, - 38.319393210666135, + 2.3417733842449655, + 6.058883462167398, + 9.549507514423109, + 13.314712839646953, + 17.308406303470562, + 22.06634539018819, + 28.33697255981416, + 36.16186391880063, + 48.35158049464502, 233.00683345812263, ), } diff --git a/src/alphajudge/report.py b/src/alphajudge/report.py index f2df8af0..2bc615c5 100644 --- a/src/alphajudge/report.py +++ b/src/alphajudge/report.py @@ -81,7 +81,7 @@ _AJ_DARK = "#111111" _REPORT_TITLE = "AlphaJudge Interface validation Report" -_BENCHMARK_TAG = "benchmark_26 (final_sync_20260523, n=7,756 AF2/AF3 rows)" +_BENCHMARK_TAG = "benchmark_26 positives (final_sync_20260523, n=3,878 interacting AF2/AF3 pairs)" _GRADIENT = np.tile(np.linspace(0.0, 1.0, 1024), (2, 1)) @@ -105,9 +105,9 @@ "interface_solv_en": "kcal/mol", } -# Metric grouping for the slider panel. Lines are drawn only WITHIN each group -# (AF-derived vs. biophysical); the Meta-score row stays separate and is never -# joined to a polyline. +# Metric grouping for the slider panel. The grouping (AF-derived vs. +# biophysical, with the Meta-score row kept separate) drives the inter-group +# vertical spacing in the panel. # # Per-interface vs. complex-level: features that are scalars per predicted # complex (not per chain pair) are pulled out of the per-interface slider @@ -183,13 +183,15 @@ def _read_csv_rows(path: Path) -> list[dict[str, str]]: def _row_meta_score(row: Mapping[str, Any]) -> float | None: - direct = _safe_float(row.get("interface_meta_score")) - if direct is not None: - return direct + # Always recompute from the current (positives-only) calibration so the + # Meta marker stays consistent with the freshly recalibrated feature + # sliders. A precomputed ``interface_meta_score`` column from an older or + # externally merged CSV could carry the legacy all-rows calibration; only + # fall back to it when the raw feature columns are unavailable. computed = interface_meta_score(row) if isinstance(computed, float) and math.isfinite(computed): return computed - return None + return _safe_float(row.get("interface_meta_score")) def _feature_view(row: Mapping[str, Any]) -> "OrderedDict[str, tuple[float | None, float | None]]": @@ -606,8 +608,7 @@ def _metric_rows_for_slider_panel( Group is one of "overall" (the Meta-score row), "af" (AlphaFold- derived confidence features), "biophys" (biophysical features), or "complex" (per-complex scalars). The grouping is used by - ``_draw_slider_panel`` to add vertical spacing between groups and to - draw polylines only within a group. + ``_draw_slider_panel`` to add vertical spacing between groups. ``groups`` lets callers swap the per-interface feature list for a different set (e.g. just complex-level metrics on the end-of-report @@ -652,7 +653,7 @@ def _draw_percentile_legend( x: float, y: float, w: float, - label: str = "Percentile relative to AlphaJudge benchmark", + label: str = "Percentile vs interacting (positive) benchmark pairs", ) -> None: ax = fig.add_axes((x, y, w, 0.032)) ax.set_xlim(0, 1) @@ -695,9 +696,9 @@ def _draw_slider_panel( The Meta-score row (if included) is rendered first and visually offset from the rest. Each group passed in ``groups`` is rendered as its own - block, with its own connecting polyline; lines never cross the - Meta-score row or a group boundary. When ``groups`` is ``None`` the - standard per-interface layout (AF-derived + biophysical) is used. + block, separated by extra vertical spacing. Each row's percentile is shown + by a black marker on its bar. When ``groups`` is ``None`` the standard + per-interface layout (AF-derived + biophysical) is used. Returns the bottom y coordinate of the graphic. """ @@ -815,20 +816,6 @@ def _draw_slider_panel( def _row_y(idx: int) -> float: return centers[idx] - # Polyline segments per metric group (skip "overall" - the Meta-score row - # is intentionally not connected to any feature row). - by_group: dict[str, list[tuple[float, float]]] = {} - for idx, pct, group in pct_positions: - if group == "overall": - continue - by_group.setdefault(group, []).append((pct, _row_y(idx))) - - for points in by_group.values(): - if len(points) >= 2: - xs = [p for p, _y in points] - ys = [y for _p, y in points] - line_ax.plot(xs, ys, color="#0b0b0b", linewidth=0.75, zorder=4) - marker_w = 0.012 marker_h = max(0.0042, min(0.0070, bar_h * 1.35)) for idx, pct, group in pct_positions: @@ -1464,7 +1451,7 @@ def _aggregate_cover_page( info = [ "This report scores AlphaFold-predicted complexes against the", - "AlphaJudge benchmark_26 reference set.", + "AlphaJudge benchmark_26 interacting (positive) reference set.", "All percentiles are archive percentiles; higher is better.", ] _draw_info_box(fig, x=0.13, y=0.54, w=0.74, h=0.11, lines=info) @@ -1595,8 +1582,8 @@ def _interface_summary_page( note_ax.text( 0.5, 1.0, - "Black marker shows this interface's percentile rank against the AlphaJudge benchmark " - "(higher = better).", + "Black marker shows this interface's percentile rank against the AlphaJudge " + "interacting (positive) benchmark pairs (higher = better).", ha="center", va="top", fontsize=9, @@ -1701,8 +1688,8 @@ def generate_per_run_report( info_lines = [ "AlphaJudge interface validation report.", "Each metric is converted to its archive percentile against the frozen", - "benchmark distribution; the overall meta score is the unweighted mean over", - "available features.", + "distribution of interacting (positive) benchmark pairs; the overall meta", + "score is the unweighted mean over available features.", ] software_lines: list[tuple[str, str]] = [ ("Reference distribution", _BENCHMARK_TAG),