Calibrate percentile sliders on positives only; drop slider poly-line

scripts/freeze_metascore_quantiles.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Freeze the AlphaJudge metascore calibration deciles from the benchmark.
+
+The percentile sliders in AlphaJudge reports map each raw interface descriptor
+onto a frozen percentile scale (``BENCHMARK_QUANTILES`` in
+``alphajudge.meta_score``). Those deciles must describe the distribution of
+*real, interacting* complexes, so a new prediction is ranked against the
+population of true interfaces rather than against a benchmark that is half
+non-interacting decoys. This script therefore calibrates on **positive
+(interacting) pairs only** by default; the database-negative re-pairings are
+excluded.
+
+For each metascore feature the value is oriented "higher is better" using
+``FEATURE_DIRECTIONS`` (PAE and solvation energy are sign-flipped), NaNs are
+dropped, and ``numpy.quantile`` is evaluated at ``CALIBRATION_LEVELS``
+(deciles 0.0..1.0). The printed dictionary can be pasted into
+``src/alphajudge/meta_score.py``.
+
+Only ``numpy`` and the standard library are used (no pandas), so the script
+runs in a stock AlphaJudge install.
+
+Usage:
+    python scripts/freeze_metascore_quantiles.py \
+        --input-csv .../benchmark_best....csv \
+        [--label-filter positive]   # use "all" to reproduce the legacy scale
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import math
+from pathlib import Path
+
+import numpy as np
+
+from alphajudge.meta_score import (
+    BENCHMARK_QUANTILES,
+    CALIBRATION_LEVELS,
+    FEATURE_DIRECTIONS,
+)
+
+DEFAULT_BENCHMARK_CSV = Path(
+    "/scratch/dima/benchmark_26/final_sync_20260523_225722/staged_benchmark/"
+    "benchmark_best.final_sync_20260523_225722_force_recompute_nointerfacefix.csv"
+)
+
+
+def _safe_float(value) -> float:
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        return float("nan")
+    return parsed if math.isfinite(parsed) else float("nan")
+
+
+def feature_deciles(rows: list[dict[str, str]], feature: str, direction: float) -> tuple[np.ndarray, int]:
+    oriented = np.asarray([_safe_float(row.get(feature)) for row in rows], dtype=float) * direction
+    oriented = oriented[np.isfinite(oriented)]
+    return np.quantile(oriented, list(CALIBRATION_LEVELS)), len(oriented)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input-csv", default=str(DEFAULT_BENCHMARK_CSV))
+    parser.add_argument(
+        "--label-filter",
+        default="positive",
+        choices=("positive", "negative", "all"),
+        help="Subset of rows to calibrate on (default: positive interacting pairs).",
+    )
+    args = parser.parse_args()
+
+    with Path(args.input_csv).open(newline="") as handle:
+        rows = list(csv.DictReader(handle))
+    if not rows:
+        raise SystemExit(f"no rows found in {args.input_csv}")
+
+    if args.label_filter != "all":
+        rows = [r for r in rows if str(r.get("label", "")).strip().lower() == args.label_filter]
+        if not rows:
+            raise SystemExit(f"no rows with label '{args.label_filter}' in {args.input_csv}")
+
+    header = set(rows[0])
+    print(f"# input: {args.input_csv}")
+    print(f"# label-filter: {args.label_filter}  rows: {len(rows)}")
+    print("BENCHMARK_QUANTILES = {")
+    for feature in BENCHMARK_QUANTILES:
+        if feature not in header:
+            raise SystemExit(f"missing column in CSV: {feature}")
+        quantiles, n_finite = feature_deciles(rows, feature, FEATURE_DIRECTIONS[feature])
+        print(f'    "{feature}": (  # n_finite={n_finite}')
+        for value in quantiles:
+            print(f"        {float(value)!r},")
+        print("    ),")
+    print("}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/alphajudge/meta_score.py

@@ -47,113 +47,118 @@
 )
 
 # Frozen deciles from the benchmark_26 final synchronized best-interface run
-# (7,756 AF2/AF3 positive/negative rows; final_sync_20260523_225722, after
-# the missing pair-matched predictions were back-filled). Values are already
-# oriented so larger is better; e.g. PAE and solvation energy are stored
-# after sign flip.
+# (final_sync_20260523_225722, after the missing pair-matched predictions were
+# back-filled). Calibrated on POSITIVE (interacting) pairs ONLY: 3,878 AF2/AF3
+# positive rows out of the 7,756-row balanced table. The database-negative
+# re-pairings are deliberately excluded so a new prediction is ranked against
+# the distribution of real interfaces, not against a 50% non-interacting decoy
+# population. Regenerate with
+# `python scripts/freeze_metascore_quantiles.py --label-filter positive`.
+# Values are already oriented so larger is better; e.g. PAE and solvation
+# energy are stored after sign flip.
 BENCHMARK_QUANTILES = {
     "interface_LIS": (
         0.0,
         0.0,
-        0.0,
-        0.0,
-        0.0,
-        0.04087949643906165,
-        0.13930053033306905,
-        0.2748901434822449,
-        0.3884159952769791,
-        0.5095176486875836,
+        0.04060645514906991,
+        0.16033887103863478,
+        0.2576658136092226,
+        0.32442128815185134,
+        0.3851385587805126,
+        0.44345538655101113,
+        0.5088408754804322,
+        0.5774309078720992,
         0.7683597309793258,
     ),
     "interface_ipSAE": (
         0.0,
         0.0,
-        0.0,
-        0.0,
-        0.0,
-        0.01145996318417455,
-        0.04849651380227124,
-        0.39191523664041816,
-        0.631408256291724,
-        0.7685336920803372,
+        0.010959105568327996,
+        0.11185110880195845,
+        0.3555340463896866,
+        0.5186285859409205,
+        0.6285148344492705,
+        0.702100751375003,
+        0.7684464485394504,
+        0.8257356696917769,
         0.955598788837354,
     ),
     "interface_pDockQ2": (
         0.0,
-        0.0088894923938337,
-        0.0093154249757626,
-        0.0097853519732417,
-        0.0105167871915631,
-        0.0121137962077202,
-        0.01700470113788611,
-        0.044079106104955995,
-        0.147310180270119,
-        0.4604512911372348,
+        0.009453880310358351,
+        0.0111891126550556,
+        0.016886274027138628,
+        0.03397041297897107,
+        0.07344648145371385,
+        0.14133308419757337,
+        0.2696698397779966,
+        0.45400387748598986,
+        0.6963144657009852,
         0.950422615923692,
     ),
     "iptm": (
-        0.04,
-        0.15,
-        0.18,
-        0.21,
-        0.2422413975000381,
-        0.2993520200252533,
-        0.38812878727913014,
-        0.51,
-        0.66,
-        0.8086847960948944,
+        0.08,
+        0.2,
+        0.271808648109436,
+        0.37,
+        0.4707051336765289,
+        0.5603788185119629,
+        0.6500772428512572,
+        0.73,
+        0.8022430896759034,
+        0.87,
         0.9710875749588012,
     ),
     "confidence_score": (
         -99.73,
-        0.24,
-        0.279434748489446,
-        0.32,
-        0.3648142071999158,
-        0.42,
-        0.5,
-        0.6002808797233267,
-        0.7262544258927405,
-        0.84,
-        1.17,
+        0.29009815345375634,
+        0.38,
+        0.4718669364580657,
+        0.5582932754510956,
+        0.6378477968232499,
+        0.7118574504742852,
+        0.78,
+        0.8399975446618876,
+        0.8912135719685363,
+        1.16,
     ),
     "average_interface_pae": (
-        -31.466666666666665,
-        -28.001159075666123,
-        -26.179307692307695,
-        -24.495731538992413,
-        -22.704047619047614,
-        -20.205151515151503,
-        -17.110826882477134,
-        -11.98133340586016,
-        -7.214102564102556,
-        -3.7483788429752054,
+        -31.39791666666667,
+        -25.547096774193545,
+        -21.96980016550708,
+        -17.9543,
+        -13.584722222222224,
+        -10.055666561613616,
+        -7.456405228758169,
+        -5.489171195652174,
+        -3.7889943074003773,
+        -2.4152747252747253,
         -1.0446969696969706,
     ),
     "pDockQ/mpDockQ": (
         0.0,
-        0.04342955378732375,
-        0.0744205498951194,
-        0.11156935841275703,
-        0.154663210767691,
-        0.20778370916876399,
-        0.28249571167245185,
-        0.3758437409692674,
-        0.4920927853075837,
-        0.6177077320794855,
+        0.08588056107981762,
+        0.16023056468016583,
+        0.24182412751855897,
+        0.3247870448351307,
+        0.4041803181799995,
+        0.48010708308928685,
+        0.5508260817138925,
+        0.6163167605164049,
+        0.6713316493073628,
         0.7403745371795283,
     ),
     "interface_sc": (
         -0.0909274325112387,
-        0.2727699720080689,
-        0.3812050398280308,
-        0.4213154557548237,
-        0.4488086853592169,
-        0.47065393187337334,
-        0.4938328663057683,
-        0.5188898407198262,
-        0.54885285443392,
-        0.5914616385120754,
+        0.3767070462297797,
+        0.43997561327658047,
+        0.46854106915590277,
+        0.4903169925194296,
+        0.5111733070545363,
+        0.5327125306473834,
+        0.5566747944324164,
+        0.5840561089134118,
+        0.6193315076367524,
         0.744024124091351,
     ),
     # Deciles for interface_hb replace interface_area in METASCORE: H-bond
@@ -163,41 +168,41 @@
     # benchmark).
     "interface_hb": (
         0.0,
-        2.0,
         4.0,
         6.0,
-        8.0,
-        10.0,
-        12.0,
-        15.0,
+        9.0,
+        11.0,
+        13.0,
+        16.0,
         20.0,
-        28.0,
+        25.0,
+        34.0,
         129.0,
     ),
     "interface_area": (
         0.0,
-        542.1321074137181,
-        813.3952069762071,
-        1038.3245889227087,
-        1289.939806795484,
-        1559.6427719907642,
-        1897.2281626043305,
-        2361.926012313381,
-        2928.3984815923395,
-        3943.2489991553953,
-        19027.209490273777,
+        811.1528034047047,
+        1107.604424428278,
+        1420.6931573271686,
+        1716.9149871911516,
+        2073.670645394412,
+        2488.5877792892425,
+        2942.3373065313444,
+        3529.63873252237,
+        4669.550388777939,
+        17039.462876150043,
     ),
     "interface_solv_en": (
         -26.14067293563187,
-        0.03920834959131975,
-        2.8962665146694064,
-        5.443213873630101,
-        8.057221500635805,
-        10.930657812354951,
-        14.625751361301866,
-        19.243051691872445,
-        26.38771636631324,
-        38.319393210666135,
+        2.3417733842449655,
+        6.058883462167398,
+        9.549507514423109,
+        13.314712839646953,
+        17.308406303470562,
+        22.06634539018819,
+        28.33697255981416,
+        36.16186391880063,
+        48.35158049464502,
         233.00683345812263,
     ),
 }