op4b: held-out memorization gate (diff-in-diff, shadow-first)

eval/memorization.py

@@ -0,0 +1,157 @@
+"""op4b — held-out MEMORIZATION gate (difference-in-differences).
+
+Catches the training-time held-out-memorization fraud class (2026-07-06 fraud king
+2c3d59e3: a canonical checkpoint that passed every gate but had TRAINED ON the
+leaked eval shard -> artificially low val_bpb, true generalization worse than GPT-2).
+Compute/arch/attestation gates are blind to this: the fraud is in the loss surface,
+not the code path, so HOSB (eval-time answer-blanking) does not touch it either.
+
+The signal (the exact control that unmasked the fraud king): a memorizer scores the
+PINNED shard P (what op4 crowns on) anomalously LOW relative to FRESH disjoint text F,
+compared to how a CLEAN reference model R (public GPT-2, never trained on P) ranks the
+same two. Difference-in-differences cancels "P is just intrinsically easier/harder text":
+
+    DD = (meanNLL_M(P) - meanNLL_M(F)) - (meanNLL_R(P) - meanNLL_R(F))
+
+Honest model: DD >= 0 (it finds P about as hard as, or harder than, F -- same as R).
+Memorizer:    DD << 0 (it finds P much easier than F, against R's verdict).
+  Fraud king measured: DD = (3.42-3.81) - (3.61-3.38) = -0.62.
+  #1593 (honest):      DD = (3.44-2.94) - (3.61-3.38) = +0.27.
+
+Plus a TAIL statistic for CONCENTRATED (near-verbatim) recall that a diffuse memorizer
+can hide from the mean: fraction of P-windows the model predicts near-perfectly.
+
+This module keeps `memorization_verdict` pure (numpy only, no torch) so it is unit-
+tested against the real measured numbers; the GPU harness is lazily imported.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+# Defaults; overridable via RALPH_MEMGATE_TAU / _TAU_TAIL / _TAIL_FLOOR env in the caller.
+DEFAULT_TAU = 0.15        # DD reject threshold (nats). Fraud king -0.62; honest >= +0.2. Wide margin.
+DEFAULT_TAU_TAIL = 0.05   # max fraction of near-verbatim windows before reject.
+DEFAULT_TAIL_FLOOR = 0.5  # a window mean-NLL below this (nats) is near-verbatim recall.
+
+
+def window_means(flat_nlls: np.ndarray, seq_len: int) -> np.ndarray:
+    """Reshape a flat per-token NLL array (len == n_windows*seq_len, as produced by
+    eval.val_bpb.per_position_nlls) into per-window mean NLL."""
+    flat = np.asarray(flat_nlls, dtype=np.float64)
+    n = (len(flat) // seq_len) * seq_len
+    if n == 0:
+        return np.zeros(0, dtype=np.float64)
+    return flat[:n].reshape(-1, seq_len).mean(axis=1)
+
+
+def memorization_verdict(
+    m_on_p: np.ndarray,
+    m_on_f: np.ndarray,
+    r_on_p: np.ndarray,
+    r_on_f: np.ndarray,
+    *,
+    tau: float = DEFAULT_TAU,
+    tau_tail: float = DEFAULT_TAU_TAIL,
+    tail_floor: float = DEFAULT_TAIL_FLOOR,
+) -> dict:
+    """Pure diff-in-diff + tail verdict on per-window mean NLLs.
+
+    Args are per-window mean-NLL arrays: submission model M and clean reference R,
+    each on the pinned shard P and the fresh disjoint control F.
+    Returns a dict: ok + every stat (so a CPU auditor can reproduce the decision).
+    """
+    for name, a in (("m_on_p", m_on_p), ("m_on_f", m_on_f), ("r_on_p", r_on_p), ("r_on_f", r_on_f)):
+        if len(a) == 0:
+            return {"ok": True, "skipped": True, "detail": f"memgate skipped: empty {name}"}
+    m_p, m_f = float(np.mean(m_on_p)), float(np.mean(m_on_f))
+    r_p, r_f = float(np.mean(r_on_p)), float(np.mean(r_on_f))
+    m_adv = m_p - m_f          # M's pinned-vs-fresh gap (negative => M finds P easier)
+    r_adv = r_p - r_f          # clean reference's intrinsic pinned-vs-fresh gap
+    dd = m_adv - r_adv
+    tail_frac = float(np.mean(np.asarray(m_on_p, dtype=np.float64) < tail_floor))
+    reject_dd = dd < -tau
+    reject_tail = tail_frac > tau_tail
+    ok = not (reject_dd or reject_tail)
+    reasons = []
+    if reject_dd:
+        reasons.append(
+            f"diff-in-diff {dd:.3f} < -{tau}: scores the pinned eval shard {-dd:.3f} nats "
+            f"easier than a clean reference would rank it vs fresh text — consistent with "
+            f"having TRAINED ON the held-out (memorization)"
+        )
+    if reject_tail:
+        reasons.append(
+            f"tail_frac {tail_frac:.3f} > {tau_tail}: near-verbatim recall on "
+            f"{tail_frac*100:.1f}% of pinned windows (mean-NLL < {tail_floor})"
+        )
+    detail = (
+        f"memgate ok: dd={dd:.3f} (M {m_adv:+.3f} vs ref {r_adv:+.3f}), tail={tail_frac:.3f}"
+        if ok else "MEMORIZATION REJECT: " + "; ".join(reasons)
+    )
+    return {
+        "ok": ok, "dd": round(dd, 4), "m_adv": round(m_adv, 4), "r_adv": round(r_adv, 4),
+        "tail_frac": round(tail_frac, 4), "m_p": round(m_p, 4), "m_f": round(m_f, 4),
+        "r_p": round(r_p, 4), "r_f": round(r_f, 4),
+        "tau": tau, "tau_tail": tau_tail, "n_windows_p": int(len(m_on_p)), "detail": detail,
+    }
+
+
+# ---------------------------------------------------------------------------
+# GPU harness (lazily imports torch/transformers so the verdict above stays pure).
+# ---------------------------------------------------------------------------
+
+class _GPT2Ref:
+    """Adapter so eval.val_bpb.per_position_nlls (which calls `logits, _ = model(x)`)
+    can score public GPT-2 (same gpt2 BPE / 50257 vocab as the held-out)."""
+
+    def __init__(self, device):
+        import torch
+        from transformers import GPT2LMHeadModel
+        self.m = GPT2LMHeadModel.from_pretrained("gpt2").to(device).eval().to(torch.bfloat16)
+        self.device = device
+
+    def eval(self):
+        return self
+
+    def parameters(self):
+        return self.m.parameters()
+
+    def __call__(self, inp):
+        return self.m(inp).logits, None
+
+
+def reference_window_means(p_tokens, f_tokens, seq_len, device, batch_size=16):
+    """Per-window mean NLLs of the clean GPT-2 reference on P and F. Deterministic in
+    (P, F, seq_len) — the caller caches these keyed by the shard fingerprints so GPT-2
+    runs once per shard-pair, not once per submission."""
+    from eval.val_bpb import per_position_nlls
+    ref = _GPT2Ref(device)
+    r_p = window_means(per_position_nlls(ref, p_tokens, seq_len, batch_size, device), seq_len)
+    r_f = window_means(per_position_nlls(ref, f_tokens, seq_len, batch_size, device), seq_len)
+    return r_p, r_f
+
+
+def model_window_means(model, tokens, seq_len, device, batch_size=16):
+    """Per-window mean NLLs of the submission model on a token stream."""
+    from eval.val_bpb import per_position_nlls
+    return window_means(per_position_nlls(model, tokens, seq_len, batch_size, device), seq_len)
+
+
+def run_memorization_gate(
+    model, p_tokens, f_tokens, seq_len, device, *,
+    ref_window_means=None, batch_size=16,
+    tau=DEFAULT_TAU, tau_tail=DEFAULT_TAU_TAIL, tail_floor=DEFAULT_TAIL_FLOOR,
+):
+    """End-to-end op4b: score M on the pinned shard P and the fresh control F, obtain the
+    clean GPT-2 reference on the same (computed here, or passed pre-cached as
+    (r_on_p, r_on_f)), and return the memorization_verdict dict.
+    ref_window_means lets the caller cache GPT-2 across submissions (it never changes for
+    a fixed shard-pair)."""
+    m_on_p = model_window_means(model, p_tokens, seq_len, device, batch_size)
+    m_on_f = model_window_means(model, f_tokens, seq_len, device, batch_size)
+    if ref_window_means is None:
+        r_on_p, r_on_f = reference_window_means(p_tokens, f_tokens, seq_len, device, batch_size)
+    else:
+        r_on_p, r_on_f = ref_window_means
+    return memorization_verdict(
+        m_on_p, m_on_f, r_on_p, r_on_f, tau=tau, tau_tail=tau_tail, tail_floor=tail_floor)

tests/test_memorization_gate.py

@@ -0,0 +1,74 @@
+"""op4b memorization-gate verdict tests, calibrated on the REAL measured numbers from
+the 2026-07-06 investigation (per-window mean NLLs, nats):
+  fraud king 2c3d59e3: P=3.42 F=3.81 ; GPT-2 ref: P=3.61 F=3.38  -> DD=-0.62 (memorized)
+  #1593 (honest):      P=3.44 F=2.94 ; same ref                   -> DD=+0.27 (honest)
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+# pure verdict tests need only numpy; eval.memorization lazily imports torch/transformers
+from eval.memorization import memorization_verdict, window_means  # noqa: E402
+
+REF_P, REF_F = 3.61, 3.38  # public GPT-2 reference (never trained on the shard)
+
+
+def _const(mean, n=200):
+    return np.full(n, mean, dtype=np.float64)
+
+
+def test_fraud_king_rejected():
+    v = memorization_verdict(_const(3.42), _const(3.81), _const(REF_P), _const(REF_F))
+    assert not v["ok"], v
+    assert v["dd"] < -0.15
+    assert "MEMORIZATION" in v["detail"]
+
+
+def test_1593_honest_passes():
+    v = memorization_verdict(_const(3.44), _const(2.94), _const(REF_P), _const(REF_F))
+    assert v["ok"], v
+    assert v["dd"] > 0  # finds pinned HARDER than fresh -> not memorized
+
+
+def test_symmetric_honest_passes():
+    # a model that ranks P vs F exactly like the reference -> DD == 0 -> pass
+    v = memorization_verdict(_const(4.0), _const(3.77), _const(REF_P), _const(REF_F))
+    assert v["ok"] and abs(v["dd"]) < 1e-9
+
+
+def test_borderline_below_tau_passes():
+    # small negative DD within noise must NOT reject at tau=0.15:
+    # dd = (3.60-3.45) - (3.61-3.38) = 0.15 - 0.23 = -0.08  (|dd| < tau)
+    v = memorization_verdict(_const(3.60), _const(3.45), _const(REF_P), _const(REF_F))
+    assert v["ok"], v
+    assert -0.15 < v["dd"] < 0
+
+
+def test_concentrated_memorizer_caught_by_tail():
+    # diffuse mean looks fine (DD ~ 0) but 20% of pinned windows are near-verbatim (~0 nats)
+    m_p = np.concatenate([_const(0.02, 40), _const(4.5, 160)])  # mean ~3.6, matches ref-ish
+    v = memorization_verdict(m_p, _const(3.4), _const(REF_P), _const(REF_F))
+    assert not v["ok"], v
+    assert v["tail_frac"] >= 0.2
+
+
+def test_empty_skips():
+    v = memorization_verdict(np.array([]), _const(3.0), _const(REF_P), _const(REF_F))
+    assert v["ok"] and v.get("skipped")
+
+
+def test_window_means_reshape():
+    flat = np.array([1.0, 3.0, 2.0, 4.0], dtype=np.float32)  # seq_len=2 -> windows [1,3],[2,4]
+    wm = window_means(flat, 2)
+    assert np.allclose(wm, [2.0, 3.0])
+
+
+def test_stats_present_for_auditor():
+    v = memorization_verdict(_const(3.42), _const(3.81), _const(REF_P), _const(REF_F))
+    for k in ("dd", "m_adv", "r_adv", "tail_frac", "m_p", "m_f", "r_p", "r_f", "n_windows_p"):
+        assert k in v

validator/validator.py

@@ -1573,6 +1573,111 @@
     return ok, detail, result
 
 
+def _memgate_mode() -> str:
+    import os
+    return os.environ.get("RALPH_MEMGATE", "off").strip().lower()
+
+
+def _memgate_reference(eval_dir, p_tokens, f_tokens, seq_len, device):
+    """Cache the clean GPT-2 reference window-means keyed by the (P, F, seq_len) content
+    hash so GPT-2 runs ONCE per shard-pair, not once per submission."""
+    import hashlib
+    import numpy as np
+    from eval.memorization import reference_window_means
+    key = hashlib.sha256(
+        p_tokens.tobytes() + b"|" + f_tokens.tobytes() + b"|" + str(seq_len).encode()
+    ).hexdigest()[:16]
+    cache = Path(eval_dir) / f".memgate_ref_{key}.npz"
+    if cache.exists():
+        try:
+            d = np.load(cache)
+            return d["r_p"], d["r_f"]
+        except (OSError, ValueError):
+            pass
+    r_p, r_f = reference_window_means(p_tokens, f_tokens, seq_len, device)
+    try:
+        np.savez(cache, r_p=r_p, r_f=r_f)
+    except OSError:
+        pass
+    return r_p, r_f
+
+
+def _memgate_shadow_log(ralph_root, proof_dir, v) -> None:
+    try:
+        rec = {"bundle": proof_dir.name}
+        rec.update({k: v.get(k) for k in ("ok", "dd", "tail_frac", "m_adv", "r_adv", "detail")})
+        with (Path(ralph_root) / "memgate_shadow.jsonl").open("a") as f:
+            f.write(json.dumps(rec) + "\n")
+    except OSError:
+        pass
+
+
+def op_memorization_gate(ralph_root: Path, proof_dir: Path, chain=None):
+    """op4b: reject a checkpoint that MEMORIZED the held-out (trained on the eval shard),
+    which every compute/arch/attestation gate AND HOSB is blind to (the fraud lives in the
+    loss surface, not the code path). Difference-in-differences vs a clean GPT-2 reference on
+    the pinned shard P vs a FRESH disjoint control F -- the exact control that caught the
+    2026-07-06 fraud king (canonical ckpt, all gates green, worse than GPT-2 on fresh text).
+
+    RALPH_MEMGATE: off (default) / shadow (log to memgate_shadow.jsonl, never rejects) /
+    enforce (fail-CLOSED). Needs a fresh control at eval/private/fresh_control.bin (or
+    RALPH_MEMGATE_FRESH); the control MUST never overlap training -- in the durable design it
+    is drawn POST-COMMIT (recency-gating) so no frozen model can have trained on it. Fail-OPEN
+    + loud if the control is missing (it IS the test) or on any infra error (never crash the
+    epoch loop). Efficiency: like op_rederive_trajectory this should ideally run only for a
+    would-beat-king bundle; today it runs for every op4-passing submission when enabled.
+    """
+    import os
+    mode = _memgate_mode()
+    if mode not in ("shadow", "enforce"):
+        return True, "memgate off", None
+    eval_dir = ralph_root / "eval" / "private"
+    p_path = eval_dir / "active_tokens.bin"
+    f_path = Path(os.environ.get("RALPH_MEMGATE_FRESH", str(eval_dir / "fresh_control.bin")))
+    if not p_path.exists() or not f_path.exists():
+        return True, f"memgate skipped: fresh control {f_path.name} not materialized", None
+    try:
+        import torch
+        from eval.val_bpb import load_eval_tokens, pinned_eval_seq_len
+        from eval.memorization import run_memorization_gate
+        ckpt = proof_dir / "training" / "checkpoint.pt"
+        saved = _safe_load_checkpoint_config(ckpt)
+        sd = _safe_load_checkpoint_weights(ckpt)
+        cfg = RalphConfig(
+            vocab_size=saved["vocab_size"], dim=saved["dim"], n_layers=saved["n_layers"],
+            n_heads=saved["n_heads"], head_dim=saved["head_dim"],
+            ffn_mult=saved.get("ffn_mult", 8 / 3), max_seq_len=saved["max_seq_len"],
+        )
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model = RalphBase(cfg).to(device).eval().to(torch.bfloat16)
+        model.load_state_dict(sd)
+        seq_len = pinned_eval_seq_len(saved["max_seq_len"])
+        p_tokens = load_eval_tokens(p_path)
+        f_tokens = load_eval_tokens(f_path)
+        ref = _memgate_reference(eval_dir, p_tokens, f_tokens, seq_len, device)
+
+        def _envf(name, default):
+            try:
+                return float(os.environ.get(name, default))
+            except (TypeError, ValueError):
+                return default
+        v = run_memorization_gate(
+            model, p_tokens, f_tokens, seq_len, device, ref_window_means=ref,
+            tau=_envf("RALPH_MEMGATE_TAU", 0.15),
+            tau_tail=_envf("RALPH_MEMGATE_TAU_TAIL", 0.05),
+        )
+        del model
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception as e:  # noqa: BLE001 -- memgate must never crash the epoch loop
+        return True, f"memgate error (skipped): {e}", None
+    if mode == "shadow":
+        _memgate_shadow_log(ralph_root, proof_dir, v)
+        return True, f"memgate shadow (would-{'pass' if v['ok'] else 'REJECT'}): {v['detail']}", v
+    return v["ok"], v["detail"], v
+
+
+
 def judge_submission(
     ralph_root: Path,
     proof_dir: Path,
@@ -1645,6 +1750,16 @@
         return result
     result.hidden_eval = hidden_eval
 
+    # op4b: memorization gate (RALPH_MEMGATE). Rejects a checkpoint that trained on the
+    # held-out (invisible to op1-op4 + HOSB). Fail-closed in enforce; shadow only logs.
+    ok_m, detail_m, mem_stats = op_memorization_gate(ralph_root, proof_dir, chain=chain)
+    result.operations["op4b_memorization_gate"] = {"ok": ok_m, "detail": detail_m}
+    if mem_stats is not None:
+        result.operations["op4b_memorization_gate"]["stats"] = mem_stats
+    if not ok_m:
+        result.rejected = ValidatorReject("op4b_memorization_gate", detail_m)
+        return result
+
     # Attach training + calibration summaries for downstream scoring.
     final_state_path = proof_dir / "training" / "final_state.json"
     if final_state_path.exists():