From be9b2f57ade3299784da6941786a92c43d95f53c Mon Sep 17 00:00:00 2001 From: karpabot Date: Mon, 6 Jul 2026 01:16:49 +0000 Subject: [PATCH 1/2] op4b: held-out memorization gate (diff-in-diff, shadow-first) - eval/memorization.py: pure diff-in-diff verdict + GPT-2-reference harness. A memorizer scores the pinned shard anomalously easier than a fresh disjoint control vs how clean public GPT-2 ranks the same two (cancels intrinsic text difficulty). Plus a tail statistic for concentrated near-verbatim recall. - Wired as op4b in judge_submission after op4; RALPH_MEMGATE off/shadow/enforce, fail-CLOSED in enforce, fail-OPEN+loud without a fresh control. Reference is cached per shard-pair so GPT-2 runs once, not per submission. - Catches a training-time held-out memorizer that every compute/arch/attestation gate AND HOSB is blind to (the fraud is in the loss surface): verified reject of the 2026-07-06 fraud king (dd -0.65), pass of the honest challenger (dd +0.28). - Fresh control MUST never overlap training; durable design draws it post-commit (recency-gating) so no frozen model can have trained on it. Co-Authored-By: Claude Opus 4.8 --- eval/memorization.py | 157 ++++++++++++++++++++++++++++++++ tests/test_memorization_gate.py | 74 +++++++++++++++ validator/validator.py | 109 ++++++++++++++++++++++ 3 files changed, 340 insertions(+) create mode 100644 eval/memorization.py create mode 100644 tests/test_memorization_gate.py diff --git a/eval/memorization.py b/eval/memorization.py new file mode 100644 index 0000000..7154d78 --- /dev/null +++ b/eval/memorization.py @@ -0,0 +1,157 @@ +"""op4b — held-out MEMORIZATION gate (difference-in-differences). + +Catches the training-time held-out-memorization fraud class (2026-07-06 fraud king +2c3d59e3: a canonical checkpoint that passed every gate but had TRAINED ON the +leaked eval shard -> artificially low val_bpb, true generalization worse than GPT-2). +Compute/arch/attestation gates are blind to this: the fraud is in the loss surface, +not the code path, so HOSB (eval-time answer-blanking) does not touch it either. + +The signal (the exact control that unmasked the fraud king): a memorizer scores the +PINNED shard P (what op4 crowns on) anomalously LOW relative to FRESH disjoint text F, +compared to how a CLEAN reference model R (public GPT-2, never trained on P) ranks the +same two. Difference-in-differences cancels "P is just intrinsically easier/harder text": + + DD = (meanNLL_M(P) - meanNLL_M(F)) - (meanNLL_R(P) - meanNLL_R(F)) + +Honest model: DD >= 0 (it finds P about as hard as, or harder than, F -- same as R). +Memorizer: DD << 0 (it finds P much easier than F, against R's verdict). + Fraud king measured: DD = (3.42-3.81) - (3.61-3.38) = -0.62. + #1593 (honest): DD = (3.44-2.94) - (3.61-3.38) = +0.27. + +Plus a TAIL statistic for CONCENTRATED (near-verbatim) recall that a diffuse memorizer +can hide from the mean: fraction of P-windows the model predicts near-perfectly. + +This module keeps `memorization_verdict` pure (numpy only, no torch) so it is unit- +tested against the real measured numbers; the GPU harness is lazily imported. +""" +from __future__ import annotations + +import numpy as np + +# Defaults; overridable via RALPH_MEMGATE_TAU / _TAU_TAIL / _TAIL_FLOOR env in the caller. +DEFAULT_TAU = 0.15 # DD reject threshold (nats). Fraud king -0.62; honest >= +0.2. Wide margin. +DEFAULT_TAU_TAIL = 0.05 # max fraction of near-verbatim windows before reject. +DEFAULT_TAIL_FLOOR = 0.5 # a window mean-NLL below this (nats) is near-verbatim recall. + + +def window_means(flat_nlls: np.ndarray, seq_len: int) -> np.ndarray: + """Reshape a flat per-token NLL array (len == n_windows*seq_len, as produced by + eval.val_bpb.per_position_nlls) into per-window mean NLL.""" + flat = np.asarray(flat_nlls, dtype=np.float64) + n = (len(flat) // seq_len) * seq_len + if n == 0: + return np.zeros(0, dtype=np.float64) + return flat[:n].reshape(-1, seq_len).mean(axis=1) + + +def memorization_verdict( + m_on_p: np.ndarray, + m_on_f: np.ndarray, + r_on_p: np.ndarray, + r_on_f: np.ndarray, + *, + tau: float = DEFAULT_TAU, + tau_tail: float = DEFAULT_TAU_TAIL, + tail_floor: float = DEFAULT_TAIL_FLOOR, +) -> dict: + """Pure diff-in-diff + tail verdict on per-window mean NLLs. + + Args are per-window mean-NLL arrays: submission model M and clean reference R, + each on the pinned shard P and the fresh disjoint control F. + Returns a dict: ok + every stat (so a CPU auditor can reproduce the decision). + """ + for name, a in (("m_on_p", m_on_p), ("m_on_f", m_on_f), ("r_on_p", r_on_p), ("r_on_f", r_on_f)): + if len(a) == 0: + return {"ok": True, "skipped": True, "detail": f"memgate skipped: empty {name}"} + m_p, m_f = float(np.mean(m_on_p)), float(np.mean(m_on_f)) + r_p, r_f = float(np.mean(r_on_p)), float(np.mean(r_on_f)) + m_adv = m_p - m_f # M's pinned-vs-fresh gap (negative => M finds P easier) + r_adv = r_p - r_f # clean reference's intrinsic pinned-vs-fresh gap + dd = m_adv - r_adv + tail_frac = float(np.mean(np.asarray(m_on_p, dtype=np.float64) < tail_floor)) + reject_dd = dd < -tau + reject_tail = tail_frac > tau_tail + ok = not (reject_dd or reject_tail) + reasons = [] + if reject_dd: + reasons.append( + f"diff-in-diff {dd:.3f} < -{tau}: scores the pinned eval shard {-dd:.3f} nats " + f"easier than a clean reference would rank it vs fresh text — consistent with " + f"having TRAINED ON the held-out (memorization)" + ) + if reject_tail: + reasons.append( + f"tail_frac {tail_frac:.3f} > {tau_tail}: near-verbatim recall on " + f"{tail_frac*100:.1f}% of pinned windows (mean-NLL < {tail_floor})" + ) + detail = ( + f"memgate ok: dd={dd:.3f} (M {m_adv:+.3f} vs ref {r_adv:+.3f}), tail={tail_frac:.3f}" + if ok else "MEMORIZATION REJECT: " + "; ".join(reasons) + ) + return { + "ok": ok, "dd": round(dd, 4), "m_adv": round(m_adv, 4), "r_adv": round(r_adv, 4), + "tail_frac": round(tail_frac, 4), "m_p": round(m_p, 4), "m_f": round(m_f, 4), + "r_p": round(r_p, 4), "r_f": round(r_f, 4), + "tau": tau, "tau_tail": tau_tail, "n_windows_p": int(len(m_on_p)), "detail": detail, + } + + +# --------------------------------------------------------------------------- +# GPU harness (lazily imports torch/transformers so the verdict above stays pure). +# --------------------------------------------------------------------------- + +class _GPT2Ref: + """Adapter so eval.val_bpb.per_position_nlls (which calls `logits, _ = model(x)`) + can score public GPT-2 (same gpt2 BPE / 50257 vocab as the held-out).""" + + def __init__(self, device): + import torch + from transformers import GPT2LMHeadModel + self.m = GPT2LMHeadModel.from_pretrained("gpt2").to(device).eval().to(torch.bfloat16) + self.device = device + + def eval(self): + return self + + def parameters(self): + return self.m.parameters() + + def __call__(self, inp): + return self.m(inp).logits, None + + +def reference_window_means(p_tokens, f_tokens, seq_len, device, batch_size=16): + """Per-window mean NLLs of the clean GPT-2 reference on P and F. Deterministic in + (P, F, seq_len) — the caller caches these keyed by the shard fingerprints so GPT-2 + runs once per shard-pair, not once per submission.""" + from eval.val_bpb import per_position_nlls + ref = _GPT2Ref(device) + r_p = window_means(per_position_nlls(ref, p_tokens, seq_len, batch_size, device), seq_len) + r_f = window_means(per_position_nlls(ref, f_tokens, seq_len, batch_size, device), seq_len) + return r_p, r_f + + +def model_window_means(model, tokens, seq_len, device, batch_size=16): + """Per-window mean NLLs of the submission model on a token stream.""" + from eval.val_bpb import per_position_nlls + return window_means(per_position_nlls(model, tokens, seq_len, batch_size, device), seq_len) + + +def run_memorization_gate( + model, p_tokens, f_tokens, seq_len, device, *, + ref_window_means=None, batch_size=16, + tau=DEFAULT_TAU, tau_tail=DEFAULT_TAU_TAIL, tail_floor=DEFAULT_TAIL_FLOOR, +): + """End-to-end op4b: score M on the pinned shard P and the fresh control F, obtain the + clean GPT-2 reference on the same (computed here, or passed pre-cached as + (r_on_p, r_on_f)), and return the memorization_verdict dict. + ref_window_means lets the caller cache GPT-2 across submissions (it never changes for + a fixed shard-pair).""" + m_on_p = model_window_means(model, p_tokens, seq_len, device, batch_size) + m_on_f = model_window_means(model, f_tokens, seq_len, device, batch_size) + if ref_window_means is None: + r_on_p, r_on_f = reference_window_means(p_tokens, f_tokens, seq_len, device, batch_size) + else: + r_on_p, r_on_f = ref_window_means + return memorization_verdict( + m_on_p, m_on_f, r_on_p, r_on_f, tau=tau, tau_tail=tau_tail, tail_floor=tail_floor) diff --git a/tests/test_memorization_gate.py b/tests/test_memorization_gate.py new file mode 100644 index 0000000..6b2993d --- /dev/null +++ b/tests/test_memorization_gate.py @@ -0,0 +1,74 @@ +"""op4b memorization-gate verdict tests, calibrated on the REAL measured numbers from +the 2026-07-06 investigation (per-window mean NLLs, nats): + fraud king 2c3d59e3: P=3.42 F=3.81 ; GPT-2 ref: P=3.61 F=3.38 -> DD=-0.62 (memorized) + #1593 (honest): P=3.44 F=2.94 ; same ref -> DD=+0.27 (honest) +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +# pure verdict tests need only numpy; eval.memorization lazily imports torch/transformers +from eval.memorization import memorization_verdict, window_means # noqa: E402 + +REF_P, REF_F = 3.61, 3.38 # public GPT-2 reference (never trained on the shard) + + +def _const(mean, n=200): + return np.full(n, mean, dtype=np.float64) + + +def test_fraud_king_rejected(): + v = memorization_verdict(_const(3.42), _const(3.81), _const(REF_P), _const(REF_F)) + assert not v["ok"], v + assert v["dd"] < -0.15 + assert "MEMORIZATION" in v["detail"] + + +def test_1593_honest_passes(): + v = memorization_verdict(_const(3.44), _const(2.94), _const(REF_P), _const(REF_F)) + assert v["ok"], v + assert v["dd"] > 0 # finds pinned HARDER than fresh -> not memorized + + +def test_symmetric_honest_passes(): + # a model that ranks P vs F exactly like the reference -> DD == 0 -> pass + v = memorization_verdict(_const(4.0), _const(3.77), _const(REF_P), _const(REF_F)) + assert v["ok"] and abs(v["dd"]) < 1e-9 + + +def test_borderline_below_tau_passes(): + # small negative DD within noise must NOT reject at tau=0.15: + # dd = (3.60-3.45) - (3.61-3.38) = 0.15 - 0.23 = -0.08 (|dd| < tau) + v = memorization_verdict(_const(3.60), _const(3.45), _const(REF_P), _const(REF_F)) + assert v["ok"], v + assert -0.15 < v["dd"] < 0 + + +def test_concentrated_memorizer_caught_by_tail(): + # diffuse mean looks fine (DD ~ 0) but 20% of pinned windows are near-verbatim (~0 nats) + m_p = np.concatenate([_const(0.02, 40), _const(4.5, 160)]) # mean ~3.6, matches ref-ish + v = memorization_verdict(m_p, _const(3.4), _const(REF_P), _const(REF_F)) + assert not v["ok"], v + assert v["tail_frac"] >= 0.2 + + +def test_empty_skips(): + v = memorization_verdict(np.array([]), _const(3.0), _const(REF_P), _const(REF_F)) + assert v["ok"] and v.get("skipped") + + +def test_window_means_reshape(): + flat = np.array([1.0, 3.0, 2.0, 4.0], dtype=np.float32) # seq_len=2 -> windows [1,3],[2,4] + wm = window_means(flat, 2) + assert np.allclose(wm, [2.0, 3.0]) + + +def test_stats_present_for_auditor(): + v = memorization_verdict(_const(3.42), _const(3.81), _const(REF_P), _const(REF_F)) + for k in ("dd", "m_adv", "r_adv", "tail_frac", "m_p", "m_f", "r_p", "r_f", "n_windows_p"): + assert k in v diff --git a/validator/validator.py b/validator/validator.py index 69476ce..4458b55 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -1573,6 +1573,105 @@ def op4_hidden_eval( return ok, detail, result +def _memgate_mode() -> str: + import os + return os.environ.get("RALPH_MEMGATE", "off").strip().lower() + + +def _memgate_reference(eval_dir, p_tokens, f_tokens, seq_len, device): + """Cache the clean GPT-2 reference window-means keyed by the (P, F, seq_len) content + hash so GPT-2 runs ONCE per shard-pair, not once per submission.""" + import hashlib + import numpy as np + from eval.memorization import reference_window_means + key = hashlib.sha256( + p_tokens.tobytes() + b"|" + f_tokens.tobytes() + b"|" + str(seq_len).encode() + ).hexdigest()[:16] + cache = Path(eval_dir) / f".memgate_ref_{key}.npz" + if cache.exists(): + try: + d = np.load(cache) + return d["r_p"], d["r_f"] + except (OSError, ValueError): + pass + r_p, r_f = reference_window_means(p_tokens, f_tokens, seq_len, device) + try: + np.savez(cache, r_p=r_p, r_f=r_f) + except OSError: + pass + return r_p, r_f + + +def _memgate_shadow_log(ralph_root, proof_dir, v) -> None: + try: + rec = {"bundle": proof_dir.name} + rec.update({k: v.get(k) for k in ("ok", "dd", "tail_frac", "m_adv", "r_adv", "detail")}) + with (Path(ralph_root) / "memgate_shadow.jsonl").open("a") as f: + f.write(json.dumps(rec) + "\n") + except OSError: + pass + + +def op_memorization_gate(ralph_root: Path, proof_dir: Path, chain=None): + """op4b: reject a checkpoint that MEMORIZED the held-out (trained on the eval shard), + which every compute/arch/attestation gate AND HOSB is blind to (the fraud lives in the + loss surface, not the code path). Difference-in-differences vs a clean GPT-2 reference on + the pinned shard P vs a FRESH disjoint control F -- the exact control that caught the + 2026-07-06 fraud king (canonical ckpt, all gates green, worse than GPT-2 on fresh text). + + RALPH_MEMGATE: off (default) / shadow (log to memgate_shadow.jsonl, never rejects) / + enforce (fail-CLOSED). Needs a fresh control at eval/private/fresh_control.bin (or + RALPH_MEMGATE_FRESH); the control MUST never overlap training -- in the durable design it + is drawn POST-COMMIT (recency-gating) so no frozen model can have trained on it. Fail-OPEN + + loud if the control is missing (it IS the test) or on any infra error (never crash the + epoch loop). Efficiency: like op_rederive_trajectory this should ideally run only for a + would-beat-king bundle; today it runs for every op4-passing submission when enabled. + """ + import os + mode = _memgate_mode() + if mode not in ("shadow", "enforce"): + return True, "memgate off", None + eval_dir = ralph_root / "eval" / "private" + p_path = eval_dir / "active_tokens.bin" + f_path = Path(os.environ.get("RALPH_MEMGATE_FRESH", str(eval_dir / "fresh_control.bin"))) + if not p_path.exists() or not f_path.exists(): + return True, f"memgate skipped: fresh control {f_path.name} not materialized", None + try: + import torch + from eval.val_bpb import load_eval_tokens, pinned_eval_seq_len + from eval.memorization import run_memorization_gate + ckpt = proof_dir / "training" / "checkpoint.pt" + saved = _safe_load_checkpoint_config(ckpt) + sd = _safe_load_checkpoint_weights(ckpt) + cfg = RalphConfig( + vocab_size=saved["vocab_size"], dim=saved["dim"], n_layers=saved["n_layers"], + n_heads=saved["n_heads"], head_dim=saved["head_dim"], + ffn_mult=saved.get("ffn_mult", 8 / 3), max_seq_len=saved["max_seq_len"], + ) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = RalphBase(cfg).to(device).eval().to(torch.bfloat16) + model.load_state_dict(sd) + seq_len = pinned_eval_seq_len(saved["max_seq_len"]) + p_tokens = load_eval_tokens(p_path) + f_tokens = load_eval_tokens(f_path) + ref = _memgate_reference(eval_dir, p_tokens, f_tokens, seq_len, device) + v = run_memorization_gate( + model, p_tokens, f_tokens, seq_len, device, ref_window_means=ref, + tau=_envf("RALPH_MEMGATE_TAU", 0.15), + tau_tail=_envf("RALPH_MEMGATE_TAU_TAIL", 0.05), + ) + del model + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception as e: # noqa: BLE001 -- memgate must never crash the epoch loop + return True, f"memgate error (skipped): {e}", None + if mode == "shadow": + _memgate_shadow_log(ralph_root, proof_dir, v) + return True, f"memgate shadow (would-{'pass' if v['ok'] else 'REJECT'}): {v['detail']}", v + return v["ok"], v["detail"], v + + + def judge_submission( ralph_root: Path, proof_dir: Path, @@ -1645,6 +1744,16 @@ def judge_submission( return result result.hidden_eval = hidden_eval + # op4b: memorization gate (RALPH_MEMGATE). Rejects a checkpoint that trained on the + # held-out (invisible to op1-op4 + HOSB). Fail-closed in enforce; shadow only logs. + ok_m, detail_m, mem_stats = op_memorization_gate(ralph_root, proof_dir, chain=chain) + result.operations["op4b_memorization_gate"] = {"ok": ok_m, "detail": detail_m} + if mem_stats is not None: + result.operations["op4b_memorization_gate"]["stats"] = mem_stats + if not ok_m: + result.rejected = ValidatorReject("op4b_memorization_gate", detail_m) + return result + # Attach training + calibration summaries for downstream scoring. final_state_path = proof_dir / "training" / "final_state.json" if final_state_path.exists(): From 18ed75a452d5c68df56ef562a2c89ab5d31e8a5e Mon Sep 17 00:00:00 2001 From: karpabot Date: Mon, 6 Jul 2026 04:56:46 +0000 Subject: [PATCH 2/2] op4b: define _envf locally in op_memorization_gate (module-scope name was op1-local -> gate always fail-open-skipped) Co-Authored-By: Claude Opus 4.8 --- validator/validator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/validator/validator.py b/validator/validator.py index 4458b55..3e7577f 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -1655,6 +1655,12 @@ def op_memorization_gate(ralph_root: Path, proof_dir: Path, chain=None): p_tokens = load_eval_tokens(p_path) f_tokens = load_eval_tokens(f_path) ref = _memgate_reference(eval_dir, p_tokens, f_tokens, seq_len, device) + + def _envf(name, default): + try: + return float(os.environ.get(name, default)) + except (TypeError, ValueError): + return default v = run_memorization_gate( model, p_tokens, f_tokens, seq_len, device, ref_window_means=ref, tau=_envf("RALPH_MEMGATE_TAU", 0.15),