op4: fail closed on arch-divergent checkpoints (close the miner-eval bypass)

tests/test_op4_fail_closed_arch.py

@@ -0,0 +1,59 @@
+"""op4 must FAIL CLOSED on an arch-divergent checkpoint (one that can't load into the
+validator's canonical model) rather than falling back to the miner's own eval code.
+
+The exploit (Kaizen0304, 2026-07-05): a patch adds value_embeddings to model/_v4skip.py,
+so the checkpoint carries params the canonical RalphBase lacks -> load_state_dict fails ->
+op4 fell back to _patched_hidden_eval, which runs the MINER's model code to compute the
+crown metric. With HOSB off that eval is not answer-blanked, so the crown was never
+independently verified. Fail-closed refuses to crown a model the validator can't build.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import ralph_bootstrap  # noqa: F401,E402
+import validator.validator as V  # noqa: E402
+
+
+def _force_arch_mismatch(tmp_path, monkeypatch):
+    (tmp_path / "training").mkdir(parents=True, exist_ok=True)
+    (tmp_path / "training" / "checkpoint.pt").write_bytes(b"x")  # must exist
+    monkeypatch.setenv("RALPH_SANDBOX", "0")
+    saved = {"vocab_size": 50257, "dim": 1024, "n_layers": 16, "n_heads": 16,
+             "head_dim": 64, "ffn_mult": 2.6875, "max_seq_len": 1024}
+    monkeypatch.setattr(V, "_safe_load_checkpoint_config", lambda p: saved)
+    monkeypatch.setattr(V, "_safe_load_checkpoint_weights", lambda p: {"value_embed.weight": 1})
+
+    class _FakeModel:
+        def __init__(self, cfg):
+            pass
+
+        def load_state_dict(self, sd):
+            raise RuntimeError("Unexpected key(s) in state_dict: value_embed.weight; size mismatch")
+
+    monkeypatch.setattr(V, "RalphBase", _FakeModel)
+    monkeypatch.setattr(V, "_is_state_dict_shape_mismatch", lambda e: True)
+    calls = []
+    monkeypatch.setattr(V, "_patched_hidden_eval",
+                        lambda *a, **k: (calls.append(1) or (True, "ran MINER code", object())))
+    return calls
+
+
+def test_arch_divergent_fails_closed(tmp_path, monkeypatch):
+    calls = _force_arch_mismatch(tmp_path, monkeypatch)
+    monkeypatch.delenv("RALPH_ALLOW_PATCHED_EVAL", raising=False)
+    ok, detail, res = V._legacy_hidden_eval(tmp_path, tmp_path)
+    assert not ok and res is None
+    assert "arch-divergent" in detail and "not independently verifiable" in detail
+    assert calls == []  # the miner's eval code was NEVER run
+
+
+def test_override_restores_patched_eval(tmp_path, monkeypatch):
+    # RALPH_ALLOW_PATCHED_EVAL=1 (testnet/debug or once HOSB is enforced) restores fallback.
+    calls = _force_arch_mismatch(tmp_path, monkeypatch)
+    monkeypatch.setenv("RALPH_ALLOW_PATCHED_EVAL", "1")
+    ok, detail, res = V._legacy_hidden_eval(tmp_path, tmp_path)
+    assert ok and calls == [1]  # fallback ran

validator/validator.py

@@ -1380,10 +1380,25 @@ def _legacy_hidden_eval(
         model = RalphBase(cfg)
         model.load_state_dict(state_dict)
     except RuntimeError as e:
-        # Architecture divergence (a structural patch that adds parameters):
-        # retry under the patched workdir so the actually-trained model scores.
+        # Architecture divergence (a structural patch that adds parameters the validator's
+        # canonical RalphBase does not have — e.g. value_embeddings / readout-calib).
         if _is_state_dict_shape_mismatch(e):
-            return _patched_hidden_eval(ralph_root, proof_dir, ckpt_path)
+            # FAIL CLOSED. Falling back to _patched_hidden_eval would rebuild the model from
+            # the MINER's patched model/ code and run it to compute the crown metric. On this
+            # HOSB-off legacy path that eval is NOT answer-blanked, so the miner's own code
+            # sees the raw held-out and the crown is not independently verifiable — the
+            # value_embeddings/novel-arch exploit (Kaizen0304 2026-07-05). We refuse to crown a
+            # model the validator cannot build + score itself. Arch innovation is re-enabled
+            # under HOSB (answer-blanked eval), or explicitly via RALPH_ALLOW_PATCHED_EVAL=1
+            # (testnet/debug only — do NOT set on mainnet while HOSB is off).
+            import os
+            if os.environ.get("RALPH_ALLOW_PATCHED_EVAL") == "1":
+                return _patched_hidden_eval(ralph_root, proof_dir, ckpt_path)
+            return False, (
+                "arch-divergent checkpoint: does not load into the validator's canonical "
+                "model, and op4 will NOT fall back to the miner's own eval code (HOSB off = "
+                f"not independently verifiable). Rejected. [{str(e)[:100]}]"
+            ), None
         raise
     # The CPU load above only ROUTED canonical-vs-patched; free it and run the GPU
     # forward in a SUBPROCESS so a fatal CUDA fault kills only the child.