diff --git a/tests/test_op4_fail_closed_arch.py b/tests/test_op4_fail_closed_arch.py new file mode 100644 index 0000000..ee08ae0 --- /dev/null +++ b/tests/test_op4_fail_closed_arch.py @@ -0,0 +1,59 @@ +"""op4 must FAIL CLOSED on an arch-divergent checkpoint (one that can't load into the +validator's canonical model) rather than falling back to the miner's own eval code. + +The exploit (Kaizen0304, 2026-07-05): a patch adds value_embeddings to model/_v4skip.py, +so the checkpoint carries params the canonical RalphBase lacks -> load_state_dict fails -> +op4 fell back to _patched_hidden_eval, which runs the MINER's model code to compute the +crown metric. With HOSB off that eval is not answer-blanked, so the crown was never +independently verified. Fail-closed refuses to crown a model the validator can't build. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import ralph_bootstrap # noqa: F401,E402 +import validator.validator as V # noqa: E402 + + +def _force_arch_mismatch(tmp_path, monkeypatch): + (tmp_path / "training").mkdir(parents=True, exist_ok=True) + (tmp_path / "training" / "checkpoint.pt").write_bytes(b"x") # must exist + monkeypatch.setenv("RALPH_SANDBOX", "0") + saved = {"vocab_size": 50257, "dim": 1024, "n_layers": 16, "n_heads": 16, + "head_dim": 64, "ffn_mult": 2.6875, "max_seq_len": 1024} + monkeypatch.setattr(V, "_safe_load_checkpoint_config", lambda p: saved) + monkeypatch.setattr(V, "_safe_load_checkpoint_weights", lambda p: {"value_embed.weight": 1}) + + class _FakeModel: + def __init__(self, cfg): + pass + + def load_state_dict(self, sd): + raise RuntimeError("Unexpected key(s) in state_dict: value_embed.weight; size mismatch") + + monkeypatch.setattr(V, "RalphBase", _FakeModel) + monkeypatch.setattr(V, "_is_state_dict_shape_mismatch", lambda e: True) + calls = [] + monkeypatch.setattr(V, "_patched_hidden_eval", + lambda *a, **k: (calls.append(1) or (True, "ran MINER code", object()))) + return calls + + +def test_arch_divergent_fails_closed(tmp_path, monkeypatch): + calls = _force_arch_mismatch(tmp_path, monkeypatch) + monkeypatch.delenv("RALPH_ALLOW_PATCHED_EVAL", raising=False) + ok, detail, res = V._legacy_hidden_eval(tmp_path, tmp_path) + assert not ok and res is None + assert "arch-divergent" in detail and "not independently verifiable" in detail + assert calls == [] # the miner's eval code was NEVER run + + +def test_override_restores_patched_eval(tmp_path, monkeypatch): + # RALPH_ALLOW_PATCHED_EVAL=1 (testnet/debug or once HOSB is enforced) restores fallback. + calls = _force_arch_mismatch(tmp_path, monkeypatch) + monkeypatch.setenv("RALPH_ALLOW_PATCHED_EVAL", "1") + ok, detail, res = V._legacy_hidden_eval(tmp_path, tmp_path) + assert ok and calls == [1] # fallback ran diff --git a/validator/validator.py b/validator/validator.py index 69476ce..40d7571 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -1380,10 +1380,25 @@ def _legacy_hidden_eval( model = RalphBase(cfg) model.load_state_dict(state_dict) except RuntimeError as e: - # Architecture divergence (a structural patch that adds parameters): - # retry under the patched workdir so the actually-trained model scores. + # Architecture divergence (a structural patch that adds parameters the validator's + # canonical RalphBase does not have — e.g. value_embeddings / readout-calib). if _is_state_dict_shape_mismatch(e): - return _patched_hidden_eval(ralph_root, proof_dir, ckpt_path) + # FAIL CLOSED. Falling back to _patched_hidden_eval would rebuild the model from + # the MINER's patched model/ code and run it to compute the crown metric. On this + # HOSB-off legacy path that eval is NOT answer-blanked, so the miner's own code + # sees the raw held-out and the crown is not independently verifiable — the + # value_embeddings/novel-arch exploit (Kaizen0304 2026-07-05). We refuse to crown a + # model the validator cannot build + score itself. Arch innovation is re-enabled + # under HOSB (answer-blanked eval), or explicitly via RALPH_ALLOW_PATCHED_EVAL=1 + # (testnet/debug only — do NOT set on mainnet while HOSB is off). + import os + if os.environ.get("RALPH_ALLOW_PATCHED_EVAL") == "1": + return _patched_hidden_eval(ralph_root, proof_dir, ckpt_path) + return False, ( + "arch-divergent checkpoint: does not load into the validator's canonical " + "model, and op4 will NOT fall back to the miner's own eval code (HOSB off = " + f"not independently verifiable). Rejected. [{str(e)[:100]}]" + ), None raise # The CPU load above only ROUTED canonical-vs-patched; free it and run the GPU # forward in a SUBPROCESS so a fatal CUDA fault kills only the child.