diff --git a/tests/test_rederive_gate.py b/tests/test_rederive_gate.py new file mode 100644 index 0000000..691a3cb --- /dev/null +++ b/tests/test_rederive_gate.py @@ -0,0 +1,263 @@ +"""Pre-crown re-derivation gate (proof-of-EXECUTION). + +op_rederive_trajectory re-runs the first steps of a challenger's declared training on +the validator's CANONICAL data and requires the loss trajectory to reproduce. It is: + * OFF by default; shadow logs the verdict but never blocks; enforce withholds the crown + on a genuine trajectory mismatch (RALPH_REDERIVE, _rederive_mode). + * run ONLY for a bundle that would take the crown (router gate) -> cost O(king-changes), + not O(submissions). + * fail-OPEN on every ambiguity (disabled / no canonical data / timeout / too-few points) + so a validator glitch never wrongly withholds a crown; a failed verdict only PREVENTS a + promotion, it never dethrones the sitting king. + * env-tunable tolerances (_rederive_tols) so the honest cross-GPU spread is calibrated in + shadow without a code change. +Also guards the seed-clobber fix: re-derivation must NOT pass --seed (train.py --seed +clobbers both init_seed AND data_seed -> false-reject of any honest init_seed!=data_seed run). +""" +from __future__ import annotations + +import sys +import types +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import ralph_bootstrap # noqa: F401,E402 +import validator.router as router # noqa: E402 +from validator.validator import ( # noqa: E402 + _apply_rederive_mode, + _rederive_mode, + _rederive_tols, + op_rederive_trajectory, +) + + +# ---------------------------------------------------------------- mode parsing +@pytest.mark.parametrize("val,expect", [ + (None, "off"), ("", "off"), ("0", "off"), ("off", "off"), + ("1", "enforce"), ("on", "enforce"), ("true", "enforce"), + ("enforce", "enforce"), ("ENFORCE", "enforce"), + ("shadow", "shadow"), ("Shadow", "shadow"), +]) +def test_rederive_mode(monkeypatch, val, expect): + if val is None: + monkeypatch.delenv("RALPH_REDERIVE", raising=False) + else: + monkeypatch.setenv("RALPH_REDERIVE", val) + assert _rederive_mode() == expect + + +# ------------------------------------------------------------ tolerance knobs +def test_tols_default(monkeypatch): + for k in ("STEP0", "ABS", "REL"): + monkeypatch.delenv(f"RALPH_REDERIVE_{k}_TOL", raising=False) + assert _rederive_tols() == (0.10, 0.40, 0.10) + + +def test_tols_env_override(monkeypatch): + monkeypatch.setenv("RALPH_REDERIVE_STEP0_TOL", "0.30") + monkeypatch.setenv("RALPH_REDERIVE_ABS_TOL", "0.60") + monkeypatch.setenv("RALPH_REDERIVE_REL_TOL", "0.15") + assert _rederive_tols() == (0.30, 0.60, 0.15) + + +def test_tols_malformed_falls_back(monkeypatch): + monkeypatch.setenv("RALPH_REDERIVE_STEP0_TOL", "not-a-float") + assert _rederive_tols()[0] == 0.10 + + +# --------------------------------------------------------------- shadow wrap +def test_shadow_never_gates_on_fail(): + ok, detail = _apply_rederive_mode("shadow", False, "mismatch at step 0") + assert ok is True and "would_gate=FAIL" in detail # logged, but never blocks + + +def test_shadow_reports_pass(): + ok, detail = _apply_rederive_mode("shadow", True, "reproduced 3/3") + assert ok is True and "would_gate=PASS" in detail + + +def test_enforce_passes_verdict_through(): + assert _apply_rederive_mode("enforce", False, "x") == (False, "x") + assert _apply_rederive_mode("enforce", True, "y") == (True, "y") + + +# ------------------------------------------------------------- fail-open skips +def test_disabled_is_fail_open(monkeypatch, tmp_path): + monkeypatch.delenv("RALPH_REDERIVE", raising=False) + ok, detail = op_rederive_trajectory(tmp_path, tmp_path) + assert ok is True and "disabled" in detail + + +def test_no_canonical_data_is_fail_open(monkeypatch, tmp_path): + # enabled, but the validator has not materialized the canonical shards/manifest -> + # skip (never a false-reject), the load-bearing pre-P2 behaviour. + monkeypatch.setenv("RALPH_REDERIVE", "1") + import ralph_bootstrap as rb + monkeypatch.setattr(rb, "RECIPE_DIR", tmp_path / "no_recipe_here") + ok, detail = op_rederive_trajectory(tmp_path, tmp_path) + assert ok is True and "not materialized" in detail + + +# ------------------------------------------------- seed-clobber fix (source guard) +def test_rederive_does_not_pass_seed(): + """train.py --seed clobbers both init_seed and data_seed; re-derivation must rely on + the full cfg via --config instead, or any honest init_seed!=data_seed run false-rejects.""" + import inspect + src = inspect.getsource(op_rederive_trajectory) + assert '"--seed"' not in src and "'--seed'" not in src + + +# ============================================================ router crown gate +def _fake_judge(*, val_bpb, benchmark_accuracy, tier="verified"): + return types.SimpleNamespace( + rejected=None, miner_hotkey="5F_challenger", bundle_hash="bh_challenger", + handshake_nonce="nonce", + hidden_eval=types.SimpleNamespace(val_bpb=val_bpb, benchmark_accuracy=benchmark_accuracy), + calibration={"matmul_ms": 10.0}, training_summary={"wall_clock_s": 1.0}, + operations={"op2_attestation": {"tier": tier}}, to_dict=lambda: {}, + ) + + +@pytest.fixture +def patched(monkeypatch): + holder: dict = {"rederive": (True, "disabled")} + calls: list = [] + + def set_judge(**kw): + holder["result"] = _fake_judge(**kw) + + def set_rederive(ok, detail="v"): + holder["rederive"] = (ok, detail) + + def _spy(ralph_root, proof_dir): + calls.append(proof_dir) + return holder["rederive"] + + monkeypatch.setattr(router, "judge_submission", lambda *a, **k: holder["result"]) + monkeypatch.setattr(router, "op_rederive_trajectory", _spy) + return types.SimpleNamespace(set_judge=set_judge, set_rederive=set_rederive, calls=calls) + + +def test_would_beat_king_and_rederive_pass_crowns(tmp_path, patched): + patched.set_judge(val_bpb=2.0, benchmark_accuracy=0.4) # genesis + patched.set_rederive(True) + out = router.process_submission(tmp_path, tmp_path / "p_a") + assert out["status"] == "accepted" + assert router._load_king(tmp_path)["val_bpb"] == 2.0 + assert len(patched.calls) == 1 # ran for the crown + + +def test_would_beat_king_but_rederive_fail_withholds_crown(tmp_path, patched): + # genesis crowns cleanly (rederive passes), sets the bar at 2.0 + patched.set_judge(val_bpb=2.0, benchmark_accuracy=0.4) + patched.set_rederive(True) + router.process_submission(tmp_path, tmp_path / "p_a") + + # a decisively-better challenger, but re-derivation FAILS -> crown withheld, king holds + patched.set_judge(val_bpb=1.0, benchmark_accuracy=0.4) + patched.set_rederive(False, "re-derivation mismatch at step 0") + out = router.process_submission(tmp_path, tmp_path / "p_b") + + assert out["status"] == "rederive_withheld" + assert out["event"]["accepted_as_king"] is False + assert out["event"]["rederive_blocked"] is True + assert router._load_king(tmp_path)["val_bpb"] == 2.0 # sitting king NOT dethroned + + +def test_below_threshold_challenger_skips_rederive(tmp_path, patched): + # genesis crown @ 1.0 + patched.set_judge(val_bpb=1.0, benchmark_accuracy=0.4) + patched.set_rederive(True) + router.process_submission(tmp_path, tmp_path / "p_a") + patched.calls.clear() + + # a worse challenger (2.0 > 1.0 val_bpb, does not beat the bar) -> rederive NOT invoked + patched.set_judge(val_bpb=2.0, benchmark_accuracy=0.4) + out = router.process_submission(tmp_path, tmp_path / "p_b") + assert out["status"] == "below_threshold" + assert patched.calls == [] # cost gate: O(king-changes) + + +# ============================================ PRODUCTION crown gate (service.py) +def _svc_result(val_bpb=1.0): + return types.SimpleNamespace( + rejected=None, + hidden_eval=types.SimpleNamespace( + val_bpb=val_bpb, benchmark_accuracy=0.5, val_seq_len=1024, + sealed_stream_manifest_hash="h", tail_val_bpb=None, + ), + calibration={"matmul_ms": 10.0}, training_summary={"wall_clock_s": 1.0}, + operations={"op2_attestation": {"tier": "verified"}}, + miner_hotkey="5F", miner_github="gh", pr_url=None, bundle_hash="bh", + ) + + +class _Chain: + def __init__(self, king=None): + self._king = king + + def get_king(self): + return self._king + + def get_high_water_mark(self): + return None + + +@pytest.fixture +def svc(monkeypatch): + import validator.service as S + calls: list = [] + seen: dict = {} + + monkeypatch.setattr(S, "_verify_pr_if_required", lambda *a, **k: (True, "ok")) + + def _classify(*, decisively, **k): + seen["decisively"] = decisively + return ("king_change" if decisively else "plain_failure", + 1.0 if decisively else 0.0) + + monkeypatch.setattr(S, "_classify_outcome", _classify) + + def set_judge(val_bpb): + monkeypatch.setattr(S, "judge_submission", lambda *a, **k: _svc_result(val_bpb)) + + def set_rederive(ok, detail="v"): + def _spy(ralph_root, bundle_dir): + calls.append(bundle_dir) + return (ok, detail) + monkeypatch.setattr(S, "op_rederive_trajectory", _spy) + + return types.SimpleNamespace( + S=S, calls=calls, seen=seen, set_judge=set_judge, set_rederive=set_rederive, + ) + + +def test_service_rederive_pass_allows_crown(tmp_path, svc): + svc.set_judge(1.0) + svc.set_rederive(True) + out = svc.S.score_and_decide(_Chain(king=None), tmp_path, 0.02) # genesis + assert out["status"] == "accepted" and out["accepted"] is True + assert out["rederive_withheld"] is False + assert svc.seen["decisively"] is True and len(svc.calls) == 1 + + +def test_service_rederive_fail_withholds_crown(tmp_path, svc): + svc.set_judge(1.0) + svc.set_rederive(False, "re-derivation mismatch at step 0") + out = svc.S.score_and_decide(_Chain(king=None), tmp_path, 0.02) + assert out["status"] == "rederive_withheld" + assert out["rederive_withheld"] is True and out["accepted"] is False + assert svc.seen["decisively"] is False # gate flipped it before classify + + +def test_service_non_decisive_skips_rederive(tmp_path, svc): + # sitting king @ 1.0; a worse challenger @ 2.0 does not beat it -> rederive NOT called + king = types.SimpleNamespace(val_bpb=1.0, benchmark_accuracy=0.5) + svc.set_judge(2.0) + svc.set_rederive(True) + out = svc.S.score_and_decide(_Chain(king=king), tmp_path, 0.02) + assert out["accepted"] is False and out["rederive_withheld"] is False + assert svc.calls == [] # cost gate: O(king-changes) diff --git a/validator/router.py b/validator/router.py index 3d4266f..88457e9 100644 --- a/validator/router.py +++ b/validator/router.py @@ -23,7 +23,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from validator.scoring import score_bundle -from validator.validator import judge_submission +from validator.validator import judge_submission, op_rederive_trajectory def _chain_dir(ralph_root: Path) -> Path: @@ -126,6 +126,22 @@ def process_submission( is_first = king is None and hwm is None accepted = score.decisively_beats_king if bar else False + # Pre-crown proof-of-EXECUTION: re-run the first steps on CANONICAL data and require + # the declared loss trajectory to reproduce. Run ONLY for a bundle that would take the + # crown, so cost is O(king-changes) not O(submissions). off/shadow never block a crown + # (shadow logs the verdict for calibration); enforce withholds the crown on a genuine + # trajectory mismatch. Every ambiguity (disabled, no canonical data, timeout, too-few + # points) stays fail-OPEN, so a validator glitch never wrongly withholds a crown, and a + # failed re-derivation only PREVENTS a promotion — it never dethrones the sitting king. + rederive_blocked = False + if accepted or is_first: + ok_rd, detail_rd = op_rederive_trajectory(ralph_root, proof_dir) + result.operations["op_rederive"] = {"ok": ok_rd, "detail": detail_rd} + if not ok_rd: + rederive_blocked = True + accepted = False + is_first = False + event = { "type": "submission_scored", "timestamp": time.time(), @@ -140,6 +156,7 @@ def process_submission( "score": score.score, "decisively_beats_king": score.decisively_beats_king, "accepted_as_king": accepted or is_first, + "rederive_blocked": rederive_blocked, "operations": result.operations, } _append_event(ralph_root, event) @@ -166,8 +183,14 @@ def process_submission( "new_king": new_king, }) + if accepted or is_first: + status = "accepted" + elif rederive_blocked: + status = "rederive_withheld" + else: + status = "below_threshold" return { - "status": "accepted" if (accepted or is_first) else "below_threshold", + "status": status, "result": result.to_dict(), "score": asdict(score), "event": event, diff --git a/validator/service.py b/validator/service.py index 513b00e..59546bc 100644 --- a/validator/service.py +++ b/validator/service.py @@ -45,7 +45,7 @@ from validator.hf_poller import DEFAULT_REPO as DEFAULT_HF_REPO from validator.hf_poller import poll_hub from validator.scoring import score_bundle -from validator.validator import judge_submission +from validator.validator import judge_submission, op_rederive_trajectory # Bittensor's bt.logging hijacks Python's logging module and raises the root @@ -428,6 +428,22 @@ def score_and_decide( is_first = king is None and hwm is None decisively = score.decisively_beats_king or is_first + # Pre-crown proof-of-EXECUTION: re-run the first steps on the validator's CANONICAL + # data and require the declared loss trajectory to reproduce. Run ONLY for a would-be + # king, so cost is O(king-changes) not O(submissions). off/shadow never block (shadow + # logs the verdict for calibration); enforce withholds the crown on a genuine + # trajectory mismatch. Every ambiguity (disabled / no canonical data / timeout / + # too-few points) stays fail-OPEN, and a failed verdict only PREVENTS this promotion — + # it never dethrones the sitting king or auto-blacklists. + rederive_withheld = False + if decisively: + ok_rd, detail_rd = op_rederive_trajectory(RALPH_ROOT, bundle_dir) + result.operations["op_rederive"] = {"ok": ok_rd, "detail": detail_rd} + if not ok_rd: + rederive_withheld = True + decisively = False + is_first = False + classification, weight_credit = _classify_outcome( decisively=decisively, val_bpb=result.hidden_eval.val_bpb, @@ -436,7 +452,9 @@ def score_and_decide( bundle_dir=bundle_dir, ) accepted = classification == "king_change" - if classification == "king_change": + if rederive_withheld: + status = "rederive_withheld" + elif classification == "king_change": status = "accepted" elif classification == "meaningful_failure": status = "meaningful_failure" @@ -466,6 +484,7 @@ def score_and_decide( "decisive": score.decisively_beats_king, "accepted": accepted, "is_first": is_first, + "rederive_withheld": rederive_withheld, "result": result, "score_report": score, } diff --git a/validator/validator.py b/validator/validator.py index 69476ce..5b816d8 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -1410,6 +1410,50 @@ def _read_training_log_points(log_path: Path, max_step: int | None = None) -> li return out +def _rederive_mode() -> str: + """RALPH_REDERIVE selects the re-derivation regime: + off (default) -> gate never runs; + shadow -> gate RUNS + logs the verdict but NEVER blocks a crown + (calibration data collection on a live box, cannot false-reject); + enforce (=1) -> a trajectory mismatch blocks the crown (fail-closed on POSITIVE + mismatch only; every ambiguity stays fail-OPEN skip). + """ + import os + v = os.environ.get("RALPH_REDERIVE", "").strip().lower() + if v in ("1", "on", "true", "enforce"): + return "enforce" + if v == "shadow": + return "shadow" + return "off" + + +def _rederive_tols() -> tuple[float, float, float]: + """(step0_tol, abs_tol, rel_tol) for compare_loss_trajectory, env-tunable so the + honest cross-GPU/bf16 spread can be calibrated during shadow without a code change. + Defaults match the compare_loss_trajectory defaults.""" + import os + + def _f(name: str, default: float) -> float: + try: + return float(os.environ.get(name, "") or default) + except ValueError: + return default + + return ( + _f("RALPH_REDERIVE_STEP0_TOL", 0.10), + _f("RALPH_REDERIVE_ABS_TOL", 0.40), + _f("RALPH_REDERIVE_REL_TOL", 0.10), + ) + + +def _apply_rederive_mode(mode: str, ok_v: bool, detail_v: str) -> tuple[bool, str]: + """shadow NEVER gates: return PASS but surface the would-be verdict for the audit + log + calibration. enforce returns the real verdict.""" + if mode == "shadow": + return True, f"[shadow] would_gate={'PASS' if ok_v else 'FAIL'}: {detail_v}" + return ok_v, detail_v + + def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str]: """Pre-crown proof-of-EXECUTION: re-run the FIRST few steps of the declared training on CANONICAL data and require the loss trajectory to reproduce. @@ -1420,12 +1464,13 @@ def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str train.py with the miner's exact config+seed but pinned to CANONICAL data, collect the first few logged points, and compare the trajectory (validator.integrity.compare_loss_trajectory). - OFF by default (RALPH_REDERIVE=1 to enable). Fail-OPEN (skip) when disabled or when the - canonical training data is not materialized on the validator — enable only AFTER - materializing data/data_manifest.json + shards and calibrating the tolerance band on a - known-honest bundle. Expensive (GPU-minutes); the caller should run it only on a bundle - that would actually beat the king. NOTE: written but UNVALIDATED until canonical data - exists on a validator to run it against. + Regime via RALPH_REDERIVE (_rederive_mode): off (default) / shadow / enforce. Runs + only when the canonical data are materialized on the validator; otherwise fail-OPEN + skip. In shadow it always returns (True, "[shadow] ...") so it can NEVER block — used + to collect the honest cross-GPU trajectory spread before calibrating the tolerance + band. Tolerances are env-tunable (RALPH_REDERIVE_STEP0_TOL / _ABS_TOL / _REL_TOL) so + calibration needs no code change. Expensive (GPU-minutes); the CALLER runs it only for + a bundle that would actually beat the king (cost is O(king-changes), not O(submissions)). """ import os import shutil @@ -1433,8 +1478,9 @@ def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str import tempfile import time - if os.environ.get("RALPH_REDERIVE") != "1": - return True, "re-derivation disabled (RALPH_REDERIVE!=1)" + mode = _rederive_mode() + if mode == "off": + return True, "re-derivation disabled (RALPH_REDERIVE off)" from ralph_bootstrap import RECIPE_DIR canon_manifest = RECIPE_DIR / "data" / "data_manifest.json" @@ -1482,6 +1528,12 @@ def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str train_py = workdir / "recipe" / "train.py" if not train_py.exists(): return True, "re-derivation skipped: recipe/train.py not found in workdir" + # Do NOT pass --seed: train.py --seed clobbers BOTH init_seed AND data_seed + # (recipe/train.py main()), so collapsing them here would give an honest miner + # whose run used init_seed != data_seed different init weights AND a different + # data order on re-derivation -> a guaranteed step-0 false-reject. The full cfg + # (with the miner's real, distinct init_seed + data_seed) is already written to + # cfg_file and applied via --config, so re-derivation reproduces the exact run. cmd = [ sys.executable, str(train_py), "--config", str(cfg_file), @@ -1489,9 +1541,6 @@ def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str "--data-base-dir", str((RECIPE_DIR / "data").resolve()), "--out-dir", str(out_dir), ] - seed = cfg.get("init_seed", cfg.get("data_seed")) - if seed is not None: - cmd += ["--seed", str(int(seed))] env = _sanitized_env(extra={"PYTHONPATH": str(workdir)}) timeout_s = int(os.environ.get("RALPH_REDERIVE_TIMEOUT_S", "1200")) @@ -1522,7 +1571,12 @@ def op_rederive_trajectory(ralph_root: Path, proof_dir: Path) -> tuple[bool, str return True, f"re-derivation inconclusive: only {len(rederived)} point(s) produced in {timeout_s}s (skip)" from validator.integrity import compare_loss_trajectory - return compare_loss_trajectory(declared, rederived) + + step0_tol, abs_tol, rel_tol = _rederive_tols() + ok_v, detail_v = compare_loss_trajectory( + declared, rederived, step0_tol=step0_tol, abs_tol=abs_tol, rel_tol=rel_tol, + ) + return _apply_rederive_mode(mode, ok_v, detail_v) def op4_hidden_eval( @@ -1618,19 +1672,12 @@ def judge_submission( result.rejected = ValidatorReject("op3_log_plausibility", detail) return result - # Pre-crown proof-of-EXECUTION (OFF by default; RALPH_REDERIVE=1). Re-runs the - # first training steps on CANONICAL data and requires the loss trajectory to - # reproduce — the only check that off-protocol training can't satisfy without - # actually running the canonical recipe. Skips cleanly when disabled or when the - # canonical data isn't materialized. Expensive: for efficiency the crown path - # should ideally invoke this only for a bundle that would beat the king (it still - # short-circuits behind op1-op3 here). - ok, detail = op_rederive_trajectory(ralph_root, proof_dir) - result.operations["op_rederive"] = {"ok": ok, "detail": detail} - if not ok: - result.rejected = ValidatorReject("op_rederive_trajectory", detail) - return result - + # NOTE: proof-of-EXECUTION re-derivation (op_rederive_trajectory) is intentionally + # NOT run here. It is expensive (GPU-minutes) and only meaningful for a bundle that + # would actually take the crown, so the caller (validator.router.process_submission) + # invokes it as a pre-crown gate — AFTER op4 scores the bundle and ONLY when it beats + # the king. Running it per-submission here would be O(submissions) GPU cost + a DoS + # surface for any op1-op3-passing bundle. ok, detail, hidden_eval = op4_hidden_eval(ralph_root, proof_dir, chain=chain) result.operations["op4_hidden_eval"] = {"ok": ok, "detail": detail} if not ok: