Anti-gaming hardening (un-gameable plan) — tracking PR

eval/benchmark.py

@@ -78,3 +78,49 @@ def make_placeholder_examples(
             "distractors": distractors,
         })
     return examples
+
+
+def benchmark_blind_forgeable(examples: list[dict], sigma: float = 3.0) -> tuple[bool, str]:
+    """Detect a non-content-whitened (blind-forgeable) benchmark file.
+
+    compute_benchmark_score ranks [target_id] + distractors by the model's logits.
+    If the correct target is distinguishable from the distractors by a monotone
+    function of the token-id alone (e.g. targets are systematically small while
+    distractors are uniform), a model that never learned anything wins by scoring
+    `-token_id` — the exact 0.785-vs-0.2 break found on the deployed file. A
+    content-whitened file (make_placeholder_examples: target + distractors from one
+    exchangeable rng.choice pool, uniform target slot) makes every blind monotone
+    score chance-level.
+
+    Returns (forgeable, reason). forgeable=True -> the file must be regenerated
+    with make_placeholder_examples before it can gate the crown.
+    """
+    import numpy as np
+
+    if not examples:
+        return False, "empty benchmark (nothing to forge)"
+    n = len(examples)
+    k = 1 + len(examples[0].get("distractors", []))
+    if k <= 1:
+        return False, "single-candidate benchmark"
+    chance = 1.0 / k
+    sd = (chance * (1.0 - chance) / n) ** 0.5
+    bar = chance + sigma * sd
+    worst = 0.0
+    worst_label = ""
+    for sign, label in ((-1.0, "-id"), (1.0, "+id")):
+        correct = 0
+        for e in examples:
+            cands = [e["target_id"]] + list(e["distractors"])
+            if int(np.argmax([sign * float(t) for t in cands])) == 0:
+                correct += 1
+        acc = correct / n
+        if acc > worst:
+            worst, worst_label = acc, label
+    if worst > bar:
+        return True, (
+            f"blind-forgeable benchmark: score={worst_label} sweep scores {worst:.3f} "
+            f"> chance {chance:.3f} + {sigma:.0f}sigma ({bar:.3f}); regenerate "
+            f"active_benchmark.json with the content-whitened make_placeholder_examples"
+        )
+    return False, f"benchmark not blind-forgeable (worst blind ±id = {worst:.3f} <= {bar:.3f})"

tests/test_benchmark_selftest.py

@@ -0,0 +1,34 @@
+"""The blind-forgeability detector must flag an id-biased benchmark (the deployed
+0.785 break) and pass the content-whitened generator."""
+from __future__ import annotations
+
+import numpy as np
+
+from eval.benchmark import benchmark_blind_forgeable, make_placeholder_examples
+
+
+def test_whitened_generator_not_forgeable():
+    ex = make_placeholder_examples(n=300, seed=1, n_candidates=5)
+    forgeable, reason = benchmark_blind_forgeable(ex)
+    assert not forgeable, reason
+
+
+def test_id_biased_benchmark_is_flagged():
+    # Targets systematically small, distractors uniform-large -> blind score=-id wins
+    # (the exact deployed break: target mean ~4550 vs distractor mean ~24917).
+    rng = np.random.default_rng(0)
+    ex = [
+        {
+            "context_ids": [1, 2, 3],
+            "target_id": int(rng.integers(0, 5000)),
+            "distractors": [int(rng.integers(20000, 50000)) for _ in range(4)],
+        }
+        for _ in range(300)
+    ]
+    forgeable, reason = benchmark_blind_forgeable(ex)
+    assert forgeable and "blind-forgeable" in reason
+
+
+def test_empty_and_single_candidate_are_safe():
+    assert not benchmark_blind_forgeable([])[0]
+    assert not benchmark_blind_forgeable([{"target_id": 1, "distractors": []}])[0]

tests/test_hf_poller_decrypt_feedback.py

@@ -0,0 +1,126 @@
+"""Tests for decrypt-failure handling in the HF poller.
+
+A bundle sealed to an outdated validator pubkey can never be decrypted. Before,
+poll_hub re-downloaded + re-failed it every epoch forever (never stamped done).
+Now:
+  - download_one returns "decrypt_failed" (permanent) vs "unavailable" (transient),
+  - poll_hub stamps "decrypt_failed" done (stops the churn) but leaves "unavailable"
+    for retry,
+  - and the miner's PR is closed with the current pubkey so they can re-seal.
+"""
+
+import huggingface_hub
+
+from validator import hf_poller as hp
+from validator.version import VALIDATOR_VERSION
+
+
+def _pr(prefix, num):
+    return {
+        "bundle_id": prefix + "0" * (64 - len(prefix)),
+        "pr_num": num,
+        "git_ref": f"refs/pr/{num}",
+        "created_at": f"2026-06-2{num}T00:00:00+00:00",
+    }
+
+
+def test_decrypt_failed_marked_done_transient_retried(tmp_path, monkeypatch):
+    prs = [_pr("ok", 1), _pr("de", 2), _pr("un", 3)]
+    monkeypatch.setattr(hp, "list_remote_submissions", lambda repo_id, token=None: prs)
+
+    status_by_prefix = {"ok": "ok", "de": "decrypt_failed", "un": "unavailable"}
+    monkeypatch.setattr(
+        hp, "download_one",
+        lambda bid, repo_id, dest, **kw: status_by_prefix[bid[:2]],
+    )
+
+    downloaded = hp.poll_hub(tmp_path)
+    assert downloaded == [prs[0]["bundle_id"]]  # only the 'ok' bundle materialised
+
+    done = hp._load_state(tmp_path)["processed"]
+    assert done.get(prs[0]["bundle_id"]) == VALIDATOR_VERSION   # ok → done
+    assert done.get(prs[1]["bundle_id"]) == VALIDATOR_VERSION   # decrypt_failed → done (no re-churn)
+    assert prs[2]["bundle_id"] not in done                      # unavailable → NOT done (retries)
+
+
+def test_close_pr_posts_current_pubkey_and_closes(tmp_path, monkeypatch):
+    from proof import bundle_crypto
+
+    calls = {}
+
+    class _Api:
+        def __init__(self, token=None):
+            pass
+
+        def change_discussion_status(self, repo_id, discussion_num, new_status,
+                                     repo_type=None, comment=None):
+            calls.update(repo_id=repo_id, num=discussion_num, status=new_status,
+                         repo_type=repo_type, comment=comment)
+
+    monkeypatch.setattr(huggingface_hub, "HfApi", _Api)
+
+    hp._close_pr_wrong_key("RalphLabsAI/proof-bundles", 42, None, "a" * 64, "MAC check failed")
+
+    assert calls["num"] == 42
+    assert calls["status"] == "closed"
+    assert calls["repo_type"] == "dataset"
+    assert bundle_crypto.DEFAULT_VALIDATOR_PUBKEY in calls["comment"]
+    assert "re-seal" in calls["comment"].lower()
+
+
+def test_close_pr_swallows_api_errors(monkeypatch):
+    """A failed close (perms/network) must not raise — dedup still proceeds."""
+    class _Api:
+        def __init__(self, token=None):
+            pass
+
+        def change_discussion_status(self, **kw):
+            raise RuntimeError("403 forbidden")
+
+    monkeypatch.setattr(huggingface_hub, "HfApi", _Api)
+    hp._close_pr_wrong_key("RalphLabsAI/proof-bundles", 7, None, "b" * 64, "boom")  # no raise
+
+
+def test_close_pr_noop_without_pr_num(monkeypatch):
+    def _boom(token=None):
+        raise AssertionError("HfApi should not be constructed when pr_num is None")
+
+    monkeypatch.setattr(huggingface_hub, "HfApi", _boom)
+    hp._close_pr_wrong_key("RalphLabsAI/proof-bundles", None, None, "c" * 64, "x")  # no raise
+
+
+# --- HF fetch timeout (hang / slowloris protection) ---------------------------
+
+def test_with_timeout_fires_on_slow_call():
+    import time
+    # a call that runs longer than the cap raises _FetchTimeout (not a hang)
+    try:
+        hp._with_timeout(1, time.sleep, 5)
+        raise AssertionError("expected _FetchTimeout")
+    except hp._FetchTimeout:
+        pass
+
+
+def test_with_timeout_passes_through_fast_call():
+    assert hp._with_timeout(5, lambda: 42) == 42
+    import signal
+    # handler + alarm are restored (no lingering alarm)
+    assert signal.getitimer(signal.ITIMER_REAL)[0] == 0.0
+
+
+def test_poll_hub_stalled_retries_then_marks_done(tmp_path, monkeypatch):
+    monkeypatch.setattr(hp, "_MAX_STALLS", 2)
+    hp._STALL_COUNTS.clear()
+    prs = [_pr("st", 1)]
+    monkeypatch.setattr(hp, "list_remote_submissions", lambda repo_id, token=None: prs)
+    monkeypatch.setattr(hp, "download_one", lambda bid, repo_id, dest, **kw: "stalled")
+    bid = prs[0]["bundle_id"]
+
+    # 1st stall: retried (not done)
+    hp.poll_hub(tmp_path)
+    assert bid not in hp._load_state(tmp_path)["processed"]
+    assert hp._STALL_COUNTS[bid] == 1
+    # 2nd stall reaches _MAX_STALLS: marked done so it can't re-stall forever
+    hp.poll_hub(tmp_path)
+    assert hp._load_state(tmp_path)["processed"].get(bid) == VALIDATOR_VERSION
+    assert bid not in hp._STALL_COUNTS

tests/test_intake_caps.py

@@ -0,0 +1,23 @@
+"""DoS intake caps: a decompression bomb must be rejected before unpack."""
+from __future__ import annotations
+
+import gzip
+import io
+
+import pytest
+
+from validator.hf_poller import _assert_blob_safe
+
+
+def test_normal_blob_passes():
+    _assert_blob_safe(gzip.compress(b"ok" * 2000))
+
+
+def test_decompression_bomb_rejected():
+    buf = io.BytesIO()
+    with gzip.GzipFile(fileobj=buf, mode="wb") as g:
+        z = b"\0" * (1 << 20)
+        for _ in range(4096):  # 4 GiB decompressed from a ~4 MB blob
+            g.write(z)
+    with pytest.raises(ValueError, match="bomb|cap"):
+        _assert_blob_safe(buf.getvalue())

tests/test_scoring.py

@@ -48,11 +48,13 @@ def test_not_decisive_inside_noise_band():
     assert sr.decisively_beats_king is False
 
 
-def test_not_decisive_when_bench_regresses_and_quality_below_dominant():
+def test_not_decisive_when_bench_regresses_and_quality_below_dominant(monkeypatch):
     """bpb wins past noise but BELOW DOMINANT_QUALITY_MULTIPLIER, and bench
     regresses past -noise — not decisive. The quality_gain (0.02 ≈ 1.5x
     noise) doesn't clear the dominant-quality branch (3x noise = 0.039)
-    and the paired-axes branch fails because bench drop is too large."""
+    and the paired-axes branch fails because bench drop is too large.
+    (Legacy benchmark-in-crown behavior — now behind RALPH_BENCHMARK_CROWN.)"""
+    monkeypatch.setenv("RALPH_BENCHMARK_CROWN", "1")
     sr = score_bundle(
         val_bpb=1.48,                # 0.02 better — above noise, below dominant
         benchmark_accuracy=0.40,     # 0.05 worse — outside -noise_floor
@@ -64,7 +66,7 @@ def test_not_decisive_when_bench_regresses_and_quality_below_dominant():
     assert sr.decisively_beats_king is False
 
 
-def test_v0_10_branch_a_no_longer_crowns_when_bench_regresses():
+def test_v0_10_branch_a_no_longer_crowns_when_bench_regresses(monkeypatch):
     """v0.10 Goodhart closure: under the GUARDED dominant-quality clause, a
     big quality_gain (~6x noise) does NOT crown if benchmark regressed past
     the noise floor. This is the seed-search backdoor the v0.10 fix closes
@@ -74,7 +76,9 @@ def test_v0_10_branch_a_no_longer_crowns_when_bench_regresses():
     the v0.10 guard the case is plain_failure because the benchmark drop
     indicates the win may be spurious. Branch B and Branch C also fail
     (Branch B needs benchmark-no-regress; Branch C needs benchmark > noise).
+    (Legacy benchmark-in-crown behavior — now behind RALPH_BENCHMARK_CROWN.)
     """
+    monkeypatch.setenv("RALPH_BENCHMARK_CROWN", "1")
     sr = score_bundle(
         val_bpb=1.42,                # 0.08 better — well above 3x noise
         benchmark_accuracy=0.40,     # 0.05 worse — outside -noise_floor
@@ -117,10 +121,11 @@ def test_dominant_quality_threshold_just_above_3x_noise():
     assert sr.decisively_beats_king is True
 
 
-def test_dominant_quality_threshold_does_not_fire_just_below():
+def test_dominant_quality_threshold_does_not_fire_just_below(monkeypatch):
     """Boundary the other side: 2.9x noise (below 3x) doesn't trigger the
     dominant-quality branch. If bench also regresses past -noise the
     submission falls back to non-decisive."""
+    monkeypatch.setenv("RALPH_BENCHMARK_CROWN", "1")
     sr = score_bundle(
         val_bpb=1.4623,              # gain = 0.0377 = 2.9 * 0.013
         benchmark_accuracy=0.00,     # bench tanked
@@ -132,9 +137,10 @@ def test_dominant_quality_threshold_does_not_fire_just_below():
     assert sr.decisively_beats_king is False
 
 
-def test_decisive_via_benchmark_axis():
-    """Benchmark wins, val_bpb roughly tied — decisive via the benchmark
-    arm of the OR-of-AND condition."""
+def test_decisive_via_benchmark_axis(monkeypatch):
+    """Benchmark wins, val_bpb roughly tied — decisive via the benchmark arm of
+    the OR-of-AND condition. (Legacy behavior — now behind RALPH_BENCHMARK_CROWN.)"""
+    monkeypatch.setenv("RALPH_BENCHMARK_CROWN", "1")
     sr = score_bundle(
         val_bpb=1.50,  # tied
         benchmark_accuracy=0.50,  # 0.05 better — outside noise floor
@@ -146,6 +152,22 @@ def test_decisive_via_benchmark_axis():
     assert sr.decisively_beats_king is True
 
 
+def test_benchmark_neutralized_by_default(monkeypatch):
+    """Default (no RALPH_BENCHMARK_CROWN): the forgeable in-container benchmark can
+    neither crown a challenger alone (Branch C gone) nor block a genuine val_bpb win."""
+    monkeypatch.delenv("RALPH_BENCHMARK_CROWN", raising=False)
+    # bench-only "win", val_bpb tied -> NO crown.
+    assert score_bundle(
+        val_bpb=1.50, benchmark_accuracy=0.50, king_val_bpb=1.5, king_benchmark=0.45,
+        noise_floor_margin=0.013, matmul_ms=5.0, wall_clock_s=60.0,
+    ).decisively_beats_king is False
+    # val_bpb win with benchmark tanked -> STILL crowns (benchmark ignored).
+    assert score_bundle(
+        val_bpb=1.42, benchmark_accuracy=0.0, king_val_bpb=1.5, king_benchmark=0.45,
+        noise_floor_margin=0.013, matmul_ms=5.0, wall_clock_s=60.0,
+    ).decisively_beats_king is True
+
+
 def test_nan_val_bpb_not_decisive():
     """NaN val_bpb must NOT crown a king — non-finite metrics rejected."""
     sr = score_bundle(val_bpb=float("nan"), **_BASE)