From dc25241aac756e6587ac765eb4f0f68bf7ea5406 Mon Sep 17 00:00:00 2001 From: karpabot Date: Sun, 5 Jul 2026 04:00:30 +0000 Subject: [PATCH] integrity: compile-aware MFU ceiling (stops the uncompiled-Muon forgery flood) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The micro_batch ceiling (PR#96) still let uncompiled Muon runs declare torch.compile-grade throughput: andreastanm 239k (37% MFU) and Kaizen 356k (54.9% MFU) both sailed under the 70% cap because the cap is blind to whether the recipe actually compiles. Measured on a real H200 (ubatch-128/seq-512): Muon UNCOMPILED = 86,436 tok/s (~13% MFU); torch.compile'd = 207,700 (~32%). - UNCOMPILED_MFU_CEILING=0.35: an uncompiled recipe declaring MFU above this is a forged-down wall_clock (honest uncompiled tops out ~27%, 5Fbh5xe). - max_plausible_mfu(...recipe_compiles): compiled keeps the micro_batch ceiling (up to 70%); uncompiled is capped at 0.35. - _recipe_uses_compile: config.compile truthy AND patch.diff actually contains torch.compile — a bare config flag is a no-op (hasattr-dropped) and can't be trusted to earn the higher ceiling. - op1 threads it into check_compute_plausibility. Verified on the real bundles: Kaizen(356k,uncompiled) + andreastanm(239k, uncompiled) REJECTED; danielortega(239k,COMPILED) + 5Fbh5xe(174k,uncompiled,27%) PASS. 8 new tests. Interim while the on-GPU re-measurement gate is built. Co-Authored-By: Claude Opus 4.8 --- tests/test_integrity_trained.py | 38 +++++++++++++++ validator/integrity.py | 84 +++++++++++++++++++++++---------- validator/validator.py | 22 ++++++++- 3 files changed, 117 insertions(+), 27 deletions(-) diff --git a/tests/test_integrity_trained.py b/tests/test_integrity_trained.py index e192a6c..afea7c5 100644 --- a/tests/test_integrity_trained.py +++ b/tests/test_integrity_trained.py @@ -221,6 +221,44 @@ def test_accepts_honest_small_batch_run(): assert check_compute_plausibility(fs, H100)[0] +# --- compile-aware ceiling (the 2026-07-04/05 uncompiled-Muon forgery flood) -------- +def test_max_plausible_mfu_compile_aware(): + # UNCOMPILED recipes are capped at UNCOMPILED_MFU_CEILING (0.35); compiled keep the + # micro_batch ceiling. Measured: uncompiled Muon ubatch-128 = ~13% MFU, compiled ~32%. + assert max_plausible_mfu(128, 512, recipe_compiles=True) == 0.70 # compiled -> full ceiling + assert max_plausible_mfu(128, 512, recipe_compiles=False) == 0.35 # uncompiled -> capped + assert max_plausible_mfu(64, 512, recipe_compiles=False) == 0.35 # min(0.60, 0.35) -> capped + assert max_plausible_mfu(8, 512, recipe_compiles=False) == 0.33 # min(0.33, 0.35) -> small-batch wins + assert max_plausible_mfu(4, 512, recipe_compiles=False) == 0.25 # M=2048 base 0.25 already below cap + assert max_plausible_mfu(None, None, recipe_compiles=False) == 0.70 # unknown cfg -> never false-reject + + +def test_rejects_kaizen_uncompiled_forgery(): + # Kaizen0304 (b1930c27): 356k tok/s at micro_batch=128, muon, NO torch.compile => 54.9% + # MFU. Passed the 70% ceiling; measured uncompiled H200 ceiling is 86k (~13% MFU). + fs = {"tokens_seen": 4_190_000_000, "wall_clock_s": 11_763, "n_params": 253_874_184, + "config": {"micro_batch_size": 128, "seq_len": 512}} # ~356k tok/s, ~55% MFU + ok, reason = check_compute_plausibility(fs, H100, recipe_compiles=False) + assert not ok + assert "torch.compile" in reason and "compile=NO" in reason + + +def test_rejects_andreastanm_uncompiled_but_accepts_danielortega_compiled(): + # SAME declared throughput (239k tok/s, ~37% MFU, ubatch-128), opposite verdicts: + # andreastanm ran UNCOMPILED (impossible -> reject); danielortega COMPILED (real -> pass). + fs = {"tokens_seen": 2_621_440_000, "wall_clock_s": 10_962, "n_params": 253_874_184, + "config": {"micro_batch_size": 128, "seq_len": 512}} # ~239k tok/s, ~37% MFU + assert not check_compute_plausibility(fs, H100, recipe_compiles=False)[0] # andreastanm + assert check_compute_plausibility(fs, H100, recipe_compiles=True)[0] # danielortega + + +def test_accepts_honest_uncompiled_field_run(): + # 5Fbh5xe (honest, verified): 174k tok/s uncompiled ubatch-128 H200 = ~27% MFU < 35% cap. + fs = {"tokens_seen": 1_913_000_000, "wall_clock_s": 10_989, "n_params": 253_874_184, + "config": {"micro_batch_size": 128, "seq_len": 512}} # ~174k tok/s, ~27% MFU + assert check_compute_plausibility(fs, H100, recipe_compiles=False)[0] + + def test_accepts_honest_large_batch_field_runs(): # The restored king 5CqhtHE7 (ubatch 64, 137k tok/s, ~21% MFU) and 5Fbh5xe (ubatch 128, # 174k, ~27%) are physically plausible and must NOT be false-rejected. diff --git a/validator/integrity.py b/validator/integrity.py index e40550c..cdf881c 100644 --- a/validator/integrity.py +++ b/validator/integrity.py @@ -116,8 +116,21 @@ def _gpu_bf16_peak_flops(gpu_name: str | None) -> float: return _DEFAULT_PEAK_TFLOPS * 1e12 -def max_plausible_mfu(micro_batch: int | None, seq_len: int | None) -> float: - """Micro-batch-aware MFU ceiling. +# An UNCOMPILED recipe physically tops out far below a torch.compile'd one: the +# per-op kernel-launch overhead (esp. Muon's Newton-Schulz loop) dominates. Measured +# on a real H200 at ubatch-128/seq-512: Muon UNCOMPILED = 86,436 tok/s = ~13% MFU; +# torch.compile'd = 207,700 tok/s = ~32% MFU (a 2.4x speedup). Honest uncompiled runs +# observed on-chain: Muon ~13-21%, lighter (AdamW) ~27% (5Fbh5xe). So an uncompiled +# recipe declaring MFU above this ceiling is a forged-down wall_clock (andreastanm 37%, +# Kaizen 54.9% — both uncompiled — sailed under the flat 70% cap). Set with ~8pp headroom +# over the honest uncompiled max (27%). Compiled recipes keep the micro_batch ceiling. +UNCOMPILED_MFU_CEILING = 0.35 + + +def max_plausible_mfu( + micro_batch: int | None, seq_len: int | None, recipe_compiles: bool = True +) -> float: + """Micro-batch- AND compile-aware MFU ceiling. The flat MAX_PLAUSIBLE_MFU cap is blind to micro_batch, which is the loophole a forged-down wall_clock exploited: the per-forward matmul M-dimension is @@ -130,26 +143,31 @@ def max_plausible_mfu(micro_batch: int | None, seq_len: int | None) -> float: M=4096->11%, M=16384->15%, M=32768->21%, M=65536->27%), so honest runs — even well-optimized ones — never trip them; only physically-impossible small-batch throughput does. Unknown/zero config -> fall back to the flat cap (never false-reject). - NOTE: a static ceiling only halves the forgery window (~6x -> ~3x at small batch); the - airtight fix is on-GPU throughput re-measurement (a pre-crown, would-beat-king gate). + + recipe_compiles=False (no real torch.compile in the recipe) tightens the ceiling to + UNCOMPILED_MFU_CEILING — the class that flooded 2026-07-04/05: Muon runs that declare + a torch.compile-grade throughput without actually compiling. NOTE: a static ceiling + only narrows the window; the airtight fix is on-GPU throughput re-measurement. """ try: m = int(micro_batch or 0) * int(seq_len or 0) except (TypeError, ValueError): return MAX_PLAUSIBLE_MFU if m <= 0: - return MAX_PLAUSIBLE_MFU + return MAX_PLAUSIBLE_MFU # unknown config -> flat cap, never false-reject if m >= 65536: - return 0.70 - if m >= 32768: - return 0.60 - if m >= 16384: - return 0.50 - if m >= 8192: - return 0.42 - if m >= 4096: - return 0.33 - return 0.25 + base = 0.70 + elif m >= 32768: + base = 0.60 + elif m >= 16384: + base = 0.50 + elif m >= 8192: + base = 0.42 + elif m >= 4096: + base = 0.33 + else: + base = 0.25 + return min(base, UNCOMPILED_MFU_CEILING) if not recipe_compiles else base def check_compute_plausibility( @@ -157,16 +175,18 @@ def check_compute_plausibility( calibration: dict | None = None, *, max_mfu: float = MAX_PLAUSIBLE_MFU, + recipe_compiles: bool = True, ) -> tuple[bool, str]: """Reject a bundle whose declared training throughput is physically impossible. tokens_seen / wall_clock_s implies ~6*N FLOPs/token; over the declared GPU's - bf16 peak that is the achieved MFU. The ceiling is micro-batch-aware - (max_plausible_mfu): the flat cap is too loose at small micro_batch, where the GPU - is underutilized and high MFU is physically impossible. An implied MFU over the + bf16 peak that is the achieved MFU. The ceiling is micro-batch- AND compile-aware + (max_plausible_mfu): the flat cap is too loose at small micro_batch and for + uncompiled recipes, where high MFU is physically impossible. An implied MFU over the ceiling means the wall_clock_s (and the efficiency-gate compute cost it drives) is - fabricated. Best-effort: a missing/incomplete training_summary is skipped (deferred - to the other gates), not rejected. Returns (ok, reason); ok=False -> reject. + fabricated. recipe_compiles=False (no real torch.compile) tightens the ceiling. + Best-effort: a missing/incomplete training_summary is skipped (deferred to the other + gates), not rejected. Returns (ok, reason); ok=False -> reject. """ fs = final_state or {} try: @@ -179,8 +199,11 @@ def check_compute_plausibility( return True, "compute-plausibility: incomplete training_summary (skipped)" gpu = (calibration or {}).get("gpu_name") or fs.get("gpu_name") or fs.get("device") or "" cfg = fs.get("config") or {} - # micro-batch-aware ceiling, tightened by any explicitly-passed max_mfu. - ceiling = min(max_mfu, max_plausible_mfu(cfg.get("micro_batch_size"), cfg.get("seq_len"))) + # micro-batch- + compile-aware ceiling, tightened by any explicitly-passed max_mfu. + ceiling = min( + max_mfu, + max_plausible_mfu(cfg.get("micro_batch_size"), cfg.get("seq_len"), recipe_compiles), + ) flops_per_s = 6.0 * n * tokens / wall # 6N FLOPs/token (fwd+bwd) mfu = flops_per_s / _gpu_bf16_peak_flops(gpu) if mfu > ceiling: @@ -189,13 +212,22 @@ def check_compute_plausibility( f" at micro_batch={mb}, seq_len={sl} (effective batch {int(mb) * int(sl)} tokens/fwd)" if isinstance(mb, int) and isinstance(sl, int) else "" ) + why = ( + "the recipe does NOT use torch.compile, so this MFU is physically impossible " + "(uncompiled Muon tops out ~13-27% MFU, measured)" + if not recipe_compiles + else "small micro_batch underutilizes the GPU, so this throughput is physically impossible" + ) return False, ( f"fabricated compute: {tokens / wall:,.0f} tok/s for a {n / 1e6:.0f}M model on " - f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {ceiling * 100:.0f}% ceiling{batch_note}); " - f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable — small micro_batch " - f"underutilizes the GPU, so this throughput is physically impossible" + f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {ceiling * 100:.0f}% ceiling{batch_note}, " + f"compile={'yes' if recipe_compiles else 'NO'}); wall_clock_s={wall:.0f}s for " + f"{tokens:,.0f} tokens is not achievable — {why}" ) - return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU (ceiling {ceiling * 100:.0f}%)" + return True, ( + f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU " + f"(ceiling {ceiling * 100:.0f}%, compile={'yes' if recipe_compiles else 'no'})" + ) # --- Training-log integrity (anti fabricated-log / forged-down wall_clock) ------ diff --git a/validator/validator.py b/validator/validator.py index 69476ce..10a94d6 100644 --- a/validator/validator.py +++ b/validator/validator.py @@ -236,6 +236,23 @@ def _canonical_code_epoch() -> float | None: return None +def _recipe_uses_compile(final_state: dict, proof_dir: Path) -> bool: + """True only if the recipe ACTUALLY torch.compiles: the config requests it AND the + patch adds a torch.compile call. A bare config `compile: true` is a no-op on the + canonical recipe (dropped by hasattr in train.py) and cannot be trusted — a forger + could set it to earn the higher compiled-MFU ceiling while running uncompiled. This + gates the compute-plausibility ceiling: uncompiled Muon tops out ~13-27% MFU + (measured), so an uncompiled recipe declaring compiled-grade throughput is forged.""" + cfg = (final_state or {}).get("config") or {} + if not cfg.get("compile"): + return False + try: + patch = (proof_dir / "patch.diff").read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return "torch.compile" in patch + + def op1_diff_and_integrity( ralph_root: Path, submission_payload: dict, @@ -401,7 +418,10 @@ def op1_diff_and_integrity( calibration = json.loads(cal_path.read_text(encoding="utf-8", errors="replace")) except (ValueError, OSError): calibration = {} - ok_c, detail_c = check_compute_plausibility(final_state, calibration) + recipe_compiles = _recipe_uses_compile(final_state, proof_dir) + ok_c, detail_c = check_compute_plausibility( + final_state, calibration, recipe_compiles=recipe_compiles + ) if not ok_c: return False, detail_c # Fabricated-log guard: reject a training_log forged to disguise a