diff --git a/tests/test_integrity_trained.py b/tests/test_integrity_trained.py index 6bf578b..e192a6c 100644 --- a/tests/test_integrity_trained.py +++ b/tests/test_integrity_trained.py @@ -16,6 +16,7 @@ check_recipe_config_matches_proof, check_training_timing, compare_loss_trajectory, + max_plausible_mfu, nats_per_token_from_bpb, ) @@ -187,6 +188,64 @@ def test_accepts_an_optimized_run_under_the_ceiling(): assert check_compute_plausibility(fs, H100)[0] +# --- micro-batch-aware MFU ceiling (the #1217 forgery class) ------------------ +def test_max_plausible_mfu_ceilings(): + # M = micro_batch * seq_len; ceiling rises with the effective batch. + assert max_plausible_mfu(8, 512) == 0.33 # M=4096 + assert max_plausible_mfu(16, 512) == 0.42 # M=8192 + assert max_plausible_mfu(32, 512) == 0.50 # M=16384 + assert max_plausible_mfu(64, 512) == 0.60 # M=32768 + assert max_plausible_mfu(128, 512) == 0.70 # M=65536 + # unknown/zero config -> flat cap, never false-reject + assert max_plausible_mfu(None, 512) == 0.70 + assert max_plausible_mfu(0, 0) == 0.70 + assert max_plausible_mfu("x", 512) == 0.70 + + +def test_rejects_1217_microbatch_forgery(): + # #1217 (5FTfrwU3): 2.62B tokens in 8074.8s at micro_batch=8/seq=512 => 324k tok/s + # => 50% MFU. Passed the flat 70% cap; measured-achievable at micro_batch=8 is ~11%. + # The micro-batch ceiling (33% at M=4096) rejects it. + fs = {"tokens_seen": 2_621_440_000, "wall_clock_s": 8074.8, "n_params": 253_874_184, + "config": {"micro_batch_size": 8, "seq_len": 512}} + ok, reason = check_compute_plausibility(fs, H100) + assert not ok + assert "micro_batch=8" in reason and "ceiling" in reason + + +def test_accepts_honest_small_batch_run(): + # An HONEST micro_batch=8 run is SLOW (~70k tok/s SXM, ~11% MFU) -> passes; the gate + # rejects only physically-impossible small-batch throughput, not small batches per se. + fs = {"tokens_seen": 2_621_440_000, "wall_clock_s": 37_000, "n_params": 253_874_184, + "config": {"micro_batch_size": 8, "seq_len": 512}} # ~71k tok/s, ~11% MFU + assert check_compute_plausibility(fs, H100)[0] + + +def test_accepts_honest_large_batch_field_runs(): + # The restored king 5CqhtHE7 (ubatch 64, 137k tok/s, ~21% MFU) and 5Fbh5xe (ubatch 128, + # 174k, ~27%) are physically plausible and must NOT be false-rejected. + king = {"tokens_seen": 1_991_000_000, "wall_clock_s": 14_491, "n_params": 253_874_184, + "config": {"micro_batch_size": 64, "seq_len": 512}} # ~137k, 21% < 60% + chal = {"tokens_seen": 1_913_000_000, "wall_clock_s": 10_989, "n_params": 253_874_184, + "config": {"micro_batch_size": 128, "seq_len": 512}} # ~174k, 27% < 70% + assert check_compute_plausibility(king, H100)[0] + assert check_compute_plausibility(chal, H100)[0] + + +def test_microbatch_forgery_at_large_batch_still_caught_by_flat_cap(): + # If a forger moves to a LARGE micro_batch to dodge the small-batch ceiling, a 3x + # wall under-declaration lands >70% MFU and the flat cap catches it. 174k*3=522k => 80%. + fs = {"tokens_seen": 1_913_000_000, "wall_clock_s": 3_663, "n_params": 253_874_184, + "config": {"micro_batch_size": 128, "seq_len": 512}} # ~522k tok/s, ~80% MFU + assert not check_compute_plausibility(fs, H100)[0] + + +def test_no_config_falls_back_to_flat_cap(): + # Backward compat: a bundle with no config field keeps the flat 70% behaviour. + fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184} # 39% MFU + assert check_compute_plausibility(fs, H100)[0] + + def test_incomplete_training_summary_is_skipped_not_rejected(): assert check_compute_plausibility({"tokens_seen": 0, "wall_clock_s": 0}, {})[0] assert check_compute_plausibility({}, None)[0] diff --git a/validator/integrity.py b/validator/integrity.py index fba4930..e40550c 100644 --- a/validator/integrity.py +++ b/validator/integrity.py @@ -116,6 +116,42 @@ def _gpu_bf16_peak_flops(gpu_name: str | None) -> float: return _DEFAULT_PEAK_TFLOPS * 1e12 +def max_plausible_mfu(micro_batch: int | None, seq_len: int | None) -> float: + """Micro-batch-aware MFU ceiling. + + The flat MAX_PLAUSIBLE_MFU cap is blind to micro_batch, which is the loophole a + forged-down wall_clock exploited: the per-forward matmul M-dimension is + micro_batch*seq_len, and small M underutilizes the tensor cores, so the ACHIEVABLE + MFU drops sharply with the effective batch. At micro_batch=8/seq=512 (M=4096) a real + H100 tops out ~11% MFU (measured, reproduced), yet the flat 70% cap let a bundle + declare 50% MFU there — a ~4.5x wall_clock under-declaration — and pass. + + These ceilings are set ~3x above the EMPIRICALLY-ACHIEVED MFU (H100, seq 512: + M=4096->11%, M=16384->15%, M=32768->21%, M=65536->27%), so honest runs — even + well-optimized ones — never trip them; only physically-impossible small-batch + throughput does. Unknown/zero config -> fall back to the flat cap (never false-reject). + NOTE: a static ceiling only halves the forgery window (~6x -> ~3x at small batch); the + airtight fix is on-GPU throughput re-measurement (a pre-crown, would-beat-king gate). + """ + try: + m = int(micro_batch or 0) * int(seq_len or 0) + except (TypeError, ValueError): + return MAX_PLAUSIBLE_MFU + if m <= 0: + return MAX_PLAUSIBLE_MFU + if m >= 65536: + return 0.70 + if m >= 32768: + return 0.60 + if m >= 16384: + return 0.50 + if m >= 8192: + return 0.42 + if m >= 4096: + return 0.33 + return 0.25 + + def check_compute_plausibility( final_state: dict, calibration: dict | None = None, @@ -125,10 +161,12 @@ def check_compute_plausibility( """Reject a bundle whose declared training throughput is physically impossible. tokens_seen / wall_clock_s implies ~6*N FLOPs/token; over the declared GPU's - bf16 peak that is the achieved MFU. An implied MFU > `max_mfu` means the - wall_clock_s (and the efficiency-gate compute cost it drives) is fabricated. - Best-effort: a missing/incomplete training_summary is skipped (deferred to the - other gates), not rejected. Returns (ok, reason); ok=False -> reject. + bf16 peak that is the achieved MFU. The ceiling is micro-batch-aware + (max_plausible_mfu): the flat cap is too loose at small micro_batch, where the GPU + is underutilized and high MFU is physically impossible. An implied MFU over the + ceiling means the wall_clock_s (and the efficiency-gate compute cost it drives) is + fabricated. Best-effort: a missing/incomplete training_summary is skipped (deferred + to the other gates), not rejected. Returns (ok, reason); ok=False -> reject. """ fs = final_state or {} try: @@ -140,15 +178,24 @@ def check_compute_plausibility( if tokens <= 0 or wall <= 0 or n <= 0: return True, "compute-plausibility: incomplete training_summary (skipped)" gpu = (calibration or {}).get("gpu_name") or fs.get("gpu_name") or fs.get("device") or "" + cfg = fs.get("config") or {} + # micro-batch-aware ceiling, tightened by any explicitly-passed max_mfu. + ceiling = min(max_mfu, max_plausible_mfu(cfg.get("micro_batch_size"), cfg.get("seq_len"))) flops_per_s = 6.0 * n * tokens / wall # 6N FLOPs/token (fwd+bwd) mfu = flops_per_s / _gpu_bf16_peak_flops(gpu) - if mfu > max_mfu: + if mfu > ceiling: + mb, sl = cfg.get("micro_batch_size"), cfg.get("seq_len") + batch_note = ( + f" at micro_batch={mb}, seq_len={sl} (effective batch {int(mb) * int(sl)} tokens/fwd)" + if isinstance(mb, int) and isinstance(sl, int) else "" + ) return False, ( f"fabricated compute: {tokens / wall:,.0f} tok/s for a {n / 1e6:.0f}M model on " - f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {max_mfu * 100:.0f}% physical max); " - f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable" + f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {ceiling * 100:.0f}% ceiling{batch_note}); " + f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable — small micro_batch " + f"underutilizes the GPU, so this throughput is physically impossible" ) - return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU" + return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU (ceiling {ceiling * 100:.0f}%)" # --- Training-log integrity (anti fabricated-log / forged-down wall_clock) ------