integrity: micro-batch-aware MFU ceiling (closes the #1217 compute-forgery class)

tests/test_integrity_trained.py

@@ -16,6 +16,7 @@
     check_recipe_config_matches_proof,
     check_training_timing,
     compare_loss_trajectory,
+    max_plausible_mfu,
     nats_per_token_from_bpb,
 )
 
@@ -187,6 +188,64 @@ def test_accepts_an_optimized_run_under_the_ceiling():
     assert check_compute_plausibility(fs, H100)[0]
 
 
+# --- micro-batch-aware MFU ceiling (the #1217 forgery class) ------------------
+def test_max_plausible_mfu_ceilings():
+    # M = micro_batch * seq_len; ceiling rises with the effective batch.
+    assert max_plausible_mfu(8, 512) == 0.33     # M=4096
+    assert max_plausible_mfu(16, 512) == 0.42    # M=8192
+    assert max_plausible_mfu(32, 512) == 0.50    # M=16384
+    assert max_plausible_mfu(64, 512) == 0.60    # M=32768
+    assert max_plausible_mfu(128, 512) == 0.70   # M=65536
+    # unknown/zero config -> flat cap, never false-reject
+    assert max_plausible_mfu(None, 512) == 0.70
+    assert max_plausible_mfu(0, 0) == 0.70
+    assert max_plausible_mfu("x", 512) == 0.70
+
+
+def test_rejects_1217_microbatch_forgery():
+    # #1217 (5FTfrwU3): 2.62B tokens in 8074.8s at micro_batch=8/seq=512 => 324k tok/s
+    # => 50% MFU. Passed the flat 70% cap; measured-achievable at micro_batch=8 is ~11%.
+    # The micro-batch ceiling (33% at M=4096) rejects it.
+    fs = {"tokens_seen": 2_621_440_000, "wall_clock_s": 8074.8, "n_params": 253_874_184,
+          "config": {"micro_batch_size": 8, "seq_len": 512}}
+    ok, reason = check_compute_plausibility(fs, H100)
+    assert not ok
+    assert "micro_batch=8" in reason and "ceiling" in reason
+
+
+def test_accepts_honest_small_batch_run():
+    # An HONEST micro_batch=8 run is SLOW (~70k tok/s SXM, ~11% MFU) -> passes; the gate
+    # rejects only physically-impossible small-batch throughput, not small batches per se.
+    fs = {"tokens_seen": 2_621_440_000, "wall_clock_s": 37_000, "n_params": 253_874_184,
+          "config": {"micro_batch_size": 8, "seq_len": 512}}  # ~71k tok/s, ~11% MFU
+    assert check_compute_plausibility(fs, H100)[0]
+
+
+def test_accepts_honest_large_batch_field_runs():
+    # The restored king 5CqhtHE7 (ubatch 64, 137k tok/s, ~21% MFU) and 5Fbh5xe (ubatch 128,
+    # 174k, ~27%) are physically plausible and must NOT be false-rejected.
+    king = {"tokens_seen": 1_991_000_000, "wall_clock_s": 14_491, "n_params": 253_874_184,
+            "config": {"micro_batch_size": 64, "seq_len": 512}}  # ~137k, 21% < 60%
+    chal = {"tokens_seen": 1_913_000_000, "wall_clock_s": 10_989, "n_params": 253_874_184,
+            "config": {"micro_batch_size": 128, "seq_len": 512}}  # ~174k, 27% < 70%
+    assert check_compute_plausibility(king, H100)[0]
+    assert check_compute_plausibility(chal, H100)[0]
+
+
+def test_microbatch_forgery_at_large_batch_still_caught_by_flat_cap():
+    # If a forger moves to a LARGE micro_batch to dodge the small-batch ceiling, a 3x
+    # wall under-declaration lands >70% MFU and the flat cap catches it. 174k*3=522k => 80%.
+    fs = {"tokens_seen": 1_913_000_000, "wall_clock_s": 3_663, "n_params": 253_874_184,
+          "config": {"micro_batch_size": 128, "seq_len": 512}}  # ~522k tok/s, ~80% MFU
+    assert not check_compute_plausibility(fs, H100)[0]
+
+
+def test_no_config_falls_back_to_flat_cap():
+    # Backward compat: a bundle with no config field keeps the flat 70% behaviour.
+    fs = {"tokens_seen": 5_557_452_800, "wall_clock_s": 22_000, "n_params": 253_874_184}  # 39% MFU
+    assert check_compute_plausibility(fs, H100)[0]
+
+
 def test_incomplete_training_summary_is_skipped_not_rejected():
     assert check_compute_plausibility({"tokens_seen": 0, "wall_clock_s": 0}, {})[0]
     assert check_compute_plausibility({}, None)[0]

validator/integrity.py

@@ -116,6 +116,42 @@ def _gpu_bf16_peak_flops(gpu_name: str | None) -> float:
     return _DEFAULT_PEAK_TFLOPS * 1e12
 
 
+def max_plausible_mfu(micro_batch: int | None, seq_len: int | None) -> float:
+    """Micro-batch-aware MFU ceiling.
+
+    The flat MAX_PLAUSIBLE_MFU cap is blind to micro_batch, which is the loophole a
+    forged-down wall_clock exploited: the per-forward matmul M-dimension is
+    micro_batch*seq_len, and small M underutilizes the tensor cores, so the ACHIEVABLE
+    MFU drops sharply with the effective batch. At micro_batch=8/seq=512 (M=4096) a real
+    H100 tops out ~11% MFU (measured, reproduced), yet the flat 70% cap let a bundle
+    declare 50% MFU there — a ~4.5x wall_clock under-declaration — and pass.
+
+    These ceilings are set ~3x above the EMPIRICALLY-ACHIEVED MFU (H100, seq 512:
+    M=4096->11%, M=16384->15%, M=32768->21%, M=65536->27%), so honest runs — even
+    well-optimized ones — never trip them; only physically-impossible small-batch
+    throughput does. Unknown/zero config -> fall back to the flat cap (never false-reject).
+    NOTE: a static ceiling only halves the forgery window (~6x -> ~3x at small batch); the
+    airtight fix is on-GPU throughput re-measurement (a pre-crown, would-beat-king gate).
+    """
+    try:
+        m = int(micro_batch or 0) * int(seq_len or 0)
+    except (TypeError, ValueError):
+        return MAX_PLAUSIBLE_MFU
+    if m <= 0:
+        return MAX_PLAUSIBLE_MFU
+    if m >= 65536:
+        return 0.70
+    if m >= 32768:
+        return 0.60
+    if m >= 16384:
+        return 0.50
+    if m >= 8192:
+        return 0.42
+    if m >= 4096:
+        return 0.33
+    return 0.25
+
+
 def check_compute_plausibility(
     final_state: dict,
     calibration: dict | None = None,
@@ -125,10 +161,12 @@ def check_compute_plausibility(
     """Reject a bundle whose declared training throughput is physically impossible.
 
     tokens_seen / wall_clock_s implies ~6*N FLOPs/token; over the declared GPU's
-    bf16 peak that is the achieved MFU. An implied MFU > `max_mfu` means the
-    wall_clock_s (and the efficiency-gate compute cost it drives) is fabricated.
-    Best-effort: a missing/incomplete training_summary is skipped (deferred to the
-    other gates), not rejected. Returns (ok, reason); ok=False -> reject.
+    bf16 peak that is the achieved MFU. The ceiling is micro-batch-aware
+    (max_plausible_mfu): the flat cap is too loose at small micro_batch, where the GPU
+    is underutilized and high MFU is physically impossible. An implied MFU over the
+    ceiling means the wall_clock_s (and the efficiency-gate compute cost it drives) is
+    fabricated. Best-effort: a missing/incomplete training_summary is skipped (deferred
+    to the other gates), not rejected. Returns (ok, reason); ok=False -> reject.
     """
     fs = final_state or {}
     try:
@@ -140,15 +178,24 @@ def check_compute_plausibility(
     if tokens <= 0 or wall <= 0 or n <= 0:
         return True, "compute-plausibility: incomplete training_summary (skipped)"
     gpu = (calibration or {}).get("gpu_name") or fs.get("gpu_name") or fs.get("device") or ""
+    cfg = fs.get("config") or {}
+    # micro-batch-aware ceiling, tightened by any explicitly-passed max_mfu.
+    ceiling = min(max_mfu, max_plausible_mfu(cfg.get("micro_batch_size"), cfg.get("seq_len")))
     flops_per_s = 6.0 * n * tokens / wall  # 6N FLOPs/token (fwd+bwd)
     mfu = flops_per_s / _gpu_bf16_peak_flops(gpu)
-    if mfu > max_mfu:
+    if mfu > ceiling:
+        mb, sl = cfg.get("micro_batch_size"), cfg.get("seq_len")
+        batch_note = (
+            f" at micro_batch={mb}, seq_len={sl} (effective batch {int(mb) * int(sl)} tokens/fwd)"
+            if isinstance(mb, int) and isinstance(sl, int) else ""
+        )
         return False, (
             f"fabricated compute: {tokens / wall:,.0f} tok/s for a {n / 1e6:.0f}M model on "
-            f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {max_mfu * 100:.0f}% physical max); "
-            f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable"
+            f"'{gpu or 'unknown'}' => {mfu * 100:.0f}% MFU (> {ceiling * 100:.0f}% ceiling{batch_note}); "
+            f"wall_clock_s={wall:.0f}s for {tokens:,.0f} tokens is not achievable — small micro_batch "
+            f"underutilizes the GPU, so this throughput is physically impossible"
         )
-    return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU"
+    return True, f"compute plausible: {tokens / wall:,.0f} tok/s, {mfu * 100:.0f}% MFU (ceiling {ceiling * 100:.0f}%)"
 
 
 # --- Training-log integrity (anti fabricated-log / forged-down wall_clock) ------