From b0b9e94a7cf0c6a8bd1fc4df5edbadfa609bbbe5 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:25:02 +0300 Subject: [PATCH] Calibrate the killswitch perf guards to the claim, not to a machine `assert authz["median"] < 5.0` is red on main-merged PRs. It failed at 5.418 and 5.453 on PR #566, whose diff cannot touch authz at all, minutes after passing on main with the same test. Same code, different runner. The ceiling was never careless, and that is the interesting part: developer machine 0.253us what it was calibrated against old ceiling 5.0us ~20x the dev number; reads as generous shared CI runner ~5.44us ~21x SLOWER than dev for this in-memory call Written on a machine where it had 20x headroom, it lands a hair under the actual cost on a runner that is ~21x slower at a no-I/O `Policy.evaluate`. Nobody would guess that ratio, which is why the number survived review. Main cleared it by luck; the next PR did not. Assert the shape of the claim instead. Policy.evaluate is an in-memory decision, so it costs microseconds; a regression that matters means someone put a lookup on the decision path, and that costs milliseconds. Between runner noise and a real regression sit three orders of magnitude, so any ceiling in that gap catches everything this test exists to catch while ignoring which CPU GitHub assigned. 50us is ~200x the dev median, ~9x the CI median, and mirrors the headroom the replay floor already had (78k/s observed against a 10k/s floor) which is exactly why that assertion never flaked and this one did. The paper's numbers are unaffected: they come from a deliberate CORA_PERF_OUT run on the developer testcontainer, not from this guard. Co-Authored-By: Claude Opus 4.8 --- ...hority_revocation_perf_against_postgres.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/apps/api/tests/integration/test_authority_revocation_perf_against_postgres.py b/apps/api/tests/integration/test_authority_revocation_perf_against_postgres.py index 66909887e1a..f3ac60fe69b 100644 --- a/apps/api/tests/integration/test_authority_revocation_perf_against_postgres.py +++ b/apps/api/tests/integration/test_authority_revocation_perf_against_postgres.py @@ -61,6 +61,34 @@ _REPLAY_STREAM = 100 _REPLAY_ITERS = 20000 +# Order-of-magnitude guards, NOT budgets. The paper's numbers come from a +# deliberate CORA_PERF_OUT run on the developer testcontainer; these two +# assertions run on whatever shared CI runner GitHub assigns, so they must be +# calibrated to the claim's SHAPE rather than to a machine. +# +# The shape being guarded: Policy.evaluate is an in-memory decision with no I/O, +# so it costs microseconds. A regression that matters means someone put a lookup +# on the decision path, and that costs milliseconds: a 1000x move, not a 10% one. +# +# Why the old `< 5.0` looked safe and was not, measured 2026-07-16: +# +# developer machine 0.253us <- what the author calibrated against +# old ceiling 5.0us <- ~20x the dev number; reads as generous +# shared CI runner ~5.44us <- ~21x SLOWER than dev for this in-memory call +# +# The ceiling was never careless; it was ~20x headroom on the machine it was +# written on. It failed because CI is ~21x slower here, which no author would +# guess, landing the budget a hair under the runner's actual cost. It passed on +# main and failed twice on a PR minutes later with code that cannot touch authz. +# +# 50us is ~200x the dev median and ~9x the CI median, and mirrors the headroom +# the replay floor already had (78k/s observed against a 10k/s floor, ~8x) which +# is why that assertion never flaked and this one did. Tighten only against a +# same-run baseline; an absolute wall-clock number cannot be tightened safely on +# infrastructure whose speed varies 20x from the machine you wrote it on. +_AUTHZ_MEDIAN_CEILING_US = 50.0 +_REPLAY_FLOOR_EVENTS_PER_S = 10_000.0 + _ENVIRONMENT = ( "developer testcontainer (pgvector/pgvector:pg18 via testcontainers, asyncpg); " "wall-clock, single-threaded, not a tuned benchmark" @@ -271,5 +299,14 @@ async def test_authority_revocation_perf_characterizes_authz_replay_and_killswit assert scales[str(k)]["runs_held"] == k, ( f"K={k}: only {scales[str(k)]['runs_held']}/{k} held" ) - assert float(authz["median"]) < 5.0 - assert float(replay["events_per_s"]) > 10_000.0 + assert float(authz["median"]) < _AUTHZ_MEDIAN_CEILING_US, ( + f"authz median {authz['median']:.3f}us exceeded the order-of-magnitude ceiling " + f"{_AUTHZ_MEDIAN_CEILING_US}us. This guards the SHAPE of the claim (Policy.evaluate " + "is an in-memory microsecond-scale decision), not a tuned budget: a real regression " + "here means someone gave the decision path I/O, which costs milliseconds. If this " + "fires in the hundreds, look for a lookup that crept in. See the ceiling's rationale." + ) + assert float(replay["events_per_s"]) > _REPLAY_FLOOR_EVENTS_PER_S, ( + f"replay {replay['events_per_s']:.0f} events/s fell below the order-of-magnitude " + f"floor {_REPLAY_FLOOR_EVENTS_PER_S:.0f}/s" + )