From cc1c5def992a1a3b0531c3bdf47d6df0709bb950 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 11:47:48 -0400 Subject: [PATCH 1/7] Refresh the sota-v3 lineup and make route health a gate, not a note Qwen 3.7 Plus -> Qwen 3.8 Max (released 2026-08-03), and the DeepSeek slot moves off the deranked first-party route onto Cloudflare FP8. Cohort size stays at ten, so the Holm family, the 16x1 allocation, and the power selection are all untouched. The DeepSeek substitution was chosen on 24h availability (99.75%, best of sixteen endpoints), not on spot health -- the first-party route read status 0 with the cohort's best throughput hours before it was deranked to -5. Same FP8 quantization, identical published rates, so the swap costs nothing. Reservation moves $107.81 -> $119.76. That is entirely the Max tier, which bills 6.25x/4.69x the Plus tier per token; a 55.1% discount that appeared on the GLM Novita route absorbs part of it. Call count is unchanged at 3,240. Both drifts were invisible until the snapshot was re-read, which is the case for checking prices rather than trusting them. Two runner changes, both mutation-tested: - The zero-call phase now probes every route and reports the complete set of failures. Exiting on the first one leaves every later route unchecked, which reads as "one route is broken" when it may be four -- that happened twice in two days. Paid phases still abort on the first bad route, since there the next cell costs money. - Endpoint eligibility now enforces availability floors on two windows. The 24h figure cannot see an outage in progress: the deranked DeepSeek route still read 99.24% over 24h while serving 78% of requests. The 30m figure is what moved. Floors are 90% (30m) and 95% (24h), both well below the noise band -- these readings drift half a point between polls, and a 99% 24h floor rejected two healthy cohort members while passing the route that had actually failed. --- config/sota_v3_models.json | 80 +++++----- config/sota_v3_pricing_snapshot.json | 25 ++- .../sota-v3-pre-smoke-cost-estimate.json | 36 ++--- scripts/run_publication_matrix.py | 43 +++++- tests/test_publication_cost.py | 8 +- tests/test_publication_runner.py | 143 ++++++++++++++++++ 6 files changed, 264 insertions(+), 71 deletions(-) diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index c7dd595..1f1a258 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -13,7 +13,7 @@ "selection_frozen_at_utc": null, "selection_revision": "2026-08-03-public-catalog-cohort-v2", "catalog_snapshot_status": "frozen-public-metadata-only", - "catalog_checked_at_utc": "2026-08-03T15:53:59Z", + "catalog_checked_at_utc": "2026-08-04T03:31:26Z", "catalog_sources": [ "https://openrouter.ai/api/v1/models", "https://openrouter.ai/api/v1/models/{model_id}/endpoints" @@ -32,7 +32,7 @@ "endpoint_tag": "openai", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.31771054990628, + "catalog_uptime_last_30m": 99.73181045082407, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -79,7 +79,7 @@ "endpoint_tag": "amazon-bedrock/global", "endpoint_name": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.56639566395664, + "catalog_uptime_last_30m": 99.941792782305, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -126,7 +126,7 @@ "endpoint_tag": "google-ai-studio", "endpoint_name": "Google AI Studio | google/gemini-3.6-flash-20260721", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.78925184404636, + "catalog_uptime_last_30m": 98.72192579724532, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -172,7 +172,7 @@ "endpoint_tag": "xai/zdr", "endpoint_name": "xAI | x-ai/grok-4.5-20260708", "catalog_route_status": 0, - "catalog_uptime_last_30m": 100.0, + "catalog_uptime_last_30m": 99.95086799868982, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -222,7 +222,7 @@ "endpoint_tag": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.70362506358791, + "catalog_uptime_last_30m": 99.84704120327586, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -271,7 +271,7 @@ "endpoint_tag": "minimax/fp8", "endpoint_name": "Minimax | minimax/minimax-m3-20260531", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.45594322885867, + "catalog_uptime_last_30m": 98.48329048843188, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -296,18 +296,18 @@ "role": "MiniMax open-weight anchor on the first-party FP8 route" }, { - "id": "openrouter-qwen3.7-plus-alibaba", + "id": "openrouter-qwen3.8-max-alibaba", "provider": "openrouter", - "model": "qwen/qwen3.7-plus", - "canonical_slug": "qwen/qwen3.7-plus-20260602", + "model": "qwen/qwen3.8-max", + "canonical_slug": "qwen/qwen3.8-max-20260803", "transport": "gateway-api", "cohort": "open-weight", "upstream_provider": "Alibaba", - "upstream_provider_slug": "alibaba/fp8", - "endpoint_tag": "alibaba/fp8", - "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", + "upstream_provider_slug": "alibaba", + "endpoint_tag": "alibaba", + "endpoint_name": "Alibaba | qwen/qwen3.8-max-20260803", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.99335742374322, + "catalog_uptime_last_30m": 100, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -317,11 +317,15 @@ "seed", "presence_penalty", "response_format", - "logprobs", - "top_logprobs", "tools", "tool_choice", - "structured_outputs" + "structured_outputs", + "logprobs", + "top_logprobs", + "top_k", + "frequency_penalty", + "stop", + "reasoning_effort" ], "reasoning_policy": "disabled", "reasoning_effort": null, @@ -335,7 +339,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "Qwen open-weight frontier anchor" + "role": "Qwen frontier anchor on the first-party Alibaba route" }, { "id": "openrouter-mistral-medium-3.5-mistral", @@ -349,7 +353,7 @@ "endpoint_tag": "mistral", "endpoint_name": "Mistral | mistralai/mistral-medium-3.5-20260430", "catalog_route_status": 0, - "catalog_uptime_last_30m": 100.0, + "catalog_uptime_last_30m": 100, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -385,32 +389,38 @@ "role": "Mistral European open-weight anchor" }, { - "id": "openrouter-deepseek-v4-flash-0731-deepseek", + "id": "openrouter-deepseek-v4-flash-0731-cloudflare", "provider": "openrouter", "model": "deepseek/deepseek-v4-flash-0731", "canonical_slug": "deepseek/deepseek-v4-flash-20260731", "transport": "gateway-api", "cohort": "open-weight", - "upstream_provider": "DeepSeek", - "upstream_provider_slug": "deepseek/fp8", - "endpoint_tag": "deepseek/fp8", - "endpoint_name": "DeepSeek | deepseek/deepseek-v4-flash-20260731", + "upstream_provider": "Cloudflare", + "upstream_provider_slug": "cloudflare/fp8", + "endpoint_tag": "cloudflare/fp8", + "endpoint_name": "Cloudflare | deepseek/deepseek-v4-flash-20260731", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.99164403593065, + "catalog_uptime_last_30m": 98.77663772691398, "catalog_supported_parameters": [ "reasoning", "include_reasoning", "max_tokens", "temperature", "top_p", - "stop", + "top_k", + "seed", + "repetition_penalty", "frequency_penalty", "presence_penalty", - "logprobs", - "top_logprobs", + "min_p", + "stop", + "logit_bias", + "response_format", + "structured_outputs", "tools", "tool_choice", - "response_format", + "logprobs", + "top_logprobs", "reasoning_effort" ], "reasoning_policy": "disabled", @@ -431,7 +441,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "DeepSeek open-weight anchor on the first-party FP8 route" + "role": "DeepSeek open-weight anchor, substituted onto the Cloudflare FP8 route on 2026-08-04 after the first-party DeepSeek route was deranked to status -5 (30m uptime 77%). Same FP8 quantization and identical published rates; selected on 24h uptime (99.75%, best of 16 endpoints), not spot health." }, { "id": "openrouter-hy3-tencent", @@ -445,7 +455,7 @@ "endpoint_tag": "tencent/fp8", "endpoint_name": "Tencent | tencent/hy3-20260706", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.88329118848473, + "catalog_uptime_last_30m": 99.90525817148271, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -579,7 +589,7 @@ "evidence_sha256": null } }, - "openrouter-qwen3.7-plus-alibaba": { + "openrouter-qwen3.8-max-alibaba": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -611,7 +621,7 @@ "evidence_sha256": null } }, - "openrouter-deepseek-v4-flash-0731-deepseek": { + "openrouter-deepseek-v4-flash-0731-cloudflare": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -652,9 +662,9 @@ "openrouter-grok-4.5-xai", "openrouter-glm-5.2-novita", "openrouter-minimax-m3-minimax", - "openrouter-qwen3.7-plus-alibaba", + "openrouter-qwen3.8-max-alibaba", "openrouter-mistral-medium-3.5-mistral", - "openrouter-deepseek-v4-flash-0731-deepseek", + "openrouter-deepseek-v4-flash-0731-cloudflare", "openrouter-hy3-tencent" ], "shared_fixed_options": { diff --git a/config/sota_v3_pricing_snapshot.json b/config/sota_v3_pricing_snapshot.json index 4577ec9..020a25a 100644 --- a/config/sota_v3_pricing_snapshot.json +++ b/config/sota_v3_pricing_snapshot.json @@ -3,7 +3,7 @@ "contract": "sota-v3", "contract_fingerprint": "a523bdfcebe47bbd", "status": "catalog-frozen-public-metadata-only", - "checked_at_utc": "2026-08-03T15:53:59Z", + "checked_at_utc": "2026-08-04T03:31:26Z", "source": "Unauthenticated HTTP GET of https://openrouter.ai/api/v1/models and each selected model's https://openrouter.ai/api/v1/models/{model_id}/endpoints response; no completion or chat endpoint was called.", "currency": "USD", "rates_are_per_token": true, @@ -29,8 +29,8 @@ "completion": 1e-05 }, "deepseek/deepseek-v4-flash-0731": { - "provider_slug": "deepseek/fp8", - "endpoint_name": "DeepSeek | deepseek/deepseek-v4-flash-20260731", + "provider_slug": "cloudflare/fp8", + "endpoint_name": "Cloudflare | deepseek/deepseek-v4-flash-20260731", "prompt": 1.4e-07, "completion": 2.8e-07 }, @@ -64,16 +64,11 @@ "completion": 9e-07 } }, - "qwen/qwen3.7-plus": { - "provider_slug": "alibaba/fp8", - "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", - "prompt": 3.2e-07, - "completion": 1.28e-06, - "long_context_override": { - "min_prompt_tokens": 256000, - "prompt": 9.6e-07, - "completion": 3.84e-06 - } + "qwen/qwen3.8-max": { + "provider_slug": "alibaba", + "endpoint_name": "Alibaba | qwen/qwen3.8-max-20260803", + "prompt": 2e-06, + "completion": 6e-06 }, "tencent/hy3": { "provider_slug": "tencent/fp8", @@ -95,8 +90,8 @@ "z-ai/glm-5.2": { "provider_slug": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", - "prompt": 7.266e-07, - "completion": 2.2836e-06 + "prompt": 6.286e-07, + "completion": 1.9756e-06 } }, "public_metadata_limitations": [ diff --git a/results/analysis/sota-v3-pre-smoke-cost-estimate.json b/results/analysis/sota-v3-pre-smoke-cost-estimate.json index fe7c1a9..5472dd4 100644 --- a/results/analysis/sota-v3-pre-smoke-cost-estimate.json +++ b/results/analysis/sota-v3-pre-smoke-cost-estimate.json @@ -26,10 +26,10 @@ "total_calls": 3240 }, "costs_usd": { - "panel": 88.735895552, - "smoke": 1.1091986944, - "total_unrounded": 89.8450942464, - "total_with_1_2x_contingency": 107.81411309568 + "panel": 98.568712192, + "smoke": 1.2321089024, + "total_unrounded": 99.8008210944, + "total_with_1_2x_contingency": 119.76098531328 }, "models": [ { @@ -87,17 +87,17 @@ "smoke_cost_usd": 0.260608 }, { - "applied_completion_rate_usd": 2.2836e-06, + "applied_completion_rate_usd": 1.9756e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 7.266e-07, - "cost_per_decision_usd": 0.0151664256, + "applied_prompt_rate_usd": 6.286e-07, + "cost_per_decision_usd": 0.0131208576, "experiment_id": "openrouter-glm-5.2-novita", "internal_reasoning_tokens_per_decision": 0, "model": "z-ai/glm-5.2", "panel_calls": 320, - "panel_cost_usd": 4.853256192, + "panel_cost_usd": 4.198674432, "smoke_calls": 4, - "smoke_cost_usd": 0.0606657024 + "smoke_cost_usd": 0.0524834304 }, { "applied_completion_rate_usd": 1.2e-06, @@ -113,17 +113,17 @@ "smoke_cost_usd": 0.0292608 }, { - "applied_completion_rate_usd": 1.28e-06, + "applied_completion_rate_usd": 6e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 3.2e-07, - "cost_per_decision_usd": 0.00780288, - "experiment_id": "openrouter-qwen3.7-plus-alibaba", + "applied_prompt_rate_usd": 2e-06, + "cost_per_decision_usd": 0.040576, + "experiment_id": "openrouter-qwen3.8-max-alibaba", "internal_reasoning_tokens_per_decision": 0, - "model": "qwen/qwen3.7-plus", + "model": "qwen/qwen3.8-max", "panel_calls": 320, - "panel_cost_usd": 2.4969216, + "panel_cost_usd": 12.98432, "smoke_calls": 4, - "smoke_cost_usd": 0.03121152 + "smoke_cost_usd": 0.162304 }, { "applied_completion_rate_usd": 7.5e-06, @@ -143,7 +143,7 @@ "applied_internal_reasoning_rate_usd": 0.0, "applied_prompt_rate_usd": 1.4e-07, "cost_per_decision_usd": 0.00226688, - "experiment_id": "openrouter-deepseek-v4-flash-0731-deepseek", + "experiment_id": "openrouter-deepseek-v4-flash-0731-cloudflare", "internal_reasoning_tokens_per_decision": 0, "model": "deepseek/deepseek-v4-flash-0731", "panel_calls": 320, @@ -165,7 +165,7 @@ "smoke_cost_usd": 0.012874752 } ], - "pricing_checked_at_utc": "2026-08-03T15:53:59Z", + "pricing_checked_at_utc": "2026-08-04T03:31:26Z", "runtime": { "note": "Regenerate this artifact from accepted smoke telemetry before approving the full panel; latency is reported only for models with committed observations.", "observation_source": null, diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index d58e86b..dfa7a50 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -31,6 +31,21 @@ PROTOCOL_CONFIG = ROOT / "config" / "publication_protocol.json" SMOKE_MANIFEST = ROOT / "config" / "sota_v2_smoke_manifest.json" RUN_STATE_FORMAT = "gm-bench-publication-run-v1" +# Availability floors for a pinned endpoint to stay eligible. +# +# Two windows, because they detect different failures. The 24h figure is a +# chronic filter and is far too slow to see an outage: on 2026-08-04 the +# first-party DeepSeek route was deranked to status -5 while still reporting +# 99.24% over 24h. The 30m figure is what moved (78.93%), so that is the +# acute gate. +# +# Both floors sit well below the noise band. These readings drift by half a +# point between consecutive polls, so a threshold set near the observed values +# would block healthy routes at random -- a 99% 24h floor rejected two +# perfectly healthy cohort members on the day it was written while still +# passing the route that had actually failed. +MIN_UPTIME_LAST_30M_PCT = 90.0 +MIN_UPTIME_LAST_1D_PCT = 95.0 CONTRACT_CONFIGS = { "sota-v2": ( ROOT / "config" / "sota_v2_models.json", @@ -490,6 +505,17 @@ def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: capable.append(endpoint) if not capable: return [f"matching endpoint cannot honor required parameters {sorted(required)!r} and cap={cell.cap_label}"] + floors = (("uptime_last_30m", MIN_UPTIME_LAST_30M_PCT, "30m"), ("uptime_last_1d", MIN_UPTIME_LAST_1D_PCT, "24h")) + for field, floor, label in floors: + # A route that does not publish the figure is not penalised for it; + # only a published figure below the floor disqualifies. + durable = [e for e in capable if not isinstance(e.get(field), (int, float)) or e[field] >= floor] + if not durable: + observed = max(e[field] for e in capable if isinstance(e.get(field), (int, float))) + return [ + f"matching endpoint is below the {floor}% {label} uptime floor (best matching route: {observed:.2f}%)" + ] + capable = durable return [] @@ -1464,6 +1490,7 @@ def main(argv: list[str] | None = None) -> int: if not args.dry_run and not args.preflight_only and args.phase != "route-preflight": _write_run_state(run_dir, args.phase, cells, args.max_spend_usd) budget_start: float | None = None + preflight_failures: list[str] = [] for cell in cells: env = cell_environment(cell) command = cell_command( @@ -1513,7 +1540,16 @@ def main(argv: list[str] | None = None) -> int: KeyError, json.JSONDecodeError, ) as exc: - raise SystemExit(f"OpenRouter endpoint preflight failed for {cell.experiment_id}: {exc}") from exc + failure = f"OpenRouter endpoint preflight failed for {cell.experiment_id}: {exc}" + # A phase that spends money must stop at the first bad route. + # The zero-call phase must not: exiting early leaves every + # later route unchecked, which reads as "one route is broken" + # when the truth may be four. Collect and report them all. + if args.phase != "route-preflight": + raise SystemExit(failure) from exc + preflight_failures.append(failure) + print(failure) + continue if args.phase == "route-preflight": print(f"zero-completion-call route preflight passed: {cell.experiment_id}") continue @@ -1569,6 +1605,11 @@ def main(argv: list[str] | None = None) -> int: print(f"measured OpenRouter spend for this run directory: ${spent:.4f}") if spent > args.max_spend_usd: raise SystemExit(f"spend ceiling exceeded after attempted cell: ${spent:.4f}") + if preflight_failures: + raise SystemExit( + f"zero-completion-call route preflight failed for " + f"{len(preflight_failures)} of {len(cells)} routes:\n " + "\n ".join(preflight_failures) + ) return 0 diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index c8c97dd..34d3571 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -46,8 +46,12 @@ def test_v3_cost_plan_uses_registered_private_seed_count() -> None: assert result["calls"]["panel_decisions_per_model"] == 320 assert result["calls"]["panel_calls"] == 3_200 assert result["calls"]["total_calls"] == 3_240 - assert result["costs_usd"]["total_unrounded"] == pytest.approx(89.8450942464) - assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(107.81411309568) + # Repriced 2026-08-04 when the Qwen slot moved from qwen3.7-plus to + # qwen3.8-max and the GLM 5.2 Novita route picked up a 55.1% discount. + # The Max tier costs 6.25x/4.69x the Plus tier per token, so the swap + # dominates: the reserve moves +11.1% on an unchanged 3,240-call plan. + assert result["costs_usd"]["total_unrounded"] == pytest.approx(99.8008210944) + assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(119.76098531328) grok = next(row for row in result["models"] if row["model"] == "x-ai/grok-4.5") assert grok["internal_reasoning_tokens_per_decision"] == 4096 assert grok["applied_internal_reasoning_rate_usd"] == pytest.approx(grok["applied_completion_rate_usd"]) diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index bae167a..6f90c71 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1325,6 +1325,86 @@ def test_zero_call_route_preflight_has_separate_authorization_and_never_launches assert not (tmp_path / "checkpoints").exists() +def test_route_preflight_checks_every_route_before_failing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A bad route must not hide the routes queued behind it. + + Exiting on the first failure understates how much is broken: the operator + sees one dead route, fixes it, and only then learns about the next one. + The zero-call phase is free, so it has no reason to stop early -- it must + probe every route and report the complete set in one pass. The paid + phases keep failing fast, which is asserted separately below. + """ + monkeypatch.setenv("OPENROUTER_API_KEY", "test-route-preflight-key") + registry, lane, _manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + registry["selection_status"] = "route-preflight-ready" + lane["preregistration_status"] = "provisional-blocked" + lane["route_preflight_authorized"] = True + publication_runner.PANEL_CONFIG.write_text(json.dumps(registry)) + publication_runner.LANE_CONFIG.write_text(json.dumps(lane)) + + model_ids = [model["id"] for model in registry["models"]] + assert len(model_ids) >= 3, "this test needs a route queued behind both failures" + doomed = {model_ids[0], model_ids[2]} + checked: list[str] = [] + child_calls: list[str] = [] + + def fake_validate(cell, _env): + checked.append(cell.experiment_id) + if cell.experiment_id in doomed: + raise RuntimeError("no healthy OpenRouter endpoint matches") + + monkeypatch.setattr(publication_runner, "_validate_openrouter_endpoint", fake_validate) + monkeypatch.setattr( + publication_runner.subprocess, + "run", + lambda *_args, **_kwargs: child_calls.append("child"), + ) + + with pytest.raises(SystemExit) as exc_info: + main(["route-preflight", "--run-dir", str(tmp_path)]) + + message = str(exc_info.value.code) + # Every route was probed, including the ones queued behind both failures. + assert checked == model_ids + assert f"failed for {len(doomed)} of {len(model_ids)} routes" in message + for model_id in doomed: + assert model_id in message + # Still zero-call and still stateless, exactly as on the passing path. + assert child_calls == [] + assert not (tmp_path / "run-state.json").exists() + assert not (tmp_path / "raw").exists() + + +def test_paid_phases_still_abort_on_the_first_bad_route( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Collecting failures is a zero-call affordance, not a general one. + + A phase that spends money must stop the instant a route is wrong, so the + later cells are never probed at all. + """ + monkeypatch.setenv("OPENROUTER_API_KEY", "test-smoke-key") + registry, _lane, _manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + model_ids = [model["id"] for model in registry["models"]] + checked: list[str] = [] + + def fake_validate(cell, _env): + checked.append(cell.experiment_id) + raise RuntimeError("no healthy OpenRouter endpoint matches") + + monkeypatch.setattr(publication_runner, "_validate_openrouter_endpoint", fake_validate) + + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--run-dir", str(tmp_path), "--max-spend-usd", "1.00"]) + + assert checked == model_ids[:1], "a paid phase kept probing after a bad route" + assert model_ids[0] in str(exc_info.value.code) + + def test_v3_route_preflight_requires_bearer_credential_before_endpoint_request( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1511,6 +1591,69 @@ def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] +def _healthy_endpoint(cell) -> dict: + return { + "provider_name": "Alibaba", + "tag": "alibaba", + "name": cell.endpoint_name, + "status": 0, + "max_completion_tokens": 65536, + "supported_parameters": ["max_tokens", "response_format", "reasoning"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, + } + + +def test_endpoint_preflight_enforces_both_uptime_floors() -> None: + """The two windows catch different failures, so both have to gate. + + The 24h figure is a chronic filter and cannot see an outage in progress: + on 2026-08-04 the first-party DeepSeek route was deranked to status -5 + while still reporting 99.24% over 24h. The 30m figure is the one that + moved, so it carries the acute signal. A route that publishes neither + figure is not penalised for the omission. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + endpoint = _healthy_endpoint(cell) + payload = {"data": {"endpoints": [endpoint]}} + assert _endpoint_issues(cell, payload) == [] + + # The acute case: the exact shape of the DeepSeek derank, which a 24h-only + # floor waves straight through. + endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] = 78.93, 99.24 + issues = _endpoint_issues(cell, payload) + assert issues and "30m uptime floor" in issues[0] and "78.93%" in issues[0] + + # The chronic case: recent traffic looks fine, the whole day did not. + endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] = 99.9, 80.0 + issues = _endpoint_issues(cell, payload) + assert issues and "24h uptime floor" in issues[0] and "80.00%" in issues[0] + + for field, floor in ( + ("uptime_last_30m", publication_runner.MIN_UPTIME_LAST_30M_PCT), + ("uptime_last_1d", publication_runner.MIN_UPTIME_LAST_1D_PCT), + ): + endpoint.update(_healthy_endpoint(cell)) + endpoint[field] = floor + assert _endpoint_issues(cell, payload) == [], f"{field} floor must be inclusive" + + endpoint.update(_healthy_endpoint(cell)) + del endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] + assert _endpoint_issues(cell, payload) == [], "an unpublished figure must not fail the route" + + +def test_uptime_floors_sit_below_the_healthy_cohort_noise_band() -> None: + """The floors must not flap. + + These readings drift by roughly half a point between consecutive polls, + so a floor set near real values rejects healthy routes at random. A 99% + 24h floor rejected two healthy cohort members on the day it was written. + Both floors are therefore pinned well clear of the observed band. + """ + assert publication_runner.MIN_UPTIME_LAST_30M_PCT <= 90.0 + assert publication_runner.MIN_UPTIME_LAST_1D_PCT <= 95.0 + + def test_endpoint_preflight_allows_registered_prompt_only_json_route() -> None: cell = build_cells("smoke", model_id="openrouter-tencent-hy3-free-novita", cap=4096)[0] assert cell.fixed_options["OPENROUTER_JSON_MODE"] == "false" From 47ab32452d4acb1b1a68fdbf953012cd8b9ec18d Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 11:57:31 -0400 Subject: [PATCH 2/7] Make the committed spend ceiling bind, and stop tests billing a real account budget_policy.operator_ceiling_usd was declared in config and read by nothing. The only thing between a mistyped --max-spend-usd and an unbounded run was the operator retyping the right number from memory. It is now enforced ahead of the cell loop -- proven by asserting that a run over the cap reaches neither the endpoint probe nor a child process -- and set to the owner's $120 hard cap for sota-v3. A null ceiling stays permissive so contracts that have not chosen a number are not silently given one. Writing that test exposed something worse. The runner calls load_environment_files(ROOT) at startup, which reads the gitignored .env.local out of the working tree. Any test driving main() through a paid phase without stubbing the child process therefore runs the real benchmark against real routes and bills a real account -- and nothing fails, because the run succeeds. The test written to assert that a ceiling BLOCKS a run instead spent $0.436198 across 38 live calls to nine models. It resolved --contract sota-v3 to the shared fixture lane, which is fully unlocked by construction and inherits a null ceiling from the v2 protocol, so the guard correctly declined to fire and the smoke ran for real. An autouse fixture now neutralises the loader and clears provider keys for the whole suite, so no test can inherit a live credential. Tests that need a key present still set one explicitly with monkeypatch.setenv; what they can no longer do is pick one up by accident. --- config/sota_v3_publication_protocol.json | 3 +- scripts/run_publication_matrix.py | 33 +++++++++ tests/conftest.py | 24 +++++++ tests/test_publication_runner.py | 89 ++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/config/sota_v3_publication_protocol.json b/config/sota_v3_publication_protocol.json index ad32fd2..5c74389 100644 --- a/config/sota_v3_publication_protocol.json +++ b/config/sota_v3_publication_protocol.json @@ -84,7 +84,8 @@ "cost_estimate_artifact": "results/analysis/sota-v3-pre-smoke-cost-estimate.json", "operator_must_pass_max_spend_usd": true, "spend_authorized": false, - "operator_ceiling_usd": null + "operator_ceiling_usd": 120.0, + "operator_ceiling_basis": "Owner-set hard cap, 2026-08-04. Above the $119.76 pre-smoke reservation so the committed plan fits, and enforced by the runner: --max-spend-usd above this value is rejected before any cell runs. Projected actual spend is ~$34 from July smoke telemetry reprojected at current rates (models emit 48-640 output tokens per decision against a 4,096 reservation), so the cap is a backstop rather than the expected bill. Raising it is a deliberate edit here." }, "publication_authorized": false } diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index dfa7a50..d95405c 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -475,6 +475,34 @@ def _openrouter_usage_usd(env: dict[str, str]) -> float: return float(payload["data"]["total_usage"]) +def _enforce_operator_ceiling(max_spend_usd: float, contract: str | None) -> None: + """Reject a `--max-spend-usd` above the contract's committed hard cap. + + `budget_policy.operator_ceiling_usd` was declared but never read, so the + only thing standing between a typo and an unbounded run was the operator + retyping the right number. A committed ceiling that nothing enforces is a + comment. A null ceiling stays permissive: contracts that have not + committed to a number are not silently given one. + """ + _, _, _, protocol_path, _ = CONTRACT_CONFIGS.get(contract or "", (None,) * 5) + if protocol_path is None: + protocol_path = PROTOCOL_CONFIG + try: + budget_policy = (_read_json(protocol_path) or {}).get("budget_policy") or {} + except (OSError, ValueError, json.JSONDecodeError): + return + ceiling = budget_policy.get("operator_ceiling_usd") + if ceiling is None: + return + if not isinstance(ceiling, (int, float)) or isinstance(ceiling, bool) or ceiling <= 0: + raise ValueError(f"budget_policy.operator_ceiling_usd must be a positive number, got {ceiling!r}") + if max_spend_usd > ceiling: + raise ValueError( + f"--max-spend-usd ${max_spend_usd:.2f} exceeds the committed operator ceiling " + f"${float(ceiling):.2f} in {protocol_path.name}; raise the ceiling deliberately or lower the run" + ) + + def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: endpoints = (payload.get("data") or {}).get("endpoints") or [] expected_provider = cell.upstream_provider @@ -1483,6 +1511,11 @@ def main(argv: list[str] | None = None) -> int: and args.max_spend_usd is None ): parser.error("paid OpenRouter runs require an explicit --max-spend-usd ceiling") + if args.max_spend_usd is not None: + try: + _enforce_operator_ceiling(args.max_spend_usd, args.contract) + except ValueError as exc: + parser.error(str(exc)) run_dir = args.run_dir.resolve() for directory in (run_dir / "raw", run_dir / "checkpoints"): if not args.dry_run and not args.preflight_only and args.phase != "route-preflight": diff --git a/tests/conftest.py b/tests/conftest.py index ae66821..89ec216 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import pytest +import scripts.run_publication_matrix as publication_runner + @pytest.fixture(autouse=True) def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -12,3 +14,25 @@ def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N Also inherited by subprocess-based CLI tests via the environment. """ monkeypatch.setenv("GM_BENCH_BASELINE_CACHE", str(tmp_path / "baseline_cache.json")) + + +@pytest.fixture(autouse=True) +def block_real_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + """Stop the test suite from ever authenticating against a paid provider. + + The publication runner calls ``load_environment_files(ROOT)`` at startup, + which reads the gitignored ``.env.local`` out of the working tree. Any test + that drives ``main()`` through a paid phase without stubbing out the child + process therefore runs the real benchmark against real routes and bills a + real account -- with no failure to signal it, because the run succeeds. + That is exactly what happened on 2026-08-04: a test written to assert that + a spend ceiling *blocks* a run instead spent $0.44 across 38 live calls, + because the fixture lane it resolved to had every gate already unlocked. + + Neutralising the loader is enough. Tests that need a credential present + still set one explicitly with ``monkeypatch.setenv``, which continues to + work; what they cannot do any more is silently inherit a live key. + """ + monkeypatch.setattr(publication_runner, "load_environment_files", lambda _root: []) + for name in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.delenv(name, raising=False) diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 6f90c71..3df1acd 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1604,6 +1604,95 @@ def _healthy_endpoint(cell) -> dict: } +def test_the_suite_cannot_inherit_a_live_provider_credential() -> None: + """Pin the guard that stops a test from quietly billing a real account. + + The runner loads the gitignored `.env.local` itself, so before this guard + existed any test driving `main()` through a paid phase without stubbing + the child process ran the real benchmark against real routes. On + 2026-08-04 a test written to assert a spend ceiling *blocks* a run instead + spent $0.44 across 38 live calls, and passed no judgement on it -- the run + simply succeeded. + """ + assert os.environ.get("OPENROUTER_API_KEY") is None + assert publication_runner.load_environment_files(Path(".")) == [] + + +def test_operator_ceiling_rejects_a_run_that_could_outspend_the_committed_cap() -> None: + """The committed ceiling has to bind, or it is a comment. + + `budget_policy.operator_ceiling_usd` sat in the config unread, so the only + thing between a mistyped `--max-spend-usd` and an unbounded run was the + operator retyping the right number from memory. + """ + ceiling = json.loads(Path("config/sota_v3_publication_protocol.json").read_text()) + ceiling = ceiling["budget_policy"]["operator_ceiling_usd"] + assert ceiling == 120.00 + + publication_runner._enforce_operator_ceiling(ceiling, "sota-v3") + publication_runner._enforce_operator_ceiling(ceiling - 0.01, "sota-v3") + + with pytest.raises(ValueError, match="exceeds the committed operator ceiling"): + publication_runner._enforce_operator_ceiling(ceiling + 0.01, "sota-v3") + with pytest.raises(ValueError, match=r"\$1200\.00"): + publication_runner._enforce_operator_ceiling(1200.00, "sota-v3") + + +def test_operator_ceiling_stays_permissive_when_no_cap_is_committed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A contract that has not chosen a number must not be given one silently.""" + protocol_path = tmp_path / "protocol.json" + protocol_path.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": None}})) + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-test", + (protocol_path, protocol_path, protocol_path, protocol_path, protocol_path), + ) + publication_runner._enforce_operator_ceiling(10_000.00, "sota-test") + + protocol_path.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": "lots"}})) + with pytest.raises(ValueError, match="must be a positive number"): + publication_runner._enforce_operator_ceiling(1.00, "sota-test") + + +def test_paid_run_above_the_ceiling_is_refused_before_any_cell_runs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The gate has to sit ahead of the cell loop, not inside it. + + Checked by proving that neither the endpoint probe nor a child process is + reached: a ceiling enforced after the first cell has already spent money + is not a ceiling. + """ + _frozen_panel_files(tmp_path, monkeypatch) + protocol = json.loads(publication_runner.PROTOCOL_CONFIG.read_text()) + protocol["budget_policy"]["operator_ceiling_usd"] = 120.00 + publication_runner.PROTOCOL_CONFIG.write_text(json.dumps(protocol)) + + checked: list[str] = [] + child_calls: list[str] = [] + monkeypatch.setattr( + publication_runner, + "_validate_openrouter_endpoint", + lambda cell, _env: checked.append(cell.experiment_id), + ) + monkeypatch.setattr( + publication_runner.subprocess, + "run", + lambda *_args, **_kwargs: child_calls.append("child"), + ) + + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", "500"]) + + assert exc_info.value.code == 2 + assert checked == [], "a run over the ceiling reached the endpoint probe" + assert child_calls == [], "a run over the ceiling launched a model" + + def test_endpoint_preflight_enforces_both_uptime_floors() -> None: """The two windows catch different failures, so both have to gate. From 651043a00541a72f8df398026c4602a7fb2f26d8 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 12:05:06 -0400 Subject: [PATCH 3/7] Check prices on every probe, and stop reserving against promotional rates The pricing snapshot is what the reservation was computed from, so nothing was comparing it to reality. Every probe now checks the pinned route's live base rates: an increase fails the route, a decrease is reported and allowed. The asymmetry is deliberate -- a rate that rose makes the budget wrong in the direction that costs money, and a rate that fell only means coming in under reserve. It found two things on its first live run. MiniMax's first-party route had been deranked to status -2. Substituted onto deepinfra/fp8 under the new written policy: same model, same FP8 quantization, highest 24h availability among eligible routes, identical rates. More seriously, openai/gpt-5.6-luna and z-ai/glm-5.2 were both pinned at 50%-off promotional rates. The GLM discount moved 55.1% -> 50% within hours of being recorded, which is what surfaced it. A promo is not a floor and a reservation computed from one is wrong the moment it ends, so the snapshot now pins undiscounted list rates throughout. That changes the picture: the reservation moves $119.76 -> $127.29, which EXCEEDS the committed $120.00 ceiling. The plan only ever appeared to fit because two routes were discounted. Projected actual spend remains ~$35-45 from July smoke telemetry, so the ceiling is a backstop rather than a forecast, but the committed number now needs an owner decision either way. Also adds docs/ROUTE_SUBSTITUTION_POLICY.md, which fixes eligibility, forbids price, throughput, and first-party status as substitution criteria, and requires route and privacy acceptance to be re-established for any new counterparty -- Cloudflare and DeepInfra have never been reviewed for this project. Throughput is called out specifically because the DeepSeek route had the cohort's best throughput hours before it failed. The 2026-08-03 run log is left unedited as the historical record; today's state is a new log. --- config/sota_v3_models.json | 46 ++++--- config/sota_v3_pricing_snapshot.json | 17 +-- docs/PUBLISH_READINESS.md | 3 + docs/ROUTE_SUBSTITUTION_POLICY.md | 114 ++++++++++++++++++ .../sota-v3-lineup-refresh-2026-08-04.md | 109 +++++++++++++++++ .../sota-v3-pre-smoke-cost-estimate.json | 32 ++--- scripts/run_publication_matrix.py | 56 ++++++++- tests/test_publication_cost.py | 14 ++- tests/test_publication_runner.py | 48 ++++++++ 9 files changed, 389 insertions(+), 50 deletions(-) create mode 100644 docs/ROUTE_SUBSTITUTION_POLICY.md create mode 100644 docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index 1f1a258..f233e57 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -13,7 +13,7 @@ "selection_frozen_at_utc": null, "selection_revision": "2026-08-03-public-catalog-cohort-v2", "catalog_snapshot_status": "frozen-public-metadata-only", - "catalog_checked_at_utc": "2026-08-04T03:31:26Z", + "catalog_checked_at_utc": "2026-08-04T16:00:40Z", "catalog_sources": [ "https://openrouter.ai/api/v1/models", "https://openrouter.ai/api/v1/models/{model_id}/endpoints" @@ -32,7 +32,7 @@ "endpoint_tag": "openai", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.73181045082407, + "catalog_uptime_last_30m": 99.62251080512623, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -79,7 +79,7 @@ "endpoint_tag": "amazon-bedrock/global", "endpoint_name": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.941792782305, + "catalog_uptime_last_30m": 99.62314342717801, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -126,7 +126,7 @@ "endpoint_tag": "google-ai-studio", "endpoint_name": "Google AI Studio | google/gemini-3.6-flash-20260721", "catalog_route_status": 0, - "catalog_uptime_last_30m": 98.72192579724532, + "catalog_uptime_last_30m": 97.20506031185643, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -172,7 +172,7 @@ "endpoint_tag": "xai/zdr", "endpoint_name": "xAI | x-ai/grok-4.5-20260708", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.95086799868982, + "catalog_uptime_last_30m": 99.97348886532343, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -222,7 +222,7 @@ "endpoint_tag": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.84704120327586, + "catalog_uptime_last_30m": 98.38854073410921, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -260,27 +260,35 @@ "role": "Z.ai open-weight anchor on the previously smoke-validated Novita FP8 route" }, { - "id": "openrouter-minimax-m3-minimax", + "id": "openrouter-minimax-m3-deepinfra", "provider": "openrouter", "model": "minimax/minimax-m3", "canonical_slug": "minimax/minimax-m3-20260531", "transport": "gateway-api", "cohort": "open-weight", - "upstream_provider": "Minimax", - "upstream_provider_slug": "minimax/fp8", - "endpoint_tag": "minimax/fp8", - "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "upstream_provider": "DeepInfra", + "upstream_provider_slug": "deepinfra/fp8", + "endpoint_tag": "deepinfra/fp8", + "endpoint_name": "DeepInfra | minimax/minimax-m3-20260531", "catalog_route_status": 0, - "catalog_uptime_last_30m": 98.48329048843188, + "catalog_uptime_last_30m": 99.82795698924731, "catalog_supported_parameters": [ "reasoning", "include_reasoning", "max_tokens", "temperature", "top_p", + "stop", + "frequency_penalty", + "presence_penalty", + "repetition_penalty", + "top_k", + "seed", + "min_p", "response_format", - "tool_choice", - "tools" + "logit_bias", + "tools", + "tool_choice" ], "reasoning_policy": "disabled", "reasoning_effort": null, @@ -293,7 +301,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "MiniMax open-weight anchor on the first-party FP8 route" + "role": "MiniMax open-weight anchor, substituted onto the DeepInfra FP8 route on 2026-08-04 after the first-party route was deranked to status -2. Highest 24h availability among eligible FP8 routes; identical published rates." }, { "id": "openrouter-qwen3.8-max-alibaba", @@ -400,7 +408,7 @@ "endpoint_tag": "cloudflare/fp8", "endpoint_name": "Cloudflare | deepseek/deepseek-v4-flash-20260731", "catalog_route_status": 0, - "catalog_uptime_last_30m": 98.77663772691398, + "catalog_uptime_last_30m": 99.28610653487095, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -455,7 +463,7 @@ "endpoint_tag": "tencent/fp8", "endpoint_name": "Tencent | tencent/hy3-20260706", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.90525817148271, + "catalog_uptime_last_30m": 99.80510276399717, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -573,7 +581,7 @@ "evidence_sha256": null } }, - "openrouter-minimax-m3-minimax": { + "openrouter-minimax-m3-deepinfra": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -661,7 +669,7 @@ "openrouter-gemini-3.6-flash-google-ai-studio", "openrouter-grok-4.5-xai", "openrouter-glm-5.2-novita", - "openrouter-minimax-m3-minimax", + "openrouter-minimax-m3-deepinfra", "openrouter-qwen3.8-max-alibaba", "openrouter-mistral-medium-3.5-mistral", "openrouter-deepseek-v4-flash-0731-cloudflare", diff --git a/config/sota_v3_pricing_snapshot.json b/config/sota_v3_pricing_snapshot.json index 020a25a..592baa4 100644 --- a/config/sota_v3_pricing_snapshot.json +++ b/config/sota_v3_pricing_snapshot.json @@ -3,7 +3,7 @@ "contract": "sota-v3", "contract_fingerprint": "a523bdfcebe47bbd", "status": "catalog-frozen-public-metadata-only", - "checked_at_utc": "2026-08-04T03:31:26Z", + "checked_at_utc": "2026-08-04T16:00:40Z", "source": "Unauthenticated HTTP GET of https://openrouter.ai/api/v1/models and each selected model's https://openrouter.ai/api/v1/models/{model_id}/endpoints response; no completion or chat endpoint was called.", "currency": "USD", "rates_are_per_token": true, @@ -42,8 +42,8 @@ "internal_reasoning": 7.5e-06 }, "minimax/minimax-m3": { - "provider_slug": "minimax/fp8", - "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "provider_slug": "deepinfra/fp8", + "endpoint_name": "DeepInfra | minimax/minimax-m3-20260531", "prompt": 3e-07, "completion": 1.2e-06 }, @@ -56,8 +56,8 @@ "openai/gpt-5.6-luna": { "provider_slug": "openai", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", - "prompt": 1e-07, - "completion": 6e-07, + "prompt": 2e-07, + "completion": 1.2e-06, "long_context_override": { "min_prompt_tokens": 272000, "prompt": 2e-07, @@ -90,8 +90,8 @@ "z-ai/glm-5.2": { "provider_slug": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", - "prompt": 6.286e-07, - "completion": 1.9756e-06 + "prompt": 1.4e-06, + "completion": 4.4e-06 } }, "public_metadata_limitations": [ @@ -103,5 +103,6 @@ "route_preflight_authorized": false, "smoke_execution_authorized": false, "panel_execution_authorized": false, - "publication_authorized": false + "publication_authorized": false, + "pricing_basis": "Undiscounted list rates for the exact pinned route. Promotional discounts are deliberately not reserved against: the GLM 5.2 Novita discount moved from 55.1% to 50% within hours on 2026-08-04, and a reservation computed from a promo is wrong the moment the promo ends. A live discount only ever brings the run in under reserve." } diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 81f7bdc..d1fba67 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -860,6 +860,9 @@ decision and why. | Date | Decision | Evidence / rationale | Effect | | --- | --- | --- | --- | +| 2026-08-04 | Adopt `qwen/qwen3.8-max` in place of `qwen/qwen3.7-plus`, and adopt a written [route substitution policy](ROUTE_SUBSTITUTION_POLICY.md). | Qwen 3.8 Max shipped 2026-08-03 and the benchmark is intended to track new releases rather than freeze once. Three substitutions in two days had each been decided ad hoc, which is how a benchmark quietly starts measuring whichever host is cheapest this week. | Cohort stays at ten, so the Holm family, the 16x1 allocation, and the power selection are untouched. The Max tier bills 6.25x/4.69x the Plus tier per token. The policy fixes eligibility, forbids price/throughput/first-party status as substitution criteria, and requires re-establishing route and privacy acceptance for any new counterparty. | +| 2026-08-04 | Substitute `deepseek/fp8` -> `cloudflare/fp8` and `minimax/fp8` -> `deepinfra/fp8`. | Both first-party routes were deranked the same day (status `-5` at 78% availability, and `-2`). Both replacements are the same model at the same FP8 quantization, chosen on highest 24h availability, at identical published rates. | No cost or cohort-size effect. The DeepSeek route recovered on its own within the day, so in hindsight that substitution was not strictly necessary; it is kept because Cloudflare holds the better 24h record. Route and privacy acceptance do **not** carry over — Cloudflare and DeepInfra are counterparties this project has never reviewed. | +| 2026-08-04 | Pin undiscounted list rates instead of promotional rates, and enforce `operator_ceiling_usd` in the runner. | `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both pinned at 50%-off promos; the GLM discount moved 55.1% -> 50% within hours of being recorded. A reservation computed from a promo is wrong the moment it ends. Separately, `operator_ceiling_usd` had sat in config unread, so nothing enforced the committed cap. | Reservation $119.76 -> **$127.29**, which **exceeds the owner's $120.00 ceiling** — the plan only ever appeared to fit because of the discounts. Awaiting an owner decision on the cap. Projected actual spend is ~$35-45 from July smoke telemetry, so the ceiling is a backstop rather than the expected bill. Logged in [`docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md`](run_logs/sota-v3-lineup-refresh-2026-08-04.md). | | 2026-07-13 | Treat v1 model rows as archived historical evidence, not a current ranking. | Scout-key mismatch affected models unevenly; failed queries were invisible. | Current claims require `sota-v2`; v1 remains auditable under `sota-v1`. | | 2026-07-13 | Separate API and coding-harness lanes. | Archived rows mixed provider API behavior with uncontrolled CLI harness context and very different output usage. | API becomes the headline lane; CLI harnesses remain diagnostic. | | 2026-07-13 | Withhold the v2 ranking pending an output-budget sweep. | Archived scores tracked output allowance strongly enough to confound model comparison. | Run the planned cap matrix and freeze a compute policy before the full panel. | diff --git a/docs/ROUTE_SUBSTITUTION_POLICY.md b/docs/ROUTE_SUBSTITUTION_POLICY.md new file mode 100644 index 0000000..4bf042d --- /dev/null +++ b/docs/ROUTE_SUBSTITUTION_POLICY.md @@ -0,0 +1,114 @@ +# Route substitution policy + +When a pinned OpenRouter route stops being eligible, this is how it gets +replaced. The point is to make the decision mechanical, so that a route failing +at an inconvenient moment does not turn into an improvised choice about what the +benchmark measures. + +Adopted 2026-08-04, after three substitutions in two days were each decided ad +hoc. + +## Why this exists + +Route health is volatile on a timescale shorter than a panel run. Measured over +roughly twelve hours on 2026-08-04: + +| Route | Observed | +| --- | --- | +| `deepseek/deepseek-v4-flash-0731` @ `deepseek/fp8` | status 0, best throughput in the cohort → deranked to `-5`, serving 78% of requests → recovered to status 0 | +| `minimax/minimax-m3` @ `minimax/fp8` | status 0 → deranked to `-2` | +| `qwen/qwen3.7-plus` @ `alibaba` | endpoint tag silently moved to `alibaba/fp8` | +| `z-ai/glm-5.2` @ `novita/fp8` | discount moved 55.1% → 50% | + +None of these were announced. All were found by running the free probe. + +## What makes a route ineligible + +A pinned route is ineligible if **any** of the following holds. These are all +enforced by `_endpoint_issues` in `scripts/run_publication_matrix.py`; the free +zero-call probe reports every failing route in one pass. + +1. No endpoint matches the pinned `provider_name`, `tag`, and `name`. +2. Endpoint `status` is not `0`. +3. The endpoint does not advertise `max_tokens`, `reasoning`, and — when the + lane sets `OPENROUTER_JSON_MODE=true` — `response_format`. +4. `max_completion_tokens` is below the registered output cap. +5. `uptime_last_30m` is below **90%**. +6. `uptime_last_1d` is below **95%**. +7. The live base rate for the pinned route is **above** the committed pricing + snapshot. A rate below the snapshot is reported, not blocked. + +On (5) and (6): both windows gate because they detect different failures. The +24h figure cannot see an outage in progress — the deranked DeepSeek route still +read 99.24% over 24h while serving 78% of requests. Both floors sit well below +the observed noise band, because these readings drift about half a point between +consecutive polls; a 99% 24h floor was measured rejecting two healthy cohort +members while still passing the route that had actually failed. + +## Choosing the replacement + +**Prefer waiting.** Deranking is frequently transient — the first-party DeepSeek +route recovered on its own within a day. If the paid phase is not imminent, +re-probe before substituting. + +Where a substitution is required, take the eligible endpoint for the **same +model** with the **highest `uptime_last_1d`**, subject to: + +- **Same quantization** as the outgoing route. A different precision is a + different numerical model, not a different host, and is out of scope for a + substitution — treat it as a cohort amendment. +- All eligibility criteria above. +- `max_completion_tokens` at or above the registered cap. + +Deliberately *not* criteria: price, throughput, latency, and first-party status. + +- **Price** must not steer the choice; that is how a benchmark quietly starts + measuring whichever host is cheapest this week. Record the new rate and let + the reservation move. +- **Throughput and latency** are the worst possible signals here. The DeepSeek + route had the cohort's best throughput hours before it failed. +- **First-party status** is not load-bearing. It was framed as one in the + original route roles, but on the axis that actually matters mid-panel — + staying up — a model's author has no particular advantage. Cloudflare beat + DeepSeek on its own model. + +## Recording it + +A substitution is a cohort **route** change, not a cohort **membership** change, +so it does not touch the Holm family size, the allocation, or the power +selection. It does require, in the same change: + +1. `config/sota_v3_models.json` — `id`, `upstream_provider`, + `upstream_provider_slug`, `endpoint_tag`, `endpoint_name`, + `catalog_route_status`, `catalog_uptime_last_30m`, + `catalog_supported_parameters`, and a `role` naming the date, the outgoing + route, and the observed reason. +2. The same `id` in `required_smokes` and in `exact_route_acceptance.entries`. +3. `config/sota_v3_pricing_snapshot.json` — `provider_slug`, `endpoint_name`, + and the **undiscounted list rates** (see below). +4. Regenerate `results/analysis/sota-v3-pre-smoke-cost-estimate.json` and check + the result against `budget_policy.operator_ceiling_usd`. +5. A dated entry in the decision log, and a run log if a probe was run. + +Substituting a host does **not** carry over route acceptance. `authenticated`, +`verified_at_utc`, `route_evidence_sha256`, and the whole `privacy_acceptance` +block describe a specific provider and must be re-established for the new one. + +## Pricing + +Pin the **undiscounted list rate**, never a promotional rate. + +The reservation is a safe upper bound, and a promo is not a floor. On +2026-08-04 the GLM 5.2 Novita discount moved from 55.1% to 50% within hours, +and `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both found pinned at 50%-off +prices — which is the only reason the panel had appeared to fit under the +$120 ceiling. Where the endpoint reports a `discount`, record +`live_rate / (1 - discount)`. + +A live discount then only ever brings the run in under reserve, which is the +direction that cannot hurt. + +## When to re-probe + +Immediately before any paid phase, every time. A passing probe has a shelf life +measured in hours, and a stale pass is not a pass. diff --git a/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md new file mode 100644 index 0000000..3c3a8b2 --- /dev/null +++ b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md @@ -0,0 +1,109 @@ +# sota-v3 lineup refresh — 2026-08-04 + +Supersedes the route state recorded in +[`sota-v3-route-preflight-2026-08-03.md`](sota-v3-route-preflight-2026-08-03.md), +which is left unedited as the historical record of that day's probe. Two of the +ten routes it describes are no longer in the lineup, and its closing claim that +all ten pass was true when written and is not a standing claim. + +**Zero completion calls, zero spend.** No gate was granted or changed; every +paid authorization remains `false`. + +## Cohort changes + +Cohort size stays at **ten**, so the Holm family size, the 16x1 allocation, and +the power selection are untouched. + +| Slot | Was | Now | Reason | +| --- | --- | --- | --- | +| Qwen | `qwen/qwen3.7-plus` @ `alibaba/fp8` | `qwen/qwen3.8-max` @ `alibaba` | Released 2026-08-03; owner elected to adopt immediately | +| DeepSeek | `deepseek/deepseek-v4-flash-0731` @ `deepseek/fp8` | same model @ `cloudflare/fp8` | First-party route deranked to status `-5` | +| MiniMax | `minimax/minimax-m3` @ `minimax/fp8` | same model @ `deepinfra/fp8` | First-party route deranked to status `-2` | + +Both substitutions follow +[`docs/ROUTE_SUBSTITUTION_POLICY.md`](../ROUTE_SUBSTITUTION_POLICY.md): same +model, same FP8 quantization, highest 24h availability among eligible routes, +identical published rates. + +The Qwen change is a **cohort amendment**, not a substitution — a different +model, adopted deliberately. It replaces Alibaba's cost-effective tier with +their flagship, which bills 6.25x/4.69x per token. + +## How volatile the routes actually are + +Everything below was observed within roughly twelve hours. None was announced. + +| Route | Observed | +| --- | --- | +| `deepseek/fp8` | status 0 with the cohort's best throughput → deranked to `-5`, 30m availability 78% → recovered to status 0 the same day | +| `minimax/fp8` | status 0 → deranked to `-2` | +| `alibaba` (qwen3.7-plus) | endpoint tag silently moved to `alibaba/fp8` | +| `novita/fp8` (glm-5.2) | discount moved 55.1% → 50% | + +The DeepSeek recovery is worth stating plainly: the substitution was not +strictly necessary in hindsight. It is kept because Cloudflare holds the better +24h record, but the honest reading is that the route was transiently deranked, +not lost. + +## Pricing: promotional rates are no longer reserved against + +`openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both found pinned at 50%-off +promotional rates. The GLM discount moved 55.1% → 50% within hours of being +recorded, which is what surfaced the problem. + +The snapshot now pins **undiscounted list rates** for every route. A promo is +not a floor, and a reservation computed from one is wrong the moment it ends. + +| Route | Was pinned (promo) | Now pinned (list) | +| --- | --- | --- | +| `openai/gpt-5.6-luna` | 1e-07 / 6e-07 | 2e-07 / 1.2e-06 | +| `z-ai/glm-5.2` | 7e-07 / 2.2e-06 | 1.4e-06 / 4.4e-06 | + +## Reservation + +| | Amount | +| --- | --- | +| Previous reservation | $107.81 | +| After the Qwen amendment | $119.76 | +| **After pinning list rates** | **$127.29** | +| Committed operator ceiling | $120.00 | + +**The plan no longer fits under the ceiling, and only ever appeared to because +two routes were priced at a discount.** Call count is unchanged at 3,240. + +Reserve is concentrated in four models: Gemini 3.6 Flash (22.4%), Grok 4.5 +(19.9%), Claude Sonnet 5 (17.4%), and Mistral Medium 3.5 (13.0%). + +### Projected actual spend is far below either figure + +From July smoke telemetry reprojected at current rates, models emit **48–640 +output tokens per decision** against a 4,096-token reservation, and Grok's +observed internal reasoning was **516 tokens** against 4,096 reserved. + +Projected actual spend is roughly **$35–45**. The reservation is a backstop, not +a forecast, and the gap is concentrated in the reserved reasoning headroom for +Gemini and Grok. + +## Runner changes made alongside + +- The zero-call probe now checks **every** route and reports the full set of + failures. Exiting on the first one had twice presented a partial picture as a + complete one. Paid phases still abort on the first bad route. +- Endpoint eligibility now enforces availability floors on two windows, 90% + (30m) and 95% (24h). +- Live rates are compared against the committed snapshot on every probe. + An increase fails the route; a decrease is reported and allowed. +- `budget_policy.operator_ceiling_usd` is now enforced ahead of the cell loop. + It had been declared in config and read by nothing. +- An autouse test fixture blocks the suite from inheriting a live provider + credential, after a test spent $0.436198 across 38 live calls. + +## What this does not establish + +Unchanged: `spend_authorized`, `smoke_execution_authorized`, +`panel_execution_authorized`, `publication_authorized` all `false`; registry +`route-preflight-ready`, not `frozen`; `exact_route_acceptance` `unresolved`. + +Route acceptance does **not** carry across a substitution. Cloudflare and +DeepInfra are new counterparties whose data-handling terms have never been +reviewed for this project. diff --git a/results/analysis/sota-v3-pre-smoke-cost-estimate.json b/results/analysis/sota-v3-pre-smoke-cost-estimate.json index 5472dd4..a8ffdfa 100644 --- a/results/analysis/sota-v3-pre-smoke-cost-estimate.json +++ b/results/analysis/sota-v3-pre-smoke-cost-estimate.json @@ -26,24 +26,24 @@ "total_calls": 3240 }, "costs_usd": { - "panel": 98.568712192, - "smoke": 1.2321089024, - "total_unrounded": 99.8008210944, - "total_with_1_2x_contingency": 119.76098531328 + "panel": 104.76363776, + "smoke": 1.309545472, + "total_unrounded": 106.073183232, + "total_with_1_2x_contingency": 127.2878198784 }, "models": [ { - "applied_completion_rate_usd": 6e-07, + "applied_completion_rate_usd": 1.2e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 1e-07, - "cost_per_decision_usd": 0.0032576, + "applied_prompt_rate_usd": 2e-07, + "cost_per_decision_usd": 0.0065152, "experiment_id": "openrouter-gpt-5.6-luna-openai", "internal_reasoning_tokens_per_decision": 0, "model": "openai/gpt-5.6-luna", "panel_calls": 320, - "panel_cost_usd": 1.042432, + "panel_cost_usd": 2.084864, "smoke_calls": 4, - "smoke_cost_usd": 0.0130304 + "smoke_cost_usd": 0.0260608 }, { "applied_completion_rate_usd": 1e-05, @@ -87,24 +87,24 @@ "smoke_cost_usd": 0.260608 }, { - "applied_completion_rate_usd": 1.9756e-06, + "applied_completion_rate_usd": 4.4e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 6.286e-07, - "cost_per_decision_usd": 0.0131208576, + "applied_prompt_rate_usd": 1.4e-06, + "cost_per_decision_usd": 0.0292224, "experiment_id": "openrouter-glm-5.2-novita", "internal_reasoning_tokens_per_decision": 0, "model": "z-ai/glm-5.2", "panel_calls": 320, - "panel_cost_usd": 4.198674432, + "panel_cost_usd": 9.351168, "smoke_calls": 4, - "smoke_cost_usd": 0.0524834304 + "smoke_cost_usd": 0.1168896 }, { "applied_completion_rate_usd": 1.2e-06, "applied_internal_reasoning_rate_usd": 0.0, "applied_prompt_rate_usd": 3e-07, "cost_per_decision_usd": 0.0073152, - "experiment_id": "openrouter-minimax-m3-minimax", + "experiment_id": "openrouter-minimax-m3-deepinfra", "internal_reasoning_tokens_per_decision": 0, "model": "minimax/minimax-m3", "panel_calls": 320, @@ -165,7 +165,7 @@ "smoke_cost_usd": 0.012874752 } ], - "pricing_checked_at_utc": "2026-08-04T03:31:26Z", + "pricing_checked_at_utc": "2026-08-04T16:00:40Z", "runtime": { "note": "Regenerate this artifact from accepted smoke telemetry before approving the full panel; latency is reported only for models with committed observations.", "observation_source": null, diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index d95405c..5eccc2e 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -560,8 +560,62 @@ def _openrouter_endpoints(model: str, env: dict[str, str]) -> dict[str, Any]: return json.load(response) +def _pricing_drift_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: + """Compare the pinned route's live rates against the committed snapshot. + + The snapshot is what the reservation was computed from, so a rate that has + risen since it was taken makes the committed budget wrong in the direction + that costs money. That is an error. A rate that has *fallen* only means the + run will come in under reserve, so it is reported and allowed -- the GLM + Novita route quietly picked up a 55.1% discount that went unnoticed for two + weeks precisely because nothing ever looked. + + Only the base rates are compared. Long-context override tiers are priced + separately in the snapshot and are not exercised by the registered + 8,000-token decision, so a drift there cannot move this plan's cost. + """ + try: + rates = (_read_json(PRICING_CONFIG).get("models") or {}).get(cell.model) + except (OSError, ValueError, json.JSONDecodeError): + return [] + if not isinstance(rates, dict): + return [] + endpoint = next( + ( + e + for e in ((payload.get("data") or {}).get("endpoints") or []) + if e.get("tag") == cell.endpoint_tag and e.get("name") == cell.endpoint_name + ), + None, + ) + if endpoint is None: + return [] + issues = [] + for field in ("prompt", "completion"): + committed = rates.get(field) + raw = (endpoint.get("pricing") or {}).get(field) + if not isinstance(committed, (int, float)) or raw is None: + continue + try: + live = float(raw) + except (TypeError, ValueError): + continue + if live > committed: + issues.append( + f"live {field} rate {live:.10g} exceeds the committed snapshot rate {committed:.10g} " + f"for {cell.experiment_id}; the reservation was computed from the snapshot" + ) + elif live < committed: + print( + f"note: live {field} rate for {cell.experiment_id} fell from {committed:.10g} " + f"to {live:.10g}; the run will come in under its reservation" + ) + return issues + + def _validate_openrouter_endpoint(cell: Cell, env: dict[str, str]) -> None: - issues = _endpoint_issues(cell, _openrouter_endpoints(cell.model, env)) + payload = _openrouter_endpoints(cell.model, env) + issues = _endpoint_issues(cell, payload) + _pricing_drift_issues(cell, payload) if issues: raise RuntimeError("; ".join(issues)) diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index 34d3571..e8edd61 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -46,12 +46,14 @@ def test_v3_cost_plan_uses_registered_private_seed_count() -> None: assert result["calls"]["panel_decisions_per_model"] == 320 assert result["calls"]["panel_calls"] == 3_200 assert result["calls"]["total_calls"] == 3_240 - # Repriced 2026-08-04 when the Qwen slot moved from qwen3.7-plus to - # qwen3.8-max and the GLM 5.2 Novita route picked up a 55.1% discount. - # The Max tier costs 6.25x/4.69x the Plus tier per token, so the swap - # dominates: the reserve moves +11.1% on an unchanged 3,240-call plan. - assert result["costs_usd"]["total_unrounded"] == pytest.approx(99.8008210944) - assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(119.76098531328) + # Repriced 2026-08-04. Two moves, on an unchanged 3,240-call plan: + # the Qwen slot went from qwen3.7-plus to qwen3.8-max, which bills + # 6.25x/4.69x per token; and the snapshot now pins undiscounted list + # rates, after gpt-5.6-luna and glm-5.2 were both found pinned at + # 50%-off promotional prices. A promo is not a floor -- the GLM + # discount moved 55.1% -> 50% within hours of being recorded. + assert result["costs_usd"]["total_unrounded"] == pytest.approx(106.073183232) + assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(127.2878198784) grok = next(row for row in result["models"] if row["model"] == "x-ai/grok-4.5") assert grok["internal_reasoning_tokens_per_decision"] == 4096 assert grok["applied_internal_reasoning_rate_usd"] == pytest.approx(grok["applied_completion_rate_usd"]) diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 3df1acd..23de2a6 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1604,6 +1604,54 @@ def _healthy_endpoint(cell) -> dict: } +def test_pricing_drift_fails_upward_and_only_reports_downward( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A rate that rose invalidates the reservation; a rate that fell does not. + + The committed snapshot is what the budget was computed from, so an + increase makes the plan wrong in the direction that costs money. A + decrease only means coming in under reserve -- the GLM Novita route picked + up a 55.1% discount that nobody noticed for two weeks because nothing + compared the snapshot to reality. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + committed = json.loads(publication_runner.PRICING_CONFIG.read_text())["models"][cell.model] + + def payload(prompt: float, completion: float) -> dict: + return { + "data": { + "endpoints": [ + { + "tag": cell.endpoint_tag, + "name": cell.endpoint_name, + "pricing": {"prompt": str(prompt), "completion": str(completion)}, + } + ] + } + } + + unchanged = payload(committed["prompt"], committed["completion"]) + assert publication_runner._pricing_drift_issues(cell, unchanged) == [] + + risen = payload(committed["prompt"] * 2, committed["completion"]) + issues = publication_runner._pricing_drift_issues(cell, risen) + assert issues and "exceeds the committed snapshot rate" in issues[0] + + capsys.readouterr() + fallen = payload(committed["prompt"] / 2, committed["completion"] / 2) + assert publication_runner._pricing_drift_issues(cell, fallen) == [], "a discount must not block a run" + out = capsys.readouterr().out + assert "fell from" in out and "under its reservation" in out + + # An unpinned route in the payload is not this cell's price. + other = payload(committed["prompt"] * 10, committed["completion"]) + other["data"]["endpoints"][0]["tag"] = "someone-else/fp8" + assert publication_runner._pricing_drift_issues(cell, other) == [] + + def test_the_suite_cannot_inherit_a_live_provider_credential() -> None: """Pin the guard that stops a test from quietly billing a real account. From 4462d2c29a74549d758d67b54de77546348eddcd Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 17:49:59 -0400 Subject: [PATCH 4/7] Raise the operator ceiling to $150 and make the two budget numbers agree The $120 ceiling was chosen against a $119.76 reservation that turned out to depend on 50%-off promotional rates on openai/gpt-5.6-luna and z-ai/glm-5.2. Pinning undiscounted list rates moved the reservation to $127.29 and put the committed plan over its own committed ceiling. Owner raised the cap to $150.00, which clears it with headroom for a further route substitution or list-price move without another ceiling decision. Nothing had detected the breach, because the reservation and the ceiling live in different files and nothing compared them. Two invariants now do, at zero cost, and both were verified to fail when broken: - the committed reservation must fit under the committed ceiling, so adding a model, substituting onto a pricier host, or losing a discount fails a test rather than surfacing when someone tries to authorize a run; - the committed cost artifact must match the configs it claims to describe, so the number the readiness docs and the ceiling decision quote cannot go stale against the plan that would actually run. No gate moves. spend_authorized, smoke_execution_authorized, panel_execution_authorized, and publication_authorized all remain false, the registry is still route-preflight-ready rather than frozen, and exact_route_acceptance is still unresolved. Projected actual spend remains ~$35-45, so the ceiling stays a backstop rather than a forecast. --- config/sota_v3_publication_protocol.json | 4 +- docs/PUBLISH_READINESS.md | 2 +- docs/ROUTE_SUBSTITUTION_POLICY.md | 2 +- .../sota-v3-lineup-refresh-2026-08-04.md | 16 ++++++-- tests/test_publication_cost.py | 39 +++++++++++++++++++ tests/test_publication_runner.py | 2 +- 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/config/sota_v3_publication_protocol.json b/config/sota_v3_publication_protocol.json index 5c74389..483c493 100644 --- a/config/sota_v3_publication_protocol.json +++ b/config/sota_v3_publication_protocol.json @@ -84,8 +84,8 @@ "cost_estimate_artifact": "results/analysis/sota-v3-pre-smoke-cost-estimate.json", "operator_must_pass_max_spend_usd": true, "spend_authorized": false, - "operator_ceiling_usd": 120.0, - "operator_ceiling_basis": "Owner-set hard cap, 2026-08-04. Above the $119.76 pre-smoke reservation so the committed plan fits, and enforced by the runner: --max-spend-usd above this value is rejected before any cell runs. Projected actual spend is ~$34 from July smoke telemetry reprojected at current rates (models emit 48-640 output tokens per decision against a 4,096 reservation), so the cap is a backstop rather than the expected bill. Raising it is a deliberate edit here." + "operator_ceiling_usd": 150.0, + "operator_ceiling_basis": "Owner-set hard cap, raised 120.00 -> 150.00 on 2026-08-04. The $120 figure was chosen against a $119.76 reservation that turned out to depend on 50%-off promotional rates on openai/gpt-5.6-luna and z-ai/glm-5.2; pinning undiscounted list rates moved the reservation to $127.29 and put the committed plan over its own ceiling. $150.00 clears the current reservation with headroom for a further route substitution or list-price move without another ceiling decision. Enforced by the runner ahead of the cell loop: --max-spend-usd above this value is rejected before any endpoint probe or child process. Projected actual spend is ~$35-45 from July smoke telemetry reprojected at current rates (models emit 48-640 output tokens per decision against a 4,096-token reservation), so this is a backstop, not a forecast. Raising it again is a deliberate edit here." }, "publication_authorized": false } diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index d1fba67..4e6ff52 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -862,7 +862,7 @@ decision and why. | --- | --- | --- | --- | | 2026-08-04 | Adopt `qwen/qwen3.8-max` in place of `qwen/qwen3.7-plus`, and adopt a written [route substitution policy](ROUTE_SUBSTITUTION_POLICY.md). | Qwen 3.8 Max shipped 2026-08-03 and the benchmark is intended to track new releases rather than freeze once. Three substitutions in two days had each been decided ad hoc, which is how a benchmark quietly starts measuring whichever host is cheapest this week. | Cohort stays at ten, so the Holm family, the 16x1 allocation, and the power selection are untouched. The Max tier bills 6.25x/4.69x the Plus tier per token. The policy fixes eligibility, forbids price/throughput/first-party status as substitution criteria, and requires re-establishing route and privacy acceptance for any new counterparty. | | 2026-08-04 | Substitute `deepseek/fp8` -> `cloudflare/fp8` and `minimax/fp8` -> `deepinfra/fp8`. | Both first-party routes were deranked the same day (status `-5` at 78% availability, and `-2`). Both replacements are the same model at the same FP8 quantization, chosen on highest 24h availability, at identical published rates. | No cost or cohort-size effect. The DeepSeek route recovered on its own within the day, so in hindsight that substitution was not strictly necessary; it is kept because Cloudflare holds the better 24h record. Route and privacy acceptance do **not** carry over — Cloudflare and DeepInfra are counterparties this project has never reviewed. | -| 2026-08-04 | Pin undiscounted list rates instead of promotional rates, and enforce `operator_ceiling_usd` in the runner. | `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both pinned at 50%-off promos; the GLM discount moved 55.1% -> 50% within hours of being recorded. A reservation computed from a promo is wrong the moment it ends. Separately, `operator_ceiling_usd` had sat in config unread, so nothing enforced the committed cap. | Reservation $119.76 -> **$127.29**, which **exceeds the owner's $120.00 ceiling** — the plan only ever appeared to fit because of the discounts. Awaiting an owner decision on the cap. Projected actual spend is ~$35-45 from July smoke telemetry, so the ceiling is a backstop rather than the expected bill. Logged in [`docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md`](run_logs/sota-v3-lineup-refresh-2026-08-04.md). | +| 2026-08-04 | Pin undiscounted list rates instead of promotional rates, and enforce `operator_ceiling_usd` in the runner. | `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both pinned at 50%-off promos; the GLM discount moved 55.1% -> 50% within hours of being recorded. A reservation computed from a promo is wrong the moment it ends. Separately, `operator_ceiling_usd` had sat in config unread, so nothing enforced the committed cap. | Reservation $119.76 -> **$127.29**, which exceeded the then-committed $120.00 ceiling — the plan only ever appeared to fit because of the discounts. **Ceiling raised to $150.00 on 2026-08-04** to clear the reservation with headroom for a further substitution or list-price move. The reservation and the ceiling are now compared by `test_the_committed_plan_fits_under_the_committed_ceiling`, and the committed cost artifact is checked against the configs it claims to describe, so neither can drift silently again. Projected actual spend is ~$35-45 from July smoke telemetry, so the ceiling is a backstop rather than the expected bill. Logged in [`docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md`](run_logs/sota-v3-lineup-refresh-2026-08-04.md). | | 2026-07-13 | Treat v1 model rows as archived historical evidence, not a current ranking. | Scout-key mismatch affected models unevenly; failed queries were invisible. | Current claims require `sota-v2`; v1 remains auditable under `sota-v1`. | | 2026-07-13 | Separate API and coding-harness lanes. | Archived rows mixed provider API behavior with uncontrolled CLI harness context and very different output usage. | API becomes the headline lane; CLI harnesses remain diagnostic. | | 2026-07-13 | Withhold the v2 ranking pending an output-budget sweep. | Archived scores tracked output allowance strongly enough to confound model comparison. | Run the planned cap matrix and freeze a compute policy before the full panel. | diff --git a/docs/ROUTE_SUBSTITUTION_POLICY.md b/docs/ROUTE_SUBSTITUTION_POLICY.md index 4bf042d..d4bcaef 100644 --- a/docs/ROUTE_SUBSTITUTION_POLICY.md +++ b/docs/ROUTE_SUBSTITUTION_POLICY.md @@ -102,7 +102,7 @@ The reservation is a safe upper bound, and a promo is not a floor. On 2026-08-04 the GLM 5.2 Novita discount moved from 55.1% to 50% within hours, and `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both found pinned at 50%-off prices — which is the only reason the panel had appeared to fit under the -$120 ceiling. Where the endpoint reports a `discount`, record +$120 ceiling in force at the time. Where the endpoint reports a `discount`, record `live_rate / (1 - discount)`. A live discount then only ever brings the run in under reserve, which is the diff --git a/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md index 3c3a8b2..e7d6cee 100644 --- a/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md +++ b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md @@ -66,10 +66,20 @@ not a floor, and a reservation computed from one is wrong the moment it ends. | Previous reservation | $107.81 | | After the Qwen amendment | $119.76 | | **After pinning list rates** | **$127.29** | -| Committed operator ceiling | $120.00 | +| Committed operator ceiling | **$150.00** (raised from $120.00, 2026-08-04) | -**The plan no longer fits under the ceiling, and only ever appeared to because -two routes were priced at a discount.** Call count is unchanged at 3,240. +Call count is unchanged at 3,240. + +The $120 ceiling was chosen against a $119.76 reservation that turned out to +depend on two 50%-off promotional rates. Pinning list rates pushed the +committed plan over its own committed ceiling — a state nothing detected, +because the reservation and the ceiling lived in different files and nothing +compared them. `test_the_committed_plan_fits_under_the_committed_ceiling` now +does, at zero cost, so adding a model or losing a discount fails a test rather +than surfacing when someone tries to authorize a run. + +The ceiling was raised to **$150.00** to clear the current reservation with +headroom for a further substitution or list-price move. Reserve is concentrated in four models: Gemini 3.6 Flash (22.4%), Grok 4.5 (19.9%), Claude Sonnet 5 (17.4%), and Mistral Medium 3.5 (13.0%). diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index e8edd61..87b6b6a 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -60,6 +60,45 @@ def test_v3_cost_plan_uses_registered_private_seed_count() -> None: assert grok["internal_reasoning_billing_basis"] == "completion" +def test_the_committed_plan_fits_under_the_committed_ceiling() -> None: + """The reservation and the hard cap must not drift apart silently. + + These are two committed numbers in two different files, and on 2026-08-04 + they crossed: pinning undiscounted list rates moved the reservation to + $127.29 against a $120.00 ceiling, so the committed plan could not legally + run. Nothing caught it, because nothing compared them. + + Adding a model, substituting a route onto a pricier host, or a provider + ending a discount all move the reservation. Any of them silently breaching + the ceiling should fail here, at zero cost, rather than at the point + someone tries to authorize a run. + """ + protocol = json.loads(Path("config/sota_v3_publication_protocol.json").read_text()) + ceiling = protocol["budget_policy"]["operator_ceiling_usd"] + reserved = estimate(*_v3_inputs())["costs_usd"]["total_with_1_2x_contingency"] + + assert isinstance(ceiling, (int, float)) and ceiling > 0 + assert reserved <= ceiling, ( + f"the committed reservation ${reserved:.2f} exceeds the committed operator ceiling " + f"${ceiling:.2f}; raise the ceiling deliberately or reduce the plan" + ) + + +def test_the_committed_cost_artifact_matches_the_committed_configs() -> None: + """A stale cost artifact is a reservation nobody recomputed. + + The runner reserves per cell from the pricing snapshot, but the artifact is + what the readiness docs and the ceiling decision quote. If someone edits a + price or a route without regenerating it, the number people reason about + stops describing the plan they would actually run. + """ + committed = json.loads(Path("results/analysis/sota-v3-pre-smoke-cost-estimate.json").read_text()) + recomputed = estimate(*_v3_inputs()) + + assert committed["costs_usd"] == pytest.approx(recomputed["costs_usd"]) + assert {row["model"] for row in committed["models"]} == {row["model"] for row in recomputed["models"]} + + def test_costs_sum_unrounded_rows_before_contingency() -> None: result = estimate(*_committed_inputs()) rows = result["models"] diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 23de2a6..396d40b 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1675,7 +1675,7 @@ def test_operator_ceiling_rejects_a_run_that_could_outspend_the_committed_cap() """ ceiling = json.loads(Path("config/sota_v3_publication_protocol.json").read_text()) ceiling = ceiling["budget_policy"]["operator_ceiling_usd"] - assert ceiling == 120.00 + assert ceiling == 150.00 publication_runner._enforce_operator_ceiling(ceiling, "sota-v3") publication_runner._enforce_operator_ceiling(ceiling - 0.01, "sota-v3") From 868c84c9782e548e2a11ff654eb10e77d1198bf3 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 18:04:49 -0400 Subject: [PATCH 5/7] Close an unbounded-spend hole and fail closed on unverifiable prices Both found in review of the previous two commits; both verified before fixing. --max-spend-usd nan satisfied every guard while bounding nothing. NaN makes `nan <= 0`, `nan > ceiling`, and `spent >= nan` all false, so the positivity check, the operator ceiling, and the in-loop spend check each waved it through -- an unbounded paid run that looks fully authorized. Infinity is the same hole wherever no ceiling is configured. Non-finite limits and non-finite configured ceilings are now refused. _pricing_drift_issues failed open on every path that was not a clean comparison: unreadable snapshot, missing committed rate, missing or malformed live rate. "The price could not be verified" is not "the price is unchanged", and only one of those is safe to spend against, so each now blocks. It also matched endpoints on tag and name alone while the preflight pins provider_name, tag, and name; it now uses the full registered identity, so a same-tag endpoint from another provider can neither satisfy the check nor be compared against as though it were the pinned route. Also: preflight failures go to stderr rather than stdout, and both substitution roles now name the exact outgoing route and carry the correct observed 30m figure (78.93% for deepseek/fp8, 94.59% for minimax/fp8). All ten routes still pass the live probe. No gate moves. --- config/sota_v3_models.json | 4 +- scripts/run_publication_matrix.py | 42 ++++++++++--- tests/test_publication_runner.py | 97 ++++++++++++++++++++++++++++++- 3 files changed, 129 insertions(+), 14 deletions(-) diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index f233e57..503f78e 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -301,7 +301,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "MiniMax open-weight anchor, substituted onto the DeepInfra FP8 route on 2026-08-04 after the first-party route was deranked to status -2. Highest 24h availability among eligible FP8 routes; identical published rates." + "role": "MiniMax open-weight anchor, substituted from the first-party `minimax/fp8` route onto `deepinfra/fp8` on 2026-08-04 after `minimax/fp8` was deranked to status -2 (30m uptime 94.59%). Same FP8 quantization and identical published rates; highest 24h uptime (99.63%) among eligible FP8 routes. Per docs/ROUTE_SUBSTITUTION_POLICY.md." }, { "id": "openrouter-qwen3.8-max-alibaba", @@ -449,7 +449,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "DeepSeek open-weight anchor, substituted onto the Cloudflare FP8 route on 2026-08-04 after the first-party DeepSeek route was deranked to status -5 (30m uptime 77%). Same FP8 quantization and identical published rates; selected on 24h uptime (99.75%, best of 16 endpoints), not spot health." + "role": "DeepSeek open-weight anchor, substituted from the first-party `deepseek/fp8` route onto `cloudflare/fp8` on 2026-08-04 after `deepseek/fp8` was deranked to status -5 (30m uptime 78.93%). Same FP8 quantization and identical published rates; selected on 24h uptime (99.75%, best of sixteen endpoints), not spot health. Per docs/ROUTE_SUBSTITUTION_POLICY.md." }, { "id": "openrouter-hy3-tencent", diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index 5eccc2e..39174b3 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -12,6 +12,7 @@ import argparse import hashlib import json +import math import os import subprocess import sys @@ -494,8 +495,8 @@ def _enforce_operator_ceiling(max_spend_usd: float, contract: str | None) -> Non ceiling = budget_policy.get("operator_ceiling_usd") if ceiling is None: return - if not isinstance(ceiling, (int, float)) or isinstance(ceiling, bool) or ceiling <= 0: - raise ValueError(f"budget_policy.operator_ceiling_usd must be a positive number, got {ceiling!r}") + if not isinstance(ceiling, (int, float)) or isinstance(ceiling, bool) or not math.isfinite(ceiling) or ceiling <= 0: + raise ValueError(f"budget_policy.operator_ceiling_usd must be a positive finite number, got {ceiling!r}") if max_spend_usd > ceiling: raise ValueError( f"--max-spend-usd ${max_spend_usd:.2f} exceeds the committed operator ceiling " @@ -576,29 +577,46 @@ def _pricing_drift_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: """ try: rates = (_read_json(PRICING_CONFIG).get("models") or {}).get(cell.model) - except (OSError, ValueError, json.JSONDecodeError): - return [] + except (OSError, ValueError, json.JSONDecodeError) as exc: + return [f"committed pricing snapshot for {cell.experiment_id} could not be read: {exc}"] if not isinstance(rates, dict): - return [] + return [f"committed pricing snapshot has no rates for {cell.model}"] + # Match the full registered route identity, exactly as `_endpoint_issues` + # does. Comparing a price against a route we did not pin is worse than not + # comparing at all. endpoint = next( ( e for e in ((payload.get("data") or {}).get("endpoints") or []) - if e.get("tag") == cell.endpoint_tag and e.get("name") == cell.endpoint_name + if e.get("provider_name") == cell.upstream_provider + and e.get("tag") == cell.endpoint_tag + and e.get("name") == cell.endpoint_name ), None, ) if endpoint is None: - return [] + return [f"no endpoint matching the pinned route identity for {cell.experiment_id} to price-check"] issues = [] for field in ("prompt", "completion"): committed = rates.get(field) raw = (endpoint.get("pricing") or {}).get(field) - if not isinstance(committed, (int, float)) or raw is None: + # Fail closed. "The price could not be verified" is not the same as + # "the price is unchanged", and only one of them is safe to spend on. + if ( + not isinstance(committed, (int, float)) + or isinstance(committed, bool) + or not math.isfinite(committed) + or committed < 0 + ): + issues.append(f"committed {field} rate for {cell.experiment_id} is not a usable number: {committed!r}") continue try: live = float(raw) except (TypeError, ValueError): + issues.append(f"live {field} rate for {cell.experiment_id} is unreadable: {raw!r}") + continue + if not math.isfinite(live) or live < 0: + issues.append(f"live {field} rate for {cell.experiment_id} is not a usable number: {raw!r}") continue if live > committed: issues.append( @@ -1542,6 +1560,12 @@ def main(argv: list[str] | None = None) -> int: except (OSError, ValueError, json.JSONDecodeError) as exc: parser.error(str(exc)) return _record_smoke(args.model_id, args.artifact, manifest_path) + if args.max_spend_usd is not None and not math.isfinite(args.max_spend_usd): + # NaN defeats every downstream guard silently: `nan <= 0`, + # `nan > ceiling`, and `spent >= nan` are all false, so a NaN limit + # satisfies "the operator passed a ceiling" while bounding nothing. + # Infinity is the same hole whenever no ceiling is configured. + parser.error("--max-spend-usd must be a finite number") if args.max_spend_usd is not None and args.max_spend_usd <= 0: parser.error("--max-spend-usd must be positive") if args.phase in {"route-preflight", "smoke", "panel"}: @@ -1635,7 +1659,7 @@ def main(argv: list[str] | None = None) -> int: if args.phase != "route-preflight": raise SystemExit(failure) from exc preflight_failures.append(failure) - print(failure) + print(failure, file=sys.stderr) continue if args.phase == "route-preflight": print(f"zero-completion-call route preflight passed: {cell.experiment_id}") diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 396d40b..1a3c5d4 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1604,6 +1604,92 @@ def _healthy_endpoint(cell) -> dict: } +def test_non_finite_spend_limits_are_refused( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`--max-spend-usd nan` must not read as a ceiling. + + NaN defeats every downstream guard at once and silently: `nan <= 0`, + `nan > ceiling`, and `spent >= nan` are all False, so a NaN limit satisfies + "the operator passed a ceiling" while bounding nothing at all -- an + unbounded paid run that looks fully authorized. Infinity is the same hole + wherever no ceiling is configured. + """ + _frozen_panel_files(tmp_path, monkeypatch) + checked: list[str] = [] + monkeypatch.setattr( + publication_runner, + "_validate_openrouter_endpoint", + lambda cell, _env: checked.append(cell.experiment_id), + ) + + for literal in ("nan", "inf", "-inf", "NaN", "Infinity"): + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", literal]) + assert exc_info.value.code == 2, literal + assert checked == [], "a non-finite spend limit reached the endpoint probe" + + nonfinite = tmp_path / "nonfinite-protocol.json" + nonfinite.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": float("inf")}})) + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-nonfinite-ceiling", + (nonfinite,) * 5, + ) + with pytest.raises(ValueError, match="positive finite number"): + publication_runner._enforce_operator_ceiling(1.0, "sota-nonfinite-ceiling") + + +def test_pricing_drift_fails_closed_when_a_rate_cannot_be_verified( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ "Could not verify the price" is not "the price is unchanged". + + Only one of those is safe to spend against, so every unverifiable case + blocks rather than passing quietly. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + committed = json.loads(publication_runner.PRICING_CONFIG.read_text())["models"][cell.model] + + def payload(pricing: dict, **overrides) -> dict: + endpoint = { + "provider_name": cell.upstream_provider, + "tag": cell.endpoint_tag, + "name": cell.endpoint_name, + "pricing": pricing, + } + endpoint.update(overrides) + return {"data": {"endpoints": [endpoint]}} + + good = {"prompt": str(committed["prompt"]), "completion": str(committed["completion"])} + assert publication_runner._pricing_drift_issues(cell, payload(good)) == [] + + for pricing, expected in ( + ({"completion": good["completion"]}, "unreadable"), + ({**good, "prompt": "not-a-number"}, "unreadable"), + ({**good, "prompt": "nan"}, "not a usable number"), + ({**good, "prompt": "-1e-07"}, "not a usable number"), + ): + issues = publication_runner._pricing_drift_issues(cell, payload(pricing)) + assert issues and expected in issues[0], (pricing, issues) + + # The pinned identity is provider + tag + name, matching the preflight. + # A same-tag endpoint from another provider is not this cell's price. + issues = publication_runner._pricing_drift_issues(cell, payload(good, provider_name="Somebody Else")) + assert issues and "pinned route identity" in issues[0] + + # A model absent from the snapshot cannot be price-checked at all. + snapshot = json.loads(publication_runner.PRICING_CONFIG.read_text()) + del snapshot["models"][cell.model] + stripped = tmp_path / "pricing.json" + stripped.write_text(json.dumps(snapshot)) + monkeypatch.setattr(publication_runner, "PRICING_CONFIG", stripped) + issues = publication_runner._pricing_drift_issues(cell, payload(good)) + assert issues and "no rates for" in issues[0] + + def test_pricing_drift_fails_upward_and_only_reports_downward( capsys: pytest.CaptureFixture[str], tmp_path: Path, @@ -1625,6 +1711,7 @@ def payload(prompt: float, completion: float) -> dict: "data": { "endpoints": [ { + "provider_name": cell.upstream_provider, "tag": cell.endpoint_tag, "name": cell.endpoint_name, "pricing": {"prompt": str(prompt), "completion": str(completion)}, @@ -1646,10 +1733,14 @@ def payload(prompt: float, completion: float) -> dict: out = capsys.readouterr().out assert "fell from" in out and "under its reservation" in out - # An unpinned route in the payload is not this cell's price. + # Another provider's rate is not this cell's rate, so it must not be read + # as one -- but nor may the absence of the pinned route pass as "unchanged". + # Both are "the price could not be verified", which now blocks. other = payload(committed["prompt"] * 10, committed["completion"]) other["data"]["endpoints"][0]["tag"] = "someone-else/fp8" - assert publication_runner._pricing_drift_issues(cell, other) == [] + issues = publication_runner._pricing_drift_issues(cell, other) + assert issues and "pinned route identity" in issues[0] + assert "exceeds the committed snapshot rate" not in issues[0], "priced against the wrong route" def test_the_suite_cannot_inherit_a_live_provider_credential() -> None: @@ -1701,7 +1792,7 @@ def test_operator_ceiling_stays_permissive_when_no_cap_is_committed( publication_runner._enforce_operator_ceiling(10_000.00, "sota-test") protocol_path.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": "lots"}})) - with pytest.raises(ValueError, match="must be a positive number"): + with pytest.raises(ValueError, match="must be a positive finite number"): publication_runner._enforce_operator_ceiling(1.00, "sota-test") From a65c79033f39419f9e90abe7ca589650ae353df3 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 18:10:58 -0400 Subject: [PATCH 6/7] Address review: explicit contract in tests, stronger artifact freshness check Two route-preflight tests invoked main() without --contract, which is required of every operator invocation. They asserted real behaviour -- both still catch their own regression under mutation -- but they were not exercising the path an operator actually takes. Both now pass --contract sota-v3. The cost-artifact freshness check now also compares call counts, the pricing timestamp, and per-model cost-per-decision. The timestamp is what tells a reader which snapshot a number describes, so a stale one is its own defect even when the totals happen to agree. --- tests/test_publication_cost.py | 11 ++++++++++- tests/test_publication_runner.py | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index 87b6b6a..a2c5a70 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -96,7 +96,16 @@ def test_the_committed_cost_artifact_matches_the_committed_configs() -> None: recomputed = estimate(*_v3_inputs()) assert committed["costs_usd"] == pytest.approx(recomputed["costs_usd"]) - assert {row["model"] for row in committed["models"]} == {row["model"] for row in recomputed["models"]} + assert committed["calls"] == recomputed["calls"] + # The timestamp is what tells a reader which snapshot the number describes, + # so a stale one is its own defect even when the totals happen to agree. + assert committed["pricing_checked_at_utc"] == recomputed["pricing_checked_at_utc"] + committed_rows = {row["experiment_id"]: row for row in committed["models"]} + recomputed_rows = {row["experiment_id"]: row for row in recomputed["models"]} + assert set(committed_rows) == set(recomputed_rows) + for experiment_id, row in recomputed_rows.items(): + assert committed_rows[experiment_id]["model"] == row["model"] + assert committed_rows[experiment_id]["cost_per_decision_usd"] == pytest.approx(row["cost_per_decision_usd"]) def test_costs_sum_unrounded_rows_before_contingency() -> None: diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 1a3c5d4..e4ba549 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1364,7 +1364,7 @@ def fake_validate(cell, _env): ) with pytest.raises(SystemExit) as exc_info: - main(["route-preflight", "--run-dir", str(tmp_path)]) + main(["route-preflight", "--contract", "sota-v3", "--run-dir", str(tmp_path)]) message = str(exc_info.value.code) # Every route was probed, including the ones queued behind both failures. @@ -1399,7 +1399,7 @@ def fake_validate(cell, _env): monkeypatch.setattr(publication_runner, "_validate_openrouter_endpoint", fake_validate) with pytest.raises(SystemExit) as exc_info: - main(["smoke", "--run-dir", str(tmp_path), "--max-spend-usd", "1.00"]) + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", "1.00"]) assert checked == model_ids[:1], "a paid phase kept probing after a bad route" assert model_ids[0] in str(exc_info.value.code) From 4d86038e5cd9aa86e608b85cdab55e8a3d07fb36 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Tue, 4 Aug 2026 18:23:02 -0400 Subject: [PATCH 7/7] Fail closed on unverifiable route health --- config/sota_v3_models.json | 4 ++-- docs/ROUTE_SUBSTITUTION_POLICY.md | 11 ++++++---- scripts/run_publication_matrix.py | 30 ++++++++++++++++++++++------ tests/test_publication_runner.py | 31 ++++++++++++++++++++++++++--- tests/test_sota_v3_route_catalog.py | 5 +++++ 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index 503f78e..0226b8c 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -11,14 +11,14 @@ "repeats": 1, "selection_status": "route-preflight-ready", "selection_frozen_at_utc": null, - "selection_revision": "2026-08-03-public-catalog-cohort-v2", + "selection_revision": "2026-08-04-public-catalog-lineup-refresh-v3", "catalog_snapshot_status": "frozen-public-metadata-only", "catalog_checked_at_utc": "2026-08-04T16:00:40Z", "catalog_sources": [ "https://openrouter.ai/api/v1/models", "https://openrouter.ai/api/v1/models/{model_id}/endpoints" ], - "selection_policy": "Ten-model pre-data cohort selected from the 2026-08-03 public OpenRouter catalog, superseding the eight-model 2026-07-28 cohort before any smoke or panel evidence exists. GPT-5.6 Luna replaces GPT-5.6 Luna Pro: the plain Luna route that was unhealthy at the prior snapshot has recovered, and the Pro variant carried an unresolved reasoning.mode inconsistency between its public description and the structured catalog. DeepSeek V4 Flash 0731 and Tencent Hy3 (both on first-party FP8 routes) are added as open-weight anchors. Thinking Machines Inkling Small was evaluated for the tenth slot and found ineligible at this snapshot: no healthy route advertises the response_format parameter the lane's frozen JSON-mode and require-parameters options demand. Exact public endpoint metadata and prices are pinned below, but the registry remains provisional-blocked because public metadata does not prove authenticated exact-route access or provider privacy and retention behavior.", + "selection_policy": "Ten-model pre-data cohort originally selected from the 2026-08-03 public OpenRouter catalog and refreshed on 2026-08-04 before any smoke or panel evidence existed. The refresh replaces Qwen 3.7 Plus with Qwen 3.8 Max and substitutes the same-model MiniMax M3 and DeepSeek V4 Flash 0731 slots from minimax/fp8 and deepseek/fp8 onto deepinfra/fp8 and cloudflare/fp8 under docs/ROUTE_SUBSTITUTION_POLICY.md. Cohort size, Holm family, and the 16x1 allocation are unchanged. Exact public endpoint metadata and undiscounted list prices are pinned below, but the registry remains route-preflight-ready rather than frozen because authenticated exact-route access, parameter behavior, and provider privacy and retention acceptance remain unresolved.", "models": [ { "id": "openrouter-gpt-5.6-luna-openai", diff --git a/docs/ROUTE_SUBSTITUTION_POLICY.md b/docs/ROUTE_SUBSTITUTION_POLICY.md index d4bcaef..1fd5835 100644 --- a/docs/ROUTE_SUBSTITUTION_POLICY.md +++ b/docs/ROUTE_SUBSTITUTION_POLICY.md @@ -32,9 +32,10 @@ zero-call probe reports every failing route in one pass. 2. Endpoint `status` is not `0`. 3. The endpoint does not advertise `max_tokens`, `reasoning`, and — when the lane sets `OPENROUTER_JSON_MODE=true` — `response_format`. -4. `max_completion_tokens` is below the registered output cap. -5. `uptime_last_30m` is below **90%**. -6. `uptime_last_1d` is below **95%**. +4. `max_completion_tokens` is missing, non-integer, or below the registered + output cap. +5. `uptime_last_30m` is missing, non-numeric, non-finite, or below **90%**. +6. `uptime_last_1d` is missing, non-numeric, non-finite, or below **95%**. 7. The live base rate for the pinned route is **above** the committed pricing snapshot. A rate below the snapshot is reported, not blocked. @@ -43,7 +44,9 @@ On (5) and (6): both windows gate because they detect different failures. The read 99.24% over 24h while serving 78% of requests. Both floors sit well below the observed noise band, because these readings drift about half a point between consecutive polls; a 99% 24h floor was measured rejecting two healthy cohort -members while still passing the route that had actually failed. +members while still passing the route that had actually failed. An absent or +malformed figure is unknown health, not evidence that the route clears the +floor, so it blocks the same way unverifiable pricing blocks. ## Choosing the replacement diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index 39174b3..f8f09a6 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -529,20 +529,38 @@ def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: for endpoint in matches: supported = set(endpoint.get("supported_parameters") or []) maximum = endpoint.get("max_completion_tokens") - cap_fits = cell.cap is None or maximum is None or (isinstance(maximum, int) and cell.cap <= maximum) + cap_fits = cell.cap is None or ( + isinstance(maximum, int) and not isinstance(maximum, bool) and cell.cap <= maximum + ) if required <= supported and cap_fits: capable.append(endpoint) if not capable: return [f"matching endpoint cannot honor required parameters {sorted(required)!r} and cap={cell.cap_label}"] floors = (("uptime_last_30m", MIN_UPTIME_LAST_30M_PCT, "30m"), ("uptime_last_1d", MIN_UPTIME_LAST_1D_PCT, "24h")) for field, floor, label in floors: - # A route that does not publish the figure is not penalised for it; - # only a published figure below the floor disqualifies. - durable = [e for e in capable if not isinstance(e.get(field), (int, float)) or e[field] >= floor] + # Fail closed. Missing or malformed health telemetry does not establish + # that the pinned route clears the declared availability floor. + durable = [ + endpoint + for endpoint in capable + if isinstance(endpoint.get(field), (int, float)) + and not isinstance(endpoint[field], bool) + and math.isfinite(endpoint[field]) + and endpoint[field] >= floor + ] if not durable: - observed = max(e[field] for e in capable if isinstance(e.get(field), (int, float))) + observed = [ + endpoint[field] + for endpoint in capable + if isinstance(endpoint.get(field), (int, float)) + and not isinstance(endpoint[field], bool) + and math.isfinite(endpoint[field]) + ] + if not observed: + return [f"matching endpoint has no finite numeric {label} uptime telemetry"] return [ - f"matching endpoint is below the {floor}% {label} uptime floor (best matching route: {observed:.2f}%)" + f"matching endpoint is below the {floor}% {label} uptime floor " + f"(best matching route: {max(observed):.2f}%)" ] capable = durable return [] diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index e4ba549..e878c1b 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1328,6 +1328,7 @@ def test_zero_call_route_preflight_has_separate_authorization_and_never_launches def test_route_preflight_checks_every_route_before_failing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: """A bad route must not hide the routes queued behind it. @@ -1372,6 +1373,9 @@ def fake_validate(cell, _env): assert f"failed for {len(doomed)} of {len(model_ids)} routes" in message for model_id in doomed: assert model_id in message + captured = capsys.readouterr() + assert captured.err.count("OpenRouter endpoint preflight failed") == len(doomed) + assert "OpenRouter endpoint preflight failed" not in captured.out # Still zero-call and still stateless, exactly as on the passing path. assert child_calls == [] assert not (tmp_path / "run-state.json").exists() @@ -1576,6 +1580,8 @@ def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: "status": 0, "max_completion_tokens": 65536, "supported_parameters": ["max_tokens", "response_format", "reasoning"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, } ] } @@ -1589,6 +1595,11 @@ def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: valid["data"]["endpoints"][0]["name"] = cell.endpoint_name valid["data"]["endpoints"][0]["supported_parameters"] = ["max_tokens", "response_format"] assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + valid["data"]["endpoints"][0]["supported_parameters"] = ["max_tokens", "response_format", "reasoning"] + valid["data"]["endpoints"][0].pop("max_completion_tokens") + assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + valid["data"]["endpoints"][0]["max_completion_tokens"] = "65536" + assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] def _healthy_endpoint(cell) -> dict: @@ -1838,8 +1849,8 @@ def test_endpoint_preflight_enforces_both_uptime_floors() -> None: The 24h figure is a chronic filter and cannot see an outage in progress: on 2026-08-04 the first-party DeepSeek route was deranked to status -5 while still reporting 99.24% over 24h. The 30m figure is the one that - moved, so it carries the acute signal. A route that publishes neither - figure is not penalised for the omission. + moved, so it carries the acute signal. Missing or malformed telemetry is + unknown health, so it must fail closed rather than read as a passing route. """ cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] endpoint = _healthy_endpoint(cell) @@ -1865,9 +1876,21 @@ def test_endpoint_preflight_enforces_both_uptime_floors() -> None: endpoint[field] = floor assert _endpoint_issues(cell, payload) == [], f"{field} floor must be inclusive" + for field, value in ( + ("uptime_last_30m", None), + ("uptime_last_30m", "unknown"), + ("uptime_last_30m", float("nan")), + ("uptime_last_1d", float("inf")), + ): + endpoint.update(_healthy_endpoint(cell)) + endpoint[field] = value + issues = _endpoint_issues(cell, payload) + assert issues and "no finite numeric" in issues[0], (field, value, issues) + endpoint.update(_healthy_endpoint(cell)) del endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] - assert _endpoint_issues(cell, payload) == [], "an unpublished figure must not fail the route" + issues = _endpoint_issues(cell, payload) + assert issues and "no finite numeric" in issues[0] def test_uptime_floors_sit_below_the_healthy_cohort_noise_band() -> None: @@ -1895,6 +1918,8 @@ def test_endpoint_preflight_allows_registered_prompt_only_json_route() -> None: "status": 0, "max_completion_tokens": 262144, "supported_parameters": ["max_tokens", "reasoning", "structured_outputs"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, } ] } diff --git a/tests/test_sota_v3_route_catalog.py b/tests/test_sota_v3_route_catalog.py index 7cffd4e..8270bfd 100644 --- a/tests/test_sota_v3_route_catalog.py +++ b/tests/test_sota_v3_route_catalog.py @@ -44,6 +44,11 @@ def test_v3_catalog_freezes_exact_balanced_cohort_without_unlocking_execution() assert registry["catalog_snapshot_status"] == "frozen-public-metadata-only" assert registry["selection_status"] == "route-preflight-ready" assert registry["selection_frozen_at_utc"] is None + assert registry["selection_revision"] == "2026-08-04-public-catalog-lineup-refresh-v3" + policy = registry["selection_policy"] + assert "Qwen 3.8 Max" in policy + assert "deepinfra/fp8" in policy and "cloudflare/fp8" in policy + assert "route-preflight-ready rather than frozen" in policy assert registry["catalog_checked_at_utc"] assert set(registry["required_smokes"]) == {model["id"] for model in models} assert registry["output_token_cap"] == 4_096