diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index c7dd595..0226b8c 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -11,14 +11,14 @@ "repeats": 1, "selection_status": "route-preflight-ready", "selection_frozen_at_utc": null, - "selection_revision": "2026-08-03-public-catalog-cohort-v2", + "selection_revision": "2026-08-04-public-catalog-lineup-refresh-v3", "catalog_snapshot_status": "frozen-public-metadata-only", - "catalog_checked_at_utc": "2026-08-03T15:53:59Z", + "catalog_checked_at_utc": "2026-08-04T16:00:40Z", "catalog_sources": [ "https://openrouter.ai/api/v1/models", "https://openrouter.ai/api/v1/models/{model_id}/endpoints" ], - "selection_policy": "Ten-model pre-data cohort selected from the 2026-08-03 public OpenRouter catalog, superseding the eight-model 2026-07-28 cohort before any smoke or panel evidence exists. GPT-5.6 Luna replaces GPT-5.6 Luna Pro: the plain Luna route that was unhealthy at the prior snapshot has recovered, and the Pro variant carried an unresolved reasoning.mode inconsistency between its public description and the structured catalog. DeepSeek V4 Flash 0731 and Tencent Hy3 (both on first-party FP8 routes) are added as open-weight anchors. Thinking Machines Inkling Small was evaluated for the tenth slot and found ineligible at this snapshot: no healthy route advertises the response_format parameter the lane's frozen JSON-mode and require-parameters options demand. Exact public endpoint metadata and prices are pinned below, but the registry remains provisional-blocked because public metadata does not prove authenticated exact-route access or provider privacy and retention behavior.", + "selection_policy": "Ten-model pre-data cohort originally selected from the 2026-08-03 public OpenRouter catalog and refreshed on 2026-08-04 before any smoke or panel evidence existed. The refresh replaces Qwen 3.7 Plus with Qwen 3.8 Max and substitutes the same-model MiniMax M3 and DeepSeek V4 Flash 0731 slots from minimax/fp8 and deepseek/fp8 onto deepinfra/fp8 and cloudflare/fp8 under docs/ROUTE_SUBSTITUTION_POLICY.md. Cohort size, Holm family, and the 16x1 allocation are unchanged. Exact public endpoint metadata and undiscounted list prices are pinned below, but the registry remains route-preflight-ready rather than frozen because authenticated exact-route access, parameter behavior, and provider privacy and retention acceptance remain unresolved.", "models": [ { "id": "openrouter-gpt-5.6-luna-openai", @@ -32,7 +32,7 @@ "endpoint_tag": "openai", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.31771054990628, + "catalog_uptime_last_30m": 99.62251080512623, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -79,7 +79,7 @@ "endpoint_tag": "amazon-bedrock/global", "endpoint_name": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.56639566395664, + "catalog_uptime_last_30m": 99.62314342717801, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -126,7 +126,7 @@ "endpoint_tag": "google-ai-studio", "endpoint_name": "Google AI Studio | google/gemini-3.6-flash-20260721", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.78925184404636, + "catalog_uptime_last_30m": 97.20506031185643, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -172,7 +172,7 @@ "endpoint_tag": "xai/zdr", "endpoint_name": "xAI | x-ai/grok-4.5-20260708", "catalog_route_status": 0, - "catalog_uptime_last_30m": 100.0, + "catalog_uptime_last_30m": 99.97348886532343, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -222,7 +222,7 @@ "endpoint_tag": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.70362506358791, + "catalog_uptime_last_30m": 98.38854073410921, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -260,27 +260,35 @@ "role": "Z.ai open-weight anchor on the previously smoke-validated Novita FP8 route" }, { - "id": "openrouter-minimax-m3-minimax", + "id": "openrouter-minimax-m3-deepinfra", "provider": "openrouter", "model": "minimax/minimax-m3", "canonical_slug": "minimax/minimax-m3-20260531", "transport": "gateway-api", "cohort": "open-weight", - "upstream_provider": "Minimax", - "upstream_provider_slug": "minimax/fp8", - "endpoint_tag": "minimax/fp8", - "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "upstream_provider": "DeepInfra", + "upstream_provider_slug": "deepinfra/fp8", + "endpoint_tag": "deepinfra/fp8", + "endpoint_name": "DeepInfra | minimax/minimax-m3-20260531", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.45594322885867, + "catalog_uptime_last_30m": 99.82795698924731, "catalog_supported_parameters": [ "reasoning", "include_reasoning", "max_tokens", "temperature", "top_p", + "stop", + "frequency_penalty", + "presence_penalty", + "repetition_penalty", + "top_k", + "seed", + "min_p", "response_format", - "tool_choice", - "tools" + "logit_bias", + "tools", + "tool_choice" ], "reasoning_policy": "disabled", "reasoning_effort": null, @@ -293,21 +301,21 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "MiniMax open-weight anchor on the first-party FP8 route" + "role": "MiniMax open-weight anchor, substituted from the first-party `minimax/fp8` route onto `deepinfra/fp8` on 2026-08-04 after `minimax/fp8` was deranked to status -2 (30m uptime 94.59%). Same FP8 quantization and identical published rates; highest 24h uptime (99.63%) among eligible FP8 routes. Per docs/ROUTE_SUBSTITUTION_POLICY.md." }, { - "id": "openrouter-qwen3.7-plus-alibaba", + "id": "openrouter-qwen3.8-max-alibaba", "provider": "openrouter", - "model": "qwen/qwen3.7-plus", - "canonical_slug": "qwen/qwen3.7-plus-20260602", + "model": "qwen/qwen3.8-max", + "canonical_slug": "qwen/qwen3.8-max-20260803", "transport": "gateway-api", "cohort": "open-weight", "upstream_provider": "Alibaba", - "upstream_provider_slug": "alibaba/fp8", - "endpoint_tag": "alibaba/fp8", - "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", + "upstream_provider_slug": "alibaba", + "endpoint_tag": "alibaba", + "endpoint_name": "Alibaba | qwen/qwen3.8-max-20260803", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.99335742374322, + "catalog_uptime_last_30m": 100, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -317,11 +325,15 @@ "seed", "presence_penalty", "response_format", - "logprobs", - "top_logprobs", "tools", "tool_choice", - "structured_outputs" + "structured_outputs", + "logprobs", + "top_logprobs", + "top_k", + "frequency_penalty", + "stop", + "reasoning_effort" ], "reasoning_policy": "disabled", "reasoning_effort": null, @@ -335,7 +347,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "Qwen open-weight frontier anchor" + "role": "Qwen frontier anchor on the first-party Alibaba route" }, { "id": "openrouter-mistral-medium-3.5-mistral", @@ -349,7 +361,7 @@ "endpoint_tag": "mistral", "endpoint_name": "Mistral | mistralai/mistral-medium-3.5-20260430", "catalog_route_status": 0, - "catalog_uptime_last_30m": 100.0, + "catalog_uptime_last_30m": 100, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -385,32 +397,38 @@ "role": "Mistral European open-weight anchor" }, { - "id": "openrouter-deepseek-v4-flash-0731-deepseek", + "id": "openrouter-deepseek-v4-flash-0731-cloudflare", "provider": "openrouter", "model": "deepseek/deepseek-v4-flash-0731", "canonical_slug": "deepseek/deepseek-v4-flash-20260731", "transport": "gateway-api", "cohort": "open-weight", - "upstream_provider": "DeepSeek", - "upstream_provider_slug": "deepseek/fp8", - "endpoint_tag": "deepseek/fp8", - "endpoint_name": "DeepSeek | deepseek/deepseek-v4-flash-20260731", + "upstream_provider": "Cloudflare", + "upstream_provider_slug": "cloudflare/fp8", + "endpoint_tag": "cloudflare/fp8", + "endpoint_name": "Cloudflare | deepseek/deepseek-v4-flash-20260731", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.99164403593065, + "catalog_uptime_last_30m": 99.28610653487095, "catalog_supported_parameters": [ "reasoning", "include_reasoning", "max_tokens", "temperature", "top_p", - "stop", + "top_k", + "seed", + "repetition_penalty", "frequency_penalty", "presence_penalty", - "logprobs", - "top_logprobs", + "min_p", + "stop", + "logit_bias", + "response_format", + "structured_outputs", "tools", "tool_choice", - "response_format", + "logprobs", + "top_logprobs", "reasoning_effort" ], "reasoning_policy": "disabled", @@ -431,7 +449,7 @@ "absent_options": [ "OPENROUTER_REASONING_EFFORT" ], - "role": "DeepSeek open-weight anchor on the first-party FP8 route" + "role": "DeepSeek open-weight anchor, substituted from the first-party `deepseek/fp8` route onto `cloudflare/fp8` on 2026-08-04 after `deepseek/fp8` was deranked to status -5 (30m uptime 78.93%). Same FP8 quantization and identical published rates; selected on 24h uptime (99.75%, best of sixteen endpoints), not spot health. Per docs/ROUTE_SUBSTITUTION_POLICY.md." }, { "id": "openrouter-hy3-tencent", @@ -445,7 +463,7 @@ "endpoint_tag": "tencent/fp8", "endpoint_name": "Tencent | tencent/hy3-20260706", "catalog_route_status": 0, - "catalog_uptime_last_30m": 99.88329118848473, + "catalog_uptime_last_30m": 99.80510276399717, "catalog_supported_parameters": [ "reasoning", "include_reasoning", @@ -563,7 +581,7 @@ "evidence_sha256": null } }, - "openrouter-minimax-m3-minimax": { + "openrouter-minimax-m3-deepinfra": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -579,7 +597,7 @@ "evidence_sha256": null } }, - "openrouter-qwen3.7-plus-alibaba": { + "openrouter-qwen3.8-max-alibaba": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -611,7 +629,7 @@ "evidence_sha256": null } }, - "openrouter-deepseek-v4-flash-0731-deepseek": { + "openrouter-deepseek-v4-flash-0731-cloudflare": { "route_identity_sha256": null, "authenticated": false, "verified_at_utc": null, @@ -651,10 +669,10 @@ "openrouter-gemini-3.6-flash-google-ai-studio", "openrouter-grok-4.5-xai", "openrouter-glm-5.2-novita", - "openrouter-minimax-m3-minimax", - "openrouter-qwen3.7-plus-alibaba", + "openrouter-minimax-m3-deepinfra", + "openrouter-qwen3.8-max-alibaba", "openrouter-mistral-medium-3.5-mistral", - "openrouter-deepseek-v4-flash-0731-deepseek", + "openrouter-deepseek-v4-flash-0731-cloudflare", "openrouter-hy3-tencent" ], "shared_fixed_options": { diff --git a/config/sota_v3_pricing_snapshot.json b/config/sota_v3_pricing_snapshot.json index 4577ec9..592baa4 100644 --- a/config/sota_v3_pricing_snapshot.json +++ b/config/sota_v3_pricing_snapshot.json @@ -3,7 +3,7 @@ "contract": "sota-v3", "contract_fingerprint": "a523bdfcebe47bbd", "status": "catalog-frozen-public-metadata-only", - "checked_at_utc": "2026-08-03T15:53:59Z", + "checked_at_utc": "2026-08-04T16:00:40Z", "source": "Unauthenticated HTTP GET of https://openrouter.ai/api/v1/models and each selected model's https://openrouter.ai/api/v1/models/{model_id}/endpoints response; no completion or chat endpoint was called.", "currency": "USD", "rates_are_per_token": true, @@ -29,8 +29,8 @@ "completion": 1e-05 }, "deepseek/deepseek-v4-flash-0731": { - "provider_slug": "deepseek/fp8", - "endpoint_name": "DeepSeek | deepseek/deepseek-v4-flash-20260731", + "provider_slug": "cloudflare/fp8", + "endpoint_name": "Cloudflare | deepseek/deepseek-v4-flash-20260731", "prompt": 1.4e-07, "completion": 2.8e-07 }, @@ -42,8 +42,8 @@ "internal_reasoning": 7.5e-06 }, "minimax/minimax-m3": { - "provider_slug": "minimax/fp8", - "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "provider_slug": "deepinfra/fp8", + "endpoint_name": "DeepInfra | minimax/minimax-m3-20260531", "prompt": 3e-07, "completion": 1.2e-06 }, @@ -56,24 +56,19 @@ "openai/gpt-5.6-luna": { "provider_slug": "openai", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", - "prompt": 1e-07, - "completion": 6e-07, + "prompt": 2e-07, + "completion": 1.2e-06, "long_context_override": { "min_prompt_tokens": 272000, "prompt": 2e-07, "completion": 9e-07 } }, - "qwen/qwen3.7-plus": { - "provider_slug": "alibaba/fp8", - "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", - "prompt": 3.2e-07, - "completion": 1.28e-06, - "long_context_override": { - "min_prompt_tokens": 256000, - "prompt": 9.6e-07, - "completion": 3.84e-06 - } + "qwen/qwen3.8-max": { + "provider_slug": "alibaba", + "endpoint_name": "Alibaba | qwen/qwen3.8-max-20260803", + "prompt": 2e-06, + "completion": 6e-06 }, "tencent/hy3": { "provider_slug": "tencent/fp8", @@ -95,8 +90,8 @@ "z-ai/glm-5.2": { "provider_slug": "novita/fp8", "endpoint_name": "Novita | z-ai/glm-5.2-20260616", - "prompt": 7.266e-07, - "completion": 2.2836e-06 + "prompt": 1.4e-06, + "completion": 4.4e-06 } }, "public_metadata_limitations": [ @@ -108,5 +103,6 @@ "route_preflight_authorized": false, "smoke_execution_authorized": false, "panel_execution_authorized": false, - "publication_authorized": false + "publication_authorized": false, + "pricing_basis": "Undiscounted list rates for the exact pinned route. Promotional discounts are deliberately not reserved against: the GLM 5.2 Novita discount moved from 55.1% to 50% within hours on 2026-08-04, and a reservation computed from a promo is wrong the moment the promo ends. A live discount only ever brings the run in under reserve." } diff --git a/config/sota_v3_publication_protocol.json b/config/sota_v3_publication_protocol.json index ad32fd2..483c493 100644 --- a/config/sota_v3_publication_protocol.json +++ b/config/sota_v3_publication_protocol.json @@ -84,7 +84,8 @@ "cost_estimate_artifact": "results/analysis/sota-v3-pre-smoke-cost-estimate.json", "operator_must_pass_max_spend_usd": true, "spend_authorized": false, - "operator_ceiling_usd": null + "operator_ceiling_usd": 150.0, + "operator_ceiling_basis": "Owner-set hard cap, raised 120.00 -> 150.00 on 2026-08-04. The $120 figure was chosen against a $119.76 reservation that turned out to depend on 50%-off promotional rates on openai/gpt-5.6-luna and z-ai/glm-5.2; pinning undiscounted list rates moved the reservation to $127.29 and put the committed plan over its own ceiling. $150.00 clears the current reservation with headroom for a further route substitution or list-price move without another ceiling decision. Enforced by the runner ahead of the cell loop: --max-spend-usd above this value is rejected before any endpoint probe or child process. Projected actual spend is ~$35-45 from July smoke telemetry reprojected at current rates (models emit 48-640 output tokens per decision against a 4,096-token reservation), so this is a backstop, not a forecast. Raising it again is a deliberate edit here." }, "publication_authorized": false } diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 81f7bdc..4e6ff52 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -860,6 +860,9 @@ decision and why. | Date | Decision | Evidence / rationale | Effect | | --- | --- | --- | --- | +| 2026-08-04 | Adopt `qwen/qwen3.8-max` in place of `qwen/qwen3.7-plus`, and adopt a written [route substitution policy](ROUTE_SUBSTITUTION_POLICY.md). | Qwen 3.8 Max shipped 2026-08-03 and the benchmark is intended to track new releases rather than freeze once. Three substitutions in two days had each been decided ad hoc, which is how a benchmark quietly starts measuring whichever host is cheapest this week. | Cohort stays at ten, so the Holm family, the 16x1 allocation, and the power selection are untouched. The Max tier bills 6.25x/4.69x the Plus tier per token. The policy fixes eligibility, forbids price/throughput/first-party status as substitution criteria, and requires re-establishing route and privacy acceptance for any new counterparty. | +| 2026-08-04 | Substitute `deepseek/fp8` -> `cloudflare/fp8` and `minimax/fp8` -> `deepinfra/fp8`. | Both first-party routes were deranked the same day (status `-5` at 78% availability, and `-2`). Both replacements are the same model at the same FP8 quantization, chosen on highest 24h availability, at identical published rates. | No cost or cohort-size effect. The DeepSeek route recovered on its own within the day, so in hindsight that substitution was not strictly necessary; it is kept because Cloudflare holds the better 24h record. Route and privacy acceptance do **not** carry over — Cloudflare and DeepInfra are counterparties this project has never reviewed. | +| 2026-08-04 | Pin undiscounted list rates instead of promotional rates, and enforce `operator_ceiling_usd` in the runner. | `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both pinned at 50%-off promos; the GLM discount moved 55.1% -> 50% within hours of being recorded. A reservation computed from a promo is wrong the moment it ends. Separately, `operator_ceiling_usd` had sat in config unread, so nothing enforced the committed cap. | Reservation $119.76 -> **$127.29**, which exceeded the then-committed $120.00 ceiling — the plan only ever appeared to fit because of the discounts. **Ceiling raised to $150.00 on 2026-08-04** to clear the reservation with headroom for a further substitution or list-price move. The reservation and the ceiling are now compared by `test_the_committed_plan_fits_under_the_committed_ceiling`, and the committed cost artifact is checked against the configs it claims to describe, so neither can drift silently again. Projected actual spend is ~$35-45 from July smoke telemetry, so the ceiling is a backstop rather than the expected bill. Logged in [`docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md`](run_logs/sota-v3-lineup-refresh-2026-08-04.md). | | 2026-07-13 | Treat v1 model rows as archived historical evidence, not a current ranking. | Scout-key mismatch affected models unevenly; failed queries were invisible. | Current claims require `sota-v2`; v1 remains auditable under `sota-v1`. | | 2026-07-13 | Separate API and coding-harness lanes. | Archived rows mixed provider API behavior with uncontrolled CLI harness context and very different output usage. | API becomes the headline lane; CLI harnesses remain diagnostic. | | 2026-07-13 | Withhold the v2 ranking pending an output-budget sweep. | Archived scores tracked output allowance strongly enough to confound model comparison. | Run the planned cap matrix and freeze a compute policy before the full panel. | diff --git a/docs/ROUTE_SUBSTITUTION_POLICY.md b/docs/ROUTE_SUBSTITUTION_POLICY.md new file mode 100644 index 0000000..1fd5835 --- /dev/null +++ b/docs/ROUTE_SUBSTITUTION_POLICY.md @@ -0,0 +1,117 @@ +# Route substitution policy + +When a pinned OpenRouter route stops being eligible, this is how it gets +replaced. The point is to make the decision mechanical, so that a route failing +at an inconvenient moment does not turn into an improvised choice about what the +benchmark measures. + +Adopted 2026-08-04, after three substitutions in two days were each decided ad +hoc. + +## Why this exists + +Route health is volatile on a timescale shorter than a panel run. Measured over +roughly twelve hours on 2026-08-04: + +| Route | Observed | +| --- | --- | +| `deepseek/deepseek-v4-flash-0731` @ `deepseek/fp8` | status 0, best throughput in the cohort → deranked to `-5`, serving 78% of requests → recovered to status 0 | +| `minimax/minimax-m3` @ `minimax/fp8` | status 0 → deranked to `-2` | +| `qwen/qwen3.7-plus` @ `alibaba` | endpoint tag silently moved to `alibaba/fp8` | +| `z-ai/glm-5.2` @ `novita/fp8` | discount moved 55.1% → 50% | + +None of these were announced. All were found by running the free probe. + +## What makes a route ineligible + +A pinned route is ineligible if **any** of the following holds. These are all +enforced by `_endpoint_issues` in `scripts/run_publication_matrix.py`; the free +zero-call probe reports every failing route in one pass. + +1. No endpoint matches the pinned `provider_name`, `tag`, and `name`. +2. Endpoint `status` is not `0`. +3. The endpoint does not advertise `max_tokens`, `reasoning`, and — when the + lane sets `OPENROUTER_JSON_MODE=true` — `response_format`. +4. `max_completion_tokens` is missing, non-integer, or below the registered + output cap. +5. `uptime_last_30m` is missing, non-numeric, non-finite, or below **90%**. +6. `uptime_last_1d` is missing, non-numeric, non-finite, or below **95%**. +7. The live base rate for the pinned route is **above** the committed pricing + snapshot. A rate below the snapshot is reported, not blocked. + +On (5) and (6): both windows gate because they detect different failures. The +24h figure cannot see an outage in progress — the deranked DeepSeek route still +read 99.24% over 24h while serving 78% of requests. Both floors sit well below +the observed noise band, because these readings drift about half a point between +consecutive polls; a 99% 24h floor was measured rejecting two healthy cohort +members while still passing the route that had actually failed. An absent or +malformed figure is unknown health, not evidence that the route clears the +floor, so it blocks the same way unverifiable pricing blocks. + +## Choosing the replacement + +**Prefer waiting.** Deranking is frequently transient — the first-party DeepSeek +route recovered on its own within a day. If the paid phase is not imminent, +re-probe before substituting. + +Where a substitution is required, take the eligible endpoint for the **same +model** with the **highest `uptime_last_1d`**, subject to: + +- **Same quantization** as the outgoing route. A different precision is a + different numerical model, not a different host, and is out of scope for a + substitution — treat it as a cohort amendment. +- All eligibility criteria above. +- `max_completion_tokens` at or above the registered cap. + +Deliberately *not* criteria: price, throughput, latency, and first-party status. + +- **Price** must not steer the choice; that is how a benchmark quietly starts + measuring whichever host is cheapest this week. Record the new rate and let + the reservation move. +- **Throughput and latency** are the worst possible signals here. The DeepSeek + route had the cohort's best throughput hours before it failed. +- **First-party status** is not load-bearing. It was framed as one in the + original route roles, but on the axis that actually matters mid-panel — + staying up — a model's author has no particular advantage. Cloudflare beat + DeepSeek on its own model. + +## Recording it + +A substitution is a cohort **route** change, not a cohort **membership** change, +so it does not touch the Holm family size, the allocation, or the power +selection. It does require, in the same change: + +1. `config/sota_v3_models.json` — `id`, `upstream_provider`, + `upstream_provider_slug`, `endpoint_tag`, `endpoint_name`, + `catalog_route_status`, `catalog_uptime_last_30m`, + `catalog_supported_parameters`, and a `role` naming the date, the outgoing + route, and the observed reason. +2. The same `id` in `required_smokes` and in `exact_route_acceptance.entries`. +3. `config/sota_v3_pricing_snapshot.json` — `provider_slug`, `endpoint_name`, + and the **undiscounted list rates** (see below). +4. Regenerate `results/analysis/sota-v3-pre-smoke-cost-estimate.json` and check + the result against `budget_policy.operator_ceiling_usd`. +5. A dated entry in the decision log, and a run log if a probe was run. + +Substituting a host does **not** carry over route acceptance. `authenticated`, +`verified_at_utc`, `route_evidence_sha256`, and the whole `privacy_acceptance` +block describe a specific provider and must be re-established for the new one. + +## Pricing + +Pin the **undiscounted list rate**, never a promotional rate. + +The reservation is a safe upper bound, and a promo is not a floor. On +2026-08-04 the GLM 5.2 Novita discount moved from 55.1% to 50% within hours, +and `openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both found pinned at 50%-off +prices — which is the only reason the panel had appeared to fit under the +$120 ceiling in force at the time. Where the endpoint reports a `discount`, record +`live_rate / (1 - discount)`. + +A live discount then only ever brings the run in under reserve, which is the +direction that cannot hurt. + +## When to re-probe + +Immediately before any paid phase, every time. A passing probe has a shelf life +measured in hours, and a stale pass is not a pass. diff --git a/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md new file mode 100644 index 0000000..e7d6cee --- /dev/null +++ b/docs/run_logs/sota-v3-lineup-refresh-2026-08-04.md @@ -0,0 +1,119 @@ +# sota-v3 lineup refresh — 2026-08-04 + +Supersedes the route state recorded in +[`sota-v3-route-preflight-2026-08-03.md`](sota-v3-route-preflight-2026-08-03.md), +which is left unedited as the historical record of that day's probe. Two of the +ten routes it describes are no longer in the lineup, and its closing claim that +all ten pass was true when written and is not a standing claim. + +**Zero completion calls, zero spend.** No gate was granted or changed; every +paid authorization remains `false`. + +## Cohort changes + +Cohort size stays at **ten**, so the Holm family size, the 16x1 allocation, and +the power selection are untouched. + +| Slot | Was | Now | Reason | +| --- | --- | --- | --- | +| Qwen | `qwen/qwen3.7-plus` @ `alibaba/fp8` | `qwen/qwen3.8-max` @ `alibaba` | Released 2026-08-03; owner elected to adopt immediately | +| DeepSeek | `deepseek/deepseek-v4-flash-0731` @ `deepseek/fp8` | same model @ `cloudflare/fp8` | First-party route deranked to status `-5` | +| MiniMax | `minimax/minimax-m3` @ `minimax/fp8` | same model @ `deepinfra/fp8` | First-party route deranked to status `-2` | + +Both substitutions follow +[`docs/ROUTE_SUBSTITUTION_POLICY.md`](../ROUTE_SUBSTITUTION_POLICY.md): same +model, same FP8 quantization, highest 24h availability among eligible routes, +identical published rates. + +The Qwen change is a **cohort amendment**, not a substitution — a different +model, adopted deliberately. It replaces Alibaba's cost-effective tier with +their flagship, which bills 6.25x/4.69x per token. + +## How volatile the routes actually are + +Everything below was observed within roughly twelve hours. None was announced. + +| Route | Observed | +| --- | --- | +| `deepseek/fp8` | status 0 with the cohort's best throughput → deranked to `-5`, 30m availability 78% → recovered to status 0 the same day | +| `minimax/fp8` | status 0 → deranked to `-2` | +| `alibaba` (qwen3.7-plus) | endpoint tag silently moved to `alibaba/fp8` | +| `novita/fp8` (glm-5.2) | discount moved 55.1% → 50% | + +The DeepSeek recovery is worth stating plainly: the substitution was not +strictly necessary in hindsight. It is kept because Cloudflare holds the better +24h record, but the honest reading is that the route was transiently deranked, +not lost. + +## Pricing: promotional rates are no longer reserved against + +`openai/gpt-5.6-luna` and `z-ai/glm-5.2` were both found pinned at 50%-off +promotional rates. The GLM discount moved 55.1% → 50% within hours of being +recorded, which is what surfaced the problem. + +The snapshot now pins **undiscounted list rates** for every route. A promo is +not a floor, and a reservation computed from one is wrong the moment it ends. + +| Route | Was pinned (promo) | Now pinned (list) | +| --- | --- | --- | +| `openai/gpt-5.6-luna` | 1e-07 / 6e-07 | 2e-07 / 1.2e-06 | +| `z-ai/glm-5.2` | 7e-07 / 2.2e-06 | 1.4e-06 / 4.4e-06 | + +## Reservation + +| | Amount | +| --- | --- | +| Previous reservation | $107.81 | +| After the Qwen amendment | $119.76 | +| **After pinning list rates** | **$127.29** | +| Committed operator ceiling | **$150.00** (raised from $120.00, 2026-08-04) | + +Call count is unchanged at 3,240. + +The $120 ceiling was chosen against a $119.76 reservation that turned out to +depend on two 50%-off promotional rates. Pinning list rates pushed the +committed plan over its own committed ceiling — a state nothing detected, +because the reservation and the ceiling lived in different files and nothing +compared them. `test_the_committed_plan_fits_under_the_committed_ceiling` now +does, at zero cost, so adding a model or losing a discount fails a test rather +than surfacing when someone tries to authorize a run. + +The ceiling was raised to **$150.00** to clear the current reservation with +headroom for a further substitution or list-price move. + +Reserve is concentrated in four models: Gemini 3.6 Flash (22.4%), Grok 4.5 +(19.9%), Claude Sonnet 5 (17.4%), and Mistral Medium 3.5 (13.0%). + +### Projected actual spend is far below either figure + +From July smoke telemetry reprojected at current rates, models emit **48–640 +output tokens per decision** against a 4,096-token reservation, and Grok's +observed internal reasoning was **516 tokens** against 4,096 reserved. + +Projected actual spend is roughly **$35–45**. The reservation is a backstop, not +a forecast, and the gap is concentrated in the reserved reasoning headroom for +Gemini and Grok. + +## Runner changes made alongside + +- The zero-call probe now checks **every** route and reports the full set of + failures. Exiting on the first one had twice presented a partial picture as a + complete one. Paid phases still abort on the first bad route. +- Endpoint eligibility now enforces availability floors on two windows, 90% + (30m) and 95% (24h). +- Live rates are compared against the committed snapshot on every probe. + An increase fails the route; a decrease is reported and allowed. +- `budget_policy.operator_ceiling_usd` is now enforced ahead of the cell loop. + It had been declared in config and read by nothing. +- An autouse test fixture blocks the suite from inheriting a live provider + credential, after a test spent $0.436198 across 38 live calls. + +## What this does not establish + +Unchanged: `spend_authorized`, `smoke_execution_authorized`, +`panel_execution_authorized`, `publication_authorized` all `false`; registry +`route-preflight-ready`, not `frozen`; `exact_route_acceptance` `unresolved`. + +Route acceptance does **not** carry across a substitution. Cloudflare and +DeepInfra are new counterparties whose data-handling terms have never been +reviewed for this project. diff --git a/results/analysis/sota-v3-pre-smoke-cost-estimate.json b/results/analysis/sota-v3-pre-smoke-cost-estimate.json index fe7c1a9..a8ffdfa 100644 --- a/results/analysis/sota-v3-pre-smoke-cost-estimate.json +++ b/results/analysis/sota-v3-pre-smoke-cost-estimate.json @@ -26,24 +26,24 @@ "total_calls": 3240 }, "costs_usd": { - "panel": 88.735895552, - "smoke": 1.1091986944, - "total_unrounded": 89.8450942464, - "total_with_1_2x_contingency": 107.81411309568 + "panel": 104.76363776, + "smoke": 1.309545472, + "total_unrounded": 106.073183232, + "total_with_1_2x_contingency": 127.2878198784 }, "models": [ { - "applied_completion_rate_usd": 6e-07, + "applied_completion_rate_usd": 1.2e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 1e-07, - "cost_per_decision_usd": 0.0032576, + "applied_prompt_rate_usd": 2e-07, + "cost_per_decision_usd": 0.0065152, "experiment_id": "openrouter-gpt-5.6-luna-openai", "internal_reasoning_tokens_per_decision": 0, "model": "openai/gpt-5.6-luna", "panel_calls": 320, - "panel_cost_usd": 1.042432, + "panel_cost_usd": 2.084864, "smoke_calls": 4, - "smoke_cost_usd": 0.0130304 + "smoke_cost_usd": 0.0260608 }, { "applied_completion_rate_usd": 1e-05, @@ -87,24 +87,24 @@ "smoke_cost_usd": 0.260608 }, { - "applied_completion_rate_usd": 2.2836e-06, + "applied_completion_rate_usd": 4.4e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 7.266e-07, - "cost_per_decision_usd": 0.0151664256, + "applied_prompt_rate_usd": 1.4e-06, + "cost_per_decision_usd": 0.0292224, "experiment_id": "openrouter-glm-5.2-novita", "internal_reasoning_tokens_per_decision": 0, "model": "z-ai/glm-5.2", "panel_calls": 320, - "panel_cost_usd": 4.853256192, + "panel_cost_usd": 9.351168, "smoke_calls": 4, - "smoke_cost_usd": 0.0606657024 + "smoke_cost_usd": 0.1168896 }, { "applied_completion_rate_usd": 1.2e-06, "applied_internal_reasoning_rate_usd": 0.0, "applied_prompt_rate_usd": 3e-07, "cost_per_decision_usd": 0.0073152, - "experiment_id": "openrouter-minimax-m3-minimax", + "experiment_id": "openrouter-minimax-m3-deepinfra", "internal_reasoning_tokens_per_decision": 0, "model": "minimax/minimax-m3", "panel_calls": 320, @@ -113,17 +113,17 @@ "smoke_cost_usd": 0.0292608 }, { - "applied_completion_rate_usd": 1.28e-06, + "applied_completion_rate_usd": 6e-06, "applied_internal_reasoning_rate_usd": 0.0, - "applied_prompt_rate_usd": 3.2e-07, - "cost_per_decision_usd": 0.00780288, - "experiment_id": "openrouter-qwen3.7-plus-alibaba", + "applied_prompt_rate_usd": 2e-06, + "cost_per_decision_usd": 0.040576, + "experiment_id": "openrouter-qwen3.8-max-alibaba", "internal_reasoning_tokens_per_decision": 0, - "model": "qwen/qwen3.7-plus", + "model": "qwen/qwen3.8-max", "panel_calls": 320, - "panel_cost_usd": 2.4969216, + "panel_cost_usd": 12.98432, "smoke_calls": 4, - "smoke_cost_usd": 0.03121152 + "smoke_cost_usd": 0.162304 }, { "applied_completion_rate_usd": 7.5e-06, @@ -143,7 +143,7 @@ "applied_internal_reasoning_rate_usd": 0.0, "applied_prompt_rate_usd": 1.4e-07, "cost_per_decision_usd": 0.00226688, - "experiment_id": "openrouter-deepseek-v4-flash-0731-deepseek", + "experiment_id": "openrouter-deepseek-v4-flash-0731-cloudflare", "internal_reasoning_tokens_per_decision": 0, "model": "deepseek/deepseek-v4-flash-0731", "panel_calls": 320, @@ -165,7 +165,7 @@ "smoke_cost_usd": 0.012874752 } ], - "pricing_checked_at_utc": "2026-08-03T15:53:59Z", + "pricing_checked_at_utc": "2026-08-04T16:00:40Z", "runtime": { "note": "Regenerate this artifact from accepted smoke telemetry before approving the full panel; latency is reported only for models with committed observations.", "observation_source": null, diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index d58e86b..f8f09a6 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -12,6 +12,7 @@ import argparse import hashlib import json +import math import os import subprocess import sys @@ -31,6 +32,21 @@ PROTOCOL_CONFIG = ROOT / "config" / "publication_protocol.json" SMOKE_MANIFEST = ROOT / "config" / "sota_v2_smoke_manifest.json" RUN_STATE_FORMAT = "gm-bench-publication-run-v1" +# Availability floors for a pinned endpoint to stay eligible. +# +# Two windows, because they detect different failures. The 24h figure is a +# chronic filter and is far too slow to see an outage: on 2026-08-04 the +# first-party DeepSeek route was deranked to status -5 while still reporting +# 99.24% over 24h. The 30m figure is what moved (78.93%), so that is the +# acute gate. +# +# Both floors sit well below the noise band. These readings drift by half a +# point between consecutive polls, so a threshold set near the observed values +# would block healthy routes at random -- a 99% 24h floor rejected two +# perfectly healthy cohort members on the day it was written while still +# passing the route that had actually failed. +MIN_UPTIME_LAST_30M_PCT = 90.0 +MIN_UPTIME_LAST_1D_PCT = 95.0 CONTRACT_CONFIGS = { "sota-v2": ( ROOT / "config" / "sota_v2_models.json", @@ -460,6 +476,34 @@ def _openrouter_usage_usd(env: dict[str, str]) -> float: return float(payload["data"]["total_usage"]) +def _enforce_operator_ceiling(max_spend_usd: float, contract: str | None) -> None: + """Reject a `--max-spend-usd` above the contract's committed hard cap. + + `budget_policy.operator_ceiling_usd` was declared but never read, so the + only thing standing between a typo and an unbounded run was the operator + retyping the right number. A committed ceiling that nothing enforces is a + comment. A null ceiling stays permissive: contracts that have not + committed to a number are not silently given one. + """ + _, _, _, protocol_path, _ = CONTRACT_CONFIGS.get(contract or "", (None,) * 5) + if protocol_path is None: + protocol_path = PROTOCOL_CONFIG + try: + budget_policy = (_read_json(protocol_path) or {}).get("budget_policy") or {} + except (OSError, ValueError, json.JSONDecodeError): + return + ceiling = budget_policy.get("operator_ceiling_usd") + if ceiling is None: + return + if not isinstance(ceiling, (int, float)) or isinstance(ceiling, bool) or not math.isfinite(ceiling) or ceiling <= 0: + raise ValueError(f"budget_policy.operator_ceiling_usd must be a positive finite number, got {ceiling!r}") + if max_spend_usd > ceiling: + raise ValueError( + f"--max-spend-usd ${max_spend_usd:.2f} exceeds the committed operator ceiling " + f"${float(ceiling):.2f} in {protocol_path.name}; raise the ceiling deliberately or lower the run" + ) + + def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: endpoints = (payload.get("data") or {}).get("endpoints") or [] expected_provider = cell.upstream_provider @@ -485,11 +529,40 @@ def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: for endpoint in matches: supported = set(endpoint.get("supported_parameters") or []) maximum = endpoint.get("max_completion_tokens") - cap_fits = cell.cap is None or maximum is None or (isinstance(maximum, int) and cell.cap <= maximum) + cap_fits = cell.cap is None or ( + isinstance(maximum, int) and not isinstance(maximum, bool) and cell.cap <= maximum + ) if required <= supported and cap_fits: capable.append(endpoint) if not capable: return [f"matching endpoint cannot honor required parameters {sorted(required)!r} and cap={cell.cap_label}"] + floors = (("uptime_last_30m", MIN_UPTIME_LAST_30M_PCT, "30m"), ("uptime_last_1d", MIN_UPTIME_LAST_1D_PCT, "24h")) + for field, floor, label in floors: + # Fail closed. Missing or malformed health telemetry does not establish + # that the pinned route clears the declared availability floor. + durable = [ + endpoint + for endpoint in capable + if isinstance(endpoint.get(field), (int, float)) + and not isinstance(endpoint[field], bool) + and math.isfinite(endpoint[field]) + and endpoint[field] >= floor + ] + if not durable: + observed = [ + endpoint[field] + for endpoint in capable + if isinstance(endpoint.get(field), (int, float)) + and not isinstance(endpoint[field], bool) + and math.isfinite(endpoint[field]) + ] + if not observed: + return [f"matching endpoint has no finite numeric {label} uptime telemetry"] + return [ + f"matching endpoint is below the {floor}% {label} uptime floor " + f"(best matching route: {max(observed):.2f}%)" + ] + capable = durable return [] @@ -506,8 +579,79 @@ def _openrouter_endpoints(model: str, env: dict[str, str]) -> dict[str, Any]: return json.load(response) +def _pricing_drift_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: + """Compare the pinned route's live rates against the committed snapshot. + + The snapshot is what the reservation was computed from, so a rate that has + risen since it was taken makes the committed budget wrong in the direction + that costs money. That is an error. A rate that has *fallen* only means the + run will come in under reserve, so it is reported and allowed -- the GLM + Novita route quietly picked up a 55.1% discount that went unnoticed for two + weeks precisely because nothing ever looked. + + Only the base rates are compared. Long-context override tiers are priced + separately in the snapshot and are not exercised by the registered + 8,000-token decision, so a drift there cannot move this plan's cost. + """ + try: + rates = (_read_json(PRICING_CONFIG).get("models") or {}).get(cell.model) + except (OSError, ValueError, json.JSONDecodeError) as exc: + return [f"committed pricing snapshot for {cell.experiment_id} could not be read: {exc}"] + if not isinstance(rates, dict): + return [f"committed pricing snapshot has no rates for {cell.model}"] + # Match the full registered route identity, exactly as `_endpoint_issues` + # does. Comparing a price against a route we did not pin is worse than not + # comparing at all. + endpoint = next( + ( + e + for e in ((payload.get("data") or {}).get("endpoints") or []) + if e.get("provider_name") == cell.upstream_provider + and e.get("tag") == cell.endpoint_tag + and e.get("name") == cell.endpoint_name + ), + None, + ) + if endpoint is None: + return [f"no endpoint matching the pinned route identity for {cell.experiment_id} to price-check"] + issues = [] + for field in ("prompt", "completion"): + committed = rates.get(field) + raw = (endpoint.get("pricing") or {}).get(field) + # Fail closed. "The price could not be verified" is not the same as + # "the price is unchanged", and only one of them is safe to spend on. + if ( + not isinstance(committed, (int, float)) + or isinstance(committed, bool) + or not math.isfinite(committed) + or committed < 0 + ): + issues.append(f"committed {field} rate for {cell.experiment_id} is not a usable number: {committed!r}") + continue + try: + live = float(raw) + except (TypeError, ValueError): + issues.append(f"live {field} rate for {cell.experiment_id} is unreadable: {raw!r}") + continue + if not math.isfinite(live) or live < 0: + issues.append(f"live {field} rate for {cell.experiment_id} is not a usable number: {raw!r}") + continue + if live > committed: + issues.append( + f"live {field} rate {live:.10g} exceeds the committed snapshot rate {committed:.10g} " + f"for {cell.experiment_id}; the reservation was computed from the snapshot" + ) + elif live < committed: + print( + f"note: live {field} rate for {cell.experiment_id} fell from {committed:.10g} " + f"to {live:.10g}; the run will come in under its reservation" + ) + return issues + + def _validate_openrouter_endpoint(cell: Cell, env: dict[str, str]) -> None: - issues = _endpoint_issues(cell, _openrouter_endpoints(cell.model, env)) + payload = _openrouter_endpoints(cell.model, env) + issues = _endpoint_issues(cell, payload) + _pricing_drift_issues(cell, payload) if issues: raise RuntimeError("; ".join(issues)) @@ -1434,6 +1578,12 @@ def main(argv: list[str] | None = None) -> int: except (OSError, ValueError, json.JSONDecodeError) as exc: parser.error(str(exc)) return _record_smoke(args.model_id, args.artifact, manifest_path) + if args.max_spend_usd is not None and not math.isfinite(args.max_spend_usd): + # NaN defeats every downstream guard silently: `nan <= 0`, + # `nan > ceiling`, and `spent >= nan` are all false, so a NaN limit + # satisfies "the operator passed a ceiling" while bounding nothing. + # Infinity is the same hole whenever no ceiling is configured. + parser.error("--max-spend-usd must be a finite number") if args.max_spend_usd is not None and args.max_spend_usd <= 0: parser.error("--max-spend-usd must be positive") if args.phase in {"route-preflight", "smoke", "panel"}: @@ -1457,6 +1607,11 @@ def main(argv: list[str] | None = None) -> int: and args.max_spend_usd is None ): parser.error("paid OpenRouter runs require an explicit --max-spend-usd ceiling") + if args.max_spend_usd is not None: + try: + _enforce_operator_ceiling(args.max_spend_usd, args.contract) + except ValueError as exc: + parser.error(str(exc)) run_dir = args.run_dir.resolve() for directory in (run_dir / "raw", run_dir / "checkpoints"): if not args.dry_run and not args.preflight_only and args.phase != "route-preflight": @@ -1464,6 +1619,7 @@ def main(argv: list[str] | None = None) -> int: if not args.dry_run and not args.preflight_only and args.phase != "route-preflight": _write_run_state(run_dir, args.phase, cells, args.max_spend_usd) budget_start: float | None = None + preflight_failures: list[str] = [] for cell in cells: env = cell_environment(cell) command = cell_command( @@ -1513,7 +1669,16 @@ def main(argv: list[str] | None = None) -> int: KeyError, json.JSONDecodeError, ) as exc: - raise SystemExit(f"OpenRouter endpoint preflight failed for {cell.experiment_id}: {exc}") from exc + failure = f"OpenRouter endpoint preflight failed for {cell.experiment_id}: {exc}" + # A phase that spends money must stop at the first bad route. + # The zero-call phase must not: exiting early leaves every + # later route unchecked, which reads as "one route is broken" + # when the truth may be four. Collect and report them all. + if args.phase != "route-preflight": + raise SystemExit(failure) from exc + preflight_failures.append(failure) + print(failure, file=sys.stderr) + continue if args.phase == "route-preflight": print(f"zero-completion-call route preflight passed: {cell.experiment_id}") continue @@ -1569,6 +1734,11 @@ def main(argv: list[str] | None = None) -> int: print(f"measured OpenRouter spend for this run directory: ${spent:.4f}") if spent > args.max_spend_usd: raise SystemExit(f"spend ceiling exceeded after attempted cell: ${spent:.4f}") + if preflight_failures: + raise SystemExit( + f"zero-completion-call route preflight failed for " + f"{len(preflight_failures)} of {len(cells)} routes:\n " + "\n ".join(preflight_failures) + ) return 0 diff --git a/tests/conftest.py b/tests/conftest.py index ae66821..89ec216 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import pytest +import scripts.run_publication_matrix as publication_runner + @pytest.fixture(autouse=True) def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -12,3 +14,25 @@ def isolate_baseline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> N Also inherited by subprocess-based CLI tests via the environment. """ monkeypatch.setenv("GM_BENCH_BASELINE_CACHE", str(tmp_path / "baseline_cache.json")) + + +@pytest.fixture(autouse=True) +def block_real_provider_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + """Stop the test suite from ever authenticating against a paid provider. + + The publication runner calls ``load_environment_files(ROOT)`` at startup, + which reads the gitignored ``.env.local`` out of the working tree. Any test + that drives ``main()`` through a paid phase without stubbing out the child + process therefore runs the real benchmark against real routes and bills a + real account -- with no failure to signal it, because the run succeeds. + That is exactly what happened on 2026-08-04: a test written to assert that + a spend ceiling *blocks* a run instead spent $0.44 across 38 live calls, + because the fixture lane it resolved to had every gate already unlocked. + + Neutralising the loader is enough. Tests that need a credential present + still set one explicitly with ``monkeypatch.setenv``, which continues to + work; what they cannot do any more is silently inherit a live key. + """ + monkeypatch.setattr(publication_runner, "load_environment_files", lambda _root: []) + for name in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.delenv(name, raising=False) diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index c8c97dd..a2c5a70 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -46,14 +46,68 @@ def test_v3_cost_plan_uses_registered_private_seed_count() -> None: assert result["calls"]["panel_decisions_per_model"] == 320 assert result["calls"]["panel_calls"] == 3_200 assert result["calls"]["total_calls"] == 3_240 - assert result["costs_usd"]["total_unrounded"] == pytest.approx(89.8450942464) - assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(107.81411309568) + # Repriced 2026-08-04. Two moves, on an unchanged 3,240-call plan: + # the Qwen slot went from qwen3.7-plus to qwen3.8-max, which bills + # 6.25x/4.69x per token; and the snapshot now pins undiscounted list + # rates, after gpt-5.6-luna and glm-5.2 were both found pinned at + # 50%-off promotional prices. A promo is not a floor -- the GLM + # discount moved 55.1% -> 50% within hours of being recorded. + assert result["costs_usd"]["total_unrounded"] == pytest.approx(106.073183232) + assert result["costs_usd"]["total_with_1_2x_contingency"] == pytest.approx(127.2878198784) grok = next(row for row in result["models"] if row["model"] == "x-ai/grok-4.5") assert grok["internal_reasoning_tokens_per_decision"] == 4096 assert grok["applied_internal_reasoning_rate_usd"] == pytest.approx(grok["applied_completion_rate_usd"]) assert grok["internal_reasoning_billing_basis"] == "completion" +def test_the_committed_plan_fits_under_the_committed_ceiling() -> None: + """The reservation and the hard cap must not drift apart silently. + + These are two committed numbers in two different files, and on 2026-08-04 + they crossed: pinning undiscounted list rates moved the reservation to + $127.29 against a $120.00 ceiling, so the committed plan could not legally + run. Nothing caught it, because nothing compared them. + + Adding a model, substituting a route onto a pricier host, or a provider + ending a discount all move the reservation. Any of them silently breaching + the ceiling should fail here, at zero cost, rather than at the point + someone tries to authorize a run. + """ + protocol = json.loads(Path("config/sota_v3_publication_protocol.json").read_text()) + ceiling = protocol["budget_policy"]["operator_ceiling_usd"] + reserved = estimate(*_v3_inputs())["costs_usd"]["total_with_1_2x_contingency"] + + assert isinstance(ceiling, (int, float)) and ceiling > 0 + assert reserved <= ceiling, ( + f"the committed reservation ${reserved:.2f} exceeds the committed operator ceiling " + f"${ceiling:.2f}; raise the ceiling deliberately or reduce the plan" + ) + + +def test_the_committed_cost_artifact_matches_the_committed_configs() -> None: + """A stale cost artifact is a reservation nobody recomputed. + + The runner reserves per cell from the pricing snapshot, but the artifact is + what the readiness docs and the ceiling decision quote. If someone edits a + price or a route without regenerating it, the number people reason about + stops describing the plan they would actually run. + """ + committed = json.loads(Path("results/analysis/sota-v3-pre-smoke-cost-estimate.json").read_text()) + recomputed = estimate(*_v3_inputs()) + + assert committed["costs_usd"] == pytest.approx(recomputed["costs_usd"]) + assert committed["calls"] == recomputed["calls"] + # The timestamp is what tells a reader which snapshot the number describes, + # so a stale one is its own defect even when the totals happen to agree. + assert committed["pricing_checked_at_utc"] == recomputed["pricing_checked_at_utc"] + committed_rows = {row["experiment_id"]: row for row in committed["models"]} + recomputed_rows = {row["experiment_id"]: row for row in recomputed["models"]} + assert set(committed_rows) == set(recomputed_rows) + for experiment_id, row in recomputed_rows.items(): + assert committed_rows[experiment_id]["model"] == row["model"] + assert committed_rows[experiment_id]["cost_per_decision_usd"] == pytest.approx(row["cost_per_decision_usd"]) + + def test_costs_sum_unrounded_rows_before_contingency() -> None: result = estimate(*_committed_inputs()) rows = result["models"] diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index bae167a..e878c1b 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -1325,6 +1325,90 @@ def test_zero_call_route_preflight_has_separate_authorization_and_never_launches assert not (tmp_path / "checkpoints").exists() +def test_route_preflight_checks_every_route_before_failing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A bad route must not hide the routes queued behind it. + + Exiting on the first failure understates how much is broken: the operator + sees one dead route, fixes it, and only then learns about the next one. + The zero-call phase is free, so it has no reason to stop early -- it must + probe every route and report the complete set in one pass. The paid + phases keep failing fast, which is asserted separately below. + """ + monkeypatch.setenv("OPENROUTER_API_KEY", "test-route-preflight-key") + registry, lane, _manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + registry["selection_status"] = "route-preflight-ready" + lane["preregistration_status"] = "provisional-blocked" + lane["route_preflight_authorized"] = True + publication_runner.PANEL_CONFIG.write_text(json.dumps(registry)) + publication_runner.LANE_CONFIG.write_text(json.dumps(lane)) + + model_ids = [model["id"] for model in registry["models"]] + assert len(model_ids) >= 3, "this test needs a route queued behind both failures" + doomed = {model_ids[0], model_ids[2]} + checked: list[str] = [] + child_calls: list[str] = [] + + def fake_validate(cell, _env): + checked.append(cell.experiment_id) + if cell.experiment_id in doomed: + raise RuntimeError("no healthy OpenRouter endpoint matches") + + monkeypatch.setattr(publication_runner, "_validate_openrouter_endpoint", fake_validate) + monkeypatch.setattr( + publication_runner.subprocess, + "run", + lambda *_args, **_kwargs: child_calls.append("child"), + ) + + with pytest.raises(SystemExit) as exc_info: + main(["route-preflight", "--contract", "sota-v3", "--run-dir", str(tmp_path)]) + + message = str(exc_info.value.code) + # Every route was probed, including the ones queued behind both failures. + assert checked == model_ids + assert f"failed for {len(doomed)} of {len(model_ids)} routes" in message + for model_id in doomed: + assert model_id in message + captured = capsys.readouterr() + assert captured.err.count("OpenRouter endpoint preflight failed") == len(doomed) + assert "OpenRouter endpoint preflight failed" not in captured.out + # Still zero-call and still stateless, exactly as on the passing path. + assert child_calls == [] + assert not (tmp_path / "run-state.json").exists() + assert not (tmp_path / "raw").exists() + + +def test_paid_phases_still_abort_on_the_first_bad_route( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Collecting failures is a zero-call affordance, not a general one. + + A phase that spends money must stop the instant a route is wrong, so the + later cells are never probed at all. + """ + monkeypatch.setenv("OPENROUTER_API_KEY", "test-smoke-key") + registry, _lane, _manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + model_ids = [model["id"] for model in registry["models"]] + checked: list[str] = [] + + def fake_validate(cell, _env): + checked.append(cell.experiment_id) + raise RuntimeError("no healthy OpenRouter endpoint matches") + + monkeypatch.setattr(publication_runner, "_validate_openrouter_endpoint", fake_validate) + + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", "1.00"]) + + assert checked == model_ids[:1], "a paid phase kept probing after a bad route" + assert model_ids[0] in str(exc_info.value.code) + + def test_v3_route_preflight_requires_bearer_credential_before_endpoint_request( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1496,6 +1580,8 @@ def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: "status": 0, "max_completion_tokens": 65536, "supported_parameters": ["max_tokens", "response_format", "reasoning"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, } ] } @@ -1509,6 +1595,314 @@ def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: valid["data"]["endpoints"][0]["name"] = cell.endpoint_name valid["data"]["endpoints"][0]["supported_parameters"] = ["max_tokens", "response_format"] assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + valid["data"]["endpoints"][0]["supported_parameters"] = ["max_tokens", "response_format", "reasoning"] + valid["data"]["endpoints"][0].pop("max_completion_tokens") + assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + valid["data"]["endpoints"][0]["max_completion_tokens"] = "65536" + assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + + +def _healthy_endpoint(cell) -> dict: + return { + "provider_name": "Alibaba", + "tag": "alibaba", + "name": cell.endpoint_name, + "status": 0, + "max_completion_tokens": 65536, + "supported_parameters": ["max_tokens", "response_format", "reasoning"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, + } + + +def test_non_finite_spend_limits_are_refused( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`--max-spend-usd nan` must not read as a ceiling. + + NaN defeats every downstream guard at once and silently: `nan <= 0`, + `nan > ceiling`, and `spent >= nan` are all False, so a NaN limit satisfies + "the operator passed a ceiling" while bounding nothing at all -- an + unbounded paid run that looks fully authorized. Infinity is the same hole + wherever no ceiling is configured. + """ + _frozen_panel_files(tmp_path, monkeypatch) + checked: list[str] = [] + monkeypatch.setattr( + publication_runner, + "_validate_openrouter_endpoint", + lambda cell, _env: checked.append(cell.experiment_id), + ) + + for literal in ("nan", "inf", "-inf", "NaN", "Infinity"): + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", literal]) + assert exc_info.value.code == 2, literal + assert checked == [], "a non-finite spend limit reached the endpoint probe" + + nonfinite = tmp_path / "nonfinite-protocol.json" + nonfinite.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": float("inf")}})) + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-nonfinite-ceiling", + (nonfinite,) * 5, + ) + with pytest.raises(ValueError, match="positive finite number"): + publication_runner._enforce_operator_ceiling(1.0, "sota-nonfinite-ceiling") + + +def test_pricing_drift_fails_closed_when_a_rate_cannot_be_verified( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ "Could not verify the price" is not "the price is unchanged". + + Only one of those is safe to spend against, so every unverifiable case + blocks rather than passing quietly. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + committed = json.loads(publication_runner.PRICING_CONFIG.read_text())["models"][cell.model] + + def payload(pricing: dict, **overrides) -> dict: + endpoint = { + "provider_name": cell.upstream_provider, + "tag": cell.endpoint_tag, + "name": cell.endpoint_name, + "pricing": pricing, + } + endpoint.update(overrides) + return {"data": {"endpoints": [endpoint]}} + + good = {"prompt": str(committed["prompt"]), "completion": str(committed["completion"])} + assert publication_runner._pricing_drift_issues(cell, payload(good)) == [] + + for pricing, expected in ( + ({"completion": good["completion"]}, "unreadable"), + ({**good, "prompt": "not-a-number"}, "unreadable"), + ({**good, "prompt": "nan"}, "not a usable number"), + ({**good, "prompt": "-1e-07"}, "not a usable number"), + ): + issues = publication_runner._pricing_drift_issues(cell, payload(pricing)) + assert issues and expected in issues[0], (pricing, issues) + + # The pinned identity is provider + tag + name, matching the preflight. + # A same-tag endpoint from another provider is not this cell's price. + issues = publication_runner._pricing_drift_issues(cell, payload(good, provider_name="Somebody Else")) + assert issues and "pinned route identity" in issues[0] + + # A model absent from the snapshot cannot be price-checked at all. + snapshot = json.loads(publication_runner.PRICING_CONFIG.read_text()) + del snapshot["models"][cell.model] + stripped = tmp_path / "pricing.json" + stripped.write_text(json.dumps(snapshot)) + monkeypatch.setattr(publication_runner, "PRICING_CONFIG", stripped) + issues = publication_runner._pricing_drift_issues(cell, payload(good)) + assert issues and "no rates for" in issues[0] + + +def test_pricing_drift_fails_upward_and_only_reports_downward( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A rate that rose invalidates the reservation; a rate that fell does not. + + The committed snapshot is what the budget was computed from, so an + increase makes the plan wrong in the direction that costs money. A + decrease only means coming in under reserve -- the GLM Novita route picked + up a 55.1% discount that nobody noticed for two weeks because nothing + compared the snapshot to reality. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + committed = json.loads(publication_runner.PRICING_CONFIG.read_text())["models"][cell.model] + + def payload(prompt: float, completion: float) -> dict: + return { + "data": { + "endpoints": [ + { + "provider_name": cell.upstream_provider, + "tag": cell.endpoint_tag, + "name": cell.endpoint_name, + "pricing": {"prompt": str(prompt), "completion": str(completion)}, + } + ] + } + } + + unchanged = payload(committed["prompt"], committed["completion"]) + assert publication_runner._pricing_drift_issues(cell, unchanged) == [] + + risen = payload(committed["prompt"] * 2, committed["completion"]) + issues = publication_runner._pricing_drift_issues(cell, risen) + assert issues and "exceeds the committed snapshot rate" in issues[0] + + capsys.readouterr() + fallen = payload(committed["prompt"] / 2, committed["completion"] / 2) + assert publication_runner._pricing_drift_issues(cell, fallen) == [], "a discount must not block a run" + out = capsys.readouterr().out + assert "fell from" in out and "under its reservation" in out + + # Another provider's rate is not this cell's rate, so it must not be read + # as one -- but nor may the absence of the pinned route pass as "unchanged". + # Both are "the price could not be verified", which now blocks. + other = payload(committed["prompt"] * 10, committed["completion"]) + other["data"]["endpoints"][0]["tag"] = "someone-else/fp8" + issues = publication_runner._pricing_drift_issues(cell, other) + assert issues and "pinned route identity" in issues[0] + assert "exceeds the committed snapshot rate" not in issues[0], "priced against the wrong route" + + +def test_the_suite_cannot_inherit_a_live_provider_credential() -> None: + """Pin the guard that stops a test from quietly billing a real account. + + The runner loads the gitignored `.env.local` itself, so before this guard + existed any test driving `main()` through a paid phase without stubbing + the child process ran the real benchmark against real routes. On + 2026-08-04 a test written to assert a spend ceiling *blocks* a run instead + spent $0.44 across 38 live calls, and passed no judgement on it -- the run + simply succeeded. + """ + assert os.environ.get("OPENROUTER_API_KEY") is None + assert publication_runner.load_environment_files(Path(".")) == [] + + +def test_operator_ceiling_rejects_a_run_that_could_outspend_the_committed_cap() -> None: + """The committed ceiling has to bind, or it is a comment. + + `budget_policy.operator_ceiling_usd` sat in the config unread, so the only + thing between a mistyped `--max-spend-usd` and an unbounded run was the + operator retyping the right number from memory. + """ + ceiling = json.loads(Path("config/sota_v3_publication_protocol.json").read_text()) + ceiling = ceiling["budget_policy"]["operator_ceiling_usd"] + assert ceiling == 150.00 + + publication_runner._enforce_operator_ceiling(ceiling, "sota-v3") + publication_runner._enforce_operator_ceiling(ceiling - 0.01, "sota-v3") + + with pytest.raises(ValueError, match="exceeds the committed operator ceiling"): + publication_runner._enforce_operator_ceiling(ceiling + 0.01, "sota-v3") + with pytest.raises(ValueError, match=r"\$1200\.00"): + publication_runner._enforce_operator_ceiling(1200.00, "sota-v3") + + +def test_operator_ceiling_stays_permissive_when_no_cap_is_committed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A contract that has not chosen a number must not be given one silently.""" + protocol_path = tmp_path / "protocol.json" + protocol_path.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": None}})) + monkeypatch.setitem( + publication_runner.CONTRACT_CONFIGS, + "sota-test", + (protocol_path, protocol_path, protocol_path, protocol_path, protocol_path), + ) + publication_runner._enforce_operator_ceiling(10_000.00, "sota-test") + + protocol_path.write_text(json.dumps({"budget_policy": {"operator_ceiling_usd": "lots"}})) + with pytest.raises(ValueError, match="must be a positive finite number"): + publication_runner._enforce_operator_ceiling(1.00, "sota-test") + + +def test_paid_run_above_the_ceiling_is_refused_before_any_cell_runs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The gate has to sit ahead of the cell loop, not inside it. + + Checked by proving that neither the endpoint probe nor a child process is + reached: a ceiling enforced after the first cell has already spent money + is not a ceiling. + """ + _frozen_panel_files(tmp_path, monkeypatch) + protocol = json.loads(publication_runner.PROTOCOL_CONFIG.read_text()) + protocol["budget_policy"]["operator_ceiling_usd"] = 120.00 + publication_runner.PROTOCOL_CONFIG.write_text(json.dumps(protocol)) + + checked: list[str] = [] + child_calls: list[str] = [] + monkeypatch.setattr( + publication_runner, + "_validate_openrouter_endpoint", + lambda cell, _env: checked.append(cell.experiment_id), + ) + monkeypatch.setattr( + publication_runner.subprocess, + "run", + lambda *_args, **_kwargs: child_calls.append("child"), + ) + + with pytest.raises(SystemExit) as exc_info: + main(["smoke", "--contract", "sota-v3", "--run-dir", str(tmp_path), "--max-spend-usd", "500"]) + + assert exc_info.value.code == 2 + assert checked == [], "a run over the ceiling reached the endpoint probe" + assert child_calls == [], "a run over the ceiling launched a model" + + +def test_endpoint_preflight_enforces_both_uptime_floors() -> None: + """The two windows catch different failures, so both have to gate. + + The 24h figure is a chronic filter and cannot see an outage in progress: + on 2026-08-04 the first-party DeepSeek route was deranked to status -5 + while still reporting 99.24% over 24h. The 30m figure is the one that + moved, so it carries the acute signal. Missing or malformed telemetry is + unknown health, so it must fail closed rather than read as a passing route. + """ + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] + endpoint = _healthy_endpoint(cell) + payload = {"data": {"endpoints": [endpoint]}} + assert _endpoint_issues(cell, payload) == [] + + # The acute case: the exact shape of the DeepSeek derank, which a 24h-only + # floor waves straight through. + endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] = 78.93, 99.24 + issues = _endpoint_issues(cell, payload) + assert issues and "30m uptime floor" in issues[0] and "78.93%" in issues[0] + + # The chronic case: recent traffic looks fine, the whole day did not. + endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] = 99.9, 80.0 + issues = _endpoint_issues(cell, payload) + assert issues and "24h uptime floor" in issues[0] and "80.00%" in issues[0] + + for field, floor in ( + ("uptime_last_30m", publication_runner.MIN_UPTIME_LAST_30M_PCT), + ("uptime_last_1d", publication_runner.MIN_UPTIME_LAST_1D_PCT), + ): + endpoint.update(_healthy_endpoint(cell)) + endpoint[field] = floor + assert _endpoint_issues(cell, payload) == [], f"{field} floor must be inclusive" + + for field, value in ( + ("uptime_last_30m", None), + ("uptime_last_30m", "unknown"), + ("uptime_last_30m", float("nan")), + ("uptime_last_1d", float("inf")), + ): + endpoint.update(_healthy_endpoint(cell)) + endpoint[field] = value + issues = _endpoint_issues(cell, payload) + assert issues and "no finite numeric" in issues[0], (field, value, issues) + + endpoint.update(_healthy_endpoint(cell)) + del endpoint["uptime_last_30m"], endpoint["uptime_last_1d"] + issues = _endpoint_issues(cell, payload) + assert issues and "no finite numeric" in issues[0] + + +def test_uptime_floors_sit_below_the_healthy_cohort_noise_band() -> None: + """The floors must not flap. + + These readings drift by roughly half a point between consecutive polls, + so a floor set near real values rejects healthy routes at random. A 99% + 24h floor rejected two healthy cohort members on the day it was written. + Both floors are therefore pinned well clear of the observed band. + """ + assert publication_runner.MIN_UPTIME_LAST_30M_PCT <= 90.0 + assert publication_runner.MIN_UPTIME_LAST_1D_PCT <= 95.0 def test_endpoint_preflight_allows_registered_prompt_only_json_route() -> None: @@ -1524,6 +1918,8 @@ def test_endpoint_preflight_allows_registered_prompt_only_json_route() -> None: "status": 0, "max_completion_tokens": 262144, "supported_parameters": ["max_tokens", "reasoning", "structured_outputs"], + "uptime_last_30m": 99.8, + "uptime_last_1d": 99.75, } ] } diff --git a/tests/test_sota_v3_route_catalog.py b/tests/test_sota_v3_route_catalog.py index 7cffd4e..8270bfd 100644 --- a/tests/test_sota_v3_route_catalog.py +++ b/tests/test_sota_v3_route_catalog.py @@ -44,6 +44,11 @@ def test_v3_catalog_freezes_exact_balanced_cohort_without_unlocking_execution() assert registry["catalog_snapshot_status"] == "frozen-public-metadata-only" assert registry["selection_status"] == "route-preflight-ready" assert registry["selection_frozen_at_utc"] is None + assert registry["selection_revision"] == "2026-08-04-public-catalog-lineup-refresh-v3" + policy = registry["selection_policy"] + assert "Qwen 3.8 Max" in policy + assert "deepinfra/fp8" in policy and "cloudflare/fp8" in policy + assert "route-preflight-ready rather than frozen" in policy assert registry["catalog_checked_at_utc"] assert set(registry["required_smokes"]) == {model["id"] for model in models} assert registry["output_token_cap"] == 4_096