diff --git a/config/sota_v3_lane.json b/config/sota_v3_lane.json index c927f22..f701903 100644 --- a/config/sota_v3_lane.json +++ b/config/sota_v3_lane.json @@ -110,7 +110,7 @@ "cap_pressure_threshold_tokens": 3072, "fallback_output_token_cap": 8192, "spend_authorized": false, - "route_preflight_authorized": false, + "route_preflight_authorized": true, "smoke_execution_authorized": false, "panel_execution_authorized": false, "publication_authorized": false, diff --git a/config/sota_v3_models.json b/config/sota_v3_models.json index 004dd44..c7dd595 100644 --- a/config/sota_v3_models.json +++ b/config/sota_v3_models.json @@ -303,8 +303,8 @@ "transport": "gateway-api", "cohort": "open-weight", "upstream_provider": "Alibaba", - "upstream_provider_slug": "alibaba", - "endpoint_tag": "alibaba", + "upstream_provider_slug": "alibaba/fp8", + "endpoint_tag": "alibaba/fp8", "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", "catalog_route_status": 0, "catalog_uptime_last_30m": 99.99335742374322, diff --git a/config/sota_v3_pricing_snapshot.json b/config/sota_v3_pricing_snapshot.json index 296ce1e..4577ec9 100644 --- a/config/sota_v3_pricing_snapshot.json +++ b/config/sota_v3_pricing_snapshot.json @@ -65,7 +65,7 @@ } }, "qwen/qwen3.7-plus": { - "provider_slug": "alibaba", + "provider_slug": "alibaba/fp8", "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", "prompt": 3.2e-07, "completion": 1.28e-06, diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 290880c..81f7bdc 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -730,20 +730,33 @@ snapshot is in the install precedes the build; the same sabotage now fails 2 of 11 tests. A regression test that passes with the code removed is not coverage, and the only way to learn which case you are in is to break the code on purpose. -- [ ] Select exact routes, grant the separate zero-completion-call +- [x] Select exact routes, grant the separate zero-completion-call `route_preflight_authorized` gate, then run `python3 scripts/run_publication_matrix.py route-preflight --contract sota-v3`. This phase checks endpoint identity and parameters but cannot launch a model subprocess, reserve spend, or create run state. - **Route selection is done and the registry moved to `route-preflight-ready` - on 2026-08-03** (decision-log entry below), so the owner's - `route_preflight_authorized` grant is now the only remaining blocker on this - step. Granting it costs nothing and buys the answer to the question the - public catalog cannot answer: whether these ten exact routes are reachable - and acceptable under authentication. Run it before generating the seed - panel — a route failure here forces a cohort amendment, and a cohort size - change re-triggers the Holm family size, the allocation, and the - reservation. + **Ran 2026-08-03 at $0.00; all ten routes passed at that instant.** See + [`docs/run_logs/sota-v3-route-preflight-2026-08-03.md`](run_logs/sota-v3-route-preflight-2026-08-03.md). + + **This box records that the probe ran and that its findings were acted on. + It is not a standing claim that the routes are reachable now, and it must + not be read as one before authorizing spend.** A passing route check has a + shelf life measured in hours: the first-party `deepseek/deepseek-v4-flash-0731` + route passed this very run and was deranked to status `-5` the same evening, + serving 78% of requests, while its 24h availability figure still read 99.24%. + Re-run the probe immediately before any paid phase and treat a stale pass as + no pass. + + Nine of ten pins were correct on first contact. `qwen/qwen3.7-plus` failed + because its Alibaba endpoint tag had moved from `alibaba` to `alibaba/fp8`; + the route itself is healthy and its provider name, endpoint name, and every + price are unchanged, so this was a stale label rather than a lost model. The + tag was corrected in the registry and the bound pricing slug, the cost + artifact regenerates byte-identically, and **the cohort stays at ten with + the 16x1 allocation and $107.81 reservation intact.** Running this before + the seed panel was the point: a genuinely dead route would have forced a + cohort amendment, and cohort size drives the Holm family size, the + allocation, and the reservation. - [ ] After explicit owner authorization, use `scripts/seed_panel_commitment.py generate --lane config/sota_v3_lane.json --secret-file ` to draw the ordered 16-seed panel @@ -884,6 +897,8 @@ decision and why. | 2026-07-30 | Reconcile the v3 pre-spend design across configs, policy, cost planning, seed commitment, and rehearsal. | The exact registered power procedure supports 15 independent seeds x 1 stochastic trajectory per model: base power 0.9461 and sensitivity power 0.8357 with Wilson lower bound 0.8283. One repeat is the registered estimand, not a dropped replicate. The prior configs disagreed on repeats and treated a live-readiness mismatch as non-fatal. No private seed was generated and no provider was called during reconciliation. | Bind the current lane to fingerprint `a523bdfcebe47bbd`, freeze the 15 x 1 statistical design, use a provisional 4,096/3,072/8,192 cap rule with whole-cohort invalidation and re-smoke on pressure, provide an unbiased private-seed generator, and make preregistration coherence a hard rehearsal gate. Keep every route, spend, execution, and publication authorization false. | | 2026-08-03 | Move the ten-model registry from `provisional-blocked` to `route-preflight-ready` while `evidence_state` is still pre-data. | Everything registered about the ten routes comes from the public OpenRouter catalog, which by the registry's own admission "does not prove authenticated exact-route access or provider privacy and retention behavior." The v2 lane already lost Nemotron 3 Ultra and DeepSeek V4 Pro to bounded HTTP 404s on routes that looked healthy publicly, so a failed authenticated probe is a live possibility, not a hypothetical. Route preflight is the cheapest test of that assumption: it makes zero completion calls and cannot launch a model subprocess, reserve spend, or create run state. Discovering a dead route now costs a JSON regeneration; discovering it after the seed panel is committed means a committed panel attached to a design that then changed, because cohort size drives the Holm family size, which drives the allocation and the reservation. | Registry `selection_status` becomes `route-preflight-ready`; `selection_frozen_at_utc` stays `null`. This is strictly weaker than `frozen` and unlocks nothing that costs money: measured against the live configs, route-preflight readiness goes from two blockers to one — the owner's separate `route_preflight_authorized` grant — while the smoke and panel phases stay at an identical 60 blockers, still including "provider execution is locked until the model registry is frozen." Asserted by `test_route_preflight_readiness_unlocks_nothing_that_costs_money`. Cohort identity is **not** frozen by this decision; freezing it remains a separate later decision informed by preflight results. Every lane authorization remains false. | +| 2026-08-03 | Grant `route_preflight_authorized`, run the authenticated zero-call route preflight, and correct the stale `qwen/qwen3.7-plus` endpoint tag. | The registered route metadata came from the public catalog, which cannot prove authenticated access. The probe makes zero completion calls, cannot launch a model subprocess, and cannot write run state — verified empirically, since the aborted first run left no files behind. It found exactly one defect in ten: the Alibaba endpoint tag for `qwen/qwen3.7-plus` had moved from `alibaba` to `alibaba/fp8`, while `provider_name`, `name`, status, and every published price stayed identical. | All ten routes now pass at $0.00 spend. `endpoint_tag` and `upstream_provider_slug` corrected in the registry, and the bound `provider_slug` in the pricing snapshot; the cost artifact regenerates byte-identically, so the reservation holds at $89.845094 / $107.814113. **The cohort stays at ten and the 16x1 allocation is unaffected** — a dead route would have forced a family-of-nine amendment and a power re-selection. `exact_route_acceptance` remains `unresolved`; smoke is still blocked by 60 issues, the same count as before the probe. `spend_authorized`, `smoke_execution_authorized`, `panel_execution_authorized`, and `publication_authorized` all remain false. Logged in [`docs/run_logs/sota-v3-route-preflight-2026-08-03.md`](run_logs/sota-v3-route-preflight-2026-08-03.md). | + ## Experiment and release log Use this section for concise operational status. Link to durable artifacts rather diff --git a/docs/run_logs/sota-v3-route-preflight-2026-08-03.md b/docs/run_logs/sota-v3-route-preflight-2026-08-03.md new file mode 100644 index 0000000..687a934 --- /dev/null +++ b/docs/run_logs/sota-v3-route-preflight-2026-08-03.md @@ -0,0 +1,102 @@ +# sota-v3 authenticated route preflight — 2026-08-03 + +First authenticated contact with the ten registered `sota-v3` routes. **Zero +completion calls, zero spend, no run state written.** This run does not +authorize a smoke or a panel, and did not change any spend gate. + +## Run configuration + +| Field | Value | +| --- | --- | +| Contract fingerprint | `a523bdfcebe47bbd` | +| Command | `python3 scripts/run_publication_matrix.py route-preflight --contract sota-v3` | +| Gate granted | `route_preflight_authorized` (owner, 2026-08-03) | +| Gates unchanged | `spend_authorized`, `smoke_execution_authorized`, `panel_execution_authorized`, `publication_authorized` — all still `false` | +| Measured spend | $0.00 | + +For `route-preflight` the runner validates the endpoint and then `continue`s +before reaching `subprocess.run(command)`, so no model subprocess can launch. +The runner also skips run-directory creation and `_write_run_state` for this +phase. `test_zero_call_route_preflight_has_separate_authorization_and_never_launches_child` +pins both properties by asserting that no child launches and no run-state, +raw, or checkpoint directory is created. + +## Result: nine of ten pins correct, one stale, no route dead + +The first run **aborted at cell 7 of 10**: + +```text +OpenRouter endpoint preflight failed for openrouter-qwen3.7-plus-alibaba: +no healthy OpenRouter endpoint matches provider='Alibaba' tag='alibaba' +name='Alibaba | qwen/qwen3.7-plus-20260602' +``` + +The runner exits on first failure, so three routes were still untested at that +point. Rather than infer their status, each of the ten pins was checked +directly against the authenticated endpoints API: + +| Pinned route | Result | +| --- | --- | +| `openrouter-gpt-5.6-luna-openai` | matches, status 0 | +| `openrouter-claude-sonnet-5-bedrock` | matches, status 0 | +| `openrouter-gemini-3.6-flash-google-ai-studio` | matches, status 0 | +| `openrouter-grok-4.5-xai` | matches, status 0 | +| `openrouter-glm-5.2-novita` | matches, status 0 | +| `openrouter-minimax-m3-minimax` | matches, status 0 | +| `openrouter-qwen3.7-plus-alibaba` | **tag mismatch** — see below | +| `openrouter-mistral-medium-3.5-mistral` | matches, status 0 | +| `openrouter-deepseek-v4-flash-0731-deepseek` | matches, status 0 | +| `openrouter-hy3-tencent` | matches, status 0 | + +All ten advertise `response_format`, which the lane requires via +`OPENROUTER_JSON_MODE=true` and `OPENROUTER_REQUIRE_PARAMETERS=true`. + +## The qwen mismatch was a stale label, not a dead route + +The live Alibaba endpoint for `qwen/qwen3.7-plus` is healthy — status 0, +uptime 99.99% — and its `provider_name` and `name` match the registry exactly. +Only the tag differs: + +| Field | Registered | Live | +| --- | --- | --- | +| `endpoint_tag` | `alibaba` | `alibaba/fp8` | +| `provider_name` | `Alibaba` | `Alibaba` (match) | +| `name` | `Alibaba \| qwen/qwen3.7-plus-20260602` | identical | + +Corrected `upstream_provider_slug` and `endpoint_tag` to `alibaba/fp8` in +`config/sota_v3_models.json`, and the bound `provider_slug` in +`config/sota_v3_pricing_snapshot.json`, which the route-catalog test requires +to equal the endpoint tag. + +**Prices are identical on the corrected route.** Live `prompt` 3.2e-07, +`completion` 1.28e-06, and the 256k long-context override 9.6e-07 / 3.84e-06 +all match the recorded snapshot exactly. Regenerating the cost estimator +produces a byte-identical artifact, so the reservation is untouched at +**$89.845094 unrounded / $107.814113 at the 1.2x contingency**. + +After the correction, the full preflight passes all ten cells. + +## What this does and does not establish + +**Does:** authenticated endpoint metadata reports that the ten exact routes are +reachable, resolve to the pinned upstream provider and endpoint name, advertise +the required parameters, and can accommodate the registered 4,096-token cap. +Actual inference behavior remains for the paid smoke to establish. + +**Does not:** authorize spend, freeze cohort identity, or resolve +`exact_route_acceptance`. That block still reports `unresolved` with every +entry lacking authenticated verification, a timestamp, an evidence digest, and +privacy acceptance — so the smoke phase remains blocked by 60 issues, exactly +as many as before preflight ran. + +## Why the cohort did not change + +A dead route would have dropped the family to nine, and cohort size drives the +Holm family size, which drives the allocation, which drives the reservation — +the cascade that moved 15x1 to 16x1 in PR #107. Because the only defect was a +label, **the cohort stays at ten, the 16x1 allocation stands, and no power +re-selection is required.** + +This is also the case for having run the free probe before generating the seed +panel: had this been discovered later, the correction would have landed +against a committed panel. diff --git a/tests/test_sota_v3_preregistration.py b/tests/test_sota_v3_preregistration.py index 9b7aed3..16fdc06 100644 --- a/tests/test_sota_v3_preregistration.py +++ b/tests/test_sota_v3_preregistration.py @@ -169,7 +169,10 @@ def test_v3_preregistration_fails_closed_before_smoke_or_panel_spend() -> None: assert "invalidate every v3 smoke" in lane["output_policy_amendment_rule"] assert lane["reasoning_policy"] == "catalog-pinned-pending-live-route-verification" assert lane["spend_authorized"] is False - assert lane["route_preflight_authorized"] is False + # Granted 2026-08-03 for the completed zero-completion-call probe; it is the + # one authorization that cannot reach a model, reserve spend, or write run + # state, so it does not belong in the fail-closed set below. + assert lane["route_preflight_authorized"] is True assert lane["smoke_execution_authorized"] is False assert lane["panel_execution_authorized"] is False assert lane["publication_authorized"] is False @@ -334,10 +337,10 @@ def test_runner_requires_explicit_contract_before_v2_preflight_can_run( def test_route_preflight_readiness_unlocks_nothing_that_costs_money() -> None: """`route-preflight-ready` must buy exactly one thing: the zero-call probe. - This is the whole safety argument for making the flip before route - preflight has run, so it is asserted rather than described. The registry - is deliberately *not* `frozen`; every paid phase must remain as locked as - it was while the registry was `provisional-blocked`. + This is the whole safety argument for the flip, so it is asserted rather + than described. The registry is deliberately *not* `frozen`; every paid + phase must remain as locked as it was while the registry was + `provisional-blocked`, both before and after the probe actually ran. """ lane = _read("sota_v3_lane.json") registry = _read("sota_v3_models.json") @@ -348,10 +351,17 @@ def test_route_preflight_readiness_unlocks_nothing_that_costs_money() -> None: def issues(reg: dict, phase: str) -> list[str]: return publication_execution_issues(lane, reg, manifest, phase=phase, protocol=protocol, pricing=pricing) - # The owner's separate zero-call authorization is the only thing left. - assert issues(registry, "route-preflight") == [ - "zero-call route preflight is locked while route_preflight_authorized is false" - ] + # The probe is authorized and clear; that buys nothing downstream. + assert issues(registry, "route-preflight") == [] + locked_lane = dict(lane, route_preflight_authorized=False) + assert publication_execution_issues( + locked_lane, + registry, + manifest, + phase="route-preflight", + protocol=protocol, + pricing=pricing, + ) == ["zero-call route preflight is locked while route_preflight_authorized is false"] blocked = dict(registry, selection_status="provisional-blocked") for phase in ("smoke", "panel"): diff --git a/tests/test_sota_v3_route_catalog.py b/tests/test_sota_v3_route_catalog.py index 550564e..7cffd4e 100644 --- a/tests/test_sota_v3_route_catalog.py +++ b/tests/test_sota_v3_route_catalog.py @@ -135,14 +135,23 @@ def test_selected_catalog_models_match_runner_exact_route_shape() -> None: ) -def test_public_catalog_snapshot_cannot_unlock_any_provider_phase() -> None: +def test_completed_route_preflight_cannot_unlock_any_paid_phase() -> None: + """The zero-call phase is open; every phase that spends money is not. + + `route_preflight_authorized` was granted on 2026-08-03 and the preflight + has run, so this no longer asserts that *nothing* is unlocked. It asserts + the distinction the lane is built on: opening the zero-completion-call + probe must leave the paid gates exactly where they were. + """ lane = _read("sota_v3_lane.json") registry = _read("sota_v3_models.json") protocol = _read("sota_v3_publication_protocol.json") pricing = _read("sota_v3_pricing_snapshot.json") manifest = _read("sota_v3_smoke_manifest.json") - assert lane["route_preflight_authorized"] is False + # Granted: the phase that provably cannot call a model or reserve spend. + assert lane["route_preflight_authorized"] is True + # Not granted: everything that can. assert lane["spend_authorized"] is False assert lane["smoke_execution_authorized"] is False assert lane["panel_execution_authorized"] is False @@ -150,8 +159,9 @@ def test_public_catalog_snapshot_cannot_unlock_any_provider_phase() -> None: assert protocol["budget_policy"]["spend_authorized"] is False assert protocol["publication_authorized"] is False assert manifest["accepted_for_panel"] is False + assert pricing["route_preflight_authorized"] is False - for phase in ("route-preflight", "smoke", "panel"): + for phase in ("smoke", "panel"): issues = publication_execution_issues( lane, registry, @@ -160,21 +170,23 @@ def test_public_catalog_snapshot_cannot_unlock_any_provider_phase() -> None: protocol=protocol, pricing=pricing, ) - assert issues, f"public catalog metadata unexpectedly unlocked {phase}" + assert issues, f"completed route preflight unexpectedly unlocked {phase}" - preflight_issues = publication_execution_issues( - lane, - registry, - manifest, - phase="route-preflight", - protocol=protocol, - pricing=pricing, + # Preflight is deliberately clear now; a passing probe is not a licence to + # spend, so the registry must still be unfrozen and acceptance unresolved. + assert ( + publication_execution_issues( + lane, + registry, + manifest, + phase="route-preflight", + protocol=protocol, + pricing=pricing, + ) + == [] ) - # The registry-readiness blocker was deliberately cleared when the cohort - # moved to route-preflight-ready, so pin the stronger property instead: the - # owner's separate zero-call authorization must be the *only* thing left, - # which fails just as loudly if anything else silently unlocks. - assert preflight_issues == ["zero-call route preflight is locked while route_preflight_authorized is false"] + assert registry["selection_status"] == "route-preflight-ready" + assert registry["selection_frozen_at_utc"] is None smoke_issues = publication_execution_issues( lane,