Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions config/sota_v3_lane.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"contract_fingerprint": "a523bdfcebe47bbd",
"mechanics_status": "frozen-for-sota-v3-panel",
"mechanics_change_policy": "Any score-, action-, observation-, or simulator-semantic change after this preregistration requires a new contract fingerprint, invalidates all sota-v3 smoke evidence, and re-locks provider spend.",
"preregistration_status": "provisional-pre-smoke",
"preregistration_status": "frozen",
"preregistered_at_utc": "2026-07-27T16:58:41Z",
"design_amended_at_utc": "2026-08-03T00:00:00Z",
"design_amendment": {
Expand Down Expand Up @@ -89,10 +89,14 @@
"decision_required": null
},
"seed_panel": {
"status": "pending-authorized-generation",
"name": null,
"status": "frozen",
"name": "private-env",
"count": 16,
"sha256": null
"sha256": "291fa61cc3dfd8b23fdd79cce3c80a0a98f918f6c8757d35c21b4d8131cc6099",
"hiding_commitment_sha256": "7f8da7ca4db4a698ea2b0506af8568c89744e18508d8dedbfeb1c87e90a2b5f8",
"generation_method": "uniform-rejection-sampling-secrets-randbelow-63bit-v1",
"secret_escrow": "macos-keychain:gm-bench-sota-v3-private-panel",
"seed_values_included": false
},
"reference_agent": "pick-trader",
"protocol_repair_attempts": 1,
Expand All @@ -102,24 +106,22 @@
"publication_protocol": "config/sota_v3_publication_protocol.json",
"pricing_snapshot": "config/sota_v3_pricing_snapshot.json",
"minimum_headline_models": 10,
"reasoning_policy": "catalog-pinned-pending-live-route-verification",
"reasoning_policy": "catalog-pinned-pending-strict-smoke-behavior-verification",
"output_token_cap": 4096,
"output_budget_status": "provisional-pre-smoke-validation",
"output_budget_status": "frozen-native-reasoning-cap",
"output_policy_basis": "fixed-safety-ceiling",
"output_policy_amendment_rule": "The 4,096-token cap is a symmetric pre-smoke safety ceiling, not a result selected from v3 model behavior. If any accepted-route smoke is truncated, reaches 3,072 output tokens, or cannot satisfy a route's mandatory minimum reasoning policy, invalidate every v3 smoke, amend the common cap before panel data, and re-smoke the full registered family. Model score or apparent quality never authorizes a cap change.",
"cap_pressure_threshold_tokens": 3072,
"fallback_output_token_cap": 8192,
"spend_authorized": false,
"spend_authorized": true,
"route_preflight_authorized": true,
"smoke_execution_authorized": false,
"smoke_execution_authorized": true,
"panel_execution_authorized": false,
"publication_authorized": false,
"blockers": [
"Generate and commit the 16-seed private panel under separate owner authorization; seed identity is unfrozen until its salted commitment hash is recorded.",
"Complete authenticated exact-route and privacy verification for the catalog-selected model cohort before any provider call.",
"Live-verify and pin each exact provider route, endpoint name, supported parameters, privacy policy, and reasoning requirements.",
"Validate the provisional common 4,096-token safety ceiling on one strict smoke per registered route; any predeclared cap-pressure trigger invalidates all v3 smokes and requires one symmetric pre-panel amendment.",
"Record one accepted strict-fallback smoke for every registered model before panel execution.",
"Grant separate zero-completion-call route-preflight authorization after routes are selected, then grant operator spend authorization only after that route gate passes."
"Run exactly one serial strict smoke per registered route at the frozen 4,096-token cap; this is the only currently authorized paid phase.",
"Grok 4.5 and Mistral Medium 3.5 advertise max_tokens but omit max_completion_tokens metadata, so their request-cap behavior must be established by the strict smoke before any panel authorization.",
"If any smoke is truncated, reaches 3,072 output tokens, or cannot satisfy mandatory reasoning, invalidate every v3 smoke and amend the common cap symmetrically before panel data.",
"Record and accept one strict-fallback smoke for every registered model before separately authorizing panel execution."
]
}
306 changes: 168 additions & 138 deletions config/sota_v3_models.json

Large diffs are not rendered by default.

14 changes: 7 additions & 7 deletions config/sota_v3_pricing_snapshot.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
"schema_version": 2,
"contract": "sota-v3",
"contract_fingerprint": "a523bdfcebe47bbd",
"status": "catalog-frozen-public-metadata-only",
"status": "frozen",
"checked_at_utc": "2026-08-04T16:00:40Z",
"source": "Unauthenticated HTTP GET of https://openrouter.ai/api/v1/models and each selected model's https://openrouter.ai/api/v1/models/{model_id}/endpoints response; no completion or chat endpoint was called.",
"source": "Undiscounted rates frozen from the public OpenRouter catalog and rechecked against authenticated exact-route endpoint metadata before smoke authorization; no completion or chat endpoint was called.",
"currency": "USD",
"rates_are_per_token": true,
"pricing_scope": "Exact selected upstream route, not the model-level cheapest-route summary.",
Expand Down Expand Up @@ -96,12 +96,12 @@
},
"public_metadata_limitations": [
"Rates are a point-in-time public catalog snapshot and can change before authenticated preflight or smoke execution.",
"No runtime observations exist for the revised cohort and the token assumptions are pre-smoke reservations, so this file is not yet an executable spend plan.",
"Pricing metadata does not establish route privacy, retention, availability to the configured account, or successful parameter handling."
"No runtime observations exist for the revised cohort and the token assumptions remain conservative pre-smoke reservations; refresh them from accepted smoke telemetry before panel authorization.",
"Pricing metadata does not establish successful inference or parameter behavior; the strict smoke remains mandatory."
],
"spend_authorized": false,
"route_preflight_authorized": false,
"smoke_execution_authorized": false,
"spend_authorized": true,
"route_preflight_authorized": true,
"smoke_execution_authorized": true,
"panel_execution_authorized": false,
"publication_authorized": false,
"pricing_basis": "Undiscounted list rates for the exact pinned route. Promotional discounts are deliberately not reserved against: the GLM 5.2 Novita discount moved from 55.1% to 50% within hours on 2026-08-04, and a reservation computed from a promo is wrong the moment the promo ends. A live discount only ever brings the run in under reserve."
Expand Down
12 changes: 6 additions & 6 deletions config/sota_v3_publication_protocol.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"schema_version": 1,
"contract": "sota-v3",
"contract_fingerprint": "a523bdfcebe47bbd",
"status": "provisional-pre-smoke",
"status": "frozen",
"evidence_state": "pre-data-no-v3-model-smokes-or-panel-results",
"research_question": "Whether each pre-registered model-plus-compact-scaffold system trails the named pick-trader reference under the frozen sota-v3 mechanics, replicating the frozen sota-v2 finding on a new contract fingerprint.",
"panel_design": {
Expand All @@ -21,14 +21,14 @@
},
"selection_rule": "Smallest allocation in the predeclared 9-16 seed and 1-3 repeat grid whose conservative-sensitivity familywise all-reject power Wilson 95% lower bound is at least 0.80.",
"result": "16 seeds x 1 repeat (16 episodes/model) qualifies; sensitivity power 0.8488 with Wilson 95% CI [0.841645, 0.855688] and base power 0.9527 with Wilson 95% CI [0.948363, 0.95669].",
"seed_identity_status": "pending-authorized-generation"
"seed_identity_status": "frozen-private-commitment"
},
"output_policy": {
"status": "provisional-pre-smoke-validation",
"status": "frozen-native-reasoning-cap",
"output_token_cap": 4096,
"cap_pressure_threshold_tokens": 3072,
"fallback_output_token_cap": 8192,
"reasoning_policy": "catalog-pinned-pending-live-route-verification",
"reasoning_policy": "catalog-pinned-pending-strict-smoke-behavior-verification",
"amendment_rule": "The common 4,096-token cap is frozen for the first strict smoke of every route. If any smoke is truncated, reaches 3,072 output tokens, or cannot satisfy mandatory minimum reasoning, invalidate all v3 smokes, amend the cap once before panel data, and re-smoke the entire family. Scores and apparent model quality are never cap-selection inputs."
},
"rerun_policy": {
Expand Down Expand Up @@ -77,13 +77,13 @@
]
},
"ranking_rule": "No model tiers or ordinal ranking; only each registered model's predeclared contrast versus pick-trader is supported.",
"remaining_blocker": "Seed identity is not frozen. The 16-seed private panel must be generated and its salted commitment hash committed under separate owner authorization before any provider call."
"remaining_blocker": "No statistical-design blocker remains before smoke. Panel execution stays blocked until all ten strict smokes are accepted under the frozen route, cap, seed, and failure policy."
},
"budget_policy": {
"provider": "openrouter",
"cost_estimate_artifact": "results/analysis/sota-v3-pre-smoke-cost-estimate.json",
"operator_must_pass_max_spend_usd": true,
"spend_authorized": false,
"spend_authorized": true,
"operator_ceiling_usd": 150.0,
"operator_ceiling_basis": "Owner-set hard cap, raised 120.00 -> 150.00 on 2026-08-04. The $120 figure was chosen against a $119.76 reservation that turned out to depend on 50%-off promotional rates on openai/gpt-5.6-luna and z-ai/glm-5.2; pinning undiscounted list rates moved the reservation to $127.29 and put the committed plan over its own ceiling. $150.00 clears the current reservation with headroom for a further route substitution or list-price move without another ceiling decision. Enforced by the runner ahead of the cell loop: --max-spend-usd above this value is rejected before any endpoint probe or child process. Projected actual spend is ~$35-45 from July smoke telemetry reprojected at current rates (models emit 48-640 output tokens per decision against a 4,096-token reservation), so this is a backstop, not a forecast. Raising it again is a deliberate edit here."
},
Expand Down
Loading