From 929c3a2d0ab253e1c33a5c1ce03cb01b7a783c9e Mon Sep 17 00:00:00 2001 From: Peyton-Spencer Date: Tue, 28 Jul 2026 12:03:17 -0400 Subject: [PATCH 1/2] fix(inference): normalize harmless request fields instead of 400ing the run The v7 chat gate refused any top-level field outside a flat allowlist, and did so 43 lines before begin_inference_request -- so no inference_requests row was written and the rejections were invisible in telemetry. The error body did not name the offending key either. `Cooking` (rank 15 on v1) hit this by adding one line that sent `reasoning: {"effort": "medium"}`. The platform already stamps `{"effort": "medium", "exclude": True}` on every v7 request, so the miner was refused for redundantly requesting exactly what it was being given. ~81 of ~480 chat calls died per run; v3 shipped byte-identical and still failed. Split the allowlist into three sets by what a field can actually do: - pinned (model, max_tokens/max_completion_tokens, n, reasoning): accepted, then overwritten with the ticket's own value. - dropped (best_of, reasoning_effort, service_tier, user, metadata, store, stream_options, response_format, logprobs, top_logprobs, logit_bias): accepted, then stripped before the provider call, each for a stated reason. - forwarded (messages, temperature, top_p, seed, stop, tools, tool_choice, parallel_tool_calls, frequency_penalty, presence_penalty). Unknown keys stay fail-closed -- OpenRouter's surface is additive and a field invented later is exactly the one that might buy compute -- but the refusal now names them. max_tokens over-asks and disagreeing aliases now clamp downward rather than kill the run. Anti-cheat is unchanged and now testable as a partition: nothing is accepted without being pinned, dropped, or deliberately forwarded, so no request can obtain compute, a model, or a reasoning effort its ticket did not grant. `effort: "high"` is served as medium; so is `effort: "none"`, which would otherwise let an agent opt out of v7's mandatory reasoning. Co-Authored-By: Claude Opus 5 --- ditto/api_server/endpoints/inference.py | 230 ++++++++-- .../api_server/endpoints/test_inference.py | 190 +++++++- .../test_inference_reasoning_integration.py | 431 ++++++++++++++++++ 3 files changed, 807 insertions(+), 44 deletions(-) create mode 100644 ditto/tests/integration/test_inference_reasoning_integration.py diff --git a/ditto/api_server/endpoints/inference.py b/ditto/api_server/endpoints/inference.py index 8a2d949d..ed78de6a 100644 --- a/ditto/api_server/endpoints/inference.py +++ b/ditto/api_server/endpoints/inference.py @@ -439,11 +439,100 @@ def _provider_preferences( return preferences -_ALLOWED_REQUEST_FIELDS = { +# How this gate treats an unexpected request field, and why it changed +# --------------------------------------------------------------------- +# +# This used to be one flat allowlist, and anything outside it was a 400 raised +# *before* ``begin_inference_request``. That combination was quietly expensive: +# no ``inference_requests`` row is written on that path, so the rejections were +# invisible in telemetry, and the error body did not name the offending key. A +# rank-15 miner (``Cooking``) burned three submissions adding a single line that +# sent ``reasoning: {"effort": "medium"}`` -- the very value this platform was +# already stamping on their request for them (``benchmark_reasoning`` below) -- +# and had no way to discover which field killed the run. +# +# The philosophy is now the broker's: normalise rather than refuse, wherever +# refusing buys no safety. ``dittobench-api`` already overwrites the caller's +# ``model`` instead of refusing a request that names one, and pins its own +# embedding model and nonce. The same reasoning applies field by field here, so +# the allowlist is split into three sets by what the field can actually do: +# +# ``_PINNED_REQUEST_FIELDS`` accepted, then *overwritten* with the value the +# ticket grants, in ``_locked_upstream_payload``. +# The caller's value never reaches the provider, so +# asking for more than the ticket granted is inert +# rather than fatal. +# ``_DROPPED_REQUEST_FIELDS`` accepted, then removed before the provider call. +# These have no bearing on the answer the harness +# gets, and each one is dropped for a stated reason +# (cost lever, egress channel, determinism, or dead +# weight against a contract we already refuse). +# ``_FORWARDED_REQUEST_FIELDS`` validated and passed through unchanged. +# +# Anything still outside all three is refused -- and refused *by name*. Unknown +# stays fail-closed on purpose: OpenRouter's request surface is additive, and a +# field invented after this code was written is exactly the one that might buy +# compute. The fix for the incident above is not "accept everything", it is +# "accept everything harmless, and say which key was not". + +# Route identity or a compute lever. Accepted and then replaced wholesale with +# the ticket's own value; see ``_locked_upstream_payload``. +_PINNED_REQUEST_FIELDS = { + # Substituted by ``_locked_grant_model``: the grant pins the model. "model", - "messages", + # Clamped down to the ticket ceiling by ``_output_token_limit``. "max_tokens", "max_completion_tokens", + # Forced to 1. Extra completions are extra billed generations. + "n", + # Nested reasoning control. Replaced with ``benchmark_reasoning(model)``, so + # ``{"effort": "high"}`` is served as the pinned ``medium``. + "reasoning", +} + +# Accepted, then stripped before the request leaves this process. +_DROPPED_REQUEST_FIELDS = { + # Buys N server-side generations and bills for all of them. Dropping is + # strictly cheaper than the ticket already allows. + "best_of", + # The *flat* sibling of ``reasoning`` -- a different key, and a harness may + # plausibly send either. Dropped rather than pinned: the pinned effort + # already arrives via ``reasoning``, and forwarding both leaves provider-side + # precedence between them undefined. + "reasoning_effort", + # Selects a provider priority/cost tier. A compute lever the ticket did not + # grant. + "service_tier", + # Free-form caller-controlled strings shipped verbatim to a third party. + # The harness runs sandboxed with no egress; forwarding these would hand it + # one. Neither affects the completion. + "user", + "metadata", + # Asks the provider to retain the completion, contradicting the + # ``data_collection: "deny"`` / ``zdr: true`` preferences this proxy pins. + "store", + # Only meaningful alongside ``stream: true``, which this lane refuses. + "stream_options", + # Constrained/grammar decoding materially changes the output distribution + # and its latency profile, so two agents differing only in this field are no + # longer comparable runs. Dropped for benchmark comparability, not safety. + "response_format", + # Do not change sampling, but inflate the response body hard (``top_logprobs`` + # carries up to 20 alternatives per token) against ``response_body_bytes``, + # and ``_public_provider_response`` strips them, so the caller could never + # read them back regardless. + "logprobs", + "top_logprobs", + # Unbounded in size, and keyed by *tokenizer-specific* token ids. Because the + # served model is the ticket's rather than the one the caller named, a bias + # map built against another tokenizer silently biases unrelated tokens. + # Dropped for correctness and comparability. + "logit_bias", +} + +# Validated and passed through untouched. +_FORWARDED_REQUEST_FIELDS = { + "messages", "temperature", "top_p", "seed", @@ -451,18 +540,35 @@ def _provider_preferences( "tools", "tool_choice", "parallel_tool_calls", - "n", - "best_of", + # Same class as ``temperature``/``top_p``/``seed``, which this lane has + # always forwarded: per-request sampling knobs that cost nothing extra and + # that the miner's own agent design is entitled to choose. Silently dropping + # a deliberately-set sampling knob would change an agent's behaviour behind + # its back -- the exact failure mode this change exists to remove. + "frequency_penalty", + "presence_penalty", + # Refused when true (see below), but must parse as a known field so the + # refusal can explain itself. "stream", } +_ALLOWED_REQUEST_FIELDS = ( + _PINNED_REQUEST_FIELDS | _DROPPED_REQUEST_FIELDS | _FORWARDED_REQUEST_FIELDS +) + def _validate_request_schema(payload: dict[str, Any]) -> None: """Accept only the text/tool subset used by the benchmark harness.""" unknown = set(payload) - _ALLOWED_REQUEST_FIELDS if unknown: - raise HTTPException(status_code=400, detail="unsupported inference parameter") - for name in ("temperature", "top_p"): + # Name the keys. A harness author reading this over stderr has no other + # way to learn which of their fields was the problem, and the broker now + # forwards this detail verbatim (dittobench-api ``forwardChatCompletion``). + raise HTTPException( + status_code=400, + detail=f"unsupported inference parameter: {', '.join(sorted(unknown))}", + ) + for name in ("temperature", "top_p", "frequency_penalty", "presence_penalty"): value = payload.get(name) if value is not None and ( not isinstance(value, (int, float)) @@ -470,6 +576,10 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: or not math.isfinite(value) ): raise HTTPException(status_code=400, detail=f"invalid {name}") + for name in ("frequency_penalty", "presence_penalty"): + penalty = payload.get(name) + if penalty is not None and not -2 <= penalty <= 2: + raise HTTPException(status_code=400, detail=f"invalid {name}") temperature = payload.get("temperature") if temperature is not None and not 0 <= temperature <= 2: raise HTTPException(status_code=400, detail="invalid temperature") @@ -493,16 +603,31 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: ) ): raise HTTPException(status_code=400, detail="invalid stop") - for name in ("parallel_tool_calls", "stream"): + for name in ("parallel_tool_calls", "stream", "store"): if name in payload and not isinstance(payload[name], bool): raise HTTPException(status_code=400, detail=f"invalid {name}") - n = payload.get("n", 1) - if not isinstance(n, int) or isinstance(n, bool) or n != 1: + # One of the few genuine refusals left. ``stream`` is not normalised to + # false because this lane answers with a single non-streaming JSON body: a + # caller that asked for SSE and silently received one would fail parsing it, + # which is a worse and far less legible outcome than being told. + if payload.get("stream") not in {None, False}: raise HTTPException( - status_code=400, detail="multiple completions are not supported" + status_code=400, + detail=( + "unsupported inference parameter: stream " + "(this lane answers with a single non-streaming response)" + ), ) - if "best_of" in payload: - raise HTTPException(status_code=400, detail="best_of is not supported") + # ``n`` and ``best_of`` are pinned/dropped in ``_locked_upstream_payload`` + # rather than refused: the provider is asked for exactly one completion no + # matter what arrives here, so an over-ask cannot buy a second generation. + # A malformed value is still a malformed request and is still named. + for name in ("n", "best_of"): + value = payload.get(name) + if value is not None and ( + not isinstance(value, int) or isinstance(value, bool) or value < 1 + ): + raise HTTPException(status_code=400, detail=f"invalid {name}") tool_choice = payload.get("tool_choice") if tool_choice is not None: valid_named_choice = ( @@ -586,31 +711,39 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: def _output_token_limit(payload: dict[str, Any], maximum: int) -> int: - """Normalize OpenAI's aliases without allowing one to bypass the other.""" + """Normalize OpenAI's aliases without allowing one to bypass the other. + + Both an over-ask and a disagreement between the two aliases are resolved + *downward* rather than refused. Asking for more output than the ticket + grants is the single most likely way for an ordinary OpenAI-compatible + harness to trip this gate, and killing the run over it buys nothing: the + clamp already guarantees the ticket's ceiling holds, which is the only + property that was ever at stake. A caller cannot obtain more output by + asking for more, by asking twice, or by disagreeing with itself. + """ max_tokens_value = payload.get("max_tokens") max_completion_tokens = payload.get("max_completion_tokens") - if ( - max_tokens_value is not None - and max_completion_tokens is not None - and max_tokens_value != max_completion_tokens + for name, candidate in ( + ("max_tokens", max_tokens_value), + ("max_completion_tokens", max_completion_tokens), ): - raise HTTPException(status_code=400, detail="conflicting output token limits") - value = ( - max_tokens_value - if max_tokens_value is not None - else max_completion_tokens - if max_completion_tokens is not None - else maximum - ) - if ( - not isinstance(value, int) - or isinstance(value, bool) - or not 1 <= value <= maximum - ): - raise HTTPException( - status_code=400, detail="max_tokens exceeds the ticket limit" - ) - return value + if candidate is not None and ( + not isinstance(candidate, int) or isinstance(candidate, bool) + ): + raise HTTPException(status_code=400, detail=f"invalid {name}") + # A non-positive ceiling is a malformed request rather than an over-ask, + # and there is no conservative direction to normalise it in: clamping it + # up would hand the caller output it did not ask for. + if candidate is not None and candidate < 1: + raise HTTPException(status_code=400, detail=f"invalid {name}") + requested = [ + value + for value in (max_tokens_value, max_completion_tokens) + if value is not None + ] + # Aliases that disagree resolve to the smaller, so neither can raise the + # other. Absent both, the ticket ceiling is the default it always was. + return min(*requested, maximum) if requested else maximum def _locked_grant_model(grant: Any, *, requested: str, config: Any) -> str: @@ -648,14 +781,30 @@ def _locked_grant_model(grant: Any, *, requested: str, config: Any) -> str: def _locked_upstream_payload( payload: dict[str, Any], *, model: str, max_tokens: int ) -> dict[str, Any]: - """Force consensus model/reasoning fields before provider routing.""" + """Force consensus model/reasoning fields before provider routing. + + This is the half of the schema policy that makes permissiveness safe. Every + field the gate now *accepts* rather than refuses is either replaced here + with the ticket's own value or removed here before the request leaves the + process, so accepting it cannot buy the caller compute, a model, or a + reasoning effort its ticket did not grant. + """ upstream = dict(payload) + # Accepted at the door purely so the caller is not killed for sending them; + # none of them reaches the provider. See ``_DROPPED_REQUEST_FIELDS``. + for field in _DROPPED_REQUEST_FIELDS: + upstream.pop(field, None) + # Collapsed into the single clamped ``max_tokens`` computed by + # ``_output_token_limit``; forwarding both would re-open the alias bypass. upstream.pop("max_completion_tokens", None) - upstream.pop("best_of", None) upstream["model"] = model upstream["max_tokens"] = max_tokens upstream["n"] = 1 upstream["stream"] = False + # Unconditional assignment, not a default: the caller's value is discarded + # whatever its shape, so ``{"effort": "high"}`` and a malformed reasoning + # object are both served as the pinned contract. On a bench version with no + # reasoning contract the field is removed outright rather than passed on. reasoning = benchmark_reasoning(model) if reasoning is None: upstream.pop("reasoning", None) @@ -897,8 +1046,13 @@ async def proxy_chat_completions( payload = json.loads(body) except (json.JSONDecodeError, UnicodeDecodeError) as error: raise HTTPException(status_code=400, detail="invalid JSON request") from error - if not isinstance(payload, dict) or payload.get("stream") not in {None, False}: - raise HTTPException(status_code=400, detail="streaming is not supported") + if not isinstance(payload, dict): + raise HTTPException( + status_code=400, detail="inference request must be a JSON object" + ) + # Every field-level decision, including the streaming refusal, lives in one + # function now. It used to be split across here and the schema check, which + # is how the two most common refusals ended up with two different wordings. _validate_request_schema(payload) requested_model = payload.get("model") if not isinstance(requested_model, str): diff --git a/ditto/tests/api_server/endpoints/test_inference.py b/ditto/tests/api_server/endpoints/test_inference.py index aa1fc098..fe5ce929 100644 --- a/ditto/tests/api_server/endpoints/test_inference.py +++ b/ditto/tests/api_server/endpoints/test_inference.py @@ -11,6 +11,10 @@ from ditto.api_models.inference import InferenceExchangeRequest, InferenceGrantOffer from ditto.api_server.endpoints.inference import ( + _ALLOWED_REQUEST_FIELDS, + _DROPPED_REQUEST_FIELDS, + _FORWARDED_REQUEST_FIELDS, + _PINNED_REQUEST_FIELDS, _bounded_provider_cost, _estimated_tokens, _exchange_message, @@ -212,11 +216,31 @@ def test_embedding_contract_is_exact_and_response_is_sanitized() -> None: def test_output_token_alias_cannot_bypass_ticket_limit() -> None: - with pytest.raises(HTTPException): + # Disagreeing aliases resolve downward, so neither can raise the other. + assert ( _output_token_limit({"max_tokens": 1, "max_completion_tokens": 999_999}, 8192) - with pytest.raises(HTTPException): - _output_token_limit({"max_completion_tokens": 8193}, 8192) + == 1 + ) + assert ( + _output_token_limit({"max_tokens": 999_999, "max_completion_tokens": 1}, 8192) + == 1 + ) + # An over-ask is clamped to the ticket ceiling instead of killing the run. + assert _output_token_limit({"max_completion_tokens": 8193}, 8192) == 8192 + assert _output_token_limit({"max_tokens": 10**9}, 8192) == 8192 assert _output_token_limit({"max_completion_tokens": 32}, 8192) == 32 + assert _output_token_limit({}, 8192) == 8192 + # A non-positive or non-integer ceiling has no conservative normalisation + # and stays a named refusal. + for key, bad in ( + ("max_tokens", {"max_tokens": 0}), + ("max_tokens", {"max_tokens": -1}), + ("max_completion_tokens", {"max_completion_tokens": True}), + ("max_completion_tokens", {"max_completion_tokens": "many"}), + ): + with pytest.raises(HTTPException) as invalid: + _output_token_limit(bad, 8192) + assert str(invalid.value.detail) == f"invalid {key}" @pytest.mark.parametrize( @@ -225,7 +249,8 @@ def test_output_token_alias_cannot_bypass_ticket_limit() -> None: {"models": ["attacker/model"]}, {"plugins": [{"id": "web"}]}, {"provider": {"allow_fallbacks": True}}, - {"reasoning": {"effort": "high"}}, + {"transforms": ["middle-out"]}, + {"route": "fallback"}, { "messages": [ { @@ -319,8 +344,11 @@ def test_proxy_schema_rejects_non_text_content_parts(content: object) -> None: {"parallel_tool_calls": 1}, {"stream": "false"}, {"n": True}, - {"n": 2}, - {"best_of": 1}, + {"n": 0}, + {"best_of": 0}, + {"frequency_penalty": 2.5}, + {"presence_penalty": float("inf")}, + {"store": "yes"}, {"tool_choice": {"type": "function", "function": {"name": ""}}}, {"tool_choice": {"type": "function", "function": {"name": "x", "x": 1}}}, ], @@ -391,6 +419,156 @@ def test_v7_upstream_profile_pins_medium_reasoning_without_changing_v6() -> None assert "reasoning" not in v6 +def test_caller_reasoning_is_accepted_then_overwritten_with_the_pinned_effort() -> None: + """The rejection that cost `Cooking` three submissions is now a normalisation. + + The platform already stamps `{"effort": "medium", "exclude": True}` on every + v7 request (`benchmark_reasoning`), so the miner was 400'd for redundantly + requesting exactly what it was being given. Accepting the field is safe for + the same reason it was pointless to refuse it: the value is replaced. + """ + for effort in ({"effort": "high"}, {"effort": "low"}, {"max_tokens": 100_000}, {}): + payload = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "reasoning": effort, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + + # `reasoning_effort` is the *flat* OpenAI sibling and a different key. It is + # accepted so no harness dies on it, and dropped so it cannot compete with + # the pinned nested value at the provider. + flat = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "reasoning_effort": "high", + } + _validate_request_schema(flat) + upstream = _locked_upstream_payload( + flat, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert "reasoning_effort" not in upstream + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + + +def test_harmless_client_defaults_are_accepted_and_never_reach_the_provider() -> None: + """Fields mainstream OpenAI clients emit by default cost nobody a run. + + Each is accepted at the door and stripped before the upstream call, so the + harness is not killed for sending it and the provider never sees it. + """ + payload: dict[str, object] = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "user": "agent-7", + "metadata": {"run": "abc"}, + "store": False, + "stream_options": {"include_usage": True}, + "response_format": {"type": "json_object"}, + "logprobs": True, + "top_logprobs": 5, + "logit_bias": {"1234": -100}, + "service_tier": "priority", + "best_of": 4, + "n": 3, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + for dropped in ( + "user", + "metadata", + "store", + "stream_options", + "response_format", + "logprobs", + "top_logprobs", + "logit_bias", + "service_tier", + "best_of", + ): + assert dropped not in upstream, dropped + # `n` is pinned rather than dropped: exactly one billed generation. + assert upstream["n"] == 1 + + +def test_sampling_knobs_the_miner_owns_are_forwarded_unchanged() -> None: + """Same class as temperature/top_p/seed, which this lane always forwarded. + + Dropping a deliberately-set sampling knob would silently change an agent's + behaviour behind its back -- the exact failure mode this change removes. + """ + payload = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "frequency_penalty": 0.5, + "presence_penalty": -0.25, + "temperature": 0.0, + "seed": 42, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert upstream["frequency_penalty"] == 0.5 + assert upstream["presence_penalty"] == -0.25 + assert upstream["temperature"] == 0.0 + assert upstream["seed"] == 42 + + +def test_unsupported_parameter_error_names_every_offending_key() -> None: + """A miner must be able to discover which field broke the run.""" + with pytest.raises(HTTPException) as refused: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "plugins": [{"id": "web"}], + "models": ["attacker/model"], + } + ) + assert refused.value.status_code == 400 + detail = str(refused.value.detail) + assert "unsupported inference parameter" in detail + # Sorted, so the message is stable across dict ordering. + assert "models, plugins" in detail + + with pytest.raises(HTTPException) as streaming: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + } + ) + assert "stream" in str(streaming.value.detail) + + +def test_every_accepted_field_is_pinned_dropped_or_deliberately_forwarded() -> None: + """The anti-cheat property, stated as a partition over the allowlist. + + Nothing may be accepted without a decided fate. A field that is neither + pinned nor dropped is forwarded to the provider verbatim, so this partition + is what makes "accept it" reviewable one field at a time. + """ + assert ( + _PINNED_REQUEST_FIELDS | _DROPPED_REQUEST_FIELDS | _FORWARDED_REQUEST_FIELDS + == _ALLOWED_REQUEST_FIELDS + ) + assert not _PINNED_REQUEST_FIELDS & _DROPPED_REQUEST_FIELDS + assert not _PINNED_REQUEST_FIELDS & _FORWARDED_REQUEST_FIELDS + assert not _DROPPED_REQUEST_FIELDS & _FORWARDED_REQUEST_FIELDS + # Nothing that selects a model, a route, or server-side network egress may + # be accepted at any tier. + for escape in ("models", "provider", "plugins", "transforms", "route"): + assert escape not in _ALLOWED_REQUEST_FIELDS + + def test_caller_shape_rejections_do_not_cool_shared_provider_route() -> None: assert not _provider_rejection_is_route_observable(400) assert not _provider_rejection_is_route_observable(422) diff --git a/ditto/tests/integration/test_inference_reasoning_integration.py b/ditto/tests/integration/test_inference_reasoning_integration.py new file mode 100644 index 00000000..d2602603 --- /dev/null +++ b/ditto/tests/integration/test_inference_reasoning_integration.py @@ -0,0 +1,431 @@ +"""End-to-end proof that a caller-supplied ``reasoning`` effort is pinned, not refused. + +This is the regression test for the incident that motivated the schema change. +``Cooking`` (rank 15 on v1) added one line in v2 that sent +``reasoning: {"effort": "medium"}`` on every chat call. ``reasoning`` was absent +from the request allowlist, so the gate answered 400 forty-three lines *before* +``begin_inference_request`` -- no ``inference_requests`` row, nothing in +telemetry, and an error body that never named the field. Roughly 81 of ~480 chat +calls per run died that way, the run failed v7's complete-usage check, and the +miner burned three submissions on it. + +The irony that decided the fix: ``benchmark_reasoning`` already stamps +``{"effort": "medium", "exclude": True}`` on every v7 request. The miner was +rejected for redundantly asking for precisely what they were already being +given. + +So the property under test is not "``reasoning`` is allowed". It is the stronger +one that makes allowing it safe: the caller's value is *discarded and replaced*, +so a request asking for ``high`` is served, metered, and billed as ``medium``. +The anti-cheat property and the availability fix are the same mechanism. +""" + +from __future__ import annotations + +import base64 +import json +from datetime import UTC, datetime, timedelta +from typing import Any +from uuid import UUID, uuid4 + +import httpx +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from ditto.api_models.agent_status import AgentStatus +from ditto.api_models.ticket_status import TicketStatus +from ditto.api_server.config import InferenceProxyConfig +from ditto.api_server.endpoints.inference import _proxy_message, proxy_chat_completions +from ditto.api_server.inference_routing import ( + AGGREGATE_CALIBRATION_SAMPLES, + AGGREGATE_PROVIDER, + V7_MODEL, + aggregate_profile_revision, +) +from ditto.db.models import ( + Agent, + InferenceGrant, + InferenceProviderRoute, + InferenceRequest, + InferenceRoutingPolicy, + ValidatorTicket, +) +from ditto.db.queries.inference import ( + activate_inference_grant, + ensure_inference_grant, +) + +pytestmark = pytest.mark.integration + +_MAX_OUTPUT_TOKENS = 1024 + + +def _config() -> InferenceProxyConfig: + return InferenceProxyConfig( + enabled=True, + required=False, + public_base_url="https://platform.example", + openrouter_api_key="test-key", + upstream_url="https://openrouter.ai/api/v1/chat/completions", + allowed_models=(V7_MODEL,), + provider="openrouter", + routing_mode="aggregate_throughput", + request_budget=1000, + token_budget=1_000_000, + embedding_upstream_url="https://openrouter.ai/api/v1/embeddings", + embedding_model="perplexity/pplx-embed-v1-0.6b", + embedding_profile="dittobench-v7-openrouter-pplx-embed-v1-0.6b-768-v1", + embedding_provider="Perplexity", + embedding_dimensions=768, + embedding_request_budget=100_000, + embedding_token_budget=1_000_000_000, + embedding_per_ticket_concurrency=64, + embedding_per_validator_concurrency=256, + embedding_global_concurrency=1024, + embedding_per_ticket_requests_per_minute=10_000, + embedding_per_validator_requests_per_minute=40_000, + embedding_global_requests_per_minute=100_000, + embedding_request_body_bytes=1 << 20, + embedding_response_body_bytes=16 << 20, + per_ticket_concurrency=64, + per_validator_concurrency=256, + global_concurrency=1024, + per_ticket_requests_per_minute=10_000, + per_validator_requests_per_minute=40_000, + global_requests_per_minute=100_000, + request_body_bytes=256 << 10, + response_body_bytes=1 << 20, + timeout_seconds=10, + max_output_tokens=_MAX_OUTPUT_TOKENS, + ) + + +class _State: + """The three attributes ``proxy_chat_completions`` reads off ``app.state``.""" + + def __init__(self, *, config: Any, session_maker: Any, client: httpx.AsyncClient): + class _Config: + inference_proxy = config + + self.config = _Config() + self.session_maker = session_maker + self.inference_client = client + + +class _App: + def __init__(self, state: _State) -> None: + self.state = state + + +async def _seed_v7_grant( + maker: Any, *, config: InferenceProxyConfig, public_key: str +) -> tuple[UUID, str, int]: + """One agent + v7 ticket + a grant minted through the real v7 route path.""" + now = datetime.now(UTC) + async with maker() as session, session.begin(): + # A calibrated aggregate route, so ``select_route`` admits the v7 mint + # exactly as it does in production rather than being bypassed. + session.add( + InferenceRoutingPolicy( + model=V7_MODEL, + enabled=True, + speed_weight=1.0, + cost_weight=0.0, + exploration_weight=0.0, + exploration_ticket_budget=0, + min_tool_accuracy=0.0, + min_composite=0.0, + min_calibration_samples=1, + max_error_rate=1.0, + max_timeout_rate=1.0, + cooldown_seconds=1, + ewma_alpha=0.3, + updated_at=now, + ) + ) + session.add( + InferenceProviderRoute( + model=V7_MODEL, + provider=AGGREGATE_PROVIDER, + profile_revision=aggregate_profile_revision(V7_MODEL), + status="healthy", + calibration_status="eligible", + calibration_tool_accuracy=1.0, + calibration_composite=1.0, + calibration_sample_count=AGGREGATE_CALIBRATION_SAMPLES, + calibration_manifest_sha256="0" * 64, + discovered_at=now, + ewma_error_rate=0.0, + ewma_timeout_rate=0.0, + sample_count=0, + selected_ticket_count=0, + exploration_ticket_count=0, + updated_at=now, + ) + ) + await session.flush() + agent = Agent( + agent_id=uuid4(), + miner_hotkey="5FRZRm3R6ESJ4TtxaQ51vxk99hdkFdchUHFZVNSfYAUbehyR", + name="reasoning-pin", + sha256=uuid4().hex + uuid4().hex, + status=AgentStatus.EVALUATING, + created_at=now, + ) + ticket = ValidatorTicket( + agent_id=agent.agent_id, + validator_hotkey="validator-reasoning", + slot_id="slot-0", + status=TicketStatus.ISSUED, + issued_at=now, + deadline=now + timedelta(minutes=20), + bench_version=7, + attempt_count=1, + ) + session.add_all([agent, ticket]) + await session.flush() + grant = await ensure_inference_grant(session, ticket=ticket, config=config) + assert grant is not None + activated = await activate_inference_grant( + session, + grant_id=grant.grant_id, + validator_hotkey="validator-reasoning", + broker_public_key=public_key, + now=now, + config=config, + ) + assert activated is not None + live = activated[0] + # Minted through the real v7 path, so these are the platform's own + # values rather than anything this test arranged. + assert live.bench_version == 7 + assert live.allowed_models == [V7_MODEL] + assert live.route_provider == AGGREGATE_PROVIDER + return live.grant_id, activated[1], live.generation + + +def _signed_request( + *, + app: _App, + grant_id: UUID, + bearer: str, + generation: int, + private: Ed25519PrivateKey, + body: bytes, +) -> dict[str, Any]: + """Header kwargs for one authenticated proxy call over ``body``.""" + from starlette.requests import Request + + nonce = uuid4() + requested_at = datetime.now(UTC) + proof = private.sign( + _proxy_message( + grant_id=grant_id, + generation=generation, + nonce=nonce, + requested_at=requested_at, + body=body, + ) + ) + + async def receive() -> dict[str, Any]: + return {"type": "http.request", "body": body, "more_body": False} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/api/v1/inference/chat/completions", + "headers": [], + "app": app, + }, + receive, + ) + return { + "request": request, + "x_ditto_grant": grant_id, + "x_ditto_generation": generation, + "x_ditto_nonce": nonce, + "x_ditto_requested_at": requested_at, + "x_ditto_proof": base64.urlsafe_b64encode(proof).decode().rstrip("="), + "authorization": f"Bearer {bearer}", + } + + +@pytest.mark.asyncio +async def test_caller_reasoning_effort_is_accepted_pinned_to_medium_and_charged( + session_maker: async_sessionmaker[Any], +) -> None: + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + seen: list[dict[str, Any]] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": V7_MODEL, + # Opt-in router metadata; ``_upstream_provider`` requires exactly + # one selected endpoint for trusted route telemetry. + "openrouter_metadata": { + "endpoints": { + "available": [{"provider": "Fireworks", "selected": True}] + } + }, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "ok"}, + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 17, + "total_tokens": 48, + # Aggregate routing derives trusted cost from the provider's + # own figure; without it the call books as usage-unavailable. + "cost": 0.000123, + }, + }, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + + # Exactly the shape `Cooking` v2/v3 emits, but asking for the *strongest* + # effort rather than the pinned one -- the adversarial direction. + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "hello"}], + "temperature": 0.0, + "seed": 42, + "reasoning": {"effort": "high"}, + } + ).encode() + + try: + response = await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + # 1. Accepted. This is the byte-for-byte request that used to 400. + assert response.status_code == 200 + + # 2. Reached the provider with the *pinned* value, not the caller's "high". + assert len(seen) == 1 + assert seen[0]["reasoning"] == {"effort": "medium", "exclude": True} + assert seen[0]["model"] == V7_MODEL + assert seen[0]["max_tokens"] == _MAX_OUTPUT_TOKENS + assert seen[0]["n"] == 1 + assert seen[0]["stream"] is False + # The caller's own sampling choices survive untouched. + assert seen[0]["temperature"] == 0.0 + assert seen[0]["seed"] == 42 + + # 3. Charged, and charged as the medium call that actually ran. The request + # reached the ledger at all -- the old rejection never wrote a row, which + # is why this failure mode was invisible in telemetry. + async with session_maker() as session: + row = ( + await session.scalars( + select(InferenceRequest).where(InferenceRequest.grant_id == grant_id) + ) + ).one() + assert row.status == "completed" + assert row.model == V7_MODEL + assert row.prompt_tokens == 31 + assert row.completion_tokens == 17 + grant = await session.get(InferenceGrant, grant_id) + assert grant is not None + assert grant.request_count == 1 + assert grant.prompt_tokens == 31 + assert grant.completion_tokens == 17 + assert grant.cost_microusd == 123 + + +@pytest.mark.asyncio +async def test_unknown_request_field_names_itself_in_the_error( + session_maker: async_sessionmaker[Any], +) -> None: + """The legibility backstop: a refusal must say which key it refused. + + Whatever the allowlist admits, something is eventually refused, and the + miner's only channel is the harness's stderr. ``Cooking`` could not discover + ``reasoning`` was the problem because this string used to be the bare + ``"unsupported inference parameter"``. + """ + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + async def handler(_: httpx.Request) -> httpx.Response: # pragma: no cover + raise AssertionError("refused requests must never reach the provider") + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "hello"}], + "plugins": [{"id": "web"}], + "transforms": ["middle-out"], + } + ).encode() + + from fastapi import HTTPException + + try: + with pytest.raises(HTTPException) as refused: + await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + assert refused.value.status_code == 400 + detail = str(refused.value.detail) + assert "plugins" in detail + assert "transforms" in detail From efe28a8644bc354154ca01dc75b10a6cdc664704 Mon Sep 17 00:00:00 2001 From: Peyton-Spencer Date: Tue, 28 Jul 2026 12:54:36 -0400 Subject: [PATCH 2/2] fix(inference): default to forwarding request fields, pin only the grant Flips the posture: forwarding needs no justification, every pin and drop carries a concrete reason. This is a retrieval-agent competition, not a prompt-shape competition -- a harness's sampling and observability choices are its own. The rule for a drop: stripping is acceptable only when it changes neither what the model produces nor what the harness can observe. Silently discarding a knob a miner set is the Cooking bug with extra steps -- they ask for a JSON schema, get prose, their parser fails, nothing names the cause. Provider support was established empirically against the pinned v7 model rather than from docs. Two findings drove the placements: - OpenRouter EXCLUDES an endpoint lacking a requested parameter rather than routing to it and failing. Pinning order:[amazon-bedrock] (no response_format) returns 404 with the field and succeeds without it, and under the v7 aggregate route it lands on one of eight supporting providers. So forwarding response_format cannot manufacture a 400. - reasoning_effort HARD-FAILS: OpenRouter 400s when it disagrees with the pinned reasoning.effort. It moves from drop to pin for that reason. Moved to forwarded: response_format (+structured_outputs), logprobs, logit_bias, top_k, min_p, top_a, repetition_penalty, prediction, verbosity. json_schema was verified to conform on all five supporting providers WITH the pinned reasoning block, so the feared reasoning/ structured-output interaction does not exist for this model. logprobs is now plumbed through _public_provider_response; forwarding a request field whose response is stripped would be a silent no-op. top_logprobs is refused as a measured capability limit: ~80 bytes per token per alternative puts top_logprobs=3 at ~2.65 MiB and =20 at ~13.3 MiB against the 2 MiB response cap, and a breach raises a 502 that is attributed to the PROVIDER and cools the shared route -- so a caller's own choice would be charged to infrastructure. No non-zero value is safe at the 8192-token ceiling. logprobs alone is ~0.81 MiB and fits. Pin now also covers include_reasoning, service_tier, usage and prompt_cache_key. Drop is down to user/metadata/safety_identifier/store/ stream_options -- all zero-effect on the completion. Refusals are named with reasons, including the two capability limits. Co-Authored-By: Claude Opus 5 --- ditto/api_server/endpoints/inference.py | 306 +++++++++++++----- .../api_server/endpoints/test_inference.py | 260 +++++++++++++-- .../test_inference_reasoning_integration.py | 131 ++++++++ 3 files changed, 583 insertions(+), 114 deletions(-) diff --git a/ditto/api_server/endpoints/inference.py b/ditto/api_server/endpoints/inference.py index ed78de6a..923cee7f 100644 --- a/ditto/api_server/endpoints/inference.py +++ b/ditto/api_server/endpoints/inference.py @@ -395,6 +395,16 @@ def _public_provider_response(payload: dict[str, Any]) -> dict[str, Any]: if usage is None: raise HTTPException(status_code=502, detail="invalid provider response") prompt_tokens, completion_tokens, _ = usage + # ``logprobs`` is a forwarded request field, so it has to come back or + # forwarding it would be a silent no-op -- the caller sets the flag, pays for + # the larger provider response, and receives nothing. Passed through as the + # provider sent it; only the two providers serving this model that support + # logprobs populate it, and the rest return null, which is the honest + # answer. ``top_logprobs`` is refused at the door, so the per-token + # alternatives array is never present and the body stays bounded. + logprobs = choice.get("logprobs") + if logprobs is not None and not isinstance(logprobs, dict): + raise HTTPException(status_code=502, detail="invalid provider response") return { "id": response_id, "object": "chat.completion", @@ -405,6 +415,7 @@ def _public_provider_response(payload: dict[str, Any]) -> dict[str, Any]: "index": index, "finish_reason": finish_reason, "message": public_message, + "logprobs": logprobs, } ], "usage": { @@ -442,41 +453,61 @@ def _provider_preferences( # How this gate treats an unexpected request field, and why it changed # --------------------------------------------------------------------- # -# This used to be one flat allowlist, and anything outside it was a 400 raised -# *before* ``begin_inference_request``. That combination was quietly expensive: -# no ``inference_requests`` row is written on that path, so the rejections were -# invisible in telemetry, and the error body did not name the offending key. A -# rank-15 miner (``Cooking``) burned three submissions adding a single line that -# sent ``reasoning: {"effort": "medium"}`` -- the very value this platform was -# already stamping on their request for them (``benchmark_reasoning`` below) -- -# and had no way to discover which field killed the run. +# **The default is FORWARD.** Forwarding needs no justification; every pin and +# every drop below carries a concrete, stated reason. This is a retrieval-agent +# competition, not a prompt-shape competition -- a harness's own sampling and +# observability choices are its business. +# +# The default used to be the opposite, and it was expensive. Anything outside a +# flat allowlist was a 400 raised *before* ``begin_inference_request``, so no +# ``inference_requests`` row was written, the rejections were invisible in +# telemetry, and the error named no key. ``Cooking`` (rank 15 on v1) burned three +# submissions on one line sending ``reasoning: {"effort": "medium"}`` -- the very +# value this platform already stamps on their request (``benchmark_reasoning``) +# -- with no way to discover which field killed the run. +# +# The governing rule for a drop: **stripping a field is only acceptable when it +# changes neither what the model produces nor what the harness can observe.** +# Silently discarding a knob a miner deliberately set is the Cooking bug with +# extra steps -- they ask for a JSON schema, receive prose, their parser fails, +# and nothing anywhere names the cause. Fields with genuinely zero effect on +# either (``user``, ``metadata``) are inert to strip; fields that shape +# generation or output (``response_format``, ``logit_bias``, ``logprobs``) are +# not, and are forwarded. # -# The philosophy is now the broker's: normalise rather than refuse, wherever -# refusing buys no safety. ``dittobench-api`` already overwrites the caller's -# ``model`` instead of refusing a request that names one, and pins its own -# embedding model and nonce. The same reasoning applies field by field here, so -# the allowlist is split into three sets by what the field can actually do: +# The four fates: # -# ``_PINNED_REQUEST_FIELDS`` accepted, then *overwritten* with the value the -# ticket grants, in ``_locked_upstream_payload``. -# The caller's value never reaches the provider, so -# asking for more than the ticket granted is inert -# rather than fatal. -# ``_DROPPED_REQUEST_FIELDS`` accepted, then removed before the provider call. -# These have no bearing on the answer the harness -# gets, and each one is dropped for a stated reason -# (cost lever, egress channel, determinism, or dead -# weight against a contract we already refuse). +# ``_PINNED_REQUEST_FIELDS`` accepted, then overwritten or removed in +# ``_locked_upstream_payload`` so the ticket's own +# value governs. This is the anti-cheat boundary +# and it does not loosen: nothing here can obtain +# compute, a model, or an effort the ticket did +# not grant. +# ``_DROPPED_REQUEST_FIELDS`` accepted, then removed. Zero effect on the +# completion or on what the harness observes. # ``_FORWARDED_REQUEST_FIELDS`` validated and passed through unchanged. +# (refused) named explicitly in ``_validate_request_schema`` +# with the reason, including the two capability +# limits (``stream``, ``top_logprobs``). # -# Anything still outside all three is refused -- and refused *by name*. Unknown -# stays fail-closed on purpose: OpenRouter's request surface is additive, and a -# field invented after this code was written is exactly the one that might buy -# compute. The fix for the incident above is not "accept everything", it is -# "accept everything harmless, and say which key was not". - -# Route identity or a compute lever. Accepted and then replaced wholesale with -# the ticket's own value; see ``_locked_upstream_payload``. +# Provider support was established empirically against the pinned v7 model +# rather than from documentation -- see the table in the PR. Two findings drive +# the placements below: +# +# * OpenRouter EXCLUDES an endpoint that lacks a requested parameter rather +# than routing to it and failing. Pinning ``order: [amazon-bedrock]`` (which +# advertises no ``response_format``) returns 404 "No endpoints found" with +# the field present and succeeds without it. Under the v7 aggregate route +# (``allow_fallbacks: true``) the request simply lands on one of the eight +# supporting providers, so forwarding ``response_format`` cannot manufacture +# a 400. +# * ``reasoning_effort`` is the one field that HARD-FAILS: OpenRouter answers +# 400 `"reasoning_effort" and "reasoning.effort" are both provided with +# conflicting values` whenever it disagrees with the pinned block. Forwarding +# it would break every request that sets it to anything but ``medium``. + +# The anti-cheat boundary. Accepted, then replaced or removed so the ticket's +# value governs; see ``_locked_upstream_payload``. _PINNED_REQUEST_FIELDS = { # Substituted by ``_locked_grant_model``: the grant pins the model. "model", @@ -485,68 +516,96 @@ def _provider_preferences( "max_completion_tokens", # Forced to 1. Extra completions are extra billed generations. "n", - # Nested reasoning control. Replaced with ``benchmark_reasoning(model)``, so - # ``{"effort": "high"}`` is served as the pinned ``medium``. + # Buys N server-side generations and bills for all of them. + "best_of", + # Replaced with ``benchmark_reasoning(model)``, so ``{"effort": "high"}`` is + # served as the pinned ``medium`` -- and so is ``{"effort": "none"}``, which + # would otherwise let an agent opt out of v7's mandatory reasoning entirely. "reasoning", + # The flat sibling. Removed rather than forwarded: OpenRouter hard-400s when + # it disagrees with ``reasoning.effort`` (verified), so forwarding it would + # break every harness that sets it. Removing it makes the pinned nested + # value the single source of truth, which is also the anti-cheat action. + "reasoning_effort", + # OpenRouter's legacy boolean for returning reasoning. Removed so the pinned + # ``exclude: true`` cannot be overridden into echoing reasoning back. + "include_reasoning", + # Selects a provider priority/cost tier -- ``priority`` buys faster compute + # at a higher price. A compute lever the ticket did not grant. + "service_tier", + # The platform derives trusted cost from the response's usage block + # (``_bounded_provider_cost`` under aggregate routing). A caller must not be + # able to reshape the input to its own metering. + "usage", + # Controls provider-side prompt-cache bucketing. Two agents could collide on + # a bucket, or one could deliberately target another's, which is cross-miner + # interference rather than a private choice. + "prompt_cache_key", } -# Accepted, then stripped before the request leaves this process. +# Accepted, then stripped. Each has zero effect on the completion AND zero +# effect on what the harness can observe, which is what makes stripping inert +# rather than a silent behaviour change. _DROPPED_REQUEST_FIELDS = { - # Buys N server-side generations and bills for all of them. Dropping is - # strictly cheaper than the ticket already allows. - "best_of", - # The *flat* sibling of ``reasoning`` -- a different key, and a harness may - # plausibly send either. Dropped rather than pinned: the pinned effort - # already arrives via ``reasoning``, and forwarding both leaves provider-side - # precedence between them undefined. - "reasoning_effort", - # Selects a provider priority/cost tier. A compute lever the ticket did not - # grant. - "service_tier", - # Free-form caller-controlled strings shipped verbatim to a third party. - # The harness runs sandboxed with no egress; forwarding these would hand it - # one. Neither affects the completion. + # Caller-controlled strings that travel to a third party under the + # ``data_collection: "deny"`` / ``zdr: true`` posture this proxy pins, and + # that identify or fingerprint the agent behind the request. None of them + # affects the completion, so dropping cannot change a harness's results. "user", "metadata", - # Asks the provider to retain the completion, contradicting the - # ``data_collection: "deny"`` / ``zdr: true`` preferences this proxy pins. + "safety_identifier", + # Asks the provider to retain the completion, directly contradicting the + # pinned retention posture above. "store", # Only meaningful alongside ``stream: true``, which this lane refuses. "stream_options", - # Constrained/grammar decoding materially changes the output distribution - # and its latency profile, so two agents differing only in this field are no - # longer comparable runs. Dropped for benchmark comparability, not safety. - "response_format", - # Do not change sampling, but inflate the response body hard (``top_logprobs`` - # carries up to 20 alternatives per token) against ``response_body_bytes``, - # and ``_public_provider_response`` strips them, so the caller could never - # read them back regardless. - "logprobs", - "top_logprobs", - # Unbounded in size, and keyed by *tokenizer-specific* token ids. Because the - # served model is the ticket's rather than the one the caller named, a bias - # map built against another tokenizer silently biases unrelated tokens. - # Dropped for correctness and comparability. - "logit_bias", } -# Validated and passed through untouched. +# Everything else a reasonable OpenAI-compatible harness sends. Validated where +# a malformed value would be a genuine error, then passed through untouched. _FORWARDED_REQUEST_FIELDS = { "messages", + "tools", + "tool_choice", + "parallel_tool_calls", + # Sampling knobs. The miner's agent design owns these; they cost nothing + # extra and silently dropping one would change an agent's behaviour behind + # its back. ``seed`` and ``stop`` were always forwarded; the rest are the + # same class and were only ever excluded by the allowlist's silence. "temperature", "top_p", + "top_k", + "min_p", + "top_a", "seed", "stop", - "tools", - "tool_choice", - "parallel_tool_calls", - # Same class as ``temperature``/``top_p``/``seed``, which this lane has - # always forwarded: per-request sampling knobs that cost nothing extra and - # that the miner's own agent design is entitled to choose. Silently dropping - # a deliberately-set sampling knob would change an agent's behaviour behind - # its back -- the exact failure mode this change exists to remove. "frequency_penalty", "presence_penalty", + "repetition_penalty", + # Biases token selection. Buys no compute and grants no model. The earlier + # tokenizer-mismatch worry does not survive contact: the grant pins + # gpt-oss-20b and the exchange response tells the harness exactly which + # model it is talking to, so a bias map is built against the served model. + "logit_bias", + # Purely observational -- it does not alter generation at all, so there was + # never a determinism or comparability argument against it, only the + # allowlist's silence. Plumbed through ``_public_provider_response`` so it + # is genuinely returned rather than accepted and quietly discarded. + # (``top_logprobs`` is refused separately: see the response-size limit.) + "logprobs", + # Structured outputs. Verified end to end against the pinned v7 model on all + # five supporting providers tested, WITH the pinned reasoning block active: + # ``json_schema`` conforms every time. ``json_object`` is honoured by most + # providers but is advisory -- prefer ``json_schema``. + "response_format", + "structured_outputs", + # Speculative decoding. Rejected prediction tokens bill as completion + # tokens, but the prediction blob rides in the request body, so it inflates + # the caller's own byte-derived reservation and chargeable ceiling first. + # Self-limiting. + "prediction", + # Generation-length hint on newer OpenAI-compatible surfaces. + "verbosity", # Refused when true (see below), but must parse as a known field so the # refusal can explain itself. "stream", @@ -557,8 +616,59 @@ def _provider_preferences( ) +# Fields refused on purpose, each with the reason the caller gets told. A +# refusal is only acceptable when it is legible, so every entry here explains +# itself rather than saying "unsupported". +_REFUSED_REQUEST_FIELDS = { + # Route identity and model selection. The platform sets provider preferences + # itself; letting a caller choose is the evasion path this lane exists to + # close. + "models": "the model is pinned by the ticket, not chosen by the request", + "provider": "provider routing is pinned by the platform", + "route": "provider routing is pinned by the platform", + "preset": "provider routing is pinned by the platform", + "transforms": "prompt transforms would change benchmark semantics", + # Server-side network egress. The harness runs sandboxed without egress and + # these would hand it some through the provider. + "plugins": "server-side plugins are not available on this lane", + "web_search_options": "server-side web search is not available on this lane", + # Deprecated tool spellings. No provider serving this model advertises them, + # so forwarding would leave the harness silently toolless -- name the modern + # equivalent instead. + "functions": "use tools instead", + "function_call": "use tool_choice instead", + # This lane's response contract is text-only (`_public_provider_response`); + # a non-text modality would surface as a 502 blamed on the provider. + "audio": "this lane serves text completions only", + "modalities": "this lane serves text completions only", + # Capability limit, not a policy choice. `top_logprobs` multiplies the + # response body by roughly 80 bytes per token per alternative: measured + # against this model, `logprobs` alone extrapolates to ~0.81 MiB at the 8192 + # -token ceiling and fits, while top_logprobs=3 reaches ~2.65 MiB and + # top_logprobs=20 ~13.3 MiB, both past the 2 MiB response cap. Breaching it + # raises a 502 that is attributed to the PROVIDER and cools the shared + # route, so a caller's own choice would be charged to infrastructure. + # Refused rather than clamped because no non-zero value is safe at the + # ceiling. Raising `response_body_bytes` would unlock it. + "top_logprobs": ( + "logprobs is supported but top_logprobs is not: it would exceed this " + "lane's response size limit" + ), +} + + def _validate_request_schema(payload: dict[str, Any]) -> None: - """Accept only the text/tool subset used by the benchmark harness.""" + """Accept the OpenAI-compatible surface, pinning only what protects the grant.""" + # Named refusals first, so a caller sending one gets the reason rather than + # a bare "unsupported parameter". + refused = sorted(set(payload) & set(_REFUSED_REQUEST_FIELDS)) + if refused: + reasons = "; ".join( + f"{key} ({_REFUSED_REQUEST_FIELDS[key]})" for key in refused + ) + raise HTTPException( + status_code=400, detail=f"unsupported inference parameter: {reasons}" + ) unknown = set(payload) - _ALLOWED_REQUEST_FIELDS if unknown: # Name the keys. A harness author reading this over stderr has no other @@ -568,7 +678,18 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: status_code=400, detail=f"unsupported inference parameter: {', '.join(sorted(unknown))}", ) - for name in ("temperature", "top_p", "frequency_penalty", "presence_penalty"): + # Validation stays deliberately light. Over-validating a forwarded field + # recreates the problem this change exists to fix: the point is to reject + # only what is genuinely malformed, not what is merely unfamiliar. + for name in ( + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "min_p", + "top_a", + "repetition_penalty", + ): value = payload.get(name) if value is not None and ( not isinstance(value, (int, float)) @@ -580,6 +701,20 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: penalty = payload.get(name) if penalty is not None and not -2 <= penalty <= 2: raise HTTPException(status_code=400, detail=f"invalid {name}") + for name in ("min_p", "top_a"): + value = payload.get(name) + if value is not None and not 0 <= value <= 1: + raise HTTPException(status_code=400, detail=f"invalid {name}") + repetition_penalty = payload.get("repetition_penalty") + if repetition_penalty is not None and not 0 < repetition_penalty <= 2: + raise HTTPException(status_code=400, detail="invalid repetition_penalty") + top_k = payload.get("top_k") + if top_k is not None and ( + not isinstance(top_k, int) or isinstance(top_k, bool) or top_k < 0 + ): + raise HTTPException(status_code=400, detail="invalid top_k") + if "logprobs" in payload and not isinstance(payload["logprobs"], bool): + raise HTTPException(status_code=400, detail="invalid logprobs") temperature = payload.get("temperature") if temperature is not None and not 0 <= temperature <= 2: raise HTTPException(status_code=400, detail="invalid temperature") @@ -794,9 +929,22 @@ def _locked_upstream_payload( # none of them reaches the provider. See ``_DROPPED_REQUEST_FIELDS``. for field in _DROPPED_REQUEST_FIELDS: upstream.pop(field, None) - # Collapsed into the single clamped ``max_tokens`` computed by - # ``_output_token_limit``; forwarding both would re-open the alias bypass. - upstream.pop("max_completion_tokens", None) + # Pinned by removal: the ticket's value governs, and for these that value is + # "whatever the platform's own request says", so the caller's key must go. + # ``reasoning_effort`` in particular is not merely ignored upstream -- it + # hard-400s when it disagrees with the pinned ``reasoning.effort``. + for field in ( + "best_of", + "reasoning_effort", + "include_reasoning", + "service_tier", + "usage", + "prompt_cache_key", + # Collapsed into the single clamped ``max_tokens`` computed by + # ``_output_token_limit``; forwarding both re-opens the alias bypass. + "max_completion_tokens", + ): + upstream.pop(field, None) upstream["model"] = model upstream["max_tokens"] = max_tokens upstream["n"] = 1 diff --git a/ditto/tests/api_server/endpoints/test_inference.py b/ditto/tests/api_server/endpoints/test_inference.py index fe5ce929..7c3ded10 100644 --- a/ditto/tests/api_server/endpoints/test_inference.py +++ b/ditto/tests/api_server/endpoints/test_inference.py @@ -15,6 +15,7 @@ _DROPPED_REQUEST_FIELDS, _FORWARDED_REQUEST_FIELDS, _PINNED_REQUEST_FIELDS, + _REFUSED_REQUEST_FIELDS, _bounded_provider_cost, _estimated_tokens, _exchange_message, @@ -455,46 +456,158 @@ def test_caller_reasoning_is_accepted_then_overwritten_with_the_pinned_effort() assert upstream["reasoning"] == {"effort": "medium", "exclude": True} -def test_harmless_client_defaults_are_accepted_and_never_reach_the_provider() -> None: - """Fields mainstream OpenAI clients emit by default cost nobody a run. +def test_identity_fields_are_accepted_and_never_reach_the_provider() -> None: + """The only drops left: zero effect on the completion or on observation. - Each is accepted at the door and stripped before the upstream call, so the - harness is not killed for sending it and the provider never sees it. + Stripping is acceptable exactly when it changes neither what the model + produces nor what the harness can see. These carry or fingerprint agent + identity toward a third party under the pinned deny-retention posture and + affect nothing about the answer, so dropping them cannot alter a result. """ payload: dict[str, object] = { "model": "openai/gpt-oss-20b", "messages": [{"role": "user", "content": "hello"}], "user": "agent-7", "metadata": {"run": "abc"}, + "safety_identifier": "agent-7", "store": False, "stream_options": {"include_usage": True}, - "response_format": {"type": "json_object"}, - "logprobs": True, - "top_logprobs": 5, - "logit_bias": {"1234": -100}, - "service_tier": "priority", - "best_of": 4, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + for dropped in _DROPPED_REQUEST_FIELDS: + assert dropped not in upstream, dropped + + +def test_grant_protecting_fields_are_pinned_not_forwarded() -> None: + """The anti-cheat boundary, which does not loosen under default-forward.""" + payload: dict[str, object] = { + "model": "attacker/model", + "messages": [{"role": "user", "content": "hello"}], "n": 3, + "best_of": 4, + "reasoning": {"effort": "high"}, + "reasoning_effort": "high", + "include_reasoning": True, + "service_tier": "priority", + "usage": {"include": True}, + "prompt_cache_key": "shared-bucket", + "max_completion_tokens": 999_999, } _validate_request_schema(payload) upstream = _locked_upstream_payload( payload, model="openai/gpt-oss-20b", max_tokens=256 ) - for dropped in ( - "user", - "metadata", - "store", - "stream_options", - "response_format", - "logprobs", - "top_logprobs", - "logit_bias", - "service_tier", + # Replaced with the ticket's values. + assert upstream["model"] == "openai/gpt-oss-20b" + assert upstream["n"] == 1 + assert upstream["max_tokens"] == 256 + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + # Pinned by removal, so the platform's own request governs. + for removed in ( "best_of", + "reasoning_effort", + "include_reasoning", + "service_tier", + "usage", + "prompt_cache_key", + "max_completion_tokens", ): - assert dropped not in upstream, dropped - # `n` is pinned rather than dropped: exactly one billed generation. - assert upstream["n"] == 1 + assert removed not in upstream, removed + + +def test_structured_outputs_and_logprobs_are_forwarded_intact() -> None: + """Verified against the pinned v7 model on every supporting provider. + + `json_schema` conforms WITH the pinned reasoning block active, and OpenRouter + routes around providers that lack `response_format` rather than failing, so + forwarding cannot manufacture a 400. `logprobs` does not alter generation at + all, so there was never an argument against it beyond the allowlist's + silence. + """ + schema = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"color": {"type": "string"}}, + "required": ["color"], + "additionalProperties": False, + }, + }, + } + payload: dict[str, object] = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "response_format": schema, + "structured_outputs": True, + "logprobs": True, + "logit_bias": {"1234": -100}, + "prediction": {"type": "content", "content": "red"}, + "verbosity": "low", + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + # Byte-identical, not merely present: a schema is only useful intact. + assert upstream["response_format"] == schema + assert upstream["structured_outputs"] is True + assert upstream["logprobs"] is True + assert upstream["logit_bias"] == {"1234": -100} + assert upstream["prediction"] == {"type": "content", "content": "red"} + assert upstream["verbosity"] == "low" + + +def test_forwarded_logprobs_are_actually_returned_to_the_caller() -> None: + """Forwarding a field the response strips would be a silent no-op. + + The caller sets the flag, pays for the larger provider response, and + receives nothing -- which is the bug this whole change exists to remove. + """ + logprobs = {"content": [{"token": "hi", "logprob": -0.25, "top_logprobs": []}]} + public = _public_provider_response( + { + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": "openai/gpt-oss-20b", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hi"}, + "logprobs": logprobs, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + assert public["choices"][0]["logprobs"] == logprobs + + # A provider that does not support them returns null, which is the honest + # answer rather than a fabricated one. + without = _public_provider_response( + { + "id": "gen-2", + "object": "chat.completion", + "created": 1_700_000_000, + "model": "openai/gpt-oss-20b", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hi"}, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + assert without["choices"][0]["logprobs"] is None def test_sampling_knobs_the_miner_owns_are_forwarded_unchanged() -> None: @@ -535,9 +648,32 @@ def test_unsupported_parameter_error_names_every_offending_key() -> None: assert refused.value.status_code == 400 detail = str(refused.value.detail) assert "unsupported inference parameter" in detail - # Sorted, so the message is stable across dict ordering. - assert "models, plugins" in detail + # Both keys named, sorted so the message is stable across dict ordering, + # and each carrying the reason rather than a bare "unsupported". + assert "models" in detail + assert "plugins" in detail + assert detail.index("models") < detail.index("plugins") + assert "pinned by the ticket" in detail + + # A key nobody has classified still names itself. + with pytest.raises(HTTPException) as novel: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "some_future_openrouter_field": 1, + } + ) + assert "some_future_openrouter_field" in str(novel.value.detail) + +def test_capability_limits_say_they_are_limits() -> None: + """`stream` and `top_logprobs` are refusals we would rather not make. + + Both are capability limits of this lane rather than policy, so the message + has to say which key and why -- a miner reading it should know whether to + change their harness or ask us to change the platform. + """ with pytest.raises(HTTPException) as streaming: _validate_request_schema( { @@ -547,26 +683,80 @@ def test_unsupported_parameter_error_names_every_offending_key() -> None: } ) assert "stream" in str(streaming.value.detail) + assert "non-streaming" in str(streaming.value.detail) + + with pytest.raises(HTTPException) as top: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "logprobs": True, + "top_logprobs": 5, + } + ) + detail = str(top.value.detail) + assert "top_logprobs" in detail + # Says the neighbouring field DOES work, so the miner keeps what they can. + assert "logprobs is supported" in detail + assert "response size" in detail -def test_every_accepted_field_is_pinned_dropped_or_deliberately_forwarded() -> None: - """The anti-cheat property, stated as a partition over the allowlist. +def test_every_field_has_exactly_one_decided_fate() -> None: + """The partition, which is what makes default-forward reviewable. - Nothing may be accepted without a decided fate. A field that is neither - pinned nor dropped is forwarded to the provider verbatim, so this partition - is what makes "accept it" reviewable one field at a time. + Under a forward-by-default posture the risk is no longer "a normal field is + refused" but "a field that should have been pinned is forwarded by + omission". These four sets must stay disjoint and must cover the surface. """ assert ( _PINNED_REQUEST_FIELDS | _DROPPED_REQUEST_FIELDS | _FORWARDED_REQUEST_FIELDS == _ALLOWED_REQUEST_FIELDS ) - assert not _PINNED_REQUEST_FIELDS & _DROPPED_REQUEST_FIELDS - assert not _PINNED_REQUEST_FIELDS & _FORWARDED_REQUEST_FIELDS - assert not _DROPPED_REQUEST_FIELDS & _FORWARDED_REQUEST_FIELDS - # Nothing that selects a model, a route, or server-side network egress may - # be accepted at any tier. + for left, right in ( + (_PINNED_REQUEST_FIELDS, _DROPPED_REQUEST_FIELDS), + (_PINNED_REQUEST_FIELDS, _FORWARDED_REQUEST_FIELDS), + (_DROPPED_REQUEST_FIELDS, _FORWARDED_REQUEST_FIELDS), + ): + assert not left & right, sorted(left & right) + # An accepted field can never also be a named refusal. + assert not _ALLOWED_REQUEST_FIELDS & set(_REFUSED_REQUEST_FIELDS) + # Every named refusal explains itself; "unsupported" alone is what we are + # replacing, so an empty or lazy reason is a test failure. + for key, reason in _REFUSED_REQUEST_FIELDS.items(): + assert reason and len(reason) > 12, key + + # The anti-cheat boundary, asserted by name. Anything that could buy + # compute, a model, or an effort level the ticket did not grant must be + # pinned or refused -- never forwarded. + for lever in ( + "model", + "n", + "best_of", + "reasoning", + "reasoning_effort", + "include_reasoning", + "service_tier", + "max_tokens", + "max_completion_tokens", + ): + assert lever in _PINNED_REQUEST_FIELDS, lever + assert lever not in _FORWARDED_REQUEST_FIELDS, lever for escape in ("models", "provider", "plugins", "transforms", "route"): assert escape not in _ALLOWED_REQUEST_FIELDS + assert escape in _REFUSED_REQUEST_FIELDS + + # The fields the operator explicitly asked to be usable by harnesses. + for normal in ( + "seed", + "logprobs", + "response_format", + "logit_bias", + "frequency_penalty", + "presence_penalty", + "stop", + "top_k", + ): + assert normal in _FORWARDED_REQUEST_FIELDS, normal def test_caller_shape_rejections_do_not_cool_shared_provider_route() -> None: diff --git a/ditto/tests/integration/test_inference_reasoning_integration.py b/ditto/tests/integration/test_inference_reasoning_integration.py index d2602603..2f62524a 100644 --- a/ditto/tests/integration/test_inference_reasoning_integration.py +++ b/ditto/tests/integration/test_inference_reasoning_integration.py @@ -371,6 +371,137 @@ async def handler(request: httpx.Request) -> httpx.Response: assert grant.cost_microusd == 123 +@pytest.mark.asyncio +async def test_structured_outputs_reach_the_provider_and_return_unmangled( + session_maker: async_sessionmaker[Any], +) -> None: + """`response_format` is forwarded byte-identical and its answer survives. + + Verified upstream before forwarding: against the pinned v7 model, a + `json_schema` request conforms on every supporting provider WITH the pinned + reasoning block active, and OpenRouter routes around the one provider that + lacks `response_format` rather than failing the request. So forwarding + cannot manufacture the live 400 that would have made this worse than the + silent strip it replaces. + + Both halves matter. A schema that arrives mutated is useless, and a + conforming answer that the response allowlist mangles on the way back is the + same silent failure wearing a different hat. + """ + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + schema = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"color": {"type": "string"}}, + "required": ["color"], + "additionalProperties": False, + }, + }, + } + seen: list[dict[str, Any]] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": V7_MODEL, + "openrouter_metadata": { + "endpoints": { + "available": [{"provider": "Fireworks", "selected": True}] + } + }, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": '{"color":"blue"}', + }, + "logprobs": { + "content": [ + {"token": "blue", "logprob": -0.5, "top_logprobs": []} + ] + }, + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 6, + "total_tokens": 18, + "cost": 0.000045, + }, + }, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "Name one color."}], + "response_format": schema, + "logprobs": True, + "logit_bias": {"1234": -100}, + "frequency_penalty": 0.25, + "seed": 42, + } + ).encode() + + try: + response = await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + assert response.status_code == 200 + # Reached the provider byte-identical -- a mutated schema is a broken one. + assert seen[0]["response_format"] == schema + assert seen[0]["logprobs"] is True + assert seen[0]["logit_bias"] == {"1234": -100} + assert seen[0]["frequency_penalty"] == 0.25 + assert seen[0]["seed"] == 42 + # Still pinned alongside it: structured outputs buy no reasoning effort. + assert seen[0]["reasoning"] == {"effort": "medium", "exclude": True} + assert seen[0]["model"] == V7_MODEL + + # And the answer survives the response allowlist unmangled, logprobs + # included -- forwarding a request field whose response is stripped would + # be a silent no-op. + returned = json.loads(bytes(response.body)) + choice = returned["choices"][0] + assert choice["message"]["content"] == '{"color":"blue"}' + assert json.loads(choice["message"]["content"]) == {"color": "blue"} + assert choice["logprobs"]["content"][0]["token"] == "blue" + + @pytest.mark.asyncio async def test_unknown_request_field_names_itself_in_the_error( session_maker: async_sessionmaker[Any],