diff --git a/ditto/api_server/endpoints/inference.py b/ditto/api_server/endpoints/inference.py index 8a2d949d..923cee7f 100644 --- a/ditto/api_server/endpoints/inference.py +++ b/ditto/api_server/endpoints/inference.py @@ -395,6 +395,16 @@ def _public_provider_response(payload: dict[str, Any]) -> dict[str, Any]: if usage is None: raise HTTPException(status_code=502, detail="invalid provider response") prompt_tokens, completion_tokens, _ = usage + # ``logprobs`` is a forwarded request field, so it has to come back or + # forwarding it would be a silent no-op -- the caller sets the flag, pays for + # the larger provider response, and receives nothing. Passed through as the + # provider sent it; only the two providers serving this model that support + # logprobs populate it, and the rest return null, which is the honest + # answer. ``top_logprobs`` is refused at the door, so the per-token + # alternatives array is never present and the body stays bounded. + logprobs = choice.get("logprobs") + if logprobs is not None and not isinstance(logprobs, dict): + raise HTTPException(status_code=502, detail="invalid provider response") return { "id": response_id, "object": "chat.completion", @@ -405,6 +415,7 @@ def _public_provider_response(payload: dict[str, Any]) -> dict[str, Any]: "index": index, "finish_reason": finish_reason, "message": public_message, + "logprobs": logprobs, } ], "usage": { @@ -439,30 +450,246 @@ def _provider_preferences( return preferences -_ALLOWED_REQUEST_FIELDS = { +# How this gate treats an unexpected request field, and why it changed +# --------------------------------------------------------------------- +# +# **The default is FORWARD.** Forwarding needs no justification; every pin and +# every drop below carries a concrete, stated reason. This is a retrieval-agent +# competition, not a prompt-shape competition -- a harness's own sampling and +# observability choices are its business. +# +# The default used to be the opposite, and it was expensive. Anything outside a +# flat allowlist was a 400 raised *before* ``begin_inference_request``, so no +# ``inference_requests`` row was written, the rejections were invisible in +# telemetry, and the error named no key. ``Cooking`` (rank 15 on v1) burned three +# submissions on one line sending ``reasoning: {"effort": "medium"}`` -- the very +# value this platform already stamps on their request (``benchmark_reasoning``) +# -- with no way to discover which field killed the run. +# +# The governing rule for a drop: **stripping a field is only acceptable when it +# changes neither what the model produces nor what the harness can observe.** +# Silently discarding a knob a miner deliberately set is the Cooking bug with +# extra steps -- they ask for a JSON schema, receive prose, their parser fails, +# and nothing anywhere names the cause. Fields with genuinely zero effect on +# either (``user``, ``metadata``) are inert to strip; fields that shape +# generation or output (``response_format``, ``logit_bias``, ``logprobs``) are +# not, and are forwarded. +# +# The four fates: +# +# ``_PINNED_REQUEST_FIELDS`` accepted, then overwritten or removed in +# ``_locked_upstream_payload`` so the ticket's own +# value governs. This is the anti-cheat boundary +# and it does not loosen: nothing here can obtain +# compute, a model, or an effort the ticket did +# not grant. +# ``_DROPPED_REQUEST_FIELDS`` accepted, then removed. Zero effect on the +# completion or on what the harness observes. +# ``_FORWARDED_REQUEST_FIELDS`` validated and passed through unchanged. +# (refused) named explicitly in ``_validate_request_schema`` +# with the reason, including the two capability +# limits (``stream``, ``top_logprobs``). +# +# Provider support was established empirically against the pinned v7 model +# rather than from documentation -- see the table in the PR. Two findings drive +# the placements below: +# +# * OpenRouter EXCLUDES an endpoint that lacks a requested parameter rather +# than routing to it and failing. Pinning ``order: [amazon-bedrock]`` (which +# advertises no ``response_format``) returns 404 "No endpoints found" with +# the field present and succeeds without it. Under the v7 aggregate route +# (``allow_fallbacks: true``) the request simply lands on one of the eight +# supporting providers, so forwarding ``response_format`` cannot manufacture +# a 400. +# * ``reasoning_effort`` is the one field that HARD-FAILS: OpenRouter answers +# 400 `"reasoning_effort" and "reasoning.effort" are both provided with +# conflicting values` whenever it disagrees with the pinned block. Forwarding +# it would break every request that sets it to anything but ``medium``. + +# The anti-cheat boundary. Accepted, then replaced or removed so the ticket's +# value governs; see ``_locked_upstream_payload``. +_PINNED_REQUEST_FIELDS = { + # Substituted by ``_locked_grant_model``: the grant pins the model. "model", - "messages", + # Clamped down to the ticket ceiling by ``_output_token_limit``. "max_tokens", "max_completion_tokens", + # Forced to 1. Extra completions are extra billed generations. + "n", + # Buys N server-side generations and bills for all of them. + "best_of", + # Replaced with ``benchmark_reasoning(model)``, so ``{"effort": "high"}`` is + # served as the pinned ``medium`` -- and so is ``{"effort": "none"}``, which + # would otherwise let an agent opt out of v7's mandatory reasoning entirely. + "reasoning", + # The flat sibling. Removed rather than forwarded: OpenRouter hard-400s when + # it disagrees with ``reasoning.effort`` (verified), so forwarding it would + # break every harness that sets it. Removing it makes the pinned nested + # value the single source of truth, which is also the anti-cheat action. + "reasoning_effort", + # OpenRouter's legacy boolean for returning reasoning. Removed so the pinned + # ``exclude: true`` cannot be overridden into echoing reasoning back. + "include_reasoning", + # Selects a provider priority/cost tier -- ``priority`` buys faster compute + # at a higher price. A compute lever the ticket did not grant. + "service_tier", + # The platform derives trusted cost from the response's usage block + # (``_bounded_provider_cost`` under aggregate routing). A caller must not be + # able to reshape the input to its own metering. + "usage", + # Controls provider-side prompt-cache bucketing. Two agents could collide on + # a bucket, or one could deliberately target another's, which is cross-miner + # interference rather than a private choice. + "prompt_cache_key", +} + +# Accepted, then stripped. Each has zero effect on the completion AND zero +# effect on what the harness can observe, which is what makes stripping inert +# rather than a silent behaviour change. +_DROPPED_REQUEST_FIELDS = { + # Caller-controlled strings that travel to a third party under the + # ``data_collection: "deny"`` / ``zdr: true`` posture this proxy pins, and + # that identify or fingerprint the agent behind the request. None of them + # affects the completion, so dropping cannot change a harness's results. + "user", + "metadata", + "safety_identifier", + # Asks the provider to retain the completion, directly contradicting the + # pinned retention posture above. + "store", + # Only meaningful alongside ``stream: true``, which this lane refuses. + "stream_options", +} + +# Everything else a reasonable OpenAI-compatible harness sends. Validated where +# a malformed value would be a genuine error, then passed through untouched. +_FORWARDED_REQUEST_FIELDS = { + "messages", + "tools", + "tool_choice", + "parallel_tool_calls", + # Sampling knobs. The miner's agent design owns these; they cost nothing + # extra and silently dropping one would change an agent's behaviour behind + # its back. ``seed`` and ``stop`` were always forwarded; the rest are the + # same class and were only ever excluded by the allowlist's silence. "temperature", "top_p", + "top_k", + "min_p", + "top_a", "seed", "stop", - "tools", - "tool_choice", - "parallel_tool_calls", - "n", - "best_of", + "frequency_penalty", + "presence_penalty", + "repetition_penalty", + # Biases token selection. Buys no compute and grants no model. The earlier + # tokenizer-mismatch worry does not survive contact: the grant pins + # gpt-oss-20b and the exchange response tells the harness exactly which + # model it is talking to, so a bias map is built against the served model. + "logit_bias", + # Purely observational -- it does not alter generation at all, so there was + # never a determinism or comparability argument against it, only the + # allowlist's silence. Plumbed through ``_public_provider_response`` so it + # is genuinely returned rather than accepted and quietly discarded. + # (``top_logprobs`` is refused separately: see the response-size limit.) + "logprobs", + # Structured outputs. Verified end to end against the pinned v7 model on all + # five supporting providers tested, WITH the pinned reasoning block active: + # ``json_schema`` conforms every time. ``json_object`` is honoured by most + # providers but is advisory -- prefer ``json_schema``. + "response_format", + "structured_outputs", + # Speculative decoding. Rejected prediction tokens bill as completion + # tokens, but the prediction blob rides in the request body, so it inflates + # the caller's own byte-derived reservation and chargeable ceiling first. + # Self-limiting. + "prediction", + # Generation-length hint on newer OpenAI-compatible surfaces. + "verbosity", + # Refused when true (see below), but must parse as a known field so the + # refusal can explain itself. "stream", } +_ALLOWED_REQUEST_FIELDS = ( + _PINNED_REQUEST_FIELDS | _DROPPED_REQUEST_FIELDS | _FORWARDED_REQUEST_FIELDS +) + + +# Fields refused on purpose, each with the reason the caller gets told. A +# refusal is only acceptable when it is legible, so every entry here explains +# itself rather than saying "unsupported". +_REFUSED_REQUEST_FIELDS = { + # Route identity and model selection. The platform sets provider preferences + # itself; letting a caller choose is the evasion path this lane exists to + # close. + "models": "the model is pinned by the ticket, not chosen by the request", + "provider": "provider routing is pinned by the platform", + "route": "provider routing is pinned by the platform", + "preset": "provider routing is pinned by the platform", + "transforms": "prompt transforms would change benchmark semantics", + # Server-side network egress. The harness runs sandboxed without egress and + # these would hand it some through the provider. + "plugins": "server-side plugins are not available on this lane", + "web_search_options": "server-side web search is not available on this lane", + # Deprecated tool spellings. No provider serving this model advertises them, + # so forwarding would leave the harness silently toolless -- name the modern + # equivalent instead. + "functions": "use tools instead", + "function_call": "use tool_choice instead", + # This lane's response contract is text-only (`_public_provider_response`); + # a non-text modality would surface as a 502 blamed on the provider. + "audio": "this lane serves text completions only", + "modalities": "this lane serves text completions only", + # Capability limit, not a policy choice. `top_logprobs` multiplies the + # response body by roughly 80 bytes per token per alternative: measured + # against this model, `logprobs` alone extrapolates to ~0.81 MiB at the 8192 + # -token ceiling and fits, while top_logprobs=3 reaches ~2.65 MiB and + # top_logprobs=20 ~13.3 MiB, both past the 2 MiB response cap. Breaching it + # raises a 502 that is attributed to the PROVIDER and cools the shared + # route, so a caller's own choice would be charged to infrastructure. + # Refused rather than clamped because no non-zero value is safe at the + # ceiling. Raising `response_body_bytes` would unlock it. + "top_logprobs": ( + "logprobs is supported but top_logprobs is not: it would exceed this " + "lane's response size limit" + ), +} + def _validate_request_schema(payload: dict[str, Any]) -> None: - """Accept only the text/tool subset used by the benchmark harness.""" + """Accept the OpenAI-compatible surface, pinning only what protects the grant.""" + # Named refusals first, so a caller sending one gets the reason rather than + # a bare "unsupported parameter". + refused = sorted(set(payload) & set(_REFUSED_REQUEST_FIELDS)) + if refused: + reasons = "; ".join( + f"{key} ({_REFUSED_REQUEST_FIELDS[key]})" for key in refused + ) + raise HTTPException( + status_code=400, detail=f"unsupported inference parameter: {reasons}" + ) unknown = set(payload) - _ALLOWED_REQUEST_FIELDS if unknown: - raise HTTPException(status_code=400, detail="unsupported inference parameter") - for name in ("temperature", "top_p"): + # Name the keys. A harness author reading this over stderr has no other + # way to learn which of their fields was the problem, and the broker now + # forwards this detail verbatim (dittobench-api ``forwardChatCompletion``). + raise HTTPException( + status_code=400, + detail=f"unsupported inference parameter: {', '.join(sorted(unknown))}", + ) + # Validation stays deliberately light. Over-validating a forwarded field + # recreates the problem this change exists to fix: the point is to reject + # only what is genuinely malformed, not what is merely unfamiliar. + for name in ( + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "min_p", + "top_a", + "repetition_penalty", + ): value = payload.get(name) if value is not None and ( not isinstance(value, (int, float)) @@ -470,6 +697,24 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: or not math.isfinite(value) ): raise HTTPException(status_code=400, detail=f"invalid {name}") + for name in ("frequency_penalty", "presence_penalty"): + penalty = payload.get(name) + if penalty is not None and not -2 <= penalty <= 2: + raise HTTPException(status_code=400, detail=f"invalid {name}") + for name in ("min_p", "top_a"): + value = payload.get(name) + if value is not None and not 0 <= value <= 1: + raise HTTPException(status_code=400, detail=f"invalid {name}") + repetition_penalty = payload.get("repetition_penalty") + if repetition_penalty is not None and not 0 < repetition_penalty <= 2: + raise HTTPException(status_code=400, detail="invalid repetition_penalty") + top_k = payload.get("top_k") + if top_k is not None and ( + not isinstance(top_k, int) or isinstance(top_k, bool) or top_k < 0 + ): + raise HTTPException(status_code=400, detail="invalid top_k") + if "logprobs" in payload and not isinstance(payload["logprobs"], bool): + raise HTTPException(status_code=400, detail="invalid logprobs") temperature = payload.get("temperature") if temperature is not None and not 0 <= temperature <= 2: raise HTTPException(status_code=400, detail="invalid temperature") @@ -493,16 +738,31 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: ) ): raise HTTPException(status_code=400, detail="invalid stop") - for name in ("parallel_tool_calls", "stream"): + for name in ("parallel_tool_calls", "stream", "store"): if name in payload and not isinstance(payload[name], bool): raise HTTPException(status_code=400, detail=f"invalid {name}") - n = payload.get("n", 1) - if not isinstance(n, int) or isinstance(n, bool) or n != 1: + # One of the few genuine refusals left. ``stream`` is not normalised to + # false because this lane answers with a single non-streaming JSON body: a + # caller that asked for SSE and silently received one would fail parsing it, + # which is a worse and far less legible outcome than being told. + if payload.get("stream") not in {None, False}: raise HTTPException( - status_code=400, detail="multiple completions are not supported" + status_code=400, + detail=( + "unsupported inference parameter: stream " + "(this lane answers with a single non-streaming response)" + ), ) - if "best_of" in payload: - raise HTTPException(status_code=400, detail="best_of is not supported") + # ``n`` and ``best_of`` are pinned/dropped in ``_locked_upstream_payload`` + # rather than refused: the provider is asked for exactly one completion no + # matter what arrives here, so an over-ask cannot buy a second generation. + # A malformed value is still a malformed request and is still named. + for name in ("n", "best_of"): + value = payload.get(name) + if value is not None and ( + not isinstance(value, int) or isinstance(value, bool) or value < 1 + ): + raise HTTPException(status_code=400, detail=f"invalid {name}") tool_choice = payload.get("tool_choice") if tool_choice is not None: valid_named_choice = ( @@ -586,31 +846,39 @@ def _validate_request_schema(payload: dict[str, Any]) -> None: def _output_token_limit(payload: dict[str, Any], maximum: int) -> int: - """Normalize OpenAI's aliases without allowing one to bypass the other.""" + """Normalize OpenAI's aliases without allowing one to bypass the other. + + Both an over-ask and a disagreement between the two aliases are resolved + *downward* rather than refused. Asking for more output than the ticket + grants is the single most likely way for an ordinary OpenAI-compatible + harness to trip this gate, and killing the run over it buys nothing: the + clamp already guarantees the ticket's ceiling holds, which is the only + property that was ever at stake. A caller cannot obtain more output by + asking for more, by asking twice, or by disagreeing with itself. + """ max_tokens_value = payload.get("max_tokens") max_completion_tokens = payload.get("max_completion_tokens") - if ( - max_tokens_value is not None - and max_completion_tokens is not None - and max_tokens_value != max_completion_tokens - ): - raise HTTPException(status_code=400, detail="conflicting output token limits") - value = ( - max_tokens_value - if max_tokens_value is not None - else max_completion_tokens - if max_completion_tokens is not None - else maximum - ) - if ( - not isinstance(value, int) - or isinstance(value, bool) - or not 1 <= value <= maximum + for name, candidate in ( + ("max_tokens", max_tokens_value), + ("max_completion_tokens", max_completion_tokens), ): - raise HTTPException( - status_code=400, detail="max_tokens exceeds the ticket limit" - ) - return value + if candidate is not None and ( + not isinstance(candidate, int) or isinstance(candidate, bool) + ): + raise HTTPException(status_code=400, detail=f"invalid {name}") + # A non-positive ceiling is a malformed request rather than an over-ask, + # and there is no conservative direction to normalise it in: clamping it + # up would hand the caller output it did not ask for. + if candidate is not None and candidate < 1: + raise HTTPException(status_code=400, detail=f"invalid {name}") + requested = [ + value + for value in (max_tokens_value, max_completion_tokens) + if value is not None + ] + # Aliases that disagree resolve to the smaller, so neither can raise the + # other. Absent both, the ticket ceiling is the default it always was. + return min(*requested, maximum) if requested else maximum def _locked_grant_model(grant: Any, *, requested: str, config: Any) -> str: @@ -648,14 +916,43 @@ def _locked_grant_model(grant: Any, *, requested: str, config: Any) -> str: def _locked_upstream_payload( payload: dict[str, Any], *, model: str, max_tokens: int ) -> dict[str, Any]: - """Force consensus model/reasoning fields before provider routing.""" + """Force consensus model/reasoning fields before provider routing. + + This is the half of the schema policy that makes permissiveness safe. Every + field the gate now *accepts* rather than refuses is either replaced here + with the ticket's own value or removed here before the request leaves the + process, so accepting it cannot buy the caller compute, a model, or a + reasoning effort its ticket did not grant. + """ upstream = dict(payload) - upstream.pop("max_completion_tokens", None) - upstream.pop("best_of", None) + # Accepted at the door purely so the caller is not killed for sending them; + # none of them reaches the provider. See ``_DROPPED_REQUEST_FIELDS``. + for field in _DROPPED_REQUEST_FIELDS: + upstream.pop(field, None) + # Pinned by removal: the ticket's value governs, and for these that value is + # "whatever the platform's own request says", so the caller's key must go. + # ``reasoning_effort`` in particular is not merely ignored upstream -- it + # hard-400s when it disagrees with the pinned ``reasoning.effort``. + for field in ( + "best_of", + "reasoning_effort", + "include_reasoning", + "service_tier", + "usage", + "prompt_cache_key", + # Collapsed into the single clamped ``max_tokens`` computed by + # ``_output_token_limit``; forwarding both re-opens the alias bypass. + "max_completion_tokens", + ): + upstream.pop(field, None) upstream["model"] = model upstream["max_tokens"] = max_tokens upstream["n"] = 1 upstream["stream"] = False + # Unconditional assignment, not a default: the caller's value is discarded + # whatever its shape, so ``{"effort": "high"}`` and a malformed reasoning + # object are both served as the pinned contract. On a bench version with no + # reasoning contract the field is removed outright rather than passed on. reasoning = benchmark_reasoning(model) if reasoning is None: upstream.pop("reasoning", None) @@ -897,8 +1194,13 @@ async def proxy_chat_completions( payload = json.loads(body) except (json.JSONDecodeError, UnicodeDecodeError) as error: raise HTTPException(status_code=400, detail="invalid JSON request") from error - if not isinstance(payload, dict) or payload.get("stream") not in {None, False}: - raise HTTPException(status_code=400, detail="streaming is not supported") + if not isinstance(payload, dict): + raise HTTPException( + status_code=400, detail="inference request must be a JSON object" + ) + # Every field-level decision, including the streaming refusal, lives in one + # function now. It used to be split across here and the schema check, which + # is how the two most common refusals ended up with two different wordings. _validate_request_schema(payload) requested_model = payload.get("model") if not isinstance(requested_model, str): diff --git a/ditto/tests/api_server/endpoints/test_inference.py b/ditto/tests/api_server/endpoints/test_inference.py index aa1fc098..7c3ded10 100644 --- a/ditto/tests/api_server/endpoints/test_inference.py +++ b/ditto/tests/api_server/endpoints/test_inference.py @@ -11,6 +11,11 @@ from ditto.api_models.inference import InferenceExchangeRequest, InferenceGrantOffer from ditto.api_server.endpoints.inference import ( + _ALLOWED_REQUEST_FIELDS, + _DROPPED_REQUEST_FIELDS, + _FORWARDED_REQUEST_FIELDS, + _PINNED_REQUEST_FIELDS, + _REFUSED_REQUEST_FIELDS, _bounded_provider_cost, _estimated_tokens, _exchange_message, @@ -212,11 +217,31 @@ def test_embedding_contract_is_exact_and_response_is_sanitized() -> None: def test_output_token_alias_cannot_bypass_ticket_limit() -> None: - with pytest.raises(HTTPException): + # Disagreeing aliases resolve downward, so neither can raise the other. + assert ( _output_token_limit({"max_tokens": 1, "max_completion_tokens": 999_999}, 8192) - with pytest.raises(HTTPException): - _output_token_limit({"max_completion_tokens": 8193}, 8192) + == 1 + ) + assert ( + _output_token_limit({"max_tokens": 999_999, "max_completion_tokens": 1}, 8192) + == 1 + ) + # An over-ask is clamped to the ticket ceiling instead of killing the run. + assert _output_token_limit({"max_completion_tokens": 8193}, 8192) == 8192 + assert _output_token_limit({"max_tokens": 10**9}, 8192) == 8192 assert _output_token_limit({"max_completion_tokens": 32}, 8192) == 32 + assert _output_token_limit({}, 8192) == 8192 + # A non-positive or non-integer ceiling has no conservative normalisation + # and stays a named refusal. + for key, bad in ( + ("max_tokens", {"max_tokens": 0}), + ("max_tokens", {"max_tokens": -1}), + ("max_completion_tokens", {"max_completion_tokens": True}), + ("max_completion_tokens", {"max_completion_tokens": "many"}), + ): + with pytest.raises(HTTPException) as invalid: + _output_token_limit(bad, 8192) + assert str(invalid.value.detail) == f"invalid {key}" @pytest.mark.parametrize( @@ -225,7 +250,8 @@ def test_output_token_alias_cannot_bypass_ticket_limit() -> None: {"models": ["attacker/model"]}, {"plugins": [{"id": "web"}]}, {"provider": {"allow_fallbacks": True}}, - {"reasoning": {"effort": "high"}}, + {"transforms": ["middle-out"]}, + {"route": "fallback"}, { "messages": [ { @@ -319,8 +345,11 @@ def test_proxy_schema_rejects_non_text_content_parts(content: object) -> None: {"parallel_tool_calls": 1}, {"stream": "false"}, {"n": True}, - {"n": 2}, - {"best_of": 1}, + {"n": 0}, + {"best_of": 0}, + {"frequency_penalty": 2.5}, + {"presence_penalty": float("inf")}, + {"store": "yes"}, {"tool_choice": {"type": "function", "function": {"name": ""}}}, {"tool_choice": {"type": "function", "function": {"name": "x", "x": 1}}}, ], @@ -391,6 +420,345 @@ def test_v7_upstream_profile_pins_medium_reasoning_without_changing_v6() -> None assert "reasoning" not in v6 +def test_caller_reasoning_is_accepted_then_overwritten_with_the_pinned_effort() -> None: + """The rejection that cost `Cooking` three submissions is now a normalisation. + + The platform already stamps `{"effort": "medium", "exclude": True}` on every + v7 request (`benchmark_reasoning`), so the miner was 400'd for redundantly + requesting exactly what it was being given. Accepting the field is safe for + the same reason it was pointless to refuse it: the value is replaced. + """ + for effort in ({"effort": "high"}, {"effort": "low"}, {"max_tokens": 100_000}, {}): + payload = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "reasoning": effort, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + + # `reasoning_effort` is the *flat* OpenAI sibling and a different key. It is + # accepted so no harness dies on it, and dropped so it cannot compete with + # the pinned nested value at the provider. + flat = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "reasoning_effort": "high", + } + _validate_request_schema(flat) + upstream = _locked_upstream_payload( + flat, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert "reasoning_effort" not in upstream + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + + +def test_identity_fields_are_accepted_and_never_reach_the_provider() -> None: + """The only drops left: zero effect on the completion or on observation. + + Stripping is acceptable exactly when it changes neither what the model + produces nor what the harness can see. These carry or fingerprint agent + identity toward a third party under the pinned deny-retention posture and + affect nothing about the answer, so dropping them cannot alter a result. + """ + payload: dict[str, object] = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "user": "agent-7", + "metadata": {"run": "abc"}, + "safety_identifier": "agent-7", + "store": False, + "stream_options": {"include_usage": True}, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + for dropped in _DROPPED_REQUEST_FIELDS: + assert dropped not in upstream, dropped + + +def test_grant_protecting_fields_are_pinned_not_forwarded() -> None: + """The anti-cheat boundary, which does not loosen under default-forward.""" + payload: dict[str, object] = { + "model": "attacker/model", + "messages": [{"role": "user", "content": "hello"}], + "n": 3, + "best_of": 4, + "reasoning": {"effort": "high"}, + "reasoning_effort": "high", + "include_reasoning": True, + "service_tier": "priority", + "usage": {"include": True}, + "prompt_cache_key": "shared-bucket", + "max_completion_tokens": 999_999, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + # Replaced with the ticket's values. + assert upstream["model"] == "openai/gpt-oss-20b" + assert upstream["n"] == 1 + assert upstream["max_tokens"] == 256 + assert upstream["reasoning"] == {"effort": "medium", "exclude": True} + # Pinned by removal, so the platform's own request governs. + for removed in ( + "best_of", + "reasoning_effort", + "include_reasoning", + "service_tier", + "usage", + "prompt_cache_key", + "max_completion_tokens", + ): + assert removed not in upstream, removed + + +def test_structured_outputs_and_logprobs_are_forwarded_intact() -> None: + """Verified against the pinned v7 model on every supporting provider. + + `json_schema` conforms WITH the pinned reasoning block active, and OpenRouter + routes around providers that lack `response_format` rather than failing, so + forwarding cannot manufacture a 400. `logprobs` does not alter generation at + all, so there was never an argument against it beyond the allowlist's + silence. + """ + schema = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"color": {"type": "string"}}, + "required": ["color"], + "additionalProperties": False, + }, + }, + } + payload: dict[str, object] = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "response_format": schema, + "structured_outputs": True, + "logprobs": True, + "logit_bias": {"1234": -100}, + "prediction": {"type": "content", "content": "red"}, + "verbosity": "low", + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + # Byte-identical, not merely present: a schema is only useful intact. + assert upstream["response_format"] == schema + assert upstream["structured_outputs"] is True + assert upstream["logprobs"] is True + assert upstream["logit_bias"] == {"1234": -100} + assert upstream["prediction"] == {"type": "content", "content": "red"} + assert upstream["verbosity"] == "low" + + +def test_forwarded_logprobs_are_actually_returned_to_the_caller() -> None: + """Forwarding a field the response strips would be a silent no-op. + + The caller sets the flag, pays for the larger provider response, and + receives nothing -- which is the bug this whole change exists to remove. + """ + logprobs = {"content": [{"token": "hi", "logprob": -0.25, "top_logprobs": []}]} + public = _public_provider_response( + { + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": "openai/gpt-oss-20b", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hi"}, + "logprobs": logprobs, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + assert public["choices"][0]["logprobs"] == logprobs + + # A provider that does not support them returns null, which is the honest + # answer rather than a fabricated one. + without = _public_provider_response( + { + "id": "gen-2", + "object": "chat.completion", + "created": 1_700_000_000, + "model": "openai/gpt-oss-20b", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hi"}, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ) + assert without["choices"][0]["logprobs"] is None + + +def test_sampling_knobs_the_miner_owns_are_forwarded_unchanged() -> None: + """Same class as temperature/top_p/seed, which this lane always forwarded. + + Dropping a deliberately-set sampling knob would silently change an agent's + behaviour behind its back -- the exact failure mode this change removes. + """ + payload = { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "frequency_penalty": 0.5, + "presence_penalty": -0.25, + "temperature": 0.0, + "seed": 42, + } + _validate_request_schema(payload) + upstream = _locked_upstream_payload( + payload, model="openai/gpt-oss-20b", max_tokens=256 + ) + assert upstream["frequency_penalty"] == 0.5 + assert upstream["presence_penalty"] == -0.25 + assert upstream["temperature"] == 0.0 + assert upstream["seed"] == 42 + + +def test_unsupported_parameter_error_names_every_offending_key() -> None: + """A miner must be able to discover which field broke the run.""" + with pytest.raises(HTTPException) as refused: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "plugins": [{"id": "web"}], + "models": ["attacker/model"], + } + ) + assert refused.value.status_code == 400 + detail = str(refused.value.detail) + assert "unsupported inference parameter" in detail + # Both keys named, sorted so the message is stable across dict ordering, + # and each carrying the reason rather than a bare "unsupported". + assert "models" in detail + assert "plugins" in detail + assert detail.index("models") < detail.index("plugins") + assert "pinned by the ticket" in detail + + # A key nobody has classified still names itself. + with pytest.raises(HTTPException) as novel: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "some_future_openrouter_field": 1, + } + ) + assert "some_future_openrouter_field" in str(novel.value.detail) + + +def test_capability_limits_say_they_are_limits() -> None: + """`stream` and `top_logprobs` are refusals we would rather not make. + + Both are capability limits of this lane rather than policy, so the message + has to say which key and why -- a miner reading it should know whether to + change their harness or ask us to change the platform. + """ + with pytest.raises(HTTPException) as streaming: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + } + ) + assert "stream" in str(streaming.value.detail) + assert "non-streaming" in str(streaming.value.detail) + + with pytest.raises(HTTPException) as top: + _validate_request_schema( + { + "model": "openai/gpt-oss-20b", + "messages": [{"role": "user", "content": "hello"}], + "logprobs": True, + "top_logprobs": 5, + } + ) + detail = str(top.value.detail) + assert "top_logprobs" in detail + # Says the neighbouring field DOES work, so the miner keeps what they can. + assert "logprobs is supported" in detail + assert "response size" in detail + + +def test_every_field_has_exactly_one_decided_fate() -> None: + """The partition, which is what makes default-forward reviewable. + + Under a forward-by-default posture the risk is no longer "a normal field is + refused" but "a field that should have been pinned is forwarded by + omission". These four sets must stay disjoint and must cover the surface. + """ + assert ( + _PINNED_REQUEST_FIELDS | _DROPPED_REQUEST_FIELDS | _FORWARDED_REQUEST_FIELDS + == _ALLOWED_REQUEST_FIELDS + ) + for left, right in ( + (_PINNED_REQUEST_FIELDS, _DROPPED_REQUEST_FIELDS), + (_PINNED_REQUEST_FIELDS, _FORWARDED_REQUEST_FIELDS), + (_DROPPED_REQUEST_FIELDS, _FORWARDED_REQUEST_FIELDS), + ): + assert not left & right, sorted(left & right) + # An accepted field can never also be a named refusal. + assert not _ALLOWED_REQUEST_FIELDS & set(_REFUSED_REQUEST_FIELDS) + # Every named refusal explains itself; "unsupported" alone is what we are + # replacing, so an empty or lazy reason is a test failure. + for key, reason in _REFUSED_REQUEST_FIELDS.items(): + assert reason and len(reason) > 12, key + + # The anti-cheat boundary, asserted by name. Anything that could buy + # compute, a model, or an effort level the ticket did not grant must be + # pinned or refused -- never forwarded. + for lever in ( + "model", + "n", + "best_of", + "reasoning", + "reasoning_effort", + "include_reasoning", + "service_tier", + "max_tokens", + "max_completion_tokens", + ): + assert lever in _PINNED_REQUEST_FIELDS, lever + assert lever not in _FORWARDED_REQUEST_FIELDS, lever + for escape in ("models", "provider", "plugins", "transforms", "route"): + assert escape not in _ALLOWED_REQUEST_FIELDS + assert escape in _REFUSED_REQUEST_FIELDS + + # The fields the operator explicitly asked to be usable by harnesses. + for normal in ( + "seed", + "logprobs", + "response_format", + "logit_bias", + "frequency_penalty", + "presence_penalty", + "stop", + "top_k", + ): + assert normal in _FORWARDED_REQUEST_FIELDS, normal + + def test_caller_shape_rejections_do_not_cool_shared_provider_route() -> None: assert not _provider_rejection_is_route_observable(400) assert not _provider_rejection_is_route_observable(422) diff --git a/ditto/tests/integration/test_inference_reasoning_integration.py b/ditto/tests/integration/test_inference_reasoning_integration.py new file mode 100644 index 00000000..2f62524a --- /dev/null +++ b/ditto/tests/integration/test_inference_reasoning_integration.py @@ -0,0 +1,562 @@ +"""End-to-end proof that a caller-supplied ``reasoning`` effort is pinned, not refused. + +This is the regression test for the incident that motivated the schema change. +``Cooking`` (rank 15 on v1) added one line in v2 that sent +``reasoning: {"effort": "medium"}`` on every chat call. ``reasoning`` was absent +from the request allowlist, so the gate answered 400 forty-three lines *before* +``begin_inference_request`` -- no ``inference_requests`` row, nothing in +telemetry, and an error body that never named the field. Roughly 81 of ~480 chat +calls per run died that way, the run failed v7's complete-usage check, and the +miner burned three submissions on it. + +The irony that decided the fix: ``benchmark_reasoning`` already stamps +``{"effort": "medium", "exclude": True}`` on every v7 request. The miner was +rejected for redundantly asking for precisely what they were already being +given. + +So the property under test is not "``reasoning`` is allowed". It is the stronger +one that makes allowing it safe: the caller's value is *discarded and replaced*, +so a request asking for ``high`` is served, metered, and billed as ``medium``. +The anti-cheat property and the availability fix are the same mechanism. +""" + +from __future__ import annotations + +import base64 +import json +from datetime import UTC, datetime, timedelta +from typing import Any +from uuid import UUID, uuid4 + +import httpx +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from ditto.api_models.agent_status import AgentStatus +from ditto.api_models.ticket_status import TicketStatus +from ditto.api_server.config import InferenceProxyConfig +from ditto.api_server.endpoints.inference import _proxy_message, proxy_chat_completions +from ditto.api_server.inference_routing import ( + AGGREGATE_CALIBRATION_SAMPLES, + AGGREGATE_PROVIDER, + V7_MODEL, + aggregate_profile_revision, +) +from ditto.db.models import ( + Agent, + InferenceGrant, + InferenceProviderRoute, + InferenceRequest, + InferenceRoutingPolicy, + ValidatorTicket, +) +from ditto.db.queries.inference import ( + activate_inference_grant, + ensure_inference_grant, +) + +pytestmark = pytest.mark.integration + +_MAX_OUTPUT_TOKENS = 1024 + + +def _config() -> InferenceProxyConfig: + return InferenceProxyConfig( + enabled=True, + required=False, + public_base_url="https://platform.example", + openrouter_api_key="test-key", + upstream_url="https://openrouter.ai/api/v1/chat/completions", + allowed_models=(V7_MODEL,), + provider="openrouter", + routing_mode="aggregate_throughput", + request_budget=1000, + token_budget=1_000_000, + embedding_upstream_url="https://openrouter.ai/api/v1/embeddings", + embedding_model="perplexity/pplx-embed-v1-0.6b", + embedding_profile="dittobench-v7-openrouter-pplx-embed-v1-0.6b-768-v1", + embedding_provider="Perplexity", + embedding_dimensions=768, + embedding_request_budget=100_000, + embedding_token_budget=1_000_000_000, + embedding_per_ticket_concurrency=64, + embedding_per_validator_concurrency=256, + embedding_global_concurrency=1024, + embedding_per_ticket_requests_per_minute=10_000, + embedding_per_validator_requests_per_minute=40_000, + embedding_global_requests_per_minute=100_000, + embedding_request_body_bytes=1 << 20, + embedding_response_body_bytes=16 << 20, + per_ticket_concurrency=64, + per_validator_concurrency=256, + global_concurrency=1024, + per_ticket_requests_per_minute=10_000, + per_validator_requests_per_minute=40_000, + global_requests_per_minute=100_000, + request_body_bytes=256 << 10, + response_body_bytes=1 << 20, + timeout_seconds=10, + max_output_tokens=_MAX_OUTPUT_TOKENS, + ) + + +class _State: + """The three attributes ``proxy_chat_completions`` reads off ``app.state``.""" + + def __init__(self, *, config: Any, session_maker: Any, client: httpx.AsyncClient): + class _Config: + inference_proxy = config + + self.config = _Config() + self.session_maker = session_maker + self.inference_client = client + + +class _App: + def __init__(self, state: _State) -> None: + self.state = state + + +async def _seed_v7_grant( + maker: Any, *, config: InferenceProxyConfig, public_key: str +) -> tuple[UUID, str, int]: + """One agent + v7 ticket + a grant minted through the real v7 route path.""" + now = datetime.now(UTC) + async with maker() as session, session.begin(): + # A calibrated aggregate route, so ``select_route`` admits the v7 mint + # exactly as it does in production rather than being bypassed. + session.add( + InferenceRoutingPolicy( + model=V7_MODEL, + enabled=True, + speed_weight=1.0, + cost_weight=0.0, + exploration_weight=0.0, + exploration_ticket_budget=0, + min_tool_accuracy=0.0, + min_composite=0.0, + min_calibration_samples=1, + max_error_rate=1.0, + max_timeout_rate=1.0, + cooldown_seconds=1, + ewma_alpha=0.3, + updated_at=now, + ) + ) + session.add( + InferenceProviderRoute( + model=V7_MODEL, + provider=AGGREGATE_PROVIDER, + profile_revision=aggregate_profile_revision(V7_MODEL), + status="healthy", + calibration_status="eligible", + calibration_tool_accuracy=1.0, + calibration_composite=1.0, + calibration_sample_count=AGGREGATE_CALIBRATION_SAMPLES, + calibration_manifest_sha256="0" * 64, + discovered_at=now, + ewma_error_rate=0.0, + ewma_timeout_rate=0.0, + sample_count=0, + selected_ticket_count=0, + exploration_ticket_count=0, + updated_at=now, + ) + ) + await session.flush() + agent = Agent( + agent_id=uuid4(), + miner_hotkey="5FRZRm3R6ESJ4TtxaQ51vxk99hdkFdchUHFZVNSfYAUbehyR", + name="reasoning-pin", + sha256=uuid4().hex + uuid4().hex, + status=AgentStatus.EVALUATING, + created_at=now, + ) + ticket = ValidatorTicket( + agent_id=agent.agent_id, + validator_hotkey="validator-reasoning", + slot_id="slot-0", + status=TicketStatus.ISSUED, + issued_at=now, + deadline=now + timedelta(minutes=20), + bench_version=7, + attempt_count=1, + ) + session.add_all([agent, ticket]) + await session.flush() + grant = await ensure_inference_grant(session, ticket=ticket, config=config) + assert grant is not None + activated = await activate_inference_grant( + session, + grant_id=grant.grant_id, + validator_hotkey="validator-reasoning", + broker_public_key=public_key, + now=now, + config=config, + ) + assert activated is not None + live = activated[0] + # Minted through the real v7 path, so these are the platform's own + # values rather than anything this test arranged. + assert live.bench_version == 7 + assert live.allowed_models == [V7_MODEL] + assert live.route_provider == AGGREGATE_PROVIDER + return live.grant_id, activated[1], live.generation + + +def _signed_request( + *, + app: _App, + grant_id: UUID, + bearer: str, + generation: int, + private: Ed25519PrivateKey, + body: bytes, +) -> dict[str, Any]: + """Header kwargs for one authenticated proxy call over ``body``.""" + from starlette.requests import Request + + nonce = uuid4() + requested_at = datetime.now(UTC) + proof = private.sign( + _proxy_message( + grant_id=grant_id, + generation=generation, + nonce=nonce, + requested_at=requested_at, + body=body, + ) + ) + + async def receive() -> dict[str, Any]: + return {"type": "http.request", "body": body, "more_body": False} + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/api/v1/inference/chat/completions", + "headers": [], + "app": app, + }, + receive, + ) + return { + "request": request, + "x_ditto_grant": grant_id, + "x_ditto_generation": generation, + "x_ditto_nonce": nonce, + "x_ditto_requested_at": requested_at, + "x_ditto_proof": base64.urlsafe_b64encode(proof).decode().rstrip("="), + "authorization": f"Bearer {bearer}", + } + + +@pytest.mark.asyncio +async def test_caller_reasoning_effort_is_accepted_pinned_to_medium_and_charged( + session_maker: async_sessionmaker[Any], +) -> None: + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + seen: list[dict[str, Any]] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": V7_MODEL, + # Opt-in router metadata; ``_upstream_provider`` requires exactly + # one selected endpoint for trusted route telemetry. + "openrouter_metadata": { + "endpoints": { + "available": [{"provider": "Fireworks", "selected": True}] + } + }, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "ok"}, + } + ], + "usage": { + "prompt_tokens": 31, + "completion_tokens": 17, + "total_tokens": 48, + # Aggregate routing derives trusted cost from the provider's + # own figure; without it the call books as usage-unavailable. + "cost": 0.000123, + }, + }, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + + # Exactly the shape `Cooking` v2/v3 emits, but asking for the *strongest* + # effort rather than the pinned one -- the adversarial direction. + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "hello"}], + "temperature": 0.0, + "seed": 42, + "reasoning": {"effort": "high"}, + } + ).encode() + + try: + response = await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + # 1. Accepted. This is the byte-for-byte request that used to 400. + assert response.status_code == 200 + + # 2. Reached the provider with the *pinned* value, not the caller's "high". + assert len(seen) == 1 + assert seen[0]["reasoning"] == {"effort": "medium", "exclude": True} + assert seen[0]["model"] == V7_MODEL + assert seen[0]["max_tokens"] == _MAX_OUTPUT_TOKENS + assert seen[0]["n"] == 1 + assert seen[0]["stream"] is False + # The caller's own sampling choices survive untouched. + assert seen[0]["temperature"] == 0.0 + assert seen[0]["seed"] == 42 + + # 3. Charged, and charged as the medium call that actually ran. The request + # reached the ledger at all -- the old rejection never wrote a row, which + # is why this failure mode was invisible in telemetry. + async with session_maker() as session: + row = ( + await session.scalars( + select(InferenceRequest).where(InferenceRequest.grant_id == grant_id) + ) + ).one() + assert row.status == "completed" + assert row.model == V7_MODEL + assert row.prompt_tokens == 31 + assert row.completion_tokens == 17 + grant = await session.get(InferenceGrant, grant_id) + assert grant is not None + assert grant.request_count == 1 + assert grant.prompt_tokens == 31 + assert grant.completion_tokens == 17 + assert grant.cost_microusd == 123 + + +@pytest.mark.asyncio +async def test_structured_outputs_reach_the_provider_and_return_unmangled( + session_maker: async_sessionmaker[Any], +) -> None: + """`response_format` is forwarded byte-identical and its answer survives. + + Verified upstream before forwarding: against the pinned v7 model, a + `json_schema` request conforms on every supporting provider WITH the pinned + reasoning block active, and OpenRouter routes around the one provider that + lacks `response_format` rather than failing the request. So forwarding + cannot manufacture the live 400 that would have made this worse than the + silent strip it replaces. + + Both halves matter. A schema that arrives mutated is useless, and a + conforming answer that the response allowlist mangles on the way back is the + same silent failure wearing a different hat. + """ + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + schema = { + "type": "json_schema", + "json_schema": { + "name": "answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"color": {"type": "string"}}, + "required": ["color"], + "additionalProperties": False, + }, + }, + } + seen: list[dict[str, Any]] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "gen-1", + "object": "chat.completion", + "created": 1_700_000_000, + "model": V7_MODEL, + "openrouter_metadata": { + "endpoints": { + "available": [{"provider": "Fireworks", "selected": True}] + } + }, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": '{"color":"blue"}', + }, + "logprobs": { + "content": [ + {"token": "blue", "logprob": -0.5, "top_logprobs": []} + ] + }, + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 6, + "total_tokens": 18, + "cost": 0.000045, + }, + }, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "Name one color."}], + "response_format": schema, + "logprobs": True, + "logit_bias": {"1234": -100}, + "frequency_penalty": 0.25, + "seed": 42, + } + ).encode() + + try: + response = await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + assert response.status_code == 200 + # Reached the provider byte-identical -- a mutated schema is a broken one. + assert seen[0]["response_format"] == schema + assert seen[0]["logprobs"] is True + assert seen[0]["logit_bias"] == {"1234": -100} + assert seen[0]["frequency_penalty"] == 0.25 + assert seen[0]["seed"] == 42 + # Still pinned alongside it: structured outputs buy no reasoning effort. + assert seen[0]["reasoning"] == {"effort": "medium", "exclude": True} + assert seen[0]["model"] == V7_MODEL + + # And the answer survives the response allowlist unmangled, logprobs + # included -- forwarding a request field whose response is stripped would + # be a silent no-op. + returned = json.loads(bytes(response.body)) + choice = returned["choices"][0] + assert choice["message"]["content"] == '{"color":"blue"}' + assert json.loads(choice["message"]["content"]) == {"color": "blue"} + assert choice["logprobs"]["content"][0]["token"] == "blue" + + +@pytest.mark.asyncio +async def test_unknown_request_field_names_itself_in_the_error( + session_maker: async_sessionmaker[Any], +) -> None: + """The legibility backstop: a refusal must say which key it refused. + + Whatever the allowlist admits, something is eventually refused, and the + miner's only channel is the harness's stderr. ``Cooking`` could not discover + ``reasoning`` was the problem because this string used to be the bare + ``"unsupported inference parameter"``. + """ + config = _config() + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + grant_id, bearer, generation = await _seed_v7_grant( + session_maker, + config=config, + public_key=base64.urlsafe_b64encode(public).decode().rstrip("="), + ) + + async def handler(_: httpx.Request) -> httpx.Response: # pragma: no cover + raise AssertionError("refused requests must never reach the provider") + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = _App(_State(config=config, session_maker=session_maker, client=client)) + body = json.dumps( + { + "model": V7_MODEL, + "messages": [{"role": "user", "content": "hello"}], + "plugins": [{"id": "web"}], + "transforms": ["middle-out"], + } + ).encode() + + from fastapi import HTTPException + + try: + with pytest.raises(HTTPException) as refused: + await proxy_chat_completions( + **_signed_request( + app=app, + grant_id=grant_id, + bearer=bearer, + generation=generation, + private=private, + body=body, + ) + ) + finally: + await client.aclose() + + assert refused.value.status_code == 400 + detail = str(refused.value.detail) + assert "plugins" in detail + assert "transforms" in detail