From 909fa10fb8425d67c6d141478d5fe2602699bb47 Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Sat, 16 May 2026 19:31:42 -0500 Subject: [PATCH 1/2] =?UTF-8?q?fix(llm):=20OpenRouter=20reasoning-off=20?= =?UTF-8?q?=E2=80=94=20use=20effort=3Dminimal,=20not=20enabled=3Dfalse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenRouter rejects `reasoning.enabled=false` for reasoning-mandatory models like openai/gpt-5 with HTTP 400: "Reasoning is mandatory for this endpoint and cannot be disabled." Verified via direct API call against openai/gpt-5 (2026-05-16): POST /api/v1/chat/completions with reasoning.enabled=false → 400 POST /api/v1/chat/completions with reasoning.effort="minimal" → 200, reasoning_tokens=0, cost $0.0002 on a trivial prompt. Replace the OpenRouter branch of _build_reasoning_off_extras to send `reasoning.effort=minimal` instead. This key is honoured by: * reasoning-mandatory models (openai/gpt-5, etc.): emit 0 reasoning tokens — the desired behaviour. The model still routes through the reasoning endpoint but produces no reasoning content. * reasoning-capable models (openai/gpt-5-chat, anthropic/claude-sonnet-4.6 in thinking mode, etc.): treats as lowest-effort setting. * non-reasoning models (qwen, llama, etc.): silently ignored. Discovered while running the RATCHET 5-vendor CRCv2 extension smoke test (CIRISAI/RATCHET workflow crcv2_5vendor.yml). The openai/gpt-5 sweep was failing every chain at the API layer with 400 errors, masquerading as "slow" because qa_runner retries on 4xx errors. Update test_openrouter_carries_reasoning_enabled_false → test_openrouter_carries_reasoning_effort_minimal to reflect the new expected payload. Update test_openrouter_includes_provider_config's assertion accordingly. All 19 reasoning/openrouter tests pass. This is a backwards-compatible fix at the LLM-routing level — the existing 3-vendor CRCv2 cohort (Gemini, Llama, Qwen) all use models that ignore the `effort` key, so re-running them produces identical behaviour. Co-Authored-By: Claude Opus 4.7 --- .../logic/services/runtime/llm_service/service.py | 12 +++++++++++- .../llm_service/test_llm_service_coverage.py | 15 +++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ciris_engine/logic/services/runtime/llm_service/service.py b/ciris_engine/logic/services/runtime/llm_service/service.py index c741b60c0a..40484348de 100644 --- a/ciris_engine/logic/services/runtime/llm_service/service.py +++ b/ciris_engine/logic/services/runtime/llm_service/service.py @@ -1547,7 +1547,17 @@ def _build_reasoning_off_extras(base_url: str, model_name: str) -> Dict[str, Any model = (model_name or "").lower() if "openrouter.ai" in base: - return {"reasoning": {"enabled": False}} + # OpenRouter accepts `reasoning.enabled=false` for *some* models but + # explicitly rejects it for reasoning-mandatory models like + # `openai/gpt-5` (400 "Reasoning is mandatory for this endpoint and + # cannot be disabled."). Sending `reasoning.effort="minimal"` works + # universally: reasoning-mandatory models honour it (emit + # 0 reasoning tokens), reasoning-capable models treat it as the + # lowest-effort setting, non-reasoning models silently ignore it. + # Verified 2026-05-16 via direct OpenRouter API call against + # `openai/gpt-5` (returns content, reasoning_tokens=0, cost ~$0.0002 + # for a trivial prompt). + return {"reasoning": {"effort": "minimal"}} if "together" in base: # Models live in BOTH families on this endpoint; layer both keys. diff --git a/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py b/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py index 7e14297430..fbc9f992be 100644 --- a/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py +++ b/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py @@ -325,9 +325,14 @@ def test_deepinfra_carries_vllm_and_reasoning_enabled_keys(self): assert extra_body["chat_template_kwargs"] == {"enable_thinking": False} assert extra_body["reasoning"] == {"enabled": False} - def test_openrouter_carries_reasoning_enabled_false(self): + def test_openrouter_carries_reasoning_effort_minimal(self): + # OpenRouter rejects reasoning.enabled=false for reasoning-mandatory + # models like openai/gpt-5 (400 "Reasoning is mandatory…"). The safe + # universal key is reasoning.effort=minimal — honoured by reasoning- + # mandatory models (0 reasoning tokens emitted), treated as lowest + # setting by reasoning-capable models, silently ignored otherwise. extra_body = self._build_for("https://openrouter.ai/api/v1", "kimi/k2-0130-preview")["extra_body"] - assert extra_body["reasoning"] == {"enabled": False} + assert extra_body["reasoning"] == {"effort": "minimal"} def test_local_endpoint_carries_vllm_key(self): extra_body = self._build_for("http://localhost:8080/v1", "llama")["extra_body"] @@ -453,9 +458,11 @@ def test_openrouter_includes_provider_config(self): retry_state=RetryState(), ) - # Verify reasoning is disabled + # Verify reasoning is suppressed via effort=minimal + # (universal OpenRouter key — accepted by reasoning-mandatory models + # like openai/gpt-5 which reject `enabled=false` with 400). assert "extra_body" in extra_kwargs extra_body = extra_kwargs["extra_body"] # Provider config may or may not be present depending on env assert "reasoning" in extra_body - assert extra_body["reasoning"]["enabled"] is False + assert extra_body["reasoning"] == {"effort": "minimal"} From d5b78aa82f63c348c4c6b5cab7b130650f8f1c48 Mon Sep 17 00:00:00 2001 From: Eric Moore Date: Mon, 25 May 2026 08:49:20 -0500 Subject: [PATCH 2/2] =?UTF-8?q?fix(llm):=20scope=20effort=3Dminimal=20to?= =?UTF-8?q?=20gpt-5=20reasoning=20family=20=E2=80=94=20preserve=20enabled?= =?UTF-8?q?=3Dfalse=20elsewhere?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses codex P2 on PR #769. The previous version universally swapped reasoning.enabled=false → reasoning.effort=minimal on OpenRouter, but effort=minimal is the *lowest-effort* setting on reasoning-capable models, not a disable. That re-enables reasoning tokens on claude-sonnet-4.6, openai/gpt-5-chat, deepseek, etc. — violating the reasoning-off contract and adding latency/cost. Correct behavior: - Default OpenRouter path keeps reasoning.enabled=false (universal disable). - gpt-5 reasoning family (openai/gpt-5{,-mini,-nano}) takes effort=minimal because OpenRouter rejects enabled=false there with 400 "Reasoning is mandatory for this endpoint and cannot be disabled." - openai/gpt-5-chat is reasoning-capable, not mandatory — stays on enabled=false alongside the other reasoning-capable models. Tests: - openrouter_default_carries_reasoning_enabled_false - openrouter_gpt5_uses_effort_minimal (gpt-5/gpt-5-mini/gpt-5-nano) - openrouter_gpt5_chat_stays_on_enabled_false - openrouter_includes_provider_config asserts enabled=false for gpt-4o Co-Authored-By: Claude Opus 4.7 --- .../services/runtime/llm_service/service.py | 29 +++++++++----- .../llm_service/test_llm_service_coverage.py | 40 ++++++++++++++----- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/ciris_engine/logic/services/runtime/llm_service/service.py b/ciris_engine/logic/services/runtime/llm_service/service.py index 40484348de..b990cd7951 100644 --- a/ciris_engine/logic/services/runtime/llm_service/service.py +++ b/ciris_engine/logic/services/runtime/llm_service/service.py @@ -1547,17 +1547,24 @@ def _build_reasoning_off_extras(base_url: str, model_name: str) -> Dict[str, Any model = (model_name or "").lower() if "openrouter.ai" in base: - # OpenRouter accepts `reasoning.enabled=false` for *some* models but - # explicitly rejects it for reasoning-mandatory models like - # `openai/gpt-5` (400 "Reasoning is mandatory for this endpoint and - # cannot be disabled."). Sending `reasoning.effort="minimal"` works - # universally: reasoning-mandatory models honour it (emit - # 0 reasoning tokens), reasoning-capable models treat it as the - # lowest-effort setting, non-reasoning models silently ignore it. - # Verified 2026-05-16 via direct OpenRouter API call against - # `openai/gpt-5` (returns content, reasoning_tokens=0, cost ~$0.0002 - # for a trivial prompt). - return {"reasoning": {"effort": "minimal"}} + # Reasoning-OFF contract: pass `reasoning.enabled=false` so any + # reasoning-capable model that honours it (claude-sonnet-4.6, + # openai/gpt-5-chat, deepseek, etc.) emits zero reasoning tokens + # as before. `effort=minimal` is *not* a disable on those models — + # it's the lowest-effort setting and still emits reasoning tokens. + # + # OpenRouter explicitly rejects `reasoning.enabled=false` for the + # gpt-5 reasoning family with 400 "Reasoning is mandatory for this + # endpoint and cannot be disabled." (verified 2026-05-16 against + # `openai/gpt-5`). For that narrow case fall back to + # `effort=minimal`, which the gpt-5 family honours by emitting + # 0 reasoning tokens (cost ~$0.0002 on a trivial prompt). The + # chat-tuned variant `openai/gpt-5-chat` is reasoning-capable, not + # reasoning-mandatory, and stays on the universal `enabled=false` + # path. + if model.startswith("openai/gpt-5") and "chat" not in model: + return {"reasoning": {"effort": "minimal"}} + return {"reasoning": {"enabled": False}} if "together" in base: # Models live in BOTH families on this endpoint; layer both keys. diff --git a/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py b/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py index fbc9f992be..6f49ab852e 100644 --- a/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py +++ b/tests/ciris_engine/logic/services/runtime/llm_service/test_llm_service_coverage.py @@ -325,14 +325,32 @@ def test_deepinfra_carries_vllm_and_reasoning_enabled_keys(self): assert extra_body["chat_template_kwargs"] == {"enable_thinking": False} assert extra_body["reasoning"] == {"enabled": False} - def test_openrouter_carries_reasoning_effort_minimal(self): - # OpenRouter rejects reasoning.enabled=false for reasoning-mandatory - # models like openai/gpt-5 (400 "Reasoning is mandatory…"). The safe - # universal key is reasoning.effort=minimal — honoured by reasoning- - # mandatory models (0 reasoning tokens emitted), treated as lowest - # setting by reasoning-capable models, silently ignored otherwise. + def test_openrouter_default_carries_reasoning_enabled_false(self): + # Default reasoning-OFF contract on OpenRouter: send `enabled=false` + # so reasoning-capable models (claude, gpt-5-chat, deepseek, …) emit + # zero reasoning tokens. `effort=minimal` is *not* a disable on those + # models, so we only use it for the narrow reasoning-mandatory case + # below. extra_body = self._build_for("https://openrouter.ai/api/v1", "kimi/k2-0130-preview")["extra_body"] - assert extra_body["reasoning"] == {"effort": "minimal"} + assert extra_body["reasoning"] == {"enabled": False} + + def test_openrouter_gpt5_uses_effort_minimal(self): + # OpenRouter rejects reasoning.enabled=false for the gpt-5 reasoning + # family with 400 "Reasoning is mandatory for this endpoint and cannot + # be disabled." (verified 2026-05-16 against openai/gpt-5). For that + # narrow case we fall back to effort=minimal, which gpt-5 honours by + # emitting 0 reasoning tokens. + for model in ("openai/gpt-5", "openai/gpt-5-mini", "openai/gpt-5-nano"): + extra_body = self._build_for("https://openrouter.ai/api/v1", model)["extra_body"] + assert extra_body["reasoning"] == {"effort": "minimal"}, model + + def test_openrouter_gpt5_chat_stays_on_enabled_false(self): + # gpt-5-chat is the chat-tuned (reasoning-capable, not mandatory) + # variant and honours reasoning.enabled=false. It must stay on the + # universal disable path so it doesn't accidentally re-enable + # reasoning at lowest-effort under our reasoning-OFF contract. + extra_body = self._build_for("https://openrouter.ai/api/v1", "openai/gpt-5-chat")["extra_body"] + assert extra_body["reasoning"] == {"enabled": False} def test_local_endpoint_carries_vllm_key(self): extra_body = self._build_for("http://localhost:8080/v1", "llama")["extra_body"] @@ -458,11 +476,11 @@ def test_openrouter_includes_provider_config(self): retry_state=RetryState(), ) - # Verify reasoning is suppressed via effort=minimal - # (universal OpenRouter key — accepted by reasoning-mandatory models - # like openai/gpt-5 which reject `enabled=false` with 400). + # gpt-4o is reasoning-capable, not reasoning-mandatory — it honours + # `enabled=false`. Reasoning-mandatory gpt-5 models would take the + # `effort=minimal` fallback instead (covered separately above). assert "extra_body" in extra_kwargs extra_body = extra_kwargs["extra_body"] # Provider config may or may not be present depending on env assert "reasoning" in extra_body - assert extra_body["reasoning"] == {"effort": "minimal"} + assert extra_body["reasoning"] == {"enabled": False}