From 41e77548232db825bfd878c0d4395595dca6b4b1 Mon Sep 17 00:00:00 2001 From: Michael Sitarzewski Date: Mon, 22 Jun 2026 00:34:44 -0500 Subject: [PATCH] fix(anthropic): stop sending temperature to models that reject it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic's newest thinking models (Opus 4.8, 4.7) deprecated the `temperature` param and return a 400 ("`temperature` is deprecated for this model"). The provider sent it unconditionally, breaking any consensus run that used those models. Older models (Opus 4.6, Sonnet, Haiku) still accept temperature. Mirrors the existing OpenAI NO_TEMPERATURE_MODELS approach: a new ANTHROPIC_NO_TEMPERATURE_MODELS set (verified empirically against the live API) gates the param in both send() and stream(). Also extends scripts/refresh_catalog.py to probe Anthropic temperature support (it only probed OpenAI before — which is why this slipped when Opus 4.8/4.7 were added), so the set stays maintained. Fixes a test-isolation issue in the Cloudflare config tests exposed once CLOUDFLARE_* vars are present in .env (load_dotenv repopulates deleted vars; use empty strings instead). Verified live: Opus 4.8 send/stream now succeed. 1675 tests pass, mypy + ruff clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EkrekgzMAQko92UkjnXhHL --- scripts/refresh_catalog.py | 55 +++++++++++++++++++++++++- src/duh/providers/anthropic.py | 13 ++++-- src/duh/providers/catalog.py | 9 +++++ tests/unit/test_config.py | 14 +++++-- tests/unit/test_providers_anthropic.py | 42 ++++++++++++++++++++ 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/scripts/refresh_catalog.py b/scripts/refresh_catalog.py index 1c3bdd5..ee0f81b 100644 --- a/scripts/refresh_catalog.py +++ b/scripts/refresh_catalog.py @@ -38,7 +38,11 @@ from dotenv import load_dotenv -from duh.providers.catalog import MODEL_CATALOG, NO_TEMPERATURE_MODELS +from duh.providers.catalog import ( + ANTHROPIC_NO_TEMPERATURE_MODELS, + MODEL_CATALOG, + NO_TEMPERATURE_MODELS, +) # truefoundry provider directory names (differ from our provider_id keys) _TF_BASE = "https://raw.githubusercontent.com/truefoundry/models/main/providers" @@ -196,6 +200,33 @@ def probe_openai_temperature(model_id: str) -> bool | None: return True +def probe_anthropic_temperature(model_id: str) -> bool | None: + """True if the model accepts temperature, False if it rejects, None on error.""" + key = os.environ.get("ANTHROPIC_API_KEY") + if not key: + return None + resp = _post_json( + "https://api.anthropic.com/v1/messages", + { + "x-api-key": key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + { + "model": model_id, + "max_tokens": 16, + "temperature": 0.7, + "messages": [{"role": "user", "content": "hi"}], + }, + ) + if resp is None: + return None + err = resp.get("error") if isinstance(resp, dict) else None + if isinstance(err, dict): + return False if "temperature" in err.get("message", "") else None + return True + + # ── report ───────────────────────────────────────────────────── @@ -292,6 +323,28 @@ def main() -> int: else: print(f" {mid}: ok ({'no-temp' if rejects else 'temp'})") + print("\n## BEHAVIOR — Anthropic temperature probe vs catalog set") + anthropic_ids = {m["model_id"] for m in MODEL_CATALOG.get("anthropic", [])} + for mid in sorted(anthropic_ids): + accepts = probe_anthropic_temperature(mid) + if accepts is None: + print(f" {mid}: probe inconclusive") + continue + listed = mid in ANTHROPIC_NO_TEMPERATURE_MODELS + rejects = not accepts + if rejects and not listed: + print( + f" {mid}: REJECTS temp but NOT in " + "ANTHROPIC_NO_TEMPERATURE_MODELS — ADD" + ) + elif accepts and listed: + print( + f" {mid}: accepts temp but IS in " + "ANTHROPIC_NO_TEMPERATURE_MODELS — REMOVE" + ) + else: + print(f" {mid}: ok ({'no-temp' if rejects else 'temp'})") + if args.write_snapshot: _SNAPSHOT.write_text(json.dumps(new_snapshot, indent=2, sort_keys=True) + "\n") print(f"\nSnapshot written to {_SNAPSHOT}") diff --git a/src/duh/providers/anthropic.py b/src/duh/providers/anthropic.py index 42f6d1b..7c22802 100644 --- a/src/duh/providers/anthropic.py +++ b/src/duh/providers/anthropic.py @@ -23,7 +23,11 @@ TokenUsage, ToolCallData, ) -from duh.providers.catalog import MODEL_CATALOG, PROVIDER_CAPS +from duh.providers.catalog import ( + ANTHROPIC_NO_TEMPERATURE_MODELS, + MODEL_CATALOG, + PROVIDER_CAPS, +) if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -120,10 +124,12 @@ async def send( kwargs: dict[str, Any] = { "model": model_id, "max_tokens": max_tokens, - "temperature": temperature, "system": system, "messages": api_messages, } + # Newest thinking models reject temperature; older ones accept it. + if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences if tools: @@ -243,10 +249,11 @@ async def stream( kwargs: dict[str, Any] = { "model": model_id, "max_tokens": max_tokens, - "temperature": temperature, "system": system, "messages": api_messages, } + if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences diff --git a/src/duh/providers/catalog.py b/src/duh/providers/catalog.py index 93b5a3a..b703043 100644 --- a/src/duh/providers/catalog.py +++ b/src/duh/providers/catalog.py @@ -297,6 +297,15 @@ "gpt-5.5", } +# ── Anthropic models that reject the temperature param ───────── +# The newest thinking models (Opus 4.7+) deprecated `temperature`; older +# models (Opus 4.6, Sonnet, Haiku) still accept it. Verified empirically. + +ANTHROPIC_NO_TEMPERATURE_MODELS: set[str] = { + "claude-opus-4-8", + "claude-opus-4-7", +} + # ── Providers that are challenger-only (not proposer-eligible) ── CHALLENGER_ONLY_PROVIDERS: set[str] = {"perplexity"} diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index c88d45f..3b99bd4 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -337,10 +337,16 @@ def test_cloudflare_configured_from_env(self, tmp_path, monkeypatch): ) def test_cloudflare_absent_without_env(self, tmp_path, monkeypatch): - """Missing either var -> no cloudflare provider.""" + """Missing either var -> no cloudflare provider. + + Set to empty rather than delete: load_config() runs load_dotenv(), + which would otherwise repopulate these from the repo .env. An empty + env var is treated as "existing" (not overridden) and as absent by + _resolve_cloudflare. + """ self._isolate(tmp_path, monkeypatch) - monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) - monkeypatch.delenv("CLOUDFLARE_WORKERS_AI_TOKEN", raising=False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "") + monkeypatch.setenv("CLOUDFLARE_WORKERS_AI_TOKEN", "") cfg = load_config() assert "cloudflare" not in cfg.providers @@ -349,7 +355,7 @@ def test_cloudflare_requires_both(self, tmp_path, monkeypatch): """Only the account ID (no token) -> not configured.""" self._isolate(tmp_path, monkeypatch) monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct123") - monkeypatch.delenv("CLOUDFLARE_WORKERS_AI_TOKEN", raising=False) + monkeypatch.setenv("CLOUDFLARE_WORKERS_AI_TOKEN", "") cfg = load_config() assert "cloudflare" not in cfg.providers diff --git a/tests/unit/test_providers_anthropic.py b/tests/unit/test_providers_anthropic.py index 35059f4..4bb980c 100644 --- a/tests/unit/test_providers_anthropic.py +++ b/tests/unit/test_providers_anthropic.py @@ -313,3 +313,45 @@ async def test_unhealthy_on_error(self): client.messages.create.side_effect = Exception("connection failed") provider = AnthropicProvider(client=client) assert await provider.health_check() is False + + +# ── Temperature handling (newest models reject it) ─────────────── + + +class TestTemperatureHandling: + async def test_omits_temperature_for_no_temp_model(self): + """Opus 4.8 rejects temperature -> it must not be sent.""" + client = _make_client() + provider = AnthropicProvider(client=client) + await provider.send( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-8", + temperature=0.7, + ) + call_kwargs = client.messages.stream.call_args.kwargs + assert "temperature" not in call_kwargs + + async def test_sends_temperature_for_regular_model(self): + """Opus 4.6 still accepts temperature -> it is sent.""" + client = _make_client() + provider = AnthropicProvider(client=client) + await provider.send( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-6", + temperature=0.55, + ) + call_kwargs = client.messages.stream.call_args.kwargs + assert call_kwargs["temperature"] == 0.55 + + async def test_stream_omits_temperature_for_no_temp_model(self): + """The streaming path also omits temperature for Opus 4.8.""" + client = _make_client() + provider = AnthropicProvider(client=client) + async for _ in provider.stream( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-8", + temperature=0.7, + ): + pass + call_kwargs = client.messages.stream.call_args.kwargs + assert "temperature" not in call_kwargs