diff --git a/scripts/refresh_catalog.py b/scripts/refresh_catalog.py index 1c3bdd5..ee0f81b 100644 --- a/scripts/refresh_catalog.py +++ b/scripts/refresh_catalog.py @@ -38,7 +38,11 @@ from dotenv import load_dotenv -from duh.providers.catalog import MODEL_CATALOG, NO_TEMPERATURE_MODELS +from duh.providers.catalog import ( + ANTHROPIC_NO_TEMPERATURE_MODELS, + MODEL_CATALOG, + NO_TEMPERATURE_MODELS, +) # truefoundry provider directory names (differ from our provider_id keys) _TF_BASE = "https://raw.githubusercontent.com/truefoundry/models/main/providers" @@ -196,6 +200,33 @@ def probe_openai_temperature(model_id: str) -> bool | None: return True +def probe_anthropic_temperature(model_id: str) -> bool | None: + """True if the model accepts temperature, False if it rejects, None on error.""" + key = os.environ.get("ANTHROPIC_API_KEY") + if not key: + return None + resp = _post_json( + "https://api.anthropic.com/v1/messages", + { + "x-api-key": key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + { + "model": model_id, + "max_tokens": 16, + "temperature": 0.7, + "messages": [{"role": "user", "content": "hi"}], + }, + ) + if resp is None: + return None + err = resp.get("error") if isinstance(resp, dict) else None + if isinstance(err, dict): + return False if "temperature" in err.get("message", "") else None + return True + + # ── report ───────────────────────────────────────────────────── @@ -292,6 +323,28 @@ def main() -> int: else: print(f" {mid}: ok ({'no-temp' if rejects else 'temp'})") + print("\n## BEHAVIOR — Anthropic temperature probe vs catalog set") + anthropic_ids = {m["model_id"] for m in MODEL_CATALOG.get("anthropic", [])} + for mid in sorted(anthropic_ids): + accepts = probe_anthropic_temperature(mid) + if accepts is None: + print(f" {mid}: probe inconclusive") + continue + listed = mid in ANTHROPIC_NO_TEMPERATURE_MODELS + rejects = not accepts + if rejects and not listed: + print( + f" {mid}: REJECTS temp but NOT in " + "ANTHROPIC_NO_TEMPERATURE_MODELS — ADD" + ) + elif accepts and listed: + print( + f" {mid}: accepts temp but IS in " + "ANTHROPIC_NO_TEMPERATURE_MODELS — REMOVE" + ) + else: + print(f" {mid}: ok ({'no-temp' if rejects else 'temp'})") + if args.write_snapshot: _SNAPSHOT.write_text(json.dumps(new_snapshot, indent=2, sort_keys=True) + "\n") print(f"\nSnapshot written to {_SNAPSHOT}") diff --git a/src/duh/providers/anthropic.py b/src/duh/providers/anthropic.py index 42f6d1b..7c22802 100644 --- a/src/duh/providers/anthropic.py +++ b/src/duh/providers/anthropic.py @@ -23,7 +23,11 @@ TokenUsage, ToolCallData, ) -from duh.providers.catalog import MODEL_CATALOG, PROVIDER_CAPS +from duh.providers.catalog import ( + ANTHROPIC_NO_TEMPERATURE_MODELS, + MODEL_CATALOG, + PROVIDER_CAPS, +) if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -120,10 +124,12 @@ async def send( kwargs: dict[str, Any] = { "model": model_id, "max_tokens": max_tokens, - "temperature": temperature, "system": system, "messages": api_messages, } + # Newest thinking models reject temperature; older ones accept it. + if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences if tools: @@ -243,10 +249,11 @@ async def stream( kwargs: dict[str, Any] = { "model": model_id, "max_tokens": max_tokens, - "temperature": temperature, "system": system, "messages": api_messages, } + if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences diff --git a/src/duh/providers/catalog.py b/src/duh/providers/catalog.py index 93b5a3a..b703043 100644 --- a/src/duh/providers/catalog.py +++ b/src/duh/providers/catalog.py @@ -297,6 +297,15 @@ "gpt-5.5", } +# ── Anthropic models that reject the temperature param ───────── +# The newest thinking models (Opus 4.7+) deprecated `temperature`; older +# models (Opus 4.6, Sonnet, Haiku) still accept it. Verified empirically. + +ANTHROPIC_NO_TEMPERATURE_MODELS: set[str] = { + "claude-opus-4-8", + "claude-opus-4-7", +} + # ── Providers that are challenger-only (not proposer-eligible) ── CHALLENGER_ONLY_PROVIDERS: set[str] = {"perplexity"} diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index c88d45f..3b99bd4 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -337,10 +337,16 @@ def test_cloudflare_configured_from_env(self, tmp_path, monkeypatch): ) def test_cloudflare_absent_without_env(self, tmp_path, monkeypatch): - """Missing either var -> no cloudflare provider.""" + """Missing either var -> no cloudflare provider. + + Set to empty rather than delete: load_config() runs load_dotenv(), + which would otherwise repopulate these from the repo .env. An empty + env var is treated as "existing" (not overridden) and as absent by + _resolve_cloudflare. + """ self._isolate(tmp_path, monkeypatch) - monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) - monkeypatch.delenv("CLOUDFLARE_WORKERS_AI_TOKEN", raising=False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "") + monkeypatch.setenv("CLOUDFLARE_WORKERS_AI_TOKEN", "") cfg = load_config() assert "cloudflare" not in cfg.providers @@ -349,7 +355,7 @@ def test_cloudflare_requires_both(self, tmp_path, monkeypatch): """Only the account ID (no token) -> not configured.""" self._isolate(tmp_path, monkeypatch) monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct123") - monkeypatch.delenv("CLOUDFLARE_WORKERS_AI_TOKEN", raising=False) + monkeypatch.setenv("CLOUDFLARE_WORKERS_AI_TOKEN", "") cfg = load_config() assert "cloudflare" not in cfg.providers diff --git a/tests/unit/test_providers_anthropic.py b/tests/unit/test_providers_anthropic.py index 35059f4..4bb980c 100644 --- a/tests/unit/test_providers_anthropic.py +++ b/tests/unit/test_providers_anthropic.py @@ -313,3 +313,45 @@ async def test_unhealthy_on_error(self): client.messages.create.side_effect = Exception("connection failed") provider = AnthropicProvider(client=client) assert await provider.health_check() is False + + +# ── Temperature handling (newest models reject it) ─────────────── + + +class TestTemperatureHandling: + async def test_omits_temperature_for_no_temp_model(self): + """Opus 4.8 rejects temperature -> it must not be sent.""" + client = _make_client() + provider = AnthropicProvider(client=client) + await provider.send( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-8", + temperature=0.7, + ) + call_kwargs = client.messages.stream.call_args.kwargs + assert "temperature" not in call_kwargs + + async def test_sends_temperature_for_regular_model(self): + """Opus 4.6 still accepts temperature -> it is sent.""" + client = _make_client() + provider = AnthropicProvider(client=client) + await provider.send( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-6", + temperature=0.55, + ) + call_kwargs = client.messages.stream.call_args.kwargs + assert call_kwargs["temperature"] == 0.55 + + async def test_stream_omits_temperature_for_no_temp_model(self): + """The streaming path also omits temperature for Opus 4.8.""" + client = _make_client() + provider = AnthropicProvider(client=client) + async for _ in provider.stream( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-8", + temperature=0.7, + ): + pass + call_kwargs = client.messages.stream.call_args.kwargs + assert "temperature" not in call_kwargs