From 68ac62e76a27c2247cf8b35c27b28e7939031ccd Mon Sep 17 00:00:00 2001 From: Michael Sitarzewski Date: Mon, 22 Jun 2026 10:19:35 -0500 Subject: [PATCH] feat(providers): self-healing temperature retry as a cross-provider safety net When a model newly rejects the `temperature` parameter, the provider now records it and retries the request once without temperature, instead of 400ing until the catalog static set is updated by hand. This class of bug hit users twice (gpt-5.5, then Opus 4.8). - new `src/duh/providers/temperature.py`: runtime-learned `_LEARNED_NO_TEMPERATURE` set + `omit_temperature` / `record_no_temperature` / `is_temperature_error` - OpenAI + Anthropic `send()`: temperature decision uses the learned set; on a temperature-related 400 (BadRequest), record the model and retry once without temperature. `stream()` honors the learned set. - Static catalog sets remain the fast path for known models; this only adds self-correction for unknown/new ones. Live-verified: with Opus 4.8 temporarily removed from the static set, send() hit a real Anthropic 400, retried without temperature, succeeded, and recorded the model. 1681 Python tests pass, mypy + ruff clean. Closes the open follow-up; memory bank updated to reflect it. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EkrekgzMAQko92UkjnXhHL --- memory-bank/activeContext.md | 13 ++++- memory-bank/progress.md | 6 +- .../2026-06/260622_temperature-self-heal.md | 43 ++++++++++++++ memory-bank/tasks/2026-06/README.md | 12 +++- memory-bank/toc.md | 1 + src/duh/providers/anthropic.py | 25 +++++++- src/duh/providers/openai.py | 25 +++++++- src/duh/providers/temperature.py | 39 +++++++++++++ tests/unit/test_providers_anthropic.py | 57 +++++++++++++++++++ tests/unit/test_providers_openai.py | 56 ++++++++++++++++++ 10 files changed, 264 insertions(+), 13 deletions(-) create mode 100644 memory-bank/tasks/2026-06/260622_temperature-self-heal.md create mode 100644 src/duh/providers/temperature.py diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index bd3363d..b9fedbb 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -41,10 +41,17 @@ (`ThreadDetail`) identically; export per-view via an `exportSlot` - Markdown parity confirmed (same ``; `.duh-prose` has no own font-size) +### Temperature Self-Heal (cross-provider safety net) +- New `src/duh/providers/temperature.py`: runtime-learned `_LEARNED_NO_TEMPERATURE` set + + `omit_temperature` / `record_no_temperature` / `is_temperature_error` +- OpenAI + Anthropic `send()` retry once without temperature on a temperature-related 400, record + the model, and skip it thereafter. `stream()` honors the learned set. Static catalog sets remain + the fast path. Live-verified against a real Anthropic 400. Closes the prior open follow-up. + ### End-of-session state -- **1677 Python + 204 Vitest tests**, mypy clean (63 files), ruff clean, build clean -- **Open follow-ups**: (1) self-healing temperature retry as a cross-provider safety net (this bug - hit twice: gpt-5.5, then Opus 4.8); (2) gitignore `web/tsconfig.tsbuildinfo` (tracked build artifact) +- **1681 Python + 204 Vitest tests**, mypy clean (64 files), ruff clean, build clean +- **Resolved this session**: self-healing temperature retry; `web/tsconfig.tsbuildinfo` gitignored +- **No open follow-ups outstanding from this session** --- diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 1b3c0c1..0629d18 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -12,9 +12,11 @@ Merged to `main` via PRs #16–#21 (details in `tasks/2026-06/README.md` and `ac - **Incremental persistence** (`IncrementalPersister`) across CLI/WS/REST — mid-run crash leaves a real partial thread - **Cloudflare Workers AI provider** with Zhipu GLM-5.2; `OpenAIProvider` generalized to any OpenAI-compatible host - **Unified `ConsensusReport`** for live + history; history now shows the executive summary +- **Temperature self-heal**: providers retry once without temperature on a temperature-related 400 + and learn the model (`src/duh/providers/temperature.py`) — cross-provider safety net, live-verified - **Prior (2026-06-21)**: token usage tracking end-to-end + `npm/like-duh` wrapper -- **Tests**: 1677 Python + 204 Vitest, mypy clean (63 files), ruff clean, build clean -- **Open follow-ups**: self-healing temperature retry (cross-provider); gitignore `web/tsconfig.tsbuildinfo` +- **Tests**: 1681 Python + 204 Vitest, mypy clean (64 files), ruff clean, build clean +- **No open follow-ups outstanding from this session** (temperature retry + tsbuildinfo both done) --- diff --git a/memory-bank/tasks/2026-06/260622_temperature-self-heal.md b/memory-bank/tasks/2026-06/260622_temperature-self-heal.md new file mode 100644 index 0000000..c5e41f2 --- /dev/null +++ b/memory-bank/tasks/2026-06/260622_temperature-self-heal.md @@ -0,0 +1,43 @@ +# 260622_temperature-self-heal + +## Objective +Add a cross-provider safety net so a model that newly rejects the `temperature` +parameter recovers automatically, instead of 400ing until the catalog is +updated by hand. (Closes the open follow-up — this class of bug had hit users +twice: gpt-5.5, then Opus 4.8.) + +## Outcome +- New `src/duh/providers/temperature.py`: a process-local `_LEARNED_NO_TEMPERATURE` + set plus helpers `omit_temperature(model_id, static_set)`, + `record_no_temperature(model_id)`, and `is_temperature_error(exc)`. +- **OpenAI + Anthropic `send()`**: the temperature decision now uses + `omit_temperature` (static catalog set OR runtime-learned). On a 400 whose + message mentions `temperature`, the provider records the model and **retries + once without temperature**. Future calls skip it from the start. +- Both `stream()` paths honor the learned set too (so a model learned via + `send` skips temperature in `stream`). `stream()` has no consumers in the + engine, so it relies on the shared learned set rather than its own retry. +- The static sets (`NO_TEMPERATURE_MODELS`, `ANTHROPIC_NO_TEMPERATURE_MODELS`) + remain the fast path for known cases; this only adds self-correction for the + unknown ones. + +## Files +- Created: `src/duh/providers/temperature.py` +- Modified: `src/duh/providers/openai.py`, `src/duh/providers/anthropic.py` +- Tests: `test_providers_openai.py`, `test_providers_anthropic.py` + (`TestTemperatureSelfHeal` — retry-without-temperature on a temperature 400, + model recorded; non-temperature 400 is NOT retried) + +## Validation +- 1681 Python tests, mypy clean (64 files), ruff clean. +- **Live**: with Opus 4.8 temporarily removed from the static set (simulating a + brand-new model), `send()` hit a real Anthropic 400 on the first attempt, + retried without temperature, succeeded, and recorded `claude-opus-4-8` in the + learned set. + +## Design Notes +- Retry only fires when temperature was actually sent AND the caught error is a + provider `BadRequestError` AND its message mentions `temperature` — so + unrelated 400s propagate normally. +- The learned set is process-local (resets on restart, repopulates on first use) + — no persistence needed; the catalog static sets are the durable record. diff --git a/memory-bank/tasks/2026-06/README.md b/memory-bank/tasks/2026-06/README.md index 2bf650e..f7aced1 100644 --- a/memory-bank/tasks/2026-06/README.md +++ b/memory-bank/tasks/2026-06/README.md @@ -51,6 +51,14 @@ - Markdown parity verified (same shared ``; `.duh-prose` sets no own font-size) - See: [260622_unified-consensus-report.md](./260622_unified-consensus-report.md) +## 2026-06-22: Temperature Self-Heal (cross-provider safety net) +- New `src/duh/providers/temperature.py`: runtime-learned no-temperature set + helpers +- OpenAI + Anthropic `send()` retry once without `temperature` on a temperature-related 400, + record the model, and skip it thereafter; `stream()` honors the learned set +- Static catalog sets stay the fast path; this self-corrects unknown/new models +- Live-verified against a real Anthropic 400. Closes the prior open follow-up. +- See: [260622_temperature-self-heal.md](./260622_temperature-self-heal.md) + --- -**End-of-session state (2026-06-22)**: 1677 Python + 204 Vitest tests passing, mypy clean -(63 files), ruff clean, build clean. All work merged to `main` via PRs #16–#21. +**End-of-session state (2026-06-22)**: 1681 Python + 204 Vitest tests passing, mypy clean +(64 files), ruff clean, build clean. All work merged to `main` via PRs #16–#22 (+ this fix). diff --git a/memory-bank/toc.md b/memory-bank/toc.md index ca7bb30..d53cff2 100644 --- a/memory-bank/toc.md +++ b/memory-bank/toc.md @@ -36,6 +36,7 @@ - [tasks/2026-06/260622_incremental-persistence.md](./tasks/2026-06/260622_incremental-persistence.md) — IncrementalPersister + REST unification - [tasks/2026-06/260622_cloudflare-glm-provider.md](./tasks/2026-06/260622_cloudflare-glm-provider.md) — Cloudflare Workers AI + Zhipu GLM-5.2 - [tasks/2026-06/260622_unified-consensus-report.md](./tasks/2026-06/260622_unified-consensus-report.md) — shared ConsensusReport + overview in history +- [tasks/2026-06/260622_temperature-self-heal.md](./tasks/2026-06/260622_temperature-self-heal.md) — cross-provider self-healing temperature retry ## Phase 0 Code (not memory bank — in repo root) - `phase0/` — Benchmark framework (config, models, prompts, methods, questions, runner, judge, analyze) diff --git a/src/duh/providers/anthropic.py b/src/duh/providers/anthropic.py index 7c22802..829ff37 100644 --- a/src/duh/providers/anthropic.py +++ b/src/duh/providers/anthropic.py @@ -28,6 +28,11 @@ MODEL_CATALOG, PROVIDER_CAPS, ) +from duh.providers.temperature import ( + is_temperature_error, + omit_temperature, + record_no_temperature, +) if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -128,7 +133,7 @@ async def send( "messages": api_messages, } # Newest thinking models reject temperature; older ones accept it. - if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + if not omit_temperature(model_id, ANTHROPIC_NO_TEMPERATURE_MODELS): kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences @@ -158,7 +163,21 @@ async def send( # timeout on non-streaming requests with large max_tokens. response = await self._collect_stream(kwargs) except anthropic.APIError as e: - raise _map_error(e) from e + # Self-heal: a model that newly rejects temperature -> record it and + # retry once without it (handles models not yet in the catalog set). + if ( + isinstance(e, anthropic.BadRequestError) + and "temperature" in kwargs + and is_temperature_error(e) + ): + record_no_temperature(model_id) + kwargs.pop("temperature", None) + try: + response = await self._collect_stream(kwargs) + except anthropic.APIError as retry_exc: + raise _map_error(retry_exc) from retry_exc + else: + raise _map_error(e) from e latency_ms = (time.monotonic() - start) * 1000 @@ -252,7 +271,7 @@ async def stream( "system": system, "messages": api_messages, } - if model_id not in ANTHROPIC_NO_TEMPERATURE_MODELS: + if not omit_temperature(model_id, ANTHROPIC_NO_TEMPERATURE_MODELS): kwargs["temperature"] = temperature if stop_sequences: kwargs["stop_sequences"] = stop_sequences diff --git a/src/duh/providers/openai.py b/src/duh/providers/openai.py index 0833dba..81b27a3 100644 --- a/src/duh/providers/openai.py +++ b/src/duh/providers/openai.py @@ -27,6 +27,11 @@ NO_TEMPERATURE_MODELS, PROVIDER_CAPS, ) +from duh.providers.temperature import ( + is_temperature_error, + omit_temperature, + record_no_temperature, +) if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -160,7 +165,7 @@ async def send( "max_completion_tokens": max_tokens, "messages": api_messages, } - if model_id not in _NO_TEMPERATURE_MODELS: + if not omit_temperature(model_id, _NO_TEMPERATURE_MODELS): kwargs["temperature"] = temperature if stop_sequences: kwargs["stop"] = stop_sequences @@ -190,7 +195,21 @@ async def send( try: response = await self._client.chat.completions.create(**kwargs) except openai.APIError as e: - raise _map_error(e) from e + # Self-heal: a model that newly rejects temperature -> record it and + # retry once without it (handles models not yet in the catalog set). + if ( + isinstance(e, openai.BadRequestError) + and "temperature" in kwargs + and is_temperature_error(e) + ): + record_no_temperature(model_id) + kwargs.pop("temperature", None) + try: + response = await self._client.chat.completions.create(**kwargs) + except openai.APIError as retry_exc: + raise _map_error(retry_exc) from retry_exc + else: + raise _map_error(e) from e latency_ms = (time.monotonic() - start) * 1000 @@ -250,7 +269,7 @@ async def stream( "messages": api_messages, "stream_options": {"include_usage": True}, } - if model_id not in _NO_TEMPERATURE_MODELS: + if not omit_temperature(model_id, _NO_TEMPERATURE_MODELS): kwargs["temperature"] = temperature if model_id in _REASONING_EFFORT_MODELS: kwargs["reasoning_effort"] = "high" diff --git a/src/duh/providers/temperature.py b/src/duh/providers/temperature.py new file mode 100644 index 0000000..7778e49 --- /dev/null +++ b/src/duh/providers/temperature.py @@ -0,0 +1,39 @@ +"""Self-healing for models that reject the ``temperature`` parameter. + +Static sets in ``catalog.py`` cover the known cases (OpenAI gpt-5.x reasoning +models via ``NO_TEMPERATURE_MODELS``; Anthropic Opus 4.7+ via +``ANTHROPIC_NO_TEMPERATURE_MODELS``). But a newly released model can drop +``temperature`` before the catalog is updated — which has bitten us twice +(gpt-5.5, then Opus 4.8). + +This module is the cross-provider safety net: when a request 400s with a +temperature-related message, the provider records the model here and retries +the request without ``temperature``. Future calls skip it from the start, so +the system self-corrects without a catalog change or a redeploy. +""" + +from __future__ import annotations + +# Models learned at runtime to reject temperature. Augments the static catalog +# sets; process-local (resets on restart, repopulates on first use). +_LEARNED_NO_TEMPERATURE: set[str] = set() + + +def omit_temperature(model_id: str, static_set: frozenset[str] | set[str]) -> bool: + """Return True if ``temperature`` should NOT be sent for this model.""" + return model_id in static_set or model_id in _LEARNED_NO_TEMPERATURE + + +def record_no_temperature(model_id: str) -> None: + """Remember that a model rejects temperature (after a 400).""" + _LEARNED_NO_TEMPERATURE.add(model_id) + + +def is_temperature_error(exc: Exception) -> bool: + """Heuristic: does this error look like 'temperature not supported'? + + Called only after a 400/BadRequest has been caught, so a message mention of + ``temperature`` is a reliable signal (e.g. "`temperature` is deprecated for + this model" / "temperature does not support 0.7 with this model"). + """ + return "temperature" in str(exc).lower() diff --git a/tests/unit/test_providers_anthropic.py b/tests/unit/test_providers_anthropic.py index 4bb980c..f35c01b 100644 --- a/tests/unit/test_providers_anthropic.py +++ b/tests/unit/test_providers_anthropic.py @@ -355,3 +355,60 @@ async def test_stream_omits_temperature_for_no_temp_model(self): pass call_kwargs = client.messages.stream.call_args.kwargs assert "temperature" not in call_kwargs + + +# ─── Temperature self-heal (cross-provider safety net) ──────── + + +class TestTemperatureSelfHeal: + def _bad_request(self, msg: str) -> anthropic.BadRequestError: + response = MagicMock() + response.status_code = 400 + response.headers = {} + return anthropic.BadRequestError(message=msg, response=response, body=None) + + async def test_retries_without_temperature_on_400(self): + """A 'temperature is deprecated' 400 is retried without temperature and + the model is recorded so later calls skip it.""" + from duh.providers import temperature as temp_mod + + temp_mod._LEARNED_NO_TEMPERATURE.discard("brand-new-claude") + client = _make_client() + stream_cm = client.messages.stream.return_value + client.messages.stream = MagicMock( + side_effect=[ + self._bad_request("`temperature` is deprecated for this model"), + stream_cm, + ] + ) + provider = AnthropicProvider(client=client) + try: + resp = await provider.send( + [PromptMessage(role="user", content="hi")], + "brand-new-claude", + temperature=0.7, + ) + assert resp.content is not None + assert client.messages.stream.call_count == 2 + first = client.messages.stream.call_args_list[0].kwargs + second = client.messages.stream.call_args_list[1].kwargs + assert "temperature" in first + assert "temperature" not in second + assert temp_mod.omit_temperature("brand-new-claude", set()) + finally: + temp_mod._LEARNED_NO_TEMPERATURE.discard("brand-new-claude") + + async def test_non_temperature_400_not_retried(self): + """A 400 unrelated to temperature is not retried.""" + client = _make_client() + client.messages.stream = MagicMock( + side_effect=self._bad_request("invalid request: bad messages") + ) + provider = AnthropicProvider(client=client) + with pytest.raises(Exception): # noqa: B017 + await provider.send( + [PromptMessage(role="user", content="hi")], + "claude-opus-4-6", + temperature=0.7, + ) + assert client.messages.stream.call_count == 1 diff --git a/tests/unit/test_providers_openai.py b/tests/unit/test_providers_openai.py index d98c63a..fd4505a 100644 --- a/tests/unit/test_providers_openai.py +++ b/tests/unit/test_providers_openai.py @@ -481,3 +481,59 @@ def test_unknown_provider_falls_back_to_openai_catalog(self): assert provider.provider_id == "mystery" # No catalog for "mystery" -> falls back to openai's known models. assert provider._known_models is not None + + +# ─── Temperature self-heal (cross-provider safety net) ──────── + + +class TestTemperatureSelfHeal: + def _bad_request(self, msg: str) -> openai.BadRequestError: + response = MagicMock() + response.status_code = 400 + response.headers = {} + return openai.BadRequestError(message=msg, response=response, body=None) + + async def test_retries_without_temperature_on_400(self): + """A temperature-deprecated 400 is retried without temperature and + the model is recorded so later calls skip it.""" + from duh.providers import temperature as temp_mod + + temp_mod._LEARNED_NO_TEMPERATURE.discard("brand-new-model") + client = _make_client() + client.chat.completions.create = AsyncMock( + side_effect=[ + self._bad_request( + "Unsupported value: 'temperature' does not support 0.7" + ), + _make_response(), + ] + ) + provider = OpenAIProvider(client=client) + try: + resp = await provider.send( + [PromptMessage(role="user", content="hi")], + "brand-new-model", + temperature=0.7, + ) + assert resp.content + assert client.chat.completions.create.call_count == 2 + first = client.chat.completions.create.call_args_list[0].kwargs + second = client.chat.completions.create.call_args_list[1].kwargs + assert "temperature" in first + assert "temperature" not in second + assert temp_mod.omit_temperature("brand-new-model", set()) + finally: + temp_mod._LEARNED_NO_TEMPERATURE.discard("brand-new-model") + + async def test_non_temperature_400_not_retried(self): + """A 400 unrelated to temperature is not retried.""" + client = _make_client() + client.chat.completions.create = AsyncMock( + side_effect=self._bad_request("messages: invalid role") + ) + provider = OpenAIProvider(client=client) + with pytest.raises(Exception): # noqa: B017 + await provider.send( + [PromptMessage(role="user", content="hi")], "gpt-4o", temperature=0.7 + ) + assert client.chat.completions.create.call_count == 1