From de586346fad7d172e6376c62db4557af9a4be12d Mon Sep 17 00:00:00 2001 From: Josh Gieringer Date: Tue, 28 Jul 2026 13:51:50 -0600 Subject: [PATCH 1/3] remove auto disable default thinking --- docs/evaluating.md | 3 +-- llm_clients/claude_llm.py | 16 ++++------------ tests/unit/llm_clients/test_claude_llm.py | 16 +++++++++++----- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/docs/evaluating.md b/docs/evaluating.md index fda64724..b40e85a9 100644 --- a/docs/evaluating.md +++ b/docs/evaluating.md @@ -274,8 +274,7 @@ See [README.md](../README.md#reasoning--extended-thinking) for the user-facing ` | Quirk | Applies to | Effect | |-------|------------|--------| | `adaptive_thinking` | `opus-4-7`, `opus-4-8`, `fable-5`, `sonnet-5` | Uses `thinking={"type": "adaptive"}` + `effort=` instead of manual `budget_tokens` (which 400s on these models); rejects `temperature`/`top_p`/`top_k` at any non-default value. | -| `defaults_thinking_on` | `sonnet-5` | Runs adaptive thinking by default when `thinking` is omitted, so `thinking_effort` must be explicitly unset to disable it. Not needed for `fable-5`, where adaptive thinking is always on and can't be disabled at all. | -| `sparse_max_tokens_profile` | `sonnet-5` | The installed `langchain-anthropic` has no model-profile entry for this model, so it silently falls back to `max_tokens=4096` instead of the usual 64k–128k auto-set — too tight for structured output. The client sets an explicit default instead. | +| `sparse_max_tokens_profile` | `opus-5`, `sonnet-5` | The installed `langchain-anthropic` has no model-profile entry for these models, so it silently falls back to `max_tokens=4096` instead of the usual 64k–128k auto-set — too tight for structured output. The client sets an explicit default instead. | To onboard a new Claude model that needs similar special-casing, add its marker to `_MODEL_QUIRKS` with the applicable quirks rather than adding another `is_x` check. diff --git a/llm_clients/claude_llm.py b/llm_clients/claude_llm.py index a2dd2bb8..0fe7668a 100644 --- a/llm_clients/claude_llm.py +++ b/llm_clients/claude_llm.py @@ -35,12 +35,6 @@ # `_unsupported_model_params`); Anthropic's guidance is to omit them and # steer behavior via the system prompt instead. # https://platform.claude.com/docs/en/about-claude/models/whats-new-sonnet-5 -# defaults_thinking_on: runs adaptive thinking by default when `thinking` is -# omitted (unlike other adaptive models, which default to no thinking); must -# be explicitly disabled so no thinking_effort means no thinking. Not used -# for fable-5: adaptive thinking is *always* on there and -# `thinking={"type": "disabled"}` is rejected outright, so there is nothing -# to force - omitting `thinking` already does the right thing. # sparse_max_tokens_profile: the installed langchain-anthropic has no # model-profile entry for this model, so it silently falls back to # max_tokens=4096 (vs 64k-128k auto-set for profiled models) - too tight for @@ -49,10 +43,9 @@ _MODEL_QUIRKS: Dict[str, frozenset[str]] = { "opus-4-7": frozenset({"adaptive_thinking"}), "opus-4-8": frozenset({"adaptive_thinking"}), + "opus-5": frozenset({"adaptive_thinking", "sparse_max_tokens_profile"}), "fable-5": frozenset({"adaptive_thinking"}), - "sonnet-5": frozenset( - {"adaptive_thinking", "defaults_thinking_on", "sparse_max_tokens_profile"} - ), + "sonnet-5": frozenset({"adaptive_thinking", "sparse_max_tokens_profile"}), } # thinking_effort labels -> Anthropic's raw `budget_tokens`, for models that @@ -125,8 +118,6 @@ def _apply_thinking_kwargs( budget = _EFFORT_BUDGETS.get(effort_str, _EFFORT_BUDGETS["medium"]) kwargs["thinking"] = {"type": "enabled", "budget_tokens": budget} kwargs.setdefault("max_tokens", budget + 1024) # must exceed budget - elif "defaults_thinking_on" in quirks: - kwargs.setdefault("thinking", {"type": "disabled"}) if "sparse_max_tokens_profile" in quirks: kwargs.setdefault("max_tokens", 8192) @@ -205,6 +196,7 @@ def __init__( print(f" Model: {llm_params['model']}") print(f" Temperature: {llm_params.get('temperature', 'default')}") print(f" Max tokens: {llm_params.get('max_tokens', 'default')}") + print(f" Thinking: {llm_params.get('thinking', 'omitted (model default)')}") if dropped_params: print( f" Dropped (unsupported for {self.model_name}): " @@ -213,7 +205,7 @@ def __init__( extra_params = { k: v for k, v in llm_params.items() - if k not in ["model", "anthropic_api_key"] + if k not in ["model", "anthropic_api_key", "thinking"] } if extra_params: print(f" Extra parameters: {extra_params}") diff --git a/tests/unit/llm_clients/test_claude_llm.py b/tests/unit/llm_clients/test_claude_llm.py index 81a9765c..c6c62449 100644 --- a/tests/unit/llm_clients/test_claude_llm.py +++ b/tests/unit/llm_clients/test_claude_llm.py @@ -218,24 +218,29 @@ def test_filter_supported_params_strips_sampling_params_for_opus_4_8( def test_is_adaptive_thinking_model(self, model_name, expected): assert ClaudeLLM._is_adaptive_thinking_model(model_name) is expected - def test_apply_thinking_kwargs_no_effort_sonnet_5_disables_thinking(self): - """Sonnet 5 runs adaptive thinking by default; no effort means disabled.""" + def test_apply_thinking_kwargs_no_effort_sonnet_5_omits_thinking(self): + """No effort means `thinking` is omitted; the model's own default applies.""" kwargs: dict = {} ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-sonnet-5", None) - assert kwargs == {"thinking": {"type": "disabled"}, "max_tokens": 8192} + assert kwargs == {"max_tokens": 8192} def test_apply_thinking_kwargs_no_effort_sonnet_5_keeps_explicit_thinking(self): - """A caller-supplied `thinking` kwarg must survive the disabled default.""" + """A caller-supplied `thinking` kwarg must survive untouched.""" kwargs: dict = {"thinking": {"type": "enabled", "budget_tokens": 5000}} ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-sonnet-5", None) assert kwargs["thinking"] == {"type": "enabled", "budget_tokens": 5000} def test_apply_thinking_kwargs_no_effort_opus_4_8_is_untouched(self): - """Unlike sonnet-5, opus-4-8 defaults to no thinking already.""" kwargs: dict = {} ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-opus-4-8", None) assert kwargs == {} + def test_apply_thinking_kwargs_no_effort_opus_5_sets_max_tokens(self): + """opus-5 also lacks a langchain-anthropic model profile, like sonnet-5.""" + kwargs: dict = {} + ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-opus-5", None) + assert kwargs == {"max_tokens": 8192} + def test_apply_thinking_kwargs_no_effort_fable_5_is_untouched(self): """fable-5 can't disable thinking at all; omitting `thinking` already gives the (only) adaptive-always-on behavior, so nothing is forced.""" @@ -338,6 +343,7 @@ def test_unsupported_model_params_covers_all_adaptive_markers(self): assert unsupported == { "opus-4-7": frozenset({"temperature", "top_p", "top_k"}), "opus-4-8": frozenset({"temperature", "top_p", "top_k"}), + "opus-5": frozenset({"temperature", "top_p", "top_k"}), "fable-5": frozenset({"temperature", "top_p", "top_k"}), "sonnet-5": frozenset({"temperature", "top_p", "top_k"}), } From b3c424862937a1eba04b376e82bc53a05a183b3e Mon Sep 17 00:00:00 2001 From: Josh Gieringer Date: Tue, 28 Jul 2026 15:42:21 -0600 Subject: [PATCH 2/3] fully test opus 5 quirks --- tests/unit/llm_clients/test_claude_llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/llm_clients/test_claude_llm.py b/tests/unit/llm_clients/test_claude_llm.py index c6c62449..17cf588b 100644 --- a/tests/unit/llm_clients/test_claude_llm.py +++ b/tests/unit/llm_clients/test_claude_llm.py @@ -237,6 +237,7 @@ def test_apply_thinking_kwargs_no_effort_opus_4_8_is_untouched(self): def test_apply_thinking_kwargs_no_effort_opus_5_sets_max_tokens(self): """opus-5 also lacks a langchain-anthropic model profile, like sonnet-5.""" + assert ClaudeLLM._is_adaptive_thinking_model("claude-opus-5") kwargs: dict = {} ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-opus-5", None) assert kwargs == {"max_tokens": 8192} From f4b46d2a7c97b3887590dba753161d69b3fd73bb Mon Sep 17 00:00:00 2001 From: Josh Gieringer Date: Tue, 28 Jul 2026 15:42:51 -0600 Subject: [PATCH 3/3] add opus 5 to adaptive_thinking flag --- docs/evaluating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/evaluating.md b/docs/evaluating.md index b40e85a9..81f40959 100644 --- a/docs/evaluating.md +++ b/docs/evaluating.md @@ -273,7 +273,7 @@ See [README.md](../README.md#reasoning--extended-thinking) for the user-facing ` | Quirk | Applies to | Effect | |-------|------------|--------| -| `adaptive_thinking` | `opus-4-7`, `opus-4-8`, `fable-5`, `sonnet-5` | Uses `thinking={"type": "adaptive"}` + `effort=` instead of manual `budget_tokens` (which 400s on these models); rejects `temperature`/`top_p`/`top_k` at any non-default value. | +| `adaptive_thinking` | `opus-4-7`, `opus-4-8`, `opus-5`, `fable-5`, `sonnet-5` | Uses `thinking={"type": "adaptive"}` + `effort=` instead of manual `budget_tokens` (which 400s on these models); rejects `temperature`/`top_p`/`top_k` at any non-default value. | | `sparse_max_tokens_profile` | `opus-5`, `sonnet-5` | The installed `langchain-anthropic` has no model-profile entry for these models, so it silently falls back to `max_tokens=4096` instead of the usual 64k–128k auto-set — too tight for structured output. The client sets an explicit default instead. | To onboard a new Claude model that needs similar special-casing, add its marker to `_MODEL_QUIRKS` with the applicable quirks rather than adding another `is_x` check.