Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions docs/evaluating.md
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,8 @@ See [README.md](../README.md#reasoning--extended-thinking) for the user-facing `

| Quirk | Applies to | Effect |
|-------|------------|--------|
| `adaptive_thinking` | `opus-4-7`, `opus-4-8`, `fable-5`, `sonnet-5` | Uses `thinking={"type": "adaptive"}` + `effort=<level>` instead of manual `budget_tokens` (which 400s on these models); rejects `temperature`/`top_p`/`top_k` at any non-default value. |
| `defaults_thinking_on` | `sonnet-5` | Runs adaptive thinking by default when `thinking` is omitted, so `thinking_effort` must be explicitly unset to disable it. Not needed for `fable-5`, where adaptive thinking is always on and can't be disabled at all. |
| `sparse_max_tokens_profile` | `sonnet-5` | The installed `langchain-anthropic` has no model-profile entry for this model, so it silently falls back to `max_tokens=4096` instead of the usual 64k–128k auto-set — too tight for structured output. The client sets an explicit default instead. |
| `adaptive_thinking` | `opus-4-7`, `opus-4-8`, `opus-5`, `fable-5`, `sonnet-5` | Uses `thinking={"type": "adaptive"}` + `effort=<level>` instead of manual `budget_tokens` (which 400s on these models); rejects `temperature`/`top_p`/`top_k` at any non-default value. |

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if there is a way of auto populating those by looking at the code

| `sparse_max_tokens_profile` | `opus-5`, `sonnet-5` | The installed `langchain-anthropic` has no model-profile entry for these models, so it silently falls back to `max_tokens=4096` instead of the usual 64k–128k auto-set — too tight for structured output. The client sets an explicit default instead. |

To onboard a new Claude model that needs similar special-casing, add its marker to `_MODEL_QUIRKS` with the applicable quirks rather than adding another `is_x` check.

Expand Down
16 changes: 4 additions & 12 deletions llm_clients/claude_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,6 @@
# `_unsupported_model_params`); Anthropic's guidance is to omit them and
# steer behavior via the system prompt instead.
# https://platform.claude.com/docs/en/about-claude/models/whats-new-sonnet-5
# defaults_thinking_on: runs adaptive thinking by default when `thinking` is
# omitted (unlike other adaptive models, which default to no thinking); must
# be explicitly disabled so no thinking_effort means no thinking. Not used
# for fable-5: adaptive thinking is *always* on there and
# `thinking={"type": "disabled"}` is rejected outright, so there is nothing
# to force - omitting `thinking` already does the right thing.
# sparse_max_tokens_profile: the installed langchain-anthropic has no
# model-profile entry for this model, so it silently falls back to
# max_tokens=4096 (vs 64k-128k auto-set for profiled models) - too tight for
Expand All @@ -49,10 +43,9 @@
_MODEL_QUIRKS: Dict[str, frozenset[str]] = {
"opus-4-7": frozenset({"adaptive_thinking"}),
"opus-4-8": frozenset({"adaptive_thinking"}),
"opus-5": frozenset({"adaptive_thinking", "sparse_max_tokens_profile"}),
"fable-5": frozenset({"adaptive_thinking"}),
"sonnet-5": frozenset(
{"adaptive_thinking", "defaults_thinking_on", "sparse_max_tokens_profile"}
),
"sonnet-5": frozenset({"adaptive_thinking", "sparse_max_tokens_profile"}),
}

# thinking_effort labels -> Anthropic's raw `budget_tokens`, for models that
Expand Down Expand Up @@ -125,8 +118,6 @@ def _apply_thinking_kwargs(
budget = _EFFORT_BUDGETS.get(effort_str, _EFFORT_BUDGETS["medium"])
kwargs["thinking"] = {"type": "enabled", "budget_tokens": budget}
kwargs.setdefault("max_tokens", budget + 1024) # must exceed budget
elif "defaults_thinking_on" in quirks:
kwargs.setdefault("thinking", {"type": "disabled"})

if "sparse_max_tokens_profile" in quirks:
kwargs.setdefault("max_tokens", 8192)
Expand Down Expand Up @@ -205,6 +196,7 @@ def __init__(
print(f" Model: {llm_params['model']}")
print(f" Temperature: {llm_params.get('temperature', 'default')}")
print(f" Max tokens: {llm_params.get('max_tokens', 'default')}")
print(f" Thinking: {llm_params.get('thinking', 'omitted (model default)')}")
if dropped_params:
print(
f" Dropped (unsupported for {self.model_name}): "
Expand All @@ -213,7 +205,7 @@ def __init__(
extra_params = {
k: v
for k, v in llm_params.items()
if k not in ["model", "anthropic_api_key"]
if k not in ["model", "anthropic_api_key", "thinking"]
}
if extra_params:
print(f" Extra parameters: {extra_params}")
Expand Down
17 changes: 12 additions & 5 deletions tests/unit/llm_clients/test_claude_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,24 +218,30 @@ def test_filter_supported_params_strips_sampling_params_for_opus_4_8(
def test_is_adaptive_thinking_model(self, model_name, expected):
assert ClaudeLLM._is_adaptive_thinking_model(model_name) is expected

def test_apply_thinking_kwargs_no_effort_sonnet_5_disables_thinking(self):
"""Sonnet 5 runs adaptive thinking by default; no effort means disabled."""
def test_apply_thinking_kwargs_no_effort_sonnet_5_omits_thinking(self):
"""No effort means `thinking` is omitted; the model's own default applies."""
kwargs: dict = {}
ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-sonnet-5", None)
assert kwargs == {"thinking": {"type": "disabled"}, "max_tokens": 8192}
assert kwargs == {"max_tokens": 8192}

def test_apply_thinking_kwargs_no_effort_sonnet_5_keeps_explicit_thinking(self):
"""A caller-supplied `thinking` kwarg must survive the disabled default."""
"""A caller-supplied `thinking` kwarg must survive untouched."""
kwargs: dict = {"thinking": {"type": "enabled", "budget_tokens": 5000}}
ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-sonnet-5", None)
assert kwargs["thinking"] == {"type": "enabled", "budget_tokens": 5000}

def test_apply_thinking_kwargs_no_effort_opus_4_8_is_untouched(self):
"""Unlike sonnet-5, opus-4-8 defaults to no thinking already."""
kwargs: dict = {}
ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-opus-4-8", None)
assert kwargs == {}

def test_apply_thinking_kwargs_no_effort_opus_5_sets_max_tokens(self):
"""opus-5 also lacks a langchain-anthropic model profile, like sonnet-5."""
assert ClaudeLLM._is_adaptive_thinking_model("claude-opus-5")
kwargs: dict = {}
ClaudeLLM._apply_thinking_kwargs(kwargs, "claude-opus-5", None)
assert kwargs == {"max_tokens": 8192}

def test_apply_thinking_kwargs_no_effort_fable_5_is_untouched(self):
"""fable-5 can't disable thinking at all; omitting `thinking` already
gives the (only) adaptive-always-on behavior, so nothing is forced."""
Expand Down Expand Up @@ -338,6 +344,7 @@ def test_unsupported_model_params_covers_all_adaptive_markers(self):
assert unsupported == {
"opus-4-7": frozenset({"temperature", "top_p", "top_k"}),
"opus-4-8": frozenset({"temperature", "top_p", "top_k"}),
"opus-5": frozenset({"temperature", "top_p", "top_k"}),
"fable-5": frozenset({"temperature", "top_p", "top_k"}),
"sonnet-5": frozenset({"temperature", "top_p", "top_k"}),
}
Expand Down