diff --git a/providers/asione.py b/providers/asione.py index 0237ec7f..b911c3be 100644 --- a/providers/asione.py +++ b/providers/asione.py @@ -54,6 +54,10 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content llm._log_raw(self._name, self._model_name, raw) + llm._log_chat_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = llm._llm_empty_response_command() resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/lib_llm_ext.py b/providers/lib_llm_ext.py index 31363f6b..96473ed3 100644 --- a/providers/lib_llm_ext.py +++ b/providers/lib_llm_ext.py @@ -2,8 +2,13 @@ import openai from typing import Optional, Tuple, Dict, Any from config import config_get_by_key +from src.helper import quote_arg PROMPT_DELIMITER = ":-:-:-:" +LLM_EMPTY_RESPONSE_MESSAGE = ( + 'The entire configured "maxOutputTokens" budget was consumed by thinking, ' + "leaving no tokens available to generate the final answer." +) from src.logger import get_logger @@ -13,6 +18,46 @@ def _log_raw(provider: str, model: str, raw: str) -> None: logger.debug(f"[LLM_RAW] provider={provider} model={model} chars={len(raw or '')} raw={raw!r}") +def _log_chat_completion(provider: str, model: str, response) -> None: + """Report how the completion budget was actually spent (Chat Completions API).""" + finish_reason = getattr(response.choices[0], "finish_reason", None) + usage = getattr(response, "usage", None) + details = getattr(usage, "completion_tokens_details", None) + prompt_details = getattr(usage, "prompt_tokens_details", None) + line = ( + f"[LLM_USAGE] provider={provider} model={model} " + f"finish_reason={finish_reason} " + f"prompt_tokens={getattr(usage, 'prompt_tokens', None)} " + f"cached_tokens={getattr(prompt_details, 'cached_tokens', None)} " + f"completion_tokens={getattr(usage, 'completion_tokens', None)} " + f"reasoning_tokens={getattr(details, 'reasoning_tokens', None)} " + ) + logger.debug(line) + +def _log_responses_completion(provider: str, model: str, response) -> None: + """Report how the completion budget was actually spent (Responses API). + """ + incomplete_details = getattr(response, "incomplete_details", None) + usage = getattr(response, "usage", None) + input_details = getattr(usage, "input_tokens_details", None) + output_details = getattr(usage, "output_tokens_details", None) + line = ( + f"[LLM_USAGE] provider={provider} model={model} " + f"status={getattr(response, 'status', None)} " + f"incomplete_reason={getattr(incomplete_details, 'reason', None)} " + f"input_tokens={getattr(usage, 'input_tokens', None)} " + f"cached_tokens={getattr(input_details, 'cached_tokens', None)} " + f"output_tokens={getattr(usage, 'output_tokens', None)} " + f"reasoning_tokens={getattr(output_details, 'reasoning_tokens', None)} " + ) + logger.debug(line) + +def _llm_empty_response_command() -> str: + """Return an explanatory message as a MeTTa `send` command when the LLM + spends the entire output token budget on reasoning and returns no content. + """ + return f"(send {quote_arg(LLM_EMPTY_RESPONSE_MESSAGE)})" + def _split_system_user(content: str) -> Tuple[str, str]: """ MeTTa sends: @@ -130,6 +175,10 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content or "" _log_raw(self._name, self._model_name, raw) + _log_chat_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = _llm_empty_response_command() resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/openai.py b/providers/openai.py index 5491ca9e..a60b5aef 100644 --- a/providers/openai.py +++ b/providers/openai.py @@ -55,22 +55,12 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", response = self._client.responses.create(**create_kwargs) - usage = getattr(response, "usage", None) - if usage: - input_tokens = getattr(usage, "input_tokens", None) - output_tokens = getattr(usage, "output_tokens", None) - total_tokens = getattr(usage, "total_tokens", None) - details = getattr(usage, "input_tokens_details", None) - cached_tokens = getattr(details, "cached_tokens", None) if details else None - - logger.info( - f"[LLM_USAGE] provider={self._name} model={self._model_name} " - f"input_tokens={input_tokens} output_tokens={output_tokens} " - f"total_tokens={total_tokens} cached_tokens={cached_tokens}" - ) - raw = response.output_text or "" llm._log_raw(self._name, self._model_name, raw) + llm._log_responses_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = llm._llm_empty_response_command() return self._clean_text(raw) except Exception as e: logger.exception(f"[OpenAIProviderImpl.chat]: Exception while communicating with LLM: {e}") diff --git a/providers/openrouter.py b/providers/openrouter.py index 40aca84b..29dd7c4a 100644 --- a/providers/openrouter.py +++ b/providers/openrouter.py @@ -8,6 +8,24 @@ logger = get_logger(__name__) +# Share of the completion budget OpenRouter reserves for reasoning at each effort level. +# Models that accept only reasoning.max_tokens get the same split, computed here. +# See: https://openrouter.ai/docs/guides/best-practices/reasoning-tokens#reasoning-effort-level +REASONING_EFFORT_RATIO = { + "none": 0.0, + "minimal": 0.10, + "low": 0.20, + "medium": 0.50, + "high": 0.80, + "xhigh": 0.95, + "max": 0.95, +} + +def _reasoning_budget(max_tokens: int, effort: str) -> int: + """Tokens reserved for reasoning; the rest of max_tokens stays for the answer.""" + ratio = REASONING_EFFORT_RATIO.get((effort or "none").lower(), 0.0) + return int(max_tokens * ratio) + class OpenRouterProvider(providers.LLMProvider): def __init__(self): @@ -46,12 +64,19 @@ def _create_client(self) -> Optional[openai.OpenAI]: return None - def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any]: + def _openrouter_extra_body(self, content: str, max_tokens: int, reasoning: str) -> Dict[str, Any]: + is_anthropic = self._model_name.lower().startswith("anthropic/") sysmsg, _ = llm._split_system_user(content) + # OpenRouter Anthropic models support `max_tokens` for reasoning, + # while other models expect an effort level. + reasoning_config = ( + {"max_tokens": _reasoning_budget(max_tokens, reasoning)} if is_anthropic + else {"effort": reasoning} + ) body = { "reasoning": { "enabled": True, - "max_tokens": max_tokens, + **reasoning_config, "exclude": True, } } @@ -65,10 +90,8 @@ def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any if session_id: body["session_id"] = session_id[:256] - model = self._model_name.lower() - # OpenRouter supports top-level cache_control for Anthropic Claude routes. - if model.startswith("anthropic/"): + if is_anthropic: body["cache_control"] = { "type": "ephemeral", "ttl": config_get_by_key("OPENROUTER_CACHE_TTL", "5m"), @@ -79,7 +102,7 @@ def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", **kwargs) -> str: extra_body = llm._merge_dicts( - self._openrouter_extra_body(content, max_tokens), + self._openrouter_extra_body(content, max_tokens, reasoning), kwargs.pop("extra_body", None), )