From 8f2cbdf296b01d5b12427f6d7d8a99d6679bfe8b Mon Sep 17 00:00:00 2001 From: Paul Vasiletz Date: Thu, 3 Sep 2026 16:48:03 +0300 Subject: [PATCH 1/3] draft: providers | cap OpenRouter reasoning budget, log token usage, and fall back to a send command on empty replies --- providers/asione.py | 4 ++++ providers/lib_llm_ext.py | 28 ++++++++++++++++++++++++++++ providers/openai.py | 18 ++++-------------- providers/openrouter.py | 35 +++++++++++++++++++++++++++++------ 4 files changed, 65 insertions(+), 20 deletions(-) diff --git a/providers/asione.py b/providers/asione.py index 0237ec7f..a4ba40a4 100644 --- a/providers/asione.py +++ b/providers/asione.py @@ -54,6 +54,10 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content llm._log_raw(self._name, self._model_name, raw) + llm._log_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = llm._llm_empty_response_command() resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/lib_llm_ext.py b/providers/lib_llm_ext.py index 31363f6b..42ab795a 100644 --- a/providers/lib_llm_ext.py +++ b/providers/lib_llm_ext.py @@ -2,8 +2,13 @@ import openai from typing import Optional, Tuple, Dict, Any from config import config_get_by_key +from src.helper import quote_arg PROMPT_DELIMITER = ":-:-:-:" +LLM_EMPTY_RESPONSE_MESSAGE = ( + 'The entire configured "maxOutputTokens" budget was consumed by thinking, ' + "leaving no tokens available to generate the final answer." +) from src.logger import get_logger @@ -13,6 +18,25 @@ def _log_raw(provider: str, model: str, raw: str) -> None: logger.debug(f"[LLM_RAW] provider={provider} model={model} chars={len(raw or '')} raw={raw!r}") +def _log_completion(provider: str, model: str, response) -> None: + """Report how the completion budget was actually spent.""" + finish_reason = getattr(response.choices[0], "finish_reason", None) + usage = getattr(response, "usage", None) + details = getattr(usage, "completion_tokens_details", None) + line = ( + f"[LLM_USAGE] provider={provider} model={model} " + f"finish_reason={finish_reason} " + f"completion_tokens={getattr(usage, 'completion_tokens', None)} " + f"reasoning_tokens={getattr(details, 'reasoning_tokens', None)} " + ) + logger.debug(line) + +def _llm_empty_response_command() -> str: + """Return an explanatory message as a MeTTa `send` command when the LLM + spends the entire output token budget on reasoning and returns no content. + """ + return f"(send {quote_arg(LLM_EMPTY_RESPONSE_MESSAGE)})" + def _split_system_user(content: str) -> Tuple[str, str]: """ MeTTa sends: @@ -130,6 +154,10 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content or "" _log_raw(self._name, self._model_name, raw) + _log_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = _llm_empty_response_command() resp = self._clean_text(raw) return resp except Exception as e: diff --git a/providers/openai.py b/providers/openai.py index 5491ca9e..46da788a 100644 --- a/providers/openai.py +++ b/providers/openai.py @@ -55,22 +55,12 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", response = self._client.responses.create(**create_kwargs) - usage = getattr(response, "usage", None) - if usage: - input_tokens = getattr(usage, "input_tokens", None) - output_tokens = getattr(usage, "output_tokens", None) - total_tokens = getattr(usage, "total_tokens", None) - details = getattr(usage, "input_tokens_details", None) - cached_tokens = getattr(details, "cached_tokens", None) if details else None - - logger.info( - f"[LLM_USAGE] provider={self._name} model={self._model_name} " - f"input_tokens={input_tokens} output_tokens={output_tokens} " - f"total_tokens={total_tokens} cached_tokens={cached_tokens}" - ) - raw = response.output_text or "" llm._log_raw(self._name, self._model_name, raw) + llm._log_completion(self._name, self._model_name, response) + if not raw: + logger.warning("LLM returned an empty response") + raw = llm._llm_empty_response_command() return self._clean_text(raw) except Exception as e: logger.exception(f"[OpenAIProviderImpl.chat]: Exception while communicating with LLM: {e}") diff --git a/providers/openrouter.py b/providers/openrouter.py index 40aca84b..29dd7c4a 100644 --- a/providers/openrouter.py +++ b/providers/openrouter.py @@ -8,6 +8,24 @@ logger = get_logger(__name__) +# Share of the completion budget OpenRouter reserves for reasoning at each effort level. +# Models that accept only reasoning.max_tokens get the same split, computed here. +# See: https://openrouter.ai/docs/guides/best-practices/reasoning-tokens#reasoning-effort-level +REASONING_EFFORT_RATIO = { + "none": 0.0, + "minimal": 0.10, + "low": 0.20, + "medium": 0.50, + "high": 0.80, + "xhigh": 0.95, + "max": 0.95, +} + +def _reasoning_budget(max_tokens: int, effort: str) -> int: + """Tokens reserved for reasoning; the rest of max_tokens stays for the answer.""" + ratio = REASONING_EFFORT_RATIO.get((effort or "none").lower(), 0.0) + return int(max_tokens * ratio) + class OpenRouterProvider(providers.LLMProvider): def __init__(self): @@ -46,12 +64,19 @@ def _create_client(self) -> Optional[openai.OpenAI]: return None - def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any]: + def _openrouter_extra_body(self, content: str, max_tokens: int, reasoning: str) -> Dict[str, Any]: + is_anthropic = self._model_name.lower().startswith("anthropic/") sysmsg, _ = llm._split_system_user(content) + # OpenRouter Anthropic models support `max_tokens` for reasoning, + # while other models expect an effort level. + reasoning_config = ( + {"max_tokens": _reasoning_budget(max_tokens, reasoning)} if is_anthropic + else {"effort": reasoning} + ) body = { "reasoning": { "enabled": True, - "max_tokens": max_tokens, + **reasoning_config, "exclude": True, } } @@ -65,10 +90,8 @@ def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any if session_id: body["session_id"] = session_id[:256] - model = self._model_name.lower() - # OpenRouter supports top-level cache_control for Anthropic Claude routes. - if model.startswith("anthropic/"): + if is_anthropic: body["cache_control"] = { "type": "ephemeral", "ttl": config_get_by_key("OPENROUTER_CACHE_TTL", "5m"), @@ -79,7 +102,7 @@ def _openrouter_extra_body(self, content: str, max_tokens: int) -> Dict[str, Any def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", **kwargs) -> str: extra_body = llm._merge_dicts( - self._openrouter_extra_body(content, max_tokens), + self._openrouter_extra_body(content, max_tokens, reasoning), kwargs.pop("extra_body", None), ) From 80bb4525f491e7ec5796811afab696beb67750d6 Mon Sep 17 00:00:00 2001 From: Paul Vasiletz Date: Thu, 3 Sep 2026 17:38:56 +0300 Subject: [PATCH 2/3] draft: providers | add cached and prompt token counts to completion usage logging --- providers/lib_llm_ext.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/providers/lib_llm_ext.py b/providers/lib_llm_ext.py index 42ab795a..73017d80 100644 --- a/providers/lib_llm_ext.py +++ b/providers/lib_llm_ext.py @@ -23,9 +23,12 @@ def _log_completion(provider: str, model: str, response) -> None: finish_reason = getattr(response.choices[0], "finish_reason", None) usage = getattr(response, "usage", None) details = getattr(usage, "completion_tokens_details", None) + prompt_details = getattr(usage, "prompt_tokens_details", None) line = ( f"[LLM_USAGE] provider={provider} model={model} " f"finish_reason={finish_reason} " + f"prompt_tokens={getattr(usage, 'prompt_tokens', None)} " + f"cached_tokens={getattr(prompt_details, 'cached_tokens', None)} " f"completion_tokens={getattr(usage, 'completion_tokens', None)} " f"reasoning_tokens={getattr(details, 'reasoning_tokens', None)} " ) From 321f474af56fa37d9bd1b7e2e341cd310d500b35 Mon Sep 17 00:00:00 2001 From: Paul Vasiletz Date: Fri, 4 Sep 2026 15:54:05 +0300 Subject: [PATCH 3/3] draft: providers | add separate usage logging for the Responses API --- providers/asione.py | 2 +- providers/lib_llm_ext.py | 24 +++++++++++++++++++++--- providers/openai.py | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/providers/asione.py b/providers/asione.py index a4ba40a4..b911c3be 100644 --- a/providers/asione.py +++ b/providers/asione.py @@ -54,7 +54,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content llm._log_raw(self._name, self._model_name, raw) - llm._log_completion(self._name, self._model_name, response) + llm._log_chat_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") raw = llm._llm_empty_response_command() diff --git a/providers/lib_llm_ext.py b/providers/lib_llm_ext.py index 73017d80..96473ed3 100644 --- a/providers/lib_llm_ext.py +++ b/providers/lib_llm_ext.py @@ -18,8 +18,8 @@ def _log_raw(provider: str, model: str, raw: str) -> None: logger.debug(f"[LLM_RAW] provider={provider} model={model} chars={len(raw or '')} raw={raw!r}") -def _log_completion(provider: str, model: str, response) -> None: - """Report how the completion budget was actually spent.""" +def _log_chat_completion(provider: str, model: str, response) -> None: + """Report how the completion budget was actually spent (Chat Completions API).""" finish_reason = getattr(response.choices[0], "finish_reason", None) usage = getattr(response, "usage", None) details = getattr(usage, "completion_tokens_details", None) @@ -34,6 +34,24 @@ def _log_completion(provider: str, model: str, response) -> None: ) logger.debug(line) +def _log_responses_completion(provider: str, model: str, response) -> None: + """Report how the completion budget was actually spent (Responses API). + """ + incomplete_details = getattr(response, "incomplete_details", None) + usage = getattr(response, "usage", None) + input_details = getattr(usage, "input_tokens_details", None) + output_details = getattr(usage, "output_tokens_details", None) + line = ( + f"[LLM_USAGE] provider={provider} model={model} " + f"status={getattr(response, 'status', None)} " + f"incomplete_reason={getattr(incomplete_details, 'reason', None)} " + f"input_tokens={getattr(usage, 'input_tokens', None)} " + f"cached_tokens={getattr(input_details, 'cached_tokens', None)} " + f"output_tokens={getattr(usage, 'output_tokens', None)} " + f"reasoning_tokens={getattr(output_details, 'reasoning_tokens', None)} " + ) + logger.debug(line) + def _llm_empty_response_command() -> str: """Return an explanatory message as a MeTTa `send` command when the LLM spends the entire output token budget on reasoning and returns no content. @@ -157,7 +175,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.choices[0].message.content or "" _log_raw(self._name, self._model_name, raw) - _log_completion(self._name, self._model_name, response) + _log_chat_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") raw = _llm_empty_response_command() diff --git a/providers/openai.py b/providers/openai.py index 46da788a..a60b5aef 100644 --- a/providers/openai.py +++ b/providers/openai.py @@ -57,7 +57,7 @@ def chat(self, content: str, max_tokens: int = 6000, reasoning: str = "medium", raw = response.output_text or "" llm._log_raw(self._name, self._model_name, raw) - llm._log_completion(self._name, self._model_name, response) + llm._log_responses_completion(self._name, self._model_name, response) if not raw: logger.warning("LLM returned an empty response") raw = llm._llm_empty_response_command()