From 117f94a4e835472a288d6b08b993beb561ba3202 Mon Sep 17 00:00:00 2001 From: Ahmad Hammad Date: Thu, 25 Jun 2026 22:47:15 +0300 Subject: [PATCH] fix(cost): fold reasoning tokens into calc_cost (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calc_cost() priced only input + output tokens, so for providers that report reasoning/thinking tokens *outside* output_tokens (e.g. Gemini 2.5 Flash via OpenRouter) the cost was undercounted ~10x even though reasoning tokens were already captured for display. calc_cost now bills reasoning tokens at the output rate, adding them only when reasoning_tok > output_tok — this corrects the non-compliant providers without double-counting compliant ones (gpt-oss already folds reasoning into output_tokens). chat.py and save_turn pass the captured reasoning_tokens through. Part B (follow-up-questions UI) verified already wired in ChatPage.tsx (state + done-event capture + clickable chips); no change needed. Adds tests/test_agent/test_cost.py covering the undercount case, the no-double-count case, and the no-reasoning passthrough. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 1 + apps/backend/src/api/routers/chat.py | 11 ++--- apps/backend/tests/test_agent/test_cost.py | 48 ++++++++++++++++++++ apps/core/src/opendevops_core/agent/turns.py | 16 +++++-- 4 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 apps/backend/tests/test_agent/test_cost.py diff --git a/AGENTS.md b/AGENTS.md index 4a4d94a..5993bf3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,7 @@ Everything below is built and working in the codebase: - **Skills system:** several skills ship (e.g. `lambda-throttling`). The system prompt is built at import time by scanning `apps/core/src/opendevops_core/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`. - **Summarization:** `maybe_summarize()` runs before each agent call; compacts old messages when total chars exceed `SUMMARIZATION_THRESHOLD_CHARS`; tracks the event in `usage_events` with `metadata.summarization=True`. - **Cancellation:** `DELETE /chat/{session_id}` sets an `asyncio.Event` that stops the streaming loop at the next chunk boundary. +- **Cost accounting:** `calc_cost()` (`agent/turns.py`) prices `input + billable_output`, where reasoning/thinking tokens are billed at the output rate. Per the LangChain `usage_metadata` standard, `output_token_details.reasoning` is a *subset* of `output_tokens`, so it is normally already counted; but some providers (e.g. Gemini 2.5 via OpenRouter) report reasoning *outside* `output_tokens` (issue #59). The fold is detection-based: reasoning is added only when `reasoning_tok > output_tok`, which adds it for the broken providers without double-counting compliant ones (gpt-oss). Reasoning tokens are captured into `usage["reasoning_tokens"]` in `routers/chat.py` and passed through `save_turn` → `calc_cost`. ### Storage - Three backends all implementing `DatabaseBackend` ABC: `memory` (default, zero config), `sqlite` (aiosqlite + LangGraph SQLite checkpointer), `postgres` (psycopg3 async + `AsyncPostgresSaver`). diff --git a/apps/backend/src/api/routers/chat.py b/apps/backend/src/api/routers/chat.py index 990d325..d790c94 100644 --- a/apps/backend/src/api/routers/chat.py +++ b/apps/backend/src/api/routers/chat.py @@ -304,12 +304,10 @@ def _flush_text_buf(): usage["input_tokens"] = _field(usage_meta, "input_tokens", 0) or 0 usage["output_tokens"] = _field(usage_meta, "output_tokens", 0) or 0 # Capture reasoning tokens (gpt-oss, o1-family, Gemini thinking) for visibility - # in the cost card and downstream analysis. We intentionally do NOT add them - # to output_tokens for the cost calc here: providers differ on whether - # output_tokens already includes reasoning (gpt-oss: YES — 53 total includes - # 38 reasoning; Gemini: NO — but reasoning is also missing from the metadata, - # which is the actual undercount bug). Per-provider normalization is tracked - # in issue #59. + # in the cost card and downstream analysis. calc_cost folds them into the + # billable output only when the provider reports them *outside* output_tokens + # (issue #59) — gpt-oss already includes reasoning in output_tokens, so it is + # not double-counted; Gemini reports them separately, so they are added. details = _field(usage_meta, "output_token_details", None) or {} reasoning = _field(details, "reasoning", 0) or 0 if reasoning and reasoning > 0: @@ -318,6 +316,7 @@ def _flush_text_buf(): usage["model"], usage.get("input_tokens", 0), usage.get("output_tokens", 0), + usage.get("reasoning_tokens", 0), ) if cost is not None: usage["cost_usd"] = cost diff --git a/apps/backend/tests/test_agent/test_cost.py b/apps/backend/tests/test_agent/test_cost.py new file mode 100644 index 0000000..6dca18e --- /dev/null +++ b/apps/backend/tests/test_agent/test_cost.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from opendevops_core.agent.turns import _FALLBACK_PRICING, calc_cost + +# A model present in the fallback pricing config — used so the test asserts the +# folding math against known per-token rates without depending on LiteLLM's +# (mutable) pricing database. +_MODEL = "openrouter/google/gemma-4-26b-a4b-it" +_RATES = _FALLBACK_PRICING[_MODEL] + + +def _expected(input_tok: int, output_tok: int) -> float: + return (input_tok / 1e6) * _RATES["input"] + (output_tok / 1e6) * _RATES["output"] + + +def test_reasoning_tokens_folded_when_reported_separately(): + # Gemini-style undercount (issue #59): the provider reports reasoning tokens + # *outside* output_tokens (visible output is ~0), so they must be added to the + # billable output or the cost is undercounted. + undercounted = calc_cost(_MODEL, input_tok=10_000, output_tok=6) + correct = calc_cost(_MODEL, input_tok=10_000, output_tok=6, reasoning_tok=800) + + assert undercounted == _expected(10_000, 6) + assert correct == _expected(10_000, 6 + 800) + assert correct > undercounted + + +def test_reasoning_not_double_counted_when_already_in_output(): + # gpt-oss style: output_tokens already includes reasoning (53 total, 38 of + # which are reasoning). reasoning <= output_tokens, so it must NOT be re-added. + cost = calc_cost(_MODEL, input_tok=1_000, output_tok=53, reasoning_tok=38) + assert cost == _expected(1_000, 53) + + +def test_no_reasoning_is_unchanged(): + assert calc_cost(_MODEL, input_tok=1_000, output_tok=200, reasoning_tok=0) == _expected( + 1_000, 200 + ) + + +def test_litellm_priced_model_includes_reasoning(): + # Real reported model from issue #59. Don't assert an absolute price (LiteLLM's + # table changes); assert that folding reasoning in raises the cost. + model = "openrouter/google/gemini-2.5-flash" + without = calc_cost(model, input_tok=14_000, output_tok=6) + with_reasoning = calc_cost(model, input_tok=14_000, output_tok=6, reasoning_tok=500) + if without is not None: # only meaningful when LiteLLM knows this model + assert with_reasoning > without diff --git a/apps/core/src/opendevops_core/agent/turns.py b/apps/core/src/opendevops_core/agent/turns.py index e9e0fb9..2f3ab57 100644 --- a/apps/core/src/opendevops_core/agent/turns.py +++ b/apps/core/src/opendevops_core/agent/turns.py @@ -19,18 +19,27 @@ } -def calc_cost(model: str, input_tok: int, output_tok: int) -> float | None: +def calc_cost(model: str, input_tok: int, output_tok: int, reasoning_tok: int = 0) -> float | None: + # Reasoning/thinking tokens are billed at the output rate. Per the LangChain + # usage_metadata standard, output_token_details.reasoning is a *subset* of + # output_tokens — but some providers (e.g. Gemini 2.5 via OpenRouter) violate + # this and report reasoning *outside* output_tokens, which undercounted cost + # (issue #59). Detect that case: if reasoning exceeds the reported output + # count, the provider did not fold it in, so add it to the billable output. + billable_output = output_tok + reasoning_tok if reasoning_tok > output_tok else output_tok try: import litellm info = litellm.model_cost.get(model) if info: - return input_tok * info.get("input_cost_per_token", 0) + output_tok * info.get( + return input_tok * info.get("input_cost_per_token", 0) + billable_output * info.get( "output_cost_per_token", 0 ) fallback = _FALLBACK_PRICING.get(model) if fallback: - return (input_tok / 1e6) * fallback["input"] + (output_tok / 1e6) * fallback["output"] + return (input_tok / 1e6) * fallback["input"] + (billable_output / 1e6) * fallback[ + "output" + ] return None except Exception: return None @@ -80,6 +89,7 @@ async def save_turn( usage.get("model", ""), usage.get("input_tokens", 0), usage.get("output_tokens", 0), + usage.get("reasoning_tokens", 0), ) await db.save_usage_event( session_id,