Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ Everything below is built and working in the codebase:
- **Skills system:** several skills ship (e.g. `lambda-throttling`). The system prompt is built at import time by scanning `apps/core/src/opendevops_core/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`.
- **Summarization:** `maybe_summarize()` runs before each agent call; compacts old messages when total chars exceed `SUMMARIZATION_THRESHOLD_CHARS`; tracks the event in `usage_events` with `metadata.summarization=True`.
- **Cancellation:** `DELETE /chat/{session_id}` sets an `asyncio.Event` that stops the streaming loop at the next chunk boundary.
- **Cost accounting:** `calc_cost()` (`agent/turns.py`) prices `input + billable_output`, where reasoning/thinking tokens are billed at the output rate. Per the LangChain `usage_metadata` standard, `output_token_details.reasoning` is a *subset* of `output_tokens`, so it is normally already counted; but some providers (e.g. Gemini 2.5 via OpenRouter) report reasoning *outside* `output_tokens` (issue #59). The fold is detection-based: reasoning is added only when `reasoning_tok > output_tok`, which adds it for the broken providers without double-counting compliant ones (gpt-oss). Reasoning tokens are captured into `usage["reasoning_tokens"]` in `routers/chat.py` and passed through `save_turn` → `calc_cost`.

### Storage
- Three backends all implementing `DatabaseBackend` ABC: `memory` (default, zero config), `sqlite` (aiosqlite + LangGraph SQLite checkpointer), `postgres` (psycopg3 async + `AsyncPostgresSaver`).
Expand Down
11 changes: 5 additions & 6 deletions apps/backend/src/api/routers/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,12 +304,10 @@ def _flush_text_buf():
usage["input_tokens"] = _field(usage_meta, "input_tokens", 0) or 0
usage["output_tokens"] = _field(usage_meta, "output_tokens", 0) or 0
# Capture reasoning tokens (gpt-oss, o1-family, Gemini thinking) for visibility
# in the cost card and downstream analysis. We intentionally do NOT add them
# to output_tokens for the cost calc here: providers differ on whether
# output_tokens already includes reasoning (gpt-oss: YES — 53 total includes
# 38 reasoning; Gemini: NO — but reasoning is also missing from the metadata,
# which is the actual undercount bug). Per-provider normalization is tracked
# in issue #59.
# in the cost card and downstream analysis. calc_cost folds them into the
# billable output only when the provider reports them *outside* output_tokens
# (issue #59) — gpt-oss already includes reasoning in output_tokens, so it is
# not double-counted; Gemini reports them separately, so they are added.
details = _field(usage_meta, "output_token_details", None) or {}
reasoning = _field(details, "reasoning", 0) or 0
if reasoning and reasoning > 0:
Expand All @@ -318,6 +316,7 @@ def _flush_text_buf():
usage["model"],
usage.get("input_tokens", 0),
usage.get("output_tokens", 0),
usage.get("reasoning_tokens", 0),
)
if cost is not None:
usage["cost_usd"] = cost
Expand Down
48 changes: 48 additions & 0 deletions apps/backend/tests/test_agent/test_cost.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from __future__ import annotations

from opendevops_core.agent.turns import _FALLBACK_PRICING, calc_cost

# A model present in the fallback pricing config — used so the test asserts the
# folding math against known per-token rates without depending on LiteLLM's
# (mutable) pricing database.
_MODEL = "openrouter/google/gemma-4-26b-a4b-it"
_RATES = _FALLBACK_PRICING[_MODEL]


def _expected(input_tok: int, output_tok: int) -> float:
return (input_tok / 1e6) * _RATES["input"] + (output_tok / 1e6) * _RATES["output"]


def test_reasoning_tokens_folded_when_reported_separately():
# Gemini-style undercount (issue #59): the provider reports reasoning tokens
# *outside* output_tokens (visible output is ~0), so they must be added to the
# billable output or the cost is undercounted.
undercounted = calc_cost(_MODEL, input_tok=10_000, output_tok=6)
correct = calc_cost(_MODEL, input_tok=10_000, output_tok=6, reasoning_tok=800)

assert undercounted == _expected(10_000, 6)
assert correct == _expected(10_000, 6 + 800)
assert correct > undercounted


def test_reasoning_not_double_counted_when_already_in_output():
# gpt-oss style: output_tokens already includes reasoning (53 total, 38 of
# which are reasoning). reasoning <= output_tokens, so it must NOT be re-added.
cost = calc_cost(_MODEL, input_tok=1_000, output_tok=53, reasoning_tok=38)
assert cost == _expected(1_000, 53)


def test_no_reasoning_is_unchanged():
assert calc_cost(_MODEL, input_tok=1_000, output_tok=200, reasoning_tok=0) == _expected(
1_000, 200
)


def test_litellm_priced_model_includes_reasoning():
# Real reported model from issue #59. Don't assert an absolute price (LiteLLM's
# table changes); assert that folding reasoning in raises the cost.
model = "openrouter/google/gemini-2.5-flash"
without = calc_cost(model, input_tok=14_000, output_tok=6)
with_reasoning = calc_cost(model, input_tok=14_000, output_tok=6, reasoning_tok=500)
if without is not None: # only meaningful when LiteLLM knows this model
assert with_reasoning > without
16 changes: 13 additions & 3 deletions apps/core/src/opendevops_core/agent/turns.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,27 @@
}


def calc_cost(model: str, input_tok: int, output_tok: int) -> float | None:
def calc_cost(model: str, input_tok: int, output_tok: int, reasoning_tok: int = 0) -> float | None:
# Reasoning/thinking tokens are billed at the output rate. Per the LangChain
# usage_metadata standard, output_token_details.reasoning is a *subset* of
# output_tokens — but some providers (e.g. Gemini 2.5 via OpenRouter) violate
# this and report reasoning *outside* output_tokens, which undercounted cost
# (issue #59). Detect that case: if reasoning exceeds the reported output
# count, the provider did not fold it in, so add it to the billable output.
billable_output = output_tok + reasoning_tok if reasoning_tok > output_tok else output_tok
try:
import litellm

info = litellm.model_cost.get(model)
if info:
return input_tok * info.get("input_cost_per_token", 0) + output_tok * info.get(
return input_tok * info.get("input_cost_per_token", 0) + billable_output * info.get(
"output_cost_per_token", 0
)
fallback = _FALLBACK_PRICING.get(model)
if fallback:
return (input_tok / 1e6) * fallback["input"] + (output_tok / 1e6) * fallback["output"]
return (input_tok / 1e6) * fallback["input"] + (billable_output / 1e6) * fallback[
"output"
]
return None
except Exception:
return None
Expand Down Expand Up @@ -80,6 +89,7 @@ async def save_turn(
usage.get("model", ""),
usage.get("input_tokens", 0),
usage.get("output_tokens", 0),
usage.get("reasoning_tokens", 0),
)
await db.save_usage_event(
session_id,
Expand Down
Loading