From 97306a0283fd509649686cf5c06fc8c9fc1e7971 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Mon, 27 Jul 2026 11:46:06 -0300 Subject: [PATCH] (MOT-4251) feat(console): add session usage metrics --- console/web/e2e/ui-metrics.spec.ts | 83 ++++ console/web/src/components/chat/ChatView.tsx | 25 + .../web/src/components/chat/ContextUsage.tsx | 85 +++- console/web/src/components/chat/Message.tsx | 23 +- .../web/src/components/chat/MessageList.tsx | 13 + .../components/chat/SessionMetricsButton.tsx | 135 +++++ .../components/chat/TurnUsageChip.stories.tsx | 134 +++++ .../web/src/components/chat/TurnUsageChip.tsx | 106 ++++ .../src/components/metrics/MetricTable.tsx | 89 ++++ .../metrics/SessionMetricsPanel.stories.tsx | 195 ++++++++ .../metrics/SessionMetricsPanel.tsx | 375 ++++++++++++++ .../web/src/lib/backend/harness-metrics.ts | 89 ++++ console/web/src/lib/session-usage.test.ts | 402 +++++++++++++++ console/web/src/lib/session-usage.ts | 461 ++++++++++++++++++ .../web/src/lib/sessions/entry-mapper.test.ts | 86 ++++ console/web/src/lib/sessions/entry-mapper.ts | 22 +- console/web/src/lib/sessions/types.ts | 26 + console/web/src/lib/storage.ts | 23 + console/web/src/types/chat.ts | 16 + 19 files changed, 2370 insertions(+), 18 deletions(-) create mode 100644 console/web/e2e/ui-metrics.spec.ts create mode 100644 console/web/src/components/chat/SessionMetricsButton.tsx create mode 100644 console/web/src/components/chat/TurnUsageChip.stories.tsx create mode 100644 console/web/src/components/chat/TurnUsageChip.tsx create mode 100644 console/web/src/components/metrics/MetricTable.tsx create mode 100644 console/web/src/components/metrics/SessionMetricsPanel.stories.tsx create mode 100644 console/web/src/components/metrics/SessionMetricsPanel.tsx create mode 100644 console/web/src/lib/backend/harness-metrics.ts create mode 100644 console/web/src/lib/session-usage.test.ts create mode 100644 console/web/src/lib/session-usage.ts diff --git a/console/web/e2e/ui-metrics.spec.ts b/console/web/e2e/ui-metrics.spec.ts new file mode 100644 index 000000000..db8d99c31 --- /dev/null +++ b/console/web/e2e/ui-metrics.spec.ts @@ -0,0 +1,83 @@ +import { expect, expectPassingResult, openSession, test } from './harness-stack' + +test.use({ scenario: 'console-streamed-text' }) + +/** + * The session metrics dialog opens from either of its two triggers and shows + * the provider-reported numbers the harness persisted during the turn. + * + * Selectors are attributes rather than text, matching `ui-send.spec.ts` — the + * copy in this panel is expected to change. + */ +test('opens session metrics from the header and from the ctx widget', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await openSession(page, stack) + + const composer = page.getByLabel('message composer') + await composer.pressSequentially(stack.ready.message) + await page.getByRole('button', { name: 'send message' }).click() + await completed + await expect( + page.locator('[data-message-role="assistant"]', { + hasText: 'console fixture complete', + }), + ).toHaveCount(1) + + const dialog = page.locator('[data-testid="session-metrics"]') + + // Trigger 1: the header button. Addressed by testid, not accessible name — + // `getByRole(name:)` matches substrings, so a sidebar row for a session + // whose title contains "metrics" would win instead. + await page.getByTestId('session-metrics-trigger').click() + await expect(dialog).toBeVisible() + + // The three sections are structural, not decorative: which one a number + // sits in is the statement about how much to trust it. + await expect(dialog.getByText('exact', { exact: true })).toBeVisible() + await expect(dialog.getByText('counted', { exact: true })).toBeVisible() + await expect(dialog.getByText('estimated', { exact: true })).toBeVisible() + + // The scripted fixture reports usage, so input tokens must not be a dash. + const inputRow = dialog.locator('div', { hasText: /^input tokens/ }).last() + await expect(inputRow).not.toContainText('—') + + await dialog.getByRole('tab', { name: 'turns' }).click() + await expect(dialog.getByRole('columnheader', { name: 'turn' })).toBeVisible() + + await dialog.getByRole('tab', { name: 'tree' }).click() + await expect(dialog).toBeVisible() + + await page.keyboard.press('Escape') + await expect(dialog).toBeHidden() + + // Trigger 2: clicking the ctx widget — the discoverability path. + await page.getByRole('button', { name: /^ctx/ }).click() + await expect(dialog).toBeVisible() + + expectPassingResult(await stack.finish()) +}) + +test('shows a per-turn usage chip in the transcript', async ({ + page, + stack, +}) => { + const completed = stack.waitForTurnCompleted() + await openSession(page, stack) + + const composer = page.getByLabel('message composer') + await composer.pressSequentially(stack.ready.message) + await page.getByRole('button', { name: 'send message' }).click() + await completed + + const chip = page.locator('[data-turn-usage]').first() + await expect(chip).toBeVisible() + + // Expanding is the point of the chip: per-step is where cache warmth shows. + await chip.getByRole('button').first().click() + await expect(chip).toContainText('step 0') + + expectPassingResult(await stack.finish()) +}) diff --git a/console/web/src/components/chat/ChatView.tsx b/console/web/src/components/chat/ChatView.tsx index e68b7a4ec..535358e81 100644 --- a/console/web/src/components/chat/ChatView.tsx +++ b/console/web/src/components/chat/ChatView.tsx @@ -38,6 +38,8 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context' import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions' import { formatStopReason } from '@/lib/format-stop-reason' import { newMessageId } from '@/lib/session-id' +import { sessionUsage } from '@/lib/session-usage' +import { loadShowTurnMetrics } from '@/lib/storage' import { cn } from '@/lib/utils' import { fetchDefaultWorkingDir, validateWorkspaceDir } from '@/lib/working-dir' import { @@ -74,6 +76,7 @@ import { Composer, type ComposerSubmitPayload } from './Composer' import { ContextUsage } from './ContextUsage' import { ExportSessionButton } from './ExportSessionButton' import { MessageList } from './MessageList' +import { SessionMetricsButton } from './SessionMetricsButton' import { SessionTriggers } from './SessionTriggers' import { WorktreeBadge } from './WorktreeBadge' @@ -762,6 +765,16 @@ export function ChatView({ const filesystemGrants = useFilesystemGrants(sessionId) const [filesystemDialogOpen, setFilesystemDialogOpen] = useState(false) + const [metricsOpen, setMetricsOpen] = useState(false) + const [showTurnMetrics, setShowTurnMetrics] = useState(loadShowTurnMetrics) + // Computed once here and shared: the dialog needs the whole rollup, and the + // ctx widget needs `lastCall` to cross-check its chars/4 estimate against a + // number the provider actually reported. + const usageRollup = useMemo( + () => sessionUsage(conversation.messages), + [conversation.messages], + ) + const lastModelCall = usageRollup.lastCall const handleManageFilesystemAccess = useCallback(() => { setFilesystemDialogOpen(true) }, []) @@ -1630,6 +1643,17 @@ export function ChatView({ setMetricsOpen(true)} + /> + diff --git a/console/web/src/components/chat/ContextUsage.tsx b/console/web/src/components/chat/ContextUsage.tsx index 222a60fb4..a92235288 100644 --- a/console/web/src/components/chat/ContextUsage.tsx +++ b/console/web/src/components/chat/ContextUsage.tsx @@ -1,4 +1,5 @@ import { useMemo } from 'react' +import type { SessionUsage } from '@/lib/session-usage' import { estimateConversationTokens, formatTokenCount, @@ -6,27 +7,72 @@ import { import { cn } from '@/lib/utils' import type { Message } from '@/types/chat' +/** + * How close this conversation is to overflowing its context window. + * + * This is deliberately an ESTIMATE (chars ÷ 4) and stays one even now that + * real provider usage is available, because the two measure different things: + * this is current occupancy, while `Σ usage.input` is cumulative billing that + * counts the same prompt once per step. The obvious substitute — the last + * call's `input + cache_read + cache_write` — is right for anthropic (whose + * `input` excludes cached tokens) and double-counts for openai/codex (whose + * `input` includes them), so it cannot be computed portably in the browser. + * + * What we do instead: mark the number `~` so it never implies precision, and + * put the last provider-reported prompt size in the tooltip as a calibration + * reference. A genuinely exact gauge belongs in llm-router, where provider + * semantics are already known. + */ + interface ContextUsageProps { messages: readonly Message[] contextWindow?: number + /** Last provider-reported call, for the tooltip cross-check. */ + lastCall?: SessionUsage['lastCall'] + /** When set the widget becomes a button that opens the metrics dialog. */ + onClick?: () => void } const WARN_THRESHOLD = 0.75 const DANGER_THRESHOLD = 0.9 -export function ContextUsage({ messages, contextWindow }: ContextUsageProps) { +function calibration(lastCall: ContextUsageProps['lastCall']): string { + if (typeof lastCall?.usage.input !== 'number') return '' + return `\nlast provider prompt: ${lastCall.usage.input.toLocaleString()} tokens (measured)` +} + +export function ContextUsage({ + messages, + contextWindow, + lastCall, + onClick, +}: ContextUsageProps) { const tokens = useMemo(() => estimateConversationTokens(messages), [messages]) + const shell = cn( + 'flex items-center gap-1.5 font-mono text-[11px] uppercase tracking-[0.06em] text-ink-faint', + onClick && + 'hover:text-ink transition-colors focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent', + ) + const openHint = onClick ? '\nclick for session metrics' : '' + if (!contextWindow || contextWindow <= 0) { - return ( -
+ const body = ( + <> ctx - {formatTokenCount(tokens)} + ~{formatTokenCount(tokens)} + + ) + const title = `~${tokens.toLocaleString()} tokens estimated (context window unknown)${calibration(lastCall)}${openHint}` + return onClick ? ( + + ) : ( +
+ {body}
) } @@ -58,13 +104,12 @@ export function ContextUsage({ messages, contextWindow }: ContextUsageProps) { ? 'consider /compact' : 'pre-flight compaction imminent' - return ( -
+ const title = `~${tokens.toLocaleString()} / ${contextWindow.toLocaleString()} tokens estimated (${pct}%)${ + hint ? ` — ${hint}` : '' + }${calibration(lastCall)}${openHint}` + + const body = ( + <> ctx
{pct}% - {formatTokenCount(tokens)}/{formatTokenCount(contextWindow)} + ~{formatTokenCount(tokens)}/{formatTokenCount(contextWindow)} + + ) + + return onClick ? ( + + ) : ( +
+ {body}
) } diff --git a/console/web/src/components/chat/Message.tsx b/console/web/src/components/chat/Message.tsx index 4784c6f9d..ea316a422 100644 --- a/console/web/src/components/chat/Message.tsx +++ b/console/web/src/components/chat/Message.tsx @@ -3,6 +3,7 @@ import type { FilesystemAccessAction } from '@/components/permissions/Filesystem import { Caret } from '@/components/ui/Caret' import { Prompt } from '@/components/ui/Prompt' import { Markdown } from '@/lib/markdown' +import type { TurnUsage } from '@/lib/session-usage' import { JsonHighlight } from '@/lib/syntax' import { cn } from '@/lib/utils' import type { @@ -15,6 +16,7 @@ import { AttachmentChip } from './AttachmentChip' import { CopyMessageButton } from './CopyMessageButton' import { MemoryChip } from './MemoryChip' import { ThoughtMessage } from './ThoughtMessage' +import { TurnUsageChip } from './TurnUsageChip' interface MessageProps { message: MessageType @@ -38,6 +40,9 @@ interface MessageProps { /** Copy payload for an assistant turn (prose + its function calls). Lazy so the string is built on click, not on every streaming re-render. */ copyText?: string | (() => string) + /** Rollup for the turn this message closes; absent unless it is the anchor. */ + turnUsage?: TurnUsage + compactTurnUsage?: boolean } export function Message({ @@ -48,6 +53,8 @@ export function Message({ onManageFilesystemAccess, workingDir, copyText, + turnUsage, + compactTurnUsage, }: MessageProps) { switch (message.role) { case 'user': @@ -61,7 +68,14 @@ export function Message({ ) case 'assistant': - return + return ( + + ) case 'thought': return case 'function-trigger': { @@ -307,9 +321,13 @@ function UserMessage({ message }: { message: UserMessageType }) { function AssistantMessage({ message, copyText, + turnUsage, + compactTurnUsage, }: { message: AssistantMessageType copyText?: string | (() => string) + turnUsage?: TurnUsage + compactTurnUsage?: boolean }) { const showCaret = !!message.streaming // A tool-only turn has no prose but still carries a copy payload (its @@ -330,6 +348,9 @@ function AssistantMessage({ · {message.mode} ) : null} {message.memory ? : null} + {turnUsage ? ( + + ) : null} {copySource !== undefined && !message.streaming ? ( Promise onManageFilesystemAccess?: () => void workingDir?: string | null + /** Per-turn usage chips in the transcript (user preference, default on). */ + showTurnMetrics?: boolean } type RenderItem = @@ -99,6 +102,7 @@ export function MessageList({ onResolveFilesystemAccess, onManageFilesystemAccess, workingDir, + showTurnMetrics = true, }: MessageListProps) { const bottomRef = useRef(null) const containerRef = useRef(null) @@ -109,6 +113,13 @@ export function MessageList({ () => functionTriggersByAssistant(messages), [messages], ) + // Turn rollup keyed by the message the chip hangs on. Grouping cannot live + // in the entry mapper — that sees one entry at a time, while a turn spans + // several — so it happens here, once per transcript change. + const turnUsage = useMemo( + () => (showTurnMetrics ? turnUsageByAnchor(messages) : null), + [messages, showTurnMetrics], + ) // Read optionally so isolated renders (Storybook) still work without the // ConversationsProvider; the empty state falls back to `ready` there. @@ -205,6 +216,8 @@ export function MessageList({ key={item.key} message={m} copyText={copyText} + turnUsage={turnUsage?.get(m.id)} + compactTurnUsage={density === 'dock'} onResolveApproval={onResolveApproval} onAlwaysAllow={onAlwaysAllow} onResolveFilesystemAccess={onResolveFilesystemAccess} diff --git a/console/web/src/components/chat/SessionMetricsButton.tsx b/console/web/src/components/chat/SessionMetricsButton.tsx new file mode 100644 index 000000000..ee6ecd6aa --- /dev/null +++ b/console/web/src/components/chat/SessionMetricsButton.tsx @@ -0,0 +1,135 @@ +import { Gauge } from 'lucide-react' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { SessionMetricsPanel } from '@/components/metrics/SessionMetricsPanel' +import { + Dialog, + DialogContent, + DialogDescription, + DialogTitle, +} from '@/components/ui/Dialog' +import { + fetchHarnessMetrics, + type HarnessMetricsState, +} from '@/lib/backend/harness-metrics' +import type { SessionUsage } from '@/lib/session-usage' +import { loadShowTurnMetrics, saveShowTurnMetrics } from '@/lib/storage' +import { estimateConversationTokens } from '@/lib/token-estimate' +import { cn } from '@/lib/utils' +import type { Conversation } from '@/types/chat' + +/** + * Header trigger + dialog for session metrics. Owns every piece of data + * access so `SessionMetricsPanel` can stay presentational. + * + * The dialog is a modal rather than an inline panel because the chat dock is + * resizable down to 320px and a metrics table does not fit in it — the modal + * portals to the viewport centre and clears the dock at any width. + */ + +interface SessionMetricsButtonProps { + conversation: Conversation + /** Shared with the ctx widget so the rollup is computed once per render. */ + usage: SessionUsage + contextWindow?: number + /** Dock density: icon only, no visible label. */ + compact?: boolean + open: boolean + onOpenChange: (open: boolean) => void + onShowTurnMetricsChange?: (show: boolean) => void + className?: string +} + +export function SessionMetricsButton({ + conversation, + usage, + contextWindow, + compact, + open, + onOpenChange, + onShowTurnMetricsChange, + className, +}: SessionMetricsButtonProps) { + const [tree, setTree] = useState(null) + const [showTurnChips, setShowTurnChips] = useState(loadShowTurnMetrics) + const disabled = conversation.messages.length === 0 + + const contextEstimate = useMemo( + () => estimateConversationTokens(conversation.messages), + [conversation.messages], + ) + + const loadTree = useCallback(() => { + setTree('loading') + void fetchHarnessMetrics(conversation.id).then(setTree) + }, [conversation.id]) + + // Only on first open: `harness::metrics` walks every descendant session, so + // it must never run just because a chat is mounted. + useEffect(() => { + if (open && tree === null) loadTree() + }, [open, tree, loadTree]) + + const toggleTurnChips = useCallback( + (next: boolean) => { + setShowTurnChips(next) + saveShowTurnMetrics(next) + onShowTurnMetricsChange?.(next) + }, + [onShowTurnMetricsChange], + ) + + return ( + <> + + + + + + session metrics + + + {conversation.id} + + { + onOpenChange(false) + window.location.hash = '#/traces' + }} + showTurnChips={showTurnChips} + onToggleTurnChips={toggleTurnChips} + /> + + + + ) +} diff --git a/console/web/src/components/chat/TurnUsageChip.stories.tsx b/console/web/src/components/chat/TurnUsageChip.stories.tsx new file mode 100644 index 000000000..eba8ae724 --- /dev/null +++ b/console/web/src/components/chat/TurnUsageChip.stories.tsx @@ -0,0 +1,134 @@ +import type { Meta, StoryObj } from '@storybook/react-vite' +import type { TurnUsage } from '@/lib/session-usage' +import { TurnUsageChip } from './TurnUsageChip' + +const totals = ( + over: Partial = {}, +): TurnUsage['totals'] => ({ + input: 412_908, + output: 18_204, + cacheRead: 388_110, + cacheWrite: 12_400, + reasoning: 0, + costUsd: 0.0482, + reported: { + input: 3, + output: 3, + cacheRead: 3, + cacheWrite: 3, + reasoning: 0, + cost: 3, + }, + total: 431_112, + ...over, +}) + +/** Three steps of one tool loop: cold prompt, then two cache-warm calls. */ +const warmingTurn: TurnUsage = { + turnId: 't_9f1a', + steps: 3, + stepUsage: [ + { + entryId: 'e_t_9f1a_0_assistant', + usage: { input: 12_404, output: 288, cache_read: 0, cost_usd: 0.0021 }, + }, + { + entryId: 'e_t_9f1a_1_assistant', + usage: { + input: 12_910, + output: 1_044, + cache_read: 12_400, + cost_usd: 0.0038, + }, + }, + { + entryId: 'e_t_9f1a_2_assistant', + usage: { + input: 13_882, + output: 402, + cache_read: 12_400, + cost_usd: 0.0029, + }, + }, + ], + totals: totals(), + functionCalls: 2, + functionCallErrors: 0, + startedAt: 0, + endedAt: 62_000, + durationMs: 62_000, + anchorId: 'e_t_9f1a_2_assistant', + streaming: false, +} + +const meta = { + title: 'chat/TurnUsageChip', + component: TurnUsageChip, +} satisfies Meta + +export default meta +type Story = StoryObj + +export const Collapsed: Story = { args: { turn: warmingTurn } } + +/** Dock density drops the cost figure so the header still fits at 320px. */ +export const Compact: Story = { args: { turn: warmingTurn, compact: true } } + +/** Usage typically lands only on the final frame, so a live turn shows dashes. */ +export const Streaming: Story = { + args: { + turn: { + ...warmingTurn, + steps: 0, + stepUsage: [], + totals: totals({ + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + costUsd: 0, + total: 0, + reported: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + cost: 0, + }, + }), + streaming: true, + }, + }, +} + +/** + * A turn with nothing measured and nothing running renders NOTHING — sessions + * written before usage was persisted look exactly as they do today. + */ +export const NullGuard: Story = { + args: { + turn: { + ...warmingTurn, + steps: 0, + stepUsage: [], + totals: totals({ + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + costUsd: 0, + total: 0, + reported: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + cost: 0, + }, + }), + streaming: false, + }, + }, +} diff --git a/console/web/src/components/chat/TurnUsageChip.tsx b/console/web/src/components/chat/TurnUsageChip.tsx new file mode 100644 index 000000000..ed064d2f8 --- /dev/null +++ b/console/web/src/components/chat/TurnUsageChip.tsx @@ -0,0 +1,106 @@ +import { useState } from 'react' +import { + formatUsageValue, + hasReportedUsage, + reportedValue, + type TurnUsage, +} from '@/lib/session-usage' +import { formatTokenCount } from '@/lib/token-estimate' +import { cn } from '@/lib/utils' + +/** + * Per-turn usage, in the assistant message header beside the memory chip. + * + * Modeled on `MemoryChip`: same slot, same typography, same click-to-expand, + * and the same "render nothing when there is nothing to say" guard — so a + * session with no recorded usage looks exactly as it does today. + * + * The chip totals the turn; the expansion breaks it down per step. That + * expansion is the point: it is where cache warmth becomes legible (step 0 + * cold, later steps warm), which no aggregate can show. + */ + +interface TurnUsageChipProps { + turn: TurnUsage + /** Dock density — drops the cost figure to fit a narrow header. */ + compact?: boolean +} + +export function TurnUsageChip({ turn, compact }: TurnUsageChipProps) { + const [open, setOpen] = useState(false) + const reported = hasReportedUsage(turn.totals) + + // Nothing measured and nothing in flight — stay invisible rather than + // render a row of em dashes on every reply. + if (!reported && !turn.streaming) return null + + const label = !reported + ? '↑— ↓— · running' + : [ + `↑${formatTokenCount(turn.totals.input)}`, + `↓${formatTokenCount(turn.totals.output)}`, + ...(compact || turn.totals.reported.cost === 0 + ? [] + : [`· ${formatUsageValue(turn.totals.costUsd, 'cost')}`]), + ...(turn.streaming ? ['· running'] : []), + ].join(' ') + + return ( + + + {open ? ( + + {turn.stepUsage.length === 0 ? ( + + no per-step usage recorded for this turn + + ) : ( + turn.stepUsage.map(({ entryId, usage }, i) => ( + + step {i} + {' ↑ '} + {formatUsageValue(usage.input)} + {' ↓ '} + {formatUsageValue(usage.output)} + {typeof usage.cache_read === 'number' ? ( + + {' cache r '} + {formatUsageValue(usage.cache_read)} + + ) : null} + {typeof usage.cost_usd === 'number' ? ( + {` ${formatUsageValue(usage.cost_usd, 'cost')}`} + ) : null} + + )) + )} + + turn total ↑{reportedValue(turn.totals, 'input', turn.totals.input)}{' '} + ↓{reportedValue(turn.totals, 'output', turn.totals.output)} ·{' '} + {reportedValue(turn.totals, 'cost', turn.totals.costUsd, 'cost')} + + + ) : null} + + ) +} diff --git a/console/web/src/components/metrics/MetricTable.tsx b/console/web/src/components/metrics/MetricTable.tsx new file mode 100644 index 000000000..bd9c97b91 --- /dev/null +++ b/console/web/src/components/metrics/MetricTable.tsx @@ -0,0 +1,89 @@ +import { cn } from '@/lib/utils' + +/** + * The metric table shared by the session metrics surfaces — a Tailwind port + * of eval's aggregate table (`eval/ui/src/page/EvaluationDetail.tsx:399-454`) + * in the console's own idiom. Eval's stylesheet is scoped under + * `[data-iii-ui="eval"]` and unreachable from this SPA, so this only *looks* + * like eval; nothing is shared but the visual language. + * + * Two columns rather than eval's four: there is no A/B comparison here, just + * a metric, its value, and an optional muted note. + */ + +export interface MetricRow { + label: string + /** Preformatted — callers use `formatUsageValue` / `reportedValue`. */ + value: string + /** Muted right-hand annotation: a unit, a caveat, a derivation. */ + note?: string + /** + * Descriptive-only metric with no better/worse direction, marked with a + * mid dot. Carries eval's `neutral` meaning: reading more cached tokens is + * not "better", it means cost and latency reflect cache warmth. + */ + neutral?: boolean + tone?: 'default' | 'faint' | 'alert' +} + +interface MetricTableProps { + title: string + /** One line under the title explaining where these numbers come from. */ + caption?: string + rows: MetricRow[] + className?: string +} + +export function MetricTable({ + title, + caption, + rows, + className, +}: MetricTableProps) { + return ( +
+

+ {title} +

+ {caption ? ( +

{caption}

+ ) : null} +
+ {rows.map((row) => ( +
+
+ {row.label} +
+ {row.note ? ( + + {row.note} + + ) : null} + {row.neutral ? ( + + · + + ) : null} +
+ {row.value} +
+
+ ))} +
+
+ ) +} diff --git a/console/web/src/components/metrics/SessionMetricsPanel.stories.tsx b/console/web/src/components/metrics/SessionMetricsPanel.stories.tsx new file mode 100644 index 000000000..d3403db67 --- /dev/null +++ b/console/web/src/components/metrics/SessionMetricsPanel.stories.tsx @@ -0,0 +1,195 @@ +import type { Meta, StoryObj } from '@storybook/react-vite' +import type { SessionUsage, UsageTotals } from '@/lib/session-usage' +import { SessionMetricsPanel } from './SessionMetricsPanel' + +function totals(over: Partial = {}): UsageTotals { + const base: UsageTotals = { + input: 412_908, + output: 18_204, + cacheRead: 388_110, + cacheWrite: 12_400, + reasoning: 0, + costUsd: 0.4821, + reported: { + input: 39, + output: 39, + cacheRead: 39, + cacheWrite: 39, + // anthropic never reports reasoning — this must render `—`, not `0`. + reasoning: 0, + cost: 39, + }, + total: 431_112, + } + return { ...base, ...over } +} + +const emptyTotals = totals({ + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + costUsd: 0, + total: 0, + reported: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + cost: 0, + }, +}) + +function turn(id: string, i: number, streaming = false) { + return { + turnId: id, + steps: 3, + stepUsage: [], + totals: totals({ total: 62_100 * i, costUsd: 0.031 * i }), + functionCalls: 4, + functionCallErrors: i === 2 ? 1 : 0, + startedAt: 0, + endedAt: 62_000 * i, + durationMs: 62_000 * i, + streaming, + } +} + +const full: SessionUsage = { + totals: totals(), + turns: [turn('t_9f1a', 1), turn('t_a02b', 2), turn('t_b17c', 3)], + steps: 39, + stepsMissingUsage: 3, + functionCalls: 61, + functionCallErrors: 3, + startedAt: 0, + endedAt: 724_000, + durationMs: 724_000, + lastCall: { usage: { input: 96_410, output: 288 }, at: 724_000 }, +} + +const meta = { + title: 'metrics/SessionMetricsPanel', + component: SessionMetricsPanel, + args: { + contextEstimate: 84_200, + contextWindow: 200_000, + showTurnChips: true, + onToggleTurnChips: () => {}, + onViewTraces: () => {}, + onRetryTree: () => {}, + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +} satisfies Meta + +export default meta +type Story = StoryObj + +export const Full: Story = { + args: { + usage: full, + tree: { + status: 'ok', + metrics: { + root_session_id: 's_3f2a', + complete: true, + totals: { + sessions: 3, + turns: 52, + function_calls: 88, + function_call_errors: 4, + input_tokens: 610_400, + output_tokens: 24_900, + cache_read_tokens: 502_100, + cache_write_tokens: 18_200, + reasoning_tokens: null, + cost_usd: 0.7412, + }, + by_session: [ + { + session_id: 's_3f2a', + depth: 0, + sessions: 1, + turns: 39, + function_calls: 61, + function_call_errors: 3, + input_tokens: 412_908, + output_tokens: 18_204, + }, + { + session_id: 's_child1', + parent_session_id: 's_3f2a', + depth: 1, + sessions: 1, + turns: 13, + function_calls: 27, + function_call_errors: 1, + input_tokens: 197_492, + output_tokens: 6_696, + }, + ], + traces: { + trace_count: 39, + span_count: 412, + error_span_count: 2, + duration_ms: 724_000, + }, + }, + }, + }, +} + +/** Before the backend persisted usage — exact rows dash out, counted rows work. */ +export const NoUsage: Story = { + args: { + usage: { + ...full, + totals: emptyTotals, + stepsMissingUsage: 39, + turns: [], + lastCall: undefined, + }, + tree: { status: 'unavailable' }, + }, +} + +/** codex reports no cache_write; anthropic reports no reasoning. */ +export const PartialProvider: Story = { + args: { + usage: { + ...full, + totals: totals({ + cacheWrite: 0, + reported: { + input: 39, + output: 39, + cacheRead: 39, + cacheWrite: 0, + reasoning: 0, + cost: 0, + }, + }), + }, + tree: { status: 'unavailable' }, + }, +} + +export const Streaming: Story = { + args: { + usage: { ...full, turns: [turn('t_9f1a', 1), turn('t_live', 2, true)] }, + tree: { status: 'incomplete' }, + }, +} + +/** The common case for an active chat — never render the zeroed payload. */ +export const TreeUnavailable: Story = { + args: { usage: full, tree: { status: 'incomplete' } }, +} diff --git a/console/web/src/components/metrics/SessionMetricsPanel.tsx b/console/web/src/components/metrics/SessionMetricsPanel.tsx new file mode 100644 index 000000000..b2deef94a --- /dev/null +++ b/console/web/src/components/metrics/SessionMetricsPanel.tsx @@ -0,0 +1,375 @@ +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/Tabs' +import type { HarnessMetricsState } from '@/lib/backend/harness-metrics' +import { + formatSpan, + formatUsageValue, + hasReportedUsage, + reportedValue, + type SessionUsage, +} from '@/lib/session-usage' +import { formatTokenCount } from '@/lib/token-estimate' +import { type MetricRow, MetricTable } from './MetricTable' + +/** + * The session metrics view: what this chat consumed, per session and per turn. + * + * Purely presentational — props in, nothing fetched, no conversation context. + * That is deliberate: it keeps this component portable to a right-pane route + * or a TracesV2 tab as a props change rather than a refactor. Treat an import + * of `@/lib/iii-client` or `@/lib/conversations-context` here as a bug. + * + * The exact/counted/estimated split is structural, not a footnote: which + * section a number sits in *is* the statement about how much to trust it. + */ + +interface SessionMetricsPanelProps { + usage: SessionUsage + /** chars/4 heuristic from `lib/token-estimate.ts`. */ + contextEstimate: number + contextWindow?: number + tree: HarnessMetricsState | 'loading' | null + onRetryTree?: () => void + onViewTraces?: () => void + showTurnChips: boolean + onToggleTurnChips: (next: boolean) => void +} + +export function SessionMetricsPanel({ + usage, + contextEstimate, + contextWindow, + tree, + onRetryTree, + onViewTraces, + showTurnChips, + onToggleTurnChips, +}: SessionMetricsPanelProps) { + const { totals } = usage + const anyUsage = hasReportedUsage(totals) + + const exactRows: MetricRow[] = [ + { + label: 'input tokens', + value: reportedValue(totals, 'input', totals.input), + }, + { + label: 'output tokens', + value: reportedValue(totals, 'output', totals.output), + }, + { + label: 'total tokens', + value: anyUsage ? formatUsageValue(totals.total) : '—', + note: 'input + output', + }, + { + label: 'cache read', + value: reportedValue(totals, 'cacheRead', totals.cacheRead), + note: totals.reported.cacheRead === 0 ? 'not reported' : undefined, + neutral: true, + }, + { + label: 'cache write', + value: reportedValue(totals, 'cacheWrite', totals.cacheWrite), + note: totals.reported.cacheWrite === 0 ? 'not reported' : undefined, + neutral: true, + }, + { + label: 'reasoning tokens', + value: reportedValue(totals, 'reasoning', totals.reasoning), + note: totals.reported.reasoning === 0 ? 'not reported' : undefined, + }, + { + label: 'cost', + value: reportedValue(totals, 'cost', totals.costUsd, 'cost'), + note: totals.reported.cost === 0 ? 'not reported' : undefined, + }, + ] + + const countedRows: MetricRow[] = [ + { label: 'turns', value: formatUsageValue(usage.turns.length) }, + { + label: 'steps', + value: formatUsageValue(usage.steps), + note: 'model calls', + }, + { label: 'function calls', value: formatUsageValue(usage.functionCalls) }, + { + label: 'function-call errors', + value: formatUsageValue(usage.functionCallErrors), + tone: usage.functionCallErrors > 0 ? 'alert' : 'default', + }, + { label: 'session time', value: formatSpan(usage.durationMs) }, + ] + + const estimatedRows: MetricRow[] = [ + { + label: 'context in use', + value: `~${formatTokenCount(contextEstimate)}`, + note: + contextWindow && contextWindow > 0 + ? `${Math.round((contextEstimate / contextWindow) * 100)}% of ${formatTokenCount(contextWindow)}` + : 'context window unknown', + }, + ] + + if (usage.lastCall) { + // Raw and un-summed on purpose: `input` means different things across + // providers (anthropic excludes cached, openai includes it), so any + // arithmetic here would be wrong for one of them. + const last = usage.lastCall.usage + estimatedRows.push({ + label: 'last model call', + value: + typeof last.input === 'number' + ? `${formatUsageValue(last.input)} in` + : '—', + note: 'measured prompt, un-summed', + tone: 'faint', + }) + } + + return ( + + + session + turns + tree + + + + {!anyUsage ? ( +

+ {usage.steps === 0 + ? 'no model calls in this session yet.' + : 'no provider usage recorded for this session — the counted and estimated figures below are still exact.'} +

+ ) : null} + + + +
+ + {usage.stepsMissingUsage > 0 && usage.steps > 0 + ? `${usage.stepsMissingUsage} of ${usage.steps} model calls reported no usage` + : ''} + + {onViewTraces ? ( + + ) : null} +
+ + + + + + + + + + + + ) +} + +function TurnsTable({ usage }: { usage: SessionUsage }) { + const turns = usage.turns.filter((t) => t.steps > 0 || t.functionCalls > 0) + if (turns.length === 0) { + return ( +

+ no completed turns yet. +

+ ) + } + return ( +
+ + + + + + + + + + + + {turns.map((turn, i) => ( + + + + + + + + ))} + +
turnstepstokenscosttime
+ {turn.turnId.startsWith('local:') ? `#${i + 1}` : turn.turnId} + {turn.streaming ? ( + · running + ) : null} + + {turn.steps} + + {hasReportedUsage(turn.totals) + ? formatUsageValue(turn.totals.total) + : '—'} + + {reportedValue( + turn.totals, + 'cost', + turn.totals.costUsd, + 'cost', + )} + + {formatSpan(turn.durationMs)} +
+
+ ) +} + +function TreePanel({ + tree, + onRetry, +}: { + tree: HarnessMetricsState | 'loading' | null + onRetry?: () => void +}) { + if (tree === 'loading' || tree === null) { + return ( +

+ {tree === 'loading' ? 'loading…' : 'not loaded.'} +

+ ) + } + + if (tree.status !== 'ok') { + return ( +
+

+ {tree.status === 'incomplete' + ? 'sub-agent and trace totals need every session in the tree to be idle — unavailable while a turn is running.' + : 'harness::metrics is unavailable.'} +

+ {onRetry ? ( + + ) : null} +
+ ) + } + + const { totals, by_session, traces } = tree.metrics + const num = (v: number | null | undefined, kind?: 'cost') => + typeof v === 'number' + ? formatUsageValue(v, kind === 'cost' ? 'cost' : 'number') + : '—' + + const rows: MetricRow[] = [ + { label: 'sessions', value: formatUsageValue(totals.sessions) }, + { + label: 'steps', + value: formatUsageValue(totals.turns), + note: 'model calls, whole tree', + }, + { label: 'input tokens', value: num(totals.input_tokens) }, + { label: 'output tokens', value: num(totals.output_tokens) }, + { + label: 'cache read', + value: num(totals.cache_read_tokens), + neutral: true, + }, + { + label: 'cache write', + value: num(totals.cache_write_tokens), + neutral: true, + }, + { label: 'reasoning tokens', value: num(totals.reasoning_tokens) }, + { label: 'cost', value: num(totals.cost_usd, 'cost') }, + { label: 'function calls', value: formatUsageValue(totals.function_calls) }, + { + label: 'function-call errors', + value: formatUsageValue(totals.function_call_errors), + tone: totals.function_call_errors > 0 ? 'alert' : 'default', + }, + ] + + const traceRows: MetricRow[] = traces + ? [ + { label: 'traces', value: formatUsageValue(traces.trace_count) }, + { label: 'spans', value: formatUsageValue(traces.span_count) }, + { + label: 'error spans', + value: formatUsageValue(traces.error_span_count), + tone: traces.error_span_count > 0 ? 'alert' : 'default', + }, + { + label: 'trace duration', + value: formatUsageValue(traces.duration_ms, 'duration'), + }, + ] + : [] + + return ( +
+ + {traceRows.length > 0 ? ( + + ) : null} + {by_session.length > 1 ? ( + ({ + label: `${' '.repeat(s.depth)}${s.session_id}`, + value: num( + typeof s.input_tokens === 'number' && + typeof s.output_tokens === 'number' + ? s.input_tokens + s.output_tokens + : null, + ), + note: `${s.turns} calls`, + }))} + /> + ) : null} +
+ ) +} diff --git a/console/web/src/lib/backend/harness-metrics.ts b/console/web/src/lib/backend/harness-metrics.ts new file mode 100644 index 000000000..56c5ce3ff --- /dev/null +++ b/console/web/src/lib/backend/harness-metrics.ts @@ -0,0 +1,89 @@ +/** + * Typed wrapper over `harness::metrics` — the only source of usage totals for + * a whole *session tree*, i.e. including sub-agent sessions spawned by + * `harness::spawn`, plus trace/span counts. Shapes mirror + * `harness/src/functions/metrics.rs:19-106`. + * + * Two properties make this a bonus rather than a primary source, and both are + * why the console computes its own rollup from the transcript instead: + * + * - It returns everything ZEROED with `complete: false` unless every session + * in the tree has a terminal turn record (`metrics.rs:140-147`) — so it is + * unavailable exactly while a chat is active. + * - Its token fields are all-or-nothing sums: one generation missing a field + * collapses that field to `null` for the entire tree. + * + * We therefore discard an incomplete payload entirely rather than render a + * confident, wrong `$0.000000`. + */ + +import { getIiiClient } from '@/lib/iii-client' + +const METRICS_FN = 'harness::metrics' +const TIMEOUT_MS = 10_000 + +export interface SessionUsageTotalsV1 { + sessions: number + /** + * Assistant messages, i.e. model calls — `metrics.rs:305` increments this + * once per assistant message, so a turn with three tool rounds counts as + * three. The console labels this "steps" and keeps "turns" for real turns. + */ + turns: number + function_calls: number + function_call_errors: number + input_tokens?: number | null + output_tokens?: number | null + cache_read_tokens?: number | null + cache_write_tokens?: number | null + reasoning_tokens?: number | null + cost_usd?: number | null +} + +export interface SessionUsageV1 extends SessionUsageTotalsV1 { + session_id: string + parent_session_id?: string | null + depth: number +} + +export interface SessionTraceMetricsV1 { + trace_count: number + span_count: number + error_span_count: number + duration_ms: number +} + +export interface SessionMetricsResponseV1 { + root_session_id: string + complete: boolean + totals: SessionUsageTotalsV1 + by_session: SessionUsageV1[] + traces?: SessionTraceMetricsV1 | null +} + +export type HarnessMetricsState = + | { status: 'ok'; metrics: SessionMetricsResponseV1 } + /** The tree has a turn in flight — totals would be all zeros. */ + | { status: 'incomplete' } + /** The harness is absent, errored, or timed out. */ + | { status: 'unavailable' } + +export async function fetchHarnessMetrics( + rootSessionId: string, +): Promise { + try { + const client = await getIiiClient() + const raw = (await client.trigger( + METRICS_FN, + { root_session_id: rootSessionId }, + { timeoutMs: TIMEOUT_MS }, + )) as SessionMetricsResponseV1 | null + + if (!raw || typeof raw !== 'object') return { status: 'unavailable' } + // An incomplete response is not partial data — it is all zeros. + if (!raw.complete) return { status: 'incomplete' } + return { status: 'ok', metrics: raw } + } catch { + return { status: 'unavailable' } + } +} diff --git a/console/web/src/lib/session-usage.test.ts b/console/web/src/lib/session-usage.test.ts new file mode 100644 index 000000000..921632393 --- /dev/null +++ b/console/web/src/lib/session-usage.test.ts @@ -0,0 +1,402 @@ +import { describe, expect, it } from 'vitest' +import type { Usage } from '@/lib/sessions/types' +import type { Message } from '@/types/chat' +import { + formatSpan, + formatUsageValue, + normalizeUsage, + parseTurnId, + reportedValue, + sessionUsage, + turnUsageByAnchor, + turnUsages, +} from './session-usage' + +const user = (id: string, at = 0, turnId?: string): Message => ({ + id, + role: 'user', + content: 'hi', + createdAt: at, + ...(turnId ? { turnId } : {}), +}) + +const assistant = ( + id: string, + content: string, + opts: { + at?: number + usage?: Usage + turnId?: string + streaming?: boolean + } = {}, +): Message => ({ + id, + role: 'assistant', + content, + createdAt: opts.at ?? 0, + ...(opts.usage ? { usage: opts.usage } : {}), + ...(opts.turnId ? { turnId: opts.turnId } : {}), + ...(opts.streaming ? { streaming: true } : {}), +}) + +const fcall = ( + id: string, + opts: { at?: number; usage?: Usage; error?: boolean } = {}, +): Message => ({ + id, + role: 'function-trigger', + functionId: 'shell::exec', + input: {}, + output: opts.error ? { error: { kind: 'function_error' } } : { ok: true }, + createdAt: opts.at ?? 0, + ...(opts.usage ? { usage: opts.usage } : {}), +}) + +describe('parseTurnId', () => { + // Mirrors harness/src/ids.rs `assistant_entry_id(turn_id, step)`. + it('recovers the turn id from a harness assistant entry id', () => { + expect(parseTurnId('e_t_abc123_0_assistant')).toBe('t_abc123') + expect(parseTurnId('e_t_abc123_12_assistant')).toBe('t_abc123') + }) + + it('splits on the LAST separator so underscore-bearing turn ids survive', () => { + // `harness/src/ids.rs:118` asserts assistant_entry_id("t_1", 3) is + // "e_t_1_3_assistant" — splitting on the first `_` would yield "t". + expect(parseTurnId('e_t_1_3_assistant')).toBe('t_1') + expect(parseTurnId('e_t_a_b_c_7_assistant')).toBe('t_a_b_c') + }) + + it('ignores a segment-index suffix added by the mapper', () => { + expect(parseTurnId('e_t_abc123_0_assistant:2')).toBe('t_abc123') + }) + + it('returns undefined for ids that are not assistant entries', () => { + expect(parseTurnId('e_t_abc_fc_call1')).toBeUndefined() + expect(parseTurnId('e_notify_xyz')).toBeUndefined() + expect(parseTurnId('local-optimistic-1')).toBeUndefined() + expect(parseTurnId('e__assistant')).toBeUndefined() + }) +}) + +describe('normalizeUsage', () => { + it('drops an object with no usable numbers', () => { + expect(normalizeUsage(undefined)).toBeUndefined() + expect(normalizeUsage({})).toBeUndefined() + expect(normalizeUsage({ input: undefined })).toBeUndefined() + }) + + it('keeps an object with a zero value — zero is reported, not absent', () => { + expect(normalizeUsage({ cache_read: 0 })).toEqual({ cache_read: 0 }) + }) +}) + +describe('sessionUsage totals', () => { + it('totals input + output only, excluding cache and reasoning', () => { + // An OpenAI-shaped payload: `input` already INCLUDES cache_read and + // `output` already includes reasoning. Adding them would double-count. + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'hi', { + usage: { + input: 1000, + output: 200, + cache_read: 800, + reasoning: 150, + cost_usd: 0.01, + }, + }), + ] + const { totals } = sessionUsage(messages) + expect(totals.input).toBe(1000) + expect(totals.output).toBe(200) + expect(totals.total).toBe(1200) + expect(totals.cacheRead).toBe(800) + expect(totals.reasoning).toBe(150) + }) + + it('applies the same total rule to an anthropic-shaped payload', () => { + // Anthropic's `input` EXCLUDES cached tokens. `total` stays input+output + // for both providers — the number means "billed in/out", not "prompt size". + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'hi', { + usage: { + input: 200, + output: 200, + cache_read: 800, + cache_write: 100, + }, + }), + ] + const { totals } = sessionUsage(messages) + expect(totals.total).toBe(400) + expect(totals.cacheWrite).toBe(100) + }) + + it('sums across steps but counts one entry once across its segments', () => { + const usage: Usage = { input: 10, output: 2 } + const messages = [ + user('u1'), + // Two segments from the SAME entry — mapper attaches usage to the first. + assistant('e_t_1_0_assistant:0', 'part one', { usage }), + assistant('e_t_1_0_assistant:1', 'part two'), + // A second step: a separate provider request, so it sums. + assistant('e_t_1_1_assistant', 'done', { + usage: { input: 30, output: 4 }, + }), + ] + const { totals } = sessionUsage(messages) + expect(totals.input).toBe(40) + expect(totals.output).toBe(6) + expect(totals.total).toBe(46) + }) +}) + +describe('reported vs zero', () => { + it('renders an unreported field as em dash, not zero', () => { + // codex never reports cache_write; anthropic never reports reasoning. + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'hi', { usage: { input: 5, output: 1 } }), + ] + const { totals } = sessionUsage(messages) + expect(totals.reported.cacheWrite).toBe(0) + expect(reportedValue(totals, 'cacheWrite', totals.cacheWrite)).toBe('—') + expect(reportedValue(totals, 'input', totals.input)).toBe('5') + }) + + it('renders a reported zero as 0', () => { + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'hi', { + usage: { input: 5, output: 1, cache_read: 0 }, + }), + ] + const { totals } = sessionUsage(messages) + expect(totals.reported.cacheRead).toBe(1) + expect(reportedValue(totals, 'cacheRead', totals.cacheRead)).toBe('0') + }) +}) + +describe('steps and missing usage', () => { + it('counts model calls and how many reported no usage', () => { + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'thinking', { + usage: { input: 10, output: 2 }, + }), + fcall('e_t_1_fc1'), + assistant('e_t_1_1_assistant', 'done'), + ] + const usage = sessionUsage(messages) + expect(usage.steps).toBe(3) + expect(usage.stepsMissingUsage).toBe(2) + expect(usage.functionCalls).toBe(1) + }) + + it('counts errored function triggers', () => { + const messages = [ + user('u1'), + fcall('e_t_1_fc1', { error: true }), + fcall('e_t_1_fc2'), + ] + const usage = sessionUsage(messages) + expect(usage.functionCalls).toBe(2) + expect(usage.functionCallErrors).toBe(1) + }) + + it('exposes the last call raw and unsummed for the ctx cross-check', () => { + const messages = [ + user('u1'), + assistant('e_t_1_0_assistant', 'a', { usage: { input: 10, output: 2 } }), + assistant('e_t_1_1_assistant', 'b', { + at: 5, + usage: { input: 90, output: 4 }, + }), + ] + const { lastCall } = sessionUsage(messages) + expect(lastCall?.usage.input).toBe(90) + }) +}) + +describe('turn grouping', () => { + it('counts a harness turn once, prompt included', () => { + // The mapper stamps turnId only on assistant entries, so the user message + // arrives without one. If it opened its own bucket every turn would be + // counted twice. + const messages = [ + { id: 'e_idem_x', role: 'user' as const, content: 'hi', createdAt: 0 }, + assistant('e_t_9_0_assistant', 'done', { + at: 1, + turnId: 't_9', + usage: { input: 10, output: 2 }, + }), + ] + const turns = turnUsages(messages) + expect(turns).toHaveLength(1) + expect(turns[0].turnId).toBe('t_9') + // The prompt's timestamp anchors the turn's start. + expect(turns[0].startedAt).toBe(0) + }) + + it('keeps consecutive harness turns separate', () => { + const messages = [ + user('e_idem_1', 0), + assistant('e_t_1_0_assistant', 'a', { + at: 1, + turnId: 't_1', + usage: { input: 10, output: 2 }, + }), + user('e_idem_2', 2), + assistant('e_t_2_0_assistant', 'b', { + at: 3, + turnId: 't_2', + usage: { input: 20, output: 4 }, + }), + ] + const turns = turnUsages(messages) + expect(turns.map((t) => t.turnId)).toEqual(['t_1', 't_2']) + }) + + it('opens a synthetic turn for a prompt still awaiting its reply', () => { + const messages = [ + user('e_idem_1', 0), + assistant('e_t_1_0_assistant', 'a', { at: 1, turnId: 't_1' }), + // Optimistic send: no assistant entry exists yet. + user('local-pending', 2), + ] + const turns = turnUsages(messages) + expect(turns).toHaveLength(2) + expect(turns[1].turnId).toMatch(/^local:/) + }) + + it('groups steps of one tool loop into a single turn', () => { + const messages = [ + user('u1', 0), + assistant('e_t_9_0_assistant', '', { + at: 1, + usage: { input: 100, output: 10 }, + }), + fcall('e_t_9_fc1', { at: 2 }), + assistant('e_t_9_1_assistant', 'all done', { + at: 3, + usage: { input: 200, output: 20 }, + }), + ] + const turns = turnUsages(messages) + const withUsage = turns.filter((t) => t.steps > 0) + expect(withUsage).toHaveLength(1) + const turn = withUsage[0] + expect(turn.turnId).toBe('t_9') + expect(turn.steps).toBe(2) + expect(turn.totals.total).toBe(330) + expect(turn.functionCalls).toBe(1) + // Spans the prompt (t=0) through the closing reply (t=3). + expect(turn.durationMs).toBe(3) + }) + + it('prefers a mapper-supplied turnId over the parsed entry id', () => { + const messages = [ + user('u1'), + assistant('e_t_parsed_0_assistant', 'hi', { + turnId: 't_from_origin', + usage: { input: 1, output: 1 }, + }), + ] + expect(turnUsages(messages).at(-1)?.turnId).toBe('t_from_origin') + }) + + it('falls back to a user-message boundary when no turn id exists', () => { + const messages = [ + user('u1', 0), + assistant('local-1', 'first'), + user('u2', 10), + assistant('local-2', 'second'), + ] + expect(turnUsages(messages)).toHaveLength(2) + }) + + it('carries tokens from a tool-only turn but gives it no anchor', () => { + // An assistant entry with only function calls emits no prose segment, + // yet it cost a real request. Its tokens must still count. + const messages = [ + user('u1'), + fcall('e_t_5_fc1', { usage: { input: 500, output: 25 } }), + ] + const turn = turnUsages(messages).at(-1) + expect(turn?.totals.total).toBe(525) + expect(turn?.anchorId).toBeUndefined() + expect(turnUsageByAnchor(messages).size).toBe(0) + }) + + it('anchors the chip on the turn last prose segment', () => { + const messages = [ + user('u1'), + assistant('e_t_7_0_assistant', 'first', { + usage: { input: 1, output: 1 }, + }), + assistant('e_t_7_1_assistant', 'last', { + usage: { input: 2, output: 2 }, + }), + ] + const byAnchor = turnUsageByAnchor(messages) + expect(byAnchor.get('e_t_7_1_assistant')?.totals.total).toBe(6) + expect(byAnchor.has('e_t_7_0_assistant')).toBe(false) + }) + + it('marks a turn streaming while any of its segments is', () => { + const messages = [ + user('u1'), + assistant('e_t_3_0_assistant', 'partial', { streaming: true }), + ] + expect(turnUsages(messages).at(-1)?.streaming).toBe(true) + }) +}) + +describe('empty and degenerate input', () => { + it('handles an empty transcript', () => { + const usage = sessionUsage([]) + expect(usage.steps).toBe(0) + expect(usage.totals.total).toBe(0) + expect(usage.turns).toEqual([]) + expect(usage.durationMs).toBe(0) + expect(usage.lastCall).toBeUndefined() + }) + + it('handles a transcript with no usage at all (pre-fix sessions)', () => { + const messages = [user('u1'), assistant('e_t_1_0_assistant', 'hi')] + const usage = sessionUsage(messages) + expect(usage.totals.total).toBe(0) + expect(reportedValue(usage.totals, 'input', usage.totals.input)).toBe('—') + expect(usage.steps).toBe(1) + expect(usage.stepsMissingUsage).toBe(1) + }) +}) + +describe('formatting', () => { + it('formats durations like eval formatMetric', () => { + expect(formatUsageValue(450, 'duration')).toBe('450ms') + expect(formatUsageValue(1500, 'duration')).toBe('1.50s') + }) + + it('formats cost to six decimals like eval', () => { + expect(formatUsageValue(0.0482, 'cost')).toBe('$0.048200') + }) + + it('formats token counts with the console dialect', () => { + expect(formatUsageValue(412908, 'tokens')).toBe('413k') + expect(formatUsageValue(900, 'tokens')).toBe('900') + }) + + it('returns em dash for undefined and non-finite', () => { + expect(formatUsageValue(undefined)).toBe('—') + expect(formatUsageValue(Number.NaN)).toBe('—') + }) + + it('formats session spans', () => { + expect(formatSpan(38_000)).toBe('38s') + expect(formatSpan(724_000)).toBe('12m 04s') + expect(formatSpan(14_820_000)).toBe('4h 07m') + expect(formatSpan(0)).toBe('—') + }) +}) diff --git a/console/web/src/lib/session-usage.ts b/console/web/src/lib/session-usage.ts new file mode 100644 index 000000000..3a9b32984 --- /dev/null +++ b/console/web/src/lib/session-usage.ts @@ -0,0 +1,461 @@ +/** + * Session and per-turn usage rollups, computed from the transcript the + * console already holds. Pure: no React, no iii client, no fetching — so the + * arithmetic (which is the part that is easy to get wrong) is unit-testable + * on its own. + * + * Two rules govern everything here: + * + * 1. **`total = input + output`, never more.** Cache and reasoning tokens are + * not portably additive. Anthropic's `input` excludes cached tokens and + * reports them separately; OpenAI's `input` already includes `cache_read` + * and its `output` already includes `reasoning`. Adding them would + * double-count on OpenAI. This matches the convention eval already uses + * (`eval/src/report.rs:59-63`, `eval/src/limits.rs:128-134`). + * + * 2. **Absent is not zero.** Providers report disjoint subsets — codex never + * reports `cache_write`, anthropic never reports `reasoning`. Every total + * carries a `reported` count so the UI can render `—` ("not reported") + * rather than a confident `0`. + * + * Within one entry, provider usage is a cumulative running total that gets + * overwritten as the stream progresses. Across entries (steps of a tool loop) + * it sums — each step is a separate provider request. + */ + +import type { Usage } from '@/lib/sessions/types' +import { formatTokenCount } from '@/lib/token-estimate' +import type { Message } from '@/types/chat' + +export type UsageField = + | 'input' + | 'output' + | 'cacheRead' + | 'cacheWrite' + | 'reasoning' + | 'cost' + +const USAGE_FIELDS: UsageField[] = [ + 'input', + 'output', + 'cacheRead', + 'cacheWrite', + 'reasoning', + 'cost', +] + +export interface UsageTotals { + input: number + output: number + cacheRead: number + cacheWrite: number + reasoning: number + costUsd: number + /** + * How many contributing entries reported each field. Zero means the field + * was never reported and must render `—`, not `0`. + */ + reported: Record + /** `input + output`. Deliberately excludes cache and reasoning — see above. */ + total: number +} + +export interface StepUsage { + entryId: string + usage: Usage +} + +export interface TurnUsage { + turnId: string + /** Assistant entries in this turn — one per model call in the tool loop. */ + steps: number + stepUsage: StepUsage[] + totals: UsageTotals + functionCalls: number + functionCallErrors: number + startedAt: number + endedAt: number + durationMs: number + /** + * Message id of the turn's last assistant prose segment — where the chip + * hangs. Undefined for a tool-only turn, which has no prose to hang it on. + */ + anchorId?: string + streaming: boolean +} + +export interface SessionUsage { + totals: UsageTotals + turns: TurnUsage[] + /** Model calls (assistant entries). Not the same as turns. */ + steps: number + /** Steps that reported no usage at all — drives the "N of M" footer. */ + stepsMissingUsage: number + functionCalls: number + functionCallErrors: number + startedAt: number + endedAt: number + durationMs: number + /** + * The most recent step's raw, unsummed numbers. Used to cross-check the + * chars/4 context estimate against something the provider actually said, + * without doing any cross-field arithmetic on it. + */ + lastCall?: { usage: Usage; model?: string; at: number } +} + +function emptyTotals(): UsageTotals { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + costUsd: 0, + reported: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + cost: 0, + }, + total: 0, + } +} + +function addNumber( + totals: UsageTotals, + key: Exclude, + field: UsageField, + value: number | undefined, +): void { + if (typeof value !== 'number' || !Number.isFinite(value)) return + totals[key] += value + totals.reported[field] += 1 +} + +/** Fold one entry's usage into a running total. */ +export function accumulate(totals: UsageTotals, usage: Usage): void { + addNumber(totals, 'input', 'input', usage.input) + addNumber(totals, 'output', 'output', usage.output) + addNumber(totals, 'cacheRead', 'cacheRead', usage.cache_read) + addNumber(totals, 'cacheWrite', 'cacheWrite', usage.cache_write) + addNumber(totals, 'reasoning', 'reasoning', usage.reasoning) + addNumber(totals, 'costUsd', 'cost', usage.cost_usd) + // The only cross-provider-safe total. + totals.total = totals.input + totals.output +} + +/** True when at least one field of `totals` was actually reported. */ +export function hasReportedUsage(totals: UsageTotals): boolean { + return USAGE_FIELDS.some((f) => totals.reported[f] > 0) +} + +/** Drop a usage object that carries no usable numbers at all. */ +export function normalizeUsage(usage: Usage | undefined): Usage | undefined { + if (!usage) return undefined + const hasAny = ( + [ + 'input', + 'output', + 'cache_read', + 'cache_write', + 'reasoning', + 'cost_usd', + ] as const + ).some((k) => typeof usage[k] === 'number' && Number.isFinite(usage[k])) + return hasAny ? usage : undefined +} + +/** + * Recover the turn id from a harness assistant entry id. + * + * The harness mints these as `e___assistant` + * (`harness/src/ids.rs:63`), and **turn ids themselves contain underscores** + * (`t_`), so the split has to come off the *last* separator, not the + * first. A segment id may carry a `:` suffix from the mapper. + */ +export function parseTurnId(messageId: string): string | undefined { + const entryId = messageId.split(':')[0] ?? '' + if (!entryId.startsWith('e_') || !entryId.endsWith('_assistant')) { + return undefined + } + const middle = entryId.slice('e_'.length, -'_assistant'.length) + const lastSep = middle.lastIndexOf('_') + if (lastSep <= 0) return undefined + const turnId = middle.slice(0, lastSep) + return turnId.length > 0 ? turnId : undefined +} + +/** A function-trigger message whose output is an error envelope. */ +function isErroredTrigger(message: Message): boolean { + if (message.role !== 'function-trigger') return false + const output: unknown = message.output + return typeof output === 'object' && output !== null && 'error' in output +} + +interface TurnBucket { + turnId: string + entries: Map + order: string[] + functionCalls: number + functionCallErrors: number + startedAt: number + endedAt: number + anchorId?: string + streaming: boolean +} + +/** + * Assign every message the turn it belongs to. + * + * The mapper only stamps `turnId` on assistant entries, so a user message + * arrives without one. It must still join the turn it *starts* — otherwise + * every harness turn splits into two buckets (the prompt and the work) and + * the turn count doubles. Hence the backward pass: an unkeyed user message + * adopts the key of whatever follows it. + * + * Everything else falls back forward, to the turn already in progress. When + * no key exists anywhere — optimistic local sends, `e_idem_*` webhook + * entries, sessions written before the harness stamped origins — each user + * message opens a synthetic `local:N` turn. + */ +function resolveTurnKeys(messages: readonly Message[]): string[] { + const keys: (string | undefined)[] = messages.map( + (m) => m.turnId ?? parseTurnId(m.id), + ) + + // Backward: a prompt belongs to the turn it opens. + for (let i = messages.length - 2; i >= 0; i--) { + if (!keys[i] && messages[i].role === 'user') keys[i] = keys[i + 1] + } + + // Forward: continuations inherit; unkeyed prompts open a synthetic turn. + let fallbackIndex = 0 + let previous: string | undefined + const resolved: string[] = [] + for (let i = 0; i < messages.length; i++) { + let key = keys[i] + if (!key) { + if (messages[i].role === 'user' || !previous) { + fallbackIndex += 1 + key = `local:${fallbackIndex}` + } else { + key = previous + } + } + resolved.push(key) + previous = key + } + return resolved +} + +function bucketTurns(messages: readonly Message[]): TurnBucket[] { + const buckets: TurnBucket[] = [] + const byId = new Map() + const keys = resolveTurnKeys(messages) + + const bucketFor = (turnId: string): TurnBucket => { + let bucket = byId.get(turnId) + if (!bucket) { + bucket = { + turnId, + entries: new Map(), + order: [], + functionCalls: 0, + functionCallErrors: 0, + startedAt: Number.POSITIVE_INFINITY, + endedAt: 0, + streaming: false, + } + byId.set(turnId, bucket) + buckets.push(bucket) + } + return bucket + } + + for (const [index, message] of messages.entries()) { + const current = bucketFor(keys[index]) + + current.startedAt = Math.min(current.startedAt, message.createdAt) + current.endedAt = Math.max(current.endedAt, message.createdAt) + + // Usage is keyed by entry so that the several segments produced from one + // assistant entry contribute their (identical) usage exactly once. + const usage = normalizeUsage(message.usage) + if (usage) { + const entryId = message.id.split(':')[0] ?? message.id + if (!current.entries.has(entryId)) current.order.push(entryId) + current.entries.set(entryId, usage) + } + + if (message.role === 'function-trigger') { + current.functionCalls += 1 + if (isErroredTrigger(message)) current.functionCallErrors += 1 + } + if (message.role === 'assistant') { + if (message.content.length > 0) current.anchorId = message.id + if (message.streaming) current.streaming = true + } + } + + return buckets +} + +function finishTurn(bucket: TurnBucket): TurnUsage { + const totals = emptyTotals() + const stepUsage: StepUsage[] = [] + for (const entryId of bucket.order) { + const usage = bucket.entries.get(entryId) + if (!usage) continue + stepUsage.push({ entryId, usage }) + accumulate(totals, usage) + } + const startedAt = Number.isFinite(bucket.startedAt) ? bucket.startedAt : 0 + return { + turnId: bucket.turnId, + steps: stepUsage.length, + stepUsage, + totals, + functionCalls: bucket.functionCalls, + functionCallErrors: bucket.functionCallErrors, + startedAt, + endedAt: bucket.endedAt, + durationMs: Math.max(0, bucket.endedAt - startedAt), + ...(bucket.anchorId ? { anchorId: bucket.anchorId } : {}), + streaming: bucket.streaming, + } +} + +export function turnUsages(messages: readonly Message[]): TurnUsage[] { + return bucketTurns(messages).map(finishTurn) +} + +/** Index turns by their anchor message id, for rendering the in-transcript chip. */ +export function turnUsageByAnchor( + messages: readonly Message[], +): Map { + const map = new Map() + for (const turn of turnUsages(messages)) { + if (turn.anchorId) map.set(turn.anchorId, turn) + } + return map +} + +export function sessionUsage(messages: readonly Message[]): SessionUsage { + const turns = turnUsages(messages) + const totals = emptyTotals() + + // Count model calls across the whole transcript rather than per turn: an + // assistant entry that produced only tool calls still cost a request, and + // it may not belong to any turn that has prose. + const seenEntries = new Set() + let steps = 0 + let stepsMissingUsage = 0 + let functionCalls = 0 + let functionCallErrors = 0 + let startedAt = Number.POSITIVE_INFINITY + let endedAt = 0 + let lastCall: SessionUsage['lastCall'] + + for (const message of messages) { + startedAt = Math.min(startedAt, message.createdAt) + endedAt = Math.max(endedAt, message.createdAt) + + if (message.role === 'function-trigger') { + functionCalls += 1 + if (isErroredTrigger(message)) functionCallErrors += 1 + } + + const entryId = message.id.split(':')[0] ?? message.id + if (seenEntries.has(entryId)) continue + const isModelCall = + message.role === 'assistant' || + message.role === 'thought' || + message.role === 'function-trigger' + if (!isModelCall) continue + seenEntries.add(entryId) + steps += 1 + + const usage = normalizeUsage(message.usage) + if (usage) { + accumulate(totals, usage) + lastCall = { + usage, + ...(message.role === 'assistant' && message.model + ? { model: message.model } + : {}), + at: message.createdAt, + } + } else { + stepsMissingUsage += 1 + } + } + + return { + totals, + turns, + steps, + stepsMissingUsage, + functionCalls, + functionCallErrors, + startedAt: Number.isFinite(startedAt) ? startedAt : 0, + endedAt, + durationMs: Number.isFinite(startedAt) + ? Math.max(0, endedAt - startedAt) + : 0, + ...(lastCall ? { lastCall } : {}), + } +} + +/** + * Value formatting, ported from eval's `formatMetric` + * (`eval/ui/src/components.tsx:55-68`) so the two surfaces read the same. + * Token counts delegate to the console's existing `formatTokenCount`. + */ +export function formatUsageValue( + value: number | undefined, + kind: 'number' | 'tokens' | 'duration' | 'cost' = 'number', +): string { + if (value === undefined || !Number.isFinite(value)) return '—' + switch (kind) { + case 'tokens': + return formatTokenCount(value) + case 'duration': + return value >= 1000 + ? `${(value / 1000).toFixed(2)}s` + : `${value.toFixed(0)}ms` + case 'cost': + return `$${value.toFixed(6)}` + default: + return new Intl.NumberFormat().format(value) + } +} + +/** + * The value for a metric row: `—` when the field was never reported, so that + * "this provider does not report cache writes" never reads as "zero cache + * writes". + */ +export function reportedValue( + totals: UsageTotals, + field: UsageField, + value: number, + kind: 'number' | 'tokens' | 'duration' | 'cost' = 'number', +): string { + if (totals.reported[field] === 0) return '—' + return formatUsageValue(value, kind) +} + +/** `12m 04s` / `4h 07m` / `38s` — coarser than eval's, for session spans. */ +export function formatSpan(ms: number): string { + if (!Number.isFinite(ms) || ms <= 0) return '—' + const totalSeconds = Math.floor(ms / 1000) + const seconds = totalSeconds % 60 + const minutes = Math.floor(totalSeconds / 60) % 60 + const hours = Math.floor(totalSeconds / 3600) + if (hours > 0) return `${hours}h ${String(minutes).padStart(2, '0')}m` + if (minutes > 0) return `${minutes}m ${String(seconds).padStart(2, '0')}s` + return `${seconds}s` +} diff --git a/console/web/src/lib/sessions/entry-mapper.test.ts b/console/web/src/lib/sessions/entry-mapper.test.ts index fd7683c7b..ce1a98c75 100644 --- a/console/web/src/lib/sessions/entry-mapper.test.ts +++ b/console/web/src/lib/sessions/entry-mapper.test.ts @@ -537,6 +537,92 @@ describe('entrySegments', () => { }) }) +describe('entrySegments — usage and turn identity', () => { + function usageItem( + entryId: string, + content: Extract['content'], + usage?: Record, + origin?: TranscriptItem['origin'], + ): TranscriptItem { + return { + entry_id: entryId, + ...(origin ? { origin } : {}), + message: { + role: 'assistant', + content, + stop_reason: 'end', + model: 'm', + provider: 'p', + timestamp: 2, + ...(usage ? { usage } : {}), + }, + } + } + + it('attaches usage and turn id to the first segment', () => { + const segments = entrySegments( + usageItem('e_t_1_0_assistant', [{ type: 'text', text: 'hi' }], { + input: 10, + output: 2, + }), + ) + expect(segments[0]).toMatchObject({ + usage: { input: 10, output: 2 }, + turnId: 't_1', + }) + }) + + it('prefers origin.turn_id over the id parse', () => { + const segments = entrySegments( + usageItem( + 'e_t_parsed_0_assistant', + [{ type: 'text', text: 'hi' }], + undefined, + { turn_id: 't_from_origin' }, + ), + ) + expect(segments[0]?.turnId).toBe('t_from_origin') + }) + + it('attaches usage to a tool-only entry, which has no assistant segment', () => { + // The step emitted only function calls but still cost a request; if usage + // rode the first *assistant* segment it would be dropped here. + const segments = entrySegments( + usageItem( + 'e_t_4_0_assistant', + [ + { + type: 'function_call', + id: 'fc1', + function_id: 'shell::exec', + arguments: {}, + }, + ], + { input: 500, output: 25 }, + ), + ) + expect(segments.some((s) => s.role === 'assistant')).toBe(false) + expect(segments[0]).toMatchObject({ + role: 'function-trigger', + usage: { input: 500, output: 25 }, + }) + }) + + it('sets nothing when the entry reports no usage', () => { + const segments = entrySegments( + usageItem('e_t_1_0_assistant', [{ type: 'text', text: 'hi' }]), + ) + expect(segments[0]?.usage).toBeUndefined() + }) + + it('drops a usage object that carries no numbers', () => { + const segments = entrySegments( + usageItem('e_t_1_0_assistant', [{ type: 'text', text: 'hi' }], {}), + ) + expect(segments[0]?.usage).toBeUndefined() + }) +}) + describe('applyEntryUpsert', () => { it('replaces the optimistic user message in place (predicted entry id)', () => { const optimistic: Message = { diff --git a/console/web/src/lib/sessions/entry-mapper.ts b/console/web/src/lib/sessions/entry-mapper.ts index 57b3dac81..13275dac7 100644 --- a/console/web/src/lib/sessions/entry-mapper.ts +++ b/console/web/src/lib/sessions/entry-mapper.ts @@ -25,6 +25,7 @@ */ import { parseAttachedFileHeader } from '@/lib/file-mentions' +import { normalizeUsage, parseTurnId } from '@/lib/session-usage' import type { Attachment, FunctionTriggerMessage, @@ -406,8 +407,9 @@ export function entrySegments( | undefined const isNotif = origin?.notification === true || item.entry_id.startsWith('e_notify_') - // A react-fired task delivered into this session (origin on events, - // `e_react_` prefix on reads — session::messages carries no origin). + // A react-fired task delivered into this session. `session::messages` + // does return `origin`, but only for message entries written with one — + // the `e_react_` prefix stays as the fallback for entries that lack it. const isReaction = origin?.reaction === true || item.entry_id.startsWith('e_react_') // A direct `harness::spawn` seed task — same pattern, `e_spawn_` prefix. @@ -463,6 +465,22 @@ export function entrySegments( } } } + // Provider usage and turn identity for the metrics rollup. Unlike + // `memory` this rides `segments[0]` whatever its role: a step that only + // produced function calls emits no assistant segment at all, yet it + // cost a real request whose tokens must still be counted. The carrier + // is incidental — only `lib/session-usage.ts` reads these back. + const carrier = segments[0] + if (carrier) { + const turnId = + typeof (item.origin as { turn_id?: unknown } | undefined)?.turn_id === + 'string' + ? (item.origin as { turn_id: string }).turn_id + : parseTurnId(item.entry_id) + if (turnId) carrier.turnId = turnId + const usage = normalizeUsage(message.usage) + if (usage) carrier.usage = usage + } return segments } case 'function_result': diff --git a/console/web/src/lib/sessions/types.ts b/console/web/src/lib/sessions/types.ts index 0cb52dc64..863ccd337 100644 --- a/console/web/src/lib/sessions/types.ts +++ b/console/web/src/lib/sessions/types.ts @@ -22,6 +22,29 @@ export type ContentBlock = is_error?: boolean } +/** + * Token / cost accounting reported by the provider (session-manager `Usage`). + * + * Every field is optional and providers report disjoint subsets: codex never + * reports `cache_write`, anthropic never reports `reasoning`. Absent means + * "not reported", which is NOT the same as zero — see `lib/session-usage.ts`. + * + * The fields are also not portably additive. Anthropic's `input` EXCLUDES + * cached tokens (they arrive as separate `cache_read` / `cache_write`), while + * OpenAI's `input` INCLUDES `cache_read` and its `output` includes + * `reasoning`. So the only cross-provider total is `input + output`; cache and + * reasoning are descriptive only. (provider-anthropic/src/sse.rs:180, + * provider-openai/src/sse.rs:156; matches eval/src/report.rs:59-63.) + */ +export type Usage = { + input?: number + output?: number + cache_read?: number + cache_write?: number + reasoning?: number + cost_usd?: number +} + export type AgentMessage = | { role: 'user'; content: ContentBlock[]; timestamp: number } | { @@ -30,6 +53,9 @@ export type AgentMessage = stop_reason: 'end' | 'length' | 'function_call' | 'aborted' | 'error' native_stop_reason?: string error_message?: string + error_kind?: string + warnings?: string[] + usage?: Usage model: string provider: string timestamp: number diff --git a/console/web/src/lib/storage.ts b/console/web/src/lib/storage.ts index 729fe56b4..b44e8429e 100644 --- a/console/web/src/lib/storage.ts +++ b/console/web/src/lib/storage.ts @@ -37,6 +37,29 @@ export function saveDefaultPermissionMode(mode: PermissionMode): void { } } +const TURN_METRICS_KEY = 'iii-chat-turn-metrics' + +/** + * Whether the transcript shows a per-turn usage chip on each reply. Default + * on — a chip nobody can find is a feature nobody uses — with an opt-out in + * the session metrics dialog for readers who want a quiet transcript. + */ +export function loadShowTurnMetrics(): boolean { + try { + return localStorage.getItem(TURN_METRICS_KEY) !== 'off' + } catch { + return true + } +} + +export function saveShowTurnMetrics(show: boolean): void { + try { + localStorage.setItem(TURN_METRICS_KEY, show ? 'on' : 'off') + } catch { + /* best-effort */ + } +} + const DEFAULT_ALLOWLIST_KEY = 'iii-default-allowlist' /** User-level allowlist used to seed new conversations' backend state. */ diff --git a/console/web/src/types/chat.ts b/console/web/src/types/chat.ts index 7a87265f7..892bbe752 100644 --- a/console/web/src/types/chat.ts +++ b/console/web/src/types/chat.ts @@ -51,9 +51,25 @@ export interface Attachment { dataUrl?: string } +/** Re-exported so `types/chat.ts` stays the single UI-facing type surface. */ +export type { Usage as TokenUsage } from '@/lib/sessions/types' + interface BaseMessage { id: string createdAt: number + /** + * Provider usage for the transcript ENTRY this segment came from, and the + * turn that entry belongs to. + * + * Both live on `BaseMessage` rather than `AssistantMessage` because the + * entry mapper attaches them to the entry's FIRST segment whatever its role + * — a tool-only step emits no assistant segment at all yet still burns real + * tokens. The carrier segment is incidental: only `lib/session-usage.ts` + * reads these, and nothing should render them off the segment they happen + * to sit on. + */ + usage?: import('@/lib/sessions/types').Usage + turnId?: string } export interface UserMessage extends BaseMessage {