Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions console/web/e2e/ui-metrics.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import { expect, expectPassingResult, openSession, test } from './harness-stack'

test.use({ scenario: 'console-streamed-text' })

/**
* The session metrics dialog opens from either of its two triggers and shows
* the provider-reported numbers the harness persisted during the turn.
*
* Selectors are attributes rather than text, matching `ui-send.spec.ts` — the
* copy in this panel is expected to change.
*/
test('opens session metrics from the header and from the ctx widget', async ({
page,
stack,
}) => {
const completed = stack.waitForTurnCompleted()
await openSession(page, stack)

const composer = page.getByLabel('message composer')
await composer.pressSequentially(stack.ready.message)
await page.getByRole('button', { name: 'send message' }).click()
await completed
await expect(
page.locator('[data-message-role="assistant"]', {
hasText: 'console fixture complete',
}),
).toHaveCount(1)

const dialog = page.locator('[data-testid="session-metrics"]')

// Trigger 1: the header button. Addressed by testid, not accessible name —
// `getByRole(name:)` matches substrings, so a sidebar row for a session
// whose title contains "metrics" would win instead.
await page.getByTestId('session-metrics-trigger').click()
await expect(dialog).toBeVisible()

// The three sections are structural, not decorative: which one a number
// sits in is the statement about how much to trust it.
await expect(dialog.getByText('exact', { exact: true })).toBeVisible()
await expect(dialog.getByText('counted', { exact: true })).toBeVisible()
await expect(dialog.getByText('estimated', { exact: true })).toBeVisible()

// The scripted fixture reports usage, so input tokens must not be a dash.
const inputRow = dialog.locator('div', { hasText: /^input tokens/ }).last()
await expect(inputRow).not.toContainText('—')

await dialog.getByRole('tab', { name: 'turns' }).click()
await expect(dialog.getByRole('columnheader', { name: 'turn' })).toBeVisible()

await dialog.getByRole('tab', { name: 'tree' }).click()
await expect(dialog).toBeVisible()

await page.keyboard.press('Escape')
await expect(dialog).toBeHidden()

// Trigger 2: clicking the ctx widget — the discoverability path.
await page.getByRole('button', { name: /^ctx/ }).click()
await expect(dialog).toBeVisible()

expectPassingResult(await stack.finish())
})

test('shows a per-turn usage chip in the transcript', async ({
page,
stack,
}) => {
const completed = stack.waitForTurnCompleted()
await openSession(page, stack)

const composer = page.getByLabel('message composer')
await composer.pressSequentially(stack.ready.message)
await page.getByRole('button', { name: 'send message' }).click()
await completed

const chip = page.locator('[data-turn-usage]').first()
await expect(chip).toBeVisible()

// Expanding is the point of the chip: per-step is where cache warmth shows.
await chip.getByRole('button').first().click()
await expect(chip).toContainText('step 0')

expectPassingResult(await stack.finish())
})
25 changes: 25 additions & 0 deletions console/web/src/components/chat/ChatView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import { useConversationsCtxOptional } from '@/lib/conversations-context'
import { expandFileMentions, parseFileMentions } from '@/lib/file-mentions'
import { formatStopReason } from '@/lib/format-stop-reason'
import { newMessageId } from '@/lib/session-id'
import { sessionUsage } from '@/lib/session-usage'
import { loadShowTurnMetrics } from '@/lib/storage'
import { cn } from '@/lib/utils'
import { fetchDefaultWorkingDir, validateWorkspaceDir } from '@/lib/working-dir'
import {
Expand Down Expand Up @@ -74,6 +76,7 @@ import { Composer, type ComposerSubmitPayload } from './Composer'
import { ContextUsage } from './ContextUsage'
import { ExportSessionButton } from './ExportSessionButton'
import { MessageList } from './MessageList'
import { SessionMetricsButton } from './SessionMetricsButton'
import { SessionTriggers } from './SessionTriggers'
import { WorktreeBadge } from './WorktreeBadge'

Expand Down Expand Up @@ -762,6 +765,16 @@ export function ChatView({

const filesystemGrants = useFilesystemGrants(sessionId)
const [filesystemDialogOpen, setFilesystemDialogOpen] = useState(false)
const [metricsOpen, setMetricsOpen] = useState(false)
const [showTurnMetrics, setShowTurnMetrics] = useState(loadShowTurnMetrics)
// Computed once here and shared: the dialog needs the whole rollup, and the
// ctx widget needs `lastCall` to cross-check its chars/4 estimate against a
// number the provider actually reported.
const usageRollup = useMemo(
() => sessionUsage(conversation.messages),
[conversation.messages],
)
const lastModelCall = usageRollup.lastCall
const handleManageFilesystemAccess = useCallback(() => {
setFilesystemDialogOpen(true)
}, [])
Expand Down Expand Up @@ -1630,6 +1643,17 @@ export function ChatView({
<ContextUsage
messages={conversation.messages}
contextWindow={contextWindow}
lastCall={lastModelCall}
onClick={() => setMetricsOpen(true)}
/>
<SessionMetricsButton
conversation={conversation}
usage={usageRollup}
contextWindow={contextWindow}
compact={isDock}
open={metricsOpen}
onOpenChange={setMetricsOpen}
onShowTurnMetricsChange={setShowTurnMetrics}
/>
<ExportSessionButton
conversation={conversation}
Expand Down Expand Up @@ -1687,6 +1711,7 @@ export function ChatView({
onResolveFilesystemAccess={handleFilesystemResolve}
onManageFilesystemAccess={handleManageFilesystemAccess}
workingDir={conversation.workingDir ?? null}
showTurnMetrics={showTurnMetrics}
/>
<LiveRegion announcement={announcer.announcement} />

Expand Down
85 changes: 70 additions & 15 deletions console/web/src/components/chat/ContextUsage.tsx
Original file line number Diff line number Diff line change
@@ -1,32 +1,78 @@
import { useMemo } from 'react'
import type { SessionUsage } from '@/lib/session-usage'
import {
estimateConversationTokens,
formatTokenCount,
} from '@/lib/token-estimate'
import { cn } from '@/lib/utils'
import type { Message } from '@/types/chat'

/**
* How close this conversation is to overflowing its context window.
*
* This is deliberately an ESTIMATE (chars ÷ 4) and stays one even now that
* real provider usage is available, because the two measure different things:
* this is current occupancy, while `Σ usage.input` is cumulative billing that
* counts the same prompt once per step. The obvious substitute — the last
* call's `input + cache_read + cache_write` — is right for anthropic (whose
* `input` excludes cached tokens) and double-counts for openai/codex (whose
* `input` includes them), so it cannot be computed portably in the browser.
*
* What we do instead: mark the number `~` so it never implies precision, and
* put the last provider-reported prompt size in the tooltip as a calibration
* reference. A genuinely exact gauge belongs in llm-router, where provider
* semantics are already known.
*/

interface ContextUsageProps {
messages: readonly Message[]
contextWindow?: number
/** Last provider-reported call, for the tooltip cross-check. */
lastCall?: SessionUsage['lastCall']
/** When set the widget becomes a button that opens the metrics dialog. */
onClick?: () => void
}

const WARN_THRESHOLD = 0.75
const DANGER_THRESHOLD = 0.9

export function ContextUsage({ messages, contextWindow }: ContextUsageProps) {
function calibration(lastCall: ContextUsageProps['lastCall']): string {
if (typeof lastCall?.usage.input !== 'number') return ''
return `\nlast provider prompt: ${lastCall.usage.input.toLocaleString()} tokens (measured)`
}

export function ContextUsage({
messages,
contextWindow,
lastCall,
onClick,
}: ContextUsageProps) {
const tokens = useMemo(() => estimateConversationTokens(messages), [messages])

const shell = cn(
'flex items-center gap-1.5 font-mono text-[11px] uppercase tracking-[0.06em] text-ink-faint',
onClick &&
'hover:text-ink transition-colors focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-accent',
)
const openHint = onClick ? '\nclick for session metrics' : ''

if (!contextWindow || contextWindow <= 0) {
return (
<div
className="flex items-center gap-1.5 font-mono text-[11px] uppercase tracking-[0.06em] text-ink-faint"
title={`${tokens.toLocaleString()} tokens (context window unknown)`}
>
const body = (
<>
<span>ctx</span>
<span className="tabular-nums text-ink">
{formatTokenCount(tokens)}
~{formatTokenCount(tokens)}
</span>
</>
)
const title = `~${tokens.toLocaleString()} tokens estimated (context window unknown)${calibration(lastCall)}${openHint}`
return onClick ? (
<button type="button" onClick={onClick} title={title} className={shell}>
{body}
</button>
) : (
<div className={shell} title={title}>
{body}
</div>
)
}
Expand Down Expand Up @@ -58,13 +104,12 @@ export function ContextUsage({ messages, contextWindow }: ContextUsageProps) {
? 'consider /compact'
: 'pre-flight compaction imminent'

return (
<div
className="flex items-center gap-1.5 font-mono text-[11px] uppercase tracking-[0.06em] text-ink-faint"
title={`${tokens.toLocaleString()} / ${contextWindow.toLocaleString()} tokens (${pct}%)${
hint ? ` — ${hint}` : ''
}`}
>
const title = `~${tokens.toLocaleString()} / ${contextWindow.toLocaleString()} tokens estimated (${pct}%)${
hint ? ` — ${hint}` : ''
}${calibration(lastCall)}${openHint}`

const body = (
<>
<span>ctx</span>
<div
className="relative w-14 h-[6px] bg-rule-2 border border-rule overflow-hidden"
Expand All @@ -78,8 +123,18 @@ export function ContextUsage({ messages, contextWindow }: ContextUsageProps) {
</div>
<span className={cn('tabular-nums', labelToneClass)}>{pct}%</span>
<span className="text-ink-ghost normal-case tracking-normal">
{formatTokenCount(tokens)}/{formatTokenCount(contextWindow)}
~{formatTokenCount(tokens)}/{formatTokenCount(contextWindow)}
</span>
</>
)

return onClick ? (
<button type="button" onClick={onClick} title={title} className={shell}>
{body}
</button>
) : (
<div className={shell} title={title}>
{body}
</div>
)
}
23 changes: 22 additions & 1 deletion console/web/src/components/chat/Message.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { FilesystemAccessAction } from '@/components/permissions/Filesystem
import { Caret } from '@/components/ui/Caret'
import { Prompt } from '@/components/ui/Prompt'
import { Markdown } from '@/lib/markdown'
import type { TurnUsage } from '@/lib/session-usage'
import { JsonHighlight } from '@/lib/syntax'
import { cn } from '@/lib/utils'
import type {
Expand All @@ -15,6 +16,7 @@ import { AttachmentChip } from './AttachmentChip'
import { CopyMessageButton } from './CopyMessageButton'
import { MemoryChip } from './MemoryChip'
import { ThoughtMessage } from './ThoughtMessage'
import { TurnUsageChip } from './TurnUsageChip'

interface MessageProps {
message: MessageType
Expand All @@ -38,6 +40,9 @@ interface MessageProps {
/** Copy payload for an assistant turn (prose + its function calls). Lazy so
the string is built on click, not on every streaming re-render. */
copyText?: string | (() => string)
/** Rollup for the turn this message closes; absent unless it is the anchor. */
turnUsage?: TurnUsage
compactTurnUsage?: boolean
}

export function Message({
Expand All @@ -48,6 +53,8 @@ export function Message({
onManageFilesystemAccess,
workingDir,
copyText,
turnUsage,
compactTurnUsage,
}: MessageProps) {
switch (message.role) {
case 'user':
Expand All @@ -61,7 +68,14 @@ export function Message({
<UserMessage message={message} />
)
case 'assistant':
return <AssistantMessage message={message} copyText={copyText} />
return (
<AssistantMessage
message={message}
copyText={copyText}
turnUsage={turnUsage}
compactTurnUsage={compactTurnUsage}
/>
)
case 'thought':
return <ThoughtMessage message={message} />
case 'function-trigger': {
Expand Down Expand Up @@ -307,9 +321,13 @@ function UserMessage({ message }: { message: UserMessageType }) {
function AssistantMessage({
message,
copyText,
turnUsage,
compactTurnUsage,
}: {
message: AssistantMessageType
copyText?: string | (() => string)
turnUsage?: TurnUsage
compactTurnUsage?: boolean
}) {
const showCaret = !!message.streaming
// A tool-only turn has no prose but still carries a copy payload (its
Expand All @@ -330,6 +348,9 @@ function AssistantMessage({
<span className="text-ink-ghost">· {message.mode}</span>
) : null}
{message.memory ? <MemoryChip memory={message.memory} /> : null}
{turnUsage ? (
<TurnUsageChip turn={turnUsage} compact={compactTurnUsage} />
) : null}
{copySource !== undefined && !message.streaming ? (
<CopyMessageButton
text={copySource}
Expand Down
13 changes: 13 additions & 0 deletions console/web/src/components/chat/MessageList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
assistantCopyText,
functionTriggersByAssistant,
} from '@/lib/function-trigger-copy'
import { turnUsageByAnchor } from '@/lib/session-usage'
import { cn } from '@/lib/utils'
import type {
FunctionTriggerMessage as FunctionTriggerMessageType,
Expand Down Expand Up @@ -42,6 +43,8 @@ interface MessageListProps {
) => Promise<void>
onManageFilesystemAccess?: () => void
workingDir?: string | null
/** Per-turn usage chips in the transcript (user preference, default on). */
showTurnMetrics?: boolean
}

type RenderItem =
Expand Down Expand Up @@ -99,6 +102,7 @@ export function MessageList({
onResolveFilesystemAccess,
onManageFilesystemAccess,
workingDir,
showTurnMetrics = true,
}: MessageListProps) {
const bottomRef = useRef<HTMLDivElement>(null)
const containerRef = useRef<HTMLDivElement>(null)
Expand All @@ -109,6 +113,13 @@ export function MessageList({
() => functionTriggersByAssistant(messages),
[messages],
)
// Turn rollup keyed by the message the chip hangs on. Grouping cannot live
// in the entry mapper — that sees one entry at a time, while a turn spans
// several — so it happens here, once per transcript change.
const turnUsage = useMemo(
() => (showTurnMetrics ? turnUsageByAnchor(messages) : null),
[messages, showTurnMetrics],
)

// Read optionally so isolated renders (Storybook) still work without the
// ConversationsProvider; the empty state falls back to `ready` there.
Expand Down Expand Up @@ -205,6 +216,8 @@ export function MessageList({
key={item.key}
message={m}
copyText={copyText}
turnUsage={turnUsage?.get(m.id)}
compactTurnUsage={density === 'dock'}
onResolveApproval={onResolveApproval}
onAlwaysAllow={onAlwaysAllow}
onResolveFilesystemAccess={onResolveFilesystemAccess}
Expand Down
Loading
Loading