Skip to content

fix(dashboard): clarify repeat attempt rows #1636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
christso merged 1 commit into from
Jul 4, 2026
Merged

fix(dashboard): clarify repeat attempt rows #1636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 35 additions & 4 deletions apps/dashboard/src/components/EvalDetail.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,19 @@ function caseTrialPath(trial: EvalCaseTrial, index = 0): string {
return trial.attempt_path ?? trial.run_path ?? `attempt-${trial.attempt ?? index + 1}`;
}

function trialNumber(trial: EvalCaseTrial, index = 0): number {
if (typeof trial.sample_index === 'number') return trial.sample_index;
if (typeof trial.attempt === 'number') return trial.attempt + 1;
return index + 1;
}

function trialDisplayLabel(trial: EvalCaseTrial, index = 0): string {
const label = `Attempt ${trialNumber(trial, index)}`;
return typeof trial.retry_index === 'number' && trial.retry_index > 0
? `${label} retry ${trial.retry_index}`
: label;
}

function caseTrialTokenTotal(trial: EvalCaseTrial): number | undefined {
if (trial.total_tokens != null) return trial.total_tokens;
const usage = trial.token_usage;
Expand Down Expand Up @@ -564,10 +577,16 @@ function TrialActionRow({
onSelectTrial?: (trial: EvalCaseTrial, initialTab?: Tab) => void;
}) {
const label = caseTrialPath(trial, index);
const displayLabel = trialDisplayLabel(trial, index);
return (
<div className="grid gap-2 rounded-md border border-gray-800 bg-gray-950/50 p-3 text-sm md:grid-cols-[minmax(8rem,1fr)_auto] md:items-center">
<div className="min-w-0">
<div className="font-medium text-gray-200">{label}</div>
<div className="flex min-w-0 flex-wrap items-center gap-2">
<span className="font-medium text-gray-200">{displayLabel}</span>
<span className="rounded-md border border-gray-800 px-1.5 py-0.5 text-[11px] font-medium text-gray-500">
{label}
</span>
</div>
<div className="mt-1 flex flex-wrap gap-x-3 gap-y-1 text-xs text-gray-500">
<span>{formatPercent(trial.score)} score</span>
<span>{trial.verdict ?? 'unknown'}</span>
Expand Down Expand Up @@ -608,14 +627,20 @@ function RepeatAggregateChecksTab({
return (
<div className="space-y-6">
<div className="rounded-lg border border-gray-800 bg-gray-900 p-4">
<p className="mb-3 text-sm text-gray-400">
Aggregate across {group.trialCount} attempts for this test case.
</p>
<div className="grid gap-3 md:grid-cols-4">
<RunMetricRow label="Attempt success" value={formatPercent(group.passRate)} />
<RunMetricRow label="Mean score" value={formatPercent(group.meanScore)} />
<RunMetricRow
label="Passed attempts"
label="Attempts passed"
value={`${group.passedTrials}/${group.trialCount}`}
/>
<RunMetricRow label="Assertions" value={formatPercent(group.assertionPassRate)} />
<RunMetricRow label="Assertion pass" value={formatPercent(group.assertionPassRate)} />
{group.executionErrorTrials > 0 ? (
<RunMetricRow label="Execution errors" value={String(group.executionErrorTrials)} />
) : null}
</div>
</div>

Expand Down Expand Up @@ -806,6 +831,7 @@ function RepeatAggregateTranscriptTab({
</h4>
{group.trials.map((trial, index) => {
const runLabel = caseTrialPath(trial, index);
const displayLabel = trialDisplayLabel(trial, index);
const transcriptPath = trial.transcript_path;
const transcriptHref = transcriptPath
? artifactFileContentUrl({
Expand All @@ -823,7 +849,12 @@ function RepeatAggregateTranscriptTab({
className="grid gap-2 rounded-md border border-gray-800 bg-gray-950/50 p-3 text-sm md:grid-cols-[minmax(8rem,1fr)_auto] md:items-center"
>
<div className="min-w-0">
<div className="font-medium text-gray-200">{runLabel}</div>
<div className="flex min-w-0 flex-wrap items-center gap-2">
<span className="font-medium text-gray-200">{displayLabel}</span>
<span className="rounded-md border border-gray-800 px-1.5 py-0.5 text-[11px] font-medium text-gray-500">
{runLabel}
</span>
</div>
<div className="mt-1 truncate font-mono text-xs text-gray-500" title={transcriptPath}>
{transcriptPath ?? 'No transcript artifact'}
</div>
Expand Down
100 changes: 100 additions & 0 deletions apps/dashboard/src/components/ResultTable.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import { describe, expect, it } from 'bun:test';
import { renderToStaticMarkup } from 'react-dom/server';

import { buildResultTableModel } from '~/lib/result-table';
import type { EvalResult } from '~/lib/types';

import { ResultRowsTable, ResultTable } from './ResultTable';

function repeatResult(overrides: Partial<EvalResult> = {}): EvalResult {
return {
testId: 'refund-policy-flaky',
target: 'codex',
score: 0.67,
executionStatus: 'quality_failure',
eval_path: 'evals/refund-policy.eval.yaml',
timestamp: '2026-07-04T10:00:00.000Z',
scores: [{ name: 'rubric', type: 'llm-rubric', score: 0.67, verdict: 'fail' }],
attempts: [
{
attempt: 0,
sample_index: 1,
attempt_path: 'sample-1',
score: 1,
verdict: 'pass',
duration_ms: 4400,
total_tokens: 830,
cost_usd: 0.0021,
},
{
attempt: 1,
sample_index: 2,
attempt_path: 'sample-2',
score: 0.51,
verdict: 'fail',
duration_ms: 5200,
total_tokens: 910,
cost_usd: 0.0024,
},
{
attempt: 2,
sample_index: 3,
attempt_path: 'sample-3',
score: 0,
verdict: 'fail',
execution_status: 'execution_error',
error: 'target timed out',
duration_ms: 15000,
},
],
...overrides,
};
}

describe('ResultTable repeat-run rendering', () => {
it('renders repeat runs as a collapsed aggregate case by default', () => {
const html = renderToStaticMarkup(
<ResultTable results={[repeatResult()]} runId="repeat-run-2026-07-04" passThreshold={0.8} />,
);

expect(html).toContain('Aggregate case');
expect(html).toContain('Flaky');
expect(html).toContain('1/3 attempts passed');
expect(html).toContain('Attempt success');
expect(html).toContain('Mean score');
expect(html).toContain('Expand attempts for refund-policy-flaky');
expect(html).not.toContain('Under refund-policy-flaky');
expect(html).not.toContain('sample-1');
});

it('renders expanded attempts as subordinate rows under the aggregate case', () => {
const model = buildResultTableModel({
results: [repeatResult()],
passThreshold: 0.8,
});
const row = model.filteredRows[0];
const html = renderToStaticMarkup(
<ResultRowsTable
rows={model.filteredRows}
visibleColumns={model.visibleColumns}
passThreshold={0.8}
selectedRowKey={null}
selectedTrialPath={null}
repeatGroupsByRowKey={new Map(model.repeatGroups.map((group) => [group.row.key, group]))}
expandedRepeatRows={new Set([row.key])}
onToggleRepeatGroup={() => undefined}
onOpenDetail={() => undefined}
onOpenTrialDetail={() => undefined}
/>,
);

expect(html).toContain('Aggregate case');
expect(html).toContain('Collapse attempts for refund-policy-flaky');
expect(html).toContain('Attempt 1');
expect(html).toContain('Attempt 2');
expect(html).toContain('Attempt 3');
expect(html).toContain('Under refund-policy-flaky');
expect(html).toContain('sample-1');
expect(html).toContain('target timed out');
});
});
Loading
Loading