Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions apps/cli/src/commands/results/manifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ export interface ResultManifestRecord {
readonly aggregation?: Record<string, unknown>;
readonly execution_status?: string;
readonly error?: string;
/** Compact target-runtime error classification; full envelope lives at `target_execution_path`. */
readonly target_error_kind?: string;
/** Legacy pre-v5.4 fat index row field accepted when reading older bundles; superseded by `target_error_kind`. */
readonly target_execution?: { readonly error_kind?: string; readonly errorKind?: string };
readonly cost_usd?: number;
readonly duration_ms?: number;
readonly token_usage?: {
Expand Down
15 changes: 15 additions & 0 deletions apps/cli/src/commands/results/serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1203,6 +1203,19 @@ function buildRepeatTrialReadModels(
});
}

/**
* Compact target-runtime error classification for a row. New bundles carry
* `target_error_kind` directly; legacy pre-v5.4 fat rows nested it under
* `target_execution` instead.
*/
function recordTargetErrorKind(record: ResultManifestRecord): string | undefined {
return (
record.target_error_kind ??
record.target_execution?.error_kind ??
record.target_execution?.errorKind
);
}

function attachRunDetailReadModelFields<T extends Record<string, unknown>>(
results: readonly T[],
records: readonly ResultManifestRecord[],
Expand All @@ -1212,8 +1225,10 @@ function attachRunDetailReadModelFields<T extends Record<string, unknown>>(
const record = records[index];
if (!record) return result;
const samples = buildRepeatTrialReadModels(baseDir, record);
const targetErrorKind = recordTargetErrorKind(record);
return {
...result,
...(targetErrorKind && { target_error_kind: targetErrorKind }),
...(record.aggregation && { aggregation: record.aggregation }),
...(record.eval_path && { eval_path: record.eval_path }),
...(record.result_dir && { result_dir: record.result_dir }),
Expand Down
44 changes: 44 additions & 0 deletions apps/cli/test/commands/results/serve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2412,6 +2412,50 @@ describe('serve app', () => {
expect(data.results[0]?.samples?.[1]?.tool_calls).toEqual({ Read: 2 });
});

it('surfaces the compact target_error_kind field from slim index rows', async () => {
const runsDir = localResultsExperimentDir(tempDir);
const filename = '2026-03-25T10-10-00-000Z';
const runDir = path.join(runsDir, filename);
mkdirSync(runDir, { recursive: true });
writeFileSync(
path.join(runDir, 'index.jsonl'),
toJsonl({
...RESULT_EXECUTION_ERROR,
target_error_kind: 'timeout',
}),
);

const app = createApp([], tempDir, tempDir, undefined, { studioDir });
const res = await app.request(`/api/runs/${filename}`);
expect(res.status).toBe(200);
const data = (await res.json()) as {
results: Array<{ target_error_kind?: string }>;
};
expect(data.results[0]?.target_error_kind).toBe('timeout');
});

it('falls back to the legacy nested target_execution shape on older fat index rows', async () => {
const runsDir = localResultsExperimentDir(tempDir);
const filename = '2026-03-25T10-11-00-000Z';
const runDir = path.join(runsDir, filename);
mkdirSync(runDir, { recursive: true });
writeFileSync(
path.join(runDir, 'index.jsonl'),
toJsonl({
...RESULT_EXECUTION_ERROR,
target_execution: { error_kind: 'timeout' },
}),
);

const app = createApp([], tempDir, tempDir, undefined, { studioDir });
const res = await app.request(`/api/runs/${filename}`);
expect(res.status).toBe(200);
const data = (await res.json()) as {
results: Array<{ target_error_kind?: string }>;
};
expect(data.results[0]?.target_error_kind).toBe('timeout');
});

it('loads historical runs without test bundle metadata', async () => {
const runId = writeLocalRunArtifact(
tempDir,
Expand Down
50 changes: 50 additions & 0 deletions apps/dashboard/src/components/ResultTable.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,53 @@ describe('ResultTable repeat-run rendering', () => {
expect(html).toContain('target timed out');
});
});

describe('ResultTable target error kind', () => {
function renderErrorColumn(result: EvalResult): string {
const model = buildResultTableModel({
results: [result],
passThreshold: 0.8,
state: { visibleColumnIds: ['status', 'test', 'target', 'score', 'error'] },
});
return renderToStaticMarkup(
<ResultRowsTable
rows={model.filteredRows}
visibleColumns={model.visibleColumns}
passThreshold={0.8}
selectedRowKey={null}
selectedTrialPath={null}
repeatGroupsByRowKey={new Map()}
expandedRepeatRows={new Set()}
onToggleRepeatGroup={() => undefined}
onOpenDetail={() => undefined}
onOpenTrialDetail={() => undefined}
/>,
);
}

it('reads the compact target_error_kind field on new slim rows', () => {
const html = renderErrorColumn({
testId: 'billing-lookup',
target: 'codex',
score: 0,
executionStatus: 'execution_error',
error: 'target timed out',
target_error_kind: 'timeout',
});

expect(html).toContain('[target:timeout] target timed out');
});

it('falls back to the legacy nested target_execution shape on older bundles', () => {
const html = renderErrorColumn({
testId: 'billing-lookup',
target: 'codex',
score: 0,
executionStatus: 'execution_error',
error: 'target timed out',
target_execution: { error_kind: 'timeout' },
});

expect(html).toContain('[target:timeout] target timed out');
});
});
3 changes: 3 additions & 0 deletions apps/dashboard/src/components/ResultTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -643,10 +643,13 @@ function formatTargetError(
}

function targetErrorKind(value: {
target_error_kind?: string;
targetExecution?: { error_kind?: string; errorKind?: string };
target_execution?: { error_kind?: string; errorKind?: string };
}): string | undefined {
return (
value.target_error_kind ??
// Legacy pre-v5.4 fat rows/samples nested the error kind under `target_execution`.
value.target_execution?.error_kind ??
value.target_execution?.errorKind ??
value.targetExecution?.error_kind ??
Expand Down
8 changes: 8 additions & 0 deletions apps/dashboard/src/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,11 @@ export interface EvalCaseTrial {
scores?: ScoreEntry[];
assertions?: AssertionEntry[];
error?: string;
/** Compact target-runtime error classification; full envelope lives at `target_execution_path`. */
target_error_kind?: string;
/** Legacy pre-v5.4 fat row field accepted when reading older bundles; superseded by `target_error_kind`. */
targetExecution?: TargetExecutionEnvelope;
/** Legacy pre-v5.4 fat row field accepted when reading older bundles; superseded by `target_error_kind`. */
target_execution?: TargetExecutionEnvelope;
target_execution_path?: string;
stdout_path?: string;
Expand Down Expand Up @@ -280,7 +284,11 @@ export interface EvalResult {
score: number;
executionStatus?: string;
error?: string;
/** Compact target-runtime error classification; full envelope lives at `target_execution_path`. */
target_error_kind?: string;
/** Legacy pre-v5.4 fat row field accepted when reading older bundles; superseded by `target_error_kind`. */
targetExecution?: TargetExecutionEnvelope;
/** Legacy pre-v5.4 fat row field accepted when reading older bundles; superseded by `target_error_kind`. */
target_execution?: TargetExecutionEnvelope;
target_execution_path?: string;
stdout_path?: string;
Expand Down
Loading