From 0c25bee9178ba0869369f02cb90b21693fe81c87 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 6 Jul 2026 19:15:49 +1000 Subject: [PATCH] feat(results): slim index.jsonl target_execution/transcript_summary to sidecars index.jsonl rows previously inlined the full target_execution envelope (command argv, complete stdout/stderr logs, transcript) and the full transcript_summary (unbounded files/shell/web-fetch lists) on every row and every repeat-sample rollup entry, duplicating what already lives in target-execution.json and each sample's result.json sidecar. Drop both inline objects; keep target_execution_path/transcript_path (already present) as the sidecar pointers, and add a compact target_error_kind scalar (row-level and per-sample) so the existing target-error-kind table affordance doesn't need a sidecar read to know a run failed. av-cpl5.3 --- .../commands/eval/artifact-writer.test.ts | 31 ++++--------------- apps/cli/test/eval.integration.test.ts | 10 +++++- packages/core/src/evaluation/run-artifacts.ts | 23 +++++--------- .../target-execution-artifacts.test.ts | 13 +++----- 4 files changed, 27 insertions(+), 50 deletions(-) diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 87757c590..c29959470 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -959,28 +959,6 @@ describe('buildIndexArtifactEntry', () => { error: 'model drift', cost_usd: 0.25, execution_status: 'quality_failure', - transcript_summary: { - total_turns: 1, - tool_calls: { - file_read: 0, - file_write: 0, - file_edit: 0, - shell: 0, - web_fetch: 0, - web_search: 0, - glob: 0, - grep: 0, - list_dir: 0, - agent_task: 0, - unknown: 0, - }, - files_read: [], - files_modified: [], - shell_commands: [], - web_fetches: [], - errors: [{ message: 'model drift' }], - thinking_blocks: 0, - }, }, ], }); @@ -1643,7 +1621,8 @@ describe('writeArtifactsFromResults', () => { }); expect(runOneResult).not.toHaveProperty('timing'); expect(runOneResult).not.toHaveProperty('verdict'); - expect(indexEntry?.samples?.[0]?.transcript_summary).toEqual(runOneResult.transcript_summary); + expect(runOneResult.transcript_summary).toBeDefined(); + expect(indexEntry?.samples?.[0]).not.toHaveProperty('transcript_summary'); const runTwoAnswer = await readFile( path.join(paths.testArtifactDir, repeatRowDir, 'sample-2', 'outputs', 'answer.md'), @@ -1667,7 +1646,8 @@ describe('writeArtifactsFromResults', () => { }); expect(runTwoResult).not.toHaveProperty('timing'); expect(runTwoResult).not.toHaveProperty('verdict'); - expect(indexEntry?.samples?.[1]?.transcript_summary).toEqual(runTwoResult.transcript_summary); + expect(runTwoResult.transcript_summary).toBeDefined(); + expect(indexEntry?.samples?.[1]).not.toHaveProperty('transcript_summary'); }); it('keys prompt-expanded resume checks by authored test id plus prompt id', () => { @@ -1898,7 +1878,8 @@ describe('writeArtifactsFromResults', () => { expect(indexLine).not.toHaveProperty('trace_path'); expect(indexLine?.transcript_path).toBe(`${rowDir}/sample-1/transcript.json`); expect(indexLine?.transcript_raw_path).toBe(`${rowDir}/sample-1/transcript-raw.jsonl`); - expect(indexLine?.transcript_summary).toEqual(transcript.transcript_summary); + expect(transcript.transcript_summary).toBeDefined(); + expect(indexLine).not.toHaveProperty('transcript_summary'); expect(indexLine?.metrics_path).toBe(`${rowDir}/sample-1/metrics.json`); expect(indexLine.metrics_path.endsWith(CANONICAL_METRICS_ARTIFACT_PATH)).toBe(true); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index b5a920d75..cc88cd5df 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -456,7 +456,15 @@ describe('agentv eval CLI', () => { for (const row of canonicalResults) { expect(row.transcript_path).toMatch(/sample-1\/transcript\.json$/); await expectFileExists(path.join(outputDir, row.transcript_path as string)); - expect(row.transcript_summary).toBeDefined(); + expect(row).not.toHaveProperty('transcript_summary'); + const resultJsonPath = (row.transcript_path as string).replace( + /sample-1\/transcript\.json$/, + 'sample-1/result.json', + ); + const sampleResult = JSON.parse( + await readFile(path.join(outputDir, resultJsonPath), 'utf8'), + ); + expect(sampleResult.transcript_summary).toBeDefined(); expect(row.transcript_raw_path).toMatch(/sample-1\/transcript-raw\.jsonl$/); await expectFileExists(path.join(outputDir, row.transcript_raw_path as string)); } diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index 275ee1512..4bc7ff6b1 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -426,7 +426,8 @@ export type TrialResultArtifact = { readonly execution_status?: string; readonly failure_stage?: string; readonly failure_reason_code?: string; - readonly transcript_summary?: TranscriptSummaryWire; + /** Compact target-runtime error classification; full detail lives in `sample_path`'s `result.json`. */ + readonly target_error_kind?: string; }; export type TrialAggregationArtifact = @@ -609,7 +610,8 @@ export interface IndexArtifactEntry { readonly error?: string; readonly failure_stage?: string; readonly failure_reason_code?: string; - readonly target_execution?: TargetExecutionWire; + /** Compact target-runtime error classification; full envelope lives at `target_execution_path`. */ + readonly target_error_kind?: string; readonly target_execution_path?: string; readonly stdout_path?: string; readonly stderr_path?: string; @@ -621,7 +623,6 @@ export interface IndexArtifactEntry { readonly answer_path?: string; readonly transcript_path?: string; readonly transcript_raw_path?: string; - readonly transcript_summary?: TranscriptSummaryWire; readonly metrics_path?: string; readonly file_changes_path?: string; readonly environment_path?: string; @@ -1032,13 +1033,6 @@ function hasPersistedTrialRuns(result: EvaluationResult): boolean { return (result.trials ?? []).some((trial) => trial.result !== undefined); } -function toTrialTranscriptSummary(trial: TrialResult): TranscriptSummaryWire | undefined { - const result = trial.result; - return result && resultHasExecutionTraceTranscript(result) - ? buildResultTranscriptSummary(result) - : undefined; -} - function toTrialArtifacts( trials: readonly TrialResult[] | undefined, ): readonly TrialResultArtifact[] | undefined { @@ -1057,7 +1051,7 @@ function toTrialArtifacts( execution_status: trial.executionStatus, failure_stage: trial.failureStage, failure_reason_code: trial.failureReasonCode, - transcript_summary: toTrialTranscriptSummary(trial), + target_error_kind: trial.result?.targetExecution?.errorKind, })); } @@ -2481,7 +2475,7 @@ export function buildIndexArtifactEntry( error: result.error, failure_stage: result.failureStage, failure_reason_code: result.failureReasonCode, - target_execution: toTargetExecutionWire(targetExecution), + target_error_kind: targetExecution?.errorKind, target_execution_path: options.targetExecutionPath ? toRelativeArtifactPath(options.outputDir, options.targetExecutionPath) : undefined, @@ -2513,7 +2507,6 @@ export function buildIndexArtifactEntry( transcript_raw_path: options.transcriptRawPath ? toRelativeArtifactPath(options.outputDir, options.transcriptRawPath) : undefined, - transcript_summary: options.transcriptPath ? buildResultTranscriptSummary(result) : undefined, metrics_path: options.metricsPath ? toRelativeArtifactPath(options.outputDir, options.metricsPath) : undefined, @@ -2622,7 +2615,7 @@ export function buildResultIndexArtifact( error: result.error, failure_stage: result.failureStage, failure_reason_code: result.failureReasonCode, - target_execution: toTargetExecutionWire(targetExecution), + target_error_kind: targetExecution?.errorKind, target_execution_path: result.targetExecution ? path.posix.join(singleRunDir, TARGET_EXECUTION_ARTIFACT_PATH) : undefined, @@ -2655,8 +2648,6 @@ export function buildResultIndexArtifact( isSingleRun && hasTranscript ? path.posix.join(singleRunDir, 'transcript-raw.jsonl') : undefined, - transcript_summary: - isSingleRun && hasTranscript ? buildResultTranscriptSummary(result) : undefined, artifact_pointers: options?.artifactPointers, sample_index: result.sampleIndex, retry_index: result.retryIndex, diff --git a/packages/core/test/evaluation/target-execution-artifacts.test.ts b/packages/core/test/evaluation/target-execution-artifacts.test.ts index 863223f0c..48a930bc4 100644 --- a/packages/core/test/evaluation/target-execution-artifacts.test.ts +++ b/packages/core/test/evaluation/target-execution-artifacts.test.ts @@ -83,14 +83,8 @@ describe('target execution artifacts', () => { expect(row.target_execution_path).toBeTruthy(); expect(row.stdout_path).toBeTruthy(); expect(row.stderr_path).toBeTruthy(); - - const targetExecution = row.target_execution as Record; - expect(targetExecution.error_kind).toBe('signal_crash'); - expect(targetExecution.provider_kind).toBe('cli'); - expect((targetExecution.artifacts as Record).stdout_path).toBe( - row.stdout_path, - ); - expect(targetExecution.artifacts).toHaveProperty('transcript_path'); + expect(row.target_error_kind).toBe('signal_crash'); + expect(row).not.toHaveProperty('target_execution'); const stdoutPath = path.join(outputDir, row.stdout_path as string); const stderrPath = path.join(outputDir, row.stderr_path as string); @@ -100,7 +94,10 @@ describe('target execution artifacts', () => { await expect(readFile(stderrPath, 'utf8')).resolves.toContain('segmentation fault'); const envelope = JSON.parse(await readFile(envelopePath, 'utf8')) as Record; expect(envelope.error_kind).toBe('signal_crash'); + expect(envelope.provider_kind).toBe('cli'); expect(envelope.artifacts).toHaveProperty('stderr_path'); + expect((envelope.artifacts as Record).stdout_path).toBe('stdout.txt'); + expect(envelope.artifacts).toHaveProperty('transcript_path'); } finally { await rm(outputDir, { recursive: true, force: true }); }