diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index edf3704d5..b6193e611 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -105,6 +105,7 @@ export function buildIndexArtifactEntry( transcriptPath?: string; transcriptRawPath?: string; metricsPath?: string; + fileChangesPath?: string; rawProviderLogPath?: string; responsePath?: string; taskBundle?: MaterializedTaskBundlePaths; diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 60ceb3dc0..d5ee54f0b 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -4,6 +4,7 @@ import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { + CANONICAL_FILE_CHANGES_ARTIFACT_PATH, CANONICAL_METRICS_ARTIFACT_PATH, CANONICAL_TRANSCRIPT_ARTIFACT_PATH, type EvalTest, @@ -290,6 +291,10 @@ describe('buildGradingArtifact', () => { '@@ -1 +1 @@', '-old', '+new', + '--- a/deleted.ts', + '+++ /dev/null', + '@@ -1 +0,0 @@', + '-gone', ].join('\n'); const result = makeResult({ fileChanges: diff }); @@ -298,6 +303,9 @@ describe('buildGradingArtifact', () => { expect(grading.workspace_changes).toBeDefined(); expect(grading.workspace_changes?.files_created).toBe(1); expect(grading.workspace_changes?.files_modified).toBe(1); + expect(grading.workspace_changes?.files_deleted).toBe(1); + expect(grading.workspace_changes?.deleted_file_paths).toEqual(['deleted.ts']); + expect(grading.workspace_changes).not.toHaveProperty('diff_summary'); }); it('includes conversation when conversationId present', () => { @@ -766,6 +774,17 @@ describe('parseJsonlResults', () => { expect(() => parseJsonlResults(content)).toThrow(/Use "artifact_pointers"/); }); + it('rejects camelCase file changes path rows for the new wire field', () => { + const content = `${JSON.stringify({ + test_id: 'file-changes-row', + target: 'codex', + score: 1, + fileChangesPath: 'file-changes-row/run-1/outputs/file_changes.diff', + })}\n`; + + expect(() => parseJsonlResults(content)).toThrow(/Use "file_changes_path"/); + }); + it('does not treat parsed raw provider log pointers as fresh source artifacts', () => { const content = `${JSON.stringify({ test_id: 'raw-log-case', @@ -1398,6 +1417,10 @@ describe('writeArtifactsFromResults', () => { '+++ b/src/new.ts', '@@ -0,0 +1 @@', '+created', + '--- a/src/gone.ts', + '+++ /dev/null', + '@@ -1 +0,0 @@', + '-deleted', ].join('\n'); const results = [ makeResult({ @@ -1433,6 +1456,21 @@ describe('writeArtifactsFromResults', () => { const rowDir = expectRowDir(indexLine, 'summary-case'); expect(indexLine?.metrics_path).toBe(`${rowDir}/run-1/metrics.json`); + expect(indexLine?.file_changes_path).toBe( + `${rowDir}/run-1/${CANONICAL_FILE_CHANGES_ARTIFACT_PATH}`, + ); + await expect( + readFile( + runArtifactPath(testDir, indexLine, 'run-1', 'outputs', 'file_changes.diff'), + 'utf8', + ), + ).resolves.toBe(fileChanges); + + const runResult = JSON.parse( + await readFile(runArtifactPath(testDir, indexLine, 'run-1', 'result.json'), 'utf8'), + ); + expect(runResult.file_changes_path).toBe('./outputs/file_changes.diff'); + expect(runResult.output_paths.file_changes).toBe('./outputs/file_changes.diff'); const summary = MetricsArtifactWireSchema.parse( JSON.parse( @@ -1451,6 +1489,7 @@ describe('writeArtifactsFromResults', () => { transcript_path: 'transcript.jsonl', grading_path: 'grading.json', timing_path: 'timing.json', + file_changes_path: CANONICAL_FILE_CHANGES_ARTIFACT_PATH, }); expect(summary.source_artifacts).not.toHaveProperty('trace_path'); await expect( @@ -1504,6 +1543,7 @@ describe('writeArtifactsFromResults', () => { source: 'file_changes', }); expect(summary.metrics.files_created).toEqual(['src/new.ts']); + expect(summary.metrics.files_deleted).toEqual(['src/gone.ts']); expect(summary.metrics.web_fetches).toEqual([ { url: 'https://example.com/spec', diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index f4517adf6..f0c0abc02 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -197,8 +197,14 @@ describe('agentv grade prepared attempts', () => { }); expect(typeof row.metadata.prepared_attempt.baseline_commit).toBe('string'); + expect(row.file_changes_path).toMatch(/\/run-1\/outputs\/file_changes\.diff$/); + await expect(readFile(path.join(runDir, row.file_changes_path), 'utf8')).resolves.toContain( + '+manual edit', + ); + const grading = JSON.parse(await readFile(path.join(runDir, row.grading_path), 'utf8')); - expect(grading.workspace_changes.diff_summary).toContain('+manual edit'); + expect(grading.workspace_changes).not.toHaveProperty('diff_summary'); + expect(grading.workspace_changes.files_modified).toBeGreaterThanOrEqual(1); }, 20_000); it('fails clearly when the prepared manifest is missing', async () => { diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 150ea4814..fd1429997 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -141,6 +141,7 @@ my-results/ transcript.jsonl transcript-raw.jsonl outputs/answer.md + outputs/file_changes.diff # when workspace changes are captured test/ EVAL.yaml targets.yaml @@ -149,13 +150,13 @@ my-results/ ``` The `index.jsonl` row links to these generated paths with snake_case fields such -as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`, and -`graders_path`. Treat those paths as relative to the run directory. When you need -a portable artifact for audit, review, Dashboard inspection, or rerun workflows, -share the generated run directory and its `index.jsonl` manifest. Source-side -case directories are still useful for organizing bulky prompts, fixtures, or -tests while authoring an eval, but they are optional input organization rather -than a separate artifact schema. +as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`, +`file_changes_path`, and `graders_path`. Treat those paths as relative to the +run directory. When you need a portable artifact for audit, review, Dashboard +inspection, or rerun workflows, share the generated run directory and its +`index.jsonl` manifest. Source-side case directories are still useful for +organizing bulky prompts, fixtures, or tests while authoring an eval, but they +are optional input organization rather than a separate artifact schema. For the full root layout, per-attempt sidecars, pointer rules, and integration guidance, use the [Result Artifact Contract](/docs/reference/result-artifacts/). diff --git a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx index 12e530fae..1609931fb 100644 --- a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx @@ -45,6 +45,7 @@ The default local layout is: transcript-raw.jsonl outputs/ answer.md + file_changes.diff run-2/ result.json grading.json @@ -54,6 +55,7 @@ The default local layout is: transcript-raw.jsonl outputs/ answer.md + file_changes.diff ``` The `` and `` directories are storage allocation. They help @@ -83,6 +85,7 @@ query. | `result.json` | Compact per-attempt manifest for one attempt directory. | Loading one attempt without scanning the whole run index. | | `grading.json` | Grader outputs, assertions, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. | | `metrics.json` | Derived executor behavior summary, such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, metric-style graders, adapter projections, and lightweight analysis. | +| `outputs/file_changes.diff` | Full unified diff of workspace file changes when file changes are captured. | Human review and external artifact inspection; LLM and code graders still receive the same full diff through `file_changes`. | | `timing.json` | Duration, token usage, cost usage, and source labels such as `provider_reported`, `token_estimated`, `aggregate`, or `unavailable`. | Cost/latency reporting and provider-accounting audits. | | `transcript.jsonl` | AgentV-normalized transcript/timeline rows. | Portable human review, replay, transcript-aware graders, and tool-trajectory analysis. | | `transcript-raw.jsonl` | Native provider or harness evidence when available. | Parser debugging, forensic review, and preserving source bytes without making provider schemas public AgentV fields. | @@ -132,6 +135,7 @@ Example row: "transcript_raw_path": "refund-eligibility--4f9a7c2d1b6e/run-1/transcript-raw.jsonl", "output_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md", "answer_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md", + "file_changes_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/file_changes.diff", "test_dir": "refund-eligibility--4f9a7c2d1b6e/test" } ``` diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 6795a45b1..8b6352e8c 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -130,8 +130,10 @@ token/cost usage. Every case uses aggregate `summary.json`, then stores attempt details under `run-N/`. Each `run-N/` contains a compact per-attempt manifest `result.json`, `grading.json`, `metrics.json`, `timing.json`, `transcript.jsonl`, -`transcript-raw.jsonl`, and `outputs/answer.md`. The `result.json` file carries -`grading_path`, `metrics_path`, transcript, and output paths. +`transcript-raw.jsonl`, `outputs/answer.md`, and `outputs/file_changes.diff` +when workspace changes were captured. The `result.json` file carries +`grading_path`, `metrics_path`, transcript, output, and `file_changes_path` +paths. `transcript-raw.jsonl` preserves native provider or harness transcript bytes when they are available, while `transcript.jsonl` is the normalized @@ -141,10 +143,10 @@ systems can be linked through safe `external_trace` metadata when available. `summary.json` remains the run-level aggregate summary. `index.jsonl` is the canonical row index for the run: one row per result, attempt, or case, carrying lightweight explicit paths such as `transcript_path`, `transcript_raw_path`, -and `metrics_path` plus artifact pointers only when detached payload publishing -needs them. Dashboard search indexes, SQLite indexes, and other read models are -derived projections over these run artifacts, not replacements for -`index.jsonl`. +`file_changes_path`, and `metrics_path` plus artifact pointers only when +detached payload publishing needs them. Dashboard search indexes, SQLite +indexes, and other read models are derived projections over these run artifacts, +not replacements for `index.jsonl`. Duration, token, and cost usage remains in `timing.json`, including source labels such as `provider_reported`, `token_estimated`, `aggregate`, or `unavailable`. @@ -154,7 +156,7 @@ while adding AgentV/Vercel-style detail: | Field group | Purpose | |-------------|---------| -| `tool_calls`, `total_tool_calls`, `total_steps`, `errors_encountered`, `output_chars`, `transcript_chars`, `files_created` | Agent Skills-compatible executor metrics | +| `tool_calls`, `total_tool_calls`, `total_steps`, `errors_encountered`, `output_chars`, `transcript_chars`, `files_created`, `files_deleted` | Agent Skills-compatible executor metrics | | `tool_call_events`, `tool_call_counts`, `tool_category_counts`, `shell_commands`, `files_read`, `files_modified`, `web_fetches`, `errors`, `reasoning_blocks`, `thinking_blocks`, `total_turns` | AgentV/Vercel-style behavior summary when source data includes it | Vercel `@vercel/agent-eval` `results.o11y` maps into AgentV like this: diff --git a/docs/adr/0011-result-output-artifact-contract.md b/docs/adr/0011-result-output-artifact-contract.md index bf3e5224a..9231f9731 100644 --- a/docs/adr/0011-result-output-artifact-contract.md +++ b/docs/adr/0011-result-output-artifact-contract.md @@ -61,6 +61,8 @@ An AgentV result output is a run-centric bundle with this root contract: transcript.jsonl transcript-raw.jsonl outputs/ + answer.md # when target output exists + file_changes.diff # when workspace file changes exist ``` `summary.json` and `index.jsonl` are complementary: @@ -82,8 +84,8 @@ aggregate summaries. ordinary per-case sidecars through explicit fields such as `result_dir`, `summary_path`, `grading_path`, `metrics_path`, `timing_path`, `transcript_path`, `transcript_raw_path`, `answer_path`, `output_path`, -`test_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` when -those artifacts exist. +`file_changes_path`, `test_dir`, `eval_path`, `targets_path`, `files_path`, and +`graders_path` when those artifacts exist. `artifact_pointers` remain an offload indirection for large detached payload bytes. They are not the discovery path for ordinary sidecars that live in the diff --git a/packages/core/src/evaluation/metrics.ts b/packages/core/src/evaluation/metrics.ts index c66d1d17b..5703db874 100644 --- a/packages/core/src/evaluation/metrics.ts +++ b/packages/core/src/evaluation/metrics.ts @@ -152,6 +152,7 @@ export const MetricsWireSchema = z files_read: z.array(FileReferenceWireSchema), files_modified: z.array(FileReferenceWireSchema), files_created: z.array(z.string()), + files_deleted: z.array(z.string()).default([]), web_fetches: z.array(WebFetchWireSchema), errors: z.array(ExecutionErrorWireSchema), errors_encountered: z.number().int().nonnegative(), @@ -184,6 +185,7 @@ export const MetricsArtifactWireSchema = z transcript_path: z.string().optional(), grading_path: z.string().optional(), timing_path: z.string().optional(), + file_changes_path: z.string().optional(), }) .strict(), metrics: MetricsWireSchema, @@ -515,11 +517,14 @@ function parseModifiedPathsFromDiff(fileChanges: string | undefined): string[] { return []; } const paths = new Set(); - for (const line of fileChanges.split('\n')) { - if (!line.startsWith('+++ b/')) { + const lines = fileChanges.split('\n'); + for (let index = 0; index < lines.length - 1; index++) { + const oldLine = lines[index]; + const newLine = lines[index + 1]; + if (!oldLine.startsWith('--- a/') || !newLine?.startsWith('+++ b/')) { continue; } - const filePath = line.slice('+++ b/'.length).trim(); + const filePath = newLine.slice('+++ b/'.length).trim(); if (filePath && filePath !== '/dev/null') { paths.add(filePath); } @@ -593,6 +598,26 @@ function buildFilesCreated(result: EvaluationResult, calls: readonly ToolCallRef return [...paths]; } +function parseDeletedPathsFromDiff(fileChanges: string | undefined): string[] { + if (!fileChanges) { + return []; + } + const paths = new Set(); + const lines = fileChanges.split('\n'); + for (let index = 0; index < lines.length - 1; index++) { + const oldLine = lines[index]; + const newLine = lines[index + 1]; + if (!oldLine.startsWith('--- a/') || newLine !== '+++ /dev/null') { + continue; + } + const filePath = oldLine.slice('--- a/'.length).trim(); + if (filePath) { + paths.add(filePath); + } + } + return [...paths]; +} + function buildWebFetches(calls: readonly ToolCallRef[]) { return calls.flatMap((call) => { if (toolCategory(call.toolCall.tool) !== 'web_fetch') { @@ -831,6 +856,7 @@ function buildMetrics(result: EvaluationResult) { files_read: buildFileReads(calls), files_modified: buildFileModifications(result, calls), files_created: buildFilesCreated(result, calls), + files_deleted: parseDeletedPathsFromDiff(result.fileChanges), web_fetches: buildWebFetches(calls), errors, errors_encountered: errors.length, @@ -854,6 +880,7 @@ export function buildMetricsArtifact( transcriptPath?: string; gradingPath?: string; timingPath?: string; + fileChangesPath?: string; generatedAt?: string; } = {}, ): MetricsArtifactWire { @@ -876,6 +903,7 @@ export function buildMetricsArtifact( transcript_path: options.transcriptPath, grading_path: options.gradingPath, timing_path: options.timingPath, + file_changes_path: options.fileChangesPath, }), metrics: buildMetrics(result), }), diff --git a/packages/core/src/evaluation/result-artifact-contract.ts b/packages/core/src/evaluation/result-artifact-contract.ts index b15312c77..41ce99347 100644 --- a/packages/core/src/evaluation/result-artifact-contract.ts +++ b/packages/core/src/evaluation/result-artifact-contract.ts @@ -7,7 +7,8 @@ * AgentV-owned artifacts belong when projected to a results ref, sidecar ref, * or object store. Use pointers for large detached payload bytes, not as the * discovery path for ordinary sidecars such as `metrics.json`; normal - * sidecars should use explicit path fields such as `metrics_path`. + * sidecars should use explicit path fields such as `metrics_path` and + * `file_changes_path`. * * Git remote publishing treats the configured results branch as the * metadata/control plane and stores transcript payload bytes whose @@ -27,6 +28,7 @@ export const AGENTV_RESULTS_REFS = { export const CANONICAL_TRANSCRIPT_ARTIFACT_PATH = 'transcript.jsonl' as const; export const CANONICAL_METRICS_ARTIFACT_PATH = 'metrics.json' as const; +export const CANONICAL_FILE_CHANGES_ARTIFACT_PATH = 'outputs/file_changes.diff' as const; export const TRANSCRIPT_SCHEMA_VERSION = 'agentv.transcript.v1' as const; export const METRICS_SCHEMA_VERSION = 'agentv.metrics.v1' as const; diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts index f222c09e2..382c2372b 100644 --- a/packages/core/src/evaluation/result-row-schema.ts +++ b/packages/core/src/evaluation/result-row-schema.ts @@ -53,6 +53,7 @@ const RESULT_ROW_ALIASES = { const NEW_SNAKE_CASE_ONLY_FIELDS = { artifactPointers: 'artifact_pointers', + fileChangesPath: 'file_changes_path', } as const; const TRACE_SUMMARY_ALIASES = { diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index f0e8dfd46..97504bba7 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -37,6 +37,7 @@ import { import type { Message } from './providers/types.js'; import { extractLastAssistantContent } from './providers/types.js'; import { + CANONICAL_FILE_CHANGES_ARTIFACT_PATH, CANONICAL_METRICS_ARTIFACT_PATH, CANONICAL_TRANSCRIPT_ARTIFACT_PATH, type ResultArtifactPointersWire, @@ -309,7 +310,8 @@ export interface GradingArtifact { readonly workspace_changes?: { readonly files_modified: number; readonly files_created: number; - readonly diff_summary: string; + readonly files_deleted: number; + readonly deleted_file_paths?: readonly string[]; }; readonly conversation?: { readonly turns: number; @@ -462,6 +464,7 @@ export interface IndexArtifactEntry { readonly transcript_path?: string; readonly transcript_raw_path?: string; readonly metrics_path?: string; + readonly file_changes_path?: string; readonly artifact_pointers?: ResultArtifactPointersWire; readonly runtime_source?: RunRuntimeSourceMetadata; readonly raw_provider_log_path?: string; @@ -512,6 +515,7 @@ export interface VercelRunResultArtifact { readonly model: string; readonly grading_path: string; readonly metrics_path: string; + readonly file_changes_path?: string; readonly transcript_path?: string; readonly transcript_raw_path?: string; readonly o11y: { @@ -521,12 +525,14 @@ export interface VercelRunResultArtifact { readonly web_fetches: readonly unknown[]; readonly files_read: readonly string[]; readonly files_modified: readonly string[]; + readonly files_deleted: readonly string[]; readonly shell_commands: readonly unknown[]; readonly errors: readonly unknown[]; readonly thinking_blocks: number; }; readonly output_paths?: { readonly answer?: string; + readonly file_changes?: string; readonly scripts?: Record; }; readonly timing?: TimingArtifact; @@ -595,26 +601,29 @@ function parseWorkspaceChanges( let filesModified = 0; let filesCreated = 0; + const deletedFilePaths = new Set(); - for (const line of fileChanges.split('\n')) { - if (line.startsWith('--- /dev/null')) { + const lines = fileChanges.split('\n'); + for (let index = 0; index < lines.length - 1; index++) { + const previousLine = lines[index]; + const nextLine = lines[index + 1]; + if (previousLine === '--- /dev/null' && nextLine.startsWith('+++ b/')) { filesCreated += 1; - } else if (line.startsWith('--- a/')) { + } else if (previousLine.startsWith('--- a/') && nextLine.startsWith('+++ b/')) { filesModified += 1; + } else if (previousLine.startsWith('--- a/') && nextLine === '+++ /dev/null') { + const filePath = previousLine.slice('--- a/'.length).trim(); + if (filePath) { + deletedFilePaths.add(filePath); + } } } - const lines = fileChanges.split('\n'); - const summaryLines = lines.slice(0, 20); - const diffSummary = - lines.length > 20 - ? `${summaryLines.join('\n')}\n... (${lines.length - 20} more lines)` - : fileChanges; - return { files_modified: filesModified, files_created: filesCreated, - diff_summary: diffSummary, + files_deleted: deletedFilePaths.size, + deleted_file_paths: deletedFilePaths.size > 0 ? [...deletedFilePaths] : undefined, }; } @@ -905,8 +914,12 @@ function buildVercelRunResultArtifact(params: { }; readonly hasTranscript: boolean; readonly hasOutput: boolean; + readonly hasFileChanges: boolean; }): VercelRunResultArtifact { const metrics = params.metricsArtifact.metrics; + const fileChangesPath = params.hasFileChanges + ? `./${CANONICAL_FILE_CHANGES_ARTIFACT_PATH}` + : undefined; return dropUndefined({ status: toVercelRunStatus(params.trial, params.result), duration_ms: resultDurationMs(params.result), @@ -914,6 +927,7 @@ function buildVercelRunResultArtifact(params: { model: params.result.target ?? 'unknown', grading_path: './grading.json', metrics_path: `./${CANONICAL_METRICS_ARTIFACT_PATH}`, + file_changes_path: fileChangesPath, transcript_path: params.hasTranscript ? `./${CANONICAL_TRANSCRIPT_ARTIFACT_PATH}` : undefined, transcript_raw_path: params.hasTranscript ? './transcript-raw.jsonl' : undefined, o11y: { @@ -923,11 +937,18 @@ function buildVercelRunResultArtifact(params: { web_fetches: metrics.web_fetches, files_read: toFilePathList(metrics.files_read), files_modified: toFilePathList(metrics.files_modified), + files_deleted: Array.isArray(metrics.files_deleted) ? metrics.files_deleted : [], shell_commands: metrics.shell_commands, errors: metrics.errors, thinking_blocks: metrics.thinking_blocks, }, - output_paths: params.hasOutput ? { answer: './outputs/answer.md' } : undefined, + output_paths: + params.hasOutput || params.hasFileChanges + ? dropUndefined({ + answer: params.hasOutput ? './outputs/answer.md' : undefined, + file_changes: fileChangesPath, + }) + : undefined, timing: params.metricsArtifact.timing, }) as unknown as VercelRunResultArtifact; } @@ -980,6 +1001,9 @@ async function writeTrialRunArtifacts(params: { const outputsDir = path.join(runDir, 'outputs'); const answerOutputPath = result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined; + const fileChangesPath = result.fileChanges + ? path.join(runDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) + : undefined; const attemptRunId = params.runId ? `${params.runId}:${runDirName}` : `${result.testId}:${result.target}:${runDirName}`; @@ -1005,6 +1029,9 @@ async function writeTrialRunArtifacts(params: { if (answerOutputPath) { await writeFile(answerOutputPath, result.output, 'utf8'); } + if (fileChangesPath && result.fileChanges) { + await writeFile(fileChangesPath, result.fileChanges, 'utf8'); + } if (transcriptPath && transcriptRawPath) { await writeNormalizedTranscriptJsonl(transcriptPath, envelope); await writeRawTranscriptJsonl(transcriptRawPath, result, envelope); @@ -1016,6 +1043,7 @@ async function writeTrialRunArtifacts(params: { transcriptArtifactPath: transcriptPath ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined, gradingArtifactPath: 'grading.json', timingArtifactPath: 'timing.json', + fileChangesArtifactPath: fileChangesPath ? CANONICAL_FILE_CHANGES_ARTIFACT_PATH : undefined, timing, }); @@ -1028,6 +1056,7 @@ async function writeTrialRunArtifacts(params: { metricsArtifact, hasTranscript, hasOutput: result.output.length > 0, + hasFileChanges: result.fileChanges !== undefined && result.fileChanges.length > 0, }), null, 2, @@ -1624,6 +1653,9 @@ function buildTraceEnvelopeSidecar(params: TraceEnvelopeSidecarParams): TraceEnv answer_path: params.result.output.length > 0 ? 'outputs/answer.md' : undefined, transcript_path: hasTranscript ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined, metrics_path: CANONICAL_METRICS_ARTIFACT_PATH, + file_changes_path: params.result.fileChanges + ? CANONICAL_FILE_CHANGES_ARTIFACT_PATH + : undefined, }, duplicatePolicy: params.duplicatePolicy, }); @@ -1642,6 +1674,7 @@ export function buildIndexArtifactEntry( transcriptPath?: string; transcriptRawPath?: string; metricsPath?: string; + fileChangesPath?: string; artifactPointers?: ResultArtifactPointersWire; rawProviderLogPath?: string; extraIndexFields?: AdditionalResultIndexFields; @@ -1699,6 +1732,9 @@ export function buildIndexArtifactEntry( metrics_path: options.metricsPath ? toRelativeArtifactPath(options.outputDir, options.metricsPath) : undefined, + file_changes_path: options.fileChangesPath + ? toRelativeArtifactPath(options.outputDir, options.fileChangesPath) + : undefined, raw_provider_log_path: options.rawProviderLogPath ? toRelativeArtifactPath(options.outputDir, options.rawProviderLogPath) : undefined, @@ -1731,6 +1767,7 @@ export function buildResultIndexArtifact( options?.projectionIdentity, ); const hasAnswer = result.output.length > 0; + const hasFileChanges = result.fileChanges !== undefined && result.fileChanges.length > 0; const hasTranscript = resultHasExecutionTraceTranscript(result); const isSingleRun = !hasPersistedTrialRuns(result); const singleRunDir = path.posix.join(artifactSubdir, trialRunDirName(0)); @@ -1768,6 +1805,10 @@ export function buildResultIndexArtifact( isSingleRun && hasAnswer ? path.posix.join(singleRunDir, 'outputs', 'answer.md') : undefined, answer_path: isSingleRun && hasAnswer ? path.posix.join(singleRunDir, 'outputs', 'answer.md') : undefined, + file_changes_path: + isSingleRun && hasFileChanges + ? path.posix.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) + : undefined, transcript_path: isSingleRun && hasTranscript ? path.posix.join(singleRunDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH) @@ -1843,6 +1884,7 @@ function buildMetricsArtifactPayload(params: { readonly transcriptArtifactPath?: string; readonly gradingArtifactPath?: string; readonly timingArtifactPath?: string | null; + readonly fileChangesArtifactPath?: string; readonly timing?: TimingArtifact; }): ReturnType & { readonly timing?: TimingArtifact } { const artifact = buildMetricsArtifact(params.result, params.envelope, { @@ -1852,6 +1894,7 @@ function buildMetricsArtifactPayload(params: { gradingPath: params.gradingArtifactPath ?? 'grading.json', timingPath: params.timingArtifactPath === null ? undefined : (params.timingArtifactPath ?? 'timing.json'), + fileChangesPath: params.fileChangesArtifactPath, }); return params.timing ? { ...artifact, timing: params.timing } : artifact; } @@ -1864,6 +1907,7 @@ async function writeMetricsArtifact(params: { readonly transcriptArtifactPath?: string; readonly gradingArtifactPath?: string; readonly timingArtifactPath?: string | null; + readonly fileChangesArtifactPath?: string; readonly timing?: TimingArtifact; }): Promise & { readonly timing?: TimingArtifact }> { const artifactWithTiming = buildMetricsArtifactPayload(params); @@ -2250,6 +2294,10 @@ export async function writePerTestArtifacts( const singleMetricsPath = isSingleRun ? path.join(singleRunDir, CANONICAL_METRICS_ARTIFACT_PATH) : undefined; + const singleFileChangesPath = + isSingleRun && result.fileChanges + ? path.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) + : undefined; const extraIndexFields = await collectAdditionalIndexFields( result, @@ -2271,6 +2319,7 @@ export async function writePerTestArtifacts( answerPath: singleAnswerPath, transcriptPath: singleTranscriptPath, transcriptRawPath: singleTranscriptRawPath, + fileChangesPath: singleFileChangesPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity, @@ -2357,6 +2406,10 @@ export async function writeArtifactsFromResults( const singleMetricsPath = isSingleRun ? path.join(singleRunDir, CANONICAL_METRICS_ARTIFACT_PATH) : undefined; + const singleFileChangesPath = + isSingleRun && result.fileChanges + ? path.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) + : undefined; return { result, testDir, @@ -2369,6 +2422,7 @@ export async function writeArtifactsFromResults( singleGradingPath, singleTimingPath, singleMetricsPath, + singleFileChangesPath, identityId, }; }); @@ -2443,6 +2497,7 @@ export async function writeArtifactsFromResults( answerPath: plan.singleAnswerPath, transcriptPath: plan.singleTranscriptPath, transcriptRawPath: plan.singleTranscriptRawPath, + fileChangesPath: plan.singleFileChangesPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity: plan.projectionIdentity, diff --git a/packages/core/test/evaluation/workspace/file-changes.test.ts b/packages/core/test/evaluation/workspace/file-changes.test.ts index ab2dca629..9d56140d2 100644 --- a/packages/core/test/evaluation/workspace/file-changes.test.ts +++ b/packages/core/test/evaluation/workspace/file-changes.test.ts @@ -134,6 +134,7 @@ describe('workspace file-changes', () => { }); it('captureFileChanges detects added/modified/deleted files', async () => { + await writeFile(path.join(workspacePath, 'delete-me.txt'), 'remove this\n', 'utf8'); const baselineCommit = await initializeBaseline(workspacePath); // Add a new file @@ -142,6 +143,9 @@ describe('workspace file-changes', () => { // Modify existing file await writeFile(path.join(workspacePath, 'hello.txt'), 'modified content\n', 'utf8'); + // Delete an existing file + await rm(path.join(workspacePath, 'delete-me.txt')); + const diff = await captureFileChanges(workspacePath, baselineCommit); // Should contain diff for modified file @@ -151,6 +155,11 @@ describe('workspace file-changes', () => { // Should contain diff for new file expect(diff).toContain('new-file.txt'); expect(diff).toContain('new content'); + + // Should contain diff for deleted file + expect(diff).toContain('delete-me.txt'); + expect(diff).toContain('deleted file mode'); + expect(diff).toContain('-remove this'); }); it('returns empty string when no changes', async () => {