diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index afb4246b5..882d3126a 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -10,8 +10,10 @@ import { type GradingArtifact, type IndexArtifactEntry, RESULT_INDEX_FILENAME, + RUN_CONFIG_FILENAME, RUN_SUMMARY_FILENAME, type ResultIndexArtifact, + type RunConfigArtifact, type RunRuntimeSourceMetadata, type RunSummaryArtifact, type TimingArtifact, @@ -27,6 +29,7 @@ import { buildTimingArtifact, deduplicateByTestIdTarget, parseJsonlResults, + readRunConfigArtifact, writeArtifacts, writeArtifactsFromResults as writeCoreArtifactsFromResults, writePerTestArtifacts as writeCorePerTestArtifacts, @@ -52,7 +55,9 @@ export { deduplicateByTestIdTarget, parseJsonlResults, RESULT_INDEX_FILENAME, + RUN_CONFIG_FILENAME, RUN_SUMMARY_FILENAME, + readRunConfigArtifact, writeArtifacts, writeInitialRunSummaryArtifact, }; @@ -61,6 +66,7 @@ export type { GradingArtifact, IndexArtifactEntry, ResultIndexArtifact, + RunConfigArtifact, RunSummaryArtifact, TimingArtifact, }; diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index b739185ab..6b6dba4ee 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -12,6 +12,7 @@ import { type GraderResult, METRICS_SCHEMA_VERSION, MetricsArtifactWireSchema, + aggregateRunDir, buildEvalTestTargetKey, buildEvaluationResultTargetKey, buildResultIndexArtifact, @@ -25,6 +26,7 @@ import { type GradingArtifact, type IndexArtifactEntry, RESULT_INDEX_FILENAME, + RUN_CONFIG_FILENAME, type RunSummaryArtifact, type TimingArtifact, buildAggregateGradingArtifact, @@ -1172,7 +1174,7 @@ describe('writeArtifactsFromResults', () => { expect(indexLines[0]?.metrics_path).toBe(`${alphaRowDir}/sample-1/metrics.json`); }); - it('writes optional runtime source metadata to summary and index rows', async () => { + it('writes optional runtime source metadata to summary only', async () => { const runtimeSource = { schema_version: 'agentv.runtime_source.v1' as const, kind: 'direct_suite' as const, @@ -1194,7 +1196,53 @@ describe('writeArtifactsFromResults', () => { .map(JSON.parse); expect(summary.metadata.runtime_source).toEqual(runtimeSource); - expect(indexLine.runtime_source).toEqual(runtimeSource); + expect(indexLine.runtime_source).toBeUndefined(); + }); + + it('moves experiment config metadata to an internal run config sidecar', async () => { + const experimentMetadata = { + name: 'native-exp', + target: 'codex-target', + threshold: 0.8, + fingerprint: 'a'.repeat(64), + }; + const paths = await writeArtifactsFromResults([makeResult({ testId: 'alpha' })], testDir, { + evalFile: 'evals/native-exp.eval.yaml', + experiment: 'native-exp', + experimentMetadata, + }); + + const summary: RunSummaryArtifact = JSON.parse(await readFile(paths.summaryPath, 'utf8')); + expect(summary.metadata).not.toHaveProperty('experiment_config'); + expect(summary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`); + + const runConfig = JSON.parse( + await readFile(path.join(paths.testArtifactDir, '.internal', RUN_CONFIG_FILENAME), 'utf8'), + ); + expect(runConfig).toEqual({ + schema_version: 'agentv.run_config.v1', + experiment_config: experimentMetadata, + }); + + await aggregateRunDir(paths.testArtifactDir); + const rewrittenSummary: RunSummaryArtifact = JSON.parse( + await readFile(paths.summaryPath, 'utf8'), + ); + expect(rewrittenSummary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`); + }); + + it('omits duplicated root instances from run summary', () => { + const summary = buildRunSummaryArtifact( + [ + makeResult({ testId: 'alpha', target: 'target-a', score: 1 }), + makeResult({ testId: 'beta', target: 'target-a', score: 0 }), + ], + 'evals/smoke.eval.yaml', + ); + + expect(summary.counts.total_instances).toBe(2); + expect(summary.cases).toHaveLength(2); + expect(summary).not.toHaveProperty('instances'); }); it('emits the resolved tags map to summary metadata and every index row', async () => { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 04ca8b9be..997cc125a 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -697,7 +697,15 @@ describe('agentv eval CLI', () => { await readFile(path.join(runDirFromIndexPath(outputPath), 'summary.json'), 'utf8'), ) as { metadata?: Record }; expect(benchmark.metadata?.experiment).toBe('native-exp'); - expect(benchmark.metadata?.experiment_config).toMatchObject({ + expect(benchmark.metadata).not.toHaveProperty('experiment_config'); + expect(benchmark.metadata?.run_config_path).toBe('.internal/run-config.json'); + const runConfig = JSON.parse( + await readFile( + path.join(runDirFromIndexPath(outputPath), '.internal', 'run-config.json'), + 'utf8', + ), + ) as { experiment_config?: Record }; + expect(runConfig.experiment_config).toMatchObject({ target: 'codex-target', repeat: { count: 2, @@ -708,9 +716,9 @@ describe('agentv eval CLI', () => { budget_usd: 3, timeout_seconds: 12, }); - expect( - (benchmark.metadata?.experiment_config as Record).fingerprint, - ).toMatch(/^[a-f0-9]{64}$/); + expect((runConfig.experiment_config as Record).fingerprint).toMatch( + /^[a-f0-9]{64}$/, + ); expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', kind: 'wrapper_eval', diff --git a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx index 20ef96e9f..24c05306e 100644 --- a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx @@ -94,7 +94,8 @@ reserved for rebuildable local state and are skipped by run discovery. | File or field | Owns | Use it for | | --- | --- | --- | -| `summary.json` | Aggregate run metadata and rollups: run id, experiment metadata, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | +| `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | +| `.internal/run-config.json` | Internal reproducibility metadata that is useful to rerun or audit the invocation but too noisy for the public aggregate summary, such as resolved experiment config and fingerprints. | Debugging run setup, reproducing wrapper-eval choices, and reviewing config provenance without repeating it in every row. | | `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. | | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. | | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. | @@ -107,11 +108,12 @@ reserved for rebuildable local state and are skipped by run discovery. | `test/` | Generated test bundle for the exact eval slice and target settings that produced a row. | Audit, external review, and rerun workflows that should not depend on a mutable source checkout. | | `artifact_pointers` | Offload indirection for large detached payload bytes. | Finding payloads published outside the primary metadata/control-plane branch, such as transcript bytes on `agentv/artifacts/v1`. | -`summary.json` and `.internal/index.jsonl` are complementary, not redundant. A run list -should not scan every row just to show pass rate or total duration, and a row -reader should not parse aggregate summary structures to find one case's grading -or transcript. Keep aggregate questions on `summary.json`; keep row and artifact -discovery on `.internal/index.jsonl`. +`summary.json`, `.internal/run-config.json`, and `.internal/index.jsonl` are +complementary, not redundant. A run list should not scan every row just to show +pass rate or total duration, and a row reader should not parse aggregate summary +structures to find one case's grading or transcript. Keep aggregate questions on +`summary.json`; keep run setup details in `.internal/run-config.json`; keep row +and artifact discovery on `.internal/index.jsonl`. ## Grading Contract @@ -175,6 +177,8 @@ adds providers and projections, but stable rows follow these rules: - Field names are `snake_case`. - Identity and filter fields live on the row, not only in directory names. - Sidecar references are explicit path fields, relative to the run directory. +- Run-level provenance such as `runtime_source` belongs in `summary.json`, not + repeated on every row. - Large detached payloads may also have `artifact_pointers`, but ordinary sidecars should still be discoverable through path fields. - Unknown fields should be preserved by adapters when they rewrite or project diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index 9eafb7983..d4d5b5a89 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -70,6 +70,7 @@ import type { export const RESULT_INDEX_FILENAME = 'index.jsonl'; export const RUN_SUMMARY_FILENAME = 'summary.json'; export const RUN_INTERNAL_DIRNAME = '.internal'; +export const RUN_CONFIG_FILENAME = 'run-config.json'; export const CROSS_RUN_INDEX_DIRNAME = '.indexes'; export const CROSS_RUN_RUNS_INDEX_FILENAME = 'runs.jsonl'; export const CROSS_RUN_CASES_INDEX_FILENAME = 'cases.jsonl'; @@ -217,6 +218,9 @@ export async function aggregateRunDir( const previousMetadata = await readRunSummaryMetadata(path.join(runDir, RUN_SUMMARY_FILENAME)); const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount; + const runConfigPath = options?.experimentMetadata + ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` + : previousMetadata.runConfigPath; const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource; const tags = options?.tags ?? previousMetadata.tags; @@ -229,9 +233,11 @@ export async function aggregateRunDir( options?.experimentMetadata, runtimeSource, tags, + runConfigPath, ); const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME); await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); + await writeRunConfigArtifact(runDir, options?.experimentMetadata); const targetSet = new Set(results.map((r) => r.target ?? 'unknown')); return { summaryPath, testCount: results.length, targetCount: targetSet.size }; @@ -289,6 +295,7 @@ async function resolveExistingResultManifestPath(runDir: string): Promise; }> { @@ -297,6 +304,7 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ const parsed = JSON.parse(raw) as { metadata?: { planned_test_count?: number; + run_config_path?: unknown; runtime_source?: RunRuntimeSourceMetadata; tags?: unknown; }; @@ -307,9 +315,15 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ const runtimeSource = isRunRuntimeSourceMetadata(parsed.metadata?.runtime_source) ? parsed.metadata.runtime_source : undefined; + const runConfigPath = + typeof parsed.metadata?.run_config_path === 'string' && + parsed.metadata.run_config_path.trim().length > 0 + ? parsed.metadata.run_config_path + : undefined; const tags = normalizeStringRecord(parsed.metadata?.tags); return { ...(plannedTestCount !== undefined && { plannedTestCount }), + ...(runConfigPath !== undefined && { runConfigPath }), ...(runtimeSource !== undefined && { runtimeSource }), ...(tags !== undefined && { tags }), }; @@ -524,7 +538,6 @@ export interface RunSummaryArtifact { readonly reasons: readonly { readonly reason: string; readonly count: number }[]; }; readonly cases: readonly Record[]; - readonly instances: readonly Record[]; readonly metadata: { readonly run_id?: string; readonly eval_file: string; @@ -533,7 +546,7 @@ export interface RunSummaryArtifact { readonly variants?: readonly string[]; readonly tests_run: readonly string[]; readonly experiment?: string; - readonly experiment_config?: ExperimentArtifactMetadata; + readonly run_config_path?: string; readonly runtime_source?: RunRuntimeSourceMetadata; readonly planned_test_count?: number; /** @@ -559,6 +572,11 @@ export interface RunSummaryArtifact { readonly notes: readonly string[]; } +export interface RunConfigArtifact { + readonly schema_version: 'agentv.run_config.v1'; + readonly experiment_config?: ExperimentArtifactMetadata; +} + export interface AggregateGradingArtifact { readonly score: number; readonly verdict: 'pass' | 'fail' | 'skip'; @@ -623,7 +641,6 @@ export interface IndexArtifactEntry { readonly metrics_path?: string; readonly file_changes_path?: string; readonly artifact_pointers?: ResultArtifactPointersWire; - readonly runtime_source?: RunRuntimeSourceMetadata; readonly sample_index?: number; readonly retry_index?: number; readonly raw_provider_log_path?: string; @@ -1649,6 +1666,7 @@ export function buildRunSummaryArtifact( experimentMetadata?: ExperimentArtifactMetadata, runtimeSource?: RunRuntimeSourceMetadata, tags?: Record, + runConfigPath?: string, ): RunSummaryArtifact { const targetSet = new Set(); const variantSet = new Set(); @@ -1870,7 +1888,6 @@ export function buildRunSummaryArtifact( .map(([reason, count]) => ({ reason, count })), }, cases: caseSummaries, - instances, metadata: { run_id: runId, eval_file: evalFile, @@ -1879,7 +1896,10 @@ export function buildRunSummaryArtifact( variants: variants.length > 0 ? variants : undefined, tests_run: testIds, experiment, - experiment_config: experimentMetadata, + run_config_path: + experimentMetadata !== undefined + ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` + : runConfigPath, runtime_source: runtimeSource, planned_test_count: plannedTestCount, tags: tags && Object.keys(tags).length > 0 ? tags : undefined, @@ -1918,6 +1938,37 @@ export async function writeInitialRunSummaryArtifact( ); const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME); await writeFile(summaryPath, `${JSON.stringify(stub, null, 2)}\n`, 'utf8'); + await writeRunConfigArtifact(runDir, options.experimentMetadata); +} + +async function writeRunConfigArtifact( + runDir: string, + experimentMetadata: ExperimentArtifactMetadata | undefined, +): Promise { + if (!experimentMetadata) { + return; + } + const config: RunConfigArtifact = { + schema_version: 'agentv.run_config.v1', + experiment_config: experimentMetadata, + }; + const configPath = runInternalPath(runDir, RUN_CONFIG_FILENAME); + await mkdir(path.dirname(configPath), { recursive: true }); + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); +} + +export async function readRunConfigArtifact( + runDir: string, +): Promise { + try { + const parsed = JSON.parse(await readFile(runInternalPath(runDir, RUN_CONFIG_FILENAME), 'utf8')); + if (!isRecord(parsed) || parsed.schema_version !== 'agentv.run_config.v1') { + return undefined; + } + return parsed as unknown as RunConfigArtifact; + } catch { + return undefined; + } } export function buildAggregateGradingArtifact( @@ -2289,7 +2340,6 @@ export function buildIndexArtifactEntry( ? toRelativeArtifactPath(options.outputDir, options.rawProviderLogPath) : undefined, artifact_pointers: options.artifactPointers, - runtime_source: options.runtimeSource, sample_index: result.sampleIndex, retry_index: result.retryIndex, ...options.extraIndexFields, @@ -2415,7 +2465,6 @@ export function buildResultIndexArtifact( transcript_summary: isSingleRun && hasTranscript ? buildResultTranscriptSummary(result) : undefined, artifact_pointers: options?.artifactPointers, - runtime_source: options?.runtimeSource, sample_index: result.sampleIndex, retry_index: result.retryIndex, ...extraIndexFields, @@ -3355,6 +3404,9 @@ export async function writeArtifactsFromResults( const previousMetadata = await readRunSummaryMetadata(summaryPath); const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount; + const runConfigPath = options?.experimentMetadata + ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` + : previousMetadata.runConfigPath; const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource; const summaryTags = resolvedTags ?? previousMetadata.tags; const summary = buildRunSummaryArtifact( @@ -3366,8 +3418,10 @@ export async function writeArtifactsFromResults( options?.experimentMetadata, runtimeSource, summaryTags, + runConfigPath, ); await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); + await writeRunConfigArtifact(outputDir, options?.experimentMetadata); await mkdir(path.dirname(indexPath), { recursive: true }); await writeJsonlFile(indexPath, indexRecords); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 80b7ab2a9..b3829b187 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -56,6 +56,7 @@ export { } from './evaluation/evaluate.js'; export { RESULT_INDEX_FILENAME, + RUN_CONFIG_FILENAME, RUN_SUMMARY_FILENAME, aggregateRunDir, buildAggregateGradingArtifact, @@ -69,6 +70,7 @@ export { buildTimingArtifact, deduplicateByTestIdTarget, parseJsonlResults, + readRunConfigArtifact, writeArtifacts, writeArtifactsFromResults, writeInitialRunSummaryArtifact, @@ -84,6 +86,7 @@ export { type RunRuntimeConfigSource, type RunRuntimeSourceKind, type RunRuntimeSourceMetadata, + type RunConfigArtifact, type RunSummaryArtifact, type TimingArtifact, } from './evaluation/run-artifacts.js';