fix(results): slim canonical artifact contract

apps/cli/src/commands/eval/artifact-writer.ts

@@ -10,8 +10,10 @@ import {
   type GradingArtifact,
   type IndexArtifactEntry,
   RESULT_INDEX_FILENAME,
+  RUN_CONFIG_FILENAME,
   RUN_SUMMARY_FILENAME,
   type ResultIndexArtifact,
+  type RunConfigArtifact,
   type RunRuntimeSourceMetadata,
   type RunSummaryArtifact,
   type TimingArtifact,
@@ -27,6 +29,7 @@ import {
   buildTimingArtifact,
   deduplicateByTestIdTarget,
   parseJsonlResults,
+  readRunConfigArtifact,
   writeArtifacts,
   writeArtifactsFromResults as writeCoreArtifactsFromResults,
   writePerTestArtifacts as writeCorePerTestArtifacts,
@@ -52,7 +55,9 @@ export {
   deduplicateByTestIdTarget,
   parseJsonlResults,
   RESULT_INDEX_FILENAME,
+  RUN_CONFIG_FILENAME,
   RUN_SUMMARY_FILENAME,
+  readRunConfigArtifact,
   writeArtifacts,
   writeInitialRunSummaryArtifact,
 };
@@ -61,6 +66,7 @@ export type {
   GradingArtifact,
   IndexArtifactEntry,
   ResultIndexArtifact,
+  RunConfigArtifact,
   RunSummaryArtifact,
   TimingArtifact,
 };

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -12,6 +12,7 @@ import {
   type GraderResult,
   METRICS_SCHEMA_VERSION,
   MetricsArtifactWireSchema,
+  aggregateRunDir,
   buildEvalTestTargetKey,
   buildEvaluationResultTargetKey,
   buildResultIndexArtifact,
@@ -25,6 +26,7 @@ import {
   type GradingArtifact,
   type IndexArtifactEntry,
   RESULT_INDEX_FILENAME,
+  RUN_CONFIG_FILENAME,
   type RunSummaryArtifact,
   type TimingArtifact,
   buildAggregateGradingArtifact,
@@ -1172,7 +1174,7 @@ describe('writeArtifactsFromResults', () => {
     expect(indexLines[0]?.metrics_path).toBe(`${alphaRowDir}/sample-1/metrics.json`);
   });
 
-  it('writes optional runtime source metadata to summary and index rows', async () => {
+  it('writes optional runtime source metadata to summary only', async () => {
     const runtimeSource = {
       schema_version: 'agentv.runtime_source.v1' as const,
       kind: 'direct_suite' as const,
@@ -1194,7 +1196,53 @@ describe('writeArtifactsFromResults', () => {
       .map(JSON.parse);
 
     expect(summary.metadata.runtime_source).toEqual(runtimeSource);
-    expect(indexLine.runtime_source).toEqual(runtimeSource);
+    expect(indexLine.runtime_source).toBeUndefined();
+  });
+
+  it('moves experiment config metadata to an internal run config sidecar', async () => {
+    const experimentMetadata = {
+      name: 'native-exp',
+      target: 'codex-target',
+      threshold: 0.8,
+      fingerprint: 'a'.repeat(64),
+    };
+    const paths = await writeArtifactsFromResults([makeResult({ testId: 'alpha' })], testDir, {
+      evalFile: 'evals/native-exp.eval.yaml',
+      experiment: 'native-exp',
+      experimentMetadata,
+    });
+
+    const summary: RunSummaryArtifact = JSON.parse(await readFile(paths.summaryPath, 'utf8'));
+    expect(summary.metadata).not.toHaveProperty('experiment_config');
+    expect(summary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`);
+
+    const runConfig = JSON.parse(
+      await readFile(path.join(paths.testArtifactDir, '.internal', RUN_CONFIG_FILENAME), 'utf8'),
+    );
+    expect(runConfig).toEqual({
+      schema_version: 'agentv.run_config.v1',
+      experiment_config: experimentMetadata,
+    });
+
+    await aggregateRunDir(paths.testArtifactDir);
+    const rewrittenSummary: RunSummaryArtifact = JSON.parse(
+      await readFile(paths.summaryPath, 'utf8'),
+    );
+    expect(rewrittenSummary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`);
+  });
+
+  it('omits duplicated root instances from run summary', () => {
+    const summary = buildRunSummaryArtifact(
+      [
+        makeResult({ testId: 'alpha', target: 'target-a', score: 1 }),
+        makeResult({ testId: 'beta', target: 'target-a', score: 0 }),
+      ],
+      'evals/smoke.eval.yaml',
+    );
+
+    expect(summary.counts.total_instances).toBe(2);
+    expect(summary.cases).toHaveLength(2);
+    expect(summary).not.toHaveProperty('instances');
   });
 
   it('emits the resolved tags map to summary metadata and every index row', async () => {

apps/cli/test/eval.integration.test.ts

@@ -697,7 +697,15 @@ describe('agentv eval CLI', () => {
         await readFile(path.join(runDirFromIndexPath(outputPath), 'summary.json'), 'utf8'),
       ) as { metadata?: Record<string, unknown> };
       expect(benchmark.metadata?.experiment).toBe('native-exp');
-      expect(benchmark.metadata?.experiment_config).toMatchObject({
+      expect(benchmark.metadata).not.toHaveProperty('experiment_config');
+      expect(benchmark.metadata?.run_config_path).toBe('.internal/run-config.json');
+      const runConfig = JSON.parse(
+        await readFile(
+          path.join(runDirFromIndexPath(outputPath), '.internal', 'run-config.json'),
+          'utf8',
+        ),
+      ) as { experiment_config?: Record<string, unknown> };
+      expect(runConfig.experiment_config).toMatchObject({
         target: 'codex-target',
         repeat: {
           count: 2,
@@ -708,9 +716,9 @@ describe('agentv eval CLI', () => {
         budget_usd: 3,
         timeout_seconds: 12,
       });
-      expect(
-        (benchmark.metadata?.experiment_config as Record<string, unknown>).fingerprint,
-      ).toMatch(/^[a-f0-9]{64}$/);
+      expect((runConfig.experiment_config as Record<string, unknown>).fingerprint).toMatch(
+        /^[a-f0-9]{64}$/,
+      );
       expect(benchmark.metadata?.runtime_source).toMatchObject({
         schema_version: 'agentv.runtime_source.v1',
         kind: 'wrapper_eval',

apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx

@@ -94,7 +94,8 @@ reserved for rebuildable local state and are skipped by run discovery.
 
 | File or field | Owns | Use it for |
 | --- | --- | --- |
-| `summary.json` | Aggregate run metadata and rollups: run id, experiment metadata, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. |
+| `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. |
+| `.internal/run-config.json` | Internal reproducibility metadata that is useful to rerun or audit the invocation but too noisy for the public aggregate summary, such as resolved experiment config and fingerprints. | Debugging run setup, reproducing wrapper-eval choices, and reviewing config provenance without repeating it in every row. |
 | `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. |
 | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. |
 | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. |
@@ -107,11 +108,12 @@ reserved for rebuildable local state and are skipped by run discovery.
 | `test/` | Generated test bundle for the exact eval slice and target settings that produced a row. | Audit, external review, and rerun workflows that should not depend on a mutable source checkout. |
 | `artifact_pointers` | Offload indirection for large detached payload bytes. | Finding payloads published outside the primary metadata/control-plane branch, such as transcript bytes on `agentv/artifacts/v1`. |
 
-`summary.json` and `.internal/index.jsonl` are complementary, not redundant. A run list
-should not scan every row just to show pass rate or total duration, and a row
-reader should not parse aggregate summary structures to find one case's grading
-or transcript. Keep aggregate questions on `summary.json`; keep row and artifact
-discovery on `.internal/index.jsonl`.
+`summary.json`, `.internal/run-config.json`, and `.internal/index.jsonl` are
+complementary, not redundant. A run list should not scan every row just to show
+pass rate or total duration, and a row reader should not parse aggregate summary
+structures to find one case's grading or transcript. Keep aggregate questions on
+`summary.json`; keep run setup details in `.internal/run-config.json`; keep row
+and artifact discovery on `.internal/index.jsonl`.
 
 ## Grading Contract
 
@@ -175,6 +177,8 @@ adds providers and projections, but stable rows follow these rules:
 - Field names are `snake_case`.
 - Identity and filter fields live on the row, not only in directory names.
 - Sidecar references are explicit path fields, relative to the run directory.
+- Run-level provenance such as `runtime_source` belongs in `summary.json`, not
+  repeated on every row.
 - Large detached payloads may also have `artifact_pointers`, but ordinary
   sidecars should still be discoverable through path fields.
 - Unknown fields should be preserved by adapters when they rewrite or project

packages/core/src/evaluation/run-artifacts.ts

@@ -70,6 +70,7 @@ import type {
 export const RESULT_INDEX_FILENAME = 'index.jsonl';
 export const RUN_SUMMARY_FILENAME = 'summary.json';
 export const RUN_INTERNAL_DIRNAME = '.internal';
+export const RUN_CONFIG_FILENAME = 'run-config.json';
 export const CROSS_RUN_INDEX_DIRNAME = '.indexes';
 export const CROSS_RUN_RUNS_INDEX_FILENAME = 'runs.jsonl';
 export const CROSS_RUN_CASES_INDEX_FILENAME = 'cases.jsonl';
@@ -217,6 +218,9 @@ export async function aggregateRunDir(
 
   const previousMetadata = await readRunSummaryMetadata(path.join(runDir, RUN_SUMMARY_FILENAME));
   const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount;
+  const runConfigPath = options?.experimentMetadata
+    ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}`
+    : previousMetadata.runConfigPath;
   const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource;
   const tags = options?.tags ?? previousMetadata.tags;
 
@@ -229,9 +233,11 @@ export async function aggregateRunDir(
     options?.experimentMetadata,
     runtimeSource,
     tags,
+    runConfigPath,
   );
   const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME);
   await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+  await writeRunConfigArtifact(runDir, options?.experimentMetadata);
 
   const targetSet = new Set(results.map((r) => r.target ?? 'unknown'));
   return { summaryPath, testCount: results.length, targetCount: targetSet.size };
@@ -289,6 +295,7 @@ async function resolveExistingResultManifestPath(runDir: string): Promise<string
 
 async function readRunSummaryMetadata(summaryPath: string): Promise<{
   plannedTestCount?: number;
+  runConfigPath?: string;
   runtimeSource?: RunRuntimeSourceMetadata;
   tags?: Record<string, string>;
 }> {
@@ -297,6 +304,7 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{
     const parsed = JSON.parse(raw) as {
       metadata?: {
         planned_test_count?: number;
+        run_config_path?: unknown;
         runtime_source?: RunRuntimeSourceMetadata;
         tags?: unknown;
       };
@@ -307,9 +315,15 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{
     const runtimeSource = isRunRuntimeSourceMetadata(parsed.metadata?.runtime_source)
       ? parsed.metadata.runtime_source
       : undefined;
+    const runConfigPath =
+      typeof parsed.metadata?.run_config_path === 'string' &&
+      parsed.metadata.run_config_path.trim().length > 0
+        ? parsed.metadata.run_config_path
+        : undefined;
     const tags = normalizeStringRecord(parsed.metadata?.tags);
     return {
       ...(plannedTestCount !== undefined && { plannedTestCount }),
+      ...(runConfigPath !== undefined && { runConfigPath }),
       ...(runtimeSource !== undefined && { runtimeSource }),
       ...(tags !== undefined && { tags }),
     };
@@ -524,7 +538,6 @@ export interface RunSummaryArtifact {
     readonly reasons: readonly { readonly reason: string; readonly count: number }[];
   };
   readonly cases: readonly Record<string, unknown>[];
-  readonly instances: readonly Record<string, unknown>[];
   readonly metadata: {
     readonly run_id?: string;
     readonly eval_file: string;
@@ -533,7 +546,7 @@ export interface RunSummaryArtifact {
     readonly variants?: readonly string[];
     readonly tests_run: readonly string[];
     readonly experiment?: string;
-    readonly experiment_config?: ExperimentArtifactMetadata;
+    readonly run_config_path?: string;
     readonly runtime_source?: RunRuntimeSourceMetadata;
     readonly planned_test_count?: number;
     /**
@@ -559,6 +572,11 @@ export interface RunSummaryArtifact {
   readonly notes: readonly string[];
 }
 
+export interface RunConfigArtifact {
+  readonly schema_version: 'agentv.run_config.v1';
+  readonly experiment_config?: ExperimentArtifactMetadata;
+}
+
 export interface AggregateGradingArtifact {
   readonly score: number;
   readonly verdict: 'pass' | 'fail' | 'skip';
@@ -623,7 +641,6 @@ export interface IndexArtifactEntry {
   readonly metrics_path?: string;
   readonly file_changes_path?: string;
   readonly artifact_pointers?: ResultArtifactPointersWire;
-  readonly runtime_source?: RunRuntimeSourceMetadata;
   readonly sample_index?: number;
   readonly retry_index?: number;
   readonly raw_provider_log_path?: string;
@@ -1649,6 +1666,7 @@ export function buildRunSummaryArtifact(
   experimentMetadata?: ExperimentArtifactMetadata,
   runtimeSource?: RunRuntimeSourceMetadata,
   tags?: Record<string, string>,
+  runConfigPath?: string,
 ): RunSummaryArtifact {
   const targetSet = new Set<string>();
   const variantSet = new Set<string>();
@@ -1870,7 +1888,6 @@ export function buildRunSummaryArtifact(
         .map(([reason, count]) => ({ reason, count })),
     },
     cases: caseSummaries,
-    instances,
     metadata: {
       run_id: runId,
       eval_file: evalFile,
@@ -1879,7 +1896,10 @@ export function buildRunSummaryArtifact(
       variants: variants.length > 0 ? variants : undefined,
       tests_run: testIds,
       experiment,
-      experiment_config: experimentMetadata,
+      run_config_path:
+        experimentMetadata !== undefined
+          ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}`
+          : runConfigPath,
       runtime_source: runtimeSource,
       planned_test_count: plannedTestCount,
       tags: tags && Object.keys(tags).length > 0 ? tags : undefined,
@@ -1918,6 +1938,37 @@ export async function writeInitialRunSummaryArtifact(
   );
   const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME);
   await writeFile(summaryPath, `${JSON.stringify(stub, null, 2)}\n`, 'utf8');
+  await writeRunConfigArtifact(runDir, options.experimentMetadata);
+}
+
+async function writeRunConfigArtifact(
+  runDir: string,
+  experimentMetadata: ExperimentArtifactMetadata | undefined,
+): Promise<void> {
+  if (!experimentMetadata) {
+    return;
+  }
+  const config: RunConfigArtifact = {
+    schema_version: 'agentv.run_config.v1',
+    experiment_config: experimentMetadata,
+  };
+  const configPath = runInternalPath(runDir, RUN_CONFIG_FILENAME);
+  await mkdir(path.dirname(configPath), { recursive: true });
+  await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
+}
+
+export async function readRunConfigArtifact(
+  runDir: string,
+): Promise<RunConfigArtifact | undefined> {
+  try {
+    const parsed = JSON.parse(await readFile(runInternalPath(runDir, RUN_CONFIG_FILENAME), 'utf8'));
+    if (!isRecord(parsed) || parsed.schema_version !== 'agentv.run_config.v1') {
+      return undefined;
+    }
+    return parsed as unknown as RunConfigArtifact;
+  } catch {
+    return undefined;
+  }
 }
 
 export function buildAggregateGradingArtifact(
@@ -2289,7 +2340,6 @@ export function buildIndexArtifactEntry(
       ? toRelativeArtifactPath(options.outputDir, options.rawProviderLogPath)
       : undefined,
     artifact_pointers: options.artifactPointers,
-    runtime_source: options.runtimeSource,
     sample_index: result.sampleIndex,
     retry_index: result.retryIndex,
     ...options.extraIndexFields,
@@ -2415,7 +2465,6 @@ export function buildResultIndexArtifact(
     transcript_summary:
       isSingleRun && hasTranscript ? buildResultTranscriptSummary(result) : undefined,
     artifact_pointers: options?.artifactPointers,
-    runtime_source: options?.runtimeSource,
     sample_index: result.sampleIndex,
     retry_index: result.retryIndex,
     ...extraIndexFields,
@@ -3355,6 +3404,9 @@ export async function writeArtifactsFromResults(
 
   const previousMetadata = await readRunSummaryMetadata(summaryPath);
   const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount;
+  const runConfigPath = options?.experimentMetadata
+    ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}`
+    : previousMetadata.runConfigPath;
   const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource;
   const summaryTags = resolvedTags ?? previousMetadata.tags;
   const summary = buildRunSummaryArtifact(
@@ -3366,8 +3418,10 @@ export async function writeArtifactsFromResults(
     options?.experimentMetadata,
     runtimeSource,
     summaryTags,
+    runConfigPath,
   );
   await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+  await writeRunConfigArtifact(outputDir, options?.experimentMetadata);
 
   await mkdir(path.dirname(indexPath), { recursive: true });
   await writeJsonlFile(indexPath, indexRecords);