From 779c897a844bce60313a04b7c8112f7ae8920fc0 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 14:13:51 +0200 Subject: [PATCH 1/2] fix(artifacts): keep eval runs at timestamp root --- CONCEPTS.md | 2 +- apps/cli/src/commands/eval/run-eval.ts | 207 ++++++------------ .../test/commands/eval/result-layout.test.ts | 18 +- apps/cli/test/eval.integration.test.ts | 31 ++- .../docs/docs/evaluation/running-evals.mdx | 12 +- .../docs/docs/reference/result-artifacts.mdx | 25 ++- ...arate-experiments-from-eval-definitions.md | 4 + ...-result-identity-and-default-experiment.md | 4 + docs/adr/0012-finalize-run-artifact-layout.md | 166 ++++++++++++++ 9 files changed, 298 insertions(+), 171 deletions(-) create mode 100644 docs/adr/0012-finalize-run-artifact-layout.md diff --git a/CONCEPTS.md b/CONCEPTS.md index 41ec016e2..8ec1fd026 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -28,7 +28,7 @@ Shared domain vocabulary for this project — entities, named processes, and sta **Result source identity** — The stable source identity for a result row: repo-relative `eval_path`, `test_id`, and `target`. `suite` and `name` are display metadata, not storage or routing identity. -**Result directory** — The `result_dir` field in a `index.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs. Consumers discover it from `index.jsonl` and must not infer it from suite names, display names, test IDs, or targets. +**Result directory** — The `result_dir` field in a `index.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs, usually a readable test-id or slug prefix plus a UUID/hash-like suffix. Consumers discover it from `index.jsonl` and must not infer it from suite names, display names, test IDs, targets, models, or folder position. **Artifact sidecar** — A file beside or below a result directory that provides evidence for a result, such as `summary.json`, `grading.json`, `result.json`, transcripts, logs, or outputs. Sidecars are evidence, not the primary discovery mechanism for a run. diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 77d99201d..9baccfc85 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1008,15 +1008,6 @@ function applyVerboseOverride(selection: TargetSelection, cliVerbose: boolean): }; } -function safeRunPathSegment(value: string | undefined, fallback: string): string { - const trimmed = value?.trim(); - if (!trimmed) { - return fallback; - } - const segment = trimmed.replace(/[/\\:*?"<>|]/g, '_'); - return !segment || segment === '.' || segment === '..' ? fallback : segment; -} - function targetVariantForSelection(selection: TargetSelection): string | undefined { const target = selection.resolvedTarget; if (target.kind === 'replay') { @@ -1025,64 +1016,37 @@ function targetVariantForSelection(selection: TargetSelection): string | undefin return undefined; } -function resultBundleKey(result: Pick): string { - return JSON.stringify({ - target: result.target ?? 'unknown', - variant: result.variant ?? null, - }); -} - -function resultBundleDir( - invocationDir: string, - result: Pick, -): string { - const targetDir = safeRunPathSegment(result.target, 'unknown-target'); - const variantDir = result.variant ? safeRunPathSegment(result.variant, 'variant') : undefined; - return variantDir - ? path.join(invocationDir, targetDir, variantDir) - : path.join(invocationDir, targetDir); -} - -class BundleOutputWriter implements OutputWriter { - private readonly writers = new Map< - string, - { readonly dir: string; readonly indexPath: string; readonly writer: OutputWriter } - >(); +class RunOutputWriter implements OutputWriter { + private readonly indexPath: string; + private writer: OutputWriter | undefined; constructor( private readonly invocationDir: string, private readonly appendMode: boolean, - ) {} + ) { + this.indexPath = path.join(invocationDir, RESULT_INDEX_FILENAME); + } async append(result: EvaluationResult): Promise { - const writer = await this.writerForResult(result); + const writer = await this.writerForRun(); await writer.append(result); } async close(): Promise { - await Promise.all([...this.writers.values()].map((entry) => entry.writer.close())); - } - - bundleDirs(): readonly string[] { - return [...this.writers.values()].map((entry) => entry.dir); + await this.writer?.close(); } - bundleIndexPaths(): readonly string[] { - return [...this.writers.values()].map((entry) => entry.indexPath); + indexPaths(): readonly string[] { + return this.writer ? [this.indexPath] : []; } - private async writerForResult(result: EvaluationResult): Promise { - const key = resultBundleKey(result); - const existing = this.writers.get(key); - if (existing) { - return existing.writer; + private async writerForRun(): Promise { + if (this.writer) { + return this.writer; } - const dir = resultBundleDir(this.invocationDir, result); - mkdirSync(dir, { recursive: true }); - const indexPath = path.join(dir, RESULT_INDEX_FILENAME); - const writer = await createOutputWriter(indexPath, { append: this.appendMode }); - this.writers.set(key, { dir, indexPath, writer }); - return writer; + mkdirSync(this.invocationDir, { recursive: true }); + this.writer = await createOutputWriter(this.indexPath, { append: this.appendMode }); + return this.writer; } } @@ -1797,8 +1761,8 @@ export async function runEvalCommand( console.log(`Repository root: ${repoRoot}`); } - // Resolve artifact directory. The CLI run dir is an invocation root; each - // target/variant writes its own bundle index below it. + // Resolve artifact directory. The CLI run dir is the run bundle root; target, + // model, and variant are metadata fields, not path dimensions. // Precedence: --output > config output.dir > default const explicitDir = options.outputDir; let runDir: string; @@ -1997,9 +1961,8 @@ export async function runEvalCommand( throw new Error('--threshold must be between 0 and 1'); } - // Build the output writer. Each target/variant gets a separate bundle index - // below the invocation directory. - const outputWriter = new BundleOutputWriter(runDir, isResumeAppend); + // Build the output writer for the single run-root manifest. + const outputWriter = new RunOutputWriter(runDir, isResumeAppend); // Detect matrix mode: multiple targets for any file const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1); @@ -2008,10 +1971,6 @@ export async function runEvalCommand( // When resuming, subtract tests that will be skipped let totalEvalCount = 0; let resumeSkippedCount = 0; - const plannedBundleCounts = new Map< - string, - { readonly target: string; readonly variant?: string; count: number } - >(); for (const meta of fileMetadata.values()) { for (const test of meta.testCases) { for (const { selection } of meta.selections) { @@ -2022,13 +1981,6 @@ export async function runEvalCommand( resumeSkippedCount++; } else { totalEvalCount++; - const bundleKey = resultBundleKey({ target, variant }); - const existing = plannedBundleCounts.get(bundleKey); - if (existing) { - existing.count += 1; - } else { - plannedBundleCounts.set(bundleKey, { target, variant, count: 1 }); - } } } } @@ -2150,7 +2102,7 @@ export async function runEvalCommand( ); } - // Write a stub summary.json in each planned bundle before dispatching tests, + // Write a stub summary.json in the run bundle before dispatching tests, // carrying the planned execution count so an interrupted run can still // surface as resumable in Dashboard. The end-of-run write preserves this // value via readPlannedTestCount inside aggregateRunDir / @@ -2158,15 +2110,13 @@ export async function runEvalCommand( // Skip on resume — we want to preserve the *original* planned count. if (!isResumeAppend && totalEvalCount > 0) { const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : ''; - for (const bundle of plannedBundleCounts.values()) { - await writeInitialRunSummaryArtifact(resultBundleDir(runDir, bundle), { - evalFile, - plannedTestCount: bundle.count, - experiment: normalizeExperimentName(options.experiment), - experimentMetadata: runExperimentMetadata, - runtimeSource: runtimeSourceMetadata, - }); - } + await writeInitialRunSummaryArtifact(runDir, { + evalFile, + plannedTestCount: totalEvalCount, + experiment: normalizeExperimentName(options.experiment), + experimentMetadata: runExperimentMetadata, + runtimeSource: runtimeSourceMetadata, + }); } // Periodic WIP checkpoint loop: push partial results to a unique non-default @@ -2421,77 +2371,62 @@ export async function runEvalCommand( console.log(formatMatrixSummary(summaryResults)); } - // Write artifacts to target/variant bundle directories (always, not - // conditional on flags). The invocation root is only a container. + // Write artifacts to the run bundle root (always, not conditional on flags). + // Per-result artifact directories are allocated from row identity and + // exposed through index.jsonl fields. if (allResults.length > 0) { const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : ''; const sourceTests = activeSourceTests; const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata); - const resultsByBundle = new Map(); - for (const result of allResults) { - const key = resultBundleKey(result); - const existing = resultsByBundle.get(key); - if (existing) { - existing.push(result); - } else { - resultsByBundle.set(key, [result]); - } - } if (isResumeAppend) { // Resume mode: write per-test artifacts for newly-run tests, then - // aggregate each bundle from its full row manifest (old + new results - // with deduplication). + // aggregate the run from its full row manifest (old + new results with + // deduplication). const { writePerTestArtifacts } = await import('./artifact-writer.js'); - for (const bundleResults of resultsByBundle.values()) { - const bundleDir = resultBundleDir(runDir, bundleResults[0]); - await writePerTestArtifacts(bundleResults, bundleDir, { + await writePerTestArtifacts(allResults, runDir, { + experiment: normalizeExperimentName(options.experiment), + resultGroup: resultGroupName, + cwd, + repoRoot, + sourceTests, + taskBundleTargets, + runtimeSource: runtimeSourceMetadata, + }); + const { summaryPath } = await aggregateRunDir(runDir, { + evalFile, + experiment: normalizeExperimentName(options.experiment), + experimentMetadata: runExperimentMetadata, + runtimeSource: runtimeSourceMetadata, + }); + const indexPath = path.join(runDir, RESULT_INDEX_FILENAME); + console.log(`Artifact bundle updated: ${runDir}`); + console.log(` Run manifest: ${indexPath}`); + console.log( + ` Per-test artifacts: ${runDir} (${allResults.length} new test directories)`, + ); + console.log(` Summary: ${summaryPath}`); + } else { + const { testArtifactDir, summaryPath, indexPath } = await writeArtifactsFromResults( + allResults, + runDir, + { + evalFile, experiment: normalizeExperimentName(options.experiment), + experimentMetadata: runExperimentMetadata, resultGroup: resultGroupName, cwd, repoRoot, sourceTests, taskBundleTargets, runtimeSource: runtimeSourceMetadata, - }); - const { summaryPath } = await aggregateRunDir(bundleDir, { - evalFile, - experiment: normalizeExperimentName(options.experiment), - experimentMetadata: runExperimentMetadata, - runtimeSource: runtimeSourceMetadata, - }); - const indexPath = path.join(bundleDir, RESULT_INDEX_FILENAME); - console.log(`Artifact bundle updated: ${bundleDir}`); - console.log(` Run manifest: ${indexPath}`); - console.log( - ` Per-test artifacts: ${bundleDir} (${bundleResults.length} new test directories)`, - ); - console.log(` Summary: ${summaryPath}`); - } - } else { - for (const bundleResults of resultsByBundle.values()) { - const bundleDir = resultBundleDir(runDir, bundleResults[0]); - const { testArtifactDir, summaryPath, indexPath } = await writeArtifactsFromResults( - bundleResults, - bundleDir, - { - evalFile, - experiment: normalizeExperimentName(options.experiment), - experimentMetadata: runExperimentMetadata, - resultGroup: resultGroupName, - cwd, - repoRoot, - sourceTests, - taskBundleTargets, - runtimeSource: runtimeSourceMetadata, - }, - ); - console.log(`Artifact bundle written to: ${bundleDir}`); - console.log(` Run manifest: ${indexPath}`); - console.log( - ` Per-test artifacts: ${testArtifactDir} (${bundleResults.length} test directories)`, - ); - console.log(` Summary: ${summaryPath}`); - } + }, + ); + console.log(`Artifact bundle written to: ${runDir}`); + console.log(` Run manifest: ${indexPath}`); + console.log( + ` Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`, + ); + console.log(` Summary: ${summaryPath}`); } } @@ -2517,7 +2452,7 @@ export async function runEvalCommand( } if (allResults.length > 0) { - const writtenIndexes = outputWriter.bundleIndexPaths(); + const writtenIndexes = outputWriter.indexPaths(); outputPath = writtenIndexes[0] ?? outputPath; console.log(`\nResults written to: ${outputPath}`); console.log(`\nResults written under: ${runDir}`); diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts index 13b769668..8c703103a 100644 --- a/apps/cli/test/commands/eval/result-layout.test.ts +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -61,7 +61,7 @@ describe('result layout', () => { } }); - it('discovers one canonical index.jsonl manifest per nested bundle', () => { + it('discovers one canonical index.jsonl manifest per legacy nested bundle', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); try { const bundleDir = path.join(tempDir, 'default', '2026-run', 'target-a'); @@ -76,6 +76,22 @@ describe('result layout', () => { } }); + it('treats the root index.jsonl as authoritative when legacy nested bundles also exist', () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); + try { + const nestedBundleDir = path.join(tempDir, 'target-a'); + mkdirSync(nestedBundleDir, { recursive: true }); + const rootIndexPath = path.join(tempDir, RESULT_INDEX_FILENAME); + writeFileSync(rootIndexPath, '{"test_id":"root"}\n'); + writeFileSync(path.join(nestedBundleDir, RESULT_INDEX_FILENAME), '{"test_id":"legacy"}\n'); + + expect(discoverRunManifestPaths(tempDir)).toEqual([rootIndexPath]); + expect(resolveRunManifestPath(tempDir)).toBe(rootIndexPath); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('reports index.jsonl as the canonical missing run manifest name', () => { const dir = mkdtempSync(path.join(tmpdir(), 'agentv-result-layout-')); try { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 5faae713b..5888e8149 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -341,19 +341,18 @@ describe('agentv eval CLI', () => { ]); expect(exitCode).toBe(0); - const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); + const indexPath = path.join(outputDir, 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); expect(stdout).toContain(`Artifact directory: ${outputDir}`); const results = await readJsonLines(indexPath); expect(results).toHaveLength(2); - await expectFileExists(path.join(outputDir, 'file-target', 'summary.json')); + await expectFileExists(path.join(outputDir, 'summary.json')); for (const row of results as Array>) { const resultDir = row.result_dir as string; - await expectFileExists(path.join(outputDir, 'file-target', resultDir, 'summary.json')); - await expectFileExists( - path.join(outputDir, 'file-target', resultDir, 'run-1', 'grading.json'), - ); + expect(resultDir).not.toContain('/'); + await expectFileExists(path.join(outputDir, resultDir, 'summary.json')); + await expectFileExists(path.join(outputDir, resultDir, 'run-1', 'grading.json')); } } finally { await rm(fixture.baseDir, { recursive: true, force: true }); @@ -369,16 +368,14 @@ describe('agentv eval CLI', () => { const outputDir = path.join(fixture.suiteDir, 'configured-results'); expect(exitCode).toBe(0); - const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); + const indexPath = path.join(outputDir, 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); await expectFileExists(indexPath); - await expectFileExists(path.join(outputDir, 'file-target', 'summary.json')); + await expectFileExists(path.join(outputDir, 'summary.json')); const [firstRow] = (await readJsonLines(indexPath)) as Array>; + await expectFileExists(path.join(outputDir, firstRow.result_dir as string, 'summary.json')); await expectFileExists( - path.join(outputDir, 'file-target', firstRow.result_dir as string, 'summary.json'), - ); - await expectFileExists( - path.join(outputDir, 'file-target', firstRow.result_dir as string, 'run-1', 'grading.json'), + path.join(outputDir, firstRow.result_dir as string, 'run-1', 'grading.json'), ); } finally { await rm(fixture.baseDir, { recursive: true, force: true }); @@ -413,20 +410,18 @@ describe('agentv eval CLI', () => { ]); expect(exitCode).toBe(1); - const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); + const indexPath = path.join(outputDir, 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); expect(stdout).not.toContain('Export files:'); const canonicalResults = await readJsonLines(indexPath); expect(canonicalResults).toHaveLength(2); - await expectFileExists(path.join(outputDir, 'file-target', 'summary.json')); + await expectFileExists(path.join(outputDir, 'summary.json')); for (const row of canonicalResults) { expect(row.transcript_path).toMatch(/run-1\/transcript\.jsonl$/); - await expectFileExists(path.join(outputDir, 'file-target', row.transcript_path as string)); + await expectFileExists(path.join(outputDir, row.transcript_path as string)); expect(row.transcript_raw_path).toMatch(/run-1\/transcript-raw\.jsonl$/); - await expectFileExists( - path.join(outputDir, 'file-target', row.transcript_raw_path as string), - ); + await expectFileExists(path.join(outputDir, row.transcript_raw_path as string)); } } finally { await rm(fixture.baseDir, { recursive: true, force: true }); diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 6095d640d..7f9fafc03 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -475,11 +475,13 @@ See the [Import tool docs](/docs/tools/import/) for all providers and options. ## Transcript And Result Artifacts -Each result row's `result_dir` is a case-local folder under the timestamped -run bundle. It can include `transcript.jsonl`, `transcript-raw.jsonl`, -`grading.json`, `timing.json`, `metrics.json`, and generated outputs under -`outputs/`. The run root does not contain a mixed transcript artifact; use each -index row's `transcript_path` to find the per-result transcript. +Each result row's `result_dir` is an allocated folder under the timestamped run +bundle, usually with a readable test-id prefix plus a short hash suffix. It can +include `transcript.jsonl`, `transcript-raw.jsonl`, `grading.json`, +`timing.json`, `metrics.json`, and generated outputs under `outputs/`. The run +root does not contain target, model, or `cases/` folders, and it does not contain +a mixed transcript artifact; use each index row's `transcript_path` to find the +per-result transcript. Rows also include `artifact_pointers` for AgentV-owned artifact storage. Pointer entries such as `artifact_pointers.transcript` carry the storage `ref`, artifact diff --git a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx index dcb0c8150..12e530fae 100644 --- a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx @@ -62,6 +62,11 @@ semantic truth from folder names. Use fields in `summary.json` and `index.jsonl` for experiment, target, variant, attempt, eval path, case identity, timing, scores, and artifact paths. +The run bundle does not add target, model, variant, or `cases/` folders below +``. Per-result directories are allocated from row identity, usually with +a readable test-id or slug prefix plus a short hash suffix, and remain opaque to +consumers. + `experiment` remains the comparison and runtime-policy concept: it is how users label a condition such as `baseline`, `candidate`, `with_skills`, or `without_skills`. The folder segment is a convenient bucket for that concept, @@ -118,16 +123,16 @@ Example row: "execution_status": "ok", "score": 0.92, "duration_ms": 184200, - "result_dir": "refund-eligibility/run-1", - "summary_path": "refund-eligibility/summary.json", - "grading_path": "refund-eligibility/run-1/grading.json", - "metrics_path": "refund-eligibility/run-1/metrics.json", - "timing_path": "refund-eligibility/run-1/timing.json", - "transcript_path": "refund-eligibility/run-1/transcript.jsonl", - "transcript_raw_path": "refund-eligibility/run-1/transcript-raw.jsonl", - "output_path": "refund-eligibility/run-1/outputs/answer.md", - "answer_path": "refund-eligibility/run-1/outputs/answer.md", - "test_dir": "refund-eligibility/test" + "result_dir": "refund-eligibility--4f9a7c2d1b6e", + "summary_path": "refund-eligibility--4f9a7c2d1b6e/summary.json", + "grading_path": "refund-eligibility--4f9a7c2d1b6e/run-1/grading.json", + "metrics_path": "refund-eligibility--4f9a7c2d1b6e/run-1/metrics.json", + "timing_path": "refund-eligibility--4f9a7c2d1b6e/run-1/timing.json", + "transcript_path": "refund-eligibility--4f9a7c2d1b6e/run-1/transcript.jsonl", + "transcript_raw_path": "refund-eligibility--4f9a7c2d1b6e/run-1/transcript-raw.jsonl", + "output_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md", + "answer_path": "refund-eligibility--4f9a7c2d1b6e/run-1/outputs/answer.md", + "test_dir": "refund-eligibility--4f9a7c2d1b6e/test" } ``` diff --git a/docs/adr/0006-separate-experiments-from-eval-definitions.md b/docs/adr/0006-separate-experiments-from-eval-definitions.md index abb68ba46..0b7d42dd3 100644 --- a/docs/adr/0006-separate-experiments-from-eval-definitions.md +++ b/docs/adr/0006-separate-experiments-from-eval-definitions.md @@ -6,6 +6,10 @@ Date: 2026-06-26 Accepted +Superseded in part by [ADR 0012](0012-finalize-run-artifact-layout.md) for the +physical run artifact layout. Target and variant are metadata dimensions, not +required folders below the timestamp bundle. + Supersedes: the 2026-06-23 proposal in this file to separate experiment files from eval definitions. diff --git a/docs/adr/0009-eval-path-result-identity-and-default-experiment.md b/docs/adr/0009-eval-path-result-identity-and-default-experiment.md index 6b18791c7..f8abde8bd 100644 --- a/docs/adr/0009-eval-path-result-identity-and-default-experiment.md +++ b/docs/adr/0009-eval-path-result-identity-and-default-experiment.md @@ -11,6 +11,10 @@ Supersedes: result naming and storage-routing portions of bundle names or per-case artifact paths from eval names, suite names, or wrapper composition. +Superseded in part by [ADR 0012](0012-finalize-run-artifact-layout.md), which +removes the target/variant folder fan-out below the timestamp and keeps +`.agentv/results///` as the run bundle. + ## Context AgentV needs one simple result identity contract that works for direct eval diff --git a/docs/adr/0012-finalize-run-artifact-layout.md b/docs/adr/0012-finalize-run-artifact-layout.md new file mode 100644 index 000000000..5f617506e --- /dev/null +++ b/docs/adr/0012-finalize-run-artifact-layout.md @@ -0,0 +1,166 @@ +# 12. Finalize run artifact layout at the timestamp bundle + +Date: 2026-06-30 + +## Status + +Accepted + +Supersedes the target/variant folder fan-out portions of +[ADR 0009](0009-eval-path-result-identity-and-default-experiment.md) and +[ADR 0006](0006-separate-experiments-from-eval-definitions.md). Extends +[ADR 0011](0011-result-output-artifact-contract.md), which keeps result output +run-centric and manifest-first. + +## Context + +AgentV now treats the timestamped result directory as the run bundle boundary: + +```text +.agentv/results/// +``` + +Earlier same-week decisions used target and variant folders below the timestamp +to avoid sidecar collisions in multi-target runs. The implementation has since +settled on allocated per-row result directories with readable test-id prefixes +and short hash suffixes. That allocation already solves collisions without +making target, model, variant, suite, or test IDs path dimensions. + +The relevant implementation points are: + +- `apps/cli/src/commands/eval/result-layout.ts` creates default run roots as + `.agentv/results///` and keeps `index.jsonl` as the + manifest filename. +- `packages/core/src/evaluation/run-artifacts.ts` writes `summary.json`, + `index.jsonl`, and per-result sidecars under allocated `result_dir` folders + such as `--/run-1/`. +- `apps/cli/src/commands/results/manifest.ts`, + `apps/cli/src/commands/results/serve.ts`, and + `packages/core/src/evaluation/results-repo.ts` consume explicit manifest + fields such as `result_dir`, `summary_path`, `grading_path`, `metrics_path`, + and `transcript_path` instead of deriving sidecar locations from directory + names. + +## Decision + +New AgentV runs write one run bundle at: + +```text +.agentv/results/// + summary.json + index.jsonl + tags.json # optional mutable overlay + / + summary.json + test/ # optional generated test bundle + run-1/ + result.json + grading.json + metrics.json + timing.json + transcript.jsonl + transcript-raw.jsonl + outputs/ + run-2/ + ... +``` + +Do not add `target`, `model`, `variant`, or `cases` as required folders below +or above ``. Target, model, provider, variant, eval path, suite, and +test identity are metadata. They belong in root `summary.json.metadata` for +run-level facts and in `index.jsonl` rows for row-level filtering and artifact +discovery. + +`index.jsonl` remains the filename for the run manifest/result index. The name +is established across CLI, Dashboard, result repo sync, compare, trend, and +adapter code. Renaming it would create churn without improving the contract. +Documentation should call it the run manifest or result index where that role is +clearer. + +`result_dir` values are opaque run-local allocations. Writers should keep them +readable when possible, using a safe test-id or slug prefix plus a UUID/hash-like +suffix, but consumers must not parse identity from those names. Consumers must +resolve ordinary sidecars through explicit `index.jsonl` fields such as: + +- `result_dir` +- `summary_path` +- `grading_path` +- `timing_path` +- `metrics_path` +- `transcript_path` +- `transcript_raw_path` +- `answer_path` +- `test_dir` + +## Compatibility + +Legacy bundles that already contain target-folder manifests remain readable. +Readers may discover nested `index.jsonl` files when a run root has no direct +manifest, and they must continue to honor legacy `index.jsonl` rows whose +explicit paths point into old target-folder layouts. Do not move old artifacts +as part of this decision. + +When a root `index.jsonl` exists, it is the authoritative manifest for that run +directory. Nested target-folder manifests are legacy compatibility input, not a +new writer contract. + +## Consequences + +Positive: + +- New run bundles have one obvious root manifest and summary. +- Dashboard and results-repo listings can use root `summary.json.metadata` + fields such as `targets` without walking per-result rows for basic run facts. +- Multi-target and variant rows still avoid filesystem collisions through + allocated result directories. +- Target/model comparisons stay a query over run and row metadata instead of a + storage hierarchy. + +Negative: + +- Humans cannot browse target folders under a timestamp. They must use + `summary.json`, `index.jsonl`, Dashboard filters, or compare tooling. +- Some accepted same-week ADR text now requires this superseding ADR for the + final layout. + +## Alternatives Considered + +### Target or model folders under the timestamp + +Rejected. Target/model folders make storage look semantic and encourage readers +to infer identity from paths. They also create needless nesting for the +single-target case and become awkward when target, provider, model, variant, and +runtime policy are all useful comparison dimensions. + +### Target folders above the timestamp + +Rejected. Moving target above timestamp fragments one run invocation into +multiple run roots and makes run-level summary metadata harder to define. + +### A `cases/` parent folder + +Rejected. `index.jsonl` already distinguishes control-plane files from +per-result sidecars. Adding `cases/` would be a cosmetic migration with no +current reader or writer need. + +### Rename `index.jsonl` + +Rejected. The file acts as the run manifest/result index, but the established +filename is portable and already wired through CLI, Dashboard, result repo, and +adapter code. + +### Add `internal/` now + +Rejected for artifact-format v1. A future artifact-format v2 migration may add +an `internal/` directory for machine-facing caches or implementation details, +but v1 keeps canonical files at the run root and per-result allocations under +explicit manifest paths. + +## Non-Goals + +- Moving or rewriting existing target-folder artifacts. +- Renaming `index.jsonl`. +- Defining a new result database or derived Dashboard index. +- Finalizing a full run-level model metadata schema. Root `summary.json.metadata` + already carries `targets`; richer provider/model fields can be added + additively when the Dashboard run-list work needs them. From 24b99123aaca4d1d81435c078329eed0b5aa44bd Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 14:55:28 +0200 Subject: [PATCH 2/2] test: fix target layout ci expectations --- apps/cli/src/commands/eval/run-eval.ts | 4 +--- apps/cli/test/commands/eval/bundle.test.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 9baccfc85..a295c43e5 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -2401,9 +2401,7 @@ export async function runEvalCommand( const indexPath = path.join(runDir, RESULT_INDEX_FILENAME); console.log(`Artifact bundle updated: ${runDir}`); console.log(` Run manifest: ${indexPath}`); - console.log( - ` Per-test artifacts: ${runDir} (${allResults.length} new test directories)`, - ); + console.log(` Per-test artifacts: ${runDir} (${allResults.length} new test directories)`); console.log(` Summary: ${summaryPath}`); } else { const { testArtifactDir, summaryPath, indexPath } = await writeArtifactsFromResults( diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 2e6f3e8fa..c205e8f41 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -166,7 +166,7 @@ tests: ../data/cases.yaml expect(run.exitCode).toBe(0); expect(run.stdout).toContain('RESULT: PASS'); - await expectFileExists(path.join(bundleDir, 'run', 'inherited', 'index.jsonl')); + await expectFileExists(path.join(bundleDir, 'run', 'index.jsonl')); }, 60_000); it('reports unbundleable workspace references with their eval location', async () => {