From 1270e997b927f186f865afac795ade13e3ed456e Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 01:21:41 +0200 Subject: [PATCH] fix(results): restore index jsonl run manifest --- .github/workflows/ci.yml | 83 ++++++++++++++++- .github/workflows/evals.yml | 2 +- CONCEPTS.md | 4 +- README.md | 6 +- ROADMAP.md | 2 +- STRATEGY.md | 2 +- apps/cli/src/cli.ts | 2 +- apps/cli/src/commands/eval/commands/run.ts | 4 +- apps/cli/src/commands/eval/result-layout.ts | 61 ++---------- apps/cli/src/commands/eval/run-eval.ts | 2 +- apps/cli/src/commands/grade/index.ts | 2 +- apps/cli/src/commands/inspect/score.ts | 2 +- apps/cli/src/commands/pipeline/bench.ts | 2 +- apps/cli/src/commands/results/delete-run.ts | 5 +- apps/cli/src/commands/results/export.ts | 7 +- apps/cli/src/commands/results/validate.ts | 10 +- .../cli/test/commands/compare/compare.test.ts | 32 +++---- apps/cli/test/commands/eval/aggregate.test.ts | 10 +- .../commands/eval/artifact-writer.test.ts | 7 +- apps/cli/test/commands/eval/bundle.test.ts | 2 +- .../test/commands/eval/pipeline/bench.test.ts | 8 +- .../eval/pipeline/pipeline-e2e.test.ts | 2 +- .../test/commands/eval/result-layout.test.ts | 41 ++++----- apps/cli/test/commands/eval/run-cache.test.ts | 6 +- .../commands/grade/grade-prepared.test.ts | 14 +-- .../cli/test/commands/results/combine.test.ts | 4 +- apps/cli/test/commands/results/export.test.ts | 6 +- .../results/remote-auto-export.test.ts | 18 ++-- apps/cli/test/commands/results/serve.test.ts | 24 ++--- apps/cli/test/commands/results/shared.test.ts | 2 +- .../test/commands/results/validate.test.ts | 4 +- apps/cli/test/commands/runs/rerun.test.ts | 4 +- apps/cli/test/commands/trend/trend.test.ts | 14 +-- apps/cli/test/eval.integration.test.ts | 10 +- apps/cli/test/unit/retry-errors.test.ts | 22 ++--- .../src/components/StopRunButton.tsx | 2 +- .../docs/docs/evaluation/experiments.mdx | 6 +- .../docs/docs/evaluation/running-evals.mdx | 30 +++--- .../docs/docs/getting-started/quickstart.mdx | 2 +- .../content/docs/docs/guides/autoresearch.mdx | 4 +- .../docs/docs/guides/benchmark-provenance.mdx | 2 +- .../content/docs/docs/guides/human-review.mdx | 12 +-- .../guides/skill-improvement-workflow.mdx | 4 +- .../docs/docs/reference/result-artifacts.mdx | 7 -- .../src/content/docs/docs/tools/compare.mdx | 16 ++-- .../src/content/docs/docs/tools/dashboard.mdx | 8 +- .../src/content/docs/docs/tools/inspect.mdx | 4 +- .../src/content/docs/docs/tools/prepare.mdx | 2 +- .../src/content/docs/docs/tools/results.mdx | 40 ++++---- .../web/src/content/docs/docs/tools/trend.mdx | 12 +-- .../docs/docs/tools/wip-checkpoints.mdx | 2 +- .../0011-result-output-artifact-contract.md | 5 - packages/core/src/evaluation/evaluate.ts | 2 +- .../core/src/evaluation/result-row-schema.ts | 4 +- .../src/evaluation/results-repo-cache.test.ts | 4 +- packages/core/src/evaluation/results-repo.ts | 32 +++---- packages/core/src/evaluation/run-artifacts.ts | 20 ++-- packages/core/src/index.ts | 2 - .../core/test/evaluation/results-repo.test.ts | 92 ++++--------------- skills-data/agentv-eval-writer/SKILL.md | 10 +- 60 files changed, 332 insertions(+), 418 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f632ccb5..05ade3c1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,7 @@ jobs: ' - name: Upload build artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: agentv-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} path: ${{ runner.temp }}/agentv-build-artifact/ @@ -213,6 +213,7 @@ jobs: name: Validate Evals runs-on: ubuntu-latest timeout-minutes: 15 + needs: build steps: - uses: actions/checkout@v6 - name: Setup Bun @@ -223,8 +224,84 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile - - name: Build - run: bun run build + - name: Download build artifact + uses: actions/download-artifact@v8 + with: + name: agentv-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: ${{ runner.temp }}/agentv-build-artifact + + - name: Restore build artifact + env: + ARTIFACT_DIR: ${{ runner.temp }}/agentv-build-artifact + EXPECTED_COMMIT_SHA: ${{ github.sha }} + EXPECTED_RUNNER_ARCH: ${{ runner.arch }} + EXPECTED_RUNNER_OS: ${{ runner.os }} + run: | + set -euo pipefail + + bun -e ' + import { createHash } from "node:crypto"; + import { Buffer } from "node:buffer"; + import { cpSync, existsSync, mkdirSync, rmSync } from "node:fs"; + import path from "node:path"; + + const artifactDir = process.env.ARTIFACT_DIR; + if (!artifactDir) { + throw new Error("ARTIFACT_DIR is required"); + } + + const manifestPath = path.join(artifactDir, "manifest.json"); + if (!existsSync(manifestPath)) { + throw new Error(`Build artifact manifest is missing: ${manifestPath}`); + } + + const manifest = await Bun.file(manifestPath).json(); + const rootPackageJson = await Bun.file("package.json").json(); + const lockfile = await Bun.file("bun.lock").arrayBuffer(); + const lockHash = createHash("sha256") + .update(Buffer.from(lockfile)) + .digest("hex"); + + const expected = { + commit_sha: process.env.EXPECTED_COMMIT_SHA, + bun_lock_sha256: lockHash, + runner_os: process.env.EXPECTED_RUNNER_OS, + runner_arch: process.env.EXPECTED_RUNNER_ARCH, + bun_version_spec: rootPackageJson.packageManager ?? null, + bun_version: Bun.version, + }; + + for (const [key, value] of Object.entries(expected)) { + if (manifest[key] !== value) { + throw new Error( + `Build artifact manifest mismatch for ${key}: expected ${value}, got ${manifest[key]}`, + ); + } + } + + const requiredPaths = [ + "packages/core/dist", + "packages/sdk/dist", + "apps/cli/dist", + "apps/dashboard/dist", + ]; + const includedPaths = new Set(manifest.included_paths ?? []); + + for (const relativePath of requiredPaths) { + if (!includedPaths.has(`${relativePath}/**`)) { + throw new Error(`Build artifact manifest does not include ${relativePath}/**`); + } + + const source = path.join(artifactDir, relativePath); + if (!existsSync(source)) { + throw new Error(`Build artifact path is missing: ${source}`); + } + + rmSync(relativePath, { recursive: true, force: true }); + mkdirSync(path.dirname(relativePath), { recursive: true }); + cpSync(source, relativePath, { recursive: true }); + } + ' - name: Check evals directories have eval files run: bun scripts/validate-eval-dirs.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index c7a4290ee..85a58e583 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -105,7 +105,7 @@ jobs: - name: Upload eval artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eval-results-${{ github.run_id }} path: | diff --git a/CONCEPTS.md b/CONCEPTS.md index 5443552da..92c2136b4 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -24,11 +24,11 @@ Shared domain vocabulary for this project — entities, named processes, and sta **Workspace** — The task environment an eval prepares for the agent: repositories, templates, fixture files, and lifecycle hooks. It is not prompt input; use `input` for instructions and `workspace.repos[]` for multi-repo workspaces the agent can inspect or modify through tools. -**Run manifest** — The root `run_manifest.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `task_dir`, `summary_path`, and `grading_path`. +**Run manifest** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `task_dir`, `summary_path`, and `grading_path`. **Result source identity** — The stable source identity for a result row: repo-relative `eval_path`, `test_id`, and `target`. `suite` and `name` are display metadata, not storage or routing identity. -**Result directory** — The `result_dir` field in a `run_manifest.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs. Consumers discover it from `run_manifest.jsonl` and must not infer it from suite names, display names, test IDs, or targets. +**Result directory** — The `result_dir` field in a `index.jsonl` row. It is a run-local directory allocation for that row's sidecars and outputs. Consumers discover it from `index.jsonl` and must not infer it from suite names, display names, test IDs, or targets. **Artifact sidecar** — A file beside or below a result directory that provides evidence for a result, such as `summary.json`, `grading.json`, `result.json`, transcripts, logs, or outputs. Sidecars are evidence, not the primary discovery mechanism for a run. diff --git a/README.md b/README.md index 50da679c3..0d8688b7e 100644 --- a/README.md +++ b/README.md @@ -73,14 +73,14 @@ agentv eval evals/my-eval.yaml **5. Compare results across targets:** ```bash -agentv compare .agentv/results/default//run_manifest.jsonl +agentv compare .agentv/results/default//index.jsonl ``` ## Output formats ```bash -agentv eval evals/my-eval.yaml --output ./run # writes ./run/run_manifest.jsonl -cat ./run/run_manifest.jsonl # JSONL results for scripts/CI +agentv eval evals/my-eval.yaml --output ./run # writes ./run/index.jsonl +cat ./run/index.jsonl # JSONL results for scripts/CI ``` ## TypeScript SDK diff --git a/ROADMAP.md b/ROADMAP.md index 7096fd810..e8d9f1b49 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,7 +15,7 @@ This roadmap translates [STRATEGY.md](STRATEGY.md) into the next few product pha ## Phase 1: Finish the artifact and local inspection foundation -- Keep the canonical handoff surface centered on completed run bundles, `run_manifest.jsonl`, grading/timing/metrics artifacts, normalized transcripts, and optional `external_trace` link metadata. +- Keep the canonical handoff surface centered on completed run bundles, `index.jsonl`, grading/timing/metrics artifacts, normalized transcripts, and optional `external_trace` link metadata. - Finish the vendor-neutral local export seams that let completed runs be re-read, compared, exported, and attached to non-Phoenix adapters without vendor-specific logic in core. - Keep OTLP/OpenInference mapping generic and reusable before building backend-specific upload or import paths. diff --git a/STRATEGY.md b/STRATEGY.md index 41819a985..5f52388d9 100644 --- a/STRATEGY.md +++ b/STRATEGY.md @@ -21,7 +21,7 @@ AgentV stays repo-native and workspace-native: it runs or imports evaluations ar - **Repo-native eval success** - Share of dogfood and example eval flows that run against real workspaces, hooks, repo materialization, or imported artifacts without extra infrastructure; measured by CI and manual UAT on canonical suites. - **Time to inspect a run** - Time from completed `agentv eval` to usable local review, compare, or report output from the canonical run bundle; measured through CLI and Dashboard/report workflows. -- **Artifact portability coverage** - Share of integrations and follow-on workflows that consume `run_manifest.jsonl`, `summary.json`, trace sidecars, or imported run bundles instead of bespoke stores; measured by adapter smoke tests, docs, and example coverage. +- **Artifact portability coverage** - Share of integrations and follow-on workflows that consume `index.jsonl`, `summary.json`, trace sidecars, or imported run bundles instead of bespoke stores; measured by adapter smoke tests, docs, and example coverage. - **Git-backed results reliability** - Success rate for publish, sync, resume, and WIP checkpoint flows across local branches and dedicated results repos; measured by integration tests and manual end-to-end verification. ## Tracks diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index c2b50fa0e..9c7946ed1 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -6,7 +6,7 @@ import { runCli } from './index.js'; // Forward SIGINT/SIGTERM to spawned provider subprocesses before exiting. // Without this, Dashboard's `child.kill('SIGTERM')` against the CLI orphans // any in-flight `claude`/`codex`/`pi`/`copilot` subprocess. The partial -// `run_manifest.jsonl` is already row-by-row durable, so finished tests survive. +// `index.jsonl` is already row-by-row durable, so finished tests survive. // // First signal: kill children, exit with the conventional 128+signal code. // Second signal within the same process: hard-exit so a hung child cannot diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 97458dbf3..857383c6c 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -52,12 +52,12 @@ export const evalRunCommand = command({ long: 'output', short: 'o', description: - 'Run artifact directory (writes run_manifest.jsonl, summary.json, and per-case artifacts)', + 'Run artifact directory (writes index.jsonl, summary.json, and per-case artifacts)', }), outputFormat: option({ type: optional(string), long: 'output-format', - description: '[Removed] Run directories always write run_manifest.jsonl', + description: '[Removed] Run directories always write index.jsonl', }), experiment: option({ type: optional(string), diff --git a/apps/cli/src/commands/eval/result-layout.ts b/apps/cli/src/commands/eval/result-layout.ts index ce53cde6c..4cc8813ae 100644 --- a/apps/cli/src/commands/eval/result-layout.ts +++ b/apps/cli/src/commands/eval/result-layout.ts @@ -1,15 +1,7 @@ -import { type Dirent, existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { type Dirent, existsSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; -export const RESULT_MANIFEST_FILENAME = 'run_manifest.jsonl'; -export const LEGACY_RESULT_INDEX_FILENAME = 'index.jsonl'; -// Backward-compatible export name retained for existing callers. New writes use -// the row-level run manifest filename. -export const RESULT_INDEX_FILENAME = RESULT_MANIFEST_FILENAME; -export const RESULT_MANIFEST_FILENAMES = [ - RESULT_MANIFEST_FILENAME, - LEGACY_RESULT_INDEX_FILENAME, -] as const; +export const RESULT_INDEX_FILENAME = 'index.jsonl'; export const RUN_SUMMARY_FILENAME = 'summary.json'; export const RESULTS_DIRNAME = 'results'; export const DEFAULT_EXPERIMENT_NAME = 'default'; @@ -73,48 +65,13 @@ export function resolveRunIndexPath(runDir: string): string { } export function isRunManifestPath(filePath: string): boolean { - return RESULT_MANIFEST_FILENAMES.includes( - path.basename(filePath) as (typeof RESULT_MANIFEST_FILENAMES)[number], - ); -} - -function safeSummaryManifestPath(runDir: string, manifestPath: unknown): string | undefined { - if (typeof manifestPath !== 'string' || manifestPath.trim().length === 0) { - return undefined; - } - if (path.isAbsolute(manifestPath)) { - return undefined; - } - const normalized = path.normalize(manifestPath); - if (normalized.startsWith('..') || path.isAbsolute(normalized)) { - return undefined; - } - return path.join(runDir, normalized); -} - -function resolveSummaryManifestPath(runDir: string): string | undefined { - try { - const summary = JSON.parse(readFileSync(path.join(runDir, RUN_SUMMARY_FILENAME), 'utf8')) as { - manifest_path?: unknown; - }; - const manifestPath = safeSummaryManifestPath(runDir, summary.manifest_path); - return manifestPath && existsSync(manifestPath) ? manifestPath : undefined; - } catch { - return undefined; - } + return path.basename(filePath) === RESULT_INDEX_FILENAME; } export function resolveExistingRunPrimaryPath(runDir: string): string | undefined { - const summaryManifestPath = resolveSummaryManifestPath(runDir); - if (summaryManifestPath) { - return summaryManifestPath; - } - - for (const filename of RESULT_MANIFEST_FILENAMES) { - const manifestPath = path.join(runDir, filename); - if (existsSync(manifestPath)) { - return manifestPath; - } + const indexPath = resolveRunIndexPath(runDir); + if (existsSync(indexPath)) { + return indexPath; } return undefined; @@ -178,9 +135,7 @@ export function resolveWorkspaceOrFilePath(filePath: string): string { `Result workspace contains multiple run manifests; pass one bundle directory or manifest: ${filePath}`, ); } - throw new Error( - `Result workspace is missing ${RESULT_MANIFEST_FILENAME} or legacy ${LEGACY_RESULT_INDEX_FILENAME}: ${filePath}`, - ); + throw new Error(`Result workspace is missing ${RESULT_INDEX_FILENAME}: ${filePath}`); } export function resolveRunManifestPath(filePath: string): string { @@ -190,7 +145,7 @@ export function resolveRunManifestPath(filePath: string): string { if (!isRunManifestPath(filePath)) { throw new Error( - `Expected a run workspace directory or ${RESULT_MANIFEST_FILENAME} manifest (legacy ${LEGACY_RESULT_INDEX_FILENAME} is also readable): ${filePath}`, + `Expected a run workspace directory or ${RESULT_INDEX_FILENAME} manifest: ${filePath}`, ); } diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index d801aba20..956648073 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -140,7 +140,7 @@ interface NormalizedOptions { readonly keepWorkspaces: boolean; /** Removed: use --output instead */ readonly artifacts?: string; - /** Removed: the run directory always uses run_manifest.jsonl */ + /** Removed: the run directory always uses index.jsonl */ readonly outputFormat?: string; readonly graderTarget?: string; readonly model?: string; diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index 6d192e838..7a49f36b4 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -625,7 +625,7 @@ export const gradeCommand = command({ type: optional(string), long: 'output', short: 'o', - description: 'Run artifact directory (writes run_manifest.jsonl and per-test artifacts)', + description: 'Run artifact directory (writes index.jsonl and per-test artifacts)', }), response: option({ type: optional(string), diff --git a/apps/cli/src/commands/inspect/score.ts b/apps/cli/src/commands/inspect/score.ts index 9ed390668..75244e827 100644 --- a/apps/cli/src/commands/inspect/score.ts +++ b/apps/cli/src/commands/inspect/score.ts @@ -379,7 +379,7 @@ export const traceScoreCommand = command({ ); if (!hasTrace) { console.error( - `${c.red}Error:${c.reset} Source lacks trace metrics. Use an OTLP trace export via ${c.bold}--otel-file${c.reset} or a run manifest with summary metrics in ${c.bold}run_manifest.jsonl${c.reset}.`, + `${c.red}Error:${c.reset} Source lacks trace metrics. Use an OTLP trace export via ${c.bold}--otel-file${c.reset} or a run manifest with summary metrics in ${c.bold}index.jsonl${c.reset}.`, ); process.exit(1); } diff --git a/apps/cli/src/commands/pipeline/bench.ts b/apps/cli/src/commands/pipeline/bench.ts index 415339da6..325226c9d 100644 --- a/apps/cli/src/commands/pipeline/bench.ts +++ b/apps/cli/src/commands/pipeline/bench.ts @@ -6,7 +6,7 @@ * * Writes: * - /grading.json (per-test grading breakdown) - * - run_manifest.jsonl (one line per test) + * - index.jsonl (one line per test) * - summary.json (aggregate statistics) */ import { existsSync } from 'node:fs'; diff --git a/apps/cli/src/commands/results/delete-run.ts b/apps/cli/src/commands/results/delete-run.ts index 125f5bfc2..f398b5c7d 100644 --- a/apps/cli/src/commands/results/delete-run.ts +++ b/apps/cli/src/commands/results/delete-run.ts @@ -12,7 +12,6 @@ import { existsSync, rmSync } from 'node:fs'; import path from 'node:path'; import { - LEGACY_RESULT_INDEX_FILENAME, RESULT_INDEX_FILENAME, isRunManifestPath, relativeRunPathFromCwd, @@ -35,9 +34,7 @@ export interface DeleteRunResult extends DeleteRunTarget { function assertLocalRunManifest(cwd: string, manifestPath: string, runId: string): DeleteRunTarget { const resolvedManifestPath = path.resolve(manifestPath); if (!isRunManifestPath(resolvedManifestPath)) { - throw new Error( - `Expected a run workspace directory or ${RESULT_INDEX_FILENAME} manifest (legacy ${LEGACY_RESULT_INDEX_FILENAME} is also readable)`, - ); + throw new Error(`Expected a run workspace directory or ${RESULT_INDEX_FILENAME} manifest`); } const runDir = path.dirname(resolvedManifestPath); diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts index f6d3b0dd1..ba21b0d8e 100644 --- a/apps/cli/src/commands/results/export.ts +++ b/apps/cli/src/commands/results/export.ts @@ -5,7 +5,7 @@ * Output structure: * / * summary.json — run aggregate scores, metadata, and timing - * run_manifest.jsonl — per-test manifest with artifact pointers + * index.jsonl — per-test manifest with artifact pointers * / * summary.json — per-case aggregate * run-1/result.json — per-run result @@ -29,7 +29,6 @@ import type { EvaluationResult, ExportDuplicatePolicy, IndexArtifactEntry } from import { parseJsonlResults, writeArtifactsFromResults } from '../eval/artifact-writer.js'; import { - LEGACY_RESULT_INDEX_FILENAME, RESULT_INDEX_FILENAME, isReservedResultsNamespace, isRunManifestPath, @@ -71,9 +70,7 @@ export async function exportResults( */ export function deriveOutputDir(cwd: string, sourceFile: string): string { if (!isRunManifestPath(sourceFile)) { - throw new Error( - `Expected a run manifest named ${RESULT_INDEX_FILENAME} (legacy ${LEGACY_RESULT_INDEX_FILENAME} is also readable): ${sourceFile}`, - ); + throw new Error(`Expected a run manifest named ${RESULT_INDEX_FILENAME}: ${sourceFile}`); } const runDir = path.dirname(sourceFile); diff --git a/apps/cli/src/commands/results/validate.ts b/apps/cli/src/commands/results/validate.ts index f6047c459..0f6a5a1ee 100644 --- a/apps/cli/src/commands/results/validate.ts +++ b/apps/cli/src/commands/results/validate.ts @@ -4,7 +4,7 @@ * * Checks: * 1. Directory follows the `.agentv/results//` naming convention - * 2. run_manifest.jsonl exists and each line has required fields + * 2. index.jsonl exists and each line has required fields * 3. Per-case summary.json exists for every entry in the index * 4. Per-run result.json and grading.json exist for every materialized trial * 5. summary.json exists @@ -20,11 +20,7 @@ import path from 'node:path'; import { command, positional, string } from 'cmd-ts'; -import { - LEGACY_RESULT_INDEX_FILENAME, - RESULT_INDEX_FILENAME, - resolveExistingRunPrimaryPath, -} from '../eval/result-layout.js'; +import { RESULT_INDEX_FILENAME, resolveExistingRunPrimaryPath } from '../eval/result-layout.js'; // ── Types ──────────────────────────────────────────────────────────────── @@ -108,7 +104,7 @@ function checkIndexJsonl(runDir: string): { diagnostics: Diagnostic[]; entries: if (!indexPath || !existsSync(indexPath)) { diagnostics.push({ severity: 'error', - message: `${RESULT_INDEX_FILENAME} is missing (legacy ${LEGACY_RESULT_INDEX_FILENAME} is also readable)`, + message: `${RESULT_INDEX_FILENAME} is missing`, }); return { diagnostics, entries }; } diff --git a/apps/cli/test/commands/compare/compare.test.ts b/apps/cli/test/commands/compare/compare.test.ts index b9b38b40f..334b0a2d2 100644 --- a/apps/cli/test/commands/compare/compare.test.ts +++ b/apps/cli/test/commands/compare/compare.test.ts @@ -27,10 +27,10 @@ describe('compare command', () => { }); describe('loadJsonlResults', () => { - it('should load run_manifest.jsonl manifests from a run workspace', () => { + it('should load index.jsonl manifests from a run workspace', () => { const runDir = path.join(tempDir, 'eval_2026-03-24T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); - const filePath = path.join(runDir, 'run_manifest.jsonl'); + const filePath = path.join(runDir, 'index.jsonl'); writeFileSync( filePath, '{"test_id": "case-1", "score": 0.8, "grading_path": "case-1/grading.json", "timing_path": "case-1/timing.json"}\n{"test_id": "case-2", "score": 0.9, "grading_path": "case-2/grading.json", "timing_path": "case-2/timing.json"}\n', @@ -44,17 +44,13 @@ describe('compare command', () => { ]); }); - it('should prefer summary.json manifest_path over a legacy index.jsonl in the same workspace', () => { + it('should resolve the row index from summary.json manifest_path in a run workspace', () => { const runDir = path.join(tempDir, 'eval_2026-03-24T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); - writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), - '{"test_id": "canonical", "score": 0.8}\n', - ); - writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id": "legacy", "score": 0.1}\n'); + writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id": "canonical", "score": 0.8}\n'); writeFileSync( path.join(runDir, 'summary.json'), - `${JSON.stringify({ manifest_path: 'run_manifest.jsonl' })}\n`, + `${JSON.stringify({ manifest_path: 'index.jsonl' })}\n`, ); const results = loadJsonlResults(runDir); @@ -62,21 +58,21 @@ describe('compare command', () => { expect(results).toEqual([{ testId: 'canonical', score: 0.8 }]); }); - it('should still accept legacy index.jsonl manifests directly', () => { + it('should accept canonical index.jsonl manifests directly', () => { const runDir = path.join(tempDir, 'eval_2026-03-24T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); const filePath = path.join(runDir, 'index.jsonl'); - writeFileSync(filePath, '{"test_id": "legacy-case", "score": 0.8}\n'); + writeFileSync(filePath, '{"test_id": "case-1", "score": 0.8}\n'); const results = loadJsonlResults(filePath); - expect(results).toEqual([{ testId: 'legacy-case', score: 0.8 }]); + expect(results).toEqual([{ testId: 'case-1', score: 0.8 }]); }); - it('should handle empty lines in run_manifest.jsonl manifests', () => { + it('should handle empty lines in index.jsonl manifests', () => { const runDir = path.join(tempDir, 'eval_2026-03-24T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); - const filePath = path.join(runDir, 'run_manifest.jsonl'); + const filePath = path.join(runDir, 'index.jsonl'); writeFileSync( filePath, '{"test_id": "case-1", "score": 0.8, "grading_path": "case-1/grading.json", "timing_path": "case-1/timing.json"}\n\n{"test_id": "case-2", "score": 0.9, "grading_path": "case-2/grading.json", "timing_path": "case-2/timing.json"}\n', @@ -116,7 +112,7 @@ describe('compare command', () => { writeFileSync(filePath, '{"test_id": "case-1", "score": 0.8}\n'); expect(() => loadJsonlResults(filePath)).toThrow( - 'Expected a run workspace directory or run_manifest.jsonl manifest', + 'Expected a run workspace directory or index.jsonl manifest', ); }); }); @@ -219,10 +215,10 @@ describe('compare command', () => { expect(groups.get('a')).toHaveLength(2); }); - it('should group records from run_manifest.jsonl manifests', () => { + it('should group records from index.jsonl manifests', () => { const runDir = path.join(tempDir, 'eval_2026-03-24T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); - const filePath = path.join(runDir, 'run_manifest.jsonl'); + const filePath = path.join(runDir, 'index.jsonl'); writeFileSync( filePath, [ @@ -242,7 +238,7 @@ describe('compare command', () => { writeFileSync(filePath, '{"test_id": "t1", "score": 0.8, "target": "a"}\n'); expect(() => loadCombinedResults(filePath)).toThrow( - 'Expected a run workspace directory or run_manifest.jsonl manifest', + 'Expected a run workspace directory or index.jsonl manifest', ); }); }); diff --git a/apps/cli/test/commands/eval/aggregate.test.ts b/apps/cli/test/commands/eval/aggregate.test.ts index 7c06d44dd..165e43980 100644 --- a/apps/cli/test/commands/eval/aggregate.test.ts +++ b/apps/cli/test/commands/eval/aggregate.test.ts @@ -205,7 +205,7 @@ describe('aggregateRunDir', () => { rmSync(tmpDir, { recursive: true, force: true }); }); - it('reads run_manifest.jsonl, deduplicates, and writes summary.json with timing rollups', async () => { + it('reads index.jsonl, deduplicates, and writes summary.json with timing rollups', async () => { writeJsonlIndex(tmpDir, [ { testId: 'a', target: 'x', score: 0.1, executionStatus: 'execution_error' }, { testId: 'a', target: 'x', score: 0.9, executionStatus: 'ok' }, @@ -224,12 +224,12 @@ describe('aggregateRunDir', () => { expect(summary.timing.total_tokens).toBeGreaterThanOrEqual(0); }); - it('falls back to legacy index.jsonl bundles', async () => { + it('reads canonical index.jsonl bundles', async () => { writeJsonlIndex( tmpDir, [ - { testId: 'legacy-a', target: 'x', score: 0.9, executionStatus: 'ok' }, - { testId: 'legacy-b', target: 'x', score: 0.8, executionStatus: 'ok' }, + { testId: 'case-a', target: 'x', score: 0.9, executionStatus: 'ok' }, + { testId: 'case-b', target: 'x', score: 0.8, executionStatus: 'ok' }, ], 'index.jsonl', ); @@ -239,7 +239,7 @@ describe('aggregateRunDir', () => { const summary = JSON.parse(readFileSync(result.summaryPath, 'utf8')); expect(summary.manifest_path).toBe(RESULT_INDEX_FILENAME); - expect(summary.metadata.tests_run).toEqual(['legacy-a', 'legacy-b']); + expect(summary.metadata.tests_run).toEqual(['case-a', 'case-b']); }); it('uses last entry for duplicates in benchmark stats', async () => { diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 34076c949..8e736131f 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -896,7 +896,7 @@ describe('writeArtifactsFromResults', () => { await rm(testDir, { recursive: true, force: true }).catch(() => undefined); }); - it('writes summary, run manifest, and per-run artifact files', async () => { + it('writes summary, index.jsonl, and per-run artifact files', async () => { const results = [ makeResult({ testId: 'alpha', score: 0.9, durationMs: 5000 }), makeResult({ testId: 'beta', score: 0.6, durationMs: 8000 }), @@ -906,8 +906,9 @@ describe('writeArtifactsFromResults', () => { evalFile: 'my-eval.yaml', }); - expect(path.basename(paths.indexPath)).toBe('run_manifest.jsonl'); - expect(existsSync(path.join(testDir, 'index.jsonl'))).toBe(false); + expect(path.basename(paths.indexPath)).toBe('index.jsonl'); + expect(paths.indexPath).toBe(path.join(testDir, 'index.jsonl')); + expect(existsSync(paths.indexPath)).toBe(true); const indexLines = await readIndexLines(paths.indexPath); expect(indexLines).toHaveLength(2); const alphaRowDir = expectRowDir(indexLines[0], 'alpha'); diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 98e87390a..2e6f3e8fa 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -166,7 +166,7 @@ tests: ../data/cases.yaml expect(run.exitCode).toBe(0); expect(run.stdout).toContain('RESULT: PASS'); - await expectFileExists(path.join(bundleDir, 'run', 'inherited', 'run_manifest.jsonl')); + await expectFileExists(path.join(bundleDir, 'run', 'inherited', 'index.jsonl')); }, 60_000); it('reports unbundleable workspace references with their eval location', async () => { diff --git a/apps/cli/test/commands/eval/pipeline/bench.test.ts b/apps/cli/test/commands/eval/pipeline/bench.test.ts index 3936f40ae..a52fe210e 100644 --- a/apps/cli/test/commands/eval/pipeline/bench.test.ts +++ b/apps/cli/test/commands/eval/pipeline/bench.test.ts @@ -76,7 +76,7 @@ describe('pipeline bench', () => { expect(grading.assertions.length).toBeGreaterThan(0); expect(grading.graders).toHaveLength(2); - const indexContent = await readFile(join(OUT_DIR, 'run_manifest.jsonl'), 'utf8'); + const indexContent = await readFile(join(OUT_DIR, 'index.jsonl'), 'utf8'); const lines = indexContent .trim() .split('\n') @@ -90,7 +90,7 @@ describe('pipeline bench', () => { expect(benchmark.run_summary['test-target']).toBeDefined(); }, 30_000); - it('propagates experiment from manifest to run_manifest.jsonl and summary.json', async () => { + it('propagates experiment from manifest to index.jsonl and summary.json', async () => { // Overwrite manifest with experiment field await writeFile( join(OUT_DIR, 'manifest.json'), @@ -106,7 +106,7 @@ describe('pipeline bench', () => { const { execa } = await import('execa'); await execa('bun', [CLI_ENTRY, 'pipeline', 'bench', OUT_DIR]); - const indexContent = await readFile(join(OUT_DIR, 'run_manifest.jsonl'), 'utf8'); + const indexContent = await readFile(join(OUT_DIR, 'index.jsonl'), 'utf8'); const entry = JSON.parse(indexContent.trim().split('\n')[0]); expect(entry.experiment).toBe('without_skills'); @@ -118,7 +118,7 @@ describe('pipeline bench', () => { const { execa } = await import('execa'); await execa('bun', [CLI_ENTRY, 'pipeline', 'bench', OUT_DIR]); - const indexContent = await readFile(join(OUT_DIR, 'run_manifest.jsonl'), 'utf8'); + const indexContent = await readFile(join(OUT_DIR, 'index.jsonl'), 'utf8'); const entry = JSON.parse(indexContent.trim().split('\n')[0]); expect(entry.experiment).toBeUndefined(); diff --git a/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts b/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts index 36f8dd7ee..fef9a62cb 100644 --- a/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts +++ b/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts @@ -61,7 +61,7 @@ describe('eval pipeline e2e', () => { expect(grading.graders).toHaveLength(2); expect(grading.summary.pass_rate).toBeGreaterThan(0); - const indexContent = await readFile(join(outDir, 'run_manifest.jsonl'), 'utf8'); + const indexContent = await readFile(join(outDir, 'index.jsonl'), 'utf8'); const indexLines = indexContent .trim() .split('\n') diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts index 3844b41b5..13b769668 100644 --- a/apps/cli/test/commands/eval/result-layout.test.ts +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -11,6 +11,7 @@ import { normalizeExperimentName, relativeRunPathFromCwd, resolveExistingRunPrimaryPath, + resolveRunManifestPath, } from '../../../src/commands/eval/result-layout.js'; describe('result layout', () => { @@ -47,48 +48,42 @@ describe('result layout', () => { ).toBe('default/2026-run'); }); - it('prefers the summary manifest_path when both manifest filenames exist', () => { + it('resolves the canonical index.jsonl file in a run directory', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); try { - writeFileSync(path.join(tempDir, RESULT_INDEX_FILENAME), '{"test_id":"new"}\n'); - writeFileSync(path.join(tempDir, 'index.jsonl'), '{"test_id":"legacy"}\n'); - writeFileSync( - path.join(tempDir, 'summary.json'), - `${JSON.stringify({ manifest_path: RESULT_INDEX_FILENAME })}\n`, - ); + const indexPath = path.join(tempDir, RESULT_INDEX_FILENAME); + writeFileSync(indexPath, '{"test_id":"case"}\n'); - expect(resolveExistingRunPrimaryPath(tempDir)).toBe( - path.join(tempDir, RESULT_INDEX_FILENAME), - ); + expect(resolveExistingRunPrimaryPath(tempDir)).toBe(indexPath); + expect(resolveRunManifestPath(tempDir)).toBe(indexPath); } finally { rmSync(tempDir, { recursive: true, force: true }); } }); - it('falls back to legacy index.jsonl when no canonical manifest exists', () => { + it('discovers one canonical index.jsonl manifest per nested bundle', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); try { - writeFileSync(path.join(tempDir, 'index.jsonl'), '{"test_id":"legacy"}\n'); + const bundleDir = path.join(tempDir, 'default', '2026-run', 'target-a'); + mkdirSync(bundleDir, { recursive: true }); + writeFileSync(path.join(bundleDir, RESULT_INDEX_FILENAME), '{"test_id":"case"}\n'); - expect(resolveExistingRunPrimaryPath(tempDir)).toBe(path.join(tempDir, 'index.jsonl')); + expect(discoverRunManifestPaths(tempDir)).toEqual([ + path.join(bundleDir, RESULT_INDEX_FILENAME), + ]); } finally { rmSync(tempDir, { recursive: true, force: true }); } }); - it('discovers one manifest per nested bundle when both filenames exist', () => { - const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); + it('reports index.jsonl as the canonical missing run manifest name', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'agentv-result-layout-')); try { - const bundleDir = path.join(tempDir, 'default', '2026-run', 'target-a'); - mkdirSync(bundleDir, { recursive: true }); - writeFileSync(path.join(bundleDir, RESULT_INDEX_FILENAME), '{"test_id":"new"}\n'); - writeFileSync(path.join(bundleDir, 'index.jsonl'), '{"test_id":"legacy"}\n'); + mkdirSync(path.join(dir, 'nested')); - expect(discoverRunManifestPaths(tempDir)).toEqual([ - path.join(bundleDir, RESULT_INDEX_FILENAME), - ]); + expect(() => resolveRunManifestPath(dir)).toThrow('missing index.jsonl'); } finally { - rmSync(tempDir, { recursive: true, force: true }); + rmSync(dir, { recursive: true, force: true }); } }); }); diff --git a/apps/cli/test/commands/eval/run-cache.test.ts b/apps/cli/test/commands/eval/run-cache.test.ts index f746f522c..e8a327cbb 100644 --- a/apps/cli/test/commands/eval/run-cache.test.ts +++ b/apps/cli/test/commands/eval/run-cache.test.ts @@ -4,13 +4,13 @@ import path from 'node:path'; import { type RunCache, resolveRunCacheFile } from '../../../src/commands/eval/run-cache.js'; describe('resolveRunCacheFile', () => { - it('should resolve new directory-based cache to run_manifest.jsonl inside dir', () => { + it('should resolve new directory-based cache to index.jsonl inside dir', () => { const cache: RunCache = { lastRunDir: '/results/default/2026-03-24T00-00-00-000Z', timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/default/2026-03-24T00-00-00-000Z', 'run_manifest.jsonl'), + path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); @@ -29,7 +29,7 @@ describe('resolveRunCacheFile', () => { timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/default/2026-03-24T00-00-00-000Z', 'run_manifest.jsonl'), + path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index 8ece8973b..f4517adf6 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -169,7 +169,7 @@ describe('agentv grade prepared attempts', () => { workspace_path: path.join(preparedDir, 'workspace'), manifest_path: path.join(preparedDir, 'agentv_prepare.json'), output_dir: runDir, - index_path: path.join(runDir, 'run_manifest.jsonl'), + index_path: path.join(runDir, 'index.jsonl'), }); expect(await exists(targetMarker)).toBe(false); @@ -177,9 +177,7 @@ describe('agentv grade prepared attempts', () => { expect(graderPayload.workspace_path).toBe(path.join(preparedDir, 'workspace')); expect(graderPayload.file_changes).toContain('+manual edit'); - const row = JSON.parse( - (await readFile(path.join(runDir, 'run_manifest.jsonl'), 'utf8')).trim(), - ); + const row = JSON.parse((await readFile(path.join(runDir, 'index.jsonl'), 'utf8')).trim()); expect(row).toMatchObject({ test_id: 'case-1', target: 'codex', @@ -271,9 +269,7 @@ describe('agentv grade prepared attempts', () => { ); expect(await exists(targetMarker)).toBe(false); - const row = JSON.parse( - (await readFile(path.join(runDir, 'run_manifest.jsonl'), 'utf8')).trim(), - ); + const row = JSON.parse((await readFile(path.join(runDir, 'index.jsonl'), 'utf8')).trim()); expect(row.score).toBe(0); expect(row.scores[0]).toMatchObject({ name: 'expected-tool-sequence', @@ -380,9 +376,7 @@ describe('agentv grade prepared attempts', () => { }); expect(await exists(targetMarker)).toBe(false); - const row = JSON.parse( - (await readFile(path.join(runDir, 'run_manifest.jsonl'), 'utf8')).trim(), - ); + const row = JSON.parse((await readFile(path.join(runDir, 'index.jsonl'), 'utf8')).trim()); const answerPath = row.answer_path ?? row.response_path ?? row.output_path; expect(typeof answerPath).toBe('string'); expect((await readFile(path.join(runDir, answerPath), 'utf8')).trim()).toBe('done'); diff --git a/apps/cli/test/commands/results/combine.test.ts b/apps/cli/test/commands/results/combine.test.ts index 341bf4725..5d9642068 100644 --- a/apps/cli/test/commands/results/combine.test.ts +++ b/apps/cli/test/commands/results/combine.test.ts @@ -48,13 +48,13 @@ describe('results combine', () => { const runDir = path.join(tempDir, '.agentv', 'results', experiment, name); mkdirSync(path.join(runDir, 'demo', 'test-a'), { recursive: true }); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), toJsonl(...records.map((record) => ({ ...record, experiment }))), 'utf8', ); writeFileSync( path.join(runDir, 'summary.json'), - `${JSON.stringify({ manifest_path: 'run_manifest.jsonl' })}\n`, + `${JSON.stringify({ manifest_path: 'index.jsonl' })}\n`, 'utf8', ); writeFileSync(path.join(runDir, 'demo', 'test-a', 'grading.json'), '{"assertions":[]}\n'); diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts index 04ff971b1..88cfa4564 100644 --- a/apps/cli/test/commands/results/export.test.ts +++ b/apps/cli/test/commands/results/export.test.ts @@ -218,7 +218,7 @@ describe('results export', () => { rmSync(tempDir, { recursive: true, force: true }); }); - it('loadExportSource resolves run workspaces to run_manifest.jsonl', async () => { + it('loadExportSource resolves run workspaces to index.jsonl', async () => { const runDir = path.join(tempDir, '2026-03-18T10-00-00-000Z'); mkdirSync(runDir, { recursive: true }); const sourceFile = path.join(runDir, RESULT_INDEX_FILENAME); @@ -260,7 +260,7 @@ describe('results export', () => { it('deriveOutputDir rejects non-manifest paths', () => { expect(() => deriveOutputDir(tempDir, path.join(tempDir, 'results.jsonl'))).toThrow( - 'Expected a run manifest named run_manifest.jsonl', + 'Expected a run manifest named index.jsonl', ); }); @@ -426,7 +426,7 @@ describe('results export', () => { expect(benchmark.run_summary['gpt-4o'].pass_rate).toHaveProperty('stddev'); }); - it('should create run_manifest.jsonl with per-test artifact pointers', async () => { + it('should create index.jsonl with per-test artifact pointers', async () => { const outputDir = path.join(tempDir, 'output'); const resultWithInput = { ...RESULT_FULL, diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index 268b1c71c..c818c75ee 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -61,13 +61,13 @@ function writeRunArtifacts(projectDir: string): string { const runDir = path.join(projectDir, '.agentv', 'results', 'default', 'run-001'); mkdirSync(runDir, { recursive: true }); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ test_id: 'alpha', score: 1 })}\n`, ); writeFileSync( path.join(runDir, 'summary.json'), `${JSON.stringify( - { manifest_path: 'run_manifest.jsonl', eval_file: 'evals/example.eval.yaml', tests_run: 1 }, + { manifest_path: 'index.jsonl', eval_file: 'evals/example.eval.yaml', tests_run: 1 }, null, 2, )}\n`, @@ -89,7 +89,7 @@ function writeRunArtifactsWithPointers(projectDir: string): string { writeFileSync(path.join(artifactDir, 'transcript.jsonl'), transcriptContent); const transcriptSha = sha256Hex(transcriptContent); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ test_id: 'alpha', score: 1, @@ -111,7 +111,7 @@ function writeRunArtifactsWithPointers(projectDir: string): string { writeFileSync( path.join(runDir, 'summary.json'), `${JSON.stringify( - { manifest_path: 'run_manifest.jsonl', eval_file: 'evals/example.eval.yaml', tests_run: 1 }, + { manifest_path: 'index.jsonl', eval_file: 'evals/example.eval.yaml', tests_run: 1 }, null, 2, )}\n`, @@ -205,7 +205,7 @@ describe('maybeAutoExportRunArtifacts', () => { expect(status).toBe('published'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).toContain( - 'runs/default/run-001/run_manifest.jsonl', + 'runs/default/run-001/index.jsonl', ); }, 20_000); @@ -227,13 +227,13 @@ describe('maybeAutoExportRunArtifacts', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${resultsBranch}`, rootDir, ); - expect(resultTree).toContain('runs/default/run-002/run_manifest.jsonl'); + expect(resultTree).toContain('runs/default/run-002/index.jsonl'); expect(resultTree).toContain('runs/default/run-002/summary.json'); expect(resultTree).not.toContain('runs/default/run-002/alpha/trace.json'); expect(resultTree).not.toContain('runs/default/run-002/alpha/transcript.jsonl'); const index = JSON.parse( git( - `git --git-dir "${remoteDir}" show ${resultsBranch}:runs/default/run-002/run_manifest.jsonl`, + `git --git-dir "${remoteDir}" show ${resultsBranch}:runs/default/run-002/index.jsonl`, rootDir, ), ); @@ -321,10 +321,10 @@ describe('maybeAutoExportRunArtifacts', () => { expect(status).toBe('published'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( - 'runs/default/run-001/run_manifest.jsonl', + 'runs/default/run-001/index.jsonl', ); expect(git('git ls-tree -r --name-only main', cloneDir)).toContain( - 'runs/default/run-001/run_manifest.jsonl', + 'runs/default/run-001/index.jsonl', ); }); }); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index c3545cf49..3e867b91e 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -381,7 +381,7 @@ function writeWtgDogfoodNoncanonicalArtifact(baseDir: string): { } { const runDir = path.join(baseDir, 'wtg-dogfood-noncanonical-run'); mkdirSync(runDir, { recursive: true }); - const indexPath = path.join(runDir, 'run_manifest.jsonl'); + const indexPath = path.join(runDir, 'index.jsonl'); writeFileSync(indexPath, toJsonl({ ...RESULT_A, test_id: 'wtg-dogfood-noncanonical' })); return { runDir, indexPath }; } @@ -444,7 +444,7 @@ describe('resolveSourceFile', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-source-')); const runDir = localRunDir(tempDir, 'default', '2026-06-17T00-00-00-000Z'); mkdirSync(runDir, { recursive: true }); - const indexPath = path.join(runDir, 'run_manifest.jsonl'); + const indexPath = path.join(runDir, 'index.jsonl'); writeFileSync(indexPath, toJsonl(RESULT_A)); await expect(resolveSourceFile(undefined, tempDir)).resolves.toBe(indexPath); @@ -489,7 +489,7 @@ describe('dashboard CLI source contract', () => { expect(result.signal).toBeNull(); expect(result.stdout).not.toContain('Serving 1 result(s)'); expect(result.stderr).toContain('Unsupported Dashboard source'); - expect(result.stderr).toContain('agentv results report '); + expect(result.stderr).toContain('agentv results report '); rmSync(tempDir, { recursive: true, force: true }); }); @@ -2778,7 +2778,7 @@ describe('serve app', () => { ): { runId: string; runDir: string; manifestPath: string } { const runDir = localRunDir(opts?.baseDir ?? tempDir, opts?.experiment ?? 'default', name); mkdirSync(runDir, { recursive: true }); - const manifestPath = path.join(runDir, 'run_manifest.jsonl'); + const manifestPath = path.join(runDir, 'index.jsonl'); writeFileSync( manifestPath, toJsonl( @@ -2887,7 +2887,7 @@ describe('serve app', () => { expect(detailRes.status).toBe(200); await detailRes.json(); const records = readFileSync( - path.join(localRunDirFromRunId(tempDir, acceptedData.run_id), 'run_manifest.jsonl'), + path.join(localRunDirFromRunId(tempDir, acceptedData.run_id), 'index.jsonl'), 'utf8', ) .trim() @@ -4394,14 +4394,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: '.agentv/results/default/r0/run_manifest.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; - expect(data.command).toContain( - '--retry-errors .agentv/results/default/r0/run_manifest.jsonl', - ); + expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); }); it('rejects resume + rerun_failed combo with 400', async () => { @@ -4430,7 +4428,7 @@ describe('serve app', () => { suite_filter: 'examples/demo.eval.yaml', output: '.agentv/results/default/r1', resume: true, - retry_errors: '.agentv/results/default/r0/run_manifest.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(400); @@ -4578,14 +4576,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: '.agentv/results/default/r0/run_manifest.jsonl', + retry_errors: '.agentv/results/default/r0/index.jsonl', }), }); expect(res.status).toBe(200); const data = (await res.json()) as { command: string }; - expect(data.command).toContain( - '--retry-errors .agentv/results/default/r0/run_manifest.jsonl', - ); + expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); }); it('emits --experiment for selected experiment requests', async () => { diff --git a/apps/cli/test/commands/results/shared.test.ts b/apps/cli/test/commands/results/shared.test.ts index be71848cb..37e201103 100644 --- a/apps/cli/test/commands/results/shared.test.ts +++ b/apps/cli/test/commands/results/shared.test.ts @@ -67,7 +67,7 @@ describe('results shared source resolution', () => { writeFileSync(flatFile, '{"test_id":"t1","score":1}\n'); expect(() => resolveRunManifestPath(flatFile)).toThrow( - 'Expected a run workspace directory or run_manifest.jsonl manifest', + 'Expected a run workspace directory or index.jsonl manifest', ); }); diff --git a/apps/cli/test/commands/results/validate.test.ts b/apps/cli/test/commands/results/validate.test.ts index b230cca71..7b82d7da9 100644 --- a/apps/cli/test/commands/results/validate.test.ts +++ b/apps/cli/test/commands/results/validate.test.ts @@ -19,7 +19,7 @@ describe('results validate', () => { ); mkdirSync(runDir, { recursive: true }); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ timestamp: '2026-03-27T12:42:24.429Z', test_id: 'test-greeting', @@ -43,7 +43,7 @@ describe('results validate', () => { writeFileSync( path.join(runDir, 'summary.json'), `${JSON.stringify({ - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', schema_version: 1, metadata: { experiment: 'with-skills', diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index 903e0c74a..2a6be746a 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -147,8 +147,8 @@ async function readJsonLines(filePath: string): Promise { const entries = await readdir(dir, { withFileTypes: true }); - if (entries.some((entry) => entry.isFile() && entry.name === 'run_manifest.jsonl')) { - return [path.join(dir, 'run_manifest.jsonl')]; + if (entries.some((entry) => entry.isFile() && entry.name === 'index.jsonl')) { + return [path.join(dir, 'index.jsonl')]; } if (entries.some((entry) => entry.isFile() && entry.name === 'index.jsonl')) { return [path.join(dir, 'index.jsonl')]; diff --git a/apps/cli/test/commands/trend/trend.test.ts b/apps/cli/test/commands/trend/trend.test.ts index b96fc3387..2f772f7f7 100644 --- a/apps/cli/test/commands/trend/trend.test.ts +++ b/apps/cli/test/commands/trend/trend.test.ts @@ -37,7 +37,7 @@ async function createRunWorkspace( ): Promise<{ runDir: string; indexPath: string }> { const runDir = path.join(rootDir, '.agentv', 'results', 'default', runName); await mkdir(runDir, { recursive: true }); - const indexPath = path.join(runDir, 'run_manifest.jsonl'); + const indexPath = path.join(runDir, 'index.jsonl'); await writeFile( indexPath, `${records.map((record) => JSON.stringify(record)).join('\n')}\n`, @@ -45,7 +45,7 @@ async function createRunWorkspace( ); await writeFile( path.join(runDir, 'summary.json'), - `${JSON.stringify({ manifest_path: 'run_manifest.jsonl' })}\n`, + `${JSON.stringify({ manifest_path: 'index.jsonl' })}\n`, 'utf8', ); return { runDir, indexPath }; @@ -244,15 +244,15 @@ describe('trend command', () => { ); }); - it('still accepts legacy index.jsonl run manifests explicitly', async () => { + it('accepts canonical index.jsonl run manifests explicitly', async () => { const cwd = await createTempDir(); cleanupDirs.push(cwd); const runDir = path.join(cwd, '.agentv', 'results', 'default', '2026-03-01T10-00-00-000Z'); await mkdir(runDir, { recursive: true }); - const legacyManifest = path.join(runDir, 'index.jsonl'); + const manifestPath = path.join(runDir, 'index.jsonl'); await writeFile( - legacyManifest, + manifestPath, `${JSON.stringify({ test_id: 't1', target: 'alpha', @@ -262,10 +262,10 @@ describe('trend command', () => { 'utf8', ); - expect(resolveTrendSources(cwd, [legacyManifest])).toEqual([legacyManifest]); + expect(resolveTrendSources(cwd, [manifestPath])).toEqual([manifestPath]); }); - it('discovers legacy-only run workspaces with --last', async () => { + it('discovers canonical run workspaces with --last', async () => { const cwd = await createTempDir(); cleanupDirs.push(cwd); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 2481790f9..5efca76c5 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -338,7 +338,7 @@ describe('agentv eval CLI', () => { ]); expect(exitCode).toBe(0); - const indexPath = path.join(outputDir, 'file-target', 'run_manifest.jsonl'); + const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); expect(stdout).toContain(`Artifact directory: ${outputDir}`); @@ -366,7 +366,7 @@ describe('agentv eval CLI', () => { const outputDir = path.join(fixture.suiteDir, 'configured-results'); expect(exitCode).toBe(0); - const indexPath = path.join(outputDir, 'file-target', 'run_manifest.jsonl'); + const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); await expectFileExists(indexPath); await expectFileExists(path.join(outputDir, 'file-target', 'summary.json')); @@ -410,7 +410,7 @@ describe('agentv eval CLI', () => { ]); expect(exitCode).toBe(1); - const indexPath = path.join(outputDir, 'file-target', 'run_manifest.jsonl'); + const indexPath = path.join(outputDir, 'file-target', 'index.jsonl'); expect(extractOutputPath(stdout)).toBe(indexPath); expect(stdout).not.toContain('Export files:'); @@ -454,14 +454,14 @@ describe('agentv eval CLI', () => { }, { args: ['--output-format', 'html'], - expected: ['--output-format was removed', 'run_manifest.jsonl'], + expected: ['--output-format was removed', 'index.jsonl'], }, { args: ['--output', 'results.xml'], expected: [ '--output expects a run directory', 'JUnit XML export from agentv eval has been removed', - '/run_manifest.jsonl', + '/index.jsonl', ], }, ] as const; diff --git a/apps/cli/test/unit/retry-errors.test.ts b/apps/cli/test/unit/retry-errors.test.ts index 0b2cd0f99..2a2a4f8a0 100644 --- a/apps/cli/test/unit/retry-errors.test.ts +++ b/apps/cli/test/unit/retry-errors.test.ts @@ -20,14 +20,6 @@ describe('retry-errors', () => { }); function createRunManifestFile(lines: object[]): string { - tmpDir = mkdtempSync(path.join(tmpdir(), 'retry-errors-test-')); - const filePath = path.join(tmpDir, 'run_manifest.jsonl'); - mkdirSync(tmpDir, { recursive: true }); - writeFileSync(filePath, lines.map((l) => JSON.stringify(l)).join('\n')); - return filePath; - } - - function createLegacyIndexFile(lines: object[]): string { tmpDir = mkdtempSync(path.join(tmpdir(), 'retry-errors-test-')); const filePath = path.join(tmpDir, 'index.jsonl'); mkdirSync(tmpDir, { recursive: true }); @@ -92,7 +84,7 @@ describe('retry-errors', () => { expect(results[1].testId).toBe('case-3'); }); - it('supports run_manifest.jsonl manifests written by the CLI', async () => { + it('supports index.jsonl manifests written by the CLI', async () => { const filePath = createRunManifestFile([ { test_id: 'case-1', execution_status: 'ok', score: 0.9 }, { test_id: 'case-2', execution_status: 'execution_error', score: 0 }, @@ -115,15 +107,15 @@ describe('retry-errors', () => { ]); await expect(loadErrorTestIds(filePath)).rejects.toThrow( - 'Expected a run workspace directory or run_manifest.jsonl manifest', + 'Expected a run workspace directory or index.jsonl manifest', ); await expect(loadNonErrorResults(filePath)).rejects.toThrow( - 'Expected a run workspace directory or run_manifest.jsonl manifest', + 'Expected a run workspace directory or index.jsonl manifest', ); }); - it('supports legacy index.jsonl manifests', async () => { - const filePath = createLegacyIndexFile([ + it('supports canonical index.jsonl manifests with artifact paths', async () => { + const filePath = createRunManifestFile([ { test_id: 'case-1', execution_status: 'ok', @@ -195,9 +187,9 @@ describe('retry-errors', () => { expect(buildExclusionFilter(['!negated'])).toBe('!\\!negated'); }); - it('throws on malformed run_manifest.jsonl lines', async () => { + it('throws on malformed index.jsonl lines', async () => { tmpDir = mkdtempSync(path.join(tmpdir(), 'retry-errors-test-')); - const filePath = path.join(tmpDir, 'run_manifest.jsonl'); + const filePath = path.join(tmpDir, 'index.jsonl'); writeFileSync( filePath, [ diff --git a/apps/dashboard/src/components/StopRunButton.tsx b/apps/dashboard/src/components/StopRunButton.tsx index c6541b4d6..9ca7a15e6 100644 --- a/apps/dashboard/src/components/StopRunButton.tsx +++ b/apps/dashboard/src/components/StopRunButton.tsx @@ -2,7 +2,7 @@ * StopRunButton — stop affordance on /jobs/:runId and active run detail * views that interrupts a Dashboard-launched eval. Stop is part of the * stop → resume → complete workflow, not a destructive cancel: the - * partial run_manifest.jsonl is preserved and can be resumed in one click from + * partial index.jsonl is preserved and can be resumed in one click from * the run-detail page. * * Calls POST /api/eval/run/:id/stop (or the project-scoped variant). diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 7e852eac7..2509e4c9c 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -141,7 +141,7 @@ Suite imports are resolved as a deterministic include graph. Circular `imports.suites` imports fail validation with the import chain; raw-case shorthand does not recursively load suite runtime blocks. -Imported suite rows keep their source suite metadata in `run_manifest.jsonl`. Use each +Imported suite rows keep their source suite metadata in `index.jsonl`. Use each row's `result_dir` as the authoritative path to generated artifacts inside the run directory; do not infer layout from suite names. @@ -280,8 +280,8 @@ CLI `--experiment` sets the bucket explicitly. Without that flag, AgentV uses comparison and runtime-policy label for a run condition; folder names are only storage allocation and must not define result semantics. -Imported source suite metadata appears in `run_manifest.jsonl` rows and manifests. -Use `run_manifest.jsonl` fields such as `eval_path`, `test_id`, `target`, and +Imported source suite metadata appears in `index.jsonl` rows and manifests. +Use `index.jsonl` fields such as `eval_path`, `test_id`, `target`, and `result_dir` for identity and artifact discovery instead of reconstructing paths from suite names or wrapper layout. diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 639ae6298..bb76a9e89 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -11,7 +11,7 @@ sidebar: agentv eval evals/my-eval.yaml ``` -Results are written to `.agentv/results///run_manifest.jsonl`. +Results are written to `.agentv/results///index.jsonl`. AgentV picks the experiment bucket from `--experiment`, then `eval.yaml` `experiment.name`, then `default`. Each CLI invocation writes one timestamped run bundle. Each line is a JSON object with one result per test @@ -65,7 +65,7 @@ agentv eval evals/my-eval.yaml --experiment without_skills ``` The experiment label chooses the result bucket and is propagated to each entry -in `run_manifest.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval +in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval file. If neither is set, AgentV writes to the `default` bucket. The eval file stays the same across experiments; what changes is the runtime condition. Dashboards can filter and compare results by experiment. @@ -99,23 +99,23 @@ are unchanged. ### Custom Output Directory -Write all artifacts (run_manifest.jsonl, summary.json, per-test grading/timing) to a specific directory: +Write all artifacts (index.jsonl, summary.json, per-test grading/timing) to a specific directory: ```bash agentv eval evals/my-eval.yaml --output ./my-results ``` `--output` is a run directory, not a file path. The canonical manifest is always -`/run_manifest.jsonl`; the aggregate summary is +`/index.jsonl`; the aggregate summary is `/summary.json`. ### Read Results from the Run Manifest -The run directory is the complete artifact boundary. Use `/run_manifest.jsonl` for scripts, CI summaries, and downstream tools: +The run directory is the complete artifact boundary. Use `/index.jsonl` for scripts, CI summaries, and downstream tools: ```bash agentv eval evals/my-eval.yaml --output ./my-results -cat ./my-results/run_manifest.jsonl +cat ./my-results/index.jsonl ``` ### Generated Task Bundles @@ -129,7 +129,7 @@ Typical layout: ```text my-results/ - run_manifest.jsonl + index.jsonl summary.json / summary.json @@ -148,11 +148,11 @@ my-results/ graders/ # copied grader prompt/script files when applicable ``` -The `run_manifest.jsonl` row links to these generated paths with snake_case fields such +The `index.jsonl` row links to these generated paths with snake_case fields such as `result_dir`, `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path`. Treat those paths as relative to the run directory. When you need a portable artifact for audit, review, Dashboard inspection, or rerun workflows, -share the generated run directory and its `run_manifest.jsonl` manifest. Source-side +share the generated run directory and its `index.jsonl` manifest. Source-side case directories are still useful for organizing bulky prompts, fixtures, or tests while authoring an eval, but they are optional input organization rather than a separate artifact schema. @@ -181,12 +181,12 @@ manifest shape, and optional trace/session input with `--trace`. Export execution traces (tool calls, timing, spans) to files for debugging and analysis: -By default, AgentV writes a per-run workspace with `run_manifest.jsonl` as the canonical manifest for +By default, AgentV writes a per-run workspace with `index.jsonl` as the canonical manifest for result-oriented workflows. For full-fidelity span inspection, export OTLP JSON explicitly. ```bash # Summary-level inspection from the run manifest -agentv inspect stats .agentv/results/default//run_manifest.jsonl +agentv inspect stats .agentv/results/default//index.jsonl # Full-fidelity OTLP JSON trace (importable by OTel backends like Jaeger, Grafana) agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json @@ -195,7 +195,7 @@ agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json agentv inspect show traces/eval.otlp.json --tree ``` -`run_manifest.jsonl` contains aggregate metrics such as score, latency, cost, token usage, and summary +`index.jsonl` contains aggregate metrics such as score, latency, cost, token usage, and summary trace counters. `--otel-file` writes standard OTLP JSON that can be imported into any OpenTelemetry-compatible backend. @@ -355,7 +355,7 @@ AgentV ships three flags for picking up a partial run. They differ only in **whi | `--rerun-failed` | Only cases with `executionStatus === 'ok'` | Errors **and** test failures (assertion misses, threshold misses) | A grader change or model swap means you want to re-grade everything that wasn't already passing | | `--retry-errors ` | Anything that completed without an `execution_error` (same set as `--resume`) | Errors and missing cases | You want to point at an arbitrary prior run/manifest by path, instead of resuming the run dir you're currently writing to | -`--resume` and `--rerun-failed` both append to the existing `run_manifest.jsonl`. When `--output ` is given they target that directory; when omitted they default to the **last run dir for the current cwd**, recorded in `.agentv/cache.json` and updated after every eval. This matches promptfoo's `--resume [evalId]` and OpenCompass's `-r [timestamp]` "latest by default" convention. `--retry-errors` takes the prior run's path directly (a directory or an `run_manifest.jsonl`). +`--resume` and `--rerun-failed` both append to the existing `index.jsonl`. When `--output ` is given they target that directory; when omitted they default to the **last run dir for the current cwd**, recorded in `.agentv/cache.json` and updated after every eval. This matches promptfoo's `--resume [evalId]` and OpenCompass's `-r [timestamp]` "latest by default" convention. `--retry-errors` takes the prior run's path directly (a directory or an `index.jsonl`). ```bash # Resume the last run — no args needed; AgentV finds it from .agentv/cache.json @@ -368,7 +368,7 @@ agentv eval evals/my-eval.yaml --output .agentv/results/default/ --re agentv eval evals/my-eval.yaml --rerun-failed # Re-run only execution errors from any prior run by path -agentv eval evals/my-eval.yaml --retry-errors .agentv/results/default//run_manifest.jsonl +agentv eval evals/my-eval.yaml --retry-errors .agentv/results/default//index.jsonl ``` After any failing run, the CLI prints the exact `--rerun-failed` command for the run dir that just completed — copy/paste it. If the process or pod disappeared before you could access the local run directory and results auto-push was enabled, recover the partial run from [WIP checkpoints](/docs/tools/wip-checkpoints/) first, then use the same `--resume` flow. @@ -492,7 +492,7 @@ When automatic remote publishing sees pointers whose `ref` is `agentv/artifacts/v1` branch in the same results remote at `runs//` and rewrites the published pointer `key` to that backend object key. The configured results branch is the metadata/control -plane for `run_manifest.jsonl`, `summary.json`, tags, and pointers; it does not +plane for `index.jsonl`, `summary.json`, tags, and pointers; it does not duplicate canonical transcript payload bodies when those rows name `agentv/artifacts/v1`. Dashboard resolves the published pointers lazily when a transcript view requests the payload. AgentV keeps this explicit pointer/backend diff --git a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx index de8769dbf..ddd7ceedb 100644 --- a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx +++ b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx @@ -66,7 +66,7 @@ tests: agentv eval ./evals/example.yaml ``` -Results appear in `.agentv/results/default//run_manifest.jsonl` with scores, reasoning, and execution traces. +Results appear in `.agentv/results/default//index.jsonl` with scores, reasoning, and execution traces. ## Next Steps diff --git a/apps/web/src/content/docs/docs/guides/autoresearch.mdx b/apps/web/src/content/docs/docs/guides/autoresearch.mdx index cee37d799..963ee7d08 100644 --- a/apps/web/src/content/docs/docs/guides/autoresearch.mdx +++ b/apps/web/src/content/docs/docs/guides/autoresearch.mdx @@ -81,7 +81,7 @@ Each autoresearch session creates a self-contained experiment directory: │ ├── iterations.jsonl # Per-cycle data (score, decision, mutation) │ └── trajectory.html # Live-updating Chart.js visualization ├── 2026-04-15T10-30-00/ # Cycle 1 run artifacts -│ ├── run_manifest.jsonl +│ ├── index.jsonl │ ├── grading.json │ └── timing.json ├── 2026-04-15T10-35-00/ # Cycle 2 run artifacts @@ -101,7 +101,7 @@ Review the mutation history with `git log` after the run completes. After each eval cycle, autoresearch runs `agentv compare` between the current candidate and the best baseline: ```bash -agentv compare /run_manifest.jsonl /run_manifest.jsonl --json +agentv compare /index.jsonl /index.jsonl --json ``` The decision rule: diff --git a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx index 5e904087c..e6a195983 100644 --- a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx @@ -80,7 +80,7 @@ Benchmark task packs map cleanly onto AgentV fields at authoring time: Use this separation only when it makes the source eval easier to maintain. It is not a first-class artifact schema. After an eval runs, AgentV writes the portable audit surface into the generated run folder: each result can link from -`run_manifest.jsonl` to a run-local `task/` bundle containing `EVAL.yaml`, +`index.jsonl` to a run-local `task/` bundle containing `EVAL.yaml`, `targets.yaml`, and copied `files/` or `graders/` snapshots where applicable. Review, Dashboard files views, and rerun workflows should inspect those generated run artifacts instead of requiring authors to maintain a parallel source-side diff --git a/apps/web/src/content/docs/docs/guides/human-review.mdx b/apps/web/src/content/docs/docs/guides/human-review.mdx index 24f410fcf..26a9ad601 100644 --- a/apps/web/src/content/docs/docs/guides/human-review.mdx +++ b/apps/web/src/content/docs/docs/guides/human-review.mdx @@ -38,7 +38,7 @@ For workspace evaluations (EVAL.yaml), inspect the run manifest and generate the ```bash # View traces from a specific run -agentv inspect show results/2026-03-14T10-32-00_claude/run_manifest.jsonl +agentv inspect show results/2026-03-14T10-32-00_claude/index.jsonl # Generate the HTML report from the run workspace agentv results report results/2026-03-14T10-32-00_claude @@ -61,12 +61,12 @@ cat results/output.jsonl | jq '{id: .test_id, score: .score, verdict: .verdict}' ### Write feedback -Create a `feedback.json` file in the run workspace, alongside `run_manifest.jsonl`: +Create a `feedback.json` file in the run workspace, alongside `index.jsonl`: ``` results/ 2026-03-14T10-32-00_claude/ - run_manifest.jsonl # run manifest + index.jsonl # run manifest trace.otlp.json # optional OTLP trace export feedback.json # ← your review annotations ``` @@ -161,13 +161,13 @@ Keep feedback files alongside results to build a history of review decisions: ``` results/ 2026-03-12T09-00-00_claude/ - run_manifest.jsonl + index.jsonl feedback.json # first iteration review 2026-03-14T10-32-00_claude/ - run_manifest.jsonl + index.jsonl feedback.json # second iteration review 2026-03-15T16-00-00_claude/ - run_manifest.jsonl + index.jsonl feedback.json # third iteration review ``` diff --git a/apps/web/src/content/docs/docs/guides/skill-improvement-workflow.mdx b/apps/web/src/content/docs/docs/guides/skill-improvement-workflow.mdx index 00420c061..0bea029db 100644 --- a/apps/web/src/content/docs/docs/guides/skill-improvement-workflow.mdx +++ b/apps/web/src/content/docs/docs/guides/skill-improvement-workflow.mdx @@ -256,7 +256,7 @@ If you've been using the Agent Skills skill-creator workflow, AgentV reads your | `claude -p "prompt"` | `agentv eval evals.json --target claude` | Same eval, richer engine | | `grading.json` (read) | `/grading.json` (write) | Same per-test schema, AgentV writes one grading file per test case | | `summary.json` (read) | `/summary.json` (write) | AgentV writes the canonical run summary; convert it in a wrapper if another tool needs a narrower compatibility shape | -| n/a | `run_manifest.jsonl` (write) | AgentV-specific per-test manifest for filtering, retry, and replay workflows | +| n/a | `index.jsonl` (write) | AgentV-specific per-test manifest for filtering, retry, and replay workflows | | with-skill vs without-skill | `--target baseline --target candidate` | Structured comparison | | Graduate to richer evals | `agentv convert evals.json` → EVAL.yaml | Adds workspace, code graders, etc. | @@ -274,7 +274,7 @@ agentv pipeline run evals/my-eval.yaml --experiment without_skills agentv pipeline run evals/my-eval.yaml --experiment with_skills ``` -Both runs use the same eval file and produce separate run directories. The experiment label is recorded in `manifest.json` and `run_manifest.jsonl`, making it easy to filter and compare in dashboards. +Both runs use the same eval file and produce separate run directories. The experiment label is recorded in `manifest.json` and `index.jsonl`, making it easy to filter and compare in dashboards. This replaces the need for separate `--target baseline` / `--target candidate` configurations when the only difference between runs is the workspace setup (skills, config, etc.) rather than the target harness. diff --git a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx index d5a453c2f..37b6e39b4 100644 --- a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx @@ -18,13 +18,6 @@ The contract is run-centric: - Dashboard, search, SQLite, HTML reports, and vendor exports are rebuildable projections over the bundle. -:::note -This page names the intended canonical row index as `index.jsonl`. During the -transition from the temporary `run_manifest.jsonl` filename, readers should -accept either filename when inspecting existing bundles. New contract examples -and integrations should converge on `index.jsonl`. -::: - ## Directory Layout The default local layout is: diff --git a/apps/web/src/content/docs/docs/tools/compare.mdx b/apps/web/src/content/docs/docs/tools/compare.mdx index 8e87e9312..0db165da6 100644 --- a/apps/web/src/content/docs/docs/tools/compare.mdx +++ b/apps/web/src/content/docs/docs/tools/compare.mdx @@ -15,12 +15,12 @@ Run two evaluations and compare them: agentv eval evals/my-eval.yaml --output .agentv/results/default/before # ... make changes to your agent ... agentv eval evals/my-eval.yaml --output .agentv/results/default/after -agentv compare .agentv/results/default/before/run_manifest.jsonl .agentv/results/default/after/run_manifest.jsonl +agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl ``` -`run_manifest.jsonl` is the canonical row-level result manifest. Existing +`index.jsonl` is the canonical row-level result manifest. Existing `index.jsonl` run manifests from older AgentV runs remain readable for -compatibility, but new runs write `run_manifest.jsonl`. +compatibility, but new runs write `index.jsonl`. ## Options @@ -32,7 +32,7 @@ compatibility, but new runs write `run_manifest.jsonl`. ## How It Works -1. **Load Results** -- reads both `run_manifest.jsonl` manifests containing evaluation results +1. **Load Results** -- reads both `index.jsonl` manifests containing evaluation results 2. **Match by test_id** -- pairs results with matching `test_id` fields 3. **Compute Deltas** -- calculates `delta = score2 - score1` for each pair 4. **Compute Normalized Gain** -- calculates `g = delta / (1 - score1)` for each pair (see below) @@ -133,7 +133,7 @@ agentv eval evals/*.yaml --target gpt-4 --output .agentv/results/default/baselin agentv eval evals/*.yaml --target gpt-4o --output .agentv/results/default/candidate # Compare results -agentv compare .agentv/results/default/baseline/run_manifest.jsonl .agentv/results/default/candidate/run_manifest.jsonl +agentv compare .agentv/results/default/baseline/index.jsonl .agentv/results/default/candidate/index.jsonl ``` ### Prompt Optimization @@ -148,7 +148,7 @@ agentv eval evals/*.yaml --output .agentv/results/default/before agentv eval evals/*.yaml --output .agentv/results/default/after # Compare with strict threshold -agentv compare .agentv/results/default/before/run_manifest.jsonl .agentv/results/default/after/run_manifest.jsonl --threshold 0.05 +agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl --threshold 0.05 ``` ### CI Quality Gate @@ -158,8 +158,8 @@ Fail CI if the candidate regresses: ```bash #!/bin/bash agentv compare \ - .agentv/results/default/baseline/run_manifest.jsonl \ - .agentv/results/default/candidate/run_manifest.jsonl + .agentv/results/default/baseline/index.jsonl \ + .agentv/results/default/candidate/index.jsonl if [ $? -eq 1 ]; then echo "Regression detected! Candidate performs worse than baseline." exit 1 diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index b6cf0a001..64b59a597 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -39,7 +39,7 @@ To open a different project, pass the project root with `--dir`: agentv dashboard --dir /path/to/project ``` -Dashboard does not accept a run workspace directory or `run_manifest.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. The old `.agentv/results/runs/**` layout is not a Dashboard-visible layout. For one-off inspection of a copied run bundle, use `agentv results report `. +Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. The old `.agentv/results/runs/**` layout is not a Dashboard-visible layout. For one-off inspection of a copied run bundle, use `agentv results report `. ## Data boundary @@ -101,7 +101,7 @@ You can also set the same field globally in `$AGENTV_HOME/config.yaml` or `~/.ag ## Run Detail -Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `run_manifest.jsonl`—including per-result task bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. +Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result task bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. In the per-test results table, click a test ID to open its checks, transcript, source, files, and feedback in a row detail panel while the table, filters, and scroll position stay in place. Use **Full page** from the panel when you want the standalone eval detail route. @@ -163,7 +163,7 @@ Select 2+ rows with the checkboxes and click the sticky **Compare N** action to ### Retroactive tags -Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `run_manifest.jsonl` in the timestamped result folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. +Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the timestamped result folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs/.../tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results///` layout. Remote tag overlays use the same `tag_revision` stale-write check as local tags. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. @@ -403,7 +403,7 @@ After sync, newly fetched remote runs appear in the list with a **remote** sourc - Safe uncommitted changes under the configured results repo's owned result and metadata paths, such as remote tag overlays under `metadata/runs/**`, are committed and pushed when `auto_push: true`. - A local results repo that is ahead is pushed when `auto_push: true` and the committed paths are all under `.agentv/results/**`. - Dirty non-results files, dirty metadata plus remote changes, unresolved conflicts, missing upstream branches, non-results commits ahead, and rejected pushes are blocked instead of reset. -- Non-fast-forward result branch pushes never force-push. AgentV runs a bounded fetch → merge → push loop that absorbs concurrent remote writes with a real merge commit using artifact-aware Git merge drivers (union for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays), so the common append-mostly case auto-merges and pushes as a fast-forward. When Dashboard sync absorbs concurrent remote changes this way, the success feedback includes **Merged remote (auto)**. +- Non-fast-forward result branch pushes never force-push. AgentV runs a bounded fetch → merge → push loop that absorbs concurrent remote writes with a real merge commit using artifact-aware Git merge drivers (union for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays), so the common append-mostly case auto-merges and pushes as a fast-forward. When Dashboard sync absorbs concurrent remote changes this way, the success feedback includes **Merged remote (auto)**. - When a genuine overlay conflict cannot be auto-merged, AgentV does not touch the canonical branch. It pushes the local work to a fresh timestamped `agentv/results-sync/--` branch and reports `needs_human_merge` with a `pending_merge` block (temp branch, target branch, and a GitHub compare URL when the remote is on GitHub). The toolbar shows a **Pending merge** card: open the link to merge the branch into the canonical target on GitHub (GitHub's pull request is the conflict surface — AgentV builds no merge UI), then click **I merged it — resync**. That resumes canonical sync by fast-forward-pulling the merged target. A premature click is a safe no-op — local work stays intact and the next sync re-creates a temp branch. When sync is blocked, Dashboard keeps the local clone intact and shows the `block_reason`, `dirty_paths` or `conflicted_paths`, `git_status`, and a compact `git_diff_summary` so you can resolve the results repo manually before syncing again. diff --git a/apps/web/src/content/docs/docs/tools/inspect.mdx b/apps/web/src/content/docs/docs/tools/inspect.mdx index 6a8b93fbd..bbac42fbd 100644 --- a/apps/web/src/content/docs/docs/tools/inspect.mdx +++ b/apps/web/src/content/docs/docs/tools/inspect.mdx @@ -9,7 +9,7 @@ The `inspect` command provides headless trace inspection and analysis — no ser Supported sources: -- Run workspaces or `run_manifest.jsonl` manifests for summary-level fallback +- Run workspaces or `index.jsonl` manifests for summary-level fallback - Legacy simple trace JSONL files for read-only migration scenarios - OTLP JSON files written via `agentv eval --otel-file ...` @@ -94,7 +94,7 @@ agentv inspect show trace.otlp.json --format json \ | jq '[.[] | select(.cost_usd > 0.10) | {test_id, score, cost: .cost_usd}]' # Compare providers -agentv inspect stats .agentv/results/default//run_manifest.jsonl --group-by target --format json \ +agentv inspect stats .agentv/results/default//index.jsonl --group-by target --format json \ | jq '.groups[] | {label, score_mean: .metrics.score.mean}' ``` diff --git a/apps/web/src/content/docs/docs/tools/prepare.mdx b/apps/web/src/content/docs/docs/tools/prepare.mdx index cfc034380..c93470fd1 100644 --- a/apps/web/src/content/docs/docs/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/tools/prepare.mdx @@ -101,4 +101,4 @@ There is no `agentv watch` command. } ``` -Keep the prepared directory with the generated run directory when sharing review evidence. The `run_manifest.jsonl` row written by `grade` includes `metadata.prepared_attempt` with the manifest path, workspace path, prompt path, baseline status, and optional trace path. +Keep the prepared directory with the generated run directory when sharing review evidence. The `index.jsonl` row written by `grade` includes `metadata.prepared_attempt` with the manifest path, workspace path, prompt path, baseline status, and optional trace path. diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 93564d223..c0d0e3d5a 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -9,7 +9,7 @@ import { Image } from 'astro:assets'; import resultsReportOverview from '../../../../assets/screenshots/results-report-overview.png'; import resultsReportDetails from '../../../../assets/screenshots/results-report-details.png'; -The `results` command family works on existing local AgentV run workspaces and `run_manifest.jsonl` manifests. Use it after an eval run to inspect failures, validate manifests, export artifact layouts, combine/delete local run workspaces, or generate a shareable HTML report. +The `results` command family works on existing local AgentV run workspaces and `index.jsonl` manifests. Use it after an eval run to inspect failures, validate manifests, export artifact layouts, combine/delete local run workspaces, or generate a shareable HTML report. Remote result repository exchange is intentionally not part of `agentv results`. New eval runs publish completed artifacts to a configured results repo or branch; `auto_push: true` additionally pushes that branch to the remote. Manual remote status and sync are Dashboard/API workflows. See [Dashboard Remote Results](/docs/tools/dashboard/#remote-results) for configuration and sync behavior, and [WIP checkpoints](/docs/tools/wip-checkpoints/) for recovering in-progress runs before final publish. @@ -33,12 +33,12 @@ start with [Result Artifact Contract](/docs/reference/result-artifacts/). ## `results report` -The `results report` command turns an existing run workspace or `run_manifest.jsonl` manifest into a self-contained HTML report for sharing, inspection, and human review. +The `results report` command turns an existing run workspace or `index.jsonl` manifest into a self-contained HTML report for sharing, inspection, and human review. AgentV results report overview showing 11 tests across 2 eval files with pass, fail, pass rate, duration, and cost summary cards ```bash -agentv results report +agentv results report ``` Examples: @@ -48,7 +48,7 @@ Examples: agentv results report .agentv/results/default/2026-03-14T10-32-00_claude # Use an explicit output path -agentv results report .agentv/results/default/2026-03-14T10-32-00_claude/run_manifest.jsonl \ +agentv results report .agentv/results/default/2026-03-14T10-32-00_claude/index.jsonl \ --out ./reports/human-review.html ``` @@ -96,17 +96,17 @@ Use `--out docs/.html` when a repository should publish multiple runs. Lin Use `results export` when you need the artifact workspace layout itself rather than a rendered report. ```bash -agentv results export [--out ] [--duplicate-policy update] +agentv results export [--out ] [--duplicate-policy update] ``` -This is useful when a manifest needs to be materialized into a predictable artifact tree for other tooling, review, or archiving. The run workspace is also where generated task bundles live: `run_manifest.jsonl` rows may point to per-result `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` entries. Keep those generated artifacts with the run when sharing or auditing results. +This is useful when a manifest needs to be materialized into a predictable artifact tree for other tooling, review, or archiving. The run workspace is also where generated task bundles live: `index.jsonl` rows may point to per-result `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` entries. Keep those generated artifacts with the run when sharing or auditing results. The export source is still the canonical run bundle described in the [Result Artifact Contract](/docs/reference/result-artifacts/): `summary.json` for aggregate run facts, the row manifest for row discovery, and sidecars for detailed payloads. -Each exported trace sidecar and `run_manifest.jsonl` row includes a stable `projection_identity` derived from AgentV-owned fields: `run_id`, `suite` or `eval_path`, `test_id`, `target`, `source_target`, `attempt`, `variant`, `envelope_id`, `trace_id`, `root_span_id`, and the projection format/version. Retrying the same completed run keeps the same projection ID even when you choose a different `--out` directory, because `run_id` comes from the source run directory or source manifest name rather than the export destination. +Each exported trace sidecar and `index.jsonl` row includes a stable `projection_identity` derived from AgentV-owned fields: `run_id`, `suite` or `eval_path`, `test_id`, `target`, `source_target`, `attempt`, `variant`, `envelope_id`, `trace_id`, `root_span_id`, and the projection format/version. Retrying the same completed run keeps the same projection ID even when you choose a different `--out` directory, because `run_id` comes from the source run directory or source manifest name rather than the export destination. Duplicate policy is explicit: @@ -138,13 +138,13 @@ when they are available, while `transcript.jsonl` is the normalized conversation transcript with joined `tool_use.result` blocks. AgentV does not persist a public `trace.json` sidecar in run bundles; external observability systems can be linked through safe `external_trace` metadata when available. -`summary.json` remains the run-level aggregate summary, and `run_manifest.jsonl` -carries lightweight explicit paths such as `transcript_path`, -`transcript_raw_path`, and `metrics_path` plus artifact pointers only when -detached payload publishing needs them. -New run summaries include `manifest_path: "run_manifest.jsonl"` so tools can -discover the row manifest from `summary.json`, but row and artifact discovery -still uses `run_manifest.jsonl` as the authoritative record. +`summary.json` remains the run-level aggregate summary. `index.jsonl` is the +canonical row index for the run: one row per result, attempt, or case, carrying +lightweight explicit paths such as `transcript_path`, `transcript_raw_path`, +and `metrics_path` plus artifact pointers only when detached payload publishing +needs them. Dashboard search indexes, SQLite indexes, and other read models are +derived projections over these run artifacts, not replacements for +`index.jsonl`. Duration, token, and cost usage remains in `timing.json`, including source labels such as `provider_reported`, `token_estimated`, `aggregate`, or `unavailable`. @@ -175,7 +175,7 @@ Agent Skills eval artifacts map into AgentV like this: | Agent Skills pattern | AgentV field | Artifact location | |----------------------|--------------|-------------------| -| Authored `evals/evals.json` cases | AgentV eval cases and task bundle paths | Eval source plus optional `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `run_manifest.jsonl` | +| Authored `evals/evals.json` cases | AgentV eval cases and task bundle paths | Eval source plus optional `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `index.jsonl` | | Per-case answer | Generated target output artifact | `run-N/outputs/answer.md` | | Per-attempt sidecars | Normalized transcript, metrics, and raw provider evidence | `run-N/transcript.jsonl`, `run-N/transcript-raw.jsonl`, `run-N/metrics.json` | | Per-attempt `timing.json` | Duration, token totals, cost, and usage source labels | `run-N/timing.json` | @@ -190,7 +190,7 @@ Use the additive projection bundle path when an external adapter needs a backend-neutral handoff instead of AgentV's full artifact tree: ```bash -agentv results export --projection-bundle +agentv results export --projection-bundle ``` This writes `projection_bundle.json` next to the exported artifacts. The bundle @@ -207,7 +207,7 @@ transcripts, datasets, experiments, or indexes into Phoenix. For adapter development and CI snapshots, use dry-run mode: ```bash -agentv results export --dry-run > projection_bundle.json +agentv results export --dry-run > projection_bundle.json ``` Dry-run prints deterministic JSON and does not write export artifacts. Vendor @@ -215,7 +215,7 @@ adapters should consume either this JSON directly or the local `projection_bundle.json`. Dry-run refs are marked `artifact_refs.status: "planned_export"` because the export tree has not been written. Bundles written with `--projection-bundle` are built from the emitted -export `run_manifest.jsonl` and use `artifact_refs.status: "emitted"`. +export `index.jsonl` and use `artifact_refs.status: "emitted"`. Raw prompt text, final output, and tool arguments/results are excluded by default, and raw-bearing artifact refs such as `grading_path`, `input_path`, @@ -223,7 +223,7 @@ default, and raw-bearing artifact refs such as `grading_path`, `input_path`, include raw payloads and raw-bearing refs in the bundle, opt in explicitly: ```bash -agentv results export --dry-run --include-raw-content +agentv results export --dry-run --include-raw-content ``` Keep backend-specific anonymization in the adapter layer. For example, an Opik @@ -251,7 +251,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `results.repo` with `results.path` pointing at the source checkout and `results.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `run_manifest.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`run_manifest.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.path` without `results.repo` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `run_manifest.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `results.repo` with `results.path` pointing at the source checkout and `results.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.path` without `results.repo` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. diff --git a/apps/web/src/content/docs/docs/tools/trend.mdx b/apps/web/src/content/docs/docs/tools/trend.mdx index 8cd062c93..b28ff4908 100644 --- a/apps/web/src/content/docs/docs/tools/trend.mdx +++ b/apps/web/src/content/docs/docs/tools/trend.mdx @@ -25,12 +25,12 @@ Filter to one suite and target: agentv trend --last 8 --suite code-review --target claude-sonnet ``` -Point directly at run workspaces or `run_manifest.jsonl` manifests when you need a specific historical slice or want a reproducible example: +Point directly at run workspaces or `index.jsonl` manifests when you need a specific historical slice or want a reproducible example: ```bash agentv trend \ .agentv/results/default/2026-03-01T10-00-00-000Z/ \ - .agentv/results/default/2026-03-08T10-00-00-000Z/run_manifest.jsonl \ + .agentv/results/default/2026-03-08T10-00-00-000Z/index.jsonl \ .agentv/results/default/2026-03-15T10-00-00-000Z/ ``` @@ -46,12 +46,12 @@ agentv trend --last 8 --suite code-review --target claude-sonnet \ `trend` only accepts canonical run workspaces: - `.agentv/results///` -- `.agentv/results///run_manifest.jsonl` +- `.agentv/results///index.jsonl` Legacy `index.jsonl` manifests from older AgentV runs remain readable when passed directly or when they are the only manifest in a run workspace. Legacy flat `results.jsonl` files are rejected. The command stays on lightweight -`run_manifest.jsonl` manifests and does not require per-test artifact hydration. +`index.jsonl` manifests and does not require per-test artifact hydration. ## Options @@ -68,7 +68,7 @@ flat `results.jsonl` files are rejected. The command stays on lightweight ## How It Works -1. Loads each selected `run_manifest.jsonl` manifest. +1. Loads each selected `index.jsonl` manifest. 2. Applies `suite` and `target` filters per record. 3. By default, reduces every run to the intersection of test IDs present in all selected runs. 4. Computes one mean score per run. @@ -115,7 +115,7 @@ Regression Gate: threshold=0.010 fail_on_degrading=true triggered=true "runs": [ { "label": "2026-03-01T10:00:00.000Z", - "path": "/repo/.agentv/results/default/2026-03-01T10-00-00-000Z/run_manifest.jsonl", + "path": "/repo/.agentv/results/default/2026-03-01T10-00-00-000Z/index.jsonl", "timestamp": "2026-03-01T10:00:00.000Z", "matched_test_count": 42, "mean_score": 0.92 diff --git a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx index a98a1a278..2eb389fa9 100644 --- a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx +++ b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx @@ -23,7 +23,7 @@ If no results repo is configured, or auto-push is disabled, `agentv eval` still | Location | Path or ref | What it contains | | --- | --- | --- | | Local project | `.agentv/results///summary.json` | A run-start stub with `metadata.planned_test_count` and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | -| Local project | `.agentv/results///run_manifest.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | +| Local project | `.agentv/results///index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | | Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results//`. | | Results repo storage branch | Configured `results.repo.branch`; local checkout configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | diff --git a/docs/adr/0011-result-output-artifact-contract.md b/docs/adr/0011-result-output-artifact-contract.md index fd8ddb392..a5f2a76c9 100644 --- a/docs/adr/0011-result-output-artifact-contract.md +++ b/docs/adr/0011-result-output-artifact-contract.md @@ -85,11 +85,6 @@ ordinary per-case sidecars through explicit fields such as `result_dir`, `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` when those artifacts exist. -During the transition away from the temporary `run_manifest.jsonl` filename, -readers may accept both `run_manifest.jsonl` and `index.jsonl` for existing -bundles. The contract name for new examples and integration guidance is -`index.jsonl`. - `artifact_pointers` remain an offload indirection for large detached payload bytes. They are not the discovery path for ordinary sidecars that live in the run tree. diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index 2911e533b..be48b831a 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -205,7 +205,7 @@ export interface EvalConfig { readonly budgetUsd?: number; /** Optional run workspace directory for canonical AgentV artifacts. */ readonly outputDir?: string; - /** Optional experiment name recorded in summary.json and run_manifest.jsonl. */ + /** Optional experiment name recorded in summary.json and index.jsonl. */ readonly experiment?: string; } diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts index b2d4bba78..d5b771031 100644 --- a/packages/core/src/evaluation/result-row-schema.ts +++ b/packages/core/src/evaluation/result-row-schema.ts @@ -1,7 +1,7 @@ /** * Result JSONL row schema used at the AgentV artifact boundary. * - * Canonical AgentV run manifests are `run_manifest.jsonl` files with snake_case keys + * Canonical AgentV run manifests are `index.jsonl` files with snake_case keys * and a numeric `score`. Historical rows produced from TypeScript * `EvaluationResult` objects may contain a small set of camelCase aliases. * Normalize those aliases only at this boundary; callers should work with the @@ -16,7 +16,7 @@ export class ResultRowSchemaError extends Error { } const MIGRATION_GUIDANCE = - 'Expected an AgentV result row with a numeric score. Eval-case JSONL is input data, not a results artifact. Run `agentv eval --output ` and pass the run workspace or its run_manifest.jsonl manifest.'; + 'Expected an AgentV result row with a numeric score. Eval-case JSONL is input data, not a results artifact. Run `agentv eval --output ` and pass the run workspace or its index.jsonl manifest.'; const RESULT_ROW_ALIASES = { answerPath: 'answer_path', diff --git a/packages/core/src/evaluation/results-repo-cache.test.ts b/packages/core/src/evaluation/results-repo-cache.test.ts index b133fe23d..3cc4861b6 100644 --- a/packages/core/src/evaluation/results-repo-cache.test.ts +++ b/packages/core/src/evaluation/results-repo-cache.test.ts @@ -37,7 +37,7 @@ function writeRun( const runDir = path.join(repoDir, 'runs', experiment, timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ timestamp, test_id: `${experiment}-case`, @@ -51,7 +51,7 @@ function writeRun( path.join(runDir, 'summary.json'), `${JSON.stringify( { - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', metadata: { display_name: `${experiment} ${timestamp}`, experiment, diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index adbf5cebe..24117b031 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -66,10 +66,7 @@ const GIT_EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'; // never overwrites the user's git config. See createOrphanResultsBranch. const RESULTS_REPO_GENESIS_MESSAGE = 'chore(results): initialize AgentV results branch'; const RESULTS_REPO_GENESIS_DATE = '@0 +0000'; -const RESULT_MANIFEST_FILENAME = 'run_manifest.jsonl'; -const LEGACY_RESULT_INDEX_FILENAME = 'index.jsonl'; -const RESULT_INDEX_FILENAME = RESULT_MANIFEST_FILENAME; -const RESULT_MANIFEST_FILENAMES = [RESULT_MANIFEST_FILENAME, LEGACY_RESULT_INDEX_FILENAME] as const; +const RESULT_INDEX_FILENAME = 'index.jsonl'; // Artifact-aware merge config for the AgentV-owned results checkout. These two // pieces let `git merge` reconcile concurrent result writes automatically so @@ -84,7 +81,6 @@ const RESULT_MANIFEST_FILENAMES = [RESULT_MANIFEST_FILENAME, LEGACY_RESULT_INDEX const RESULTS_REPO_GITATTRIBUTES_FILE = '.gitattributes'; const RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes. # Append-only run manifests: union concurrent appends (lines are orthogonal). -run_manifest.jsonl merge=union index.jsonl merge=union # Editable run overlay (tags/feedback): 3-way JSON set/field union via the # agentv-json driver; a genuine scalar conflict falls through to a human merge. @@ -3019,7 +3015,7 @@ function isDeprecatedTraceArtifactPath(relativePath: string): boolean { } function isResultManifestFilename(filename: string): boolean { - return RESULT_MANIFEST_FILENAMES.includes(filename as (typeof RESULT_MANIFEST_FILENAMES)[number]); + return filename === RESULT_INDEX_FILENAME; } function safeLocalSummaryManifestPath( @@ -3050,11 +3046,9 @@ function resolveLocalResultManifestPath(sourceDir: string): string | undefined { } } catch {} - for (const filename of RESULT_MANIFEST_FILENAMES) { - const manifestPath = path.join(sourceDir, filename); - if (existsSync(manifestPath)) { - return manifestPath; - } + const manifestPath = path.join(sourceDir, RESULT_INDEX_FILENAME); + if (existsSync(manifestPath)) { + return manifestPath; } return undefined; } @@ -3901,15 +3895,13 @@ function buildGitManifestPaths( } } - for (const filename of RESULT_MANIFEST_FILENAMES) { - for (const treePath of treePaths) { - if (!treePath.endsWith(`/${filename}`)) { - continue; - } - const runDir = path.posix.dirname(treePath); - if (!manifestByRunDir.has(runDir)) { - manifestByRunDir.set(runDir, treePath); - } + for (const treePath of treePaths) { + if (!treePath.endsWith(`/${RESULT_INDEX_FILENAME}`)) { + continue; + } + const runDir = path.posix.dirname(treePath); + if (!manifestByRunDir.has(runDir)) { + manifestByRunDir.set(runDir, treePath); } } diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index 07edb924f..53d59989e 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -2,7 +2,7 @@ * Canonical AgentV run artifact helpers. * * This module owns the shared run-workspace contract used by CLI and - * programmatic evals: `run_manifest.jsonl`, run-root `summary.json`, per-case + * programmatic evals: `index.jsonl`, run-root `summary.json`, per-case * `summary.json`, `run-N/result.json`, and transcript projections. Keep wire * keys in snake_case here so every caller produces the same artifacts. */ @@ -55,11 +55,7 @@ import type { TrialResult, } from './types.js'; -export const RESULT_MANIFEST_FILENAME = 'run_manifest.jsonl'; -export const LEGACY_RESULT_INDEX_FILENAME = 'index.jsonl'; -// Backward-compatible export name retained for existing callers. New writes use -// the row-level run manifest filename. -export const RESULT_INDEX_FILENAME = RESULT_MANIFEST_FILENAME; +export const RESULT_INDEX_FILENAME = 'index.jsonl'; export const RUN_SUMMARY_FILENAME = 'summary.json'; const TIMING_SOURCE_VALUES = [ @@ -231,11 +227,9 @@ async function resolveExistingResultManifestPath(runDir: string): Promise 0 ? 'outputs/answer.md' : undefined, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 483c111c3..8c58dca1e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -58,9 +58,7 @@ export { type EvalSummary, } from './evaluation/evaluate.js'; export { - LEGACY_RESULT_INDEX_FILENAME, RESULT_INDEX_FILENAME, - RESULT_MANIFEST_FILENAME, RUN_SUMMARY_FILENAME, aggregateRunDir, buildAggregateGradingArtifact, diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index 388828f48..bb6985d9c 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -239,12 +239,12 @@ function randomToken(): string { function writeRunArtifacts(runDir: string, experiment: string, timestamp: string): void { mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'run_manifest.jsonl'), '{"test_id":"alpha"}\n'); + writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha"}\n'); writeFileSync( path.join(runDir, 'summary.json'), JSON.stringify( { - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', metadata: { timestamp, experiment, @@ -295,7 +295,7 @@ function writeRunArtifactsWithPointers( const legacyTraceSha = sha256Hex(legacyTraceContent); const transcriptSha = sha256Hex(transcriptContent); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ test_id: 'alpha', score: 1, @@ -403,7 +403,7 @@ describe('listGitRuns', () => { rmSync(repoDir, { recursive: true, force: true }); }); - it('returns committed runs derived from run manifests and legacy index.jsonl manifests', async () => { + it('returns committed runs derived from canonical index.jsonl manifests', async () => { const defaultRunDir = path.join(repoDir, 'runs', 'default', '2026-05-20T10-00-00-000Z'); mkdirSync(defaultRunDir, { recursive: true }); writeFileSync( @@ -444,7 +444,7 @@ describe('listGitRuns', () => { const experimentRunDir = path.join(repoDir, 'runs', 'with-skills', '2026-05-21T11-00-00-000Z'); mkdirSync(experimentRunDir, { recursive: true }); writeFileSync( - path.join(experimentRunDir, 'run_manifest.jsonl'), + path.join(experimentRunDir, 'index.jsonl'), `${[ JSON.stringify({ test_id: 'alpha', @@ -467,7 +467,7 @@ describe('listGitRuns', () => { path.join(experimentRunDir, 'summary.json'), JSON.stringify( { - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', metadata: { display_name: 'remote friendly run', timestamp: '2026-05-21T11:00:00.000Z', @@ -502,7 +502,7 @@ describe('listGitRuns', () => { experiment: 'with-skills', timestamp: '2026-05-21T11:00:00.000Z', display_name: 'remote friendly run', - manifest_path: 'runs/with-skills/2026-05-21T11-00-00-000Z/run_manifest.jsonl', + manifest_path: 'runs/with-skills/2026-05-21T11-00-00-000Z/index.jsonl', summary_path: 'runs/with-skills/2026-05-21T11-00-00-000Z/summary.json', test_count: 3, pass_rate: 0.75, @@ -520,58 +520,6 @@ describe('listGitRuns', () => { expect(runs[0].size_bytes).toBeGreaterThan(0); }); - it('does not double-count a remote bundle that has both manifest filenames', async () => { - const runDir = path.join(repoDir, 'runs', 'default', '2026-05-22T12-00-00-000Z'); - mkdirSync(runDir, { recursive: true }); - const canonical = `${JSON.stringify({ - test_id: 'canonical', - target: 'codex', - score: 1, - timestamp: '2026-05-22T12:00:00.000Z', - })}\n`; - writeFileSync(path.join(runDir, 'run_manifest.jsonl'), canonical); - writeFileSync( - path.join(runDir, 'index.jsonl'), - `${JSON.stringify({ - test_id: 'legacy', - target: 'codex', - score: 0, - timestamp: '2026-05-22T12:00:00.000Z', - })}\n`, - ); - writeFileSync( - path.join(runDir, 'summary.json'), - JSON.stringify( - { - manifest_path: 'run_manifest.jsonl', - metadata: { - timestamp: '2026-05-22T12:00:00.000Z', - targets: ['codex'], - tests_run: ['canonical'], - }, - run_summary: { - codex: { - pass_rate: { mean: 1 }, - }, - }, - }, - null, - 2, - ), - ); - git('git add runs && git commit -m "seed duplicate manifests"', repoDir); - - const runs = await listGitRuns(repoDir, 'HEAD'); - - expect(runs).toHaveLength(1); - expect(runs[0]).toMatchObject({ - run_id: '2026-05-22T12-00-00-000Z', - manifest_path: 'runs/default/2026-05-22T12-00-00-000Z/run_manifest.jsonl', - test_count: 1, - avg_score: 1, - }); - }); - it('returns an empty list when the ref has no committed runs', async () => { writeFileSync(path.join(repoDir, 'README.md'), '# test\n'); git('git add README.md && git commit -m "initial"', repoDir); @@ -590,7 +538,7 @@ describe('listGitRuns', () => { const runDir = path.join(repoDir, 'runs', 'default', '2026-05-20T10-00-00-000Z'); mkdirSync(runDir, { recursive: true }); writeFileSync( - path.join(runDir, 'run_manifest.jsonl'), + path.join(runDir, 'index.jsonl'), `${JSON.stringify({ test_id: 'alpha', target: 'gpt-4o', @@ -601,7 +549,7 @@ describe('listGitRuns', () => { path.join(runDir, 'summary.json'), JSON.stringify( { - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', metadata: { timestamp: '2026-05-20T10:00:00.000Z', targets: ['gpt-4o'], @@ -646,11 +594,11 @@ describe('listGitRuns', () => { it('materializes an entire run subtree atomically from git objects', async () => { const runDir = path.join(repoDir, 'runs', 'with-files', '2026-05-22T10-00-00-000Z'); mkdirSync(path.join(runDir, 'attachments'), { recursive: true }); - writeFileSync(path.join(runDir, 'run_manifest.jsonl'), '{"test_id":"alpha"}\n'); + writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha"}\n'); writeFileSync( path.join(runDir, 'summary.json'), JSON.stringify({ - manifest_path: 'run_manifest.jsonl', + manifest_path: 'index.jsonl', metadata: { timestamp: '2026-05-22T10:00:00.000Z', experiment: 'with-files', @@ -671,9 +619,7 @@ describe('listGitRuns', () => { await materializeGitRun(repoDir, 'with-files/2026-05-22T10-00-00-000Z', 'HEAD'); - expect(readFileSync(path.join(runDir, 'run_manifest.jsonl'), 'utf8')).toContain( - '"test_id":"alpha"', - ); + expect(readFileSync(path.join(runDir, 'index.jsonl'), 'utf8')).toContain('"test_id":"alpha"'); expect(readFileSync(path.join(runDir, 'attachments', 'response.md'), 'utf8')).toBe( 'hello from git\n', ); @@ -1390,7 +1336,7 @@ describe('results repo write path', () => { expect(published).toBe(true); expect(git('git branch --show-current', resultsRepoDir)).toBe('main'); const branchFiles = git(`git ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, resultsRepoDir); - expect(branchFiles).toContain(`runs/external/${runTimestamp}/run_manifest.jsonl`); + expect(branchFiles).toContain(`runs/external/${runTimestamp}/index.jsonl`); expect(branchFiles).not.toContain('README.md'); }, 20000); @@ -1601,7 +1547,7 @@ describe('results repo write path', () => { `AgentV-Run: with-skills::${runTimestamp}`, ); expect(git('git ls-tree -r --name-only main', cloneDir)).toContain( - `runs/with-skills/${runTimestamp}/run_manifest.jsonl`, + `runs/with-skills/${runTimestamp}/index.jsonl`, ); const runs = await listGitRuns(cloneDir, 'main'); @@ -1674,7 +1620,7 @@ describe('results repo write path', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir, ); - expect(resultTree).toContain(`runs/${destinationPath}/run_manifest.jsonl`); + expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); expect(resultTree).toContain(`runs/${destinationPath}/summary.json`); expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/trace.json`); expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); @@ -1686,11 +1632,11 @@ describe('results repo write path', () => { expect(artifactTree).not.toContain(`runs/${destinationPath}/alpha/trace.json`); expect(artifactTree).toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); expect(artifactTree).not.toContain(`runs/${destinationPath}/summary.json`); - expect(artifactTree).not.toContain(`runs/${destinationPath}/run_manifest.jsonl`); + expect(artifactTree).not.toContain(`runs/${destinationPath}/index.jsonl`); const index = JSON.parse( gitRaw( - `git --git-dir "${remoteDir}" show ${storageBranch}:runs/${destinationPath}/run_manifest.jsonl`, + `git --git-dir "${remoteDir}" show ${storageBranch}:runs/${destinationPath}/index.jsonl`, rootDir, ).toString('utf8'), ); @@ -1777,7 +1723,7 @@ describe('results repo write path', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir, ); - expect(resultTree).toContain(`runs/${destinationPath}/run_manifest.jsonl`); + expect(resultTree).toContain(`runs/${destinationPath}/index.jsonl`); expect(resultTree).toContain(`runs/${destinationPath}/summary.json`); expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/trace.json`); expect(resultTree).not.toContain(`runs/${destinationPath}/alpha/transcript.jsonl`); @@ -2165,7 +2111,7 @@ describe('results repo write path', () => { const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); const cloneDir = path.join(rootDir, 'results-clone'); const config = createResultsConfig(remoteDir, cloneDir); - const indexRel = path.join('runs', 'shared', '2026-05-25T12-00-00-000Z', 'run_manifest.jsonl'); + const indexRel = path.join('runs', 'shared', '2026-05-25T12-00-00-000Z', 'index.jsonl'); await ensureResultsRepoClone(config); git('git config user.email "test@example.com"', cloneDir); diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 21449f9c1..9a67452f8 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -611,16 +611,16 @@ agentv eval assert --agent-output "..." --agent-input "..." agentv import claude --session-id # Re-run only execution errors from a previous run -agentv eval --retry-errors .agentv/results/default//run_manifest.jsonl +agentv eval --retry-errors .agentv/results/default//index.jsonl # Validate eval file agentv validate # Compare results — N-way matrix from a canonical run manifest -agentv compare .agentv/results/default//run_manifest.jsonl -agentv compare .agentv/results/default//run_manifest.jsonl --baseline # CI regression gate -agentv compare .agentv/results/default//run_manifest.jsonl --baseline --candidate # pairwise -agentv compare .agentv/results/default//run_manifest.jsonl .agentv/results/default//run_manifest.jsonl +agentv compare .agentv/results/default//index.jsonl +agentv compare .agentv/results/default//index.jsonl --baseline # CI regression gate +agentv compare .agentv/results/default//index.jsonl --baseline --candidate # pairwise +agentv compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl # Author assertions directly in the eval file # Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed