From d37269c7a7da269878a56811adc2cbcdc37e9eea Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 16:22:12 +0200 Subject: [PATCH 1/4] feat(artifacts): write eval runs at results root --- CONCEPTS.md | 16 +- apps/cli/src/commands/eval/result-layout.ts | 10 +- apps/cli/src/commands/eval/run-cache.ts | 2 +- apps/cli/src/commands/eval/run-eval.ts | 14 +- apps/cli/src/commands/eval/wip-checkpoint.ts | 4 +- apps/cli/src/commands/inspect/utils.ts | 27 +- apps/cli/src/commands/pipeline/input.ts | 3 +- apps/cli/src/commands/pipeline/run.ts | 3 +- apps/cli/src/commands/results/combine-run.ts | 48 ++-- apps/cli/src/commands/results/combine.ts | 2 +- apps/cli/src/commands/results/eval-runner.ts | 2 +- .../src/commands/results/remote-metadata.ts | 2 +- apps/cli/src/commands/results/remote.ts | 6 +- .../commands/results/serve-file-tree.test.ts | 2 +- apps/cli/src/commands/results/validate.ts | 8 +- .../commands/eval/artifact-writer.test.ts | 12 + .../test/commands/eval/result-layout.test.ts | 22 +- apps/cli/test/commands/eval/run-cache.test.ts | 8 +- .../test/commands/eval/wip-checkpoint.test.ts | 2 +- .../results/remote-auto-export.test.ts | 33 +-- apps/cli/test/commands/results/serve.test.ts | 260 +++++++----------- .../test/commands/results/validate.test.ts | 20 +- apps/cli/test/commands/trace/trace.test.ts | 44 ++- .../dashboard/src/components/AnalyticsTab.tsx | 2 - .../src/components/resume-run-helpers.test.ts | 4 +- .../docs/docs/evaluation/experiments.mdx | 13 +- .../docs/docs/evaluation/running-evals.mdx | 19 +- .../docs/docs/getting-started/quickstart.mdx | 2 +- .../docs/docs/reference/result-artifacts.mdx | 113 ++++---- .../src/content/docs/docs/tools/compare.mdx | 27 +- .../src/content/docs/docs/tools/dashboard.mdx | 20 +- .../src/content/docs/docs/tools/inspect.mdx | 2 +- .../src/content/docs/docs/tools/results.mdx | 33 ++- .../web/src/content/docs/docs/tools/trend.mdx | 19 +- .../docs/docs/tools/wip-checkpoints.mdx | 8 +- .../0011-result-output-artifact-contract.md | 45 +-- docs/adr/0012-finalize-run-artifact-layout.md | 210 +++++++------- .../src/evaluation/results-repo-cache.test.ts | 12 +- packages/core/src/evaluation/results-repo.ts | 35 ++- packages/core/src/evaluation/run-artifacts.ts | 8 + .../evaluate-programmatic-api.test.ts | 8 +- .../core/test/evaluation/results-repo.test.ts | 221 ++++++--------- 42 files changed, 625 insertions(+), 726 deletions(-) diff --git a/CONCEPTS.md b/CONCEPTS.md index 8ec1fd026..1f671a3d5 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -12,19 +12,23 @@ Shared domain vocabulary for this project — entities, named processes, and sta ## Evaluation Model -**Eval / Eval YAML** — The only composable and runnable AgentV authoring primitive. An eval YAML file can be a reusable task suite that owns task context, a wrapper eval that imports suites and carries an inline `experiment:` block, or a sidecar around raw JSONL cases. AgentV does not have a separate runnable `experiment.yaml` artifact. +**Eval / Eval YAML** — The only composable and runnable AgentV authoring primitive. An eval YAML file can be a reusable task suite that owns task context, a wrapper eval that imports suites and binds top-level runtime policy, or a sidecar around raw JSONL cases. AgentV does not have a separate runnable `experiment.yaml` artifact. **Task suite** — Eval YAML that owns what is being tested: prompts, datasets, input files, fixtures, `workspace`, assertions, expected references, and judge criteria. It can run directly or be imported by another eval with `tests[].include` and `type: suite`. **Raw case file** — YAML, JSONL, or directory case data imported with `tests: ./cases.yaml`, string shorthand, or `type: tests`. Raw cases are reusable data inputs; they do not carry imported suite context such as shared `workspace`, shared `input`, or shared `assertions`. -**Wrapper eval** — Eval YAML whose main job is to import task suites and bind runtime policy with an inline `experiment:` block. Wrapper evals may live under an `experiments/` directory, but that path is an optional user-owned convention and AgentV does not infer behavior from it. A wrapper that imports suites with `type: suite` does not define parent `workspace`; imported suites own task environment. +**Wrapper eval** — Eval YAML whose main job is to import task suites and bind top-level runtime policy such as target selection, repeat count, timeout, budget, and thresholds. Wrapper evals may live under an `experiments/` directory, but that path is an optional user-owned convention and AgentV does not infer behavior from it. A wrapper that imports suites with `type: suite` does not define parent `workspace`; imported suites own task environment. -**Experiment** — The run-policy namespace for how evals are executed: target or target matrix, eval filters, repeat counts, timeouts, workers, budgets, thresholds, and related run knobs. In authored files it lives as inline `experiment:` inside eval YAML; CLI `--experiment` and `experiment.name` choose the result bucket. Lifecycle setup belongs in `workspace.hooks` or `targets[].hooks`, not in a separate experiment artifact. +**Experiment** — A string metadata/run-grouping label such as `baseline`, `candidate`, `with_skills`, or `without_skills`. It is not a runtime-policy object and not a result path namespace. Runtime policy belongs in top-level eval fields or target objects; the experiment label is recorded in `summary.json` and `index.jsonl` for Dashboard grouping and comparison. Lifecycle setup belongs in `workspace.hooks` or `targets[].hooks`, not in a separate experiment artifact. **Workspace** — The task environment an eval prepares for the agent: repositories, templates, fixture files, and lifecycle hooks. It is not prompt input; use `input` for instructions and `workspace.repos[]` for multi-repo workspaces the agent can inspect or modify through tools. -**Run manifest** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `test_dir`, `summary_path`, and `grading_path`. +**Run bundle** — A committed local result directory at `.agentv/results//`. `summary.json` records run metadata such as `run_id` and `experiment`; `index.jsonl` records per-case rows. + +**Run manifest** — The root `summary.json` file in a run bundle. It owns aggregate run metadata and rollups such as `run_id`, `experiment`, timestamps, planned/completed counts, pass rate, score summaries, duration, tokens, and cost. + +**Result index** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `test_dir`, `summary_path`, and `grading_path`. **Result source identity** — The stable source identity for a result row: repo-relative `eval_path`, `test_id`, and `target`. `suite` and `name` are display metadata, not storage or routing identity. @@ -32,9 +36,11 @@ Shared domain vocabulary for this project — entities, named processes, and sta **Artifact sidecar** — A file beside or below a result directory that provides evidence for a result, such as `summary.json`, `grading.json`, `result.json`, transcripts, logs, or outputs. Sidecars are evidence, not the primary discovery mechanism for a run. +**Artifact attempt folder** — A per-case `run-N/` folder under a result directory. It stores one materialized execution's sidecars and outputs. It is not the primary comparison dimension: stochastic samples and infrastructure retries should be represented with explicit sample/retry metadata rather than inferred from `run-1`, `run-2`, and so on. + ## Evaluation Reliability -**Repeat run** — A configured request to execute the same eval case and target more than once in the same timestamped run bundle. Repeat runs measure stochastic reliability, verifier stability, and drift; they are not the default CI path. +**Repeat run** — A configured request to execute the same eval case and target more than once in the same run bundle. Repeat runs measure stochastic reliability, verifier stability, and drift; they are not the default CI path. **Attempt** — One concrete execution inside a repeat run. Attempts keep their own score, status, timing, trace, transcript, logs, and artifacts so aggregate results never hide individual evidence. diff --git a/apps/cli/src/commands/eval/result-layout.ts b/apps/cli/src/commands/eval/result-layout.ts index 4cc8813ae..dc9f1efa4 100644 --- a/apps/cli/src/commands/eval/result-layout.ts +++ b/apps/cli/src/commands/eval/result-layout.ts @@ -8,7 +8,7 @@ export const DEFAULT_EXPERIMENT_NAME = 'default'; export const RESERVED_RESULTS_NAMESPACES = new Set(['export', 'metadata', 'runs']); export function isReservedResultsNamespace(value: string | undefined): boolean { - return value !== undefined && RESERVED_RESULTS_NAMESPACES.has(value); + return value !== undefined && (value.startsWith('.') || RESERVED_RESULTS_NAMESPACES.has(value)); } export function normalizeExperimentName(experiment?: string): string { @@ -32,8 +32,10 @@ export function createRunDirName(timestamp = new Date()): string { } function defaultRunPathSegments(experiment: string | undefined, runDirName: string): string[] { - const normalizedExperiment = normalizeExperimentName(experiment); - return [normalizedExperiment, runDirName]; + if (experiment !== undefined) { + normalizeExperimentName(experiment); + } + return [runDirName]; } export function buildResultsRootDir(cwd: string): string { @@ -159,7 +161,7 @@ export function relativeRunPathFromCwd(cwd: string, runDir: string): string | un } const parts = relative.split(path.sep).filter(Boolean); - if (parts.length < 2 || isReservedResultsNamespace(parts[0])) { + if (parts.length !== 1 || isReservedResultsNamespace(parts[0])) { return undefined; } diff --git a/apps/cli/src/commands/eval/run-cache.ts b/apps/cli/src/commands/eval/run-cache.ts index 95f8b3b46..342fa8429 100644 --- a/apps/cli/src/commands/eval/run-cache.ts +++ b/apps/cli/src/commands/eval/run-cache.ts @@ -17,7 +17,7 @@ const CACHE_FILENAME = 'cache.json'; * Keys use camelCase (not snake_case) for backward compat with pre-existing cache files. */ export interface RunCache { - /** Directory path for the run workspace (e.g. .agentv/results/default//) */ + /** Directory path for the run workspace (e.g. .agentv/results//) */ readonly lastRunDir?: string; /** @deprecated Legacy flat-file pointer from old cache files. Ignored on read. */ readonly lastResultFile?: string; diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 51a4a1041..f78854391 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1764,15 +1764,21 @@ export async function runEvalCommand( const explicitDir = options.outputDir; let runDir: string; let outputPath: string; - const runDirName = process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName(); + let runDirName = process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName(); if (explicitDir) { runDir = path.resolve(explicitDir); mkdirSync(runDir, { recursive: true }); outputPath = runDir; } else { - // Default: .agentv/results///. + // Default: .agentv/results//. runDir = buildDefaultRunDirFromName(cwd, resultGroupName, runDirName); + let suffix = 1; + while (existsSync(runDir)) { + runDirName = `${process.env.AGENTV_RUN_TIMESTAMP?.trim() || createRunDirName()}-${suffix}`; + runDir = buildDefaultRunDirFromName(cwd, resultGroupName, runDirName); + suffix++; + } mkdirSync(runDir, { recursive: true }); outputPath = runDir; } @@ -2111,6 +2117,7 @@ export async function runEvalCommand( evalFile, plannedTestCount: totalEvalCount, experiment: normalizeExperimentName(options.experiment), + runId: path.basename(runDir), experimentMetadata: runExperimentMetadata, runtimeSource: runtimeSourceMetadata, }); @@ -2382,6 +2389,7 @@ export async function runEvalCommand( const { writePerTestArtifacts } = await import('./artifact-writer.js'); await writePerTestArtifacts(allResults, runDir, { experiment: normalizeExperimentName(options.experiment), + runId: path.basename(runDir), resultGroup: resultGroupName, cwd, repoRoot, @@ -2392,6 +2400,7 @@ export async function runEvalCommand( const { summaryPath } = await aggregateRunDir(runDir, { evalFile, experiment: normalizeExperimentName(options.experiment), + runId: path.basename(runDir), experimentMetadata: runExperimentMetadata, runtimeSource: runtimeSourceMetadata, }); @@ -2407,6 +2416,7 @@ export async function runEvalCommand( { evalFile, experiment: normalizeExperimentName(options.experiment), + runId: path.basename(runDir), experimentMetadata: runExperimentMetadata, resultGroup: resultGroupName, cwd, diff --git a/apps/cli/src/commands/eval/wip-checkpoint.ts b/apps/cli/src/commands/eval/wip-checkpoint.ts index 3345d7061..b62bdf45c 100644 --- a/apps/cli/src/commands/eval/wip-checkpoint.ts +++ b/apps/cli/src/commands/eval/wip-checkpoint.ts @@ -15,8 +15,8 @@ * Manual recovery from a WIP branch: * git clone /tmp/recovery * cd /tmp/recovery && git checkout agentv/wip// - * cp -r / /.agentv/results// - * agentv eval --output /.agentv/results// --resume + * cp -r /.agentv/results/ + * agentv eval --output /.agentv/results/ --resume * * All checkpoint operations are best-effort: failures are logged as warnings * and never propagate to the eval run. diff --git a/apps/cli/src/commands/inspect/utils.ts b/apps/cli/src/commands/inspect/utils.ts index 3c95a4879..14583d15b 100644 --- a/apps/cli/src/commands/inspect/utils.ts +++ b/apps/cli/src/commands/inspect/utils.ts @@ -564,15 +564,7 @@ export interface ResultFileMeta { function buildRunId(relativeRunPath: string): string { const normalized = relativeRunPath.split(path.sep).join('/'); const segments = normalized.split('/').filter(Boolean); - if (segments.length >= 2) { - const experiment = segments.slice(0, -1).join('/'); - const timestamp = segments.at(-1); - if (experiment === 'default') { - return timestamp ?? normalized; - } - return `${experiment}::${timestamp}`; - } - return segments[0]; + return segments[0] ?? normalized; } function readRunDisplayName(runDir: string): string | undefined { @@ -604,13 +596,6 @@ function collectRunManifestPaths( }); return; } - - const entries = readdirSync(currentDir, { withFileTypes: true }); - for (const entry of entries) { - if (entry.isDirectory()) { - collectRunManifestPaths(runsDir, path.join(currentDir, entry.name), files); - } - } } function listResultFilesFromRoot( @@ -675,13 +660,15 @@ function listResultFilesFromRoot( /** * Enumerate canonical run manifests in `.agentv/results/`. * - * Reserved local namespaces such as `.agentv/results/runs/`, + * Reserved local namespaces such as `.agentv/results/.indexes/`, + * `.agentv/results/.cache/`, `.agentv/results/runs/`, * `.agentv/results/metadata/`, and `.agentv/results/export/` are intentionally - * skipped by default discovery. + * skipped by default discovery. Discovery only treats direct children of + * `.agentv/results/` as committed run bundles. */ export function listResultFiles(cwd: string, limit?: number): ResultFileMeta[] { const metas = listResultFilesFromRoot(buildResultsRootDir(cwd), { - skipTopLevelDirs: new Set(['export', 'metadata', 'runs']), + skipTopLevelDirs: new Set(['export', 'metadata', 'runs', '.indexes', '.cache']), }).sort((a, b) => { const byTimestamp = b.timestamp.localeCompare(a.timestamp); return byTimestamp !== 0 ? byTimestamp : b.displayName.localeCompare(a.displayName); @@ -692,7 +679,7 @@ export function listResultFiles(cwd: string, limit?: number): ResultFileMeta[] { export function listResultFilesFromRunsDir(runsDir: string, limit?: number): ResultFileMeta[] { return listResultFilesFromRoot(runsDir, { limit, - skipTopLevelDirs: new Set(['export', 'metadata', 'runs']), + skipTopLevelDirs: new Set(['export', 'metadata', 'runs', '.indexes', '.cache']), }); } diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index 616b09343..8e63cdf29 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -57,8 +57,7 @@ export const evalInputCommand = command({ out: option({ type: optional(string), long: 'out', - description: - 'Output directory for extracted inputs (default: .agentv/results//)', + description: 'Output directory for extracted inputs (default: .agentv/results/)', }), experiment: option({ type: optional(string), diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 1f8c7fc0f..89072679e 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -74,8 +74,7 @@ export const evalRunCommand = command({ out: option({ type: optional(string), long: 'out', - description: - 'Output directory for results (default: .agentv/results//)', + description: 'Output directory for results (default: .agentv/results/)', }), workers: option({ type: optional(number), diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts index 2d2440562..fe48ecebe 100644 --- a/apps/cli/src/commands/results/combine-run.ts +++ b/apps/cli/src/commands/results/combine-run.ts @@ -129,15 +129,17 @@ function readManifestRecords(manifestPath: string): ResultManifestRecord[] { function readSummaryMetadata(manifestPath: string): { timestamp?: string; displayName?: string; + experiment?: string; } { try { const summaryPath = path.join(path.dirname(manifestPath), 'summary.json'); const parsed = JSON.parse(readFileSync(summaryPath, 'utf8')) as { - metadata?: { timestamp?: string; display_name?: string }; + metadata?: { timestamp?: string; display_name?: string; experiment?: string }; }; return { timestamp: parsed.metadata?.timestamp, displayName: parsed.metadata?.display_name, + experiment: parsed.metadata?.experiment, }; } catch { return {}; @@ -318,21 +320,7 @@ function uniqueRunDir(baseDir: string): string { function toRunId(cwd: string, runDir: string): string { const relative = relativeRunPathFromCwd(cwd, runDir); - if (!relative) { - return path.basename(runDir); - } - const parts = relative.split(path.posix.sep); - if (parts.length <= 1) { - return relative; - } - const experiment = parts[0]; - const timestamp = parts.slice(1).join(path.posix.sep); - return experiment === 'default' ? timestamp : `${experiment}::${timestamp}`; -} - -function experimentFromRelativeRunPath(relativeRunPath: string): string { - const experiment = relativeRunPath.split(path.posix.sep).filter(Boolean)[0]; - return normalizeExperimentName(experiment); + return relative ?? path.basename(runDir); } function resolveCombinedExperiment( @@ -559,14 +547,20 @@ export function buildCombineRunSources( const relativeRunPath = relativeRunPathFromCwd(cwd, runDir); if (!relativeRunPath) { throw new Error( - `Run workspace is outside the canonical results layout: ${runDir}. Expected .agentv/results//`, + `Run workspace is outside the canonical results layout: ${runDir}. Expected .agentv/results/`, ); } + const records = readManifestRecords(manifestPath); + const summaryMetadata = readSummaryMetadata(manifestPath); + const experiment = + summaryMetadata.experiment ?? + records.find((record) => typeof record.experiment === 'string')?.experiment ?? + 'default'; return { id: options?.ids?.[index] ?? toRunId(cwd, runDir), displayName: options?.displayNames?.[index] ?? path.basename(runDir), manifestPath, - experiment: experimentFromRelativeRunPath(relativeRunPath), + experiment: normalizeExperimentName(experiment), tags: options?.tags?.[index], }; }); @@ -588,15 +582,7 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult ); const relativeOutputRunPath = relativeRunPathFromCwd(options.cwd, runDir); if (!relativeOutputRunPath) { - throw new Error( - `Output run workspace must use .agentv/results//: ${runDir}`, - ); - } - const outputExperiment = experimentFromRelativeRunPath(relativeOutputRunPath); - if (outputExperiment !== experiment) { - throw new Error( - `Output run workspace experiment "${outputExperiment}" must match combined experiment "${experiment}".`, - ); + throw new Error(`Output run workspace must use .agentv/results/: ${runDir}`); } const { rows, conflicts } = selectRows( loadedSources, @@ -616,7 +602,13 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult const manifestPath = path.join(runDir, RESULT_INDEX_FILENAME); writeJsonl(manifestPath, records); - const summary = buildRunSummaryArtifact(results, '', 'combined', results.length); + const summary = buildRunSummaryArtifact( + results, + '', + 'combined', + path.basename(runDir), + results.length, + ); const summaryWithMetadata: RunSummaryArtifact & { metadata: RunSummaryArtifact['metadata'] & { display_name: string; diff --git a/apps/cli/src/commands/results/combine.ts b/apps/cli/src/commands/results/combine.ts index 913c4fc71..c5fffdecf 100644 --- a/apps/cli/src/commands/results/combine.ts +++ b/apps/cli/src/commands/results/combine.ts @@ -103,7 +103,7 @@ export const resultsCombineCommand = command({ long: 'output', short: 'o', description: - 'Output run workspace directory (defaults to .agentv/results//)', + 'Output run workspace directory (defaults to .agentv/results/)', }), experiment: option({ type: optional(string), diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index a3d54b6af..93d776912 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -45,7 +45,7 @@ interface DashboardRun { command: string; /** Target name passed via --target (if any). Stored so the run list can show it before the first result is written. */ target?: string; - /** Absolute path to the run directory (e.g. .agentv/results/default/). Used to correlate this in-memory run with the filesystem run when the JSONL has 0 records yet. */ + /** Absolute path to the run directory (e.g. .agentv/results/). Used to correlate this in-memory run with the filesystem run when the JSONL has 0 records yet. */ outputDir?: string; startedAt: string; finishedAt?: string; diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts index b6285708f..ac2ba4a7f 100644 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ b/apps/cli/src/commands/results/remote-metadata.ts @@ -5,7 +5,7 @@ * immutable fetched payloads. Editable fields, starting with tags, live in a * small sidecar tree under `metadata/runs/**` inside the configured results repo * checkout/branch. This is a remote-results implementation detail, not part of - * the local `.agentv/results///` layout. It keeps local + * the local `.agentv/results//` layout. It keeps local * edits pushable by normal Git sync without rewriting the fetched run bundle. * * To add another mutable field: create a sibling helper that maps the remote diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index 79109d49e..6e759b863 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -182,9 +182,7 @@ export function getRelativeRunPath(cwd: string, runDir: string): string { return relative; } - throw new Error( - `Run workspace must use .agentv/results//: ${path.resolve(runDir)}`, - ); + throw new Error(`Run workspace must use .agentv/results/: ${path.resolve(runDir)}`); } function buildCommitTitle(payload: RemoteExportPayload): string { @@ -530,7 +528,7 @@ export async function ensureRemoteRunAvailable( } // On the results branch runs live flat under `runs/` (the branch namespaces - // results), so strip that prefix to recover /. + // results), so strip that prefix to recover . const relativeRunPath = path.posix.relative('runs', path.posix.dirname(relativeManifestPath)); await materializeGitRun(config.path, relativeRunPath, getResultsStorageRef(config)); } diff --git a/apps/cli/src/commands/results/serve-file-tree.test.ts b/apps/cli/src/commands/results/serve-file-tree.test.ts index 6aa722b36..dad66935c 100644 --- a/apps/cli/src/commands/results/serve-file-tree.test.ts +++ b/apps/cli/src/commands/results/serve-file-tree.test.ts @@ -40,7 +40,7 @@ function gitTranscriptEntry(prefix: string): ArtifactCatalogEntry { kind: 'transcript', storage: 'git', ref: 'agentv/artifacts/v1', - key: `runs/default/2026-06-22T01-12-44-924Z/${prefix}/transcript.jsonl`, + key: `runs/2026-06-22T01-12-44-924Z/${prefix}/transcript.jsonl`, }; } diff --git a/apps/cli/src/commands/results/validate.ts b/apps/cli/src/commands/results/validate.ts index 0f6a5a1ee..f9d102e87 100644 --- a/apps/cli/src/commands/results/validate.ts +++ b/apps/cli/src/commands/results/validate.ts @@ -3,7 +3,7 @@ * artifacts compatible with the AgentV dashboard and results commands. * * Checks: - * 1. Directory follows the `.agentv/results//` naming convention + * 1. Directory follows the `.agentv/results/` naming convention * 2. index.jsonl exists and each line has required fields * 3. Per-case summary.json exists for every entry in the index * 4. Per-run result.json and grading.json exist for every materialized trial @@ -56,13 +56,13 @@ function checkDirectoryNaming(runDir: string): Diagnostic[] { diagnostics.push({ severity: 'error', message: - 'The .agentv/results/runs namespace is no longer supported. Expected: .agentv/results//', + 'The .agentv/results/runs namespace is no longer supported. Expected: .agentv/results/', }); - } else if (resultsIndex < 0 || resultsIndex > pathSegments.length - 3) { + } else if (resultsIndex < 0 || resultsIndex !== pathSegments.length - 2) { diagnostics.push({ severity: 'warning', message: - 'Directory is not under the canonical results tree. Expected: .agentv/results//', + 'Directory is not under the canonical results tree. Expected: .agentv/results/', }); } diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 4604128f4..849808c91 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -401,6 +401,18 @@ describe('buildRunSummaryArtifact', () => { expect(benchmark.run_summary['gpt-4'].time_seconds.stddev).toBe(15); }); + it('records run_id and experiment as run metadata', () => { + const benchmark = buildRunSummaryArtifact( + [makeResult({})], + 'test.eval.yaml', + 'with-skills', + '2026-06-30T12-00-00-000Z', + ); + + expect(benchmark.metadata.run_id).toBe('2026-06-30T12-00-00-000Z'); + expect(benchmark.metadata.experiment).toBe('with-skills'); + }); + it('includes per-grader summary', () => { const results = [ makeResult({ diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts index 8c703103a..40a8783b3 100644 --- a/apps/cli/test/commands/eval/result-layout.test.ts +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -15,37 +15,37 @@ import { } from '../../../src/commands/eval/result-layout.js'; describe('result layout', () => { - it('groups default run directories under the default result group', () => { + it('writes default run directories directly under the results root', () => { const cwd = '/repo'; const timestamp = new Date('2026-06-22T12:34:56.789Z'); expect(buildDefaultRunDir(cwd, undefined, timestamp)).toBe( - path.join('/repo', '.agentv', 'results', 'default', '2026-06-22T12-34-56-789Z'), + path.join('/repo', '.agentv', 'results', '2026-06-22T12-34-56-789Z'), ); }); - it('groups named run directories under the result group', () => { + it('keeps experiment as metadata rather than a path segment', () => { expect(buildDefaultRunDirFromName('/repo', 'with-skills', '2026-run')).toBe( - path.join('/repo', '.agentv', 'results', 'with-skills', '2026-run'), + path.join('/repo', '.agentv', 'results', '2026-run'), ); }); it('reserves non-run namespaces at the results root', () => { - for (const namespace of ['export', 'metadata', 'runs']) { + for (const namespace of ['export', 'metadata', 'runs', '.indexes', '.cache']) { expect(() => normalizeExperimentName(namespace)).toThrow('reserved'); expect( - relativeRunPathFromCwd( - '/repo', - path.join('/repo', '.agentv', 'results', namespace, 'default', '2026-run'), - ), + relativeRunPathFromCwd('/repo', path.join('/repo', '.agentv', 'results', namespace)), ).toBeUndefined(); } + expect( + relativeRunPathFromCwd('/repo', path.join('/repo', '.agentv', 'results', '2026-run')), + ).toBe('2026-run'); expect( relativeRunPathFromCwd( '/repo', path.join('/repo', '.agentv', 'results', 'default', '2026-run'), ), - ).toBe('default/2026-run'); + ).toBeUndefined(); }); it('resolves the canonical index.jsonl file in a run directory', () => { @@ -61,7 +61,7 @@ describe('result layout', () => { } }); - it('discovers one canonical index.jsonl manifest per legacy nested bundle', () => { + it('discovers nested manifests only for explicit parser fallback paths', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-layout-test-')); try { const bundleDir = path.join(tempDir, 'default', '2026-run', 'target-a'); diff --git a/apps/cli/test/commands/eval/run-cache.test.ts b/apps/cli/test/commands/eval/run-cache.test.ts index e8a327cbb..2c5ccd2ba 100644 --- a/apps/cli/test/commands/eval/run-cache.test.ts +++ b/apps/cli/test/commands/eval/run-cache.test.ts @@ -6,11 +6,11 @@ import { type RunCache, resolveRunCacheFile } from '../../../src/commands/eval/r describe('resolveRunCacheFile', () => { it('should resolve new directory-based cache to index.jsonl inside dir', () => { const cache: RunCache = { - lastRunDir: '/results/default/2026-03-24T00-00-00-000Z', + lastRunDir: '/results/2026-03-24T00-00-00-000Z', timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), + path.join('/results/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); @@ -24,12 +24,12 @@ describe('resolveRunCacheFile', () => { it('should prefer lastRunDir over lastResultFile when both present', () => { const cache: RunCache = { - lastRunDir: '/results/default/2026-03-24T00-00-00-000Z', + lastRunDir: '/results/2026-03-24T00-00-00-000Z', lastResultFile: '/results/default/eval_old.jsonl', timestamp: '', }; expect(resolveRunCacheFile(cache)).toBe( - path.join('/results/default/2026-03-24T00-00-00-000Z', 'index.jsonl'), + path.join('/results/2026-03-24T00-00-00-000Z', 'index.jsonl'), ); }); diff --git a/apps/cli/test/commands/eval/wip-checkpoint.test.ts b/apps/cli/test/commands/eval/wip-checkpoint.test.ts index 21c4a8801..cfba28ba6 100644 --- a/apps/cli/test/commands/eval/wip-checkpoint.test.ts +++ b/apps/cli/test/commands/eval/wip-checkpoint.test.ts @@ -55,7 +55,7 @@ describe('WipCheckpointLoop', () => { branch_prefix: 'agentv/results', }, runDir: '/tmp/run-001', - destinationPath: 'default/run-001', + destinationPath: 'run-001', intervalMs: 1, dependencies: { buildWipBranchName: (runDir) => `agentv/wip/test/${runDir.split('/').pop()}`, diff --git a/apps/cli/test/commands/results/remote-auto-export.test.ts b/apps/cli/test/commands/results/remote-auto-export.test.ts index c818c75ee..191c9e6d2 100644 --- a/apps/cli/test/commands/results/remote-auto-export.test.ts +++ b/apps/cli/test/commands/results/remote-auto-export.test.ts @@ -58,7 +58,7 @@ ${params.branch ? ` branch: ${JSON.stringify(params.branch)}\n` : ''} path: ${ } function writeRunArtifacts(projectDir: string): string { - const runDir = path.join(projectDir, '.agentv', 'results', 'default', 'run-001'); + const runDir = path.join(projectDir, '.agentv', 'results', 'run-001'); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -80,7 +80,7 @@ function sha256Hex(content: Buffer | string): string { } function writeRunArtifactsWithPointers(projectDir: string): string { - const runDir = path.join(projectDir, '.agentv', 'results', 'default', 'run-002'); + const runDir = path.join(projectDir, '.agentv', 'results', 'run-002'); const artifactDir = path.join(runDir, 'alpha'); mkdirSync(artifactDir, { recursive: true }); const transcriptContent = Buffer.from( @@ -205,7 +205,7 @@ describe('maybeAutoExportRunArtifacts', () => { expect(status).toBe('published'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).toContain( - 'runs/default/run-001/index.jsonl', + 'runs/run-001/index.jsonl', ); }, 20_000); @@ -227,26 +227,21 @@ describe('maybeAutoExportRunArtifacts', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${resultsBranch}`, rootDir, ); - expect(resultTree).toContain('runs/default/run-002/index.jsonl'); - expect(resultTree).toContain('runs/default/run-002/summary.json'); - expect(resultTree).not.toContain('runs/default/run-002/alpha/trace.json'); - expect(resultTree).not.toContain('runs/default/run-002/alpha/transcript.jsonl'); + expect(resultTree).toContain('runs/run-002/index.jsonl'); + expect(resultTree).toContain('runs/run-002/summary.json'); + expect(resultTree).not.toContain('runs/run-002/alpha/trace.json'); + expect(resultTree).not.toContain('runs/run-002/alpha/transcript.jsonl'); const index = JSON.parse( - git( - `git --git-dir "${remoteDir}" show ${resultsBranch}:runs/default/run-002/index.jsonl`, - rootDir, - ), + git(`git --git-dir "${remoteDir}" show ${resultsBranch}:runs/run-002/index.jsonl`, rootDir), ); expect(index.artifact_pointers).not.toHaveProperty('trace'); - expect(index.artifact_pointers.transcript.key).toBe( - 'runs/default/run-002/alpha/transcript.jsonl', - ); + expect(index.artifact_pointers.transcript.key).toBe('runs/run-002/alpha/transcript.jsonl'); const artifactTree = git( `git --git-dir "${remoteDir}" ls-tree -r --name-only ${AGENTV_RESULTS_ARTIFACTS_REF}`, rootDir, ); - expect(artifactTree).not.toContain('runs/default/run-002/alpha/trace.json'); - expect(artifactTree).toContain('runs/default/run-002/alpha/transcript.jsonl'); + expect(artifactTree).not.toContain('runs/run-002/alpha/trace.json'); + expect(artifactTree).toContain('runs/run-002/alpha/transcript.jsonl'); }, 20_000); it('returns already_published when the final results branch is already up to date', async () => { @@ -321,10 +316,8 @@ describe('maybeAutoExportRunArtifacts', () => { expect(status).toBe('published'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( - 'runs/default/run-001/index.jsonl', - ); - expect(git('git ls-tree -r --name-only main', cloneDir)).toContain( - 'runs/default/run-001/index.jsonl', + 'runs/run-001/index.jsonl', ); + expect(git('git ls-tree -r --name-only main', cloneDir)).toContain('runs/run-001/index.jsonl'); }); }); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index a41fe79a9..87dbf36f7 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -93,8 +93,8 @@ function toJsonl(...records: object[]): string { return `${records.map((r) => JSON.stringify(r)).join('\n')}\n`; } -function localResultsExperimentDir(baseDir: string, experiment = 'default'): string { - return path.join(baseDir, '.agentv', 'results', experiment); +function localResultsExperimentDir(baseDir: string, _experiment = 'default'): string { + return path.join(baseDir, '.agentv', 'results'); } function localRunDir(baseDir: string, experiment: string, timestamp: string): string { @@ -102,8 +102,7 @@ function localRunDir(baseDir: string, experiment: string, timestamp: string): st } function localRunDirFromRunId(baseDir: string, runId: string): string { - const [experiment, timestamp] = runId.includes('::') ? runId.split('::') : ['default', runId]; - return localRunDir(baseDir, experiment ?? 'default', timestamp ?? runId); + return path.join(baseDir, '.agentv', 'results', runId); } function traceSessionEnvelope(input?: { @@ -258,7 +257,7 @@ function writeRemoteRunArtifact( /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, '$1T$2:$3:$4.$5Z', ); - const runDir = path.join(cloneDir, 'runs', experiment, timestamp); + const runDir = path.join(cloneDir, 'runs', timestamp); mkdirSync(runDir, { recursive: true }); const records = Array.isArray(resultRecords) ? resultRecords : [resultRecords]; writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); @@ -267,6 +266,7 @@ function writeRemoteRunArtifact( JSON.stringify( { metadata: { + run_id: timestamp, timestamp: isoTimestamp, experiment, targets: ['gpt-4o'], @@ -285,7 +285,7 @@ function writeRemoteRunArtifact( git(`git add "${runDir}" && git commit --quiet -m "add ${experiment}"`, cloneDir); git(`git push --quiet origin HEAD:${branch}`, cloneDir); git('git fetch --quiet origin --prune', cloneDir); - return `${experiment}::${timestamp}`; + return timestamp; } function writeDirtyRemoteRunArtifact( @@ -298,7 +298,7 @@ function writeDirtyRemoteRunArtifact( /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, '$1T$2:$3:$4.$5Z', ); - const runDir = path.join(cloneDir, 'runs', experiment, timestamp); + const runDir = path.join(cloneDir, 'runs', timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(resultRecord)); writeFileSync( @@ -306,6 +306,7 @@ function writeDirtyRemoteRunArtifact( JSON.stringify( { metadata: { + run_id: timestamp, timestamp: isoTimestamp, experiment, targets: ['gpt-4o'], @@ -321,7 +322,7 @@ function writeDirtyRemoteRunArtifact( 2, ), ); - return `${experiment}::${timestamp}`; + return timestamp; } function writeRemoteTagMetadataOverlay( @@ -330,7 +331,7 @@ function writeRemoteTagMetadataOverlay( timestamp: string, tags: readonly string[], ): string { - const metadataPath = path.join(repoDir, 'metadata', 'runs', experiment, timestamp, 'tags.json'); + const metadataPath = path.join(repoDir, 'metadata', 'runs', timestamp, 'tags.json'); mkdirSync(path.dirname(metadataPath), { recursive: true }); writeFileSync( metadataPath, @@ -351,12 +352,16 @@ function writeLocalRunArtifact( ); const runDir = localRunDir(projectDir, experiment, timestamp); mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl({ ...resultRecord, experiment })); + writeFileSync( + path.join(runDir, 'index.jsonl'), + toJsonl({ ...resultRecord, experiment, run_id: timestamp }), + ); writeFileSync( path.join(runDir, 'summary.json'), JSON.stringify( { metadata: { + run_id: timestamp, timestamp: isoTimestamp, experiment, targets: ['gpt-4o'], @@ -372,7 +377,7 @@ function writeLocalRunArtifact( 2, ), ); - return `${experiment}::${timestamp}`; + return timestamp; } function writeWtgDogfoodNoncanonicalArtifact(baseDir: string): { @@ -1563,7 +1568,7 @@ describe('serve app', () => { ); }); - it('infers the experiment name from the run id when live results have not written it yet', async () => { + it('does not infer the experiment name from the run id when live results have not written it yet', async () => { const runsDir = localResultsExperimentDir(tempDir, 'issue-1198-live-name'); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T12-00-00-000Z'; @@ -1580,9 +1585,9 @@ describe('serve app', () => { }; expect(data.runs).toHaveLength(1); expect(data.runs[0]).toMatchObject({ - experiment: 'issue-1198-live-name', target: 'gpt-4o', }); + expect(data.runs[0]).not.toHaveProperty('experiment'); }); it('merges cached remote runs and tags them with remote source metadata', async () => { @@ -1598,7 +1603,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'default', '2026-03-26T10-00-00-000Z', ); mkdirSync(remoteRunDir, { recursive: true }); @@ -1823,23 +1827,8 @@ describe('serve app', () => { metadata_dirty: true, }); - const localTagsPath = path.join( - tempDir, - '.agentv', - 'results', - 'runs', - experiment, - timestamp, - 'tags.json', - ); - const overlayTagsPath = path.join( - cloneDir, - 'metadata', - 'runs', - experiment, - timestamp, - 'tags.json', - ); + const localTagsPath = path.join(tempDir, '.agentv', 'results', timestamp, 'tags.json'); + const overlayTagsPath = path.join(cloneDir, 'metadata', 'runs', timestamp, 'tags.json'); expect(existsSync(localTagsPath)).toBe(false); expect(existsSync(overlayTagsPath)).toBe(true); @@ -1929,7 +1918,6 @@ describe('serve app', () => { const runManifestPath = path.join( cloneDir, 'runs', - 'external-sync', '2026-03-26T11-00-00-000Z', 'index.jsonl', ); @@ -1991,18 +1979,11 @@ describe('serve app', () => { metadata_dirty: true, }); - const artifactTagsPath = path.join( - cloneDir, - 'runs', - 'green-uat', - '2026-03-26T12-00-00-000Z', - 'tags.json', - ); + const artifactTagsPath = path.join(cloneDir, 'runs', '2026-03-26T12-00-00-000Z', 'tags.json'); const overlayTagsPath = path.join( cloneDir, 'metadata', 'runs', - 'green-uat', '2026-03-26T12-00-00-000Z', 'tags.json', ); @@ -2189,7 +2170,7 @@ describe('serve app', () => { }); describe('GET /api/projects/all-runs', () => { - it('infers experiment names for live benchmark runs before records persist them', async () => { + it('does not infer experiment names for live benchmark runs before records persist them', async () => { const homedirSpy = spyOn(os, 'homedir').mockReturnValue(path.join(tempDir, 'home')); try { @@ -2213,9 +2194,9 @@ describe('serve app', () => { expect(data.runs).toHaveLength(1); expect(data.runs[0]).toMatchObject({ project_id: project.id, - experiment: 'issue-1198-benchmark', target: 'gpt-4o', }); + expect(data.runs[0]).not.toHaveProperty('experiment'); } finally { homedirSpy.mockRestore(); } @@ -2366,17 +2347,7 @@ describe('serve app', () => { blocked: false, run_count: 1, }); - expect( - existsSync( - path.join( - cloneDir, - 'runs', - 'project-sync-pull', - runId.replace('project-sync-pull::', ''), - 'index.jsonl', - ), - ), - ).toBe(true); + expect(existsSync(path.join(cloneDir, 'runs', runId, 'index.jsonl'))).toBe(true); } finally { if (previousHome === undefined) { process.env.AGENTV_HOME = undefined; @@ -2441,7 +2412,7 @@ describe('serve app', () => { run_count: 1, }); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, tempDir)).toContain( - `metadata/runs/project-sync-push/${runTimestamp}/tags.json`, + `metadata/runs/${runTimestamp}/tags.json`, ); } finally { if (previousHome === undefined) { @@ -2537,13 +2508,7 @@ describe('serve app', () => { }); const runTimestamp = '2026-03-26T13-00-00-000Z'; - const relativeMetadataPath = path.posix.join( - 'metadata', - 'runs', - 'project-sync-conflict', - runTimestamp, - 'tags.json', - ); + const relativeMetadataPath = path.posix.join('metadata', 'runs', runTimestamp, 'tags.json'); writeRemoteTagMetadataOverlay(seedDir, 'project-sync-conflict', runTimestamp, ['base']); git('git add metadata && git commit --quiet -m "seed tag metadata"', seedDir); git('git push --quiet origin main', seedDir); @@ -2622,17 +2587,9 @@ describe('serve app', () => { expect(data.blocked).toBe(false); expect(data.run_count).toBe(1); expect( - existsSync( - path.join( - cloneDir, - 'runs', - 'confirm-merge-unscoped', - '2026-03-26T14-00-00-000Z', - 'index.jsonl', - ), - ), + existsSync(path.join(cloneDir, 'runs', '2026-03-26T14-00-00-000Z', 'index.jsonl')), ).toBe(true); - expect(runId).toBe('confirm-merge-unscoped::2026-03-26T14-00-00-000Z'); + expect(runId).toBe('2026-03-26T14-00-00-000Z'); }, 15000); it('resumes sync by pulling the target branch (project-scoped)', async () => { @@ -2685,17 +2642,7 @@ describe('serve app', () => { blocked: false, run_count: 1, }); - expect( - existsSync( - path.join( - cloneDir, - 'runs', - 'project-confirm-merge', - runId.replace('project-confirm-merge::', ''), - 'index.jsonl', - ), - ), - ).toBe(true); + expect(existsSync(path.join(cloneDir, 'runs', runId, 'index.jsonl'))).toBe(true); } finally { if (previousHome === undefined) { process.env.AGENTV_HOME = undefined; @@ -2816,7 +2763,7 @@ describe('serve app', () => { ); } return { - runId: opts?.experiment ? `${opts.experiment}::${name}` : name, + runId: name, runDir, manifestPath, }; @@ -2904,7 +2851,7 @@ describe('serve app', () => { expect(accepted.status).toBe(201); const acceptedData = (await accepted.json()) as { run_id: string; experiment: string }; expect(acceptedData.experiment).toBe('smoke-regression'); - expect(acceptedData.run_id).toStartWith('smoke-regression::'); + expect(acceptedData.run_id).not.toContain('::'); expect(existsSync(localRunDirFromRunId(tempDir, acceptedData.run_id))).toBe(true); const detailRes = await app.request(`/api/runs/${encodeURIComponent(acceptedData.run_id)}`); @@ -3054,7 +3001,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'default', '2026-06-01T11-00-00-000Z', ); mkdirSync(remoteRunDir, { recursive: true }); @@ -3125,7 +3071,7 @@ describe('serve app', () => { writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n'); return { - runId: opts?.experiment ? `${opts.experiment}::${name}` : name, + runId: name, runDir, }; } @@ -3157,7 +3103,6 @@ describe('serve app', () => { 'EntityProcess-agentv-evals', '.agentv', 'results', - 'default', '2026-06-01T11-00-00-000Z', ); mkdirSync(remoteRunDir, { recursive: true }); @@ -3219,7 +3164,7 @@ describe('serve app', () => { traceContent: string, recordOverrides?: Record, ): string { - const runId = `${experiment}::${timestamp}`; + const runId = timestamp; const runDir = localRunDir(projectDir, experiment, timestamp); const tracePath = path.join(runDir, traceArtifactPath); mkdirSync(path.dirname(tracePath), { recursive: true }); @@ -3238,7 +3183,7 @@ describe('serve app', () => { it('projects a local AgentV trace sidecar through the Dashboard read model', async () => { const traceArtifactPath = 'demo/test-greeting/trace.json'; - const runId = 'with-trace::2026-03-25T09-00-00-000Z'; + const runId = '2026-03-25T09-00-00-000Z'; writeLocalTraceRun( tempDir, 'with-trace', @@ -3341,7 +3286,7 @@ describe('serve app', () => { it('returns a typed dangling state when the trace pointer cannot be read', async () => { const runsDir = localResultsExperimentDir(tempDir, 'dangling-trace'); - const runId = 'dangling-trace::2026-03-25T10-45-00-000Z'; + const runId = '2026-03-25T10-45-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-45-00-000Z'); const artifactPath = 'demo/test-greeting/trace.json'; @@ -3454,7 +3399,7 @@ describe('serve app', () => { const secret = 'outside trace secret'; writeFileSync(path.join(tempDir, 'outside-trace.json'), secret); const runsDir = localResultsExperimentDir(tempDir, 'escaped-trace'); - const runId = 'escaped-trace::2026-03-25T11-00-00-000Z'; + const runId = '2026-03-25T11-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T11-00-00-000Z'); const artifactPath = '../../../../../outside-trace.json'; @@ -3485,7 +3430,7 @@ describe('serve app', () => { describe('GET /api/runs/:filename/evals/:evalId/transcript', () => { it('loads canonical transcript JSONL lazily from the manifest pointer', async () => { const runsDir = localResultsExperimentDir(tempDir, 'with-transcript'); - const runId = 'with-transcript::2026-03-25T10-00-00-000Z'; + const runId = '2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; const answerArtifactPath = 'demo/test-greeting/answer.md'; @@ -3547,7 +3492,7 @@ describe('serve app', () => { it('loads pointer-shaped transcript metadata when it resolves to a local artifact path', async () => { const runsDir = localResultsExperimentDir(tempDir, 'pointer-transcript'); - const runId = 'pointer-transcript::2026-03-25T11-00-00-000Z'; + const runId = '2026-03-25T11-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T11-00-00-000Z'); const artifactPath = 'demo/test-greeting/transcript.jsonl'; const transcriptPath = path.join(timestampDir, artifactPath); @@ -3598,11 +3543,11 @@ describe('serve app', () => { const resultsBranch = 'dogfood/serve-sidecar-contract'; const experiment = 'sidecar-only'; const timestamp = '2026-03-25T15-00-00-000Z'; - const runId = `remote::${experiment}::${timestamp}`; + const runId = `remote::${timestamp}`; const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; const traceArtifactPath = 'demo/test-greeting/trace.json'; - const transcriptKey = `runs/${experiment}/${timestamp}/${transcriptArtifactPath}`; - const traceKey = `runs/${experiment}/${timestamp}/${traceArtifactPath}`; + const transcriptKey = `runs/${timestamp}/${transcriptArtifactPath}`; + const traceKey = `runs/${timestamp}/${traceArtifactPath}`; const transcriptJsonl = `${JSON.stringify({ schema_version: 'agentv.transcript.v1', test_id: 'test-greeting', @@ -3622,13 +3567,14 @@ describe('serve app', () => { git(`git switch --quiet --orphan ${resultsBranch}`, seedDir); git('git rm -rf --quiet . 2>/dev/null || true', seedDir); - const runDir = path.join(seedDir, 'runs', experiment, timestamp); + const runDir = path.join(seedDir, 'runs', timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), toJsonl({ ...RESULT_A, experiment, + run_id: timestamp, artifact_pointers: { trace: { ref: AGENTV_RESULTS_ARTIFACTS_REF, @@ -3787,7 +3733,7 @@ describe('serve app', () => { it('returns a clear dangling state when the transcript pointer cannot be read', async () => { const runsDir = localResultsExperimentDir(tempDir, 'dangling-transcript'); - const runId = 'dangling-transcript::2026-03-25T13-00-00-000Z'; + const runId = '2026-03-25T13-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-00-00-000Z'); const artifactPath = 'demo/test-greeting/transcript.jsonl'; @@ -3823,7 +3769,7 @@ describe('serve app', () => { writeFileSync(outsidePath, secret); const runsDir = localResultsExperimentDir(tempDir, 'escaped-transcript'); - const runId = 'escaped-transcript::2026-03-25T13-30-00-000Z'; + const runId = '2026-03-25T13-30-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-30-00-000Z'); const artifactPath = 'demo/test-greeting/transcript.jsonl'; const symlinkPath = path.join(timestampDir, artifactPath); @@ -3858,7 +3804,7 @@ describe('serve app', () => { writeFileSync(outsidePath, secret); const runsDir = localResultsExperimentDir(tempDir, 'escaped-answer'); - const runId = 'escaped-answer::2026-03-25T13-45-00-000Z'; + const runId = '2026-03-25T13-45-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T13-45-00-000Z'); const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl'; const answerArtifactPath = 'demo/test-greeting/answer.md'; @@ -3915,14 +3861,7 @@ describe('serve app', () => { transcript_path: transcriptArtifactPath, trace_path: traceArtifactPath, }); - const timestampDir = path.join( - tempDir, - '.agentv', - 'results', - 'runs', - 'lazy-guard', - timestamp, - ); + const timestampDir = path.join(tempDir, '.agentv', 'results', timestamp); mkdirSync(path.join(timestampDir, transcriptArtifactPath), { recursive: true }); mkdirSync(path.dirname(path.join(timestampDir, traceArtifactPath)), { recursive: true }); writeFileSync( @@ -3965,54 +3904,43 @@ describe('serve app', () => { }); describe('GET /api/runs/:filename/evals/:evalId/files/*', () => { - it('discovers nested bundle indexes and loads the requested row sidecar by manifest metadata', async () => { + it('loads the requested row sidecar by direct run manifest metadata', async () => { const runsDir = localResultsExperimentDir(tempDir, 'multi-target'); const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); const alphaDir = 'case-one--111111111111'; const betaDir = 'case-one--222222222222'; - const alphaBundleDir = path.join(timestampDir, 'storage-alpha'); - const betaBundleDir = path.join(timestampDir, 'storage-beta'); - const alphaAnswer = path.join(alphaBundleDir, alphaDir, 'run-1', 'outputs', 'answer.md'); - const betaAnswer = path.join(betaBundleDir, betaDir, 'run-1', 'outputs', 'answer.md'); + const alphaAnswer = path.join(timestampDir, alphaDir, 'run-1', 'outputs', 'answer.md'); + const betaAnswer = path.join(timestampDir, betaDir, 'run-1', 'outputs', 'answer.md'); mkdirSync(path.dirname(alphaAnswer), { recursive: true }); mkdirSync(path.dirname(betaAnswer), { recursive: true }); writeFileSync(alphaAnswer, 'alpha answer'); writeFileSync(betaAnswer, 'beta answer'); writeFileSync( - path.join(alphaBundleDir, 'index.jsonl'), - toJsonl({ - ...RESULT_A, - experiment: 'multi-target', - test_id: 'case-one', - target: 'mock-alpha', - result_dir: alphaDir, - answer_path: `${alphaDir}/run-1/outputs/answer.md`, - }), - ); - writeFileSync( - path.join(betaBundleDir, 'index.jsonl'), - toJsonl({ - ...RESULT_A, - experiment: 'multi-target', - test_id: 'case-one', - target: 'mock-beta', - result_dir: betaDir, - answer_path: `${betaDir}/run-1/outputs/answer.md`, - }), + path.join(timestampDir, 'index.jsonl'), + toJsonl( + { + ...RESULT_A, + experiment: 'multi-target', + test_id: 'case-one', + target: 'mock-alpha', + result_dir: alphaDir, + answer_path: `${alphaDir}/run-1/outputs/answer.md`, + }, + { + ...RESULT_A, + experiment: 'multi-target', + test_id: 'case-one', + target: 'mock-beta', + result_dir: betaDir, + answer_path: `${betaDir}/run-1/outputs/answer.md`, + }, + ), ); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - const listRes = await app.request('/api/runs'); - expect(listRes.status).toBe(200); - const listData = (await listRes.json()) as { - runs: Array<{ filename: string; target?: string }>; - }; - const betaRun = listData.runs.find((run) => run.target === 'mock-beta'); - expect(betaRun?.filename).toBeTruthy(); - const res = await app.request( - `/api/runs/${encodeURIComponent(betaRun?.filename ?? '')}/evals/case-one/files/${betaDir}/run-1/outputs/answer.md?result_dir=${encodeURIComponent(betaDir)}`, + `/api/runs/${encodeURIComponent('2026-03-25T10-00-00-000Z')}/evals/case-one/files/${betaDir}/run-1/outputs/answer.md?result_dir=${encodeURIComponent(betaDir)}`, ); expect(res.status).toBe(200); @@ -4020,9 +3948,9 @@ describe('serve app', () => { expect(data.content).toBe('beta answer'); }); - it('loads file content for experiment-scoped run ids', async () => { + it('loads file content for run ids with experiment row metadata', async () => { const runsDir = localResultsExperimentDir(tempDir, 'with-skills'); - const runId = 'with-skills::2026-03-25T10-00-00-000Z'; + const runId = '2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); const responsePath = path.join( timestampDir, @@ -4055,7 +3983,7 @@ describe('serve app', () => { it('serves transcript JSONL artifacts as browser-visible raw text and downloads', async () => { const runsDir = localResultsExperimentDir(tempDir, 'with-transcript'); - const runId = 'with-transcript::2026-03-25T10-00-00-000Z'; + const runId = '2026-03-25T10-00-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z'); const artifactPath = 'demo/test-greeting/transcript.jsonl'; const transcriptPath = path.join(timestampDir, artifactPath); @@ -4100,7 +4028,7 @@ describe('serve app', () => { writeFileSync(outsidePath, secret); const runsDir = localResultsExperimentDir(tempDir, 'escaped-file'); - const runId = 'escaped-file::2026-03-25T10-30-00-000Z'; + const runId = '2026-03-25T10-30-00-000Z'; const timestampDir = path.join(runsDir, '2026-03-25T10-30-00-000Z'); const artifactPath = 'demo/test-greeting/outputs/response.md'; const symlinkPath = path.join(timestampDir, artifactPath); @@ -4353,7 +4281,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: '.agentv/results/default/2026-05-06T00-00-00-000Z', + output: '.agentv/results/2026-05-06T00-00-00-000Z', resume: true, }), }); @@ -4363,7 +4291,7 @@ describe('serve app', () => { expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--resume'); - expect(data.command).toContain('--output .agentv/results/default/2026-05-06T00-00-00-000Z'); + expect(data.command).toContain('--output .agentv/results/2026-05-06T00-00-00-000Z'); }); it('builds --rerun-failed + --output flags from the request', async () => { @@ -4374,14 +4302,14 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', rerun_failed: true, }), }); expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--rerun-failed'); - expect(data.command).toContain('--output .agentv/results/default/r1'); + expect(data.command).toContain('--output .agentv/results/r1'); }); it('builds a selected experiment output path and writes initial tags beside the new run', async () => { @@ -4399,7 +4327,7 @@ describe('serve app', () => { expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--experiment smoke'); - expect(data.command).toContain(path.join('.agentv', 'results', 'smoke')); + expect(data.command).toContain(path.join('.agentv', 'results')); const outputDir = data.command.match(/--output ([^\s]+)/)?.[1]; expect(outputDir).toBeString(); @@ -4418,12 +4346,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: '.agentv/results/default/r0/index.jsonl', + retry_errors: '.agentv/results/r0/index.jsonl', }), }); expect(res.status).toBe(202); const data = (await res.json()) as { command: string }; - expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); + expect(data.command).toContain('--retry-errors .agentv/results/r0/index.jsonl'); }); it('rejects resume + rerun_failed combo with 400', async () => { @@ -4433,7 +4361,7 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', resume: true, rerun_failed: true, }), @@ -4450,9 +4378,9 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', resume: true, - retry_errors: '.agentv/results/default/r0/index.jsonl', + retry_errors: '.agentv/results/r0/index.jsonl', }), }); expect(res.status).toBe(400); @@ -4465,7 +4393,7 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', resume: true, tags: ['baseline'], }), @@ -4483,7 +4411,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', resume: true, - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', }), }); expect(res.status).toBe(403); @@ -4497,7 +4425,7 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', resume: true, - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', }), }); expect(res.status).toBe(403); @@ -4565,14 +4493,14 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', target: 'gpt-4o', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', resume: true, }), }); expect(res.status).toBe(200); const data = (await res.json()) as { command: string }; expect(data.command).toContain('--resume'); - expect(data.command).toContain('--output .agentv/results/default/r1'); + expect(data.command).toContain('--output .agentv/results/r1'); expect(data.command).not.toContain('--rerun-failed'); }); @@ -4583,7 +4511,7 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/default/r1', + output: '.agentv/results/r1', rerun_failed: true, }), }); @@ -4600,12 +4528,12 @@ describe('serve app', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', - retry_errors: '.agentv/results/default/r0/index.jsonl', + retry_errors: '.agentv/results/r0/index.jsonl', }), }); expect(res.status).toBe(200); const data = (await res.json()) as { command: string }; - expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); + expect(data.command).toContain('--retry-errors .agentv/results/r0/index.jsonl'); }); it('emits --experiment for selected experiment requests', async () => { @@ -4704,7 +4632,7 @@ describe('serve app', () => { source: 'local' | 'remote'; }; expect(data.source).toBe('local'); - expect(data.run_dir).toBe(path.join('.agentv', 'results', 'default', filename)); + expect(data.run_dir).toBe(path.join('.agentv', 'results', filename)); expect(data.suite_filter).toBe('examples/demo.eval.yaml'); }); @@ -4747,7 +4675,7 @@ describe('serve app', () => { ); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - const res = await app.request(`/api/runs/${encodeURIComponent(`cli-smoke::${filename}`)}`); + const res = await app.request(`/api/runs/${encodeURIComponent(filename)}`); expect(res.status).toBe(200); const data = (await res.json()) as { diff --git a/apps/cli/test/commands/results/validate.test.ts b/apps/cli/test/commands/results/validate.test.ts index 8b483a7bc..a33bf01fb 100644 --- a/apps/cli/test/commands/results/validate.test.ts +++ b/apps/cli/test/commands/results/validate.test.ts @@ -6,17 +6,11 @@ import path from 'node:path'; import { validateRunDirectory } from '../../../src/commands/results/validate.js'; describe('results validate', () => { - it('accepts experiment-scoped result directories without layout warnings', () => { + it('accepts v2 run-root result directories without layout warnings', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-test-')); try { - const runDir = path.join( - tempDir, - '.agentv', - 'results', - 'with-skills', - '2026-03-27T12-42-24-429Z', - ); + const runDir = path.join(tempDir, '.agentv', 'results', '2026-03-27T12-42-24-429Z'); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -56,7 +50,7 @@ describe('results validate', () => { expect(diagnostics.filter((d) => d.severity === 'error')).toEqual([]); expect(diagnostics.map((d) => d.message)).not.toContain( - 'Directory is not under the canonical results tree. Expected: .agentv/results//', + 'Directory is not under the canonical results tree. Expected: .agentv/results/', ); expect( diagnostics.some((d) => d.message.includes('does not match the expected pattern')), @@ -70,13 +64,7 @@ describe('results validate', () => { const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-validate-test-')); try { - const runDir = path.join( - tempDir, - '.agentv', - 'results', - 'with-skills', - '2026-03-27T12-42-24-429Z', - ); + const runDir = path.join(tempDir, '.agentv', 'results', '2026-03-27T12-42-24-429Z'); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index dd9629bf6..c069b44c8 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -265,8 +265,8 @@ describe('trace utils', () => { expect(metas).toEqual([]); }); - it('should enumerate run workspaces in .agentv/results/default/', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); + it('should enumerate direct run workspaces in .agentv/results/', () => { + const runsDir = path.join(tempDir, '.agentv', 'results'); mkdirSync(runsDir, { recursive: true }); const olderRunDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); @@ -342,8 +342,26 @@ describe('trace utils', () => { expect(metas).toEqual([]); }); + it('should skip dot-prefixed local result namespaces by default', () => { + for (const namespace of ['.indexes', '.cache']) { + const runDir = path.join( + tempDir, + '.agentv', + 'results', + namespace, + '2026-02-20T21-38-05-833Z', + ); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), `${RESULT_WITH_TRACE}\n`); + } + + const metas = listResultFiles(tempDir); + + expect(metas).toEqual([]); + }); + it('should respect limit', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); + const runsDir = path.join(tempDir, '.agentv', 'results'); mkdirSync(runsDir, { recursive: true }); const olderRunDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); @@ -358,8 +376,8 @@ describe('trace utils', () => { expect(metas[0].filename).toBe('2026-02-21T10-00-00-000Z'); }); - it('should ignore non-directory entries in results experiments', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); + it('should ignore non-directory entries in results root', () => { + const runsDir = path.join(tempDir, '.agentv', 'results'); mkdirSync(runsDir, { recursive: true }); writeFileSync(path.join(runsDir, 'notes.txt'), 'not a result file'); @@ -369,8 +387,8 @@ describe('trace utils', () => { expect(metas).toHaveLength(0); }); - it('should discover index.jsonl inside grouped run directories', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); + it('should discover index.jsonl inside direct run directories', () => { + const runsDir = path.join(tempDir, '.agentv', 'results'); const runDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); @@ -387,7 +405,7 @@ describe('trace utils', () => { expect(metas[0].filename).toBe('2026-02-20T21-38-05-833Z'); }); - it('should discover nested experiment run directories and emit safe run ids', () => { + it('should not discover deprecated nested experiment run directories', () => { const runsDir = path.join(tempDir, '.agentv', 'results'); const runDir = path.join(runsDir, 'with-skills', '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); @@ -396,14 +414,12 @@ describe('trace utils', () => { const metas = listResultFiles(tempDir); - expect(metas).toHaveLength(1); - expect(metas[0].filename).toBe('with-skills::2026-02-20T21-38-05-833Z'); - expect(metas[0].displayName).toBe('2026-02-20T21-38-05-833Z'); + expect(metas).toEqual([]); }); it('should use benchmark metadata display names when listing run workspaces', () => { const runsDir = path.join(tempDir, '.agentv', 'results'); - const runDir = path.join(runsDir, 'combined', '2026-02-20T21-38-05-833Z'); + const runDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), `${RESULT_WITH_TRACE}\n`); @@ -419,12 +435,12 @@ describe('trace utils', () => { const metas = listResultFiles(tempDir); expect(metas).toHaveLength(1); - expect(metas[0].filename).toBe('combined::2026-02-20T21-38-05-833Z'); + expect(metas[0].filename).toBe('2026-02-20T21-38-05-833Z'); expect(metas[0].displayName).toBe('Combined run (dogfood-run-a + dogfood-run-b)'); }); it('should skip directories without index.jsonl', () => { - const runsDir = path.join(tempDir, '.agentv', 'results', 'default'); + const runsDir = path.join(tempDir, '.agentv', 'results'); const emptyDir = path.join(runsDir, '2026-02-20T21-38-05-833Z'); mkdirSync(emptyDir, { recursive: true }); diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index 0de71c52e..cd95956a4 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -1230,9 +1230,7 @@ function formatTimestamp(iso: string): string { * `encodeRemoteRunId` in `apps/cli/src/commands/results/remote.ts`, so * the shape is one of: * - `2026-04-01T10-00-00-000Z` → null - * - `with-skills::2026-04-01T10-00-00-000Z` → "with-skills" * - `remote::2026-04-01T10-00-00-000Z` → "remote" - * - `remote::with-skills::2026-04-01T10-...` → "remote · with-skills" * * The full run id stays available via the `title` attribute on the * timestamp cell so keyboard / pointer users can always recover it. diff --git a/apps/dashboard/src/components/resume-run-helpers.test.ts b/apps/dashboard/src/components/resume-run-helpers.test.ts index 70991808f..d2e977f0e 100644 --- a/apps/dashboard/src/components/resume-run-helpers.test.ts +++ b/apps/dashboard/src/components/resume-run-helpers.test.ts @@ -66,13 +66,13 @@ describe('buildResumeRequestBody', () => { expect( buildResumeRequestBody({ mode: 'resume', - runDir: '.agentv/results/default/2026-05-06T00-00-00-000Z', + runDir: '.agentv/results/2026-05-06T00-00-00-000Z', suiteFilter: 'examples/demo.eval.yaml', target: 'gpt-4o', }), ).toEqual({ suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/default/2026-05-06T00-00-00-000Z', + output: '.agentv/results/2026-05-06T00-00-00-000Z', target: 'gpt-4o', resume: true, }); diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 2ff3ea439..ce73f8947 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -225,13 +225,10 @@ top-level or case-level `workspace`. ## Repeat Runs -Use top-level `repeat` when you want AgentV to try each case more than once: +Use top-level `runs` when you want AgentV to try each case more than once: ```yaml -repeat: - count: 3 - strategy: pass_any - early_exit: false +runs: 3 ``` `repeat.strategy` controls verdict aggregation. `pass_any` treats the case as @@ -243,13 +240,13 @@ the first fail. Leave it unset or `false` when you want complete variance data. ## Result Layout -Eval runs write to the selected result bucket: +Eval runs write to a direct run bundle: ```text -.agentv/results/// +.agentv/results// ``` -CLI `--experiment` sets the bucket explicitly. Without that flag, AgentV uses +CLI `--experiment` sets the experiment label explicitly. Without that flag, AgentV uses top-level `experiment`, then top-level `name`, then the eval filename. The Dashboard uses "Experiment" for the comparison and result grouping concept; folder names are only storage allocation and must not define result semantics. diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index fd1429997..02b750eb6 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -11,13 +11,12 @@ sidebar: agentv eval evals/my-eval.yaml ``` -Results are written to `.agentv/results///index.jsonl`. -AgentV picks the experiment bucket from `--experiment`, then -`eval.yaml` `experiment.name`, then `default`. Each CLI invocation writes one -timestamped run bundle. Each line is a JSON object with one result per test -case, and the run workspace also stores the manifest and related artifacts. Use -this generated run folder as the portable audit surface: copy or sync the run -directory, not a hand-authored parallel bundle. See the +Results are written to `.agentv/results//index.jsonl`. Each CLI +invocation writes one run bundle. The experiment label is stored in +`summary.json` and row metadata. Each line is a JSON object with one result per +test case, and the run workspace also stores the summary and related artifacts. +Use this generated run folder as the portable audit surface: copy or sync the +run directory, not a hand-authored parallel bundle. See the [Result Artifact Contract](/docs/reference/result-artifacts/) for the complete run layout and reader rules. @@ -187,7 +186,7 @@ result-oriented workflows. For full-fidelity span inspection, export OTLP JSON e ```bash # Summary-level inspection from the run manifest -agentv inspect stats .agentv/results/default//index.jsonl +agentv inspect stats .agentv/results//index.jsonl # Full-fidelity OTLP JSON trace (importable by OTel backends like Jaeger, Grafana) agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json @@ -363,13 +362,13 @@ AgentV ships three flags for picking up a partial run. They differ only in **whi agentv eval evals/my-eval.yaml --resume # Or target a specific run dir explicitly -agentv eval evals/my-eval.yaml --output .agentv/results/default/ --resume +agentv eval evals/my-eval.yaml --output .agentv/results/ --resume # Re-run errors AND failed cases against the last run dir agentv eval evals/my-eval.yaml --rerun-failed # Re-run only execution errors from any prior run by path -agentv eval evals/my-eval.yaml --retry-errors .agentv/results/default//index.jsonl +agentv eval evals/my-eval.yaml --retry-errors .agentv/results//index.jsonl ``` After any failing run, the CLI prints the exact `--rerun-failed` command for the run dir that just completed — copy/paste it. If the process or pod disappeared before you could access the local run directory and results auto-push was enabled, recover the partial run from [WIP checkpoints](/docs/tools/wip-checkpoints/) first, then use the same `--resume` flow. diff --git a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx index 76ffec463..b844759d4 100644 --- a/apps/web/src/content/docs/docs/getting-started/quickstart.mdx +++ b/apps/web/src/content/docs/docs/getting-started/quickstart.mdx @@ -65,7 +65,7 @@ tests: agentv eval ./evals/example.yaml ``` -Results appear in `.agentv/results/default//index.jsonl` with scores, reasoning, and execution traces. +Results appear in `.agentv/results//index.jsonl` with scores, reasoning, and execution traces. ## Next Steps diff --git a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx index a07acbb0d..7b8575f0d 100644 --- a/apps/web/src/content/docs/docs/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/reference/result-artifacts.mdx @@ -24,58 +24,62 @@ The default local layout is: ```text .agentv/results/ - / - / - summary.json - index.jsonl - tags.json # optional mutable Dashboard tags - / - summary.json # optional per-case aggregate, especially repeats - test/ # optional generated test bundle - EVAL.yaml - targets.yaml - files/ - graders/ - run-1/ - result.json - grading.json - metrics.json - timing.json - transcript.jsonl - transcript-raw.jsonl - outputs/ - answer.md - file_changes.diff - run-2/ - result.json - grading.json - metrics.json - timing.json - transcript.jsonl - transcript-raw.jsonl - outputs/ - answer.md - file_changes.diff + / + summary.json + index.jsonl + tags.json # optional mutable Dashboard tags + / + summary.json # optional per-case aggregate, especially repeats + test/ # optional generated test bundle + EVAL.yaml + targets.yaml + files/ + graders/ + run-1/ + result.json + grading.json + metrics.json + timing.json + transcript.jsonl + transcript-raw.jsonl + outputs/ + answer.md + file_changes.diff + run-2/ + result.json + grading.json + metrics.json + timing.json + transcript.jsonl + transcript-raw.jsonl + outputs/ + answer.md + file_changes.diff + .indexes/ # reserved rebuildable/local indexes + .cache/ # reserved local cache ``` -The `` and `` directories are storage allocation. They help -AgentV put completed runs somewhere predictable, but readers must not infer -semantic truth from folder names. Use fields in `summary.json` and `index.jsonl` -for experiment, target, variant, attempt, eval path, case identity, timing, -scores, and artifact paths. +`` is the only committed run-bundle path identity. It helps AgentV put +completed runs somewhere predictable, but readers must not infer semantic truth +from folder names. Use fields in `summary.json` and `index.jsonl` for +experiment, target, variant, attempt, eval path, case identity, timing, scores, +and artifact paths. The run bundle does not add target, model, variant, or `cases/` folders below ``. Per-result directories are allocated from row identity, usually with a readable test-id or slug prefix plus a short hash suffix, and remain opaque to consumers. -`experiment` remains the comparison and runtime-policy concept: it is how users -label a condition such as `baseline`, `candidate`, `with_skills`, or -`without_skills`. The folder segment is a convenient bucket for that concept, -not an alternate schema. If a bundle is copied, combined, published, or imported -under a different directory, its rows still carry the facts consumers should +`experiment` is metadata: it is how users label a condition such as `baseline`, +`candidate`, `with_skills`, or `without_skills`. It is recorded in +`summary.json` and rows, not as a parent directory and not as a runtime-policy +object. If a bundle is copied, combined, published, or imported under a +different directory, its metadata still carries the facts consumers should query. +Top-level dot-prefixed directories such as `.indexes/` and `.cache/` are +reserved for rebuildable local state and are skipped by run discovery. + ## File Roles | File or field | Owns | Use it for | @@ -146,6 +150,13 @@ manual `prepare`/`grade` attempts, or imported provider sessions. That is why source metadata belong in `index.jsonl`: tools can filter dynamically without requiring every run to be pre-split into semantic folders. +The `run-1/`, `run-2/`, and later folders under a result directory are artifact +attempt/execution folders. Do not treat those folder names as the comparison +dimension. Repeated stochastic samples should be represented by explicit +metadata such as `sample_index` and `sample_count`; infrastructure retries +should use retry metadata such as `retry_index`, `retry_count`, and +`retry_reason` when available. + ## Reader Rules Consumers should read a bundle in this order: @@ -180,9 +191,9 @@ Run an eval and inspect the portable bundle: ```bash agentv eval evals/support/refunds.eval.yaml --experiment with_skills -ls .agentv/results/with_skills/ -cat .agentv/results/with_skills//summary.json -cat .agentv/results/with_skills//index.jsonl +ls .agentv/results/ +cat .agentv/results//summary.json +cat .agentv/results//index.jsonl ``` Find failed rows without loading every sidecar: @@ -190,21 +201,21 @@ Find failed rows without loading every sidecar: ```bash jq -r 'select(.execution_status != "ok" or .score < 0.5) | [.eval_path, .test_id, .target, .grading_path] | @tsv' \ - .agentv/results/with_skills//index.jsonl + .agentv/results//index.jsonl ``` Compare two completed runs by their row indexes: ```bash agentv compare \ - .agentv/results/baseline//index.jsonl \ - .agentv/results/candidate//index.jsonl + .agentv/results//index.jsonl \ + .agentv/results//index.jsonl ``` Generate a shareable report from the same canonical bundle: ```bash -agentv results report .agentv/results/with_skills/ +agentv results report .agentv/results/ ``` ## Integration Author Examples @@ -229,10 +240,10 @@ export async function* rows(runDir: string) { } } -for await (const row of rows(".agentv/results/with_skills/2026-run")) { +for await (const row of rows(".agentv/results/2026-run")) { const gradingPath = row.grading_path; if (typeof gradingPath === "string") { - console.log(path.join(".agentv/results/with_skills/2026-run", gradingPath)); + console.log(path.join(".agentv/results/2026-run", gradingPath)); } } ``` diff --git a/apps/web/src/content/docs/docs/tools/compare.mdx b/apps/web/src/content/docs/docs/tools/compare.mdx index 0db165da6..23a626448 100644 --- a/apps/web/src/content/docs/docs/tools/compare.mdx +++ b/apps/web/src/content/docs/docs/tools/compare.mdx @@ -12,15 +12,14 @@ The `compare` command computes deltas between two evaluation runs for A/B testin Run two evaluations and compare them: ```bash -agentv eval evals/my-eval.yaml --output .agentv/results/default/before +agentv eval evals/my-eval.yaml --output .agentv/results/before # ... make changes to your agent ... -agentv eval evals/my-eval.yaml --output .agentv/results/default/after -agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl +agentv eval evals/my-eval.yaml --output .agentv/results/after +agentv compare .agentv/results/before/index.jsonl .agentv/results/after/index.jsonl ``` -`index.jsonl` is the canonical row-level result manifest. Existing -`index.jsonl` run manifests from older AgentV runs remain readable for -compatibility, but new runs write `index.jsonl`. +`index.jsonl` is the canonical row-level result index. New runs live at +`.agentv/results//index.jsonl`. ## Options @@ -127,13 +126,13 @@ Compare different model versions: ```bash # Run baseline evaluation -agentv eval evals/*.yaml --target gpt-4 --output .agentv/results/default/baseline +agentv eval evals/*.yaml --target gpt-4 --output .agentv/results/baseline # Run candidate evaluation -agentv eval evals/*.yaml --target gpt-4o --output .agentv/results/default/candidate +agentv eval evals/*.yaml --target gpt-4o --output .agentv/results/candidate # Compare results -agentv compare .agentv/results/default/baseline/index.jsonl .agentv/results/default/candidate/index.jsonl +agentv compare .agentv/results/baseline/index.jsonl .agentv/results/candidate/index.jsonl ``` ### Prompt Optimization @@ -142,13 +141,13 @@ Compare before/after prompt changes: ```bash # Run with original prompt -agentv eval evals/*.yaml --output .agentv/results/default/before +agentv eval evals/*.yaml --output .agentv/results/before # Modify prompt, then run again -agentv eval evals/*.yaml --output .agentv/results/default/after +agentv eval evals/*.yaml --output .agentv/results/after # Compare with strict threshold -agentv compare .agentv/results/default/before/index.jsonl .agentv/results/default/after/index.jsonl --threshold 0.05 +agentv compare .agentv/results/before/index.jsonl .agentv/results/after/index.jsonl --threshold 0.05 ``` ### CI Quality Gate @@ -158,8 +157,8 @@ Fail CI if the candidate regresses: ```bash #!/bin/bash agentv compare \ - .agentv/results/default/baseline/index.jsonl \ - .agentv/results/default/candidate/index.jsonl + .agentv/results/baseline/index.jsonl \ + .agentv/results/candidate/index.jsonl if [ $? -eq 1 ]; then echo "Regression detected! Candidate performs worse than baseline." exit 1 diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 66e646d8f..6a49a222b 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -31,7 +31,7 @@ The `dashboard` command launches a web-based dashboard for browsing evaluation r agentv dashboard ``` -Dashboard auto-discovers run workspaces from `.agentv/results///` in the current directory and opens at `http://localhost:3117`. Runs without an explicit experiment use `.agentv/results/default//`. +Dashboard auto-discovers v2 run workspaces from `.agentv/results//` in the current directory and opens at `http://localhost:3117`. Experiment is read from `summary.json` or row metadata, not from the path. To open a different project, pass the project root with `--dir`: @@ -39,7 +39,7 @@ To open a different project, pass the project root with `--dir`: agentv dashboard --dir /path/to/project ``` -Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. The old `.agentv/results/runs/**` layout is not a Dashboard-visible layout. For one-off inspection of a copied run bundle, use `agentv results report `. +Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. For one-off inspection of a copied run bundle, use `agentv results report `. ## Data boundary @@ -109,7 +109,7 @@ In the per-test results table, click a test ID to open its checks, transcript, s ## Run management -In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. If all selected runs are from one experiment, the combined run inherits that experiment, including `default`; if selected runs span experiments, Dashboard asks for a new experiment name before creating the combined run. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. +In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. If all selected runs are from one experiment, the combined run records that experiment label, including `default`; if selected runs span experiments, Dashboard asks for a new experiment name before creating the combined run. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. When you launch an eval from Dashboard, set the experiment and initial tags before the run starts. The selected experiment is recorded with the new run, and tags are written to that run workspace's `tags.json` sidecar; existing runs are not changed. @@ -163,9 +163,9 @@ Select 2+ rows with the checkboxes and click the sticky **Compare N** action to ### Retroactive tags -Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the timestamped result folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. +Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the run folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. -Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs/.../tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results///` layout. Remote tag overlays use the same `tag_revision` stale-write check as local tags. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. +Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs//tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results//` layout. Remote tag overlays use the same `tag_revision` stale-write check as local tags. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. Use tags to annotate ad-hoc variants, experiment cross-cuts, or status flags you didn't plan for up front — `baseline`, `v2-prompt`, `slow`, `after-retry-fix`, `regression`, etc. Unlike `experiment` — which groups runs and is baked into the JSONL at eval-run time — tags are mutable, multi-valued, and never touch the original run data. @@ -360,15 +360,9 @@ Use project-level **Sync Project** as the results exchange workflow. It handles There is no separate `agentv results remote status` or `agentv results remote sync` command. The `agentv results` CLI stays focused on local run workspaces; manual remote exchange is Dashboard/API-only, with eval auto-export covering the common CI/publisher path. -Each run writes to a unique timestamped directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. Temporary result/PR branch names use a fixed prefix; they are not the storage branch. +Each run writes to a unique run-id directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. Temporary result/PR branch names use a fixed prefix; they are not the storage branch. -### What happens to existing local runs? - -Existing runs already present under `.agentv/results///` stay exactly where they are and continue to appear in Dashboard as **local** runs. Runs in the removed `.agentv/results/runs/**` layout are not discovered by Dashboard. - -Adding a `results` block does **not** backfill those historical runs into the results branch automatically. Result publishing only affects runs created after the results repo is configured. `auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. - -If you want older local-only runs in the remote repo, rerun them or copy the run directories into the managed clone manually before syncing the project. +Adding a `results` block does not backfill local run workspaces into the results branch automatically. Result publishing affects runs created after the results repo is configured. `auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. ### Authentication diff --git a/apps/web/src/content/docs/docs/tools/inspect.mdx b/apps/web/src/content/docs/docs/tools/inspect.mdx index bbac42fbd..e7c7913bb 100644 --- a/apps/web/src/content/docs/docs/tools/inspect.mdx +++ b/apps/web/src/content/docs/docs/tools/inspect.mdx @@ -94,7 +94,7 @@ agentv inspect show trace.otlp.json --format json \ | jq '[.[] | select(.cost_usd > 0.10) | {test_id, score, cost: .cost_usd}]' # Compare providers -agentv inspect stats .agentv/results/default//index.jsonl --group-by target --format json \ +agentv inspect stats .agentv/results//index.jsonl --group-by target --format json \ | jq '.groups[] | {label, score_mean: .metrics.score.mean}' ``` diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx index 87d378ddb..dc1a28075 100644 --- a/apps/web/src/content/docs/docs/tools/results.mdx +++ b/apps/web/src/content/docs/docs/tools/results.mdx @@ -29,7 +29,7 @@ start with [Result Artifact Contract](/docs/reference/result-artifacts/). | `results show` | Display case-level rows from a run workspace | | `results validate` | Validate that a workspace or manifest resolves correctly | -`results combine` writes the new run under the source experiment when every selected source run belongs to the same experiment, including `default`. If the source runs span multiple experiments, pass `--experiment ` for the new combined run; AgentV does not silently write mixed-experiment combines under a `combined` namespace. +`results combine` writes a new direct run workspace under `.agentv/results//` and records the selected experiment label in `summary.json` and `index.jsonl` metadata. If the source runs span multiple experiments, pass `--experiment ` for the new combined run; AgentV does not silently invent a mixed-experiment label. ## `results report` @@ -45,10 +45,10 @@ Examples: ```bash # Generate report.html next to the run manifest -agentv results report .agentv/results/default/2026-03-14T10-32-00_claude +agentv results report .agentv/results/2026-03-14T10-32-00_claude # Use an explicit output path -agentv results report .agentv/results/default/2026-03-14T10-32-00_claude/index.jsonl \ +agentv results report .agentv/results/2026-03-14T10-32-00_claude/index.jsonl \ --out ./reports/human-review.html ``` @@ -127,13 +127,16 @@ It is the compact executor behavior summary for dashboards, comparison exports, and metric-style graders; it is not canonical trace storage and does not carry token/cost usage. -Every case uses aggregate `summary.json`, then stores attempt details under -`run-N/`. Each `run-N/` contains a compact per-attempt manifest `result.json`, -`grading.json`, `metrics.json`, `timing.json`, `transcript.jsonl`, -`transcript-raw.jsonl`, `outputs/answer.md`, and `outputs/file_changes.diff` -when workspace changes were captured. The `result.json` file carries -AgentV `execution_status` and `verdict` fields plus `grading_path`, -`metrics_path`, transcript, output, and `file_changes_path` paths. +Every case uses aggregate `summary.json`, then stores execution artifact details +under `run-N/`. Each `run-N/` contains a compact per-attempt manifest +`result.json`, `grading.json`, `metrics.json`, `timing.json`, +`transcript.jsonl`, `transcript-raw.jsonl`, `outputs/answer.md`, and +`outputs/file_changes.diff` when workspace changes were captured. The +`result.json` file carries AgentV `execution_status` and `verdict` fields plus +`grading_path`, `metrics_path`, transcript, output, and `file_changes_path` +paths. Treat `run-N/` as an artifact attempt folder, not as a comparison +dimension; stochastic samples and infrastructure retries should be represented +with explicit sample/retry metadata rather than inferred from folder names. `transcript-raw.jsonl` preserves native provider or harness transcript bytes when they are available, while `transcript.jsonl` is the normalized @@ -239,10 +242,10 @@ policy so downstream processing is auditable. For lightweight terminal workflows: ```bash -agentv results summary .agentv/results/default/ -agentv results failures .agentv/results/default/ -agentv results show .agentv/results/default/ --test-id my-case -agentv results validate .agentv/results/default/ +agentv results summary .agentv/results/ +agentv results failures .agentv/results/ +agentv results show .agentv/results/ --test-id my-case +agentv results validate .agentv/results/ ``` For a review-centric workflow built around these artifacts, see [Human Review Checkpoint](/docs/guides/human-review/). @@ -253,7 +256,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a Use these supported remote workflows instead: -- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `results.repo` with `results.path` pointing at the source checkout and `results.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.path` without `results.repo` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. +- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `results.repo` with `results.path` pointing at the source checkout and `results.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs//`. The configured results branch remains the metadata/control plane (`index.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.path` without `results.repo` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled. - **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**. - **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. - **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.path` clone with `git` directly, then sync again. diff --git a/apps/web/src/content/docs/docs/tools/trend.mdx b/apps/web/src/content/docs/docs/tools/trend.mdx index b28ff4908..857732a28 100644 --- a/apps/web/src/content/docs/docs/tools/trend.mdx +++ b/apps/web/src/content/docs/docs/tools/trend.mdx @@ -29,9 +29,9 @@ Point directly at run workspaces or `index.jsonl` manifests when you need a spec ```bash agentv trend \ - .agentv/results/default/2026-03-01T10-00-00-000Z/ \ - .agentv/results/default/2026-03-08T10-00-00-000Z/index.jsonl \ - .agentv/results/default/2026-03-15T10-00-00-000Z/ + .agentv/results/2026-03-01T10-00-00-000Z/ \ + .agentv/results/2026-03-08T10-00-00-000Z/index.jsonl \ + .agentv/results/2026-03-15T10-00-00-000Z/ ``` Concrete regression-gating example: @@ -45,13 +45,12 @@ agentv trend --last 8 --suite code-review --target claude-sonnet \ `trend` only accepts canonical run workspaces: -- `.agentv/results///` -- `.agentv/results///index.jsonl` +- `.agentv/results//` +- `.agentv/results//index.jsonl` -Legacy `index.jsonl` manifests from older AgentV runs remain readable when -passed directly or when they are the only manifest in a run workspace. Legacy -flat `results.jsonl` files are rejected. The command stays on lightweight -`index.jsonl` manifests and does not require per-test artifact hydration. +Legacy flat `results.jsonl` files are rejected. The command stays on +lightweight `index.jsonl` manifests and does not require per-test artifact +hydration. ## Options @@ -115,7 +114,7 @@ Regression Gate: threshold=0.010 fail_on_degrading=true triggered=true "runs": [ { "label": "2026-03-01T10:00:00.000Z", - "path": "/repo/.agentv/results/default/2026-03-01T10-00-00-000Z/index.jsonl", + "path": "/repo/.agentv/results/2026-03-01T10-00-00-000Z/index.jsonl", "timestamp": "2026-03-01T10:00:00.000Z", "matched_test_count": 42, "mean_score": 0.92 diff --git a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx index 9baf16d93..cbb675944 100644 --- a/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx +++ b/apps/web/src/content/docs/docs/tools/wip-checkpoints.mdx @@ -22,8 +22,8 @@ If no results repo is configured, or auto-push is disabled, `agentv eval` still | Location | Path or ref | What it contains | | --- | --- | --- | -| Local project | `.agentv/results///summary.json` | A run-start stub with `metadata.planned_test_count` and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | -| Local project | `.agentv/results///index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | +| Local project | `.agentv/results//summary.json` | A run-start stub with `metadata.run_id`, `metadata.experiment`, `metadata.planned_test_count`, and the eval file path when known. This lets Dashboard recognize incomplete local runs as resumable. | +| Local project | `.agentv/results//index.jsonl` | Result rows appended as test cases finish. Rows use the normal snake_case result JSONL format. | | Results repo remote | `agentv/wip//` | A forced-updated branch containing the checkpointed run under `.agentv/results//`. | | Results repo storage branch | Configured `results.repo.branch`; local checkout configs default to `agentv/results/v1` | The final published run after `agentv eval` completes and the normal auto-export succeeds. | @@ -57,14 +57,14 @@ git switch --detach origin/agentv/wip// # 4. Inspect the checkpointed run path. find .agentv/results -name summary.json -# 5. Copy the run tree into the eval project, preserving experiment paths. +# 5. Copy the run tree into the eval project, preserving run ids. PROJECT=/path/to/eval-project mkdir -p "$PROJECT/.agentv/results" rsync -a .agentv/results/ "$PROJECT/.agentv/results/" # 6. Resume from the recovered run directory. cd "$PROJECT" -agentv eval --output .agentv/results// --resume +agentv eval --output .agentv/results/ --resume ``` If the recovered `summary.json` contains `metadata.eval_file`, use that as ``. diff --git a/docs/adr/0011-result-output-artifact-contract.md b/docs/adr/0011-result-output-artifact-contract.md index 65d318fda..69685d856 100644 --- a/docs/adr/0011-result-output-artifact-contract.md +++ b/docs/adr/0011-result-output-artifact-contract.md @@ -16,6 +16,10 @@ Extends: keeps result identity in `index.jsonl` rows and uses `default` as the fallback result experiment. +Updated by [ADR 0012](0012-finalize-run-artifact-layout.md), which makes +artifact-format v2 runs direct children of `.agentv/results/` and treats +`experiment` as run metadata rather than path identity. + ## Context AgentV needs a result output contract that works for local runs, CI gates, @@ -46,7 +50,7 @@ metadata, but AgentV's run bundle remains the source of truth. An AgentV result output is a run-centric bundle with this root contract: ```text -.agentv/results/// +.agentv/results// summary.json index.jsonl tags.json # optional mutable overlay @@ -102,23 +106,22 @@ source for run identity or artifact discovery. ## Directory Paths Are Allocation -The `.agentv/results///` path is storage allocation. It -gives AgentV a predictable place to write and discover completed bundles, but -the path does not define semantic truth. +The `.agentv/results//` path is storage allocation. It gives AgentV a +predictable place to write and discover completed bundles, but the path does +not define semantic truth. -The experiment remains AgentV's comparison and runtime-policy concept. Users -can label run conditions such as `baseline`, `candidate`, `with_skills`, or -`without_skills`, and tools can use that label for grouping and comparison. -However, readers must use row and summary metadata for semantics. If a run is -copied under a different folder, combined with another run, synced to a results -branch, or imported from another machine, the manifest fields still carry the -truth. +The experiment label remains AgentV's run grouping metadata. Users can label +conditions such as `baseline`, `candidate`, `with_skills`, or `without_skills`, +and tools can use that label for grouping and comparison. However, readers must +use row and summary metadata for semantics. If a run is copied under a different +folder, combined with another run, synced to a results branch, or imported from +another machine, the manifest fields still carry the truth. -AgentV must not use a semantic `experiments//...` folder hierarchy as the -source of truth. A repository may keep wrapper eval YAML files under a directory -named `experiments/`, but that is user-owned organization for ordinary eval -files. It does not define result identity, runtime behavior, or Dashboard -routing. +AgentV must not use a semantic `experiments//...` or +`.agentv/results///` folder hierarchy as the source of +truth. A repository may keep eval YAML files under a directory named +`experiments/`, but that is user-owned organization for ordinary eval files. It +does not define result identity, runtime behavior, or Dashboard routing. `result_dir` is also allocation. It should stay readable when possible, but it can be suffixed or otherwise allocated to avoid collisions. The public row @@ -159,9 +162,9 @@ This aligns with Margin Evals' manifest-first and run-centric lessons: model. AgentV does not copy Margin Evals' exact layout. AgentV keeps its own -`summary.json` plus `index.jsonl` split, AgentV transcript sidecars, repeat-run -attempt folders, generated test bundles, Git-backed result branch model, and -optional detached `artifact_pointers`. +`summary.json` plus `index.jsonl` split, AgentV transcript sidecars, +`run-N/` artifact attempt folders, generated test bundles, Git-backed result +branch model, and optional detached `artifact_pointers`. ## Consequences @@ -217,7 +220,9 @@ files as the canonical contract. - Projecting AgentV-owned runs, transcripts, datasets, experiments, or indexes into Phoenix. - Requiring a semantic `experiments/` directory. -- Removing compatibility readers for older bundles in this ADR. +- Preserving Dashboard discovery for older `.agentv/results///` + bundles after artifact-format v2. ADR 0012 hard-deprecates that layout for new + v2 run discovery. - Freezing every possible row field. This ADR defines ownership and discovery; field additions remain additive and versioned. diff --git a/docs/adr/0012-finalize-run-artifact-layout.md b/docs/adr/0012-finalize-run-artifact-layout.md index 5f617506e..82dd7ef49 100644 --- a/docs/adr/0012-finalize-run-artifact-layout.md +++ b/docs/adr/0012-finalize-run-artifact-layout.md @@ -1,4 +1,4 @@ -# 12. Finalize run artifact layout at the timestamp bundle +# 12. Artifact layout v2 stores committed runs at the results root Date: 2026-06-30 @@ -6,53 +6,44 @@ Date: 2026-06-30 Accepted -Supersedes the target/variant folder fan-out portions of -[ADR 0009](0009-eval-path-result-identity-and-default-experiment.md) and -[ADR 0006](0006-separate-experiments-from-eval-definitions.md). Extends -[ADR 0011](0011-result-output-artifact-contract.md), which keeps result output -run-centric and manifest-first. +Supersedes the experiment-parent result layout portions of +[ADR 0009](0009-eval-path-result-identity-and-default-experiment.md), +[ADR 0006](0006-separate-experiments-from-eval-definitions.md), and +[ADR 0011](0011-result-output-artifact-contract.md). ## Context -AgentV now treats the timestamped result directory as the run bundle boundary: +AgentV run bundles are the portable source of truth for Dashboard, reports, +compare/trend tooling, CI gates, and export adapters. The previous layout used +the experiment label as a parent directory: ```text .agentv/results/// ``` -Earlier same-week decisions used target and variant folders below the timestamp -to avoid sidecar collisions in multi-target runs. The implementation has since -settled on allocated per-row result directories with readable test-id prefixes -and short hash suffixes. That allocation already solves collisions without -making target, model, variant, suite, or test IDs path dimensions. - -The relevant implementation points are: - -- `apps/cli/src/commands/eval/result-layout.ts` creates default run roots as - `.agentv/results///` and keeps `index.jsonl` as the - manifest filename. -- `packages/core/src/evaluation/run-artifacts.ts` writes `summary.json`, - `index.jsonl`, and per-result sidecars under allocated `result_dir` folders - such as `--/run-1/`. -- `apps/cli/src/commands/results/manifest.ts`, - `apps/cli/src/commands/results/serve.ts`, and - `packages/core/src/evaluation/results-repo.ts` consume explicit manifest - fields such as `result_dir`, `summary_path`, `grading_path`, `metrics_path`, - and `transcript_path` instead of deriving sidecar locations from directory - names. +That made a mutable grouping label look like storage identity. It also forced +Dashboard and trend discovery to infer experiment from ancestry when the run +summary should carry that metadata explicitly. + +Artifact-format v2 phase 1 removes that path dependency. The active schema +direction also treats `experiment` as a string metadata/run-grouping label, not +as an object wrapper for runtime policy. Runtime fields such as `target`, +`runs`, `early_exit`, `timeout_seconds`, `budget_usd`, and `threshold` belong at +the eval root or target object as the schema defines them; this ADR does not +duplicate that schema migration. ## Decision -New AgentV runs write one run bundle at: +New committed local run bundles are written directly under the results root: ```text -.agentv/results/// +.agentv/results// summary.json index.jsonl - tags.json # optional mutable overlay - / + tags.json # optional mutable Dashboard tags + -/ summary.json - test/ # optional generated test bundle + test/ run-1/ result.json grading.json @@ -61,106 +52,109 @@ New AgentV runs write one run bundle at: transcript.jsonl transcript-raw.jsonl outputs/ - run-2/ - ... ``` -Do not add `target`, `model`, `variant`, or `cases` as required folders below -or above ``. Target, model, provider, variant, eval path, suite, and -test identity are metadata. They belong in root `summary.json.metadata` for -run-level facts and in `index.jsonl` rows for row-level filtering and artifact -discovery. - -`index.jsonl` remains the filename for the run manifest/result index. The name -is established across CLI, Dashboard, result repo sync, compare, trend, and -adapter code. Renaming it would create churn without improving the contract. -Documentation should call it the run manifest or result index where that role is -clearer. - -`result_dir` values are opaque run-local allocations. Writers should keep them -readable when possible, using a safe test-id or slug prefix plus a UUID/hash-like -suffix, but consumers must not parse identity from those names. Consumers must -resolve ordinary sidecars through explicit `index.jsonl` fields such as: - -- `result_dir` -- `summary_path` -- `grading_path` -- `timing_path` -- `metrics_path` -- `transcript_path` -- `transcript_raw_path` -- `answer_path` -- `test_dir` +`summary.json` is the run-level manifest/summary. It must include enough +metadata for Dashboard discovery without path inference, including `run_id`, +`experiment` when known, target/model/provider metadata when available, +timestamps, planned/completed counts, and aggregate stats. + +`index.jsonl` remains the per-case result index. It is intentionally not renamed +to `manifest.jsonl` in this phase. Rows continue to use explicit run-relative +path fields such as `result_dir`, `summary_path`, `grading_path`, +`metrics_path`, `timing_path`, `transcript_path`, `transcript_raw_path`, +`answer_path`, and `test_dir`. + +The top-level `.agentv/results/` namespace reserves dot-prefixed directories for +rebuildable or local state: + +```text +.agentv/results/.indexes/ +.agentv/results/.cache/ +``` + +Discovery must skip dot-prefixed top-level directories. Existing non-run +namespaces such as `metadata`, `export`, and the removed `runs` namespace remain +reserved. + +The per-case `run-1/`, `run-2/`, etc. folders stay in place. They are artifact +attempt/execution folders, not the primary comparison dimension. Repeated +stochastic evaluation should be represented by explicit sample metadata such as +`sample_index` and `sample_count`; infrastructure retries should be represented +separately with retry metadata such as `retry_index`, `retry_count`, and +`retry_reason` when that schema exists. Do not overload `run-N` names to mean +both samples and retries. + +The results repository storage branch stores committed run bundles as +`runs//` and mutable metadata overlays as `metadata/runs//`. +That branch is already a results namespace, so it does not include the +`.agentv/results/` prefix. ## Compatibility -Legacy bundles that already contain target-folder manifests remain readable. -Readers may discover nested `index.jsonl` files when a run root has no direct -manifest, and they must continue to honor legacy `index.jsonl` rows whose -explicit paths point into old target-folder layouts. Do not move old artifacts -as part of this decision. +This is a hard deprecation of the old experiment-parent layout for +artifact-format v2. New writers do not create +`.agentv/results///`, and Dashboard/result discovery is +not required to show those legacy bundles. -When a root `index.jsonl` exists, it is the authoritative manifest for that run -directory. Nested target-folder manifests are legacy compatibility input, not a -new writer contract. +Users who need old runs to appear in v2 Dashboard views should regenerate or +re-export them into `.agentv/results//` with `experiment` recorded in +`summary.json` metadata. Small parser fallbacks may remain where existing tools +need them for explicit paths, but they are not the product contract. ## Consequences Positive: -- New run bundles have one obvious root manifest and summary. -- Dashboard and results-repo listings can use root `summary.json.metadata` - fields such as `targets` without walking per-result rows for basic run facts. -- Multi-target and variant rows still avoid filesystem collisions through - allocated result directories. -- Target/model comparisons stay a query over run and row metadata instead of a - storage hierarchy. +- A run id is the only committed run-bundle path identity. +- Experiment grouping is explicit metadata, so copied or re-exported bundles do + not lose meaning when paths change. +- Dashboard and trend discovery can skip local cache/index namespaces without + special-casing experiment names. +- Compare dimensions such as experiment, target, variant, samples, retries, and + tags stay query metadata instead of storage hierarchy. Negative: -- Humans cannot browse target folders under a timestamp. They must use - `summary.json`, `index.jsonl`, Dashboard filters, or compare tooling. -- Some accepted same-week ADR text now requires this superseding ADR for the - final layout. +- Old local result directories may disappear from v2 Dashboard discovery until + regenerated or re-exported. +- Users cannot browse all runs for an experiment by opening one parent folder; + they should use Dashboard filters, `summary.json.metadata.experiment`, tags, + or CLI queries. ## Alternatives Considered -### Target or model folders under the timestamp - -Rejected. Target/model folders make storage look semantic and encourage readers -to infer identity from paths. They also create needless nesting for the -single-target case and become awkward when target, provider, model, variant, and -runtime policy are all useful comparison dimensions. - -### Target folders above the timestamp +### Preserve legacy discovery during migration -Rejected. Moving target above timestamp fragments one run invocation into -multiple run roots and makes run-level summary metadata harder to define. +Rejected. The product direction for v2 is hard deprecation. Keeping path +fallbacks as a supported discovery mode would keep experiment ancestry as an +implicit source of truth. -### A `cases/` parent folder +### Add `.agentv/results/runs//` -Rejected. `index.jsonl` already distinguishes control-plane files from -per-result sidecars. Adding `cases/` would be a cosmetic migration with no -current reader or writer need. +Rejected for this phase. The extra `runs/` segment is redundant in a directory +that already stores results and conflicts with the reserved local namespace +model. -### Rename `index.jsonl` +### Rename `index.jsonl` to `manifest.jsonl` -Rejected. The file acts as the run manifest/result index, but the established -filename is portable and already wired through CLI, Dashboard, result repo, and -adapter code. +Rejected for this phase. `index.jsonl` is already the wired row-level result +index across CLI, Dashboard, compare/trend, result repo sync, reports, and +adapters. `summary.json` now carries the run-level manifest role. -### Add `internal/` now +### Adopt Margin's full `internal/` layout now -Rejected for artifact-format v1. A future artifact-format v2 migration may add -an `internal/` directory for machine-facing caches or implementation details, -but v1 keeps canonical files at the run root and per-result allocations under -explicit manifest paths. +Rejected for v2 phase 1. Margin's distinction between portable manifests, +samples, retries, and internal state remains useful, but this phase only moves +committed run bundles to `.agentv/results//` and reserves dot-prefixed +local namespaces for rebuildable state. ## Non-Goals -- Moving or rewriting existing target-folder artifacts. -- Renaming `index.jsonl`. -- Defining a new result database or derived Dashboard index. -- Finalizing a full run-level model metadata schema. Root `summary.json.metadata` - already carries `targets`; richer provider/model fields can be added - additively when the Dashboard run-list work needs them. +- Flattening or renaming per-case `run-N/` attempt folders. +- Completing the schema-v2 repeat naming migration. User-facing docs should + prefer `pass_any` and `pass_all` when they mention repeat strategies, but that + schema migration is tracked separately. +- Moving Dashboard/search indexes into a committed run bundle. +- Projecting AgentV-owned runs, transcripts, datasets, experiments, or indexes + into Phoenix. diff --git a/packages/core/src/evaluation/results-repo-cache.test.ts b/packages/core/src/evaluation/results-repo-cache.test.ts index 3cc4861b6..38dce44c0 100644 --- a/packages/core/src/evaluation/results-repo-cache.test.ts +++ b/packages/core/src/evaluation/results-repo-cache.test.ts @@ -34,7 +34,7 @@ function writeRun( score: number, executionStatus = 'ok', ): void { - const runDir = path.join(repoDir, 'runs', experiment, timestamp); + const runDir = path.join(repoDir, 'runs', timestamp); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -114,7 +114,7 @@ describe('git results filesystem index cache', () => { const runs = await listGitRunsCached(repoDir, RESULTS_REF); expect(runs).toHaveLength(1); expect(runs[0]?.run_id).toBe('2026-06-28T00-00-00-000Z'); - expect(runs[0]?.summary_path).toBe('runs/default/2026-06-28T00-00-00-000Z/summary.json'); + expect(runs[0]?.summary_path).toBe('runs/2026-06-28T00-00-00-000Z/summary.json'); const cacheFile = resolveGitResultsIndexCacheFile({ repoDir, @@ -146,7 +146,7 @@ describe('git results filesystem index cache', () => { run_id: 'sentinel', experiment: 'default', timestamp: '2026-06-28T01-00-00-000Z', - manifest_path: 'runs/default/sentinel/index.jsonl', + manifest_path: 'runs/sentinel/index.jsonl', display_name: 'from cache', test_count: 1, avg_score: 0.5, @@ -178,7 +178,7 @@ describe('git results filesystem index cache', () => { expect(secondCommit).not.toBe(firstCommit); const runs = await listGitRunsCached(repoDir, RESULTS_REF); - expect(runs.map((run) => run.run_id)).toContain('experiment-a::2026-06-28T02-00-00-000Z'); + expect(runs.map((run) => run.run_id)).toContain('2026-06-28T02-00-00-000Z'); expect( existsSync( resolveGitResultsIndexCacheFile({ @@ -215,9 +215,7 @@ describe('git results filesystem index cache', () => { git(repoDir, ['commit', '-m', 'add execution error run']); const runs = await listGitRunsCached(repoDir, RESULTS_REF); - const errorRun = runs.find( - (run) => run.run_id === 'error-experiment::2026-06-28T03-00-00-000Z', - ); + const errorRun = runs.find((run) => run.run_id === '2026-06-28T03-00-00-000Z'); expect(errorRun?.execution_error_count).toBe(1); }); diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index 24117b031..3f64c976f 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -31,8 +31,8 @@ const execFileAsync = promisify(execFile); const RESULTS_REPO_RESULTS_DIR = '.agentv/results'; // On-branch / results-repo-clone storage layout. The results branch (e.g. // agentv/results/v1) already namespaces results, so runs are stored flat at -// runs/// and the editable tag overlays at -// metadata/runs/// — no redundant `.agentv/results/` prefix. +// runs// and the editable tag overlays at metadata/runs// — +// no redundant `.agentv/results/` prefix. const RESULTS_REPO_RUNS_DIR = 'runs'; const RESULTS_REPO_METADATA_DIR = 'metadata'; // Top-level directories AgentV owns on the results branch. The auto-sync @@ -76,7 +76,7 @@ const RESULT_INDEX_FILENAME = 'index.jsonl'; // our `agentv-json` driver. // - `merge.agentv-json.driver` (registered once in the checkout's local git // config) points at a tiny 3-way JSON set/field union script. -// Run bundles under runs///** are uniquely pathed, so a 3-way merge +// Run bundles under runs//** are uniquely pathed, so a 3-way merge // never conflicts on them and they need no attribute. const RESULTS_REPO_GITATTRIBUTES_FILE = '.gitattributes'; const RESULTS_REPO_GITATTRIBUTES_CONTENT = `# Managed by AgentV. Artifact-aware merge so results sync never force-pushes. @@ -1767,7 +1767,7 @@ async function resolveRemotePushUrl(repoDir: string, remote: string): Promise/` run directories that the diverged +// Count the unique top-level `runs/` run directories that the diverged // local commit adds on top of the remote target tip. Best-effort: returns // undefined when the diff cannot be computed. async function countContributedRunDirs( @@ -1798,9 +1798,9 @@ async function countContributedRunDirs( continue; } const segments = file.split('/'); - // runs///... - if (segments[0] === RESULTS_REPO_RUNS_DIR && segments.length >= 3) { - runDirs.add(`${segments[0]}/${segments[1]}/${segments[2]}`); + // runs//... + if (segments[0] === RESULTS_REPO_RUNS_DIR && segments.length >= 2 && segments[1]) { + runDirs.add(`${segments[0]}/${segments[1]}`); } } return runDirs.size; @@ -3889,6 +3889,10 @@ function buildGitManifestPaths( for (const [summaryPath, summary] of summaryByPath) { const runDir = path.posix.dirname(summaryPath); + const relativeRunPath = path.posix.relative(RESULTS_REPO_RUNS_DIR, runDir); + if (!isV2ResultsRepoRunPath(relativeRunPath)) { + continue; + } const manifestPath = safeGitSummaryManifestPath(runDir, summary.manifest_path); if (manifestPath && treePathSet.has(manifestPath)) { manifestByRunDir.set(runDir, manifestPath); @@ -3900,6 +3904,10 @@ function buildGitManifestPaths( continue; } const runDir = path.posix.dirname(treePath); + const relativeRunPath = path.posix.relative(RESULTS_REPO_RUNS_DIR, runDir); + if (!isV2ResultsRepoRunPath(relativeRunPath)) { + continue; + } if (!manifestByRunDir.has(runDir)) { manifestByRunDir.set(runDir, treePath); } @@ -3908,17 +3916,14 @@ function buildGitManifestPaths( return [...manifestByRunDir.values()].sort(); } +function isV2ResultsRepoRunPath(relativeRunPath: string): boolean { + const segments = relativeRunPath.split('/').filter(Boolean); + return segments.length === 1 && !segments[0].startsWith('.'); +} + function buildGitRunId(relativeRunPath: string): string { const normalized = relativeRunPath.split(path.sep).join('/'); const segments = normalized.split('/').filter(Boolean); - if (segments.length >= 2) { - const experiment = segments.slice(0, -1).join('/'); - const timestamp = segments.at(-1); - if (experiment === 'default') { - return timestamp ?? normalized; - } - return `${experiment}::${timestamp}`; - } return segments[0] ?? relativeRunPath; } diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index efb06c811..7cb8d6994 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -160,6 +160,7 @@ export async function aggregateRunDir( options?: { evalFile?: string; experiment?: string; + runId?: string; plannedTestCount?: number; experimentMetadata?: ExperimentArtifactMetadata; runtimeSource?: RunRuntimeSourceMetadata; @@ -179,6 +180,7 @@ export async function aggregateRunDir( results, options?.evalFile, options?.experiment, + options?.runId ?? path.basename(runDir), plannedTestCount, options?.experimentMetadata, runtimeSource, @@ -392,6 +394,7 @@ export interface TimingArtifact { export interface RunSummaryArtifact { readonly manifest_path: string; readonly metadata: { + readonly run_id?: string; readonly eval_file: string; readonly timestamp: string; readonly targets: readonly string[]; @@ -1290,6 +1293,7 @@ export function buildRunSummaryArtifact( results: readonly EvaluationResult[], evalFile = '', experiment?: string, + runId?: string, plannedTestCount?: number, experimentMetadata?: ExperimentArtifactMetadata, runtimeSource?: RunRuntimeSourceMetadata, @@ -1384,6 +1388,7 @@ export function buildRunSummaryArtifact( return { manifest_path: RESULT_INDEX_FILENAME, metadata: { + run_id: runId, eval_file: evalFile, timestamp, targets, @@ -1407,6 +1412,7 @@ export async function writeInitialRunSummaryArtifact( evalFile: string; plannedTestCount: number; experiment?: string; + runId?: string; experimentMetadata?: ExperimentArtifactMetadata; runtimeSource?: RunRuntimeSourceMetadata; }, @@ -1416,6 +1422,7 @@ export async function writeInitialRunSummaryArtifact( [], options.evalFile, options.experiment, + options.runId ?? path.basename(runDir), options.plannedTestCount, options.experimentMetadata, options.runtimeSource, @@ -2517,6 +2524,7 @@ export async function writeArtifactsFromResults( results, options?.evalFile, options?.experiment, + options?.runId ?? path.basename(outputDir), plannedTestCount, options?.experimentMetadata, runtimeSource, diff --git a/packages/core/test/evaluation/evaluate-programmatic-api.test.ts b/packages/core/test/evaluation/evaluate-programmatic-api.test.ts index be272ce1b..fc894370f 100644 --- a/packages/core/test/evaluation/evaluate-programmatic-api.test.ts +++ b/packages/core/test/evaluation/evaluate-programmatic-api.test.ts @@ -147,9 +147,15 @@ describe('evaluate() — programmatic API extensions', () => { const summaryArtifact = JSON.parse( await readFile(path.join(outputDir, 'summary.json'), 'utf8'), ) as { - metadata: { experiment?: string; tests_run: string[]; eval_file: string }; + metadata: { + run_id?: string; + experiment?: string; + tests_run: string[]; + eval_file: string; + }; timing: { duration_ms: number }; }; + expect(summaryArtifact.metadata.run_id).toBe(path.basename(outputDir)); expect(summaryArtifact.metadata.experiment).toBe('sdk-test'); expect(summaryArtifact.metadata.tests_run).toEqual(['programmatic-artifacts']); expect(summaryArtifact.metadata.eval_file).toBe(''); diff --git a/packages/core/test/evaluation/results-repo.test.ts b/packages/core/test/evaluation/results-repo.test.ts index bb6985d9c..6e91dbfb7 100644 --- a/packages/core/test/evaluation/results-repo.test.ts +++ b/packages/core/test/evaluation/results-repo.test.ts @@ -136,17 +136,12 @@ async function createStaleResultBranchPushFixture(params: { await directPushResults({ config, sourceDir: firstSourceDir, - destinationPath: path.join('local-stale', '2026-06-23T09-00-00-000Z'), + destinationPath: '2026-06-23T09-00-00-000Z', commitMessage: 'feat(results): local stale base', }); git(`git switch --quiet ${params.storageBranch}`, params.seedDir); - const remoteOnlyPath = path.join( - params.seedDir, - 'runs', - 'remote-only', - '2026-06-23T09-30-00-000Z', - ); + const remoteOnlyPath = path.join(params.seedDir, 'runs', '2026-06-23T09-30-00-000Z'); writeRunArtifacts(remoteOnlyPath, 'remote-only', '2026-06-23T09:30:00.000Z'); git('git add runs && git commit --quiet -m "remote result wins race"', params.seedDir); git(`git push --quiet origin HEAD:${params.storageBranch}`, params.seedDir); @@ -157,7 +152,7 @@ async function createStaleResultBranchPushFixture(params: { return { localSourceDir, - localDestinationPath: path.join('local-conflict', '2026-06-23T10-00-00-000Z'), + localDestinationPath: '2026-06-23T10-00-00-000Z', remoteAdvancedCommit: git( `git --git-dir "${params.remoteDir}" rev-parse ${params.storageBranch}`, params.rootDir, @@ -404,7 +399,7 @@ describe('listGitRuns', () => { }); it('returns committed runs derived from canonical index.jsonl manifests', async () => { - const defaultRunDir = path.join(repoDir, 'runs', 'default', '2026-05-20T10-00-00-000Z'); + const defaultRunDir = path.join(repoDir, 'runs', '2026-05-20T10-00-00-000Z'); mkdirSync(defaultRunDir, { recursive: true }); writeFileSync( path.join(defaultRunDir, 'index.jsonl'), @@ -441,7 +436,7 @@ describe('listGitRuns', () => { ), ); - const experimentRunDir = path.join(repoDir, 'runs', 'with-skills', '2026-05-21T11-00-00-000Z'); + const experimentRunDir = path.join(repoDir, 'runs', '2026-05-21T11-00-00-000Z'); mkdirSync(experimentRunDir, { recursive: true }); writeFileSync( path.join(experimentRunDir, 'index.jsonl'), @@ -495,15 +490,15 @@ describe('listGitRuns', () => { expect(runs).toHaveLength(2); expect(runs.map((run) => run.run_id)).toEqual([ - 'with-skills::2026-05-21T11-00-00-000Z', + '2026-05-21T11-00-00-000Z', '2026-05-20T10-00-00-000Z', ]); expect(runs[0]).toMatchObject({ experiment: 'with-skills', timestamp: '2026-05-21T11:00:00.000Z', display_name: 'remote friendly run', - manifest_path: 'runs/with-skills/2026-05-21T11-00-00-000Z/index.jsonl', - summary_path: 'runs/with-skills/2026-05-21T11-00-00-000Z/summary.json', + manifest_path: 'runs/2026-05-21T11-00-00-000Z/index.jsonl', + summary_path: 'runs/2026-05-21T11-00-00-000Z/summary.json', test_count: 3, pass_rate: 0.75, avg_score: 0, @@ -513,7 +508,7 @@ describe('listGitRuns', () => { experiment: 'default', display_name: '2026-05-20T10-00-00-000Z', target: 'gpt-4o', - manifest_path: 'runs/default/2026-05-20T10-00-00-000Z/index.jsonl', + manifest_path: 'runs/2026-05-20T10-00-00-000Z/index.jsonl', test_count: 2, pass_rate: 0.5, }); @@ -535,7 +530,7 @@ describe('listGitRuns', () => { }); it('ignores inherited git hook environment variables', async () => { - const runDir = path.join(repoDir, 'runs', 'default', '2026-05-20T10-00-00-000Z'); + const runDir = path.join(repoDir, 'runs', '2026-05-20T10-00-00-000Z'); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -592,7 +587,7 @@ describe('listGitRuns', () => { }); it('materializes an entire run subtree atomically from git objects', async () => { - const runDir = path.join(repoDir, 'runs', 'with-files', '2026-05-22T10-00-00-000Z'); + const runDir = path.join(repoDir, 'runs', '2026-05-22T10-00-00-000Z'); mkdirSync(path.join(runDir, 'attachments'), { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha"}\n'); writeFileSync( @@ -617,7 +612,7 @@ describe('listGitRuns', () => { rmSync(runDir, { recursive: true, force: true }); - await materializeGitRun(repoDir, 'with-files/2026-05-22T10-00-00-000Z', 'HEAD'); + await materializeGitRun(repoDir, '2026-05-22T10-00-00-000Z', 'HEAD'); expect(readFileSync(path.join(runDir, 'index.jsonl'), 'utf8')).toContain('"test_id":"alpha"'); expect(readFileSync(path.join(runDir, 'attachments', 'response.md'), 'utf8')).toBe( @@ -645,7 +640,7 @@ describe('listGitRuns', () => { const defaultBranch = git('git branch --show-current', repoDir); git('git checkout -b agentv-results', repoDir); - const runDir = path.join(repoDir, 'runs', 'branch-only', '2026-06-12T10-00-00-000Z'); + const runDir = path.join(repoDir, 'runs', '2026-06-12T10-00-00-000Z'); writeRunArtifacts(runDir, 'branch-only', '2026-06-12T10:00:00.000Z'); writeFileSync(path.join(runDir, 'attachments.txt'), 'from branch\n'); git('git add runs && git commit -m "seed branch run"', repoDir); @@ -653,9 +648,9 @@ describe('listGitRuns', () => { const runs = await listGitRuns(repoDir, 'agentv-results'); expect(runs).toHaveLength(1); - expect(runs[0].run_id).toBe('branch-only::2026-06-12T10-00-00-000Z'); + expect(runs[0].run_id).toBe('2026-06-12T10-00-00-000Z'); - await materializeGitRun(repoDir, 'branch-only/2026-06-12T10-00-00-000Z', 'agentv-results'); + await materializeGitRun(repoDir, '2026-06-12T10-00-00-000Z', 'agentv-results'); expect(readFileSync(path.join(runDir, 'attachments.txt'), 'utf8')).toBe('from branch\n'); }); }); @@ -776,14 +771,7 @@ describe('results repo write path', () => { git('git add README.md && git commit --quiet -m "seed source"', projectDir); const runTimestamp = '2026-06-17T10-00-00-000Z'; - const runDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - 'current-repo', - runTimestamp, - ); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'current-repo', '2026-06-17T10:00:00.000Z'); writeFileSync(path.join(projectDir, 'UNRELATED.txt'), 'do not publish\n'); @@ -794,14 +782,14 @@ describe('results repo write path', () => { auto_push: false, }, sourceDir: runDir, - destinationPath: path.join('current-repo', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): current-repo - 1/1 PASS (1.000)', }); expect(published).toBe(true); expect(git('git branch --show-current', projectDir)).toBe('main'); const branchFiles = git(`git ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, projectDir); - expect(branchFiles).toContain(`runs/current-repo/${runTimestamp}/summary.json`); + expect(branchFiles).toContain(`runs/${runTimestamp}/summary.json`); expect(branchFiles).not.toContain('README.md'); expect(branchFiles).not.toContain('UNRELATED.txt'); expect(git('git status --short --branch', projectDir)).toContain('## main'); @@ -817,14 +805,7 @@ describe('results repo write path', () => { git('git add README.md && git commit --quiet -m "seed source"', projectDir); const runTimestamp = '2026-06-17T10-05-00-000Z'; - const runDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - 'human-author', - runTimestamp, - ); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'human-author', '2026-06-17T10:05:00.000Z'); const published = await directPushResults({ @@ -834,7 +815,7 @@ describe('results repo write path', () => { auto_push: false, }, sourceDir: runDir, - destinationPath: path.join('human-author', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): human-author - 1/1 PASS (1.000)', }); @@ -870,14 +851,7 @@ describe('results repo write path', () => { ); const runTimestamp = '2026-06-17T10-07-00-000Z'; - const runDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - 'env-author', - runTimestamp, - ); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'env-author', '2026-06-17T10:07:00.000Z'); const published = await directPushResults({ @@ -887,7 +861,7 @@ describe('results repo write path', () => { auto_push: false, }, sourceDir: runDir, - destinationPath: path.join('env-author', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): env-author - 1/1 PASS (1.000)', }); @@ -926,14 +900,7 @@ describe('results repo write path', () => { ); const runTimestamp = '2026-06-17T10-10-00-000Z'; - const runDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - 'fallback-author', - runTimestamp, - ); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'fallback-author', '2026-06-17T10:10:00.000Z'); const published = await directPushResults({ @@ -943,7 +910,7 @@ describe('results repo write path', () => { auto_push: false, }, sourceDir: runDir, - destinationPath: path.join('fallback-author', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): fallback-author - 1/1 PASS (1.000)', }); @@ -971,14 +938,7 @@ describe('results repo write path', () => { const originalOrigin = git('git remote get-url origin', projectDir); const runTimestamp = '2026-06-22T04-00-00-000Z'; - const runDir = path.join( - projectDir, - '.agentv', - 'results', - 'runs', - 'url-backed-source', - runTimestamp, - ); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'url-backed-source', '2026-06-22T04:00:00.000Z'); const published = await directPushResults({ @@ -989,7 +949,7 @@ describe('results repo write path', () => { auto_push: true, }, sourceDir: runDir, - destinationPath: path.join('url-backed-source', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): url-backed-source - 1/1 PASS (1.000)', }); @@ -1001,7 +961,7 @@ describe('results repo write path', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, rootDir, ); - expect(remoteFiles).toContain(`runs/url-backed-source/${runTimestamp}/summary.json`); + expect(remoteFiles).toContain(`runs/${runTimestamp}/summary.json`); expect(remoteFiles).not.toContain('README.md'); }, 20000); @@ -1050,7 +1010,6 @@ describe('results repo write path', () => { projectDir, 'metadata', 'runs', - 'default', '2026-06-22T00-19-03-060Z', 'tags.json', ); @@ -1069,7 +1028,7 @@ describe('results repo write path', () => { await expect(getResultsRepoSyncStatus(config)).resolves.toMatchObject({ sync_status: 'dirty', - dirty_paths: ['metadata/runs/default/2026-06-22T00-19-03-060Z/tags.json'], + dirty_paths: ['metadata/runs/2026-06-22T00-19-03-060Z/tags.json'], }); const status = await syncResultsRepoForProject(config); @@ -1084,7 +1043,7 @@ describe('results repo write path', () => { }); expect( git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir), - ).toContain('metadata/runs/default/2026-06-22T00-19-03-060Z/tags.json'); + ).toContain('metadata/runs/2026-06-22T00-19-03-060Z/tags.json'); await expect(getResultsRepoSyncStatus(config)).resolves.toMatchObject({ sync_status: 'clean', dirty_paths: [], @@ -1107,7 +1066,6 @@ describe('results repo write path', () => { seedDir, 'metadata', 'runs', - 'remote-only', '2026-06-22T00-00-00-000Z', 'tags.json', ); @@ -1157,7 +1115,6 @@ describe('results repo write path', () => { seedDir, 'metadata', 'runs', - 'remote-only', '2026-06-22T00-00-00-000Z', 'tags.json', ); @@ -1170,7 +1127,6 @@ describe('results repo write path', () => { projectDir, 'metadata', 'runs', - 'local', '2026-06-22T01-00-00-000Z', 'tags.json', ); @@ -1199,8 +1155,8 @@ describe('results repo write path', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir, ); - expect(remoteFiles).toContain('metadata/runs/remote-only/2026-06-22T00-00-00-000Z/tags.json'); - expect(remoteFiles).toContain('metadata/runs/local/2026-06-22T01-00-00-000Z/tags.json'); + expect(remoteFiles).toContain('metadata/runs/2026-06-22T00-00-00-000Z/tags.json'); + expect(remoteFiles).toContain('metadata/runs/2026-06-22T01-00-00-000Z/tags.json'); expect(git('git branch --show-current', projectDir)).toBe('main'); }, 20000); @@ -1216,7 +1172,7 @@ describe('results repo write path', () => { projectDir, ); - const metadataPath = 'metadata/runs/shared/2026-06-22T03-00-00-000Z/tags.json'; + const metadataPath = 'metadata/runs/2026-06-22T03-00-00-000Z/tags.json'; git(`git switch --quiet ${storageBranch}`, seedDir); const remoteTagPath = path.join(seedDir, ...metadataPath.split('/')); mkdirSync(path.dirname(remoteTagPath), { recursive: true }); @@ -1270,7 +1226,6 @@ describe('results repo write path', () => { projectDir, 'metadata', 'runs', - 'local', '2026-06-22T02-00-00-000Z', 'tags.json', ); @@ -1299,11 +1254,11 @@ describe('results repo write path', () => { }); expect(status.block_reason).toContain('Results repo push was rejected'); expect(git(`git ls-tree -r --name-only ${storageBranch}`, projectDir)).toContain( - 'metadata/runs/local/2026-06-22T02-00-00-000Z/tags.json', + 'metadata/runs/2026-06-22T02-00-00-000Z/tags.json', ); expect( git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir), - ).not.toContain('metadata/runs/local/2026-06-22T02-00-00-000Z/tags.json'); + ).not.toContain('metadata/runs/2026-06-22T02-00-00-000Z/tags.json'); expect(git('git branch --show-current', projectDir)).toBe('main'); }, 20000); @@ -1319,7 +1274,7 @@ describe('results repo write path', () => { git('git add README.md && git commit --quiet -m "seed results repo"', resultsRepoDir); const runTimestamp = '2026-06-17T11-00-00-000Z'; - const runDir = path.join(projectDir, '.agentv', 'results', 'external', runTimestamp); + const runDir = path.join(projectDir, '.agentv', 'results', runTimestamp); writeRunArtifacts(runDir, 'external', '2026-06-17T11:00:00.000Z'); const published = await directPushResults({ @@ -1329,14 +1284,14 @@ describe('results repo write path', () => { auto_push: false, }, sourceDir: runDir, - destinationPath: path.join('external', runTimestamp), + destinationPath: runTimestamp, commitMessage: 'feat(results): external - 1/1 PASS (1.000)', }); expect(published).toBe(true); expect(git('git branch --show-current', resultsRepoDir)).toBe('main'); const branchFiles = git(`git ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, resultsRepoDir); - expect(branchFiles).toContain(`runs/external/${runTimestamp}/index.jsonl`); + expect(branchFiles).toContain(`runs/${runTimestamp}/index.jsonl`); expect(branchFiles).not.toContain('README.md'); }, 20000); @@ -1345,7 +1300,7 @@ describe('results repo write path', () => { const cloneDir = path.join(rootDir, 'results-clone'); const sourceDir = path.join(rootDir, 'source-run'); const runTimestamp = '2026-05-22T11-00-00-000Z'; - const destinationPath = path.join('retry', runTimestamp); + const destinationPath = runTimestamp; const config = createResultsConfig(remoteDir, cloneDir); const hookPath = path.join(remoteDir, 'hooks', 'pre-receive'); writeRunArtifacts(sourceDir, 'retry', '2026-05-22T11:00:00.000Z'); @@ -1367,7 +1322,7 @@ describe('results repo write path', () => { ).rejects.toThrow(/simulated interrupted push/); expect(git('git rev-list --count origin/main..main', cloneDir)).toBe('1'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( - `runs/retry/${runTimestamp}/summary.json`, + `runs/${runTimestamp}/summary.json`, ); rmSync(hookPath, { force: true }); @@ -1383,10 +1338,10 @@ describe('results repo write path', () => { expect(git('git rev-list --count origin/main..main', cloneDir)).toBe('0'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).toContain( - `runs/retry/${runTimestamp}/summary.json`, + `runs/${runTimestamp}/summary.json`, ); expect(git(`git --git-dir "${remoteDir}" log -1 --pretty=%B main`, rootDir)).toContain( - `AgentV-Run: retry::${runTimestamp}`, + `AgentV-Run: ${runTimestamp}`, ); }, 20000); @@ -1436,7 +1391,7 @@ describe('results repo write path', () => { rootDir, ); expect(remoteFiles).toContain(`runs/${fixture.localDestinationPath}/summary.json`); - expect(remoteFiles).toContain('runs/remote-only/2026-06-23T09-30-00-000Z/summary.json'); + expect(remoteFiles).toContain('runs/2026-06-23T09-30-00-000Z/summary.json'); // No backup ref was ever created in the merge path. expect( git(`git --git-dir "${remoteDir}" for-each-ref refs/heads/agentv/backups`, rootDir), @@ -1492,7 +1447,7 @@ describe('results repo write path', () => { const result = await directPushResultsWithDetails({ config, sourceDir, - destinationPath: path.join('race-local', '2026-06-24T10-00-00-000Z'), + destinationPath: '2026-06-24T10-00-00-000Z', commitMessage: 'feat(results): push race', }); @@ -1512,7 +1467,7 @@ describe('results repo write path', () => { rootDir, ); expect(remoteFiles).toContain('RACE.md'); - expect(remoteFiles).toContain('runs/race-local/2026-06-24T10-00-00-000Z/summary.json'); + expect(remoteFiles).toContain('runs/2026-06-24T10-00-00-000Z/summary.json'); expect( git(`git --git-dir "${remoteDir}" for-each-ref refs/heads/agentv/backups`, rootDir), ).toBe(''); @@ -1523,7 +1478,7 @@ describe('results repo write path', () => { const cloneDir = path.join(rootDir, 'results-clone'); const sourceDir = path.join(rootDir, 'source-run'); const runTimestamp = '2026-05-22T10-00-00-000Z'; - const destinationPath = path.join('with-skills', runTimestamp); + const destinationPath = runTimestamp; const config = createResultsConfig(remoteDir, cloneDir); writeRunArtifacts(sourceDir, 'with-skills', '2026-05-22T10:00:00.000Z'); @@ -1540,19 +1495,17 @@ describe('results repo write path', () => { expect(pushed).toBe(true); expect(git('git rev-parse --show-toplevel', cloneDir)).toBe(cloneDir); - expect(git('git log -1 --pretty=%B main', cloneDir)).toContain( - `AgentV-Run: with-skills::${runTimestamp}`, - ); + expect(git('git log -1 --pretty=%B main', cloneDir)).toContain(`AgentV-Run: ${runTimestamp}`); expect(git(`git --git-dir "${remoteDir}" log -1 --pretty=%B main`, rootDir)).toContain( - `AgentV-Run: with-skills::${runTimestamp}`, + `AgentV-Run: ${runTimestamp}`, ); expect(git('git ls-tree -r --name-only main', cloneDir)).toContain( - `runs/with-skills/${runTimestamp}/index.jsonl`, + `runs/${runTimestamp}/index.jsonl`, ); const runs = await listGitRuns(cloneDir, 'main'); expect(runs).toHaveLength(1); - expect(runs[0].run_id).toBe(`with-skills::${runTimestamp}`); + expect(runs[0].run_id).toBe(runTimestamp); }, 20000); it('pushes direct results to the configured storage branch', async () => { @@ -1561,7 +1514,7 @@ describe('results repo write path', () => { const cloneDir = path.join(rootDir, 'results-clone'); const sourceDir = path.join(rootDir, 'source-run'); const runTimestamp = '2026-06-12T10-00-00-000Z'; - const destinationPath = path.join('branch-storage', runTimestamp); + const destinationPath = runTimestamp; const config = { ...createResultsConfig(remoteDir, cloneDir), branch: storageBranch, @@ -1582,14 +1535,14 @@ describe('results repo write path', () => { expect(pushed).toBe(true); expect(git('git branch --show-current', cloneDir)).toBe('main'); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( - `runs/branch-storage/${runTimestamp}/summary.json`, + `runs/${runTimestamp}/summary.json`, ); expect( git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${storageBranch}`, rootDir), - ).toContain(`runs/branch-storage/${runTimestamp}/summary.json`); + ).toContain(`runs/${runTimestamp}/summary.json`); expect( git(`git --git-dir "${remoteDir}" log -1 --pretty=%B ${storageBranch}`, rootDir), - ).toContain(`AgentV-Run: branch-storage::${runTimestamp}`); + ).toContain(`AgentV-Run: ${runTimestamp}`); }, 20000); it('pushes artifact pointer payloads to the sidecar artifact branch', async () => { @@ -1598,7 +1551,7 @@ describe('results repo write path', () => { const cloneDir = path.join(rootDir, 'results-clone'); const sourceDir = path.join(rootDir, 'source-run'); const runTimestamp = '2026-06-21T12-00-00-000Z'; - const destinationPath = path.posix.join('sidecar', runTimestamp); + const destinationPath = runTimestamp; const config = { ...createResultsConfig(remoteDir, cloneDir), branch: storageBranch, @@ -1685,7 +1638,7 @@ describe('results repo write path', () => { const cloneDir = path.join(rootDir, 'results-clone'); const sourceDir = path.join(rootDir, 'source-run'); const runTimestamp = '2026-06-21T13-00-00-000Z'; - const destinationPath = path.posix.join('sidecar-backfill', runTimestamp); + const destinationPath = runTimestamp; const config = { ...createResultsConfig(remoteDir, cloneDir), branch: storageBranch, @@ -1694,7 +1647,7 @@ describe('results repo write path', () => { git(`git switch --quiet --orphan ${storageBranch}`, seedDir); git('git rm -rf --quiet . 2>/dev/null || true', seedDir); - const seededRunDir = path.join(seedDir, 'runs', ...destinationPath.split('/')); + const seededRunDir = path.join(seedDir, 'runs', destinationPath); mkdirSync(path.dirname(seededRunDir), { recursive: true }); cpSync(sourceDir, seededRunDir, { recursive: true }); git('git add runs && git commit --quiet -m "seed published run"', seedDir); @@ -1758,7 +1711,7 @@ describe('results repo write path', () => { directPushResults({ config, sourceDir, - destinationPath: path.join('missing-branch', '2026-06-12T11-00-00-000Z'), + destinationPath: '2026-06-12T11-00-00-000Z', commitMessage: 'feat(results): missing branch', }), ).resolves.toBe(true); @@ -1767,7 +1720,7 @@ describe('results repo write path', () => { ); expect( git(`git --git-dir "${remoteDir}" ls-tree -r --name-only agentv-results`, rootDir), - ).toContain('runs/missing-branch/2026-06-12T11-00-00-000Z/summary.json'); + ).toContain('runs/2026-06-12T11-00-00-000Z/summary.json'); }, 20000); it('syncResultsRepo refreshes refs without checking out the base branch', async () => { @@ -1806,7 +1759,7 @@ describe('results repo write path', () => { behind: 0, }); - const localRunDir = path.join(cloneDir, 'runs', 'local-only', '2026-05-23T10-00-00-000Z'); + const localRunDir = path.join(cloneDir, 'runs', '2026-05-23T10-00-00-000Z'); writeRunArtifacts(localRunDir, 'local-only', '2026-05-23T10:00:00.000Z'); git('git add runs && git commit --quiet -m "local result"', cloneDir); @@ -1898,7 +1851,7 @@ describe('results repo write path', () => { git('git config user.name "Test User"', cloneDir); const runTimestamp = '2026-05-24T10-00-00-000Z'; - const runDir = path.join(cloneDir, 'runs', 'metadata', runTimestamp); + const runDir = path.join(cloneDir, 'runs', runTimestamp); writeRunArtifacts(runDir, 'metadata', '2026-05-24T10:00:00.000Z'); const status = await syncResultsRepoForProject(config); @@ -1910,7 +1863,7 @@ describe('results repo write path', () => { blocked: false, }); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).toContain( - `runs/metadata/${runTimestamp}/summary.json`, + `runs/${runTimestamp}/summary.json`, ); }, 20000); @@ -1948,7 +1901,7 @@ describe('results repo write path', () => { writeFileSync(path.join(cloneDir, 'package.json'), '{"dependencies":{"agentv":"next"}}\n'); const runTimestamp = '2026-05-24T11-00-00-000Z'; - const runDir = path.join(cloneDir, 'runs', 'safe-run', runTimestamp); + const runDir = path.join(cloneDir, 'runs', runTimestamp); writeRunArtifacts(runDir, 'safe-run', '2026-05-24T11:00:00.000Z'); const status = await syncResultsRepoForProject(config); @@ -1961,7 +1914,7 @@ describe('results repo write path', () => { }); expect(status.dirty_paths).toEqual([]); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).toContain( - `runs/safe-run/${runTimestamp}/summary.json`, + `runs/${runTimestamp}/summary.json`, ); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir)).not.toContain( 'package.json', @@ -1983,7 +1936,7 @@ describe('results repo write path', () => { git('git add package.json', cloneDir); const runTimestamp = '2026-05-24T11-30-00-000Z'; - const runDir = path.join(cloneDir, 'runs', 'staged-unrelated', runTimestamp); + const runDir = path.join(cloneDir, 'runs', runTimestamp); writeRunArtifacts(runDir, 'staged-unrelated', '2026-05-24T11:30:00.000Z'); const status = await syncResultsRepoForProject(config); @@ -1995,7 +1948,7 @@ describe('results repo write path', () => { blocked: false, }); const remoteFiles = git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir); - expect(remoteFiles).toContain(`runs/staged-unrelated/${runTimestamp}/summary.json`); + expect(remoteFiles).toContain(`runs/${runTimestamp}/summary.json`); expect(remoteFiles).not.toContain('package.json'); expect(git('git status --porcelain', cloneDir)).toContain('A package.json'); }, 20000); @@ -2041,7 +1994,7 @@ describe('results repo write path', () => { git('git push --quiet origin main', seedDir); const runTimestamp = '2026-05-24T12-00-00-000Z'; - const runDir = path.join(cloneDir, 'runs', 'pulled-then-pushed', runTimestamp); + const runDir = path.join(cloneDir, 'runs', runTimestamp); writeRunArtifacts(runDir, 'pulled-then-pushed', '2026-05-24T12:00:00.000Z'); const status = await syncResultsRepoForProject(config); @@ -2055,7 +2008,7 @@ describe('results repo write path', () => { }); const remoteFiles = git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir); expect(remoteFiles).toContain('REMOTE.md'); - expect(remoteFiles).toContain(`runs/pulled-then-pushed/${runTimestamp}/summary.json`); + expect(remoteFiles).toContain(`runs/${runTimestamp}/summary.json`); expect(remoteFiles).not.toContain('package.json'); expect(readFileSync(path.join(cloneDir, 'package.json'), 'utf8')).toBe( '{"dependencies":{"agentv":"next"}}\n', @@ -2071,11 +2024,11 @@ describe('results repo write path', () => { git('git config user.email "test@example.com"', cloneDir); git('git config user.name "Test User"', cloneDir); - const runDir = path.join(cloneDir, 'runs', 'local-only', '2026-05-25T10-00-00-000Z'); + const runDir = path.join(cloneDir, 'runs', '2026-05-25T10-00-00-000Z'); writeRunArtifacts(runDir, 'local-only', '2026-05-25T10:00:00.000Z'); git('git add runs && git commit --quiet -m "local result"', cloneDir); - const remoteRunDir = path.join(seedDir, 'runs', 'remote-only', '2026-05-25T11-00-00-000Z'); + const remoteRunDir = path.join(seedDir, 'runs', '2026-05-25T11-00-00-000Z'); writeRunArtifacts(remoteRunDir, 'remote-only', '2026-05-25T11:00:00.000Z'); git('git add runs && git commit --quiet -m "remote result"', seedDir); const remoteBefore = git('git rev-parse HEAD', seedDir); @@ -2100,8 +2053,8 @@ describe('results repo write path', () => { ), ).not.toThrow(); const remoteFiles = git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, rootDir); - expect(remoteFiles).toContain('runs/local-only/2026-05-25T10-00-00-000Z/summary.json'); - expect(remoteFiles).toContain('runs/remote-only/2026-05-25T11-00-00-000Z/summary.json'); + expect(remoteFiles).toContain('runs/2026-05-25T10-00-00-000Z/summary.json'); + expect(remoteFiles).toContain('runs/2026-05-25T11-00-00-000Z/summary.json'); expect( git(`git --git-dir "${remoteDir}" for-each-ref refs/heads/agentv/backups`, rootDir), ).toBe(''); @@ -2111,7 +2064,7 @@ describe('results repo write path', () => { const { remoteDir, seedDir } = initializeRemoteRepo(rootDir); const cloneDir = path.join(rootDir, 'results-clone'); const config = createResultsConfig(remoteDir, cloneDir); - const indexRel = path.join('runs', 'shared', '2026-05-25T12-00-00-000Z', 'index.jsonl'); + const indexRel = path.join('runs', '2026-05-25T12-00-00-000Z', 'index.jsonl'); await ensureResultsRepoClone(config); git('git config user.email "test@example.com"', cloneDir); @@ -2492,7 +2445,7 @@ describe('results branch stable genesis', () => { auto_push: true, }, sourceDir, - destinationPath: path.join(params.experiment, fsTimestamp), + destinationPath: fsTimestamp, commitMessage: `feat(results): ${params.experiment}`, }); } @@ -2528,7 +2481,7 @@ describe('results branch stable genesis', () => { expect(isAncestor(remoteDir, mainSha, DEFAULT_RESULTS_BRANCH)).toBe(false); expect( git(`git --git-dir "${remoteDir}" ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, rootDir), - ).toContain('runs/expA/2026-06-19T10-00-00-000Z/summary.json'); + ).toContain('runs/2026-06-19T10-00-00-000Z/summary.json'); }, 20000); it('mints a byte-identical genesis root regardless of wall-clock time', async () => { @@ -2546,7 +2499,7 @@ describe('results branch stable genesis', () => { await directPushResults({ config: { repo_path: repoDir, branch: DEFAULT_RESULTS_BRANCH, auto_push: false }, sourceDir, - destinationPath: path.join(label, fsTimestamp), + destinationPath: fsTimestamp, commitMessage: `feat(results): ${label}`, }); return git(`git rev-list --max-parents=0 ${DEFAULT_RESULTS_BRANCH}`, repoDir); @@ -2588,8 +2541,8 @@ describe('results branch stable genesis', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, rootDir, ); - expect(tree).toContain('runs/expA/2026-06-19T10-00-00-000Z/summary.json'); - expect(tree).toContain('runs/expB/2026-06-19T11-00-00-000Z/summary.json'); + expect(tree).toContain('runs/2026-06-19T10-00-00-000Z/summary.json'); + expect(tree).toContain('runs/2026-06-19T11-00-00-000Z/summary.json'); }, 30000); it('reconciles two independent first-inits onto a single shared genesis', async () => { @@ -2610,7 +2563,7 @@ describe('results branch stable genesis', () => { await directPushResults({ config: { repo_path: cloneA, branch: DEFAULT_RESULTS_BRANCH, auto_push: true }, sourceDir: runA, - destinationPath: path.join('expA', '2026-06-19T10-00-00-000Z'), + destinationPath: '2026-06-19T10-00-00-000Z', commitMessage: 'feat(results): expA', }); @@ -2620,7 +2573,7 @@ describe('results branch stable genesis', () => { await directPushResults({ config: { repo_path: cloneB, branch: DEFAULT_RESULTS_BRANCH, auto_push: true }, sourceDir: runB, - destinationPath: path.join('expB', '2026-06-19T11-00-00-000Z'), + destinationPath: '2026-06-19T11-00-00-000Z', commitMessage: 'feat(results): expB', }); @@ -2630,14 +2583,14 @@ describe('results branch stable genesis', () => { `git --git-dir "${remoteDir}" ls-tree -r --name-only ${DEFAULT_RESULTS_BRANCH}`, rootDir, ); - expect(tree).toContain('runs/expA/2026-06-19T10-00-00-000Z/summary.json'); - expect(tree).toContain('runs/expB/2026-06-19T11-00-00-000Z/summary.json'); + expect(tree).toContain('runs/2026-06-19T10-00-00-000Z/summary.json'); + expect(tree).toContain('runs/2026-06-19T11-00-00-000Z/summary.json'); }, 30000); }); describe('buildWipBranchName', () => { it('produces an agentv/wip// branch name', () => { - const runDir = '/some/path/.agentv/results/default/2026-01-15T10-00-00'; + const runDir = '/some/path/.agentv/results/2026-01-15T10-00-00'; const branch = buildWipBranchName(runDir); expect(branch).toMatch(/^agentv\/wip\/[^/]+\/2026-01-15T10-00-00$/); }); @@ -2722,7 +2675,7 @@ describe('WIP branch helpers', () => { const pushed = await pushWipCheckpoint({ handle, sourceDir: runDir, - destinationPath: 'default/2026-01-15T10-00-00', + destinationPath: '2026-01-15T10-00-00', }); expect(pushed).toBe(true); @@ -2746,7 +2699,7 @@ describe('WIP branch helpers', () => { const first = await pushWipCheckpoint({ handle, sourceDir: runDir, - destinationPath: 'default/2026-01-15T11-00-00', + destinationPath: '2026-01-15T11-00-00', }); expect(first).toBe(true); @@ -2754,7 +2707,7 @@ describe('WIP branch helpers', () => { const second = await pushWipCheckpoint({ handle, sourceDir: runDir, - destinationPath: 'default/2026-01-15T11-00-00', + destinationPath: '2026-01-15T11-00-00', }); expect(second).toBe(false); } finally { @@ -2772,7 +2725,7 @@ describe('WIP branch helpers', () => { await pushWipCheckpoint({ handle, sourceDir: runDir, - destinationPath: 'default/2026-01-15T12-00-00', + destinationPath: '2026-01-15T12-00-00', }); } finally { await handle.cleanup(); @@ -2808,7 +2761,7 @@ describe('WIP branch helpers', () => { await pushWipCheckpoint({ handle, sourceDir: runDir, - destinationPath: 'default/2026-01-15T13-00-00', + destinationPath: '2026-01-15T13-00-00', }); } finally { await handle.cleanup(); From 177fc8f99d70c259ee6e0c900f963e910c76ade1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 22:22:44 +0200 Subject: [PATCH 2/4] docs(agents): copy main env into worktrees --- .agents/verification.md | 12 ++++++++++-- .agents/workflow.md | 8 ++++++++ AGENTS.md | 3 ++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.agents/verification.md b/.agents/verification.md index 6a0501e14..f955798ea 100644 --- a/.agents/verification.md +++ b/.agents/verification.md @@ -104,7 +104,7 @@ wait Unit tests alone are not enough for grader changes. -1. If you are in a git worktree, copy `.env` into the worktree root before claiming E2E or grader verification: +1. If you are in a git worktree, copy the ignored `.env` from the primary/main checkout into the worktree root before claiming E2E or grader verification: ```bash cp /path/to/main/.env .env @@ -114,6 +114,10 @@ cp /path/to/main/.env .env Copy-Item D:/path/to/main/.env .env ``` +Do not use `.env.example` as a credential substitute. If the primary/main +checkout has no `.env`, record the missing credentials as the exact live +provider or grader blocker. + 2. Run a real eval with a real example file: ```bash @@ -167,12 +171,16 @@ bun scripts/check-grader-scores.ts Before marking a branch ready for review: -1. Preflight: if in a git worktree, ensure `.env` exists in the worktree root. +1. Preflight: if in a git worktree, ensure `.env` exists in the worktree root by copying it from the primary/main checkout. ```bash cp "$(git worktree list --porcelain | head -1 | sed 's/worktree //')/.env" .env ``` +The copied file is local-only and must remain uncommitted. If there is no +primary/main `.env`, record that as a live-provider blocker before claiming +grader dogfood is unavailable. + 2. Run unit tests with `bun run test`. 3. Blocking manual red and green UAT: diff --git a/.agents/workflow.md b/.agents/workflow.md index 0defae69d..221d92107 100644 --- a/.agents/workflow.md +++ b/.agents/workflow.md @@ -42,6 +42,10 @@ bd where - These setup steps are required before running builds, tests, evals, or tracker updates in the worktree. +- The `.env` copy must come from the primary/main checkout and stays ignored in + the worktree. Do not copy `.env.example` as a credential substitute. If the + primary checkout has no `.env`, record the missing credentials as the exact + blocker for live provider/grader verification. - If you discover you are on a stale base or have uncoordinated dirty files, stop and fix that before changing code. - Whenever you `git checkout`, `gh pr checkout`, `git pull`, or otherwise switch to a ref that may have changed `package.json` or `bun.lock`, run `bun install` before building or testing. @@ -102,6 +106,10 @@ bun install cp "$(git worktree list --porcelain | head -1 | sed 's/worktree //')/.env" .env ``` +If the primary/main checkout does not have `.env`, stop before live eval or +grader verification and record the missing local credentials/proxy settings as +the blocker. + After the first meaningful commit, push and open a draft PR unless the user directs a different PR lifecycle: ```bash diff --git a/AGENTS.md b/AGENTS.md index 1f9dcb5f1..6ab4ece17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,10 +40,11 @@ Read the full rationale and examples in [.agents/product-boundary.md](.agents/pr - Do not use `git stash` on shared checkouts. Stage explicit paths only, and never push directly to `main`. - Every merge to `main` requires a GitHub pull request with passing GitHub Actions. Do not locally merge feature or integration branches into `main` as a substitute for opening a PR. - Prefer the primary checkout only for small, clean, bounded work. Use a dedicated worktree from the latest `origin/main` for non-trivial, risky, long-running, or parallel changes. +- When working from a dedicated worktree, copy the ignored `.env` from the primary/main checkout into the worktree before running evals, provider dogfood, grader verification, or local OpenAI OAuth proxy checks. Keep copied env files local and uncommitted; if the primary checkout has no `.env`, record that exact blocker instead of using `.env.example` as credentials. - Non-trivial work needs a plan or task list. If the implementation surface starts to balloon, stop and re-plan. - Large or high-risk PRs need meaningful, reviewable commits for each coherent change. Rewrite only the PR branch with `git push --force-with-lease` when needed to replace WIP or accidental squashed history before review. - Manual red/green UAT is blocking before a branch is ready for review. GitHub Actions is the authoritative merge gate. -- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. `agentv validate`, mock targets, replay/frozen transcript runs, and deterministic-only smoke tests are useful checks, but they are not live dogfood. Use canonical `.agentv/results//` output and publish private evidence. See [.agents/verification.md](.agents/verification.md). +- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. `agentv validate`, mock targets, replay/frozen transcript runs, and deterministic-only smoke tests are useful checks, but they are not live dogfood. Use canonical `.agentv/results//` output and publish private evidence. See [.agents/verification.md](.agents/verification.md). - For browser or screenshot UAT, keep evidence out of the public repo and publish reviewable artifacts to an `agentv-private` evidence branch. See [.agents/verification.md](.agents/verification.md). - When dogfood or review reveals a durable workflow lesson, capture it in this guide or the relevant `.agents/*.md` guide before merge; do not leave durable agent instructions only in PR comments, Bead comments, or private evidence. Use `docs/solutions/` for fuller reusable writeups. - Research-only workers must not run `bun install`, `bun run build`, tests, or evals unless the assigned work explicitly needs that command and the worker records why. From f4d643939acd1e71d4392a5dc2d64e3799125fc6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 23:44:47 +0200 Subject: [PATCH 3/4] test(results): align discovery fixtures with direct run roots --- apps/cli/test/commands/results/shared.test.ts | 18 +++--------------- apps/cli/test/commands/trend/trend.test.ts | 12 +++--------- apps/cli/test/eval.integration.test.ts | 8 ++++++-- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/apps/cli/test/commands/results/shared.test.ts b/apps/cli/test/commands/results/shared.test.ts index 37e201103..ac7144598 100644 --- a/apps/cli/test/commands/results/shared.test.ts +++ b/apps/cli/test/commands/results/shared.test.ts @@ -37,21 +37,9 @@ describe('results shared source resolution', () => { expect(resolved.sourceFile).toBe(path.join(runDir, 'index.jsonl')); }); - it('auto-discovers the most recent canonical run workspace', async () => { - const olderRunDir = path.join( - tempDir, - '.agentv', - 'results', - 'default', - '2026-03-24T10-00-00-000Z', - ); - const newerRunDir = path.join( - tempDir, - '.agentv', - 'results', - 'default', - '2026-03-25T10-00-00-000Z', - ); + it('auto-discovers the most recent direct run workspace', async () => { + const olderRunDir = path.join(tempDir, '.agentv', 'results', '2026-03-24T10-00-00-000Z'); + const newerRunDir = path.join(tempDir, '.agentv', 'results', '2026-03-25T10-00-00-000Z'); mkdirSync(olderRunDir, { recursive: true }); mkdirSync(newerRunDir, { recursive: true }); writeFileSync(path.join(olderRunDir, 'index.jsonl'), '{"test_id":"old","score":1}\n'); diff --git a/apps/cli/test/commands/trend/trend.test.ts b/apps/cli/test/commands/trend/trend.test.ts index 2f772f7f7..0147b3625 100644 --- a/apps/cli/test/commands/trend/trend.test.ts +++ b/apps/cli/test/commands/trend/trend.test.ts @@ -35,7 +35,7 @@ async function createRunWorkspace( runName: string, records: readonly RunRecordInput[], ): Promise<{ runDir: string; indexPath: string }> { - const runDir = path.join(rootDir, '.agentv', 'results', 'default', runName); + const runDir = path.join(rootDir, '.agentv', 'results', runName); await mkdir(runDir, { recursive: true }); const indexPath = path.join(runDir, 'index.jsonl'); await writeFile( @@ -269,14 +269,8 @@ describe('trend command', () => { const cwd = await createTempDir(); cleanupDirs.push(cwd); - const firstRunDir = path.join(cwd, '.agentv', 'results', 'default', '2026-03-01T10-00-00-000Z'); - const secondRunDir = path.join( - cwd, - '.agentv', - 'results', - 'default', - '2026-03-08T10-00-00-000Z', - ); + const firstRunDir = path.join(cwd, '.agentv', 'results', '2026-03-01T10-00-00-000Z'); + const secondRunDir = path.join(cwd, '.agentv', 'results', '2026-03-08T10-00-00-000Z'); await mkdir(firstRunDir, { recursive: true }); await mkdir(secondRunDir, { recursive: true }); const firstRecord = { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index e7eaac807..4050f8002 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -631,7 +631,9 @@ describe('agentv eval CLI', () => { expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); - expect(outputPath).toContain(`${path.sep}native-exp${path.sep}`); + expect(path.dirname(path.dirname(outputPath))).toBe( + path.join(fixture.suiteDir, '.agentv', 'results'), + ); const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ @@ -728,7 +730,9 @@ describe('agentv eval CLI', () => { expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); - expect(outputPath).toContain(`${path.sep}multi-eval${path.sep}`); + expect(path.dirname(path.dirname(outputPath))).toBe( + path.join(fixture.suiteDir, '.agentv', 'results'), + ); const diagnostics = await readDiagnostics(fixture); const calls = diagnostics.calls as Array>; From 1909eeab84df0104075b7a3a09351900ce87ede1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 23:51:21 +0200 Subject: [PATCH 4/4] test(results): use direct roots for delete and combine fixtures --- apps/cli/test/commands/results/combine.test.ts | 13 ++++++++----- apps/cli/test/commands/results/delete.test.ts | 12 ++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/cli/test/commands/results/combine.test.ts b/apps/cli/test/commands/results/combine.test.ts index 5d9642068..3f188f0c1 100644 --- a/apps/cli/test/commands/results/combine.test.ts +++ b/apps/cli/test/commands/results/combine.test.ts @@ -45,7 +45,7 @@ describe('results combine', () => { }); function seedRun(name: string, records: object[], experiment = 'default'): string { - const runDir = path.join(tempDir, '.agentv', 'results', experiment, name); + const runDir = path.join(tempDir, '.agentv', 'results', name); mkdirSync(path.join(runDir, 'demo', 'test-a'), { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -54,7 +54,10 @@ describe('results combine', () => { ); writeFileSync( path.join(runDir, 'summary.json'), - `${JSON.stringify({ manifest_path: 'index.jsonl' })}\n`, + `${JSON.stringify({ + manifest_path: 'index.jsonl', + metadata: { run_id: name, experiment }, + })}\n`, 'utf8', ); writeFileSync(path.join(runDir, 'demo', 'test-a', 'grading.json'), '{"assertions":[]}\n'); @@ -125,8 +128,8 @@ describe('results combine', () => { }); expect(combined.experiment).toBe('smoke'); - expect(combined.runId).toBe('smoke::2026-06-01T10-00-00-000Z'); - expect(combined.runDir).toContain(path.join('.agentv', 'results', 'smoke')); + expect(combined.runId).toBe('2026-06-01T10-00-00-000Z'); + expect(combined.runDir).toContain(path.join('.agentv', 'results')); expect(readIndex(combined.manifestPath).map((record) => record.experiment)).toEqual([ 'smoke', 'smoke', @@ -186,7 +189,7 @@ describe('results combine', () => { }); expect(combined.experiment).toBe('smoke-regression'); - expect(combined.runId).toBe('smoke-regression::2026-06-01T10-00-00-000Z'); + expect(combined.runId).toBe('2026-06-01T10-00-00-000Z'); expect(readIndex(combined.manifestPath).map((record) => record.experiment)).toEqual([ 'smoke-regression', 'smoke-regression', diff --git a/apps/cli/test/commands/results/delete.test.ts b/apps/cli/test/commands/results/delete.test.ts index a89cf0dc0..b04cdb816 100644 --- a/apps/cli/test/commands/results/delete.test.ts +++ b/apps/cli/test/commands/results/delete.test.ts @@ -24,7 +24,7 @@ describe('results delete', () => { }); function seedRun(runId: string): string { - const runDir = path.join(tempDir, '.agentv', 'results', ...runId.split('::')); + const runDir = path.join(tempDir, '.agentv', 'results', runId); mkdirSync(runDir, { recursive: true }); writeFileSync( path.join(runDir, 'index.jsonl'), @@ -41,16 +41,16 @@ describe('results delete', () => { } it('deletes a local run workspace by run ID', () => { - const runDir = seedRun('demo::2026-06-01T10-00-00-000Z'); + const runDir = seedRun('2026-06-01T10-00-00-000Z'); - const deleted = deleteLocalRun(tempDir, 'demo::2026-06-01T10-00-00-000Z'); + const deleted = deleteLocalRun(tempDir, '2026-06-01T10-00-00-000Z'); - expect(deleted.runId).toBe('demo::2026-06-01T10-00-00-000Z'); + expect(deleted.runId).toBe('2026-06-01T10-00-00-000Z'); expect(existsSync(runDir)).toBe(false); }); it('resolves and deletes by workspace path', () => { - const runDir = seedRun('default::2026-06-01T10-00-00-000Z'); + const runDir = seedRun('2026-06-01T10-00-00-000Z'); const target = resolveDeleteRunTarget(tempDir, runDir); expect(target.runDir).toBe(runDir); @@ -60,7 +60,7 @@ describe('results delete', () => { }); it('rejects remote IDs and paths outside the local runs directory', () => { - seedRun('default::2026-06-01T10-00-00-000Z'); + seedRun('2026-06-01T10-00-00-000Z'); const outsideDir = path.join(tempDir, 'outside-run'); mkdirSync(outsideDir, { recursive: true }); writeFileSync(path.join(outsideDir, 'index.jsonl'), toJsonl({ score: 1 }), 'utf8');