From 17e1e2281481c725a5fd3701524c7a319f9f104d Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 12:10:19 +0200
Subject: [PATCH 01/18] docs: plan experiment separation

---
 ...26-06-23-experiments-vs-eval-separation.md | 151 +++++++
 ...6-06-23-002-experiments-separation-plan.md | 388 ++++++++++++++++++
 2 files changed, 539 insertions(+)
 create mode 100644 docs/adr/2026-06-23-experiments-vs-eval-separation.md
 create mode 100644 docs/plans/2026-06-23-002-experiments-separation-plan.md

diff --git a/docs/adr/2026-06-23-experiments-vs-eval-separation.md b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
new file mode 100644
index 000000000..61898f85b
--- /dev/null
+++ b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
@@ -0,0 +1,151 @@
+# ADR: Separate experiments from eval definitions
+
+Date: 2026-06-23
+
+Status: Proposed
+
+## Context
+
+AgentV currently treats an experiment as a run label. The label is threaded
+through evaluation config and recorded in run artifacts, but the agent, model,
+harness, repeat count, timeout, sandbox, and setup choices mostly remain in
+`eval.yaml` `execution` fields or CLI flags.
+
+That conflates two concerns:
+
+- The eval definition is the task contract: prompt, dataset, workspace fixture
+  required by the task, and assertions or graders.
+- The experiment is the run contract: which agent or target is under test, which
+  model and harness are used, how many runs to execute, what setup is injected,
+  and which evals are selected.
+
+The public Vercel `agent-eval` ecosystem is a useful reference point. The
+`vercel/next.js` evals keep task fixtures under `evals/`, while experiments are
+generated or committed separately. `vercel/next-evals-oss` commits many
+`experiments/*.ts` variants and stores results under experiment-specific
+directories. The useful pattern is the vocabulary and ownership split, not a
+requirement that AgentV adopt Vercel's package as its core runtime.
+
+This decision must also preserve AgentV's existing product boundary:
+
+- AgentV stays repo-native and zero-infra by default.
+- Portable run artifacts remain the source of truth.
+- Core primitives should stay small and composable.
+- Public wire formats use `snake_case`; TypeScript internals use `camelCase`.
+- `project` means the run, trace, and experiment container; `benchmark` means a
+  curated eval suite.
+
+## Vocabulary
+
+An eval is a frozen task definition. It includes the prompt or dataset, expected
+behavior, task-owned workspace fixtures, and assertions. AgentV's LLM-judge,
+code-grader, deterministic assertions, and hidden or explicit evaluation
+criteria belong here.
+
+An experiment is a committed or generated run definition. It declares which
+agent, target, provider, model, harness options, setup steps, run count, timeout,
+sandbox, and eval selector are used. Setup that changes the system under test,
+such as installing dependencies or dropping an `AGENTS.md` or skill file, belongs
+here because it is an A/B variable.
+
+An execution compatibility block is the legacy `eval.yaml` location for runner
+selection and runtime controls. It remains supported during migration but should
+stop being the canonical home for experiment-level choices.
+
+## Decision
+
+AgentV will make experiments first-class configuration units separate from
+eval definitions.
+
+Eval files remain YAML-authored by default. They should describe what is tested:
+task inputs, datasets, assertions, and task fixtures. They should not be the
+canonical place for which agent, model, harness, setup injection, sandbox, or run
+matrix executes the task.
+
+Experiment files will live under `experiments/` by convention. AgentV will
+support YAML as the canonical authoring path for the abstraction story and TypeScript
+as the power-user escape hatch:
+
+```yaml
+name: copilot-gpt55-withskill
+target: copilot-gpt55
+model: openai/gpt-5.5
+evals: "evals/**/*.eval.yaml"
+scripts:
+  - build
+runs: 3
+early_exit: false
+timeout_seconds: 900
+sandbox: auto
+setup:
+  - script: bun install
+  - script: cp skills/copilot/AGENTS.md AGENTS.md
+```
+
+`config.yaml` will gain a default experiment pointer so existing `agentv eval`
+usage keeps working:
+
+```yaml
+experiments:
+  default: experiments/default.yaml
+```
+
+If no default experiment is configured, AgentV keeps the current behavior and
+uses the `default` experiment label. Existing `eval.yaml`-only repositories
+remain valid.
+
+Legacy `eval.yaml execution` fields that select targets, targets matrices,
+workers, cache, trials, budget, thresholds, and workspace runtime behavior will
+continue to parse. The migration path is additive first: experiment config
+becomes the preferred source, while legacy fields remain a compatibility shim
+until docs and examples have moved.
+
+AgentV should adopt Vercel's structure and lowest-common-denominator contract
+ideas, not depend on `@vercel/agent-eval` as core infrastructure in this phase.
+The package's `ExperimentConfig` shape is a strong public reference for
+experiment vocabulary: agent, model, agent options, eval filter, scripts, runs,
+early exit, timeout, sandbox, and setup. A direct dependency would force AgentV
+to absorb Vercel's fixture model, sandbox assumptions, result caching semantics,
+and TypeScript-first authoring story before those boundaries are stable for
+AgentV.
+
+## Consequences
+
+Positive:
+
+- Evals become portable task definitions that can be run against multiple agents
+  without editing the task file.
+- A/B setup variants such as baseline versus skill injection become reviewable,
+  committed experiment files.
+- Existing artifact paths already use experiment labels, so this decision extends
+  an established storage axis instead of introducing a parallel result concept.
+- The default experiment pointer gives old repos a non-breaking migration path.
+- AgentV can align with Vercel conventions while preserving YAML authoring,
+  LLM-judge assertions, workspace fixtures, and Git-backed artifacts.
+
+Negative:
+
+- The migration creates two valid locations for some runtime controls until
+  deprecation completes.
+- The CLI must resolve explicit experiments, configured defaults, and legacy
+  label-only runs without surprising users.
+- Artifact readers need a richer experiment fingerprint and provenance model
+  beyond the current string label.
+
+## Non-Goals
+
+- Do not replace AgentV's evaluator engine with `@vercel/agent-eval` in the
+  initial migration.
+- Do not convert AgentV eval YAML into Vercel `PROMPT.md` plus `EVAL.ts`.
+- Do not move LLM-judge assertions out of eval definitions.
+- Do not make Phoenix, Harbor, Opik, Vercel Sandbox, or another external system
+  required for local execution.
+- Do not break existing `eval.yaml` files or current result artifacts.
+
+## References
+
+- Vercel `agent-eval`: https://github.com/vercel-labs/agent-eval
+- Vercel Next.js eval results: https://github.com/vercel/next-evals-oss
+- Anthropic Skills schema vocabulary: https://github.com/anthropics/skills/blob/main/skills/skill-creator/references/schemas.md
+- Hugging Face Datasets vocabulary: https://huggingface.co/docs/datasets/en/package_reference/main_classes
+- OpenInference trace vocabulary: https://arize-ai.github.io/openinference/spec/
diff --git a/docs/plans/2026-06-23-002-experiments-separation-plan.md b/docs/plans/2026-06-23-002-experiments-separation-plan.md
new file mode 100644
index 000000000..0749d1eeb
--- /dev/null
+++ b/docs/plans/2026-06-23-002-experiments-separation-plan.md
@@ -0,0 +1,388 @@
+---
+title: "feat: Separate experiments from eval definitions"
+type: feat
+date: 2026-06-23
+origin: docs/adr/2026-06-23-experiments-vs-eval-separation.md
+---
+
+# feat: Separate experiments from eval definitions
+
+## Summary
+
+AgentV should separate eval task definitions from experiment run definitions.
+Eval YAML stays the canonical authoring layer for prompts, datasets, assertions,
+and task fixtures. Experiments become first-class committed files that select the
+agent or target under test, model, harness options, setup injection, run knobs,
+and eval filter.
+
+This should ship in phases. Phase 1 adds the non-breaking foundation:
+experiment contract types, default experiment resolution, and artifact
+attribution by resolved experiment name. Later phases move runtime controls out
+of `eval.yaml execution`, teach the CLI to run experiment matrices, and record
+full experiment provenance and fingerprints in run bundles.
+
+## Problem Frame
+
+Today `experiment` is a string label passed through
+`packages/core/src/evaluation/evaluate.ts`, `packages/core/src/evaluation/run-artifacts.ts`,
+`packages/core/src/evaluation/results-repo.ts`, and
+`packages/core/src/evaluation/trace-envelope.ts`. Runtime choices are still
+scattered across CLI flags, TypeScript config, `.agentv/config.yaml`, and
+`eval.yaml execution`.
+
+That makes it hard to review A/B variants such as `baseline` versus
+`agents-md`, because the variable under test can be hidden inside the eval
+definition. The desired model is:
+
+- Eval equals what is tested.
+- Experiment equals how and with what it is run.
+- Setup that changes the agent's environment belongs to the experiment.
+- Existing eval-only repositories keep working through a default experiment
+  fallback.
+
+## Requirements
+
+- R1. Existing `eval.yaml` files validate and run without modification.
+- R2. Experiment wire config uses `snake_case`; TypeScript types use
+  `camelCase`.
+- R3. `config.yaml` can point at a default experiment, with no pointer falling
+  back to the current `default` experiment label.
+- R4. `agentv eval --experiment <label>` keeps working as a label.
+- R5. `agentv eval --experiment <path>` can resolve a YAML or TypeScript
+  experiment file.
+- R6. Experiment config reuses existing target names and target matrices instead
+  of embedding a new provider schema.
+- R7. Workspace setup and skill injection are modeled as experiment setup steps.
+- R8. Run artifacts record the resolved experiment name immediately and later
+  record full config provenance and fingerprint.
+- R9. Documentation and examples migrate incrementally; no bulk repo migration
+  happens in Phase 1.
+
+## Public Reference Alignment
+
+Vercel `agent-eval` supplies the strongest public precedent for this split:
+eval fixtures describe the task, and experiments describe agent, model, scripts,
+runs, early exit, timeout, sandbox, and setup. AgentV should adopt that
+vocabulary and directory convention while preserving AgentV-owned YAML evals,
+LLM-judge assertions, workspace fixtures, and portable run artifacts.
+
+Anthropic Skills reinforces the value of baseline versus with-skill comparisons
+and pass-rate deltas. Hugging Face Datasets provides the lowest-common
+denominator vocabulary for datasets, records, splits, and features. OpenInference
+provides trace and span vocabulary for external observability correlation. These
+references should inform field names and docs, but none should become a required
+runtime dependency for this change.
+
+## Key Decisions
+
+- KTD1. The canonical committed experiment directory is `experiments/`.
+- KTD2. YAML is the canonical experiment authoring path; TypeScript is the
+  escape hatch for dynamic setup.
+- KTD3. `experiments.default` in `.agentv/config.yaml` is the preferred default
+  pointer. A top-level `default_experiment` compatibility key can be accepted
+  while docs teach the nested form.
+- KTD4. Experiment `target` and `targets` refer to existing AgentV target names
+  and target refs. Provider settings stay in `targets.yaml`.
+- KTD5. `eval.yaml execution` remains valid in Phase 1 and becomes deprecated
+  only after examples and docs have a working experiment equivalent.
+- KTD6. AgentV adopts Vercel's experiment structure, not the package dependency,
+  until a direct adapter has a smaller, reviewed boundary.
+- KTD7. Full experiment fingerprints should include the experiment file contents,
+  selected eval source, setup-relevant fields, scripts, runs, timeout, sandbox,
+  and target references.
+
+## Experiment Contract
+
+Wire shape, shown in YAML:
+
+```yaml
+name: baseline
+target: codex-gpt5
+targets:
+  - codex-gpt5
+  - name: copilot-gpt55
+    use_target: copilot
+agent: codex
+model: openai/gpt-5.5
+agent_options:
+  reasoning_effort: high
+evals: "evals/**/*.eval.yaml"
+scripts:
+  - build
+  - script: bun test
+    timeout_seconds: 120
+runs: 3
+early_exit: false
+timeout_seconds: 900
+sandbox: auto
+workspace:
+  mode: temp
+setup:
+  - script: bun install
+  - script: cp skills/default/AGENTS.md AGENTS.md
+```
+
+Internal TypeScript shape:
+
+```ts
+interface ExperimentConfig {
+  name?: string;
+  target?: string;
+  targets?: readonly ExperimentTargetRef[];
+  agent?: string;
+  model?: string;
+  agentOptions?: Record<string, unknown>;
+  evals?: string | readonly string[];
+  scripts?: readonly ExperimentScript[];
+  runs?: number;
+  earlyExit?: boolean;
+  timeoutSeconds?: number;
+  sandbox?: 'auto' | 'docker' | 'vercel';
+  workspace?: Record<string, unknown>;
+  setup?: readonly ExperimentSetupStep[];
+  sourcePath?: string;
+}
+```
+
+`agent` is a harness label for Vercel alignment. AgentV execution should prefer
+`target` or `targets` for actual provider resolution so this does not create a
+parallel provider registry.
+
+## CLI Behavior
+
+Default resolution order for `agentv eval`:
+
+- Explicit `--experiment <label-or-path>`.
+- `.agentv/config.yaml` `experiments.default`.
+- `.agentv/config.yaml` `default_experiment`, accepted as a compatibility alias.
+- `agentv.config.ts` `experiments.default` or `defaultExperiment`, if present.
+- Current implicit `default` label.
+
+Path-like experiment values load an experiment file. Label-like values remain
+labels. If a loaded experiment has `name`, the name is the run namespace;
+otherwise AgentV derives the name from the file basename.
+
+Later CLI phases should add:
+
+- `agentv eval --experiment experiments/baseline.yaml`.
+- `agentv eval --experiment baseline` resolving `experiments/baseline.yaml`
+  before falling back to a label.
+- `agentv eval --experiments "experiments/*.yaml"` for matrices.
+- Experiment `evals` filters when no eval paths are provided.
+
+## Migration Strategy
+
+Phase 1 is additive. It introduces experiment config loading and default
+resolution without changing how eval execution applies targets or workspace
+settings.
+
+Phase 2 moves examples to committed `experiments/default.yaml` files while
+leaving existing `eval.yaml execution` fields in place.
+
+Phase 3 applies experiment runtime fields in the runner: target selection, eval
+filters, timeout, runs, early exit, sandbox/workspace mode, setup steps, and
+scripts.
+
+Phase 4 warns when new eval files use experiment-owned `execution` fields and
+documents the replacement.
+
+Phase 5 removes or hard-errors only for a future major or same-week unreleased
+surface where compatibility is not required.
+
+## Artifact Impact
+
+Existing artifact writers already accept an experiment label. Phase 1 should
+continue writing the resolved experiment name to `benchmark.json`, `index.jsonl`,
+trace envelopes, and results repository paths.
+
+Later artifact work should add:
+
+- `experiment_config_path` for the committed file.
+- `experiment_fingerprint` for cache and comparison invalidation.
+- Redacted `experiment_config` metadata for small safe fields.
+- `setup` and `scripts` provenance as references, not large inline payloads.
+
+`artifact_pointers` must remain reserved for detached large payload bytes. Normal
+experiment sidecars should use explicit path fields.
+
+## Implementation Units
+
+### U1. ADR and Implementation Plan
+
+Files:
+
+- `docs/adr/2026-06-23-experiments-vs-eval-separation.md`
+- `docs/plans/2026-06-23-002-experiments-separation-plan.md`
+
+Approach:
+
+Capture the eval-versus-experiment decision, Vercel alignment, dependency
+boundary, compatibility strategy, and phased rollout.
+
+Verification:
+
+- Human review for vocabulary and product boundary.
+
+### U2. Experiment Contract and Loader
+
+Files:
+
+- `packages/core/src/evaluation/experiment.ts`
+- `packages/core/src/index.ts`
+- `packages/core/src/evaluation/config.ts`
+- `packages/core/src/evaluation/loaders/config-loader.ts`
+
+Approach:
+
+Add a narrow experiment wire contract, normalization to camelCase, YAML and
+TypeScript experiment loading, and default experiment config parsing. Keep the
+loader independent of runner behavior.
+
+Test Scenarios:
+
+- YAML experiment with `agent_options`, `early_exit`, `timeout_seconds`, setup,
+  and scripts normalizes to camelCase.
+- TypeScript experiment default export loads and normalizes.
+- Invalid `runs`, `timeout_seconds`, or `sandbox` fails with a targeted error.
+- `.agentv/config.yaml` parses `experiments.default`.
+- `.agentv/config.yaml` accepts top-level `default_experiment`.
+
+### U3. CLI Default Experiment Resolution
+
+Files:
+
+- `apps/cli/src/commands/eval/run-eval.ts`
+- `apps/cli/src/commands/eval/result-layout.ts`
+
+Approach:
+
+Resolve the experiment before run directory creation. Explicit labels keep
+working. Path-like values load experiment files and derive the run label from
+`name` or basename. If no experiment is configured, the current `default` label
+is preserved.
+
+Test Scenarios:
+
+- `agentv eval evals/foo/eval.yaml` still writes under
+  `.agentv/results/default/<timestamp>/`.
+- Configured `experiments.default: experiments/default.yaml` with `name:
+  baseline` writes under `.agentv/results/baseline/<timestamp>/`.
+- `--experiment smoke` writes under `.agentv/results/smoke/<timestamp>/`.
+- `--experiment experiments/smoke.yaml` uses the file's `name` when present.
+- Missing path-like experiment values fail clearly.
+
+### U4. Runner Field Application
+
+Files:
+
+- `apps/cli/src/commands/eval/run-eval.ts`
+- `packages/core/src/evaluation/evaluate.ts`
+- `packages/core/src/evaluation/yaml-parser.ts`
+- `packages/core/src/evaluation/loaders/config-loader.ts`
+- `packages/core/src/evaluation/validation/eval-file.schema.ts`
+
+Approach:
+
+Apply experiment fields in precedence order: CLI overrides, explicit experiment,
+legacy eval `execution`, project config defaults. Reuse existing target
+resolution and workspace setup paths. Move setup-owned behavior out of eval docs
+before adding warnings.
+
+Test Scenarios:
+
+- Experiment `target` selects an existing target from `targets.yaml`.
+- Experiment `targets` drives matrix evaluation.
+- Experiment `evals` selects eval files when no positional eval paths are given.
+- Experiment setup runs before agent execution and can inject an `AGENTS.md`
+  file.
+- Legacy `eval.yaml execution.target` still works when no experiment target is
+  configured.
+
+### U5. Artifact Provenance and Fingerprint
+
+Files:
+
+- `packages/core/src/evaluation/run-artifacts.ts`
+- `packages/core/src/evaluation/results-repo.ts`
+- `packages/core/src/evaluation/trace-envelope.ts`
+- `apps/cli/src/commands/eval/artifact-writer.ts`
+
+Approach:
+
+Extend artifact metadata with safe experiment provenance and a fingerprint. The
+fingerprint should cover experiment config, selected eval source content, and
+fields that affect execution.
+
+Test Scenarios:
+
+- Two experiments with different setup steps produce different fingerprints.
+- Changing an eval prompt changes the fingerprint.
+- Redacted config metadata excludes obvious secret fields.
+- Historical artifacts without fingerprints still read.
+
+### U6. Docs and Examples Migration
+
+Files:
+
+- `apps/web/src/content/docs/`
+- `examples/`
+- `CONCEPTS.md`
+
+Approach:
+
+Document eval versus experiment authoring. Add at least one example with
+`eval.yaml` plus `experiments/default.yaml`, and one A/B pair such as baseline
+versus with-skill.
+
+Test Scenarios:
+
+- Docs examples use `snake_case` wire fields.
+- Example default experiment runs without explicit `--experiment`.
+- Existing examples continue to run through the compatibility path.
+
+## Phase 1 Scope
+
+Phase 1 includes U1, U2, and U3 only.
+
+Phase 1 explicitly does not apply `setup`, `scripts`, `runs`, `early_exit`,
+`timeout_seconds`, `target`, `targets`, or `evals` to execution behavior. Those
+fields are parsed and ready for later phases, but current runner semantics stay
+unchanged except for resolving the experiment name.
+
+## Non-Goals
+
+- Do not replace AgentV's engine with `@vercel/agent-eval`.
+- Do not bulk-edit all examples in the first PR.
+- Do not remove or error on legacy `eval.yaml execution` fields.
+- Do not add a new provider schema inside experiments.
+- Do not implement experiment matrix execution in Phase 1.
+- Do not project AgentV experiments into Phoenix or another external store.
+
+## Verification Plan
+
+Phase 1 targeted checks:
+
+- `bun test packages/core/test/evaluation` for loader and config coverage once
+  tests exist.
+- CLI-level test for default experiment run layout.
+- A dry-run fixture using `experiments/default.yaml` to prove the resolved
+  experiment name reaches artifacts.
+- Existing eval-only dry-run fixture to prove fallback stays `default`.
+
+Before PR readiness:
+
+- Run the smallest targeted Bun test set covering changed loader and CLI
+  behavior.
+- Run TypeScript/package checks only if the touched packages require them or
+  targeted tests expose type errors.
+
+## Open Questions
+
+- Should the public field be only `experiments.default`, or should
+  `default_experiment` remain documented as a short alias?
+- Should `agent` be purely descriptive in AgentV, or should it become a harness
+  selector once non-target harness adapters exist?
+- Should experiment setup reuse workspace hooks exactly, or have a smaller setup
+  step schema that compiles into workspace hooks?
+- Should experiment fingerprints be written in Phase 2 with no cache behavior, or
+  wait until result reuse can consume them?

From ae79ac507c7801d5a13941119b437226783d3edc Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 12:10:26 +0200
Subject: [PATCH 02/18] feat: add experiment config foundation

---
 apps/cli/src/commands/eval/run-eval.ts        |  47 ++
 packages/core/src/evaluation/config.ts        |  12 +
 packages/core/src/evaluation/experiment.ts    | 408 ++++++++++++++++++
 .../src/evaluation/loaders/config-loader.ts   |  58 +++
 packages/core/src/index.ts                    |   3 +
 packages/core/test/evaluation/config.test.ts  |   9 +
 .../core/test/evaluation/experiment.test.ts   |  94 ++++
 .../evaluation/loaders/config-loader.test.ts  |  51 +++
 8 files changed, 682 insertions(+)
 create mode 100644 packages/core/src/evaluation/experiment.ts
 create mode 100644 packages/core/test/evaluation/experiment.test.ts

diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts
index 47a6d4940..1c8a498e8 100644
--- a/apps/cli/src/commands/eval/run-eval.ts
+++ b/apps/cli/src/commands/eval/run-eval.ts
@@ -18,10 +18,14 @@ import {
   buildTraceFromMessages,
   runEvaluation as defaultRunEvaluation,
   deriveCategory,
+  deriveExperimentNameFromPath,
   ensureVSCodeSubagents,
+  isExperimentFileReference,
   loadConfig,
+  loadExperimentConfig,
   loadTestSuite,
   loadTsConfig,
+  resolveDefaultExperimentReference,
   resolveTargetDefinition,
   shouldEnableCache,
   shouldSkipCacheForTemperature,
@@ -554,6 +558,40 @@ function buildDefaultOutputPathForExperiment(
   return path.join(runDir, 'index.jsonl');
 }
 
+function normalizeTsDefaultExperiment(
+  config: Awaited<ReturnType<typeof loadTsConfig>> | null,
+): string | undefined {
+  return (
+    normalizeString(config?.experiments?.default) ?? normalizeString(config?.defaultExperiment)
+  );
+}
+
+async function resolveExperimentNameForRun(params: {
+  readonly cwd: string;
+  readonly explicitExperiment?: string;
+  readonly yamlDefaultExperiment?: string;
+  readonly tsDefaultExperiment?: string;
+}): Promise<string | undefined> {
+  const experimentRef =
+    params.explicitExperiment ?? params.yamlDefaultExperiment ?? params.tsDefaultExperiment;
+  if (!experimentRef) {
+    return undefined;
+  }
+  if (!isExperimentFileReference(experimentRef)) {
+    return experimentRef;
+  }
+
+  const experimentPath = path.isAbsolute(experimentRef)
+    ? experimentRef
+    : path.resolve(params.cwd, experimentRef);
+  if (!existsSync(experimentPath)) {
+    throw new Error(`Experiment file not found: ${experimentRef}`);
+  }
+
+  const experiment = await loadExperimentConfig(experimentPath);
+  return experiment.name ?? deriveExperimentNameFromPath(experimentPath);
+}
+
 type ProgressReporter = {
   readonly isInteractive: boolean;
   start(): void;
@@ -1167,6 +1205,15 @@ export async function runEvalCommand(
   }
 
   let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  options = {
+    ...options,
+    experiment: await resolveExperimentNameForRun({
+      cwd,
+      explicitExperiment: options.experiment,
+      yamlDefaultExperiment: resolveDefaultExperimentReference(yamlConfig),
+      tsDefaultExperiment: normalizeTsDefaultExperiment(config),
+    }),
+  };
   if (!process.env.AGENTV_EXPERIMENT) {
     process.env.AGENTV_EXPERIMENT = normalizeExperimentName(options.experiment);
   }
diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts
index a82708ed0..7d2aa6adb 100644
--- a/packages/core/src/evaluation/config.ts
+++ b/packages/core/src/evaluation/config.ts
@@ -67,6 +67,18 @@ const AgentVConfigSchema = z.object({
     })
     .optional(),
 
+  /** Compatibility shorthand for experiments.default */
+  defaultExperiment: z.string().optional(),
+
+  /** Experiment defaults */
+  experiments: z
+    .object({
+      /** Default experiment label or path used when eval runs omit --experiment */
+      default: z.string().optional(),
+    })
+    .strict()
+    .optional(),
+
   /** Cost and duration limits */
   limits: z
     .object({
diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts
new file mode 100644
index 000000000..5ccc7bee8
--- /dev/null
+++ b/packages/core/src/evaluation/experiment.ts
@@ -0,0 +1,408 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { parseYamlValue } from './yaml-loader.js';
+
+export type ExperimentSandbox = 'auto' | 'docker' | 'vercel';
+
+export type ExperimentTargetRefWire =
+  | string
+  | {
+      readonly name: string;
+      readonly use_target?: string;
+      readonly hooks?: Record<string, unknown>;
+    };
+
+export type ExperimentTargetRef =
+  | string
+  | {
+      readonly name: string;
+      readonly useTarget?: string;
+      readonly hooks?: Record<string, unknown>;
+    };
+
+export type ExperimentScriptWire =
+  | string
+  | {
+      readonly command?: string | readonly string[];
+      readonly script?: string | readonly string[];
+      readonly timeout_seconds?: number;
+      readonly cwd?: string;
+      readonly env?: Record<string, string>;
+    };
+
+export type ExperimentScript = {
+  readonly command?: readonly string[];
+  readonly script?: string | readonly string[];
+  readonly timeoutSeconds?: number;
+  readonly cwd?: string;
+  readonly env?: Record<string, string>;
+};
+
+export type ExperimentSetupFn = (sandbox: unknown) => void | Promise<void>;
+export type ExperimentSetup = readonly ExperimentScript[] | ExperimentSetupFn;
+
+export type ExperimentConfigWire = {
+  readonly name?: string;
+  readonly agent?: string;
+  readonly target?: string;
+  readonly targets?: readonly ExperimentTargetRefWire[];
+  readonly model?: string;
+  readonly agent_options?: Record<string, unknown>;
+  readonly evals?: string | readonly string[];
+  readonly scripts?: readonly ExperimentScriptWire[];
+  readonly runs?: number;
+  readonly early_exit?: boolean;
+  readonly timeout_seconds?: number;
+  readonly sandbox?: ExperimentSandbox;
+  readonly workspace?: Record<string, unknown>;
+  readonly setup?: readonly ExperimentScriptWire[] | ExperimentSetupFn;
+};
+
+export type ExperimentConfig = {
+  readonly name?: string;
+  readonly agent?: string;
+  readonly target?: string;
+  readonly targets?: readonly ExperimentTargetRef[];
+  readonly model?: string;
+  readonly agentOptions?: Record<string, unknown>;
+  readonly evals?: string | readonly string[];
+  readonly scripts?: readonly ExperimentScript[];
+  readonly runs?: number;
+  readonly earlyExit?: boolean;
+  readonly timeoutSeconds?: number;
+  readonly sandbox?: ExperimentSandbox;
+  readonly workspace?: Record<string, unknown>;
+  readonly setup?: ExperimentSetup;
+  readonly sourcePath?: string;
+  readonly fingerprint?: string;
+};
+
+type NormalizeOptions = {
+  readonly sourcePath?: string;
+};
+
+const EXPERIMENT_FILE_EXTENSIONS = new Set(['.yaml', '.yml', '.ts', '.js', '.mts', '.mjs']);
+const VALID_SANDBOXES: ReadonlySet<string> = new Set(['auto', 'docker', 'vercel']);
+
+export function isExperimentFileReference(value: string): boolean {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return false;
+  }
+  return (
+    trimmed.includes('/') ||
+    trimmed.includes('\\') ||
+    EXPERIMENT_FILE_EXTENSIONS.has(path.extname(trimmed).toLowerCase())
+  );
+}
+
+export function deriveExperimentNameFromPath(filePath: string): string {
+  return path
+    .basename(filePath)
+    .replace(/\.experiment\.(ya?ml|[cm]?[jt]s)$/i, '')
+    .replace(/\.(ya?ml|[cm]?[jt]s)$/i, '');
+}
+
+export async function loadExperimentConfig(filePath: string): Promise<ExperimentConfig> {
+  const resolvedPath = path.resolve(filePath);
+  const ext = path.extname(resolvedPath).toLowerCase();
+  let rawConfig: unknown;
+
+  if (ext === '.yaml' || ext === '.yml') {
+    rawConfig = parseYamlValue(await readFile(resolvedPath, 'utf8'));
+  } else if (EXPERIMENT_FILE_EXTENSIONS.has(ext)) {
+    const moduleUrl = pathToFileURL(resolvedPath).href;
+    const mod = await import(moduleUrl);
+    rawConfig = mod.default ?? mod.config ?? mod;
+  } else {
+    throw new Error(
+      `Unsupported experiment file extension '${ext}'. Use .yaml, .yml, .ts, .js, .mts, or .mjs.`,
+    );
+  }
+
+  return normalizeExperimentConfig(rawConfig, { sourcePath: resolvedPath });
+}
+
+export function normalizeExperimentConfig(
+  rawConfig: unknown,
+  options: NormalizeOptions = {},
+): ExperimentConfig {
+  if (!isRecord(rawConfig)) {
+    throw new Error('Experiment config must be an object.');
+  }
+
+  const name = readOptionalString(rawConfig.name, 'name');
+  const agent = readOptionalString(rawConfig.agent, 'agent');
+  const target = readOptionalString(rawConfig.target, 'target');
+  const targets = readTargets(rawConfig.targets);
+  const model = readOptionalString(rawConfig.model, 'model');
+  const agentOptions = readOptionalRecord(rawConfig.agent_options ?? rawConfig.agentOptions);
+  const evals = readOptionalStringOrStringArray(rawConfig.evals, 'evals');
+  const scripts = readScriptArray(rawConfig.scripts, 'scripts');
+  const runs = readOptionalPositiveInteger(rawConfig.runs, 'runs');
+  const earlyExit = readOptionalBoolean(rawConfig.early_exit ?? rawConfig.earlyExit, 'early_exit');
+  const timeoutSeconds = readOptionalPositiveNumber(
+    rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
+    'timeout_seconds',
+  );
+  const sandbox = readOptionalSandbox(rawConfig.sandbox);
+  const workspace = readOptionalRecord(rawConfig.workspace);
+  const setup = readSetup(rawConfig.setup);
+
+  const configWithoutFingerprint: Omit<ExperimentConfig, 'fingerprint'> = {
+    ...(name !== undefined && { name }),
+    ...(agent !== undefined && { agent }),
+    ...(target !== undefined && { target }),
+    ...(targets !== undefined && { targets }),
+    ...(model !== undefined && { model }),
+    ...(agentOptions !== undefined && { agentOptions }),
+    ...(evals !== undefined && { evals }),
+    ...(scripts !== undefined && { scripts }),
+    ...(runs !== undefined && { runs }),
+    ...(earlyExit !== undefined && { earlyExit }),
+    ...(timeoutSeconds !== undefined && { timeoutSeconds }),
+    ...(sandbox !== undefined && { sandbox }),
+    ...(workspace !== undefined && { workspace }),
+    ...(setup !== undefined && { setup }),
+    ...(options.sourcePath !== undefined && { sourcePath: options.sourcePath }),
+  };
+
+  return {
+    ...configWithoutFingerprint,
+    fingerprint: fingerprintExperimentConfig(configWithoutFingerprint),
+  };
+}
+
+export function fingerprintExperimentConfig(config: ExperimentConfig): string {
+  const stablePayload = toStableJsonValue(config);
+  return createHash('sha256').update(JSON.stringify(stablePayload)).digest('hex');
+}
+
+function readTargets(raw: unknown): readonly ExperimentTargetRef[] | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (!Array.isArray(raw)) {
+    throw new Error('Experiment targets must be an array.');
+  }
+  return raw.map((entry, index): ExperimentTargetRef => {
+    if (typeof entry === 'string' && entry.trim().length > 0) {
+      return entry.trim();
+    }
+    if (!isRecord(entry)) {
+      throw new Error(`Experiment targets[${index}] must be a string or object.`);
+    }
+    const name = readRequiredString(entry.name, `targets[${index}].name`);
+    const useTarget = readOptionalString(
+      entry.use_target ?? entry.useTarget,
+      `targets[${index}].use_target`,
+    );
+    const hooks = readOptionalRecord(entry.hooks);
+    return {
+      name,
+      ...(useTarget !== undefined && { useTarget }),
+      ...(hooks !== undefined && { hooks }),
+    };
+  });
+}
+
+function readScriptArray(raw: unknown, location: string): readonly ExperimentScript[] | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (!Array.isArray(raw)) {
+    throw new Error(`Experiment ${location} must be an array.`);
+  }
+  return raw.map((entry, index) => readScript(entry, `${location}[${index}]`));
+}
+
+function readSetup(raw: unknown): ExperimentSetup | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw === 'function') {
+    return raw as ExperimentSetupFn;
+  }
+  return readScriptArray(raw, 'setup');
+}
+
+function readScript(raw: unknown, location: string): ExperimentScript {
+  if (typeof raw === 'string') {
+    const script = raw.trim();
+    if (!script) {
+      throw new Error(`Experiment ${location} must not be empty.`);
+    }
+    return { script };
+  }
+  if (!isRecord(raw)) {
+    throw new Error(`Experiment ${location} must be a string or object.`);
+  }
+
+  const command = readOptionalCommand(raw.command, `${location}.command`);
+  const script = readOptionalStringOrStringArray(raw.script, `${location}.script`);
+  if (command === undefined && script === undefined) {
+    throw new Error(`Experiment ${location} must define command or script.`);
+  }
+
+  const timeoutSeconds = readOptionalPositiveNumber(
+    raw.timeout_seconds ?? raw.timeoutSeconds,
+    `${location}.timeout_seconds`,
+  );
+  const cwd = readOptionalString(raw.cwd, `${location}.cwd`);
+  const env = readOptionalStringRecord(raw.env, `${location}.env`);
+
+  return {
+    ...(command !== undefined && { command }),
+    ...(script !== undefined && { script }),
+    ...(timeoutSeconds !== undefined && { timeoutSeconds }),
+    ...(cwd !== undefined && { cwd }),
+    ...(env !== undefined && { env }),
+  };
+}
+
+function readOptionalCommand(raw: unknown, location: string): readonly string[] | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw === 'string') {
+    const command = raw.trim();
+    if (!command) {
+      throw new Error(`Experiment ${location} must not be empty.`);
+    }
+    return ['sh', '-c', command];
+  }
+  if (Array.isArray(raw) && raw.every((entry) => typeof entry === 'string' && entry.trim())) {
+    return raw.map((entry) => entry.trim());
+  }
+  throw new Error(`Experiment ${location} must be a string or string array.`);
+}
+
+function readOptionalStringOrStringArray(
+  raw: unknown,
+  location: string,
+): string | readonly string[] | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw === 'string') {
+    const trimmed = raw.trim();
+    if (!trimmed) {
+      throw new Error(`Experiment ${location} must not be empty.`);
+    }
+    return trimmed;
+  }
+  if (Array.isArray(raw) && raw.every((entry) => typeof entry === 'string' && entry.trim())) {
+    return raw.map((entry) => entry.trim());
+  }
+  throw new Error(`Experiment ${location} must be a string or string array.`);
+}
+
+function readOptionalString(raw: unknown, location: string): string | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'string' || raw.trim().length === 0) {
+    throw new Error(`Experiment ${location} must be a non-empty string.`);
+  }
+  return raw.trim();
+}
+
+function readRequiredString(raw: unknown, location: string): string {
+  const value = readOptionalString(raw, location);
+  if (value === undefined) {
+    throw new Error(`Experiment ${location} is required.`);
+  }
+  return value;
+}
+
+function readOptionalBoolean(raw: unknown, location: string): boolean | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'boolean') {
+    throw new Error(`Experiment ${location} must be a boolean.`);
+  }
+  return raw;
+}
+
+function readOptionalPositiveInteger(raw: unknown, location: string): number | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'number' || !Number.isInteger(raw) || raw < 1) {
+    throw new Error(`Experiment ${location} must be a positive integer.`);
+  }
+  return raw;
+}
+
+function readOptionalPositiveNumber(raw: unknown, location: string): number | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'number' || !Number.isFinite(raw) || raw <= 0) {
+    throw new Error(`Experiment ${location} must be a positive number.`);
+  }
+  return raw;
+}
+
+function readOptionalSandbox(raw: unknown): ExperimentSandbox | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'string' || !VALID_SANDBOXES.has(raw)) {
+    throw new Error("Experiment sandbox must be one of 'auto', 'docker', or 'vercel'.");
+  }
+  return raw as ExperimentSandbox;
+}
+
+function readOptionalRecord(raw: unknown): Record<string, unknown> | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (!isRecord(raw)) {
+    throw new Error('Experiment object field must be an object.');
+  }
+  return raw;
+}
+
+function readOptionalStringRecord(
+  raw: unknown,
+  location: string,
+): Record<string, string> | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (!isRecord(raw)) {
+    throw new Error(`Experiment ${location} must be an object.`);
+  }
+  const entries = Object.entries(raw);
+  if (!entries.every((entry): entry is [string, string] => typeof entry[1] === 'string')) {
+    throw new Error(`Experiment ${location} values must be strings.`);
+  }
+  return Object.fromEntries(entries);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function toStableJsonValue(value: unknown): unknown {
+  if (typeof value === 'function') {
+    return '[function]';
+  }
+  if (value === null || typeof value !== 'object') {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map(toStableJsonValue);
+  }
+  const record = value as Record<string, unknown>;
+  return Object.fromEntries(
+    Object.keys(record)
+      .sort()
+      .map((key) => [key, toStableJsonValue(record[key])]),
+  );
+}
diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts
index 59f8d1b15..8210dd6e9 100644
--- a/packages/core/src/evaluation/loaders/config-loader.ts
+++ b/packages/core/src/evaluation/loaders/config-loader.ts
@@ -66,9 +66,17 @@ export type HooksConfig = {
   readonly before_session?: string;
 };
 
+export type ExperimentsConfig = {
+  /** Default experiment label or path used when `agentv eval` omits --experiment. */
+  readonly default?: string;
+};
+
 export type AgentVConfig = {
   readonly required_version?: string;
   readonly eval_patterns?: readonly string[];
+  /** Compatibility shorthand for experiments.default. */
+  readonly default_experiment?: string;
+  readonly experiments?: ExperimentsConfig;
   readonly execution?: ExecutionDefaults;
   readonly results?: ResultsConfig;
   readonly hooks?: HooksConfig;
@@ -140,12 +148,22 @@ async function readConfigFile(configPath: string): Promise<AgentVConfig | null>
       (parsed as Record<string, unknown>).execution,
       configPath,
     );
+    const defaultExperiment = parseDefaultExperiment(
+      (parsed as Record<string, unknown>).default_experiment,
+      configPath,
+    );
+    const experiments = parseExperimentsConfig(
+      (parsed as Record<string, unknown>).experiments,
+      configPath,
+    );
     const results = parseResultsConfig((parsed as Record<string, unknown>).results, configPath);
     const hooks = parseHooksConfig((parsed as Record<string, unknown>).hooks, configPath);
 
     return {
       required_version: requiredVersion as string | undefined,
       eval_patterns: evalPatterns as readonly string[] | undefined,
+      ...(defaultExperiment && { default_experiment: defaultExperiment }),
+      ...(experiments && { experiments }),
       execution: executionDefaults,
       results,
       ...(hooks && { hooks }),
@@ -593,6 +611,46 @@ export function parseExecutionDefaults(
   return Object.keys(result).length > 0 ? (result as ExecutionDefaults) : undefined;
 }
 
+export function resolveDefaultExperimentReference(
+  config: AgentVConfig | null | undefined,
+): string | undefined {
+  return config?.experiments?.default ?? config?.default_experiment;
+}
+
+function parseDefaultExperiment(raw: unknown, configPath: string): string | undefined {
+  if (raw === undefined || raw === null) {
+    return undefined;
+  }
+  const value = readTrimmedString(raw);
+  if (!value) {
+    logWarning(`Invalid default_experiment in ${configPath}, expected non-empty string`);
+    return undefined;
+  }
+  return value;
+}
+
+export function parseExperimentsConfig(
+  raw: unknown,
+  configPath: string,
+): ExperimentsConfig | undefined {
+  if (raw === undefined || raw === null) {
+    return undefined;
+  }
+  if (typeof raw !== 'object' || Array.isArray(raw)) {
+    logWarning(`Invalid experiments in ${configPath}, expected object`);
+    return undefined;
+  }
+
+  const obj = raw as Record<string, unknown>;
+  const defaultExperiment = readTrimmedString(obj.default);
+  if (obj.default !== undefined && !defaultExperiment) {
+    logWarning(`Invalid experiments.default in ${configPath}, expected non-empty string`);
+    return undefined;
+  }
+
+  return defaultExperiment ? { default: defaultExperiment } : undefined;
+}
+
 function isFilesystemPath(p: string): boolean {
   return (
     p.startsWith('/') ||
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index b62a061d2..ad24e28a2 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -15,6 +15,7 @@ export {
 } from './evaluation/result-row-schema.js';
 export * from './evaluation/result-artifact-contract.js';
 export { parseYamlValue } from './evaluation/yaml-loader.js';
+export * from './evaluation/experiment.js';
 export * from './evaluation/yaml-parser.js';
 export {
   isAgentSkillsFormat,
@@ -22,8 +23,10 @@ export {
 } from './evaluation/loaders/agent-skills-parser.js';
 export {
   loadConfig,
+  resolveDefaultExperimentReference,
   resolveResultsConfigForProject,
   type AgentVConfig as AgentVYamlConfig,
+  type ExperimentsConfig,
   type ResultsConfig,
 } from './evaluation/loaders/config-loader.js';
 export {
diff --git a/packages/core/test/evaluation/config.test.ts b/packages/core/test/evaluation/config.test.ts
index 818debae1..af0b7a8f0 100644
--- a/packages/core/test/evaluation/config.test.ts
+++ b/packages/core/test/evaluation/config.test.ts
@@ -41,6 +41,15 @@ describe('defineConfig execution defaults', () => {
     });
   });
 
+  it('accepts typed experiment defaults', () => {
+    const config = defineConfig({
+      defaultExperiment: 'smoke',
+      experiments: { default: 'experiments/default.yaml' },
+    });
+    expect(config.defaultExperiment).toBe('smoke');
+    expect(config.experiments?.default).toBe('experiments/default.yaml');
+  });
+
   it('rejects non-boolean verbose', () => {
     expect(() => defineConfig({ execution: { verbose: 'yes' } } as never)).toThrow();
   });
diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts
new file mode 100644
index 000000000..e9c19f4c4
--- /dev/null
+++ b/packages/core/test/evaluation/experiment.test.ts
@@ -0,0 +1,94 @@
+import { describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {
+  deriveExperimentNameFromPath,
+  isExperimentFileReference,
+  loadExperimentConfig,
+  normalizeExperimentConfig,
+} from '../../src/evaluation/experiment.js';
+
+describe('experiment config', () => {
+  it('normalizes snake_case wire fields to camelCase runtime fields', () => {
+    const config = normalizeExperimentConfig({
+      name: 'baseline',
+      target: 'codex-gpt5',
+      agent: 'codex',
+      model: 'openai/gpt-5.5',
+      agent_options: { reasoning_effort: 'high' },
+      evals: 'evals/**/*.eval.yaml',
+      scripts: ['build', { script: 'bun test', timeout_seconds: 120 }],
+      runs: 3,
+      early_exit: false,
+      timeout_seconds: 900,
+      sandbox: 'auto',
+      setup: [{ script: 'bun install' }],
+    });
+
+    expect(config).toMatchObject({
+      name: 'baseline',
+      target: 'codex-gpt5',
+      agent: 'codex',
+      model: 'openai/gpt-5.5',
+      agentOptions: { reasoning_effort: 'high' },
+      evals: 'evals/**/*.eval.yaml',
+      scripts: [{ script: 'build' }, { script: 'bun test', timeoutSeconds: 120 }],
+      runs: 3,
+      earlyExit: false,
+      timeoutSeconds: 900,
+      sandbox: 'auto',
+      setup: [{ script: 'bun install' }],
+    });
+    expect(config.fingerprint).toMatch(/^[a-f0-9]{64}$/);
+  });
+
+  it('loads a YAML experiment file', async () => {
+    const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-experiment-'));
+    try {
+      const experimentPath = path.join(tempDir, 'default.yaml');
+      writeFileSync(
+        experimentPath,
+        [
+          'name: with-skill',
+          'target: copilot',
+          'agent_options:',
+          '  cli_package: "@github/copilot@latest"',
+          'runs: 2',
+          'setup:',
+          '  - script: cp skills/AGENTS.md AGENTS.md',
+          '',
+        ].join('\n'),
+      );
+
+      const config = await loadExperimentConfig(experimentPath);
+
+      expect(config.name).toBe('with-skill');
+      expect(config.target).toBe('copilot');
+      expect(config.agentOptions).toEqual({ cli_package: '@github/copilot@latest' });
+      expect(config.runs).toBe(2);
+      expect(config.sourcePath).toBe(experimentPath);
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it('rejects invalid run counts and sandbox values', () => {
+    expect(() => normalizeExperimentConfig({ runs: 0 })).toThrow(/runs/);
+    expect(() => normalizeExperimentConfig({ sandbox: 'host' })).toThrow(/sandbox/);
+  });
+
+  it('detects experiment file references separately from labels', () => {
+    expect(isExperimentFileReference('experiments/default.yaml')).toBe(true);
+    expect(isExperimentFileReference('default.yaml')).toBe(true);
+    expect(isExperimentFileReference('baseline')).toBe(false);
+  });
+
+  it('derives experiment names from file paths', () => {
+    expect(deriveExperimentNameFromPath('/repo/experiments/baseline.experiment.ts')).toBe(
+      'baseline',
+    );
+    expect(deriveExperimentNameFromPath('/repo/experiments/with-skill.yaml')).toBe('with-skill');
+  });
+});
diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts
index dffc15652..c345486e3 100644
--- a/packages/core/test/evaluation/loaders/config-loader.test.ts
+++ b/packages/core/test/evaluation/loaders/config-loader.test.ts
@@ -14,7 +14,9 @@ import {
   extractTrialsConfig,
   loadConfig,
   parseExecutionDefaults,
+  parseExperimentsConfig,
   parseResultsConfig,
+  resolveDefaultExperimentReference,
   resolveResultsConfigForProject,
 } from '../../../src/evaluation/loaders/config-loader.js';
 import type { JsonObject } from '../../../src/evaluation/types.js';
@@ -124,6 +126,55 @@ describe('loadConfig', () => {
       rmSync(tempDir, { recursive: true, force: true });
     }
   });
+
+  it('loads configured default experiment references', async () => {
+    const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-default-experiment-'));
+    try {
+      const projectDir = path.join(tempDir, 'project');
+      const evalDir = path.join(projectDir, 'evals');
+      const localConfigDir = path.join(projectDir, '.agentv');
+      mkdirSync(evalDir, { recursive: true });
+      mkdirSync(localConfigDir, { recursive: true });
+      writeFileSync(
+        path.join(localConfigDir, 'config.yaml'),
+        'experiments:\n  default: experiments/default.yaml\n',
+      );
+
+      const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir);
+
+      expect(config?.experiments?.default).toBe('experiments/default.yaml');
+      expect(resolveDefaultExperimentReference(config)).toBe('experiments/default.yaml');
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it('supports top-level default_experiment as a compatibility shorthand', async () => {
+    const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-default-experiment-alias-'));
+    try {
+      const projectDir = path.join(tempDir, 'project');
+      const evalDir = path.join(projectDir, 'evals');
+      const localConfigDir = path.join(projectDir, '.agentv');
+      mkdirSync(evalDir, { recursive: true });
+      mkdirSync(localConfigDir, { recursive: true });
+      writeFileSync(path.join(localConfigDir, 'config.yaml'), 'default_experiment: smoke\n');
+
+      const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir);
+
+      expect(config?.default_experiment).toBe('smoke');
+      expect(resolveDefaultExperimentReference(config)).toBe('smoke');
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe('parseExperimentsConfig', () => {
+  it('parses experiments.default', () => {
+    expect(parseExperimentsConfig({ default: 'experiments/default.yaml' }, 'config.yaml')).toEqual({
+      default: 'experiments/default.yaml',
+    });
+  });
 });
 
 describe('extractTrialsConfig', () => {

From 4a7856a25092863f546081f94756d38270b8e885 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 14:35:48 +0200
Subject: [PATCH 03/18] docs: reconcile repeat-run plan to experiment-level
 placement

---
 ...3-001-feat-repeat-runs-flaky-evals-plan.md | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index dd6759281..adcb418a6 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -148,7 +148,7 @@ Public docs and implementation notes must not reference non-public sources. If a
 
 ## Key Technical Decisions
 
-- KTD1. Prefer `execution.repeat` as the durable public config surface. The existing `execution.trials` code path mixes attempt aggregation and gate behavior; implementation should either hard-correct it before stable release or keep it as a compatibility alias with explicit mapping and warnings.
+- KTD1. The repeat config attaches to the **experiment** surface, not to `eval.yaml` `execution`, per the experiments-separation decision (epic `av-991`, recorded on `av-991.1`). This aligns with Vercel agent-eval, where `runs`/`earlyExit` are experiment-level. This epic (`av-i0l`) owns the repeat **mechanics** (schema shape, gate policies, attempt aggregation, flake classification, and the run-N artifact layout); `av-991` owns **placement** (the experiment contract the repeat block lives on). The existing `execution.trials` code path is **hard-removed** (no compatibility alias) because usage is rare; its behavior is replaced by the experiment-level repeat block. Because the experiment surface is delivered by `av-991`, the schema work in `av-i0l.1` depends on that contract landing.
 - KTD2. Keep one-run CI as the default. Repeat runs are for reliability evidence unless `repeat.gate` says they are a CI gate.
 - KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json` and `run-N/` directories so existing aggregate consumers do not inflate case counts.
 - KTD4. Single-run cases keep direct case-local files instead of always nesting under `run-1`. This preserves the simple default artifact shape and makes `.agentv/results/<experiment>/<timestamp>/<case-id>/grading.json` easy to inspect. Repeat-enabled cases use `run-1/`, `run-2/`, and so on under the case directory.
@@ -165,7 +165,7 @@ Public docs and implementation notes must not reference non-public sources. If a
 
 ```mermaid
 flowchart TB
-  Config[Eval config execution.repeat] --> Runner[Eval runner]
+  Config[Experiment repeat config] --> Runner[Eval runner]
   Runner --> Attempt1[Attempt run-1]
   Runner --> Attempt2[Attempt run-2]
   Runner --> AttemptN[Attempt run-N]
@@ -214,15 +214,15 @@ Field notes:
 - `seed` is best effort. Providers that support deterministic seeds receive a per-attempt seed derived from the base seed and attempt number; providers that do not support seeds record `seed_unsupported`.
 - `budget_usd` composes with existing run-level budget controls and stops new attempts when the budget is exhausted.
 
-Compatibility with `execution.trials`:
+Migration from `execution.trials` (hard removal, no alias):
 
-- If `trials` is still prerelease-only, replace it with `repeat` and migrate tests/docs.
-- If `trials` has external consumers, keep it as an alias:
-  - `trials.count` maps to `repeat.runs`.
-  - `trials.cost_limit_usd` maps to `repeat.budget_usd`.
-  - `trials.strategy: pass_at_k` maps to `repeat.gate.policy: any_attempt_successful`.
-  - `trials.strategy: mean` maps to `mean_score_at_least` only when a threshold is present; otherwise it is report-only aggregation.
-  - `trials.strategy: confidence_interval` remains report-only until AgentV has a reviewed CI policy for confidence bounds.
+`execution.trials` is removed from `eval.yaml` outright; the repeat block lives on the experiment instead. Existing semantics map as follows so any prerelease evals can be ported by hand:
+
+- `trials.count` maps to the experiment repeat `runs`.
+- `trials.cost_limit_usd` maps to repeat `budget_usd`.
+- `trials.strategy: pass_at_k` maps to repeat `gate.policy: any_attempt_successful` (with `early_exit: on_gate_satisfied`).
+- `trials.strategy: mean` maps to `mean_score_at_least` only when a threshold is present; otherwise it is report-only aggregation.
+- `trials.strategy: confidence_interval` remains report-only until AgentV has a reviewed CI policy for confidence bounds.
 
 ---
 
@@ -455,7 +455,7 @@ Repeat runs can multiply provider spend. V1 should ship with conservative contro
 
 **Files:** `packages/core/src/evaluation/types.ts`, `packages/core/src/evaluation/loaders/config-loader.ts`, `packages/core/src/evaluation/validation/eval-file.schema.ts`, `packages/core/scripts/generate-eval-schema.ts`, `packages/core/test/evaluation/loaders/config-loader.test.ts`, `packages/core/test/evaluation/validation/eval-file-schema.test.ts`.
 
-**Approach:** Add `execution.repeat` with a narrow schema and normalize it into internal camelCase. Decide whether `execution.trials` is removed as prerelease cleanup or retained as a legacy alias with explicit mapping.
+**Approach:** Add the repeat block as a narrow schema on the **experiment** surface (delivered by `av-991`) and normalize it into internal camelCase. **Hard-remove** `execution.trials` from `eval.yaml` as prerelease cleanup (no legacy alias); port any existing usage by hand using the mapping above.
 
 **Test Scenarios:**
 

From bbdcead3250828ff8746da5f8ac1fc4ff3b4fced Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 14:37:08 +0200
Subject: [PATCH 04/18] feat: add native experiment execution support

---
 apps/cli/src/commands/eval/artifact-writer.ts |   3 +
 apps/cli/src/commands/eval/commands/run.ts    |  11 +-
 apps/cli/src/commands/eval/run-eval.ts        | 371 ++++++++++++++++--
 apps/cli/test/eval.integration.test.ts        |  77 ++++
 apps/cli/test/fixtures/mock-run-evaluation.ts |   8 +
 .../docs/docs/evaluation/eval-files.mdx       |   2 +-
 .../content/docs/docs/evaluation/examples.mdx |   2 +-
 ...26-06-23-experiments-vs-eval-separation.md |  16 +-
 ...6-06-23-002-experiments-separation-plan.md |  42 +-
 .../features/trials/evals/dataset.eval.yaml   |   9 +-
 .../features/trials/experiments/default.yaml  |   8 +
 .../showcase/multi-model-benchmark/README.md  |  37 +-
 .../evals/benchmark.eval.yaml                 |  14 +-
 .../experiments/default.yaml                  |  14 +
 packages/core/src/evaluation/experiment.ts    |  67 +++-
 .../src/evaluation/loaders/config-loader.ts   |  70 ----
 packages/core/src/evaluation/orchestrator.ts  |   6 +-
 packages/core/src/evaluation/run-artifacts.ts |  16 +-
 packages/core/src/evaluation/types.ts         |   2 +
 .../evaluation/validation/eval-file.schema.ts |  10 +-
 packages/core/src/evaluation/yaml-parser.ts   |   7 -
 .../core/test/evaluation/experiment.test.ts   |  35 ++
 .../evaluation/loaders/config-loader.test.ts  | 123 ------
 .../core/test/evaluation/orchestrator.test.ts |   3 +-
 .../validation/eval-file-schema.test.ts       |  14 +
 .../references/eval-schema.json               |  66 +---
 26 files changed, 657 insertions(+), 376 deletions(-)
 create mode 100644 examples/features/trials/experiments/default.yaml
 create mode 100644 examples/showcase/multi-model-benchmark/experiments/default.yaml

diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts
index 1d6ce52e6..09e2d098e 100644
--- a/apps/cli/src/commands/eval/artifact-writer.ts
+++ b/apps/cli/src/commands/eval/artifact-writer.ts
@@ -6,6 +6,7 @@ import {
   type BenchmarkArtifact,
   type EvalTest,
   type EvaluationResult,
+  type ExperimentArtifactMetadata,
   type ExportDuplicatePolicy,
   type GradingArtifact,
   type IndexArtifactEntry,
@@ -228,6 +229,7 @@ export async function writeArtifactsFromResults(
   options?: {
     evalFile?: string;
     experiment?: string;
+    experimentMetadata?: ExperimentArtifactMetadata;
     plannedTestCount?: number;
     runId?: string;
     duplicatePolicy?: ExportDuplicatePolicy;
@@ -245,6 +247,7 @@ export async function writeArtifactsFromResults(
   return writeCoreArtifactsFromResults(results, outputDir, {
     evalFile: options?.evalFile,
     experiment: options?.experiment,
+    experimentMetadata: options?.experimentMetadata,
     plannedTestCount: options?.plannedTestCount,
     runId: options?.runId,
     duplicatePolicy: options?.duplicatePolicy,
diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts
index 5b74c3884..a498f4ccc 100644
--- a/apps/cli/src/commands/eval/commands/run.ts
+++ b/apps/cli/src/commands/eval/commands/run.ts
@@ -11,7 +11,6 @@ import {
 } from 'cmd-ts';
 
 import { runEvalCommand } from '../run-eval.js';
-import { resolveEvalPaths } from '../shared.js';
 
 export const evalRunCommand = command({
   name: 'eval',
@@ -264,14 +263,6 @@ export const evalRunCommand = command({
     }),
   },
   handler: async (args) => {
-    // Launch interactive wizard when no eval paths and stdin is a TTY
-    if (args.evalPaths.length === 0 && process.stdin.isTTY) {
-      const { launchInteractiveWizard } = await import('../interactive.js');
-      await launchInteractiveWizard();
-      return;
-    }
-
-    const resolvedPaths = await resolveEvalPaths(args.evalPaths, process.cwd());
     if (args.budgetUsd !== undefined && args.budgetUsd <= 0) {
       console.error('Error: --budget-usd must be a positive number.');
       process.exit(2);
@@ -330,7 +321,7 @@ export const evalRunCommand = command({
       recordReplay: args.recordReplay,
       recordReplayVariant: args.recordReplayVariant,
     };
-    const result = await runEvalCommand({ testFiles: resolvedPaths, rawOptions });
+    const result = await runEvalCommand({ testFiles: args.evalPaths, rawOptions });
     if (result?.allExecutionErrors) {
       process.exit(2);
     }
diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts
index 1c8a498e8..7cfbb751d 100644
--- a/apps/cli/src/commands/eval/run-eval.ts
+++ b/apps/cli/src/commands/eval/run-eval.ts
@@ -1,20 +1,27 @@
+import { spawn } from 'node:child_process';
 import { constants, existsSync, mkdirSync } from 'node:fs';
 import { access, readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
 import {
+  DEFAULT_EVAL_PATTERNS,
   DEFAULT_THRESHOLD,
+  type EvalTargetRef,
   type EvalTest,
   type EvaluationCache,
   type EvaluationResult,
   type ExecutionDefaults,
+  type ExperimentArtifactMetadata,
+  type ExperimentConfig,
+  type ExperimentScript,
   type FailOnError,
   type OtelTraceExporter as OtelTraceExporterType,
   type ResolvedTarget,
   ResponseCache,
   RunBudgetTracker,
   type TrialsConfig,
+  buildExperimentArtifactMetadata,
   buildTraceFromMessages,
   runEvaluation as defaultRunEvaluation,
   deriveCategory,
@@ -71,7 +78,7 @@ import {
   loadNonErrorResults,
 } from './retry-errors.js';
 import { resolveCachedRunDir, saveRunCache } from './run-cache.js';
-import { findRepoRoot } from './shared.js';
+import { findRepoRoot, resolveEvalPaths } from './shared.js';
 import {
   calculateEvaluationSummary,
   formatEvaluationSummary,
@@ -145,6 +152,10 @@ interface NormalizedOptions {
   readonly recordReplay?: string;
   readonly recordReplayVariant?: string;
   readonly experiment?: string;
+  readonly experimentConfig?: ExperimentConfig;
+  readonly experimentMetadata?: ExperimentArtifactMetadata;
+  readonly experimentTargetRefs?: readonly EvalTargetRef[];
+  readonly experimentTrialsConfig?: TrialsConfig;
   readonly budgetUsd?: number;
   readonly sourceMetadataByEvalFile?: ReadonlyMap<string, Record<string, unknown>>;
   readonly resultsOverrides?: ResultsPublishOverrides;
@@ -566,30 +577,298 @@ function normalizeTsDefaultExperiment(
   );
 }
 
-async function resolveExperimentNameForRun(params: {
+type ResolvedExperimentForRun = {
+  readonly name?: string;
+  readonly config?: ExperimentConfig;
+};
+
+async function resolveExperimentForRun(params: {
   readonly cwd: string;
   readonly explicitExperiment?: string;
   readonly yamlDefaultExperiment?: string;
   readonly tsDefaultExperiment?: string;
-}): Promise<string | undefined> {
+}): Promise<ResolvedExperimentForRun> {
   const experimentRef =
     params.explicitExperiment ?? params.yamlDefaultExperiment ?? params.tsDefaultExperiment;
   if (!experimentRef) {
+    return {};
+  }
+
+  const experimentPath = resolveExperimentFilePath(params.cwd, experimentRef);
+  if (!experimentPath) {
+    if (isExperimentFileReference(experimentRef)) {
+      throw new Error(`Experiment file not found: ${experimentRef}`);
+    }
+    return { name: experimentRef };
+  }
+
+  const config = await loadExperimentConfig(experimentPath);
+  return {
+    name: config.name ?? deriveExperimentNameFromPath(experimentPath),
+    config,
+  };
+}
+
+function resolveExperimentFilePath(cwd: string, experimentRef: string): string | undefined {
+  if (isExperimentFileReference(experimentRef)) {
+    const experimentPath = path.isAbsolute(experimentRef)
+      ? experimentRef
+      : path.resolve(cwd, experimentRef);
+    return existsSync(experimentPath) ? experimentPath : undefined;
+  }
+
+  for (const ext of ['yaml', 'yml', 'ts', 'js', 'mts', 'mjs']) {
+    const candidate = path.resolve(cwd, 'experiments', `${experimentRef}.${ext}`);
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+  }
+  return undefined;
+}
+
+function applyExperimentOptions(
+  options: NormalizedOptions,
+  experiment: ExperimentConfig | undefined,
+): NormalizedOptions {
+  if (!experiment) {
+    return options;
+  }
+
+  const experimentTargetRefs = buildExperimentTargetRefs(experiment);
+  const experimentTargetNames = experimentTargetRefs?.map((target) => target.name) ?? [];
+  const experimentTarget =
+    experiment.target && experiment.target.trim().length > 0 ? experiment.target : undefined;
+  const nextCliTargets =
+    options.cliTargets.length > 0
+      ? options.cliTargets
+      : experimentTargetNames.length > 0
+        ? experimentTargetNames
+        : experimentTarget
+          ? [experimentTarget]
+          : options.cliTargets;
+
+  const workspaceMode =
+    options.workspaceMode ?? readExperimentWorkspaceMode(experiment.workspace?.mode);
+  const workspacePath = options.workspacePath ?? readExperimentWorkspacePath(experiment.workspace);
+  const experimentFilter = normalizeExperimentCaseFilter(experiment.evals);
+
+  return {
+    ...options,
+    target: options.target ?? (nextCliTargets.length === 1 ? nextCliTargets[0] : undefined),
+    cliTargets: nextCliTargets,
+    agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds,
+    workers: options.workers ?? experiment.workers,
+    workspaceMode: workspacePath ? 'static' : workspaceMode,
+    workspacePath,
+    filter: options.filter ?? experimentFilter,
+    budgetUsd: options.budgetUsd ?? experiment.budgetUsd,
+    experimentConfig: experiment,
+    experimentMetadata: buildExperimentArtifactMetadata(experiment),
+    experimentTargetRefs: options.cliTargets.length === 0 ? experimentTargetRefs : undefined,
+    experimentTrialsConfig: buildExperimentTrialsConfig(experiment),
+  };
+}
+
+function buildExperimentTargetRefs(
+  experiment: ExperimentConfig,
+): readonly EvalTargetRef[] | undefined {
+  if (!experiment.targets || experiment.targets.length === 0) {
     return undefined;
   }
-  if (!isExperimentFileReference(experimentRef)) {
-    return experimentRef;
+  return experiment.targets.map((target) => {
+    if (typeof target === 'string') {
+      return { name: target };
+    }
+    return {
+      name: target.name,
+      ...(target.useTarget !== undefined && { use_target: target.useTarget }),
+      ...(target.hooks !== undefined && {
+        hooks: target.hooks as EvalTargetRef['hooks'],
+      }),
+    };
+  });
+}
+
+function buildExperimentTrialsConfig(experiment: ExperimentConfig): TrialsConfig | undefined {
+  if (!experiment.runs || experiment.runs <= 1) {
+    return undefined;
   }
+  return {
+    count: experiment.runs,
+    strategy: 'pass_at_k',
+    earlyExit: experiment.earlyExit ?? false,
+  };
+}
+
+function readExperimentWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined {
+  return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined;
+}
 
-  const experimentPath = path.isAbsolute(experimentRef)
-    ? experimentRef
-    : path.resolve(params.cwd, experimentRef);
-  if (!existsSync(experimentPath)) {
-    throw new Error(`Experiment file not found: ${experimentRef}`);
+function readExperimentWorkspacePath(
+  workspace: Record<string, unknown> | undefined,
+): string | undefined {
+  const value = workspace?.path;
+  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
+}
+
+function normalizeExperimentCaseFilter(
+  evals: ExperimentConfig['evals'] | undefined,
+): NormalizedOptions['filter'] {
+  if (typeof evals === 'string') {
+    return evals;
+  }
+  if (!Array.isArray(evals)) {
+    return undefined;
   }
+  return evals.length === 1 ? evals[0] : [...evals];
+}
 
-  const experiment = await loadExperimentConfig(experimentPath);
-  return experiment.name ?? deriveExperimentNameFromPath(experimentPath);
+async function runExperimentSteps(params: {
+  readonly label: 'setup' | 'script';
+  readonly steps: readonly ExperimentScript[] | undefined;
+  readonly cwd: string;
+  readonly experimentConfig?: ExperimentConfig;
+}): Promise<void> {
+  const steps = params.steps ?? [];
+  if (steps.length === 0) {
+    return;
+  }
+
+  for (let index = 0; index < steps.length; index++) {
+    const step = steps[index];
+    const command = buildExperimentStepCommand(step);
+    const cwd = resolveExperimentStepCwd(params.cwd, params.experimentConfig, step.cwd);
+    console.log(`Experiment ${params.label} ${index + 1}/${steps.length}: ${command.display}`);
+    await runExperimentCommand(command.argv, {
+      cwd,
+      env: step.env,
+      timeoutMs: step.timeoutSeconds ? step.timeoutSeconds * 1000 : undefined,
+      label: `experiment ${params.label}`,
+    });
+  }
+}
+
+async function runExperimentSetup(params: {
+  readonly config: ExperimentConfig | undefined;
+  readonly cwd: string;
+  readonly runDir: string;
+}): Promise<void> {
+  const setup = params.config?.setup;
+  if (typeof setup === 'function') {
+    console.log('Experiment setup: running TypeScript setup()');
+    await setup({
+      cwd: params.cwd,
+      runDir: params.runDir,
+      experiment: params.config,
+      env: process.env,
+    });
+    return;
+  }
+  await runExperimentSteps({
+    label: 'setup',
+    steps: setup,
+    cwd: params.cwd,
+    experimentConfig: params.config,
+  });
+}
+
+function buildExperimentStepCommand(step: ExperimentScript): {
+  readonly argv: readonly string[];
+  readonly display: string;
+} {
+  if (step.command && step.command.length > 0) {
+    return { argv: step.command, display: step.command.join(' ') };
+  }
+  if (typeof step.script === 'string' && step.script.trim().length > 0) {
+    return {
+      argv: shellCommand(step.script),
+      display: step.script,
+    };
+  }
+  if (Array.isArray(step.script) && step.script.length > 0) {
+    return { argv: step.script, display: step.script.join(' ') };
+  }
+  throw new Error('Experiment step must define command or script.');
+}
+
+function shellCommand(script: string): readonly string[] {
+  return process.platform === 'win32' ? ['cmd', '/c', script] : ['sh', '-c', script];
+}
+
+function resolveExperimentStepCwd(
+  cwd: string,
+  experimentConfig: ExperimentConfig | undefined,
+  stepCwd: string | undefined,
+): string {
+  const base = experimentConfig?.sourcePath ? path.dirname(experimentConfig.sourcePath) : cwd;
+  if (!stepCwd) {
+    return base;
+  }
+  return path.isAbsolute(stepCwd) ? stepCwd : path.resolve(base, stepCwd);
+}
+
+async function runExperimentCommand(
+  argv: readonly string[],
+  options: {
+    readonly cwd: string;
+    readonly env?: Record<string, string>;
+    readonly timeoutMs?: number;
+    readonly label: string;
+  },
+): Promise<void> {
+  if (argv.length === 0) {
+    throw new Error(`${options.label} command must not be empty.`);
+  }
+
+  await new Promise<void>((resolve, reject) => {
+    const cmd = argv[0];
+    if (!cmd) {
+      reject(new Error(`${options.label} command must not be empty.`));
+      return;
+    }
+    const args = argv.slice(1);
+    const child = spawn(cmd, args, {
+      cwd: options.cwd,
+      env: options.env ? { ...process.env, ...options.env } : process.env,
+      stdio: 'inherit',
+    });
+    let completed = false;
+    const timeout =
+      options.timeoutMs !== undefined
+        ? setTimeout(() => {
+            if (!completed) {
+              completed = true;
+              child.kill('SIGKILL');
+              reject(new Error(`${options.label} timed out after ${options.timeoutMs}ms`));
+            }
+          }, options.timeoutMs)
+        : undefined;
+
+    child.on('error', (error) => {
+      if (completed) {
+        return;
+      }
+      completed = true;
+      if (timeout !== undefined) {
+        clearTimeout(timeout);
+      }
+      reject(error);
+    });
+    child.on('exit', (code) => {
+      if (completed) {
+        return;
+      }
+      completed = true;
+      if (timeout !== undefined) {
+        clearTimeout(timeout);
+      }
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`${options.label} exited with code ${code ?? 'unknown'}`));
+      }
+    });
+  });
 }
 
 type ProgressReporter = {
@@ -814,18 +1093,23 @@ async function prepareFileMetadata(params: {
     const cliTargets = options.cliTargets;
     const suiteTargets = suite.targets;
     const suiteTargetRefs = suite.targetRefs;
+    const experimentTargetRefs = options.experimentTargetRefs;
 
-    // Resolve which target names to use (precedence: CLI > suite YAML targets > default)
+    // Resolve which target names to use (precedence: CLI/experiment > suite YAML targets > default)
     let targetNames: readonly string[];
+    let targetRefs: readonly EvalTargetRef[] | undefined;
     if (cliTargets.length > 0) {
       targetNames = cliTargets;
+      targetRefs = experimentTargetRefs;
     } else if (suiteTargets && suiteTargets.length > 0) {
       targetNames = suiteTargets;
+      targetRefs = suiteTargetRefs;
     } else {
       targetNames = [];
+      targetRefs = undefined;
     }
 
-    if (targetNames.length > 1) {
+    if (targetNames.length > 1 || (targetNames.length === 1 && targetRefs)) {
       // Matrix mode: multiple targets
       const multiSelections = await selectMultipleTargets({
         testFilePath,
@@ -838,7 +1122,7 @@ async function prepareFileMetadata(params: {
         dryRunDelayMax: options.dryRunDelayMax,
         env: process.env,
         targetNames,
-        targetRefs: suiteTargetRefs,
+        targetRefs,
       });
 
       selections = multiSelections.map((sel) => ({
@@ -861,9 +1145,7 @@ async function prepareFileMetadata(params: {
       });
 
       // Attach target hooks from eval file if available
-      const singleTargetHooks = suiteTargetRefs?.find(
-        (ref) => ref.name === selection.targetName,
-      )?.hooks;
+      const singleTargetHooks = targetRefs?.find((ref) => ref.name === selection.targetName)?.hooks;
       const augmentedSelection: TargetSelection = singleTargetHooks
         ? { ...selection, targetHooks: singleTargetHooks }
         : selection;
@@ -884,7 +1166,7 @@ async function prepareFileMetadata(params: {
     testIds,
     testCases: suite.tests,
     selections,
-    trialsConfig: suite.trials,
+    trialsConfig: options.experimentTrialsConfig,
     suiteTargets,
     yamlWorkers: suite.workers,
     yamlCache: suite.cacheConfig?.enabled,
@@ -1205,15 +1487,30 @@ export async function runEvalCommand(
   }
 
   let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  const resolvedExperiment = await resolveExperimentForRun({
+    cwd,
+    explicitExperiment: options.experiment,
+    yamlDefaultExperiment: resolveDefaultExperimentReference(yamlConfig),
+    tsDefaultExperiment: normalizeTsDefaultExperiment(config),
+  });
   options = {
-    ...options,
-    experiment: await resolveExperimentNameForRun({
-      cwd,
-      explicitExperiment: options.experiment,
-      yamlDefaultExperiment: resolveDefaultExperimentReference(yamlConfig),
-      tsDefaultExperiment: normalizeTsDefaultExperiment(config),
-    }),
+    ...applyExperimentOptions(options, resolvedExperiment.config),
+    experiment: resolvedExperiment.name,
   };
+
+  const evalPathInputs =
+    input.testFiles.length > 0
+      ? [...input.testFiles]
+      : options.experimentConfig?.evals !== undefined
+        ? [...(yamlConfig?.eval_patterns ?? DEFAULT_EVAL_PATTERNS)]
+        : [];
+  if (evalPathInputs.length === 0 && process.stdin.isTTY) {
+    const { launchInteractiveWizard } = await import('./interactive.js');
+    await launchInteractiveWizard();
+    return undefined;
+  }
+  const resolvedTestFiles = await resolveEvalPaths(evalPathInputs, cwd);
+
   if (!process.env.AGENTV_EXPERIMENT) {
     process.env.AGENTV_EXPERIMENT = normalizeExperimentName(options.experiment);
   }
@@ -1429,8 +1726,13 @@ export async function runEvalCommand(
 
   console.log(`Artifact directory: ${runDir}`);
 
+  await runExperimentSetup({
+    config: options.experimentConfig,
+    cwd,
+    runDir,
+  });
+
   // Log file export paths
-  const resolvedTestFiles = input.testFiles.map((file) => path.resolve(file));
   if (options.otelFile) {
     console.log(`OTLP JSON file: ${path.resolve(options.otelFile)}`);
   }
@@ -1675,6 +1977,7 @@ export async function runEvalCommand(
       evalFile,
       plannedTestCount: totalEvalCount,
       experiment: normalizeExperimentName(options.experiment),
+      experimentMetadata: options.experimentMetadata,
     });
   }
 
@@ -1930,7 +2233,11 @@ export async function runEvalCommand(
         });
         const { benchmarkPath: workspaceBenchmarkPath, timingPath } = await aggregateRunDir(
           runDir,
-          { evalFile, experiment: normalizeExperimentName(options.experiment) },
+          {
+            evalFile,
+            experiment: normalizeExperimentName(options.experiment),
+            experimentMetadata: options.experimentMetadata,
+          },
         );
         const indexPath = path.join(runDir, 'index.jsonl');
         console.log(`Artifact workspace updated: ${runDir}`);
@@ -1947,6 +2254,7 @@ export async function runEvalCommand(
         } = await writeArtifactsFromResults(allResults, runDir, {
           evalFile,
           experiment: normalizeExperimentName(options.experiment),
+          experimentMetadata: options.experimentMetadata,
           cwd,
           repoRoot,
           sourceTests,
@@ -2049,6 +2357,13 @@ export async function runEvalCommand(
       await wipLoop.stopAndDeleteWipBranch();
     }
 
+    await runExperimentSteps({
+      label: 'script',
+      steps: options.experimentConfig?.scripts,
+      cwd,
+      experimentConfig: options.experimentConfig,
+    });
+
     return {
       executionErrorCount: summary.executionErrorCount,
       outputPath,
diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts
index 29830509b..c95de4ea2 100644
--- a/apps/cli/test/eval.integration.test.ts
+++ b/apps/cli/test/eval.integration.test.ts
@@ -518,6 +518,83 @@ describe('agentv eval CLI', () => {
     }
   }, 30_000);
 
+  it('runs a native experiment file with eval selection and run knobs', async () => {
+    const fixture = await createFixture();
+    try {
+      const experimentsDir = path.join(fixture.suiteDir, 'experiments');
+      await mkdir(experimentsDir, { recursive: true });
+      await writeFile(
+        path.join(fixture.suiteDir, '.agentv', 'config.yaml'),
+        'eval_patterns:\n  - sample.test.yaml\n',
+        'utf8',
+      );
+      const experimentPath = path.join(experimentsDir, 'default.yaml');
+      await writeFile(
+        experimentPath,
+        [
+          'name: native-exp',
+          'target: cli-target',
+          'evals: case-alpha',
+          'timeout_seconds: 12',
+          'workers: 4',
+          'runs: 2',
+          'early_exit: false',
+          'setup:',
+          '  - script: "printf setup > ../experiment-setup.txt"',
+          'scripts:',
+          '  - script: "printf script > ../experiment-script.txt"',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+
+      const { stdout, exitCode } = await runCli(fixture, [
+        'eval',
+        '--experiment',
+        'experiments/default.yaml',
+      ]);
+
+      expect(exitCode).toBe(0);
+      const outputPath = extractOutputPath(stdout);
+      expect(outputPath).toContain(`${path.sep}native-exp${path.sep}`);
+
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        target: 'cli-target',
+        agentTimeoutMs: 12000,
+        maxConcurrency: 4,
+        trials: {
+          count: 2,
+          strategy: 'pass_at_k',
+          earlyExit: false,
+        },
+      });
+
+      await expectFileExists(path.join(fixture.suiteDir, 'experiment-setup.txt'));
+      await expectFileExists(path.join(fixture.suiteDir, 'experiment-script.txt'));
+
+      const benchmark = JSON.parse(
+        await readFile(path.join(path.dirname(outputPath), 'benchmark.json'), 'utf8'),
+      ) as { metadata?: Record<string, unknown> };
+      expect(benchmark.metadata?.experiment).toBe('native-exp');
+      expect(benchmark.metadata?.experiment_config).toMatchObject({
+        name: 'native-exp',
+        source_path: experimentPath,
+        target: 'cli-target',
+        evals: 'case-alpha',
+        runs: 2,
+        early_exit: false,
+        timeout_seconds: 12,
+        workers: 4,
+      });
+      expect(
+        (benchmark.metadata?.experiment_config as Record<string, unknown>).fingerprint,
+      ).toMatch(/^[a-f0-9]{64}$/);
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
   it('honors agentv.config.ts cache.path when response cache is enabled there', async () => {
     const fixture = await createFixture();
     try {
diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts
index c0957ca5d..c501cb2b0 100644
--- a/apps/cli/test/fixtures/mock-run-evaluation.ts
+++ b/apps/cli/test/fixtures/mock-run-evaluation.ts
@@ -18,6 +18,12 @@ interface RunEvaluationOptionsLike {
   readonly filter?: string | readonly string[];
   readonly evalCases?: ReadonlyArray<unknown>;
   readonly verbose?: boolean;
+  readonly maxConcurrency?: number;
+  readonly trials?: {
+    readonly count: number;
+    readonly strategy: string;
+    readonly earlyExit?: boolean;
+  };
   readonly budgetUsd?: number;
   readonly runBudgetTracker?: {
     readonly budgetCapUsd?: number;
@@ -172,6 +178,8 @@ async function maybeWriteDiagnostics(
     envRootOnly: process.env.CLI_ENV_ROOT_ONLY ?? null,
     envLocalOnly: process.env.CLI_ENV_LOCAL_ONLY ?? null,
     budgetUsd: options.budgetUsd ?? null,
+    maxConcurrency: options.maxConcurrency ?? null,
+    trials: options.trials ?? null,
     hasRunBudgetTracker: options.runBudgetTracker !== undefined,
     runBudgetCapUsd: options.runBudgetTracker?.budgetCapUsd ?? null,
     replayRecording: options.replayRecording ?? null,
diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
index 4b123162b..7be7eabc7 100644
--- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
@@ -11,7 +11,7 @@ YAML is the canonical portable model. TypeScript helpers, generated fixtures, an
 
 ## Suites
 
-An eval file is a **suite**: it binds test cases to execution context (workspace, hooks, targets, trials). Test cases can be inline or loaded from an external file via `tests: ./cases.yaml` for reuse across suites.
+An eval file is a **suite**: it binds test cases to task context, assertions, and reusable fixtures. Runtime choices such as target matrices, setup, and run counts belong in experiments. Test cases can be inline or loaded from an external file via `tests: ./cases.yaml` for reuse across suites.
 
 ## YAML Format
 
diff --git a/apps/web/src/content/docs/docs/evaluation/examples.mdx b/apps/web/src/content/docs/docs/evaluation/examples.mdx
index 11a29c8fb..7a848930e 100644
--- a/apps/web/src/content/docs/docs/evaluation/examples.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/examples.mdx
@@ -422,5 +422,5 @@ See the [suite-level-input example](https://github.com/EntityProcess/agentv/tree
 
 For complete end-to-end workflows that combine multiple features, see the showcases in [`examples/showcase/`](https://github.com/EntityProcess/agentv/tree/main/examples/showcase):
 
-- **[Multi-Model Benchmark](https://github.com/EntityProcess/agentv/tree/main/examples/showcase/multi-model-benchmark)** — targets matrix × weighted metrics × trials × compare workflow. Runs the same tests against multiple models, scores with weighted graders, measures variability, and compares results side-by-side.
+- **[Multi-Model Benchmark](https://github.com/EntityProcess/agentv/tree/main/examples/showcase/multi-model-benchmark)** — experiment target matrix × weighted metrics × repeated runs × compare workflow. Runs the same tests against multiple models, scores with weighted graders, measures variability, and compares results side-by-side.
 - **[Export Screening](https://github.com/EntityProcess/agentv/tree/main/examples/showcase/export-screening)** — classification eval with confusion matrix metrics and CI gating.
diff --git a/docs/adr/2026-06-23-experiments-vs-eval-separation.md b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
index 61898f85b..a4432c438 100644
--- a/docs/adr/2026-06-23-experiments-vs-eval-separation.md
+++ b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
@@ -17,7 +17,7 @@ That conflates two concerns:
   required by the task, and assertions or graders.
 - The experiment is the run contract: which agent or target is under test, which
   model and harness are used, how many runs to execute, what setup is injected,
-  and which evals are selected.
+  and which eval cases are selected.
 
 The public Vercel `agent-eval` ecosystem is a useful reference point. The
 `vercel/next.js` evals keep task fixtures under `evals/`, while experiments are
@@ -44,7 +44,7 @@ criteria belong here.
 
 An experiment is a committed or generated run definition. It declares which
 agent, target, provider, model, harness options, setup steps, run count, timeout,
-sandbox, and eval selector are used. Setup that changes the system under test,
+sandbox, and case selector are used. Setup that changes the system under test,
 such as installing dependencies or dropping an `AGENTS.md` or skill file, belongs
 here because it is an A/B variable.
 
@@ -70,7 +70,7 @@ as the power-user escape hatch:
 name: copilot-gpt55-withskill
 target: copilot-gpt55
 model: openai/gpt-5.5
-evals: "evals/**/*.eval.yaml"
+evals: "agent-042-*"
 scripts:
   - build
 runs: 3
@@ -95,15 +95,15 @@ uses the `default` experiment label. Existing `eval.yaml`-only repositories
 remain valid.
 
 Legacy `eval.yaml execution` fields that select targets, targets matrices,
-workers, cache, trials, budget, thresholds, and workspace runtime behavior will
-continue to parse. The migration path is additive first: experiment config
-becomes the preferred source, while legacy fields remain a compatibility shim
-until docs and examples have moved.
+workers, cache, budget, thresholds, and workspace runtime behavior will
+continue to parse as a compatibility shim until docs and examples have moved.
+The prerelease `execution.trials` surface is hard-removed with no alias: run
+counts live on the experiment as `runs`/`early_exit`.
 
 AgentV should adopt Vercel's structure and lowest-common-denominator contract
 ideas, not depend on `@vercel/agent-eval` as core infrastructure in this phase.
 The package's `ExperimentConfig` shape is a strong public reference for
-experiment vocabulary: agent, model, agent options, eval filter, scripts, runs,
+experiment vocabulary: agent, model, agent options, case filter, scripts, runs,
 early exit, timeout, sandbox, and setup. A direct dependency would force AgentV
 to absorb Vercel's fixture model, sandbox assumptions, result caching semantics,
 and TypeScript-first authoring story before those boundaries are stable for
diff --git a/docs/plans/2026-06-23-002-experiments-separation-plan.md b/docs/plans/2026-06-23-002-experiments-separation-plan.md
index 0749d1eeb..fc534b4b2 100644
--- a/docs/plans/2026-06-23-002-experiments-separation-plan.md
+++ b/docs/plans/2026-06-23-002-experiments-separation-plan.md
@@ -13,7 +13,7 @@ AgentV should separate eval task definitions from experiment run definitions.
 Eval YAML stays the canonical authoring layer for prompts, datasets, assertions,
 and task fixtures. Experiments become first-class committed files that select the
 agent or target under test, model, harness options, setup injection, run knobs,
-and eval filter.
+and case filter.
 
 This should ship in phases. Phase 1 adds the non-breaking foundation:
 experiment contract types, default experiment resolution, and artifact
@@ -83,8 +83,9 @@ runtime dependency for this change.
   while docs teach the nested form.
 - KTD4. Experiment `target` and `targets` refer to existing AgentV target names
   and target refs. Provider settings stay in `targets.yaml`.
-- KTD5. `eval.yaml execution` remains valid in Phase 1 and becomes deprecated
-  only after examples and docs have a working experiment equivalent.
+- KTD5. Legacy `eval.yaml execution` remains valid for released fields while
+  examples migrate. The prerelease `execution.trials` field is hard-removed
+  with no alias; run count placement belongs to experiments.
 - KTD6. AgentV adopts Vercel's experiment structure, not the package dependency,
   until a direct adapter has a smaller, reviewed boundary.
 - KTD7. Full experiment fingerprints should include the experiment file contents,
@@ -106,7 +107,7 @@ agent: codex
 model: openai/gpt-5.5
 agent_options:
   reasoning_effort: high
-evals: "evals/**/*.eval.yaml"
+evals: "agent-042-*"
 scripts:
   - build
   - script: bun test
@@ -168,7 +169,9 @@ Later CLI phases should add:
 - `agentv eval --experiment baseline` resolving `experiments/baseline.yaml`
   before falling back to a label.
 - `agentv eval --experiments "experiments/*.yaml"` for matrices.
-- Experiment `evals` filters when no eval paths are provided.
+- Experiment `evals` filters AgentV case IDs. When no eval paths are provided,
+  file discovery uses `.agentv/config.yaml eval_patterns` or AgentV's default
+  eval patterns.
 
 ## Migration Strategy
 
@@ -284,15 +287,17 @@ Files:
 Approach:
 
 Apply experiment fields in precedence order: CLI overrides, explicit experiment,
-legacy eval `execution`, project config defaults. Reuse existing target
-resolution and workspace setup paths. Move setup-owned behavior out of eval docs
-before adding warnings.
+legacy eval `execution` for still-supported fields, project config defaults.
+Reuse existing target resolution and workspace setup paths. Move setup-owned
+behavior out of eval docs before adding warnings. Do not retain
+`execution.trials`; experiments are the only public input path for run counts.
 
 Test Scenarios:
 
 - Experiment `target` selects an existing target from `targets.yaml`.
 - Experiment `targets` drives matrix evaluation.
-- Experiment `evals` selects eval files when no positional eval paths are given.
+- Experiment `evals` selects case IDs; eval file discovery still comes from
+  positional paths, configured `eval_patterns`, or default eval patterns.
 - Experiment setup runs before agent execution and can inject an `AGENTS.md`
   file.
 - Legacy `eval.yaml execution.target` still works when no experiment target is
@@ -338,22 +343,27 @@ Test Scenarios:
 
 - Docs examples use `snake_case` wire fields.
 - Example default experiment runs without explicit `--experiment`.
-- Existing examples continue to run through the compatibility path.
+- Existing examples continue to run through the compatibility path except
+  prerelease `execution.trials`, which is migrated to experiments in this PR.
 
 ## Phase 1 Scope
 
-Phase 1 includes U1, U2, and U3 only.
+The first PR originally targeted U1, U2, and U3. Owner review expanded this
+branch to include native U4/U5 support for experiment resolution, eval
+selection, run knobs, setup/scripts, target reuse, artifact provenance, and the
+pre-stable hard removal of `execution.trials`.
 
-Phase 1 explicitly does not apply `setup`, `scripts`, `runs`, `early_exit`,
-`timeout_seconds`, `target`, `targets`, or `evals` to execution behavior. Those
-fields are parsed and ready for later phases, but current runner semantics stay
-unchanged except for resolving the experiment name.
+The branch now applies `setup`, `scripts`, `runs`, `early_exit`,
+`timeout_seconds`, `target`, `targets`, and `evals` to execution behavior. Matrix
+execution via multiple experiment files remains a later phase.
 
 ## Non-Goals
 
 - Do not replace AgentV's engine with `@vercel/agent-eval`.
 - Do not bulk-edit all examples in the first PR.
-- Do not remove or error on legacy `eval.yaml execution` fields.
+- Do not remove or error on released legacy `eval.yaml execution` fields in this
+  branch. `execution.trials` is intentionally excluded because it is prerelease
+  and has moved to experiments.
 - Do not add a new provider schema inside experiments.
 - Do not implement experiment matrix execution in Phase 1.
 - Do not project AgentV experiments into Phoenix or another external store.
diff --git a/examples/features/trials/evals/dataset.eval.yaml b/examples/features/trials/evals/dataset.eval.yaml
index 0dc441a72..0844c2471 100644
--- a/examples/features/trials/evals/dataset.eval.yaml
+++ b/examples/features/trials/evals/dataset.eval.yaml
@@ -1,16 +1,9 @@
 # AgentV Trial Strategy Example
-# Demonstrates execution.trials for handling LLM non-determinism
+# Demonstrates experiment-level runs for handling LLM non-determinism
 
 name: trials
 description: Trial strategy example - pass@k with 2 trials
 
-execution:
-  target: llm
-  trials:
-    count: 2
-    strategy: pass_at_k
-    cost_limit_usd: 1.00
-
 tests:
   - id: math-basics
     criteria: Assistant correctly answers the math question and shows reasoning
diff --git a/examples/features/trials/experiments/default.yaml b/examples/features/trials/experiments/default.yaml
new file mode 100644
index 000000000..41ecc5147
--- /dev/null
+++ b/examples/features/trials/experiments/default.yaml
@@ -0,0 +1,8 @@
+name: trials
+target: llm
+evals:
+  - math-basics
+  - capital-knowledge
+runs: 2
+early_exit: true
+budget_usd: 1.00
diff --git a/examples/showcase/multi-model-benchmark/README.md b/examples/showcase/multi-model-benchmark/README.md
index f9c8f7a13..1e60a690c 100644
--- a/examples/showcase/multi-model-benchmark/README.md
+++ b/examples/showcase/multi-model-benchmark/README.md
@@ -118,19 +118,28 @@ assertions:
 
 Weighted average formula: `(3×accuracy + 2×completeness + 1×clarity) / 6`
 
-### 3. Trials
+### 3. Experiment runs
 
-Each test runs twice. The `pass_at_k` strategy means a test passes if **any** trial succeeds:
+Each test runs twice through the committed experiment. `early_exit: true`
+matches pass@k ergonomics: a case can stop once any run succeeds.
 
 ```yaml
-execution:
-  trials:
-    count: 2
-    strategy: pass_at_k
-    cost_limit_usd: 2.00
+targets:
+  - copilot
+  - claude
+  - gemini-llm
+evals:
+  - factual-*
+  - analytical-comparison
+  - creative-explanation
+  - structured-list
+runs: 2
+early_exit: true
+budget_usd: 2.00
 ```
 
-This surfaces non-determinism — if a model passes on trial 1 but fails on trial 2, that signals inconsistency worth investigating.
+This surfaces non-determinism — if a model passes on run 1 but fails on run 2,
+that signals inconsistency worth investigating.
 
 ### 4. Compare
 
@@ -191,20 +200,18 @@ assertions:
     weight: 4.0    # Highest priority
 ```
 
-### Adjusting trial count
+### Adjusting run count
 
-Increase `trials.count` for more variability data (at proportional cost):
+Increase experiment `runs` for more variability data (at proportional cost):
 
 ```yaml
-execution:
-  trials:
-    count: 5          # 5 trials for higher-confidence results
-    cost_limit_usd: 5.00
+runs: 5          # 5 runs for higher-confidence results
+budget_usd: 5.00
 ```
 
 ## See Also
 
 - [`examples/features/matrix-evaluation/`](../../features/matrix-evaluation/) — minimal targets matrix example
 - [`examples/features/weighted-graders/`](../../features/weighted-graders/) — per-grader weight patterns
-- [`examples/features/trials/`](../../features/trials/) — trial strategy configuration
+- [`examples/features/trials/`](../../features/trials/) — experiment run-count configuration
 - [`examples/features/compare/`](../../features/compare/) — baseline vs candidate comparison
diff --git a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
index 9d75be316..b96d70480 100644
--- a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
+++ b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
@@ -3,27 +3,17 @@
 # Demonstrates multi-model × multi-metric × variability workflow:
 #   - Targets matrix: runs every test against multiple models
 #   - Weighted graders: accuracy (3×), completeness (2×), clarity (1×)
-#   - Trials: pass@k with 2 trials to measure variability
+#   - Experiment runs: pass@k with 2 runs to measure variability
 #
 # Default targets use low-cost models for safe experimentation.
 # Override with: agentv eval ... --target <name>
 #
 # Usage:
-#   agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
+#   agentv eval --experiment examples/showcase/multi-model-benchmark/experiments/default.yaml
 
 description: Multi-model benchmark — accuracy, completeness, and clarity across models
 tags: [multi-provider]
 
-execution:
-  targets:
-    - copilot
-    - claude
-    - gemini-llm
-  trials:
-    count: 2
-    strategy: pass_at_k
-    cost_limit_usd: 2.00
-
 assertions:
   - name: accuracy
     type: llm-grader
diff --git a/examples/showcase/multi-model-benchmark/experiments/default.yaml b/examples/showcase/multi-model-benchmark/experiments/default.yaml
new file mode 100644
index 000000000..98cf3fbcb
--- /dev/null
+++ b/examples/showcase/multi-model-benchmark/experiments/default.yaml
@@ -0,0 +1,14 @@
+name: multi-model-benchmark
+targets:
+  - copilot
+  - claude
+  - gemini-llm
+evals:
+  - factual-geography
+  - factual-science
+  - analytical-comparison
+  - creative-explanation
+  - structured-list
+runs: 2
+early_exit: true
+budget_usd: 2.00
diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts
index 5ccc7bee8..fc11295ce 100644
--- a/packages/core/src/evaluation/experiment.ts
+++ b/packages/core/src/evaluation/experiment.ts
@@ -56,6 +56,8 @@ export type ExperimentConfigWire = {
   readonly runs?: number;
   readonly early_exit?: boolean;
   readonly timeout_seconds?: number;
+  readonly workers?: number;
+  readonly budget_usd?: number;
   readonly sandbox?: ExperimentSandbox;
   readonly workspace?: Record<string, unknown>;
   readonly setup?: readonly ExperimentScriptWire[] | ExperimentSetupFn;
@@ -73,6 +75,8 @@ export type ExperimentConfig = {
   readonly runs?: number;
   readonly earlyExit?: boolean;
   readonly timeoutSeconds?: number;
+  readonly workers?: number;
+  readonly budgetUsd?: number;
   readonly sandbox?: ExperimentSandbox;
   readonly workspace?: Record<string, unknown>;
   readonly setup?: ExperimentSetup;
@@ -80,6 +84,23 @@ export type ExperimentConfig = {
   readonly fingerprint?: string;
 };
 
+export type ExperimentArtifactMetadata = {
+  readonly name?: string;
+  readonly source_path?: string;
+  readonly fingerprint?: string;
+  readonly agent?: string;
+  readonly target?: string;
+  readonly targets?: readonly string[];
+  readonly model?: string;
+  readonly evals?: string | readonly string[];
+  readonly runs?: number;
+  readonly early_exit?: boolean;
+  readonly timeout_seconds?: number;
+  readonly workers?: number;
+  readonly budget_usd?: number;
+  readonly sandbox?: ExperimentSandbox;
+};
+
 type NormalizeOptions = {
   readonly sourcePath?: string;
 };
@@ -148,6 +169,11 @@ export function normalizeExperimentConfig(
     rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
     'timeout_seconds',
   );
+  const workers = readOptionalPositiveInteger(rawConfig.workers, 'workers');
+  const budgetUsd = readOptionalPositiveNumber(
+    rawConfig.budget_usd ?? rawConfig.budgetUsd,
+    'budget_usd',
+  );
   const sandbox = readOptionalSandbox(rawConfig.sandbox);
   const workspace = readOptionalRecord(rawConfig.workspace);
   const setup = readSetup(rawConfig.setup);
@@ -164,6 +190,8 @@ export function normalizeExperimentConfig(
     ...(runs !== undefined && { runs }),
     ...(earlyExit !== undefined && { earlyExit }),
     ...(timeoutSeconds !== undefined && { timeoutSeconds }),
+    ...(workers !== undefined && { workers }),
+    ...(budgetUsd !== undefined && { budgetUsd }),
     ...(sandbox !== undefined && { sandbox }),
     ...(workspace !== undefined && { workspace }),
     ...(setup !== undefined && { setup }),
@@ -181,6 +209,33 @@ export function fingerprintExperimentConfig(config: ExperimentConfig): string {
   return createHash('sha256').update(JSON.stringify(stablePayload)).digest('hex');
 }
 
+export function buildExperimentArtifactMetadata(
+  config: ExperimentConfig | undefined,
+): ExperimentArtifactMetadata | undefined {
+  if (!config) {
+    return undefined;
+  }
+  const targets = config.targets
+    ?.map((target) => (typeof target === 'string' ? target : target.name))
+    .filter((target) => target.trim().length > 0);
+  return {
+    ...(config.name !== undefined && { name: config.name }),
+    ...(config.sourcePath !== undefined && { source_path: config.sourcePath }),
+    ...(config.fingerprint !== undefined && { fingerprint: config.fingerprint }),
+    ...(config.agent !== undefined && { agent: config.agent }),
+    ...(config.target !== undefined && { target: config.target }),
+    ...(targets && targets.length > 0 && { targets }),
+    ...(config.model !== undefined && { model: config.model }),
+    ...(config.evals !== undefined && { evals: config.evals }),
+    ...(config.runs !== undefined && { runs: config.runs }),
+    ...(config.earlyExit !== undefined && { early_exit: config.earlyExit }),
+    ...(config.timeoutSeconds !== undefined && { timeout_seconds: config.timeoutSeconds }),
+    ...(config.workers !== undefined && { workers: config.workers }),
+    ...(config.budgetUsd !== undefined && { budget_usd: config.budgetUsd }),
+    ...(config.sandbox !== undefined && { sandbox: config.sandbox }),
+  };
+}
+
 function readTargets(raw: unknown): readonly ExperimentTargetRef[] | undefined {
   if (raw === undefined) {
     return undefined;
@@ -274,7 +329,11 @@ function readOptionalCommand(raw: unknown, location: string): readonly string[]
     }
     return ['sh', '-c', command];
   }
-  if (Array.isArray(raw) && raw.every((entry) => typeof entry === 'string' && entry.trim())) {
+  if (
+    Array.isArray(raw) &&
+    raw.length > 0 &&
+    raw.every((entry) => typeof entry === 'string' && entry.trim())
+  ) {
     return raw.map((entry) => entry.trim());
   }
   throw new Error(`Experiment ${location} must be a string or string array.`);
@@ -294,7 +353,11 @@ function readOptionalStringOrStringArray(
     }
     return trimmed;
   }
-  if (Array.isArray(raw) && raw.every((entry) => typeof entry === 'string' && entry.trim())) {
+  if (
+    Array.isArray(raw) &&
+    raw.length > 0 &&
+    raw.every((entry) => typeof entry === 'string' && entry.trim())
+  ) {
     return raw.map((entry) => entry.trim());
   }
   throw new Error(`Experiment ${location} must be a string or string array.`);
diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts
index 8210dd6e9..ebe5d653c 100644
--- a/packages/core/src/evaluation/loaders/config-loader.ts
+++ b/packages/core/src/evaluation/loaders/config-loader.ts
@@ -8,8 +8,6 @@ import type {
   FailOnError,
   JsonObject,
   TargetHooksConfig,
-  TrialStrategy,
-  TrialsConfig,
   WorkspaceHookConfig,
 } from '../types.js';
 import { isJsonObject } from '../types.js';
@@ -341,74 +339,6 @@ export function extractTargetsFromTestCase(testCase: JsonObject): readonly strin
   return undefined;
 }
 
-const VALID_TRIAL_STRATEGIES: ReadonlySet<string> = new Set([
-  'pass_at_k',
-  'mean',
-  'confidence_interval',
-]);
-
-/**
- * Extract trials configuration from parsed eval suite's execution block.
- * Returns undefined when count is 1 or not specified (no-op).
- */
-export function extractTrialsConfig(suite: JsonObject): TrialsConfig | undefined {
-  const execution = suite.execution;
-  if (!execution || typeof execution !== 'object' || Array.isArray(execution)) {
-    return undefined;
-  }
-
-  const trials = (execution as Record<string, unknown>).trials;
-  if (!trials || typeof trials !== 'object' || Array.isArray(trials)) {
-    return undefined;
-  }
-
-  const trialsObj = trials as Record<string, unknown>;
-  const count = trialsObj.count;
-
-  if (count === undefined || count === null) {
-    return undefined;
-  }
-
-  if (typeof count !== 'number' || !Number.isInteger(count) || count < 1) {
-    logWarning(
-      `Invalid trials.count: ${count}, must be a positive integer. Ignoring trials config.`,
-    );
-    return undefined;
-  }
-
-  if (count === 1) {
-    return undefined;
-  }
-
-  // Parse strategy (default: pass_at_k)
-  const rawStrategy = trialsObj.strategy;
-  let strategy: TrialStrategy = 'pass_at_k';
-  if (rawStrategy !== undefined && rawStrategy !== null) {
-    if (typeof rawStrategy !== 'string' || !VALID_TRIAL_STRATEGIES.has(rawStrategy)) {
-      logWarning(
-        `Invalid trials.strategy: '${rawStrategy}'. Must be one of: pass_at_k, mean, confidence_interval. Defaulting to pass_at_k.`,
-      );
-    } else {
-      strategy = rawStrategy as TrialStrategy;
-    }
-  }
-
-  // Parse cost_limit_usd (accepts both snake_case and camelCase)
-  const rawCostLimit = trialsObj.cost_limit_usd ?? trialsObj.costLimitUsd;
-  let costLimitUsd: number | undefined;
-  if (rawCostLimit !== undefined && rawCostLimit !== null) {
-    if (typeof rawCostLimit === 'number' && rawCostLimit > 0) {
-      costLimitUsd = rawCostLimit;
-    } else {
-      logWarning(
-        `Invalid trials.cost_limit_usd: ${rawCostLimit}. Must be a positive number. Ignoring.`,
-      );
-    }
-  }
-
-  return { count, strategy, costLimitUsd };
-}
-
 /**
  * Cache configuration parsed from execution block.
  */
diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts
index 119359250..d194b8774 100644
--- a/packages/core/src/evaluation/orchestrator.ts
+++ b/packages/core/src/evaluation/orchestrator.ts
@@ -2330,7 +2330,11 @@ async function runEvalCaseWithTrials(
     }
 
     // pass_at_k early exit: short-circuit after first passing trial
-    if (trialsConfig.strategy === 'pass_at_k' && trialVerdict === 'pass') {
+    if (
+      trialsConfig.strategy === 'pass_at_k' &&
+      trialsConfig.earlyExit !== false &&
+      trialVerdict === 'pass'
+    ) {
       break;
     }
   }
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 20c6161b7..46a0ce407 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -12,6 +12,7 @@ import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import { traceEnvelopeToTranscriptJsonLines } from '../import/types.js';
+import type { ExperimentArtifactMetadata } from './experiment.js';
 import {
   type ExternalTraceMetadataWire,
   externalTraceMetadataForResult,
@@ -89,7 +90,12 @@ export function deduplicateByTestIdTarget(
 
 export async function aggregateRunDir(
   runDir: string,
-  options?: { evalFile?: string; experiment?: string; plannedTestCount?: number },
+  options?: {
+    evalFile?: string;
+    experiment?: string;
+    plannedTestCount?: number;
+    experimentMetadata?: ExperimentArtifactMetadata;
+  },
 ): Promise<{ benchmarkPath: string; timingPath: string; testCount: number; targetCount: number }> {
   const indexPath = path.join(runDir, RESULT_INDEX_FILENAME);
   const content = await readFile(indexPath, 'utf8');
@@ -108,6 +114,7 @@ export async function aggregateRunDir(
     options?.evalFile,
     options?.experiment,
     plannedTestCount,
+    options?.experimentMetadata,
   );
   const benchmarkPath = path.join(runDir, 'benchmark.json');
   await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}\n`, 'utf8');
@@ -187,6 +194,7 @@ export interface BenchmarkArtifact {
     readonly targets: readonly string[];
     readonly tests_run: readonly string[];
     readonly experiment?: string;
+    readonly experiment_config?: ExperimentArtifactMetadata;
     readonly planned_test_count?: number;
   };
   readonly run_summary: Record<
@@ -677,6 +685,7 @@ export function buildBenchmarkArtifact(
   evalFile = '',
   experiment?: string,
   plannedTestCount?: number,
+  experimentMetadata?: ExperimentArtifactMetadata,
 ): BenchmarkArtifact {
   const targetSet = new Set<string>();
   const testIdSet = new Set<string>();
@@ -767,6 +776,7 @@ export function buildBenchmarkArtifact(
       targets,
       tests_run: testIds,
       experiment,
+      experiment_config: experimentMetadata,
       planned_test_count: plannedTestCount,
     },
     run_summary: runSummary,
@@ -781,6 +791,7 @@ export async function writeInitialBenchmarkArtifact(
     evalFile: string;
     plannedTestCount: number;
     experiment?: string;
+    experimentMetadata?: ExperimentArtifactMetadata;
   },
 ): Promise<void> {
   await mkdir(runDir, { recursive: true });
@@ -789,6 +800,7 @@ export async function writeInitialBenchmarkArtifact(
     options.evalFile,
     options.experiment,
     options.plannedTestCount,
+    options.experimentMetadata,
   );
   const benchmarkPath = path.join(runDir, 'benchmark.json');
   await writeFile(benchmarkPath, `${JSON.stringify(stub, null, 2)}\n`, 'utf8');
@@ -1610,6 +1622,7 @@ export async function writeArtifactsFromResults(
   options?: {
     evalFile?: string;
     experiment?: string;
+    experimentMetadata?: ExperimentArtifactMetadata;
     plannedTestCount?: number;
     runId?: string;
     duplicatePolicy?: ExportDuplicatePolicy;
@@ -1808,6 +1821,7 @@ export async function writeArtifactsFromResults(
     options?.evalFile,
     options?.experiment,
     plannedTestCount,
+    options?.experimentMetadata,
   );
   await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}\n`, 'utf8');
 
diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts
index 01b63b398..af8c100bb 100644
--- a/packages/core/src/evaluation/types.ts
+++ b/packages/core/src/evaluation/types.ts
@@ -1065,6 +1065,8 @@ export interface TrialsConfig {
   readonly count: number;
   readonly strategy: TrialStrategy;
   readonly costLimitUsd?: number;
+  /** When false, pass_at_k runs all configured attempts instead of stopping at first pass. */
+  readonly earlyExit?: boolean;
 }
 
 /**
diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts
index 49b781b6b..06fdc05b1 100644
--- a/packages/core/src/evaluation/validation/eval-file.schema.ts
+++ b/packages/core/src/evaluation/validation/eval-file.schema.ts
@@ -346,13 +346,6 @@ const EvalTargetRefSchema = z.object({
 // Execution block
 // ---------------------------------------------------------------------------
 
-const TrialsSchema = z.object({
-  count: z.number().int().min(1),
-  strategy: z.enum(['pass_at_k', 'mean', 'confidence_interval']).optional(),
-  cost_limit_usd: z.number().min(0).optional(),
-  costLimitUsd: z.number().min(0).optional(),
-});
-
 /** Execution error tolerance: true or false */
 const FailOnErrorSchema = z.boolean();
 
@@ -364,7 +357,8 @@ const ExecutionSchema = z.object({
   evaluators: z.array(EvaluatorSchema).optional(),
   skip_defaults: z.boolean().optional(),
   cache: z.boolean().optional(),
-  trials: TrialsSchema.optional(),
+  /** Removed before stable release. Run counts belong on experiment configs. */
+  trials: z.never().optional(),
   budget_usd: z.number().min(0).optional(),
   budgetUsd: z.number().min(0).optional(),
   fail_on_error: FailOnErrorSchema.optional(),
diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts
index 0c9efccf9..ba07f57a6 100644
--- a/packages/core/src/evaluation/yaml-parser.ts
+++ b/packages/core/src/evaluation/yaml-parser.ts
@@ -20,7 +20,6 @@ import {
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
   extractThreshold,
-  extractTrialsConfig,
   extractWorkersFromSuite,
   loadConfig,
 } from './loaders/config-loader.js';
@@ -56,7 +55,6 @@ import type {
   RepoConfig,
   TestMessage,
   TestMessageContent,
-  TrialsConfig,
   TurnFailurePolicy,
   WorkspaceConfig,
   WorkspaceEnvConfig,
@@ -79,7 +77,6 @@ export {
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
   extractThreshold,
-  extractTrialsConfig,
   extractWorkersFromSuite,
   loadConfig,
 } from './loaders/config-loader.js';
@@ -237,7 +234,6 @@ export async function readTestSuiteMetadata(testFilePath: string): Promise<{
   target?: string;
   targets?: readonly string[];
   targetRefs?: readonly import('./types.js').EvalTargetRef[];
-  trials?: TrialsConfig;
 }> {
   try {
     const absolutePath = path.resolve(testFilePath);
@@ -252,7 +248,6 @@ export async function readTestSuiteMetadata(testFilePath: string): Promise<{
       target: extractTargetFromSuite(parsed),
       targets: extractTargetsFromSuite(parsed),
       targetRefs: extractTargetRefsFromSuite(parsed),
-      trials: extractTrialsConfig(parsed),
     };
   } catch {
     return {};
@@ -265,7 +260,6 @@ export async function readTestSuiteMetadata(testFilePath: string): Promise<{
  */
 export type EvalSuiteResult = {
   readonly tests: readonly EvalTest[];
-  readonly trials?: TrialsConfig;
   /** Suite-level targets from execution.targets (matrix evaluation) */
   readonly targets?: readonly string[];
   /** Suite-level target refs with hooks from execution.targets (object form) */
@@ -736,7 +730,6 @@ function buildEvalSuiteResult(
 
   return {
     tests,
-    trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
     targetRefs: extractTargetRefsFromSuite(parsed),
     workers: extractWorkersFromSuite(parsed),
diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts
index e9c19f4c4..5a3afc22c 100644
--- a/packages/core/test/evaluation/experiment.test.ts
+++ b/packages/core/test/evaluation/experiment.test.ts
@@ -4,6 +4,7 @@ import os from 'node:os';
 import path from 'node:path';
 
 import {
+  buildExperimentArtifactMetadata,
   deriveExperimentNameFromPath,
   isExperimentFileReference,
   loadExperimentConfig,
@@ -23,6 +24,8 @@ describe('experiment config', () => {
       runs: 3,
       early_exit: false,
       timeout_seconds: 900,
+      workers: 4,
+      budget_usd: 1.25,
       sandbox: 'auto',
       setup: [{ script: 'bun install' }],
     });
@@ -38,6 +41,8 @@ describe('experiment config', () => {
       runs: 3,
       earlyExit: false,
       timeoutSeconds: 900,
+      workers: 4,
+      budgetUsd: 1.25,
       sandbox: 'auto',
       setup: [{ script: 'bun install' }],
     });
@@ -79,6 +84,36 @@ describe('experiment config', () => {
     expect(() => normalizeExperimentConfig({ sandbox: 'host' })).toThrow(/sandbox/);
   });
 
+  it('builds safe snake_case artifact metadata', () => {
+    const config = normalizeExperimentConfig({
+      name: 'baseline',
+      target: 'codex',
+      agent_options: { secret: 'not persisted' },
+      setup: [{ script: 'bun install' }],
+      scripts: [{ script: 'bun test' }],
+      runs: 2,
+      early_exit: true,
+      timeout_seconds: 120,
+      workers: 3,
+      budget_usd: 0.5,
+    });
+
+    const metadata = buildExperimentArtifactMetadata(config);
+
+    expect(metadata).toMatchObject({
+      name: 'baseline',
+      target: 'codex',
+      runs: 2,
+      early_exit: true,
+      timeout_seconds: 120,
+      workers: 3,
+      budget_usd: 0.5,
+    });
+    expect(metadata).not.toHaveProperty('agent_options');
+    expect(metadata).not.toHaveProperty('setup');
+    expect(metadata).not.toHaveProperty('scripts');
+  });
+
   it('detects experiment file references separately from labels', () => {
     expect(isExperimentFileReference('experiments/default.yaml')).toBe(true);
     expect(isExperimentFileReference('default.yaml')).toBe(true);
diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts
index c345486e3..764836ad7 100644
--- a/packages/core/test/evaluation/loaders/config-loader.test.ts
+++ b/packages/core/test/evaluation/loaders/config-loader.test.ts
@@ -11,7 +11,6 @@ import {
   extractTargetsFromSuite,
   extractTargetsFromTestCase,
   extractThreshold,
-  extractTrialsConfig,
   loadConfig,
   parseExecutionDefaults,
   parseExperimentsConfig,
@@ -177,128 +176,6 @@ describe('parseExperimentsConfig', () => {
   });
 });
 
-describe('extractTrialsConfig', () => {
-  it('returns undefined when no execution block', () => {
-    const suite: JsonObject = { tests: [] };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined when no trials in execution', () => {
-    const suite: JsonObject = { execution: { target: 'default' } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined when count is 1', () => {
-    const suite: JsonObject = { execution: { trials: { count: 1 } } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('parses valid trials config with defaults', () => {
-    const suite: JsonObject = { execution: { trials: { count: 3 } } };
-    const result = extractTrialsConfig(suite);
-
-    expect(result).toEqual({
-      count: 3,
-      strategy: 'pass_at_k',
-      costLimitUsd: undefined,
-    });
-  });
-
-  it('parses trials config with all fields', () => {
-    const suite: JsonObject = {
-      execution: {
-        trials: {
-          count: 5,
-          strategy: 'mean',
-          cost_limit_usd: 10.0,
-        },
-      },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result).toEqual({
-      count: 5,
-      strategy: 'mean',
-      costLimitUsd: 10.0,
-    });
-  });
-
-  it('accepts confidence_interval strategy', () => {
-    const suite: JsonObject = {
-      execution: { trials: { count: 10, strategy: 'confidence_interval' } },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result?.strategy).toBe('confidence_interval');
-  });
-
-  it('accepts camelCase costLimitUsd', () => {
-    const suite: JsonObject = {
-      execution: { trials: { count: 3, costLimitUsd: 5.0 } },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result?.costLimitUsd).toBe(5.0);
-  });
-
-  it('returns undefined for invalid count (non-integer)', () => {
-    const suite: JsonObject = { execution: { trials: { count: 2.5 } } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined for invalid count (zero)', () => {
-    const suite: JsonObject = { execution: { trials: { count: 0 } } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined for invalid count (negative)', () => {
-    const suite: JsonObject = { execution: { trials: { count: -1 } } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined for invalid count (string)', () => {
-    const suite: JsonObject = { execution: { trials: { count: 'three' } } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('defaults to pass_at_k for invalid strategy', () => {
-    const suite: JsonObject = {
-      execution: { trials: { count: 3, strategy: 'invalid_strategy' } },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result?.strategy).toBe('pass_at_k');
-  });
-
-  it('ignores invalid cost_limit_usd', () => {
-    const suite: JsonObject = {
-      execution: { trials: { count: 3, cost_limit_usd: -5 } },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result?.costLimitUsd).toBeUndefined();
-  });
-
-  it('ignores non-numeric cost_limit_usd', () => {
-    const suite: JsonObject = {
-      execution: { trials: { count: 3, cost_limit_usd: 'five' } },
-    };
-    const result = extractTrialsConfig(suite);
-
-    expect(result?.costLimitUsd).toBeUndefined();
-  });
-
-  it('returns undefined when trials is not an object', () => {
-    const suite: JsonObject = { execution: { trials: 'invalid' } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-
-  it('returns undefined when trials is an array', () => {
-    const suite: JsonObject = { execution: { trials: [1, 2, 3] } };
-    expect(extractTrialsConfig(suite)).toBeUndefined();
-  });
-});
-
 describe('parseResultsConfig', () => {
   it('parses valid results config with explicit path', () => {
     const result = parseResultsConfig(
diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts
index 89fda3f30..bfb23d02f 100644
--- a/packages/core/test/evaluation/orchestrator.test.ts
+++ b/packages/core/test/evaluation/orchestrator.test.ts
@@ -1801,8 +1801,7 @@ describe('runEvaluation with trials', () => {
   it('count=1: no trial metadata in result (handled by orchestrator)', async () => {
     const provider = new MultiCallProvider();
 
-    // count=1 should not produce trials metadata — extractTrialsConfig returns
-    // undefined for count=1, so trials option won't be set. Verify normal behavior.
+    // Single-run callers should omit trials entirely. Verify normal behavior.
     const results = await runEvaluation({
       testFilePath: 'in-memory.yaml',
       repoRoot: 'in-memory',
diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts
index 17e6c528d..5b15429cf 100644
--- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts
+++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts
@@ -35,4 +35,18 @@ describe('EvalFileSchema input shorthand', () => {
 
     expect(result.success).toBe(false);
   });
+
+  it('rejects eval-level execution.trials because run counts belong on experiments', () => {
+    const result = EvalFileSchema.safeParse({
+      execution: {
+        trials: {
+          count: 2,
+          strategy: 'pass_at_k',
+        },
+      },
+      tests: [baseTest],
+    });
+
+    expect(result.success).toBe(false);
+  });
 });
diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval-schema.json
index b7de8cda6..2abcdd98d 100644
--- a/skills-data/agentv-eval-writer/references/eval-schema.json
+++ b/skills-data/agentv-eval-writer/references/eval-schema.json
@@ -5144,27 +5144,7 @@
                         "type": "boolean"
                       },
                       "trials": {
-                        "type": "object",
-                        "properties": {
-                          "count": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "strategy": {
-                            "type": "string",
-                            "enum": ["pass_at_k", "mean", "confidence_interval"]
-                          },
-                          "cost_limit_usd": {
-                            "type": "number",
-                            "minimum": 0
-                          },
-                          "costLimitUsd": {
-                            "type": "number",
-                            "minimum": 0
-                          }
-                        },
-                        "required": ["count"],
-                        "additionalProperties": false
+                        "not": {}
                       },
                       "budget_usd": {
                         "type": "number",
@@ -11736,27 +11716,7 @@
                         "type": "boolean"
                       },
                       "trials": {
-                        "type": "object",
-                        "properties": {
-                          "count": {
-                            "type": "integer",
-                            "minimum": 1
-                          },
-                          "strategy": {
-                            "type": "string",
-                            "enum": ["pass_at_k", "mean", "confidence_interval"]
-                          },
-                          "cost_limit_usd": {
-                            "type": "number",
-                            "minimum": 0
-                          },
-                          "costLimitUsd": {
-                            "type": "number",
-                            "minimum": 0
-                          }
-                        },
-                        "required": ["count"],
-                        "additionalProperties": false
+                        "not": {}
                       },
                       "budget_usd": {
                         "type": "number",
@@ -15858,27 +15818,7 @@
               "type": "boolean"
             },
             "trials": {
-              "type": "object",
-              "properties": {
-                "count": {
-                  "type": "integer",
-                  "minimum": 1
-                },
-                "strategy": {
-                  "type": "string",
-                  "enum": ["pass_at_k", "mean", "confidence_interval"]
-                },
-                "cost_limit_usd": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "costLimitUsd": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              },
-              "required": ["count"],
-              "additionalProperties": false
+              "not": {}
             },
             "budget_usd": {
               "type": "number",

From 8ac7d340b36d0fd2f081f3fac1fee33eedd5d022 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 14:41:13 +0200
Subject: [PATCH 05/18] fix: skip empty eval suites after experiment filtering

---
 apps/cli/src/commands/eval/run-eval.ts | 18 ++++++++++++++++++
 apps/cli/test/eval.integration.test.ts | 16 +++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts
index 7cfbb751d..687b22cb3 100644
--- a/apps/cli/src/commands/eval/run-eval.ts
+++ b/apps/cli/src/commands/eval/run-eval.ts
@@ -1016,6 +1016,24 @@ async function prepareFileMetadata(params: {
   const testIds = suite.tests.map((value) => value.id);
   const suiteTargets = suite.targets;
 
+  if (suite.tests.length === 0) {
+    return {
+      testIds,
+      testCases: suite.tests,
+      selections: [],
+      trialsConfig: options.experimentTrialsConfig,
+      suiteTargets,
+      yamlWorkers: suite.workers,
+      yamlCache: suite.cacheConfig?.enabled,
+      yamlCachePath: suite.cacheConfig?.cachePath,
+      budgetUsd: suite.budgetUsd,
+      failOnError: suite.failOnError,
+      threshold: suite.threshold,
+      tags: suite.metadata?.tags,
+      providerFactory: suite.providerFactory,
+    };
+  }
+
   let selections: { selection: TargetSelection; inlineTargetLabel: string }[];
 
   if (options.transcript) {
diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts
index c95de4ea2..0f2b7eca9 100644
--- a/apps/cli/test/eval.integration.test.ts
+++ b/apps/cli/test/eval.integration.test.ts
@@ -525,7 +525,21 @@ describe('agentv eval CLI', () => {
       await mkdir(experimentsDir, { recursive: true });
       await writeFile(
         path.join(fixture.suiteDir, '.agentv', 'config.yaml'),
-        'eval_patterns:\n  - sample.test.yaml\n',
+        'eval_patterns:\n  - sample.test.yaml\n  - unused.test.yaml\n',
+        'utf8',
+      );
+      await writeFile(
+        path.join(fixture.suiteDir, 'unused.test.yaml'),
+        [
+          'description: unmatched eval file should not resolve targets',
+          'target: missing-target',
+          'tests:',
+          '  - id: case-unused',
+          '    criteria: System responds with unused',
+          '    input: unused',
+          '    expected_output: unused',
+          '',
+        ].join('\n'),
         'utf8',
       );
       const experimentPath = path.join(experimentsDir, 'default.yaml');

From 362aa33e421f685fcb7791c4b4998e07f76997e9 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 15:05:48 +0200
Subject: [PATCH 06/18] feat: add experiment repeat schema and docs

---
 apps/cli/src/commands/eval/run-eval.ts        |  15 +-
 apps/cli/test/eval.integration.test.ts        |  14 +-
 apps/cli/test/fixtures/mock-run-evaluation.ts |   1 +
 .../docs/docs/evaluation/eval-files.mdx       |   2 +-
 .../docs/docs/evaluation/experiments.mdx      | 124 ++++++++
 ...26-06-23-experiments-vs-eval-separation.md |   8 +-
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |  79 ++---
 ...6-06-23-002-experiments-separation-plan.md |  29 +-
 examples/features/README.md                   |   2 +-
 examples/features/trials/README.md            |  43 +++
 .../features/trials/evals/dataset.eval.yaml   |   6 +-
 .../experiments/confidence-interval.yaml      |   9 +
 .../features/trials/experiments/default.yaml  |   7 +-
 .../features/trials/experiments/mean.yaml     |   9 +
 .../showcase/multi-model-benchmark/README.md  |  68 +++--
 .../evals/benchmark.eval.yaml                 |   5 +-
 .../experiments/default.yaml                  |   7 +-
 packages/core/scripts/generate-eval-schema.ts |  61 ++--
 packages/core/src/evaluation/experiment.ts    |  91 ++++++
 .../validation/experiment-file.schema.ts      |  74 +++++
 .../core/src/evaluation/validation/index.ts   |   1 +
 .../core/test/evaluation/experiment.test.ts   |  51 +++-
 .../validation/eval-schema-sync.test.ts       |  29 +-
 skills-data/agentv-eval-writer/SKILL.md       |   8 +
 .../references/experiment-schema.json         | 276 ++++++++++++++++++
 25 files changed, 897 insertions(+), 122 deletions(-)
 create mode 100644 apps/web/src/content/docs/docs/evaluation/experiments.mdx
 create mode 100644 examples/features/trials/README.md
 create mode 100644 examples/features/trials/experiments/confidence-interval.yaml
 create mode 100644 examples/features/trials/experiments/mean.yaml
 create mode 100644 packages/core/src/evaluation/validation/experiment-file.schema.ts
 create mode 100644 skills-data/agentv-eval-writer/references/experiment-schema.json

diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts
index 687b22cb3..5e2eb1058 100644
--- a/apps/cli/src/commands/eval/run-eval.ts
+++ b/apps/cli/src/commands/eval/run-eval.ts
@@ -690,13 +690,26 @@ function buildExperimentTargetRefs(
 }
 
 function buildExperimentTrialsConfig(experiment: ExperimentConfig): TrialsConfig | undefined {
+  if (experiment.repeat) {
+    if (experiment.repeat.count <= 1) {
+      return undefined;
+    }
+    return {
+      count: experiment.repeat.count,
+      strategy: experiment.repeat.strategy,
+      ...(experiment.repeat.costLimitUsd !== undefined && {
+        costLimitUsd: experiment.repeat.costLimitUsd,
+      }),
+      ...(experiment.earlyExit !== undefined && { earlyExit: experiment.earlyExit }),
+    };
+  }
   if (!experiment.runs || experiment.runs <= 1) {
     return undefined;
   }
   return {
     count: experiment.runs,
     strategy: 'pass_at_k',
-    earlyExit: experiment.earlyExit ?? false,
+    ...(experiment.earlyExit !== undefined && { earlyExit: experiment.earlyExit }),
   };
 }
 
diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts
index 0f2b7eca9..083819931 100644
--- a/apps/cli/test/eval.integration.test.ts
+++ b/apps/cli/test/eval.integration.test.ts
@@ -551,7 +551,10 @@ describe('agentv eval CLI', () => {
           'evals: case-alpha',
           'timeout_seconds: 12',
           'workers: 4',
-          'runs: 2',
+          'repeat:',
+          '  count: 2',
+          '  strategy: mean',
+          '  cost_limit_usd: 1.25',
           'early_exit: false',
           'setup:',
           '  - script: "printf setup > ../experiment-setup.txt"',
@@ -579,7 +582,8 @@ describe('agentv eval CLI', () => {
         maxConcurrency: 4,
         trials: {
           count: 2,
-          strategy: 'pass_at_k',
+          strategy: 'mean',
+          costLimitUsd: 1.25,
           earlyExit: false,
         },
       });
@@ -596,7 +600,11 @@ describe('agentv eval CLI', () => {
         source_path: experimentPath,
         target: 'cli-target',
         evals: 'case-alpha',
-        runs: 2,
+        repeat: {
+          count: 2,
+          strategy: 'mean',
+          cost_limit_usd: 1.25,
+        },
         early_exit: false,
         timeout_seconds: 12,
         workers: 4,
diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts
index c501cb2b0..32162888e 100644
--- a/apps/cli/test/fixtures/mock-run-evaluation.ts
+++ b/apps/cli/test/fixtures/mock-run-evaluation.ts
@@ -22,6 +22,7 @@ interface RunEvaluationOptionsLike {
   readonly trials?: {
     readonly count: number;
     readonly strategy: string;
+    readonly costLimitUsd?: number;
     readonly earlyExit?: boolean;
   };
   readonly budgetUsd?: number;
diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
index 7be7eabc7..65d0942da 100644
--- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
@@ -5,7 +5,7 @@ sidebar:
   order: 1
 ---
 
-Evaluation files define the test cases, targets, and graders for an evaluation run. AgentV supports two formats: YAML and JSONL.
+Evaluation files define the test cases and graders for an evaluation run. Runtime choices such as target matrices, setup, scripts, and repeat runs belong in [experiments](/docs/evaluation/experiments/). AgentV supports two eval formats: YAML and JSONL.
 
 YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract.
 
diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
new file mode 100644
index 000000000..6766dc7ea
--- /dev/null
+++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
@@ -0,0 +1,124 @@
+---
+title: Experiments
+description: Configure how AgentV evals run
+sidebar:
+  order: 2
+---
+
+Experiments define **how** eval cases run: target or target matrix, setup,
+scripts, timeout, sandbox, case filters, and repeat-run policy. Eval files stay
+focused on **what** is tested: prompts, datasets, assertions, and task fixtures.
+
+## Experiment YAML
+
+Committed experiments conventionally live under `experiments/`:
+
+```yaml
+name: baseline
+target: codex-gpt5
+evals: "agent-*"
+timeout_seconds: 720
+repeat:
+  count: 4
+  strategy: pass_at_k
+  cost_limit_usd: 2.00
+setup:
+  - script: bun install
+scripts:
+  - build
+```
+
+Wire fields use `snake_case`. AgentV translates to internal `camelCase` when it
+loads the file.
+
+## Repeat runs
+
+`repeat` is the full AgentV replacement for the old eval-level
+`execution.trials` shape. It supports the same core strategies:
+
+```yaml
+repeat:
+  count: 3
+  strategy: mean
+  cost_limit_usd: 1.50
+```
+
+Supported strategies:
+
+| Strategy | Behavior |
+| --- | --- |
+| `pass_at_k` | Uses the best passing attempt; early-exits by default unless the experiment sets `early_exit: false` |
+| `mean` | Aggregates repeated attempt scores by mean |
+| `confidence_interval` | Uses the lower bound of a 95% confidence interval as the conservative score |
+
+`repeat.cost_limit_usd` caps repeat-run spend. `repeat.costLimitUsd` is also
+accepted for prerelease trial-schema parity, but new YAML should use
+`cost_limit_usd`.
+
+## Vercel-compatible shorthand
+
+AgentV also accepts Vercel-style top-level `runs` and `early_exit`:
+
+```yaml
+runs: 4
+early_exit: true
+```
+
+This is shorthand for a `pass_at_k` repeat run. Use `repeat` when you need
+AgentV-specific strategy or cost-limit fields.
+
+Do not set both `repeat` and `runs` in the same experiment. `repeat` is the
+canonical AgentV shape; `runs` exists only for Vercel-compatible shorthand.
+
+Vercel defines the requested run count at the experiment level. Some result
+summaries show fewer actual runs for a case because `earlyExit: true` stops
+remaining attempts after the first pass; smoke runs can also force one run.
+AgentV follows the same experiment-level placement while keeping the richer
+`repeat` block for AgentV strategies.
+
+## Targets and setup
+
+Experiments reuse targets from `.agentv/targets.yaml`; they do not define a new
+provider registry.
+
+```yaml
+targets:
+  - copilot
+  - claude
+  - name: gemini-with-hooks
+    use_target: gemini
+```
+
+Setup and scripts belong on the experiment because they are often the A/B
+variable:
+
+```yaml
+setup:
+  - script: cp skills/with-docs/AGENTS.md AGENTS.md
+scripts:
+  - script: bun test
+    timeout_seconds: 120
+```
+
+## Running experiments
+
+Run a specific experiment:
+
+```bash
+bun agentv eval evals/suite.eval.yaml --experiment experiments/default.yaml
+```
+
+If no experiment is passed, AgentV checks `.agentv/config.yaml` for a default:
+
+```yaml
+experiments:
+  default: experiments/default.yaml
+```
+
+If no default is configured, AgentV keeps the old behavior and uses the
+`default` experiment label.
+
+## Schema
+
+The generated JSON Schema is available at
+`skills-data/agentv-eval-writer/references/experiment-schema.json`.
diff --git a/docs/adr/2026-06-23-experiments-vs-eval-separation.md b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
index a4432c438..aae9955f8 100644
--- a/docs/adr/2026-06-23-experiments-vs-eval-separation.md
+++ b/docs/adr/2026-06-23-experiments-vs-eval-separation.md
@@ -73,7 +73,10 @@ model: openai/gpt-5.5
 evals: "agent-042-*"
 scripts:
   - build
-runs: 3
+repeat:
+  count: 3
+  strategy: pass_at_k
+  cost_limit_usd: 2.00
 early_exit: false
 timeout_seconds: 900
 sandbox: auto
@@ -98,7 +101,8 @@ Legacy `eval.yaml execution` fields that select targets, targets matrices,
 workers, cache, budget, thresholds, and workspace runtime behavior will
 continue to parse as a compatibility shim until docs and examples have moved.
 The prerelease `execution.trials` surface is hard-removed with no alias: run
-counts live on the experiment as `runs`/`early_exit`.
+counts live on the experiment as canonical `repeat` config, with Vercel-style
+`runs`/`early_exit` accepted as shorthand for `pass_at_k`.
 
 AgentV should adopt Vercel's structure and lowest-common-denominator contract
 ideas, not depend on `@vercel/agent-eval` as core infrastructure in this phase.
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index adcb418a6..c2e8f6d18 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -153,7 +153,7 @@ Public docs and implementation notes must not reference non-public sources. If a
 - KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json` and `run-N/` directories so existing aggregate consumers do not inflate case counts.
 - KTD4. Single-run cases keep direct case-local files instead of always nesting under `run-1`. This preserves the simple default artifact shape and makes `.agentv/results/<experiment>/<timestamp>/<case-id>/grading.json` easy to inspect. Repeat-enabled cases use `run-1/`, `run-2/`, and so on under the case directory.
 - KTD5. Write `summary.json` for every case after the artifact-layout migration. In a single-run case it summarizes the direct files; in a repeat-run case it summarizes the `run-N/` children.
-- KTD6. Default repeat reporting uses full sampling with no early exit. Early exit saves cost but biases reliability statistics, so it should be explicit and recorded.
+- KTD6. `pass_at_k` keeps the existing AgentV/Vercel ergonomics: early exit is enabled unless explicitly disabled. Full reliability sampling requires `early_exit: false` on the experiment and should be recorded because it changes cost and statistics.
 - KTD7. Do not inherit Vercel's implicit CI ambiguity. All policies that can make one failed plus one passed attempt count as passing must be visible in config and artifacts.
 - KTD8. Reuse current failure classification fields before adding new enums. Add aggregate classification fields only after mapping from `execution_status`, `failure_stage`, and `failure_reason_code` proves insufficient.
 - KTD9. Do not reuse `pass_rate` for attempt success frequency. AgentV uses `attempt_success_rate` for repeat-run reliability and reserves `pass_rate` for assertion or expectation pass rate.
@@ -187,42 +187,45 @@ The runner executes the configured attempts, writes attempt artifacts, computes
 Preferred v1 shape:
 
 ```yaml
-execution:
-  repeat:
-    runs: 3
-    seed: 1234
-    max_parallel_attempts: 1
-    timeout_ms: 300000
-    budget_usd: 5
-    early_exit: never
-    retry:
-      max_attempts: 1
-      on:
-        - verifier_error
-        - infrastructure_error
-        - timeout
-    gate:
-      policy: attempt_success_rate_at_least
-      threshold: 0.8
+repeat:
+  count: 3
+  strategy: pass_at_k
+  cost_limit_usd: 5
+  seed: 1234
+  max_parallel_attempts: 1
+  timeout_ms: 300000
+  early_exit: never
+  retry:
+    max_attempts: 1
+    on:
+      - verifier_error
+      - infrastructure_error
+      - timeout
+  gate:
+    policy: attempt_success_rate_at_least
+    threshold: 0.8
 ```
 
 Field notes:
 
-- `runs` is the planned reliability sample count. Missing or `1` means normal single-run behavior.
-- `max_attempts` belongs to retry handling, not reliability sampling. For example, `runs: 3` with `retry.max_attempts: 1` may write up to six physical attempts, but only the three counted attempts feed reliability stats.
-- `early_exit` starts with `never`, `on_gate_satisfied`, and `on_gate_failed`. Reliability reports should default to `never`.
+- `count` is the planned reliability sample count. Missing repeat config or `count: 1` means normal single-run behavior.
+- `strategy` starts with the existing AgentV aggregation strategies: `pass_at_k`, `mean`, and `confidence_interval`.
+- `max_attempts` belongs to retry handling, not reliability sampling. For example, `count: 3` with `retry.max_attempts: 1` may write up to six physical attempts, but only the three counted attempts feed reliability stats.
+- `early_exit` should be represented by the experiment-level boolean in the native experiments branch. Future gate-aware modes such as `never`, `on_gate_satisfied`, and `on_gate_failed` remain non-goals until gate policies are implemented.
 - `seed` is best effort. Providers that support deterministic seeds receive a per-attempt seed derived from the base seed and attempt number; providers that do not support seeds record `seed_unsupported`.
-- `budget_usd` composes with existing run-level budget controls and stops new attempts when the budget is exhausted.
+- `cost_limit_usd` composes with existing run-level budget controls and stops new attempts when the repeat budget is exhausted.
 
 Migration from `execution.trials` (hard removal, no alias):
 
 `execution.trials` is removed from `eval.yaml` outright; the repeat block lives on the experiment instead. Existing semantics map as follows so any prerelease evals can be ported by hand:
 
-- `trials.count` maps to the experiment repeat `runs`.
-- `trials.cost_limit_usd` maps to repeat `budget_usd`.
-- `trials.strategy: pass_at_k` maps to repeat `gate.policy: any_attempt_successful` (with `early_exit: on_gate_satisfied`).
-- `trials.strategy: mean` maps to `mean_score_at_least` only when a threshold is present; otherwise it is report-only aggregation.
-- `trials.strategy: confidence_interval` remains report-only until AgentV has a reviewed CI policy for confidence bounds.
+- `trials.count` maps to `repeat.count`.
+- `trials.cost_limit_usd` maps to `repeat.cost_limit_usd`.
+- `trials.costLimitUsd` is accepted only as `repeat.costLimitUsd` for prerelease parity; new YAML should use `cost_limit_usd`.
+- `trials.strategy: pass_at_k` maps to `repeat.strategy: pass_at_k`.
+- `trials.strategy: mean` maps to `repeat.strategy: mean`.
+- `trials.strategy: confidence_interval` maps to `repeat.strategy: confidence_interval`.
+- Existing gate-policy ideas remain future work; do not overload `strategy` to imply CI policy.
 
 ---
 
@@ -432,13 +435,13 @@ Search behavior:
 
 Repeat runs can multiply provider spend. V1 should ship with conservative controls:
 
-- `runs` and retry `max_attempts` must be bounded by validation.
+- `repeat.count` and retry `max_attempts` must be bounded by validation.
 - `max_parallel_attempts` limits per-case concurrent attempts; it composes with existing eval workers and provider-specific concurrency guidance.
 - Agent-provider targets should keep the existing "limit concurrency to 3 targets" operational guidance.
-- `budget_usd` stops scheduling new attempts when exceeded and records `budget_exceeded`.
+- `repeat.cost_limit_usd` stops scheduling new attempts when exceeded and records `budget_exceeded`.
 - `timeout_ms` applies per attempt; existing top-level agent timeout remains the default if no repeat timeout is set.
-- `early_exit` must be explicit and recorded in `summary.json`.
-- Cache should be disabled or scoped by attempt when repeat runs measure stochastic behavior. Existing code already disables cache for `trials.count > 1`; keep that principle.
+- `early_exit` must be recorded in `summary.json` when it differs from the default.
+- Cache should be disabled or scoped by attempt when repeat runs measure stochastic behavior. Existing code already disables cache for repeated attempts; keep that principle.
 - Provider seed support is best effort and must be recorded per attempt so deterministic and stochastic runs are distinguishable.
 
 ---
@@ -453,17 +456,17 @@ Repeat runs can multiply provider spend. V1 should ship with conservative contro
 
 **Bead:** `av-i0l.1`.
 
-**Files:** `packages/core/src/evaluation/types.ts`, `packages/core/src/evaluation/loaders/config-loader.ts`, `packages/core/src/evaluation/validation/eval-file.schema.ts`, `packages/core/scripts/generate-eval-schema.ts`, `packages/core/test/evaluation/loaders/config-loader.test.ts`, `packages/core/test/evaluation/validation/eval-file-schema.test.ts`.
+**Files:** `packages/core/src/evaluation/types.ts`, `packages/core/src/evaluation/experiment.ts`, `packages/core/src/evaluation/validation/experiment-file.schema.ts`, `packages/core/scripts/generate-eval-schema.ts`, `packages/core/test/evaluation/experiment.test.ts`, `packages/core/test/evaluation/validation/eval-schema-sync.test.ts`.
 
 **Approach:** Add the repeat block as a narrow schema on the **experiment** surface (delivered by `av-991`) and normalize it into internal camelCase. **Hard-remove** `execution.trials` from `eval.yaml` as prerelease cleanup (no legacy alias); port any existing usage by hand using the mapping above.
 
 **Test Scenarios:**
 
-- Valid `repeat.runs: 3` parses into internal repeat config with no gate policy.
-- Invalid `runs`, `threshold`, `max_parallel_attempts`, and retry values are rejected or warned consistently with existing config parsing.
+- Valid `repeat.count: 3` parses into internal repeat config with no gate policy.
+- Invalid `repeat.count`, `threshold`, `max_parallel_attempts`, and retry values are rejected or warned consistently with existing config parsing.
 - `all_attempts_successful`, `any_attempt_successful`, `attempt_success_rate_at_least`, and `mean_pass_rate_at_least` validate; `mean_score_at_least` validates only if included in v1.
-- Legacy `trials` input maps or fails according to the chosen compatibility decision.
-- Generated eval schema stays in sync.
+- Legacy eval-level `trials` input fails; the compatibility path is explicit hand migration to experiment `repeat`.
+- Generated experiment schema stays in sync.
 
 **Verification:** Schema tests pass and docs/examples can reference the accepted YAML shape.
 
@@ -564,8 +567,8 @@ Repeat runs can multiply provider spend. V1 should ship with conservative contro
 | --- | --- |
 | Hidden CI behavior diverges across commands | Route every repeat-run gate through one policy evaluator and test CLI command paths against the same cases |
 | Repeat attempts inflate run counts in trend/compare views | Keep one aggregate row per case/target in top-level `index.jsonl` |
-| Early exit biases reliability reports | Default to `early_exit: never`, record early-exit mode, and label incomplete samples |
-| Existing `trials` behavior conflicts with the new contract | Decide compatibility first in `av-i0l.1`; do not keep two public names with different semantics |
+| Early exit biases reliability reports | Keep pass-at-k early exit compatible by default, require `early_exit: false` for full sampling, and label incomplete samples |
+| Existing `trials` behavior conflicts with the new contract | Hard-remove eval-level `execution.trials`; preserve its strategies and cost cap only through experiment `repeat` |
 | Attempt artifacts make rows too large | Keep only compact attempt references in `index.jsonl`; move large attempt indexes to case-local sidecars |
 | Artifact-layout migration lands concurrently | Depend on shared layout helpers and do not edit the `artifact-results-layout` branch from this worktree |
 
diff --git a/docs/plans/2026-06-23-002-experiments-separation-plan.md b/docs/plans/2026-06-23-002-experiments-separation-plan.md
index fc534b4b2..7c626ecd1 100644
--- a/docs/plans/2026-06-23-002-experiments-separation-plan.md
+++ b/docs/plans/2026-06-23-002-experiments-separation-plan.md
@@ -85,12 +85,12 @@ runtime dependency for this change.
   and target refs. Provider settings stay in `targets.yaml`.
 - KTD5. Legacy `eval.yaml execution` remains valid for released fields while
   examples migrate. The prerelease `execution.trials` field is hard-removed
-  with no alias; run count placement belongs to experiments.
+  with no alias; repeat/run-count placement belongs to experiments.
 - KTD6. AgentV adopts Vercel's experiment structure, not the package dependency,
   until a direct adapter has a smaller, reviewed boundary.
 - KTD7. Full experiment fingerprints should include the experiment file contents,
-  selected eval source, setup-relevant fields, scripts, runs, timeout, sandbox,
-  and target references.
+  selected eval source, setup-relevant fields, scripts, repeat config, timeout,
+  sandbox, and target references.
 
 ## Experiment Contract
 
@@ -112,7 +112,10 @@ scripts:
   - build
   - script: bun test
     timeout_seconds: 120
-runs: 3
+repeat:
+  count: 3
+  strategy: pass_at_k
+  cost_limit_usd: 2.00
 early_exit: false
 timeout_seconds: 900
 sandbox: auto
@@ -135,6 +138,11 @@ interface ExperimentConfig {
   agentOptions?: Record<string, unknown>;
   evals?: string | readonly string[];
   scripts?: readonly ExperimentScript[];
+  repeat?: {
+    count: number;
+    strategy: 'pass_at_k' | 'mean' | 'confidence_interval';
+    costLimitUsd?: number;
+  };
   runs?: number;
   earlyExit?: boolean;
   timeoutSeconds?: number;
@@ -183,8 +191,8 @@ Phase 2 moves examples to committed `experiments/default.yaml` files while
 leaving existing `eval.yaml execution` fields in place.
 
 Phase 3 applies experiment runtime fields in the runner: target selection, eval
-filters, timeout, runs, early exit, sandbox/workspace mode, setup steps, and
-scripts.
+filters, timeout, repeat/runs, early exit, sandbox/workspace mode, setup steps,
+and scripts.
 
 Phase 4 warns when new eval files use experiment-owned `execution` fields and
 documents the replacement.
@@ -244,9 +252,10 @@ loader independent of runner behavior.
 Test Scenarios:
 
 - YAML experiment with `agent_options`, `early_exit`, `timeout_seconds`, setup,
-  and scripts normalizes to camelCase.
+  `repeat`, and scripts normalizes to camelCase.
 - TypeScript experiment default export loads and normalizes.
-- Invalid `runs`, `timeout_seconds`, or `sandbox` fails with a targeted error.
+- Invalid `repeat.count`, `runs`, `timeout_seconds`, or `sandbox` fails with a
+  targeted error.
 - `.agentv/config.yaml` parses `experiments.default`.
 - `.agentv/config.yaml` accepts top-level `default_experiment`.
 
@@ -282,7 +291,7 @@ Files:
 - `packages/core/src/evaluation/evaluate.ts`
 - `packages/core/src/evaluation/yaml-parser.ts`
 - `packages/core/src/evaluation/loaders/config-loader.ts`
-- `packages/core/src/evaluation/validation/eval-file.schema.ts`
+- `packages/core/src/evaluation/validation/experiment-file.schema.ts`
 
 Approach:
 
@@ -353,7 +362,7 @@ branch to include native U4/U5 support for experiment resolution, eval
 selection, run knobs, setup/scripts, target reuse, artifact provenance, and the
 pre-stable hard removal of `execution.trials`.
 
-The branch now applies `setup`, `scripts`, `runs`, `early_exit`,
+The branch now applies `setup`, `scripts`, `repeat`, `runs`, `early_exit`,
 `timeout_seconds`, `target`, `targets`, and `evals` to execution behavior. Matrix
 execution via multiple experiment files remains a later phase.
 
diff --git a/examples/features/README.md b/examples/features/README.md
index f51f36c34..2e45285e4 100644
--- a/examples/features/README.md
+++ b/examples/features/README.md
@@ -82,7 +82,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 |---------|-------------|
 | [matrix-evaluation](matrix-evaluation/) | Run the same tests against multiple targets and display a score matrix |
 | [benchmark-tooling](benchmark-tooling/) | N-way benchmarking with `agentv compare` cross-model score matrix |
-| [trials](trials/) | Repeat each test N times with `pass@k` strategy to handle non-determinism |
+| [trials](trials/) | Configure experiment-level repeat runs with `pass_at_k`, `mean`, or `confidence_interval` |
 | [trial-output-consistency](trial-output-consistency/) | Measure output consistency across trials using pairwise cosine similarity |
 | [compare](compare/) | Compare a run against a stored baseline |
 
diff --git a/examples/features/trials/README.md b/examples/features/trials/README.md
new file mode 100644
index 000000000..e1a3e161a
--- /dev/null
+++ b/examples/features/trials/README.md
@@ -0,0 +1,43 @@
+# Repeat Runs
+
+This example keeps the eval file focused on the task and puts repeat/run-count
+behavior in committed experiment files.
+
+## Files
+
+- `evals/dataset.eval.yaml` defines the two task cases.
+- `experiments/default.yaml` runs the cases with `pass_at_k`.
+- `experiments/mean.yaml` aggregates repeated scores with `mean`.
+- `experiments/confidence-interval.yaml` aggregates repeated scores with a 95%
+  confidence interval lower bound.
+
+## Run
+
+```bash
+bun agentv eval examples/features/trials/evals/dataset.eval.yaml \
+  --experiment examples/features/trials/experiments/default.yaml
+```
+
+Swap the experiment path to try the other strategies.
+
+## Migration from old `execution.trials`
+
+The repeat block now lives on the experiment, not in `eval.yaml`:
+
+```yaml
+repeat:
+  count: 2
+  strategy: pass_at_k
+  cost_limit_usd: 1.00
+```
+
+Field mapping:
+
+- `execution.trials.count` -> `experiment.repeat.count`
+- `execution.trials.strategy` -> `experiment.repeat.strategy`
+- `execution.trials.cost_limit_usd` -> `experiment.repeat.cost_limit_usd`
+- `execution.trials.costLimitUsd` -> accepted only as `experiment.repeat.costLimitUsd`
+  for prerelease compatibility
+
+Use top-level experiment `early_exit: false` only when you want `pass_at_k` to
+run all attempts instead of stopping after the first pass.
diff --git a/examples/features/trials/evals/dataset.eval.yaml b/examples/features/trials/evals/dataset.eval.yaml
index 0844c2471..d7592621a 100644
--- a/examples/features/trials/evals/dataset.eval.yaml
+++ b/examples/features/trials/evals/dataset.eval.yaml
@@ -1,8 +1,8 @@
-# AgentV Trial Strategy Example
-# Demonstrates experiment-level runs for handling LLM non-determinism
+# AgentV Repeat Strategy Example
+# Demonstrates experiment-level repeat runs for handling LLM non-determinism
 
 name: trials
-description: Trial strategy example - pass@k with 2 trials
+description: Repeat strategy example - pass@k with 2 attempts configured by experiment
 
 tests:
   - id: math-basics
diff --git a/examples/features/trials/experiments/confidence-interval.yaml b/examples/features/trials/experiments/confidence-interval.yaml
new file mode 100644
index 000000000..27a9b915d
--- /dev/null
+++ b/examples/features/trials/experiments/confidence-interval.yaml
@@ -0,0 +1,9 @@
+name: trials-confidence-interval
+target: llm
+evals:
+  - math-basics
+  - capital-knowledge
+repeat:
+  count: 5
+  strategy: confidence_interval
+  cost_limit_usd: 2.00
diff --git a/examples/features/trials/experiments/default.yaml b/examples/features/trials/experiments/default.yaml
index 41ecc5147..af9dcb477 100644
--- a/examples/features/trials/experiments/default.yaml
+++ b/examples/features/trials/experiments/default.yaml
@@ -3,6 +3,7 @@ target: llm
 evals:
   - math-basics
   - capital-knowledge
-runs: 2
-early_exit: true
-budget_usd: 1.00
+repeat:
+  count: 2
+  strategy: pass_at_k
+  cost_limit_usd: 1.00
diff --git a/examples/features/trials/experiments/mean.yaml b/examples/features/trials/experiments/mean.yaml
new file mode 100644
index 000000000..1c534a8b9
--- /dev/null
+++ b/examples/features/trials/experiments/mean.yaml
@@ -0,0 +1,9 @@
+name: trials-mean
+target: llm
+evals:
+  - math-basics
+  - capital-knowledge
+repeat:
+  count: 3
+  strategy: mean
+  cost_limit_usd: 1.50
diff --git a/examples/showcase/multi-model-benchmark/README.md b/examples/showcase/multi-model-benchmark/README.md
index 1e60a690c..3ad5085ea 100644
--- a/examples/showcase/multi-model-benchmark/README.md
+++ b/examples/showcase/multi-model-benchmark/README.md
@@ -1,6 +1,6 @@
 # Multi-Model Benchmark Showcase
 
-Demonstrates a complete **multi-model × multi-metric × variability** evaluation workflow end-to-end. Run the same tests against multiple LLMs, score them on weighted metrics, measure variability with trials, and compare results.
+Demonstrates a complete **multi-model × multi-metric × variability** evaluation workflow end-to-end. Run the same tests against multiple LLMs, score them on weighted metrics, measure variability with experiment repeat runs, and compare results.
 
 ## What This Shows
 
@@ -8,7 +8,7 @@ Demonstrates a complete **multi-model × multi-metric × variability** evaluatio
 |---------|---------------|
 | **Targets matrix** | Every test runs against `copilot`, `claude`, and `gemini-llm` |
 | **Weighted graders** | Accuracy (3×), completeness (2×), clarity (1×) |
-| **Trials (pass@k)** | 2 trials per test to surface non-determinism |
+| **Repeat runs (pass@k)** | 2 attempts per test to surface non-determinism |
 | **Compare workflow** | Side-by-side model comparison from result files |
 
 ## Files
@@ -17,7 +17,9 @@ Demonstrates a complete **multi-model × multi-metric × variability** evaluatio
 multi-model-benchmark/
 ├── README.md                        # This file
 ├── evals/
-│   └── benchmark.eval.yaml          # Eval definition (targets + metrics + trials)
+│   └── benchmark.eval.yaml          # Eval definition (task cases + metrics)
+├── experiments/
+│   └── default.yaml                 # Targets, repeat policy, and run knobs
 └── prompts/
     ├── accuracy-rubric.md           # Factual correctness grader (weight 3.0)
     ├── completeness-rubric.md       # Coverage grader (weight 2.0)
@@ -34,19 +36,22 @@ multi-model-benchmark/
 From the repository root:
 
 ```bash
-# Run the full matrix (all targets × all tests × 2 trials)
-bun agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
+# Run the full matrix (all targets × all tests × 2 repeat attempts)
+bun agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml \
+  --experiment examples/showcase/multi-model-benchmark/experiments/default.yaml
 ```
 
 ### Cost & Safety
 
-The eval uses **low-cost models by default** (the targets defined in `.agentv/targets.yaml` such as `gpt-5-mini`, `claude-haiku`, `gemini-flash`). With 5 tests × 3 targets × 2 trials × 3 grader calls each, expect roughly **90 LLM calls**. A `cost_limit_usd: 2.00` cap is set in the eval file.
+The eval uses **low-cost models by default** (the targets defined in `.agentv/targets.yaml` such as `gpt-5-mini`, `claude-haiku`, `gemini-flash`). With 5 tests × 3 targets × 2 repeat attempts × 3 grader calls each, expect roughly **90 LLM calls**. A `repeat.cost_limit_usd: 2.00` cap is set in the experiment file.
 
 To run against a single target first:
 
 ```bash
 # Test with just one model before running the full matrix
-bun agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml --target copilot
+bun agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml \
+  --experiment examples/showcase/multi-model-benchmark/experiments/default.yaml \
+  --target copilot
 ```
 
 ## Comparing Models
@@ -86,20 +91,19 @@ Pairwise Summary:
   copilot → gemini-llm:  0 wins, 0 losses, 5 ties  (Δ -0.026)
 ```
 
-> **Note:** Actual scores will vary — LLM outputs are non-deterministic. The trials configuration helps surface this variability. Scores above are illustrative.
+> **Note:** Actual scores will vary — LLM outputs are non-deterministic. The experiment repeat configuration helps surface this variability. Scores above are illustrative.
 
 ## How It Works
 
 ### 1. Targets Matrix
 
-The `execution.targets` array runs every test against each listed model:
+The experiment `targets` array runs every test against each listed model:
 
 ```yaml
-execution:
-  targets:
-    - copilot       # e.g., gpt-5-mini
-    - claude        # e.g., claude-haiku
-    - gemini-llm   # e.g., gemini-flash
+targets:
+  - copilot       # e.g., gpt-5-mini
+  - claude        # e.g., claude-haiku
+  - gemini-llm   # e.g., gemini-flash
 ```
 
 ### 2. Weighted Graders
@@ -118,10 +122,10 @@ assertions:
 
 Weighted average formula: `(3×accuracy + 2×completeness + 1×clarity) / 6`
 
-### 3. Experiment runs
+### 3. Experiment repeat
 
-Each test runs twice through the committed experiment. `early_exit: true`
-matches pass@k ergonomics: a case can stop once any run succeeds.
+Each test runs twice through the committed experiment. `pass_at_k` uses early-exit
+ergonomics by default: a case can stop once any attempt succeeds.
 
 ```yaml
 targets:
@@ -133,9 +137,10 @@ evals:
   - analytical-comparison
   - creative-explanation
   - structured-list
-runs: 2
-early_exit: true
-budget_usd: 2.00
+repeat:
+  count: 2
+  strategy: pass_at_k
+  cost_limit_usd: 2.00
 ```
 
 This surfaces non-determinism — if a model passes on run 1 but fails on run 2,
@@ -159,7 +164,7 @@ benchmark.eval.yaml
         ▼
 ┌─────────────────────────┐
 │  agentv eval             │
-│  (per target × trials)  │
+│  (per target × repeat)  │
 └────────┬────────────────┘
          │
          ▼
@@ -177,15 +182,14 @@ benchmark.eval.yaml
 
 ### Adding a model
 
-Add a new target to `.agentv/targets.yaml`, then reference it in the eval:
+Add a new target to `.agentv/targets.yaml`, then reference it in the experiment:
 
 ```yaml
-execution:
-  targets:
-    - copilot
-    - claude
-    - gemini-llm
-    - my_new_model    # Add here
+targets:
+  - copilot
+  - claude
+  - gemini-llm
+  - my_new_model    # Add here
 ```
 
 ### Adding an grader
@@ -202,11 +206,13 @@ assertions:
 
 ### Adjusting run count
 
-Increase experiment `runs` for more variability data (at proportional cost):
+Increase experiment `repeat.count` for more variability data (at proportional cost):
 
 ```yaml
-runs: 5          # 5 runs for higher-confidence results
-budget_usd: 5.00
+repeat:
+  count: 5
+  strategy: pass_at_k
+  cost_limit_usd: 5.00
 ```
 
 ## See Also
diff --git a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
index b96d70480..d3bffb350 100644
--- a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
+++ b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml
@@ -3,13 +3,14 @@
 # Demonstrates multi-model × multi-metric × variability workflow:
 #   - Targets matrix: runs every test against multiple models
 #   - Weighted graders: accuracy (3×), completeness (2×), clarity (1×)
-#   - Experiment runs: pass@k with 2 runs to measure variability
+#   - Experiment repeat: pass@k with 2 attempts to measure variability
 #
 # Default targets use low-cost models for safe experimentation.
 # Override with: agentv eval ... --target <name>
 #
 # Usage:
-#   agentv eval --experiment examples/showcase/multi-model-benchmark/experiments/default.yaml
+#   agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml \
+#     --experiment examples/showcase/multi-model-benchmark/experiments/default.yaml
 
 description: Multi-model benchmark — accuracy, completeness, and clarity across models
 tags: [multi-provider]
diff --git a/examples/showcase/multi-model-benchmark/experiments/default.yaml b/examples/showcase/multi-model-benchmark/experiments/default.yaml
index 98cf3fbcb..007167966 100644
--- a/examples/showcase/multi-model-benchmark/experiments/default.yaml
+++ b/examples/showcase/multi-model-benchmark/experiments/default.yaml
@@ -9,6 +9,7 @@ evals:
   - analytical-comparison
   - creative-explanation
   - structured-list
-runs: 2
-early_exit: true
-budget_usd: 2.00
+repeat:
+  count: 2
+  strategy: pass_at_k
+  cost_limit_usd: 2.00
diff --git a/packages/core/scripts/generate-eval-schema.ts b/packages/core/scripts/generate-eval-schema.ts
index 3fc1d6d71..fa1877444 100644
--- a/packages/core/scripts/generate-eval-schema.ts
+++ b/packages/core/scripts/generate-eval-schema.ts
@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 /**
- * Generates eval-schema.json from the Zod schema.
+ * Generates AgentV JSON schemas from Zod schemas.
  * Run: bun run generate:schema (from packages/core)
  * Or:  bun packages/core/scripts/generate-eval-schema.ts (from repo root)
  */
@@ -8,25 +8,50 @@ import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { EvalFileSchema } from '../src/evaluation/validation/eval-file.schema.js';
+import { ExperimentFileSchema } from '../src/evaluation/validation/experiment-file.schema.js';
 
-const jsonSchema = zodToJsonSchema(EvalFileSchema, {
-  name: 'EvalFile',
-  $refStrategy: 'none',
-  target: 'jsonSchema2019-09',
-});
+async function writeSchema(options: {
+  readonly schema: Parameters<typeof zodToJsonSchema>[0];
+  readonly name: string;
+  readonly title: string;
+  readonly description: string;
+  readonly outputFile: string;
+}): Promise<void> {
+  const jsonSchema = zodToJsonSchema(options.schema, {
+    name: options.name,
+    $refStrategy: 'none',
+    target: 'jsonSchema2019-09',
+  });
+
+  const schema = {
+    $schema: 'https://json-schema.org/draft/2019-09/schema',
+    title: options.title,
+    description: options.description,
+    ...jsonSchema,
+  };
+
+  const outputPath = path.resolve(
+    import.meta.dirname,
+    '../../../skills-data/agentv-eval-writer/references',
+    options.outputFile,
+  );
+
+  await writeFile(outputPath, `${JSON.stringify(schema, null, 2)}\n`);
+  console.log(`Generated: ${outputPath}`);
+}
 
-// Add JSON Schema metadata
-const schema = {
-  $schema: 'https://json-schema.org/draft/2019-09/schema',
+await writeSchema({
+  schema: EvalFileSchema,
+  name: 'EvalFile',
   title: 'AgentV Eval File',
   description: 'Schema for AgentV evaluation YAML files (.eval.yaml)',
-  ...jsonSchema,
-};
-
-const outputPath = path.resolve(
-  import.meta.dirname,
-  '../../../skills-data/agentv-eval-writer/references/eval-schema.json',
-);
+  outputFile: 'eval-schema.json',
+});
 
-await writeFile(outputPath, `${JSON.stringify(schema, null, 2)}\n`);
-console.log(`Generated: ${outputPath}`);
+await writeSchema({
+  schema: ExperimentFileSchema,
+  name: 'ExperimentFile',
+  title: 'AgentV Experiment File',
+  description: 'Schema for AgentV experiment YAML files (experiments/*.yaml)',
+  outputFile: 'experiment-schema.json',
+});
diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts
index fc11295ce..2d1a512a8 100644
--- a/packages/core/src/evaluation/experiment.ts
+++ b/packages/core/src/evaluation/experiment.ts
@@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
+import type { TrialStrategy } from './types.js';
 import { parseYamlValue } from './yaml-loader.js';
 
 export type ExperimentSandbox = 'auto' | 'docker' | 'vercel';
@@ -44,6 +45,19 @@ export type ExperimentScript = {
 export type ExperimentSetupFn = (sandbox: unknown) => void | Promise<void>;
 export type ExperimentSetup = readonly ExperimentScript[] | ExperimentSetupFn;
 
+export type ExperimentRepeatWire = {
+  readonly count?: number;
+  readonly strategy?: TrialStrategy;
+  readonly cost_limit_usd?: number;
+  readonly costLimitUsd?: number;
+};
+
+export type ExperimentRepeat = {
+  readonly count: number;
+  readonly strategy: TrialStrategy;
+  readonly costLimitUsd?: number;
+};
+
 export type ExperimentConfigWire = {
   readonly name?: string;
   readonly agent?: string;
@@ -53,6 +67,7 @@ export type ExperimentConfigWire = {
   readonly agent_options?: Record<string, unknown>;
   readonly evals?: string | readonly string[];
   readonly scripts?: readonly ExperimentScriptWire[];
+  readonly repeat?: ExperimentRepeatWire;
   readonly runs?: number;
   readonly early_exit?: boolean;
   readonly timeout_seconds?: number;
@@ -72,6 +87,7 @@ export type ExperimentConfig = {
   readonly agentOptions?: Record<string, unknown>;
   readonly evals?: string | readonly string[];
   readonly scripts?: readonly ExperimentScript[];
+  readonly repeat?: ExperimentRepeat;
   readonly runs?: number;
   readonly earlyExit?: boolean;
   readonly timeoutSeconds?: number;
@@ -93,6 +109,11 @@ export type ExperimentArtifactMetadata = {
   readonly targets?: readonly string[];
   readonly model?: string;
   readonly evals?: string | readonly string[];
+  readonly repeat?: {
+    readonly count: number;
+    readonly strategy: TrialStrategy;
+    readonly cost_limit_usd?: number;
+  };
   readonly runs?: number;
   readonly early_exit?: boolean;
   readonly timeout_seconds?: number;
@@ -107,6 +128,11 @@ type NormalizeOptions = {
 
 const EXPERIMENT_FILE_EXTENSIONS = new Set(['.yaml', '.yml', '.ts', '.js', '.mts', '.mjs']);
 const VALID_SANDBOXES: ReadonlySet<string> = new Set(['auto', 'docker', 'vercel']);
+const VALID_REPEAT_STRATEGIES: ReadonlySet<string> = new Set([
+  'pass_at_k',
+  'mean',
+  'confidence_interval',
+]);
 
 export function isExperimentFileReference(value: string): boolean {
   const trimmed = value.trim();
@@ -163,7 +189,11 @@ export function normalizeExperimentConfig(
   const agentOptions = readOptionalRecord(rawConfig.agent_options ?? rawConfig.agentOptions);
   const evals = readOptionalStringOrStringArray(rawConfig.evals, 'evals');
   const scripts = readScriptArray(rawConfig.scripts, 'scripts');
+  const repeat = readRepeat(rawConfig.repeat);
   const runs = readOptionalPositiveInteger(rawConfig.runs, 'runs');
+  if (repeat !== undefined && runs !== undefined) {
+    throw new Error('Experiment repeat and runs cannot both be set. Use repeat for AgentV config.');
+  }
   const earlyExit = readOptionalBoolean(rawConfig.early_exit ?? rawConfig.earlyExit, 'early_exit');
   const timeoutSeconds = readOptionalPositiveNumber(
     rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds,
@@ -187,6 +217,7 @@ export function normalizeExperimentConfig(
     ...(agentOptions !== undefined && { agentOptions }),
     ...(evals !== undefined && { evals }),
     ...(scripts !== undefined && { scripts }),
+    ...(repeat !== undefined && { repeat }),
     ...(runs !== undefined && { runs }),
     ...(earlyExit !== undefined && { earlyExit }),
     ...(timeoutSeconds !== undefined && { timeoutSeconds }),
@@ -227,6 +258,15 @@ export function buildExperimentArtifactMetadata(
     ...(targets && targets.length > 0 && { targets }),
     ...(config.model !== undefined && { model: config.model }),
     ...(config.evals !== undefined && { evals: config.evals }),
+    ...(config.repeat !== undefined && {
+      repeat: {
+        count: config.repeat.count,
+        strategy: config.repeat.strategy,
+        ...(config.repeat.costLimitUsd !== undefined && {
+          cost_limit_usd: config.repeat.costLimitUsd,
+        }),
+      },
+    }),
     ...(config.runs !== undefined && { runs: config.runs }),
     ...(config.earlyExit !== undefined && { early_exit: config.earlyExit }),
     ...(config.timeoutSeconds !== undefined && { timeout_seconds: config.timeoutSeconds }),
@@ -236,6 +276,27 @@ export function buildExperimentArtifactMetadata(
   };
 }
 
+function readRepeat(raw: unknown): ExperimentRepeat | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (!isRecord(raw)) {
+    throw new Error('Experiment repeat must be an object.');
+  }
+  const count = readRequiredPositiveInteger(raw.count, 'repeat.count');
+  const strategy = readOptionalRepeatStrategy(raw.strategy);
+  const costLimitUsd = readOptionalNonNegativeNumber(
+    raw.cost_limit_usd ?? raw.costLimitUsd,
+    'repeat.cost_limit_usd',
+  );
+
+  return {
+    count,
+    strategy: strategy ?? 'pass_at_k',
+    ...(costLimitUsd !== undefined && { costLimitUsd }),
+  };
+}
+
 function readTargets(raw: unknown): readonly ExperimentTargetRef[] | undefined {
   if (raw === undefined) {
     return undefined;
@@ -381,6 +442,14 @@ function readRequiredString(raw: unknown, location: string): string {
   return value;
 }
 
+function readRequiredPositiveInteger(raw: unknown, location: string): number {
+  const value = readOptionalPositiveInteger(raw, location);
+  if (value === undefined) {
+    throw new Error(`Experiment ${location} is required.`);
+  }
+  return value;
+}
+
 function readOptionalBoolean(raw: unknown, location: string): boolean | undefined {
   if (raw === undefined) {
     return undefined;
@@ -391,6 +460,18 @@ function readOptionalBoolean(raw: unknown, location: string): boolean | undefine
   return raw;
 }
 
+function readOptionalRepeatStrategy(raw: unknown): TrialStrategy | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'string' || !VALID_REPEAT_STRATEGIES.has(raw)) {
+    throw new Error(
+      "Experiment repeat.strategy must be one of 'pass_at_k', 'mean', or 'confidence_interval'.",
+    );
+  }
+  return raw as TrialStrategy;
+}
+
 function readOptionalPositiveInteger(raw: unknown, location: string): number | undefined {
   if (raw === undefined) {
     return undefined;
@@ -401,6 +482,16 @@ function readOptionalPositiveInteger(raw: unknown, location: string): number | u
   return raw;
 }
 
+function readOptionalNonNegativeNumber(raw: unknown, location: string): number | undefined {
+  if (raw === undefined) {
+    return undefined;
+  }
+  if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) {
+    throw new Error(`Experiment ${location} must be a non-negative number.`);
+  }
+  return raw;
+}
+
 function readOptionalPositiveNumber(raw: unknown, location: string): number | undefined {
   if (raw === undefined) {
     return undefined;
diff --git a/packages/core/src/evaluation/validation/experiment-file.schema.ts b/packages/core/src/evaluation/validation/experiment-file.schema.ts
new file mode 100644
index 000000000..8a608753d
--- /dev/null
+++ b/packages/core/src/evaluation/validation/experiment-file.schema.ts
@@ -0,0 +1,74 @@
+/**
+ * Zod schema for experiment YAML file format.
+ * Used to generate experiment-schema.json for AI agent reference.
+ *
+ * IMPORTANT: This schema describes the YAML input format, not the parsed runtime types.
+ * Wire fields are snake_case. The only camelCase field accepted here is
+ * repeat.costLimitUsd, kept for parity with the prerelease trials schema.
+ */
+import { z } from 'zod';
+
+const JsonObjectSchema = z.object({}).catchall(z.unknown());
+
+const StringOrStringArraySchema = z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]);
+
+const ExperimentScriptSchema = z.union([
+  z.string().min(1),
+  z
+    .object({
+      command: StringOrStringArraySchema.optional(),
+      script: StringOrStringArraySchema.optional(),
+      timeout_seconds: z.number().gt(0).optional(),
+      cwd: z.string().min(1).optional(),
+      env: z.record(z.string()).optional(),
+    })
+    .strict()
+    .refine((value) => value.command !== undefined || value.script !== undefined, {
+      message: 'Experiment step must define command or script.',
+    }),
+]);
+
+const ExperimentRepeatSchema = z
+  .object({
+    count: z.number().int().min(1),
+    strategy: z.enum(['pass_at_k', 'mean', 'confidence_interval']).optional(),
+    cost_limit_usd: z.number().min(0).optional(),
+    costLimitUsd: z.number().min(0).optional(),
+  })
+  .strict();
+
+const ExperimentTargetRefSchema = z.union([
+  z.string().min(1),
+  z
+    .object({
+      name: z.string().min(1),
+      use_target: z.string().min(1).optional(),
+      hooks: JsonObjectSchema.optional(),
+    })
+    .strict(),
+]);
+
+export const ExperimentFileSchema = z
+  .object({
+    name: z.string().min(1).optional(),
+    agent: z.string().min(1).optional(),
+    target: z.string().min(1).optional(),
+    targets: z.array(ExperimentTargetRefSchema).min(1).optional(),
+    model: z.string().min(1).optional(),
+    agent_options: JsonObjectSchema.optional(),
+    evals: StringOrStringArraySchema.optional(),
+    scripts: z.array(ExperimentScriptSchema).optional(),
+    repeat: ExperimentRepeatSchema.optional(),
+    runs: z.number().int().min(1).optional(),
+    early_exit: z.boolean().optional(),
+    timeout_seconds: z.number().gt(0).optional(),
+    workers: z.number().int().min(1).optional(),
+    budget_usd: z.number().gt(0).optional(),
+    sandbox: z.enum(['auto', 'docker', 'vercel']).optional(),
+    workspace: JsonObjectSchema.optional(),
+    setup: z.array(ExperimentScriptSchema).optional(),
+  })
+  .strict()
+  .refine((value) => value.repeat === undefined || value.runs === undefined, {
+    message: 'Use repeat or runs, not both.',
+  });
diff --git a/packages/core/src/evaluation/validation/index.ts b/packages/core/src/evaluation/validation/index.ts
index 23093ae0b..917715615 100644
--- a/packages/core/src/evaluation/validation/index.ts
+++ b/packages/core/src/evaluation/validation/index.ts
@@ -3,6 +3,7 @@
  */
 
 export { detectFileType, isValidSchema, getExpectedSchema } from './file-type.js';
+export { ExperimentFileSchema } from './experiment-file.schema.js';
 export { validateEvalFile } from './eval-validator.js';
 export { validateCasesFile } from './cases-validator.js';
 export { validateTargetsFile } from './targets-validator.js';
diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts
index 5a3afc22c..8b3dd4c72 100644
--- a/packages/core/test/evaluation/experiment.test.ts
+++ b/packages/core/test/evaluation/experiment.test.ts
@@ -49,6 +49,37 @@ describe('experiment config', () => {
     expect(config.fingerprint).toMatch(/^[a-f0-9]{64}$/);
   });
 
+  it('normalizes experiment-level repeat config with legacy trial strategy parity', () => {
+    const config = normalizeExperimentConfig({
+      repeat: {
+        count: 4,
+        strategy: 'confidence_interval',
+        cost_limit_usd: 0,
+      },
+    });
+
+    expect(config.repeat).toEqual({
+      count: 4,
+      strategy: 'confidence_interval',
+      costLimitUsd: 0,
+    });
+  });
+
+  it('accepts the prerelease trials costLimitUsd spelling only inside repeat', () => {
+    const config = normalizeExperimentConfig({
+      repeat: {
+        count: 2,
+        costLimitUsd: 1.5,
+      },
+    });
+
+    expect(config.repeat).toEqual({
+      count: 2,
+      strategy: 'pass_at_k',
+      costLimitUsd: 1.5,
+    });
+  });
+
   it('loads a YAML experiment file', async () => {
     const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-experiment-'));
     try {
@@ -81,6 +112,16 @@ describe('experiment config', () => {
 
   it('rejects invalid run counts and sandbox values', () => {
     expect(() => normalizeExperimentConfig({ runs: 0 })).toThrow(/runs/);
+    expect(() => normalizeExperimentConfig({ repeat: {} })).toThrow(/repeat.count/);
+    expect(() =>
+      normalizeExperimentConfig({ repeat: { count: 2, strategy: 'median' } }),
+    ).toThrow(/repeat.strategy/);
+    expect(() =>
+      normalizeExperimentConfig({ repeat: { count: 2, cost_limit_usd: -1 } }),
+    ).toThrow(/repeat.cost_limit_usd/);
+    expect(() => normalizeExperimentConfig({ repeat: { count: 2 }, runs: 2 })).toThrow(
+      /repeat and runs/,
+    );
     expect(() => normalizeExperimentConfig({ sandbox: 'host' })).toThrow(/sandbox/);
   });
 
@@ -91,11 +132,10 @@ describe('experiment config', () => {
       agent_options: { secret: 'not persisted' },
       setup: [{ script: 'bun install' }],
       scripts: [{ script: 'bun test' }],
-      runs: 2,
+      repeat: { count: 2, strategy: 'mean', cost_limit_usd: 0.5 },
       early_exit: true,
       timeout_seconds: 120,
       workers: 3,
-      budget_usd: 0.5,
     });
 
     const metadata = buildExperimentArtifactMetadata(config);
@@ -103,11 +143,14 @@ describe('experiment config', () => {
     expect(metadata).toMatchObject({
       name: 'baseline',
       target: 'codex',
-      runs: 2,
+      repeat: {
+        count: 2,
+        strategy: 'mean',
+        cost_limit_usd: 0.5,
+      },
       early_exit: true,
       timeout_seconds: 120,
       workers: 3,
-      budget_usd: 0.5,
     });
     expect(metadata).not.toHaveProperty('agent_options');
     expect(metadata).not.toHaveProperty('setup');
diff --git a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts
index 2acb8a453..7db6c29a9 100644
--- a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts
+++ b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts
@@ -3,9 +3,10 @@ import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { EvalFileSchema } from '../../../src/evaluation/validation/eval-file.schema.js';
+import { ExperimentFileSchema } from '../../../src/evaluation/validation/experiment-file.schema.js';
 
-describe('eval-schema.json sync', () => {
-  it('matches the generated schema from Zod', async () => {
+describe('generated schema sync', () => {
+  it('keeps eval-schema.json synced with the Zod schema', async () => {
     const repoRoot = path.resolve(import.meta.dirname, '../../../../..');
     const schemaPath = path.join(
       repoRoot,
@@ -32,4 +33,28 @@ describe('eval-schema.json sync', () => {
     // Compare (ignoring formatting differences)
     expect(JSON.parse(JSON.stringify(committed))).toEqual(JSON.parse(JSON.stringify(expected)));
   });
+
+  it('keeps experiment-schema.json synced with the Zod schema', async () => {
+    const repoRoot = path.resolve(import.meta.dirname, '../../../../..');
+    const schemaPath = path.join(
+      repoRoot,
+      'skills-data/agentv-eval-writer/references/experiment-schema.json',
+    );
+
+    const committed = JSON.parse(await readFile(schemaPath, 'utf8'));
+    const generated = zodToJsonSchema(ExperimentFileSchema, {
+      name: 'ExperimentFile',
+      $refStrategy: 'none',
+      target: 'jsonSchema2019-09',
+    });
+
+    const expected = {
+      $schema: 'https://json-schema.org/draft/2019-09/schema',
+      title: 'AgentV Experiment File',
+      description: 'Schema for AgentV experiment YAML files (experiments/*.yaml)',
+      ...generated,
+    };
+
+    expect(JSON.parse(JSON.stringify(committed))).toEqual(JSON.parse(JSON.stringify(expected)));
+  });
 });
diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md
index ca57ac09f..950791bff 100644
--- a/skills-data/agentv-eval-writer/SKILL.md
+++ b/skills-data/agentv-eval-writer/SKILL.md
@@ -18,6 +18,11 @@ Comprehensive docs: https://agentv.dev
 
 Treat YAML as the canonical portable model. Prefer authoring `.eval.yaml` / `EVAL.yaml` first, then use TypeScript helpers, Python scripts, or executable graders only when they lower to the same fields or when the evaluation logic must actually run code.
 
+Eval files define what is tested: prompts, datasets, assertions, and task fixtures.
+Experiment files define how those evals run: targets, setup, scripts, timeout,
+sandbox, and repeat-run policy. Use `experiments/*.yaml` for committed run
+configurations.
+
 Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it was a deprecated compatibility package and has been removed from this repository.
 
 ## Evaluation Types
@@ -759,6 +764,7 @@ Do not invent a separate Opik-specific eval surface. Keep the eval definition in
 ## Schemas
 
 - Eval file: `references/eval-schema.json`
+- Experiment file: `references/experiment-schema.json`
 - Config: `references/config-schema.json`
 
 ## Accessing reference files
@@ -767,12 +773,14 @@ To load a specific reference without pulling the entire skill into context:
 
 ```bash
 agentv skills get agentv-eval-writer --ref eval-schema.json
+agentv skills get agentv-eval-writer --ref experiment-schema.json
 ```
 
 Or resolve the skill directory and read files directly:
 
 ```bash
 cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json
+cat $(agentv skills path agentv-eval-writer)/references/experiment-schema.json
 ```
 
 Use `--full` to retrieve every file in the skill at once.
diff --git a/skills-data/agentv-eval-writer/references/experiment-schema.json b/skills-data/agentv-eval-writer/references/experiment-schema.json
new file mode 100644
index 000000000..cbef02aff
--- /dev/null
+++ b/skills-data/agentv-eval-writer/references/experiment-schema.json
@@ -0,0 +1,276 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema#",
+  "title": "AgentV Experiment File",
+  "description": "Schema for AgentV experiment YAML files (experiments/*.yaml)",
+  "$ref": "#/definitions/ExperimentFile",
+  "definitions": {
+    "ExperimentFile": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "agent": {
+          "type": "string",
+          "minLength": 1
+        },
+        "target": {
+          "type": "string",
+          "minLength": 1
+        },
+        "targets": {
+          "type": "array",
+          "items": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "use_target": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "hooks": {
+                    "type": "object",
+                    "properties": {},
+                    "additionalProperties": {}
+                  }
+                },
+                "required": [
+                  "name"
+                ],
+                "additionalProperties": false
+              }
+            ]
+          },
+          "minItems": 1
+        },
+        "model": {
+          "type": "string",
+          "minLength": 1
+        },
+        "agent_options": {
+          "type": "object",
+          "properties": {},
+          "additionalProperties": {}
+        },
+        "evals": {
+          "anyOf": [
+            {
+              "type": "string",
+              "minLength": 1
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              },
+              "minItems": 1
+            }
+          ]
+        },
+        "scripts": {
+          "type": "array",
+          "items": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "command": {
+                    "anyOf": [
+                      {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "string",
+                          "minLength": 1
+                        },
+                        "minItems": 1
+                      }
+                    ]
+                  },
+                  "script": {
+                    "anyOf": [
+                      {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "string",
+                          "minLength": 1
+                        },
+                        "minItems": 1
+                      }
+                    ]
+                  },
+                  "timeout_seconds": {
+                    "type": "number",
+                    "exclusiveMinimum": true,
+                    "minimum": 0
+                  },
+                  "cwd": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "env": {
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "additionalProperties": false
+              }
+            ]
+          }
+        },
+        "repeat": {
+          "type": "object",
+          "properties": {
+            "count": {
+              "type": "integer",
+              "minimum": 1
+            },
+            "strategy": {
+              "type": "string",
+              "enum": [
+                "pass_at_k",
+                "mean",
+                "confidence_interval"
+              ]
+            },
+            "cost_limit_usd": {
+              "type": "number",
+              "minimum": 0
+            },
+            "costLimitUsd": {
+              "type": "number",
+              "minimum": 0
+            }
+          },
+          "required": [
+            "count"
+          ],
+          "additionalProperties": false
+        },
+        "runs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "early_exit": {
+          "type": "boolean"
+        },
+        "timeout_seconds": {
+          "type": "number",
+          "exclusiveMinimum": true,
+          "minimum": 0
+        },
+        "workers": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "budget_usd": {
+          "type": "number",
+          "exclusiveMinimum": true,
+          "minimum": 0
+        },
+        "sandbox": {
+          "type": "string",
+          "enum": [
+            "auto",
+            "docker",
+            "vercel"
+          ]
+        },
+        "workspace": {
+          "type": "object",
+          "properties": {},
+          "additionalProperties": {}
+        },
+        "setup": {
+          "type": "array",
+          "items": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "command": {
+                    "anyOf": [
+                      {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "string",
+                          "minLength": 1
+                        },
+                        "minItems": 1
+                      }
+                    ]
+                  },
+                  "script": {
+                    "anyOf": [
+                      {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "string",
+                          "minLength": 1
+                        },
+                        "minItems": 1
+                      }
+                    ]
+                  },
+                  "timeout_seconds": {
+                    "type": "number",
+                    "exclusiveMinimum": true,
+                    "minimum": 0
+                  },
+                  "cwd": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "env": {
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "additionalProperties": false
+              }
+            ]
+          }
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}

From b242160c327b712aa6e01fca39144cef646aa76e Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 15:08:29 +0200
Subject: [PATCH 07/18] chore: format experiment repeat test

---
 packages/core/test/evaluation/experiment.test.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts
index 8b3dd4c72..be90638e7 100644
--- a/packages/core/test/evaluation/experiment.test.ts
+++ b/packages/core/test/evaluation/experiment.test.ts
@@ -113,12 +113,12 @@ describe('experiment config', () => {
   it('rejects invalid run counts and sandbox values', () => {
     expect(() => normalizeExperimentConfig({ runs: 0 })).toThrow(/runs/);
     expect(() => normalizeExperimentConfig({ repeat: {} })).toThrow(/repeat.count/);
-    expect(() =>
-      normalizeExperimentConfig({ repeat: { count: 2, strategy: 'median' } }),
-    ).toThrow(/repeat.strategy/);
-    expect(() =>
-      normalizeExperimentConfig({ repeat: { count: 2, cost_limit_usd: -1 } }),
-    ).toThrow(/repeat.cost_limit_usd/);
+    expect(() => normalizeExperimentConfig({ repeat: { count: 2, strategy: 'median' } })).toThrow(
+      /repeat.strategy/,
+    );
+    expect(() => normalizeExperimentConfig({ repeat: { count: 2, cost_limit_usd: -1 } })).toThrow(
+      /repeat.cost_limit_usd/,
+    );
     expect(() => normalizeExperimentConfig({ repeat: { count: 2 }, runs: 2 })).toThrow(
       /repeat and runs/,
     );

From 544981e8a750b9c28c0cd9a8a3ba2cd55959d60d Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Tue, 23 Jun 2026 15:10:26 +0200
Subject: [PATCH 08/18] chore: format experiment schema

---
 .../references/experiment-schema.json         | 20 ++++---------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/skills-data/agentv-eval-writer/references/experiment-schema.json b/skills-data/agentv-eval-writer/references/experiment-schema.json
index cbef02aff..481dafc33 100644
--- a/skills-data/agentv-eval-writer/references/experiment-schema.json
+++ b/skills-data/agentv-eval-writer/references/experiment-schema.json
@@ -44,9 +44,7 @@
                     "additionalProperties": {}
                   }
                 },
-                "required": [
-                  "name"
-                ],
+                "required": ["name"],
                 "additionalProperties": false
               }
             ]
@@ -151,11 +149,7 @@
             },
             "strategy": {
               "type": "string",
-              "enum": [
-                "pass_at_k",
-                "mean",
-                "confidence_interval"
-              ]
+              "enum": ["pass_at_k", "mean", "confidence_interval"]
             },
             "cost_limit_usd": {
               "type": "number",
@@ -166,9 +160,7 @@
               "minimum": 0
             }
           },
-          "required": [
-            "count"
-          ],
+          "required": ["count"],
           "additionalProperties": false
         },
         "runs": {
@@ -194,11 +186,7 @@
         },
         "sandbox": {
           "type": "string",
-          "enum": [
-            "auto",
-            "docker",
-            "vercel"
-          ]
+          "enum": ["auto", "docker", "vercel"]
         },
         "workspace": {
           "type": "object",

From 2c2505d96966be5a1c8cf584c3768c9b6ee7cba8 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 01:31:02 +0200
Subject: [PATCH 09/18] fix: preserve repeat trial artifact metadata

---
 .../commands/eval/artifact-writer.test.ts     | 128 ++++++++++++++++++
 packages/core/src/evaluation/run-artifacts.ts | 100 +++++++++++++-
 2 files changed, 227 insertions(+), 1 deletion(-)

diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 1d05c3e6f..6b3b3531a 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -142,6 +142,56 @@ describe('buildGradingArtifact', () => {
     });
   });
 
+  it('preserves repeat trial metadata', () => {
+    const result = makeResult({
+      trials: [
+        {
+          attempt: 0,
+          score: 0.4,
+          verdict: 'fail',
+          executionStatus: 'quality_failure',
+          failureStage: 'evaluator',
+          failureReasonCode: 'threshold_not_met',
+        },
+        {
+          attempt: 1,
+          score: 1,
+          verdict: 'pass',
+          costUsd: 0.03,
+        },
+      ],
+      aggregation: {
+        strategy: 'pass_at_k',
+        passedAttempts: 1,
+        totalAttempts: 2,
+      },
+    });
+
+    const grading = buildGradingArtifact(result);
+
+    expect(grading.trials).toEqual([
+      {
+        attempt: 0,
+        score: 0.4,
+        verdict: 'fail',
+        execution_status: 'quality_failure',
+        failure_stage: 'evaluator',
+        failure_reason_code: 'threshold_not_met',
+      },
+      {
+        attempt: 1,
+        score: 1,
+        verdict: 'pass',
+        cost_usd: 0.03,
+      },
+    ]);
+    expect(grading.aggregation).toEqual({
+      strategy: 'pass_at_k',
+      passed_attempts: 1,
+      total_attempts: 2,
+    });
+  });
+
   it('uses top-level assertions when no grader scores', () => {
     const result = makeResult({
       assertions: [
@@ -563,6 +613,40 @@ describe('buildIndexArtifactEntry', () => {
       input_path: 'alpha/input.md',
     });
   });
+
+  it('includes repeat trial metadata', () => {
+    const entry = buildIndexArtifactEntry(
+      makeResult({
+        testId: 'alpha',
+        trials: [
+          { attempt: 0, score: 0.8, verdict: 'pass' },
+          { attempt: 1, score: 0.6, verdict: 'fail', error: 'missing token' },
+        ],
+        aggregation: {
+          strategy: 'mean',
+          mean: 0.7,
+          min: 0.6,
+          max: 0.8,
+        },
+      }),
+      {
+        outputDir: '/tmp/artifacts',
+        gradingPath: '/tmp/artifacts/alpha/grading.json',
+        timingPath: '/tmp/artifacts/alpha/timing.json',
+      },
+    );
+
+    expect(entry.trials).toEqual([
+      { attempt: 0, score: 0.8, verdict: 'pass' },
+      { attempt: 1, score: 0.6, verdict: 'fail', error: 'missing token' },
+    ]);
+    expect(entry.aggregation).toEqual({
+      strategy: 'mean',
+      mean: 0.7,
+      min: 0.6,
+      max: 0.8,
+    });
+  });
 });
 
 // ---------------------------------------------------------------------------
@@ -819,6 +903,50 @@ describe('writeArtifactsFromResults', () => {
     expect(indexLines[0]?.metrics_path).toBe('alpha/metrics.json');
   });
 
+  it('writes repeat trial metadata to grading and index artifacts', async () => {
+    const results = [
+      makeResult({
+        testId: 'repeat-case',
+        score: 1,
+        trials: [
+          { attempt: 0, score: 0.25, verdict: 'fail' },
+          { attempt: 1, score: 1, verdict: 'pass' },
+        ],
+        aggregation: {
+          strategy: 'confidence_interval',
+          mean: 0.625,
+          ci95Lower: 0.1,
+          ci95Upper: 1,
+          stddev: 0.53,
+        },
+      }),
+    ];
+
+    const paths = await writeArtifactsFromResults(results, testDir);
+
+    const grading: GradingArtifact = JSON.parse(
+      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'grading.json'), 'utf8'),
+    );
+    expect(grading.trials).toEqual([
+      { attempt: 0, score: 0.25, verdict: 'fail' },
+      { attempt: 1, score: 1, verdict: 'pass' },
+    ]);
+    expect(grading.aggregation).toEqual({
+      strategy: 'confidence_interval',
+      mean: 0.625,
+      ci95_lower: 0.1,
+      ci95_upper: 1,
+      stddev: 0.53,
+    });
+
+    const [indexEntry] = (await readFile(paths.indexPath, 'utf8'))
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line) as IndexArtifactEntry);
+    expect(indexEntry?.trials).toEqual(grading.trials);
+    expect(indexEntry?.aggregation).toEqual(grading.aggregation);
+  });
+
   it('handles empty results array', async () => {
     const paths = await writeArtifactsFromResults([], testDir);
 
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 46a0ce407..9ebebb52d 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -54,7 +54,13 @@ import {
   traceEnvelopeToTranscriptMessages,
 } from './trace-envelope.js';
 import { type TokenUsage, type TraceSummary, buildTraceFromMessages } from './trace.js';
-import type { EvalTest, EvaluationResult, GraderResult } from './types.js';
+import type {
+  EvalTest,
+  EvaluationResult,
+  GraderResult,
+  TrialAggregation,
+  TrialResult,
+} from './types.js';
 
 export const RESULT_INDEX_FILENAME = 'index.jsonl';
 
@@ -167,8 +173,42 @@ export interface GradingArtifact {
     readonly turns: number;
     readonly conversation_id: string;
   };
+  readonly trials?: readonly TrialResultArtifact[];
+  readonly aggregation?: TrialAggregationArtifact;
 }
 
+export type TrialResultArtifact = {
+  readonly attempt: number;
+  readonly score: number;
+  readonly verdict: string;
+  readonly scores?: IndexArtifactEntry['scores'];
+  readonly error?: string;
+  readonly cost_usd?: number;
+  readonly execution_status?: string;
+  readonly failure_stage?: string;
+  readonly failure_reason_code?: string;
+};
+
+export type TrialAggregationArtifact =
+  | {
+      readonly strategy: 'pass_at_k';
+      readonly passed_attempts: number;
+      readonly total_attempts: number;
+    }
+  | {
+      readonly strategy: 'mean';
+      readonly mean: number;
+      readonly min: number;
+      readonly max: number;
+    }
+  | {
+      readonly strategy: 'confidence_interval';
+      readonly mean: number;
+      readonly ci95_lower: number;
+      readonly ci95_upper: number;
+      readonly stddev: number;
+    };
+
 export interface TimingArtifact {
   readonly total_tokens: number;
   readonly duration_ms: number;
@@ -241,6 +281,8 @@ export interface IndexArtifactEntry {
   readonly start_time?: string;
   readonly end_time?: string;
   readonly scores?: readonly Record<string, unknown>[];
+  readonly trials?: readonly TrialResultArtifact[];
+  readonly aggregation?: TrialAggregationArtifact;
   readonly execution_status?: string;
   readonly error?: string;
   readonly failure_stage?: string;
@@ -425,6 +467,56 @@ function toIndexScores(scores: readonly GraderResult[] | undefined): IndexArtifa
   return scores?.map(toIndexScore) as IndexArtifactEntry['scores'];
 }
 
+function toTrialArtifacts(
+  trials: readonly TrialResult[] | undefined,
+): readonly TrialResultArtifact[] | undefined {
+  if (!trials || trials.length === 0) {
+    return undefined;
+  }
+  return trials.map((trial) => ({
+    attempt: trial.attempt,
+    score: trial.score,
+    verdict: trial.verdict,
+    scores: toIndexScores(trial.scores),
+    error: trial.error,
+    cost_usd: trial.costUsd,
+    execution_status: trial.executionStatus,
+    failure_stage: trial.failureStage,
+    failure_reason_code: trial.failureReasonCode,
+  }));
+}
+
+function toTrialAggregationArtifact(
+  aggregation: TrialAggregation | undefined,
+): TrialAggregationArtifact | undefined {
+  if (!aggregation) {
+    return undefined;
+  }
+  switch (aggregation.strategy) {
+    case 'pass_at_k':
+      return {
+        strategy: aggregation.strategy,
+        passed_attempts: aggregation.passedAttempts,
+        total_attempts: aggregation.totalAttempts,
+      };
+    case 'mean':
+      return {
+        strategy: aggregation.strategy,
+        mean: aggregation.mean,
+        min: aggregation.min,
+        max: aggregation.max,
+      };
+    case 'confidence_interval':
+      return {
+        strategy: aggregation.strategy,
+        mean: aggregation.mean,
+        ci95_lower: aggregation.ci95Lower,
+        ci95_upper: aggregation.ci95Upper,
+        stddev: aggregation.stddev,
+      };
+  }
+}
+
 function dropUndefined(value: Record<string, unknown>): Record<string, unknown> {
   return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
 }
@@ -548,6 +640,8 @@ export function buildGradingArtifact(result: EvaluationResult): GradingArtifact
           conversation_id: result.conversationId,
         }
       : undefined,
+    trials: toTrialArtifacts(result.trials),
+    aggregation: toTrialAggregationArtifact(result.aggregation),
   };
 }
 
@@ -1065,6 +1159,8 @@ export function buildIndexArtifactEntry(
     start_time: result.startTime,
     end_time: result.endTime,
     scores: toIndexScores(result.scores),
+    trials: toTrialArtifacts(result.trials),
+    aggregation: toTrialAggregationArtifact(result.aggregation),
     execution_status: result.executionStatus,
     error: result.error,
     failure_stage: result.failureStage,
@@ -1139,6 +1235,8 @@ export function buildResultIndexArtifact(
     start_time: result.startTime,
     end_time: result.endTime,
     scores: toIndexScores(result.scores),
+    trials: toTrialArtifacts(result.trials),
+    aggregation: toTrialAggregationArtifact(result.aggregation),
     execution_status: result.executionStatus,
     error: result.error,
     failure_stage: result.failureStage,

From a90002d6e1deac5cc5b72cb6dc91847d84ab5b7d Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 01:51:12 +0200
Subject: [PATCH 10/18] docs: require live experiment dogfood

---
 .agents/verification.md | 12 ++++++++++++
 AGENTS.md               |  1 +
 2 files changed, 13 insertions(+)

diff --git a/.agents/verification.md b/.agents/verification.md
index 6dbff41b4..0111704af 100644
--- a/.agents/verification.md
+++ b/.agents/verification.md
@@ -120,6 +120,18 @@ bun apps/cli/src/cli.ts eval examples/features/rubric/evals/dataset.eval.yaml --
 4. Update baseline files if output format changes. Baselines live next to eval YAML files as `*.baseline.jsonl`.
 5. `--dry-run` returns schema-valid mock responses, but the scores are not meaningful. Use it only for plumbing and harness checks.
 
+## Live Dogfood for Eval and Experiment Changes
+
+Use live dogfood before marking PRs ready when they affect eval execution, experiments, repeat runs, targets, providers, graders, or artifact provenance.
+
+- Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
+- Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
+- For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
+- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted, and inspect both `index.jsonl` and per-test `grading.json` for `trials[]` and `aggregation`.
+- For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
+- Preserve review evidence in `agentv-private` on an `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, and screenshots when folder structure or UI behavior is under review.
+- If comparing against an external convention such as Vercel `agent-eval`, state whether AgentV matches the physical folder layout or only the semantic provenance contract. Do not imply `run-N` directory parity unless the artifacts actually use that layout.
+
 ## Checking Grader Score Ranges
 
 Use `scripts/check-grader-scores.ts` as a post-processor after an eval run.
diff --git a/AGENTS.md b/AGENTS.md
index 110f0d3b5..8f71e9619 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -43,6 +43,7 @@ Read the full rationale and examples in [.agents/product-boundary.md](.agents/pr
 - Non-trivial work needs a plan or task list. If the implementation surface starts to balloon, stop and re-plan.
 - Large or high-risk PRs need meaningful, reviewable commits for each coherent change. Rewrite only the PR branch with `git push --force-with-lease` when needed to replace WIP or accidental squashed history before review.
 - Manual red/green UAT is blocking before a branch is ready for review. GitHub Actions is the authoritative merge gate.
+- For eval, experiment, provider, or grader changes, dogfood with a live agent target and a live grader target before calling the PR ready. Mock graders, dry-run, and deterministic-only smoke tests are useful plumbing checks, but they are not live dogfood.
 - For browser or screenshot UAT, keep evidence out of the public repo and publish reviewable artifacts to an `agentv-private` evidence branch. See [.agents/verification.md](.agents/verification.md).
 - Wire formats are `snake_case`; internal TypeScript is `camelCase`. Translate only at the boundary.
 - In AgentV, a `project` holds runs, traces, and experiments; a `benchmark` is a curated eval suite. Do not collapse those terms.

From 7b245a2fe12854eb6cc6fc7620d97e6ce3c4babd Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 02:51:57 +0200
Subject: [PATCH 11/18] feat: write repeat attempts as run folders

---
 .agents/verification.md                       |   4 +-
 .../commands/eval/artifact-writer.test.ts     |  69 +++++++-
 packages/core/src/evaluation/orchestrator.ts  |   1 +
 packages/core/src/evaluation/run-artifacts.ts | 153 ++++++++++++++++++
 packages/core/src/evaluation/types.ts         |   5 +
 5 files changed, 226 insertions(+), 6 deletions(-)

diff --git a/.agents/verification.md b/.agents/verification.md
index 0111704af..cd37cbb82 100644
--- a/.agents/verification.md
+++ b/.agents/verification.md
@@ -127,10 +127,10 @@ Use live dogfood before marking PRs ready when they affect eval execution, exper
 - Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
-- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted, and inspect both `index.jsonl` and per-test `grading.json` for `trials[]` and `aggregation`.
+- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect `index.jsonl` and per-test `grading.json` for `trials[]`, `aggregation`, and `trials[].run_path`, then inspect the per-case `run-N/` directories for `result.json`, `summary.json`, transcript, trace, grading, timing, metrics, and outputs.
 - For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
 - Preserve review evidence in `agentv-private` on an `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, and screenshots when folder structure or UI behavior is under review.
-- If comparing against an external convention such as Vercel `agent-eval`, state whether AgentV matches the physical folder layout or only the semantic provenance contract. Do not imply `run-N` directory parity unless the artifacts actually use that layout.
+- If comparing against an external convention such as Vercel `agent-eval`, verify both semantic provenance and the physical `run-N` artifact layout for repeat runs.
 
 ## Checking Grader Score Ranges
 
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 6b3b3531a..6e381a5d1 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -909,8 +909,27 @@ describe('writeArtifactsFromResults', () => {
         testId: 'repeat-case',
         score: 1,
         trials: [
-          { attempt: 0, score: 0.25, verdict: 'fail' },
-          { attempt: 1, score: 1, verdict: 'pass' },
+          {
+            attempt: 0,
+            score: 0.25,
+            verdict: 'fail',
+            result: makeResult({
+              testId: 'repeat-case',
+              score: 0.25,
+              output: 'first attempt',
+              executionStatus: 'quality_failure',
+            }),
+          },
+          {
+            attempt: 1,
+            score: 1,
+            verdict: 'pass',
+            result: makeResult({
+              testId: 'repeat-case',
+              score: 1,
+              output: 'second attempt',
+            }),
+          },
         ],
         aggregation: {
           strategy: 'confidence_interval',
@@ -928,8 +947,8 @@ describe('writeArtifactsFromResults', () => {
       await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'grading.json'), 'utf8'),
     );
     expect(grading.trials).toEqual([
-      { attempt: 0, score: 0.25, verdict: 'fail' },
-      { attempt: 1, score: 1, verdict: 'pass' },
+      { attempt: 0, run_path: 'run-1', score: 0.25, verdict: 'fail' },
+      { attempt: 1, run_path: 'run-2', score: 1, verdict: 'pass' },
     ]);
     expect(grading.aggregation).toEqual({
       strategy: 'confidence_interval',
@@ -945,6 +964,48 @@ describe('writeArtifactsFromResults', () => {
       .map((line) => JSON.parse(line) as IndexArtifactEntry);
     expect(indexEntry?.trials).toEqual(grading.trials);
     expect(indexEntry?.aggregation).toEqual(grading.aggregation);
+
+    const repeatEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case'));
+    expect(repeatEntries.sort()).toEqual([
+      'grading.json',
+      'metrics.json',
+      'outputs',
+      'run-1',
+      'run-2',
+      'timing.json',
+      'trace.json',
+      'transcript.jsonl',
+    ]);
+
+    for (const runDir of ['run-1', 'run-2']) {
+      const runEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case', runDir));
+      expect(runEntries.sort()).toEqual([
+        'grading.json',
+        'metrics.json',
+        'outputs',
+        'result.json',
+        'summary.json',
+        'timing.json',
+        'trace.json',
+        'transcript-raw.jsonl',
+        'transcript.jsonl',
+      ]);
+    }
+
+    const runOneResult = JSON.parse(
+      await readFile(
+        path.join(paths.testArtifactDir, 'repeat-case', 'run-1', 'result.json'),
+        'utf8',
+      ),
+    ) as Record<string, unknown>;
+    expect(runOneResult.run_id).toBe('run-1');
+    expect(runOneResult.score).toBe(0.25);
+
+    const runTwoAnswer = await readFile(
+      path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'outputs', 'answer.md'),
+      'utf8',
+    );
+    expect(runTwoAnswer).toBe('second attempt');
   });
 
   it('handles empty results array', async () => {
diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts
index d194b8774..fdd56526d 100644
--- a/packages/core/src/evaluation/orchestrator.ts
+++ b/packages/core/src/evaluation/orchestrator.ts
@@ -2310,6 +2310,7 @@ async function runEvalCaseWithTrials(
       executionStatus: result.executionStatus,
       failureStage: result.failureStage,
       failureReasonCode: result.failureReasonCode,
+      result,
     };
     trialResults.push(trial);
 
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 9ebebb52d..37d6b770c 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -179,6 +179,7 @@ export interface GradingArtifact {
 
 export type TrialResultArtifact = {
   readonly attempt: number;
+  readonly run_path?: string;
   readonly score: number;
   readonly verdict: string;
   readonly scores?: IndexArtifactEntry['scores'];
@@ -467,6 +468,10 @@ function toIndexScores(scores: readonly GraderResult[] | undefined): IndexArtifa
   return scores?.map(toIndexScore) as IndexArtifactEntry['scores'];
 }
 
+function trialRunDirName(attempt: number): string {
+  return `run-${attempt + 1}`;
+}
+
 function toTrialArtifacts(
   trials: readonly TrialResult[] | undefined,
 ): readonly TrialResultArtifact[] | undefined {
@@ -475,6 +480,7 @@ function toTrialArtifacts(
   }
   return trials.map((trial) => ({
     attempt: trial.attempt,
+    run_path: trial.result ? trialRunDirName(trial.attempt) : undefined,
     score: trial.score,
     verdict: trial.verdict,
     scores: toIndexScores(trial.scores),
@@ -541,6 +547,141 @@ function toIndexRerunSource(value: unknown): Record<string, unknown> | undefined
   });
 }
 
+async function writeTrialRunArtifacts(params: {
+  readonly trial: TrialResult;
+  readonly parentTestDir: string;
+  readonly outputDir: string;
+  readonly evalFile?: string;
+  readonly experiment?: string;
+  readonly runId?: string;
+  readonly duplicatePolicy: ExportDuplicatePolicy;
+  readonly testByTestId: Map<string, EvalTest>;
+}): Promise<void> {
+  const result = params.trial.result;
+  if (!result) {
+    return;
+  }
+
+  const runDirName = trialRunDirName(params.trial.attempt);
+  const runDir = path.join(params.parentTestDir, runDirName);
+  const grading = buildGradingArtifact(result);
+  const timing = buildTimingArtifact([result]);
+  const input = extractInput(result);
+  const gradingPath = path.join(runDir, 'grading.json');
+  const timingPath = path.join(runDir, 'timing.json');
+  const inputPath = input ? path.join(runDir, 'input.md') : undefined;
+  const outputsDir = path.join(runDir, 'outputs');
+  const answerPath = result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined;
+  const responsePath = result.output.length > 0 ? path.join(outputsDir, 'response.md') : undefined;
+  const attemptRunId = params.runId
+    ? `${params.runId}:${runDirName}`
+    : `${result.testId}:${result.target}:${runDirName}`;
+  const envelope = buildTraceEnvelopeSidecar({
+    result,
+    outputDir: params.outputDir,
+    testDir: runDir,
+    evalPath: resolveEnvelopeEvalPath(result, params.testByTestId, params.evalFile),
+    experiment: params.experiment,
+    runId: attemptRunId,
+    duplicatePolicy: params.duplicatePolicy,
+  });
+  const tracePath = path.join(runDir, CANONICAL_TRACE_ARTIFACT_PATH);
+  const transcriptPath = hasTranscriptProjection(result, envelope)
+    ? path.join(runDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH)
+    : undefined;
+  const metricsPath = path.join(runDir, CANONICAL_METRICS_ARTIFACT_PATH);
+  const rawProviderLogSource = rawProviderLogSourcePath(result);
+  const rawProviderLogPath = rawProviderLogSource ? rawProviderLogArtifactPath(runDir) : undefined;
+
+  await mkdir(runDir, { recursive: true });
+  await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}\n`, 'utf8');
+  await writeFile(timingPath, `${JSON.stringify(timing, null, 2)}\n`, 'utf8');
+
+  if (inputPath && input) {
+    await writeFile(inputPath, input, 'utf8');
+  }
+
+  await mkdir(outputsDir, { recursive: true });
+  if (answerPath && responsePath) {
+    await writeFile(answerPath, result.output, 'utf8');
+    await writeFile(responsePath, result.output, 'utf8');
+  }
+  if (rawProviderLogSource) {
+    await copyRawProviderLogArtifact(rawProviderLogSource, runDir);
+  }
+  await writeFile(tracePath, `${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}\n`, 'utf8');
+  if (transcriptPath) {
+    await writeTranscriptJsonl(transcriptPath, result, envelope);
+    await writeFile(
+      path.join(runDir, 'transcript-raw.jsonl'),
+      await readFile(transcriptPath, 'utf8'),
+      'utf8',
+    );
+  }
+  await writeMetricsArtifact({
+    filePath: metricsPath,
+    result,
+    envelope,
+    transcriptPath,
+  });
+
+  const artifactPointers = await buildArtifactPointers({
+    outputDir: params.outputDir,
+    tracePath,
+    transcriptPath,
+  });
+  const resultArtifact = {
+    ...buildIndexArtifactEntry(result, {
+      outputDir: params.outputDir,
+      artifactDir: runDir,
+      gradingPath,
+      timingPath,
+      outputPath: answerPath,
+      answerPath,
+      tracePath,
+      transcriptPath,
+      metricsPath,
+      artifactPointers,
+      rawProviderLogPath,
+      inputPath,
+      responsePath,
+      projectionIdentity: envelope.projectionIdentity,
+      duplicatePolicy: params.duplicatePolicy,
+    }),
+    attempt: params.trial.attempt,
+    run: params.trial.attempt + 1,
+    run_id: runDirName,
+    experiment: params.experiment,
+  };
+  await writeFile(
+    path.join(runDir, 'result.json'),
+    `${JSON.stringify(resultArtifact, null, 2)}\n`,
+    'utf8',
+  );
+  await writeFile(
+    path.join(runDir, 'summary.json'),
+    `${JSON.stringify(
+      {
+        attempt: params.trial.attempt,
+        run: params.trial.attempt + 1,
+        run_id: runDirName,
+        test_id: result.testId,
+        target: result.target,
+        score: result.score,
+        verdict: params.trial.verdict,
+        execution_status: result.executionStatus,
+        error: result.error,
+        cost_usd: result.costUsd,
+        duration_ms: result.durationMs,
+        token_usage: result.tokenUsage,
+      },
+      null,
+      2,
+    )}\n`,
+    'utf8',
+  );
+}
+
 function toIndexPreparedAttempt(value: unknown): Record<string, unknown> | undefined {
   if (!isRecord(value)) {
     return undefined;
@@ -1862,6 +2003,18 @@ export async function writeArtifactsFromResults(
       envelope,
       transcriptPath: plan.transcriptPath,
     });
+    for (const trial of result.trials ?? []) {
+      await writeTrialRunArtifacts({
+        trial,
+        parentTestDir: plan.testDir,
+        outputDir,
+        evalFile: options?.evalFile,
+        experiment: options?.experiment,
+        runId: options?.runId,
+        duplicatePolicy,
+        testByTestId,
+      });
+    }
     const artifactPointers = await buildArtifactPointers({
       outputDir,
       tracePath: plan.tracePath,
diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts
index af8c100bb..0969ae0cd 100644
--- a/packages/core/src/evaluation/types.ts
+++ b/packages/core/src/evaluation/types.ts
@@ -1085,6 +1085,11 @@ export interface TrialResult {
   readonly failureStage?: FailureStage;
   /** Machine-readable failure reason code */
   readonly failureReasonCode?: string;
+  /**
+   * Full per-attempt result used by artifact writers to materialize Vercel-style
+   * run-N folders. This is intentionally omitted from wire trial summaries.
+   */
+  readonly result?: EvaluationResult;
 }
 
 /**

From 319652f958410033ef867b4a096d869b2ddf960e Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 03:57:34 +0200
Subject: [PATCH 12/18] feat: align repeat run artifacts with vercel layout

---
 .agents/verification.md                       |   2 +-
 apps/cli/src/commands/results/combine-run.ts  |   2 +
 apps/cli/src/commands/results/manifest.ts     |   2 +
 .../src/commands/results/projection-bundle.ts |   4 +
 apps/cli/src/commands/results/serve.ts        |   2 +
 apps/cli/src/commands/results/validate.ts     |  17 +-
 .../commands/eval/artifact-writer.test.ts     | 100 +++--
 .../docs/docs/evaluation/experiments.mdx      |  23 ++
 .../src/content/docs/docs/tools/results.mdx   |   7 +-
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |  39 +-
 packages/core/src/evaluation/metrics.ts       |  31 ++
 .../core/src/evaluation/result-row-schema.ts  |   3 +
 packages/core/src/evaluation/run-artifacts.ts | 391 ++++++++++++++----
 13 files changed, 489 insertions(+), 134 deletions(-)

diff --git a/.agents/verification.md b/.agents/verification.md
index cd37cbb82..9b4091cf2 100644
--- a/.agents/verification.md
+++ b/.agents/verification.md
@@ -127,7 +127,7 @@ Use live dogfood before marking PRs ready when they affect eval execution, exper
 - Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
-- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect `index.jsonl` and per-test `grading.json` for `trials[]`, `aggregation`, and `trials[].run_path`, then inspect the per-case `run-N/` directories for `result.json`, `summary.json`, transcript, trace, grading, timing, metrics, and outputs.
+- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect root `index.jsonl`, root `benchmark.json`, and the repeated case folder. The repeated case folder should carry Vercel-compatible `summary.json` plus AgentV aggregate `grading.json` and `timing.json`; attempt-specific outputs, metrics, and transcripts live under `run-N/`. Each `run-N/` folder should contain Vercel-compatible `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/`, plus AgentV `grading.json` and `metrics.json` with embedded `timing`.
 - For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
 - Preserve review evidence in `agentv-private` on an `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, and screenshots when folder structure or UI behavior is under review.
 - If comparing against an external convention such as Vercel `agent-eval`, verify both semantic provenance and the physical `run-N` artifact layout for repeat runs.
diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts
index b44c4ed39..18d7bbb99 100644
--- a/apps/cli/src/commands/results/combine-run.ts
+++ b/apps/cli/src/commands/results/combine-run.ts
@@ -363,6 +363,8 @@ function resolveCombinedExperiment(
 
 const MANIFEST_PATH_FIELDS = [
   'artifact_dir',
+  'benchmark_path',
+  'summary_path',
   'grading_path',
   'timing_path',
   'input_path',
diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts
index c9a272e5f..679cff1ef 100644
--- a/apps/cli/src/commands/results/manifest.ts
+++ b/apps/cli/src/commands/results/manifest.ts
@@ -39,6 +39,8 @@ export interface ResultManifestRecord {
     readonly reasoning?: number;
   };
   readonly trace?: Record<string, unknown>;
+  readonly benchmark_path?: string;
+  readonly summary_path?: string;
   readonly grading_path?: string;
   readonly timing_path?: string;
   readonly input_path?: string;
diff --git a/apps/cli/src/commands/results/projection-bundle.ts b/apps/cli/src/commands/results/projection-bundle.ts
index a81b96c15..1f03a9b7c 100644
--- a/apps/cli/src/commands/results/projection-bundle.ts
+++ b/apps/cli/src/commands/results/projection-bundle.ts
@@ -87,6 +87,8 @@ export type ProjectionBundleArtifactRefs = Partial<
   Pick<
     IndexArtifactEntry,
     | 'artifact_dir'
+    | 'benchmark_path'
+    | 'summary_path'
     | 'grading_path'
     | 'timing_path'
     | 'input_path'
@@ -172,6 +174,8 @@ function artifactRefs(
   return dropUndefined({
     ...metadataRefs,
     artifact_dir: indexEntry.artifact_dir,
+    benchmark_path: indexEntry.benchmark_path,
+    summary_path: indexEntry.summary_path,
     grading_path: indexEntry.grading_path,
     input_path: indexEntry.input_path,
     output_path: indexEntry.output_path,
diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts
index 1029475d2..ef7766b93 100644
--- a/apps/cli/src/commands/results/serve.ts
+++ b/apps/cli/src/commands/results/serve.ts
@@ -742,6 +742,8 @@ function buildResultArtifactCatalog(
   addPointerArtifactCatalogEntry(entries, seen, trace, 'trace', options?.runPath);
   addPointerArtifactCatalogEntry(entries, seen, answer, 'answer', options?.runPath);
 
+  addDirectArtifactCatalogEntry(entries, seen, record.benchmark_path, 'artifact');
+  addDirectArtifactCatalogEntry(entries, seen, record.summary_path, 'artifact');
   addDirectArtifactCatalogEntry(entries, seen, record.grading_path, 'artifact');
   addDirectArtifactCatalogEntry(entries, seen, record.timing_path, 'artifact');
   addDirectArtifactCatalogEntry(entries, seen, record.input_path, 'artifact');
diff --git a/apps/cli/src/commands/results/validate.ts b/apps/cli/src/commands/results/validate.ts
index c0ffb8dc0..e0641b39e 100644
--- a/apps/cli/src/commands/results/validate.ts
+++ b/apps/cli/src/commands/results/validate.ts
@@ -6,7 +6,7 @@
  *   1. Directory follows the `.agentv/results/<experiment>/<timestamp>` naming convention
  *   2. index.jsonl exists and each line has required fields
  *   3. Per-test grading.json exists for every entry in the index
- *   4. Per-test timing.json exists (warning if missing)
+ *   4. Per-test timing.json exists for direct case rows (warning if missing)
  *   5. benchmark.json exists (warning if missing)
  *   6. Scores are within [0, 1]
  *   7. index.jsonl entries have `scores[]` array (warning if missing — dashboard needs it)
@@ -34,6 +34,7 @@ interface IndexEntry {
   readonly target?: string;
   readonly scores?: unknown[];
   readonly execution_status?: string;
+  readonly benchmark_path?: string;
   readonly grading_path?: string;
   readonly timing_path?: string;
   readonly [key: string]: unknown;
@@ -139,10 +140,10 @@ function checkIndexJsonl(runDir: string): { diagnostics: Diagnostic[]; entries:
         });
       }
 
-      if (!entry.grading_path) {
+      if (!entry.grading_path && !entry.benchmark_path) {
         diagnostics.push({
           severity: 'warning',
-          message: `index.jsonl line ${i + 1} (${entry.test_id ?? '?'}): missing 'grading_path'`,
+          message: `index.jsonl line ${i + 1} (${entry.test_id ?? '?'}): missing 'grading_path' or 'benchmark_path'`,
         });
       }
 
@@ -203,6 +204,16 @@ function checkArtifactFiles(runDir: string, entries: IndexEntry[]): Diagnostic[]
   for (const entry of entries) {
     const testId = entry.test_id ?? '?';
 
+    if (entry.benchmark_path) {
+      const benchmarkPath = path.join(runDir, entry.benchmark_path);
+      if (!existsSync(benchmarkPath)) {
+        diagnostics.push({
+          severity: 'error',
+          message: `${testId}: benchmark.json not found at '${entry.benchmark_path}'`,
+        });
+      }
+    }
+
     // Check grading.json
     if (entry.grading_path) {
       const gradingPath = path.join(runDir, entry.grading_path);
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 6e381a5d1..f6c9ea056 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -903,7 +903,7 @@ describe('writeArtifactsFromResults', () => {
     expect(indexLines[0]?.metrics_path).toBe('alpha/metrics.json');
   });
 
-  it('writes repeat trial metadata to grading and index artifacts', async () => {
+  it('writes repeat runs in Vercel-compatible case and run folders', async () => {
     const results = [
       makeResult({
         testId: 'repeat-case',
@@ -917,6 +917,7 @@ describe('writeArtifactsFromResults', () => {
               testId: 'repeat-case',
               score: 0.25,
               output: 'first attempt',
+              durationMs: 2000,
               executionStatus: 'quality_failure',
             }),
           },
@@ -928,6 +929,7 @@ describe('writeArtifactsFromResults', () => {
               testId: 'repeat-case',
               score: 1,
               output: 'second attempt',
+              durationMs: 4000,
             }),
           },
         ],
@@ -943,40 +945,69 @@ describe('writeArtifactsFromResults', () => {
 
     const paths = await writeArtifactsFromResults(results, testDir);
 
-    const grading: GradingArtifact = JSON.parse(
-      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'grading.json'), 'utf8'),
-    );
-    expect(grading.trials).toEqual([
+    const [indexEntry] = (await readFile(paths.indexPath, 'utf8'))
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line) as IndexArtifactEntry);
+    expect(indexEntry?.trials).toEqual([
       { attempt: 0, run_path: 'run-1', score: 0.25, verdict: 'fail' },
       { attempt: 1, run_path: 'run-2', score: 1, verdict: 'pass' },
     ]);
-    expect(grading.aggregation).toEqual({
+    expect(indexEntry?.aggregation).toEqual({
       strategy: 'confidence_interval',
       mean: 0.625,
       ci95_lower: 0.1,
       ci95_upper: 1,
       stddev: 0.53,
     });
-
-    const [indexEntry] = (await readFile(paths.indexPath, 'utf8'))
-      .trim()
-      .split('\n')
-      .map((line) => JSON.parse(line) as IndexArtifactEntry);
-    expect(indexEntry?.trials).toEqual(grading.trials);
-    expect(indexEntry?.aggregation).toEqual(grading.aggregation);
+    expect(indexEntry?.artifact_dir).toBe('repeat-case');
+    expect(indexEntry?.summary_path).toBe('repeat-case/summary.json');
+    expect(indexEntry?.benchmark_path).toBeUndefined();
+    expect(indexEntry?.grading_path).toBe('repeat-case/grading.json');
+    expect(indexEntry?.timing_path).toBe('repeat-case/timing.json');
+    expect(indexEntry?.metrics_path).toBeUndefined();
 
     const repeatEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case'));
     expect(repeatEntries.sort()).toEqual([
       'grading.json',
-      'metrics.json',
-      'outputs',
       'run-1',
       'run-2',
+      'summary.json',
       'timing.json',
-      'trace.json',
-      'transcript.jsonl',
     ]);
 
+    const caseSummary = JSON.parse(
+      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'summary.json'), 'utf8'),
+    ) as Record<string, unknown>;
+    expect(caseSummary).toEqual({
+      totalRuns: 2,
+      passedRuns: 1,
+      passRate: '50%',
+      meanDuration: 3,
+      fingerprint: expect.any(String),
+    });
+
+    const aggregateGrading: GradingArtifact = JSON.parse(
+      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'grading.json'), 'utf8'),
+    );
+    expect(aggregateGrading.trials).toEqual(indexEntry?.trials);
+    expect(aggregateGrading.aggregation).toEqual(indexEntry?.aggregation);
+
+    const aggregateTiming = JSON.parse(
+      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'timing.json'), 'utf8'),
+    ) as TimingArtifact;
+    expect(aggregateTiming.duration_ms).toBe(6000);
+    expect(aggregateTiming.mean_duration_seconds).toBe(3);
+    expect(aggregateTiming.duration_stats).toEqual({
+      count: 2,
+      mean_ms: 3000,
+      mean_seconds: 3,
+      stddev_ms: 1000,
+      stddev_seconds: 1,
+      min_ms: 2000,
+      max_ms: 4000,
+    });
+
     for (const runDir of ['run-1', 'run-2']) {
       const runEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case', runDir));
       expect(runEntries.sort()).toEqual([
@@ -984,11 +1015,8 @@ describe('writeArtifactsFromResults', () => {
         'metrics.json',
         'outputs',
         'result.json',
-        'summary.json',
-        'timing.json',
-        'trace.json',
         'transcript-raw.jsonl',
-        'transcript.jsonl',
+        'transcript.json',
       ]);
     }
 
@@ -998,14 +1026,38 @@ describe('writeArtifactsFromResults', () => {
         'utf8',
       ),
     ) as Record<string, unknown>;
-    expect(runOneResult.run_id).toBe('run-1');
-    expect(runOneResult.score).toBe(0.25);
+    expect(runOneResult).toMatchObject({
+      status: 'failed',
+      duration: 2,
+      model: 'test-target',
+      transcriptPath: './transcript.json',
+      transcriptRawPath: './transcript-raw.jsonl',
+      outputPaths: { eval: './outputs/eval.txt' },
+    });
 
     const runTwoAnswer = await readFile(
-      path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'outputs', 'answer.md'),
+      path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'outputs', 'eval.txt'),
       'utf8',
     );
     expect(runTwoAnswer).toBe('second attempt');
+
+    const runTwoMetrics = JSON.parse(
+      await readFile(
+        path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'metrics.json'),
+        'utf8',
+      ),
+    ) as Record<string, unknown>;
+    expect(runTwoMetrics).toMatchObject({
+      source_artifacts: {
+        trace_path: 'transcript.json',
+        transcript_path: 'transcript-raw.jsonl',
+        grading_path: 'grading.json',
+      },
+      timing: {
+        duration_ms: 4000,
+      },
+    });
+    expect((runTwoMetrics.source_artifacts as Record<string, unknown>).timing_path).toBeUndefined();
   });
 
   it('handles empty results array', async () => {
diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
index 6766dc7ea..667ad8156 100644
--- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
@@ -76,6 +76,29 @@ remaining attempts after the first pass; smoke runs can also force one run.
 AgentV follows the same experiment-level placement while keeping the richer
 `repeat` block for AgentV strategies.
 
+Repeat-enabled cases use a Vercel-style physical layout with AgentV aggregate
+provenance:
+
+```text
+<run-dir>/index.jsonl
+<run-dir>/benchmark.json
+<run-dir>/<suite>/<case-id>/summary.json
+<run-dir>/<suite>/<case-id>/grading.json
+<run-dir>/<suite>/<case-id>/timing.json
+<run-dir>/<suite>/<case-id>/run-1/result.json
+<run-dir>/<suite>/<case-id>/run-1/transcript.json
+<run-dir>/<suite>/<case-id>/run-1/transcript-raw.jsonl
+<run-dir>/<suite>/<case-id>/run-1/outputs/eval.txt
+<run-dir>/<suite>/<case-id>/run-1/grading.json
+<run-dir>/<suite>/<case-id>/run-1/metrics.json
+```
+
+The repeated case aggregate folder uses `summary.json` for run-count and mean
+duration, `grading.json` for compact trial/aggregation verdicts, and
+`timing.json` for aggregate timing totals plus mean attempt duration. Root
+`index.jsonl` and root `benchmark.json` remain stable for existing CI summary
+scripts and uploaded artifact consumers.
+
 ## Targets and setup
 
 Experiments reuse targets from `.agentv/targets.yaml`; they do not define a new
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index e5f653ed8..a06674cd4 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -112,13 +112,18 @@ Duplicate policy is explicit:
 
 ### Metrics sidecar
 
-Each per-case artifact directory includes `metrics.json`
+Each direct per-case artifact directory includes `metrics.json`
 (`schema_version: "agentv.metrics.v1"`). This is an AgentV-owned
 derived projection over `trace.json`, the result row, and
 `grading.json`. It is the compact executor behavior summary for dashboards,
 comparison exports, and metric-style graders; it is not canonical trace storage
 and does not carry token/cost usage.
 
+Repeat-enabled cases use aggregate `summary.json`, `grading.json`, and
+`timing.json`, then store attempt details under `run-N/`. Each `run-N/` contains Vercel-compatible
+`result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/eval.txt`,
+plus AgentV `grading.json` and `metrics.json` with embedded timing.
+
 `transcript.jsonl` remains the ordered conversational/log compatibility
 projection. Full trace detail stays in `trace.json`
 (`agentv.trace.v1`). `benchmark.json` remains the run-level aggregate summary,
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index c2e8f6d18..2e28d717f 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -87,8 +87,8 @@ Repeat-run reliability uses a separate metric: `attempt_success_rate`. It means
 Suggested wire shape:
 
 - `run-N/result.json.result.pass_rate`, or the equivalent current `run-N/grading.json` summary field, is assertion-level pass rate for that attempt and is omitted when the verifier has no assertion counts.
-- `<case-id>/summary.json.pass_rate` is optional aggregate assertion pass-rate stats, not binary attempt success frequency.
-- `<case-id>/summary.json.attempts.success_rate` and the index convenience field `attempt_success_rate` represent successful counted attempts divided by counted attempts.
+- `<case-id>/grading.json.summary.pass_rate` is aggregate assertion pass-rate stats, not binary attempt success frequency.
+- `<case-id>/summary.json`, the index `trials[]`/`aggregation`, and future explicit repeat fields represent successful counted attempts divided by counted attempts.
 - Binary-only harnesses write `passed: true` or `passed: false` per attempt and derive only `attempt_success_rate` across repeated attempts.
 
 ---
@@ -100,7 +100,7 @@ Before introducing AgentV-specific contract shapes, implementation should check
 | Reference | Lowest common denominator to reuse | Intentional AgentV divergence |
 | --- | --- | --- |
 | Claude Skills schema | Use assertion, expectation, grading, `passed`, `failed`, `total`, and assertion-level `pass_rate` vocabulary for graders that expose assertion counts. | Do not copy the full skill-eval artifact shape. AgentV keeps `.agentv/results/<experiment>/<timestamp>/...` as the portable run bundle and uses `attempt_success_rate` for repeat-run reliability. |
-| Vercel `agent-eval` | Reuse fixture-driven hidden verifier ergonomics, durable `run-1`, `run-2` attempt directories, and aggregate case summaries. | Rename Vercel `passRate` to `attempt_success_rate` in AgentV, because Vercel's metric is passed attempts divided by total attempts. Do not inherit ambiguous CLI gating semantics. |
+| Vercel `agent-eval` | Reuse fixture-driven hidden verifier ergonomics, case-level `summary.json`, and durable `run-1`, `run-2` attempt directories. | Keep AgentV root `benchmark.json` for current run-level compatibility, but do not write per-attempt `benchmark.json`. Rename Vercel `passRate` to `attempt_success_rate` where attempt-frequency stats are exposed in AgentV-specific artifacts. Do not inherit ambiguous CLI gating semantics. |
 | Hugging Face Datasets | Keep dataset, split, record, features, and row-oriented corpus vocabulary for eval inputs and benchmark corpora. Treat an AgentV case as a record-like unit when mapping to external datasets. | Do not require Arrow, the Hub, DatasetDict, or HF storage layout. AgentV cases remain repo files or generated case records inside benchmark/project artifacts. |
 | OpenInference | Preserve trace/span/tool-call/model-observability semantics when naming trace metadata and external trace correlation fields. | Do not require OpenTelemetry collection, Phoenix, or OpenInference export as core runtime infrastructure. AgentV stores portable traces/transcripts as artifacts and supports link-out correlation through `external_trace` metadata. |
 
@@ -150,9 +150,9 @@ Public docs and implementation notes must not reference non-public sources. If a
 
 - KTD1. The repeat config attaches to the **experiment** surface, not to `eval.yaml` `execution`, per the experiments-separation decision (epic `av-991`, recorded on `av-991.1`). This aligns with Vercel agent-eval, where `runs`/`earlyExit` are experiment-level. This epic (`av-i0l`) owns the repeat **mechanics** (schema shape, gate policies, attempt aggregation, flake classification, and the run-N artifact layout); `av-991` owns **placement** (the experiment contract the repeat block lives on). The existing `execution.trials` code path is **hard-removed** (no compatibility alias) because usage is rare; its behavior is replaced by the experiment-level repeat block. Because the experiment surface is delivered by `av-991`, the schema work in `av-i0l.1` depends on that contract landing.
 - KTD2. Keep one-run CI as the default. Repeat runs are for reliability evidence unless `repeat.gate` says they are a CI gate.
-- KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json` and `run-N/` directories so existing aggregate consumers do not inflate case counts.
+- KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json`, `grading.json`, `timing.json`, and `run-N/` directories so existing aggregate consumers do not inflate case counts.
 - KTD4. Single-run cases keep direct case-local files instead of always nesting under `run-1`. This preserves the simple default artifact shape and makes `.agentv/results/<experiment>/<timestamp>/<case-id>/grading.json` easy to inspect. Repeat-enabled cases use `run-1/`, `run-2/`, and so on under the case directory.
-- KTD5. Write `summary.json` for every case after the artifact-layout migration. In a single-run case it summarizes the direct files; in a repeat-run case it summarizes the `run-N/` children.
+- KTD5. Root run aggregates keep the existing AgentV `benchmark.json` for compatibility. Repeat case aggregates use Vercel-compatible `summary.json` plus AgentV aggregate `grading.json` and `timing.json`; repeat attempts use `run-N/` children.
 - KTD6. `pass_at_k` keeps the existing AgentV/Vercel ergonomics: early exit is enabled unless explicitly disabled. Full reliability sampling requires `early_exit: false` on the experiment and should be recorded because it changes cost and statistics.
 - KTD7. Do not inherit Vercel's implicit CI ambiguity. All policies that can make one failed plus one passed attempt count as passing must be visible in config and artifacts.
 - KTD8. Reuse current failure classification fields before adding new enums. Add aggregate classification fields only after mapping from `execution_status`, `failure_stage`, and `failure_reason_code` proves insufficient.
@@ -244,7 +244,6 @@ Single-run cases keep direct case-local files:
 ```text
 .agentv/results/<experiment>/<timestamp>/index.jsonl
 .agentv/results/<experiment>/<timestamp>/benchmark.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/summary.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/grading.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/timing.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/input.md
@@ -253,7 +252,7 @@ Single-run cases keep direct case-local files:
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/response.md
 ```
 
-Rationale: the common path stays readable, old mental models stay close, and no user pays a `run-1/` nesting tax for default CI. `summary.json` gives readers a uniform aggregate entry point without moving direct single-run sidecars.
+Rationale: the common path stays readable, old mental models stay close, and no user pays a `run-1/` nesting tax for default CI.
 
 ### Repeat-Run Case
 
@@ -261,14 +260,20 @@ Repeat-run cases use attempt directories:
 
 ```text
 .agentv/results/<experiment>/<timestamp>/<case-id>/summary.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/grading.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/timing.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/grading.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/timing.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/trace.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/transcript.jsonl
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/metrics.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript-raw.jsonl
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/eval.txt
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/grading.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/timing.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/trace.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/transcript.jsonl
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/metrics.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript.json
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript-raw.jsonl
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/eval.txt
 ```
 
 `<case-id>` should reuse the sanitized artifact key produced by the current artifact writer, including suite disambiguation where needed. Attempt directories are one-indexed because users naturally inspect `run-1`, `run-2`, and this matches the Vercel comparison.
@@ -277,7 +282,7 @@ Repeat-run cases use attempt directories:
 
 ## Aggregation Semantics
 
-Each case summary should expose:
+Each repeat case aggregate should expose:
 
 | Field | Meaning |
 | --- | --- |
@@ -369,7 +374,7 @@ Dashboard should present repeat-run cases aggregate-first:
 - Attempt drill-down lists `run-1`, `run-2`, and so on with score, status, duration, cost, failure reason, and retry/exclusion reason.
 - Selecting an attempt opens the same Checks, Transcript, Source, Files, and Feedback affordances as a normal single-run result.
 - Dashboard must not hide individual traces, transcripts, raw provider logs, or grader output behind the aggregate.
-- Historical single-run rows render as they do today, with `summary.json` absent or minimal.
+- Historical single-run rows render as they do today, with `summary.json` absent.
 
 For trend and compare views, repeat aggregates should be the default unit. Attempt-level views can be added as a filter later, but they must not silently change run-level counts.
 
@@ -480,11 +485,11 @@ Repeat runs can multiply provider spend. V1 should ship with conservative contro
 
 **Files:** `packages/core/src/evaluation/run-artifacts.ts`, `packages/core/src/evaluation/result-row-schema.ts`, `apps/cli/src/commands/eval/artifact-writer.ts`, `apps/cli/src/commands/eval/result-layout.ts`, `apps/cli/test/commands/eval/artifact-writer.test.ts`, `apps/cli/test/commands/eval/aggregate.test.ts`, `apps/cli/test/commands/results/validate.test.ts`.
 
-**Approach:** Extend the artifact writer to understand aggregate results with attempt children. Keep single-run case sidecars direct, add `summary.json`, and add optional repeat fields to index rows. Avoid putting full attempt payloads in `index.jsonl`.
+**Approach:** Extend the artifact writer to understand aggregate results with attempt children. Keep single-run case sidecars direct, add case-local `summary.json` for repeat aggregates, and add optional repeat fields to index rows. Avoid putting full attempt payloads in `index.jsonl`.
 
 **Test Scenarios:**
 
-- Single-run output writes direct case-local sidecars plus `summary.json` and remains readable by existing manifest hydration.
+- Single-run output writes direct case-local sidecars and remains readable by existing manifest hydration.
 - Repeat-run output writes `summary.json` and `run-1/`, `run-2/` sidecars with correct relative paths.
 - `index.jsonl` has one aggregate row per case/target and compact attempt references.
 - Historical rows without repeat fields parse successfully.
diff --git a/packages/core/src/evaluation/metrics.ts b/packages/core/src/evaluation/metrics.ts
index 8cbd2b2f4..b0243a9d2 100644
--- a/packages/core/src/evaluation/metrics.ts
+++ b/packages/core/src/evaluation/metrics.ts
@@ -20,6 +20,12 @@ import type { TraceEvent } from './trace.js';
 import type { EvaluationResult } from './types.js';
 
 const TOOL_STATUS_VALUES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const;
+const TIMING_SOURCE_VALUES = [
+  'provider_reported',
+  'token_estimated',
+  'aggregate',
+  'unavailable',
+] as const;
 
 const FILE_READ_KEY_SET = new Set([
   'file',
@@ -113,6 +119,30 @@ const ReasoningBlockWireSchema = z
   })
   .strict();
 
+const MetricsTimingWireSchema = z
+  .object({
+    total_tokens: z.number().int().nonnegative(),
+    duration_ms: z.number().nonnegative(),
+    total_duration_seconds: z.number().nonnegative(),
+    cost_usd: z.number().nonnegative().nullable(),
+    token_usage: z
+      .object({
+        input: z.number().int().nonnegative(),
+        output: z.number().int().nonnegative(),
+        reasoning: z.number().int().nonnegative(),
+      })
+      .strict(),
+    usage_sources: z
+      .object({
+        token_usage: z.enum(TIMING_SOURCE_VALUES),
+        total_tokens: z.enum(TIMING_SOURCE_VALUES),
+        duration: z.enum(TIMING_SOURCE_VALUES),
+        cost: z.enum(TIMING_SOURCE_VALUES),
+      })
+      .strict(),
+  })
+  .strict();
+
 export const MetricsWireSchema = z
   .object({
     tool_calls: z.record(z.string(), z.number().int().nonnegative()),
@@ -163,6 +193,7 @@ export const MetricsArtifactWireSchema = z
       })
       .strict(),
     metrics: MetricsWireSchema,
+    timing: MetricsTimingWireSchema.optional(),
   })
   .strict();
 
diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts
index 0655ecaf7..4fafbe960 100644
--- a/packages/core/src/evaluation/result-row-schema.ts
+++ b/packages/core/src/evaluation/result-row-schema.ts
@@ -21,6 +21,7 @@ const MIGRATION_GUIDANCE =
 const RESULT_ROW_ALIASES = {
   answerPath: 'answer_path',
   artifactDir: 'artifact_dir',
+  benchmarkPath: 'benchmark_path',
   conversationId: 'conversation_id',
   costUsd: 'cost_usd',
   durationMs: 'duration_ms',
@@ -38,6 +39,7 @@ const RESULT_ROW_ALIASES = {
   rawProviderLogPath: 'raw_provider_log_path',
   responsePath: 'response_path',
   startTime: 'start_time',
+  summaryPath: 'summary_path',
   targetsPath: 'targets_path',
   taskDir: 'task_dir',
   testId: 'test_id',
@@ -176,6 +178,7 @@ function looksLikeResultRow(value: Record<string, unknown>): boolean {
     Object.hasOwn(value, 'trace') ||
     Object.hasOwn(value, 'spans') ||
     Object.hasOwn(value, 'target') ||
+    Object.hasOwn(value, 'benchmark_path') ||
     Object.hasOwn(value, 'grading_path') ||
     Object.hasOwn(value, 'timing_path')
   );
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 37d6b770c..9e324a1c4 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -214,6 +214,17 @@ export interface TimingArtifact {
   readonly total_tokens: number;
   readonly duration_ms: number;
   readonly total_duration_seconds: number;
+  readonly mean_duration_ms?: number;
+  readonly mean_duration_seconds?: number;
+  readonly duration_stats?: {
+    readonly count: number;
+    readonly mean_ms: number;
+    readonly mean_seconds: number;
+    readonly stddev_ms: number;
+    readonly stddev_seconds: number;
+    readonly min_ms: number;
+    readonly max_ms: number;
+  };
   readonly cost_usd: number | null;
   readonly token_usage: {
     readonly input: number;
@@ -290,8 +301,10 @@ export interface IndexArtifactEntry {
   readonly failure_reason_code?: string;
   readonly workspace_path?: string;
   readonly artifact_dir?: string;
-  readonly grading_path: string;
-  readonly timing_path: string;
+  readonly grading_path?: string;
+  readonly timing_path?: string;
+  readonly benchmark_path?: string;
+  readonly summary_path?: string;
   readonly output_path?: string;
   readonly answer_path?: string;
   readonly trace_path?: string;
@@ -338,6 +351,37 @@ export interface AdditionalResultArtifactsContext {
   readonly sourceTestsById: ReadonlyMap<string, EvalTest>;
 }
 
+export interface VercelRunResultArtifact {
+  readonly status: 'passed' | 'failed' | 'error';
+  readonly duration: number;
+  readonly model: string;
+  readonly transcriptPath?: string;
+  readonly transcriptRawPath?: string;
+  readonly o11y: {
+    readonly totalTurns: number;
+    readonly toolCalls: Record<string, number>;
+    readonly totalToolCalls: number;
+    readonly webFetches: readonly unknown[];
+    readonly filesRead: readonly string[];
+    readonly filesModified: readonly string[];
+    readonly shellCommands: readonly unknown[];
+    readonly errors: readonly unknown[];
+    readonly thinkingBlocks: number;
+  };
+  readonly outputPaths?: {
+    readonly eval?: string;
+    readonly scripts?: Record<string, string>;
+  };
+}
+
+export interface VercelEvalSummaryArtifact {
+  readonly totalRuns: number;
+  readonly passedRuns: number;
+  readonly passRate: string;
+  readonly meanDuration: number;
+  readonly fingerprint: string;
+}
+
 export type AdditionalResultArtifactsWriter = (
   context: AdditionalResultArtifactsContext,
 ) => Promise<AdditionalResultIndexFields | undefined>;
@@ -472,6 +516,10 @@ function trialRunDirName(attempt: number): string {
   return `run-${attempt + 1}`;
 }
 
+function hasPersistedTrialRuns(result: EvaluationResult): boolean {
+  return (result.trials ?? []).some((trial) => trial.result !== undefined);
+}
+
 function toTrialArtifacts(
   trials: readonly TrialResult[] | undefined,
 ): readonly TrialResultArtifact[] | undefined {
@@ -547,6 +595,151 @@ function toIndexRerunSource(value: unknown): Record<string, unknown> | undefined
   });
 }
 
+function resultDurationSeconds(result: EvaluationResult): number {
+  const durationMs =
+    result.durationMs ?? result.trace?.durationMs ?? result.evalRun?.durationMs ?? 0;
+  return Math.round((durationMs / 1000) * 1000) / 1000;
+}
+
+function resultDurationMs(result: EvaluationResult): number | undefined {
+  const durationMs = result.durationMs ?? result.trace?.durationMs ?? result.evalRun?.durationMs;
+  return typeof durationMs === 'number' && Number.isFinite(durationMs) ? durationMs : undefined;
+}
+
+function roundMillis(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}
+
+function roundSecondsFromMs(value: number): number {
+  return Math.round((value / 1000) * 1000) / 1000;
+}
+
+function repeatAttemptResults(result: EvaluationResult): readonly EvaluationResult[] {
+  const trialResults = (result.trials ?? [])
+    .map((trial) => trial.result)
+    .filter((trialResult): trialResult is EvaluationResult => trialResult !== undefined);
+  return trialResults.length > 0 ? trialResults : [result];
+}
+
+function buildRepeatAggregateTimingArtifact(result: EvaluationResult): TimingArtifact {
+  const attemptResults = repeatAttemptResults(result);
+  const timing = buildTimingArtifact(attemptResults);
+  const durationsMs = attemptResults
+    .map(resultDurationMs)
+    .filter((durationMs): durationMs is number => durationMs !== undefined);
+  if (durationsMs.length === 0) {
+    return timing;
+  }
+
+  const stats = computeStats(durationsMs);
+  const minMs = Math.min(...durationsMs);
+  const maxMs = Math.max(...durationsMs);
+  return {
+    ...timing,
+    mean_duration_ms: stats.mean,
+    mean_duration_seconds: roundSecondsFromMs(stats.mean),
+    duration_stats: {
+      count: durationsMs.length,
+      mean_ms: stats.mean,
+      mean_seconds: roundSecondsFromMs(stats.mean),
+      stddev_ms: stats.stddev,
+      stddev_seconds: roundSecondsFromMs(stats.stddev),
+      min_ms: roundMillis(minMs),
+      max_ms: roundMillis(maxMs),
+    },
+  };
+}
+
+function formatVercelPassRate(passedRuns: number, totalRuns: number): string {
+  if (totalRuns === 0) {
+    return '0%';
+  }
+  const percent = Math.round((passedRuns / totalRuns) * 1000) / 10;
+  return `${Number.isInteger(percent) ? percent.toFixed(0) : percent.toFixed(1)}%`;
+}
+
+function fallbackRepeatFingerprint(result: EvaluationResult): string {
+  return createHash('sha256')
+    .update(
+      JSON.stringify({
+        test_id: result.testId ?? 'unknown',
+        target: result.target ?? 'unknown',
+        trial_count: result.trials?.length ?? 0,
+        aggregation: result.aggregation,
+      }),
+    )
+    .digest('hex');
+}
+
+function buildVercelEvalSummaryArtifact(
+  result: EvaluationResult,
+  fingerprint?: string,
+): VercelEvalSummaryArtifact {
+  const trials = result.trials ?? [];
+  const totalRuns = trials.length;
+  const passedRuns = trials.filter((trial) => trial.verdict === 'pass').length;
+  const durations = repeatAttemptResults(result).map(resultDurationSeconds);
+  const meanDuration =
+    durations.length === 0
+      ? 0
+      : Math.round(
+          (durations.reduce((sum, duration) => sum + duration, 0) / durations.length) * 1000,
+        ) / 1000;
+
+  return {
+    totalRuns,
+    passedRuns,
+    passRate: formatVercelPassRate(passedRuns, totalRuns),
+    meanDuration,
+    fingerprint: fingerprint ?? fallbackRepeatFingerprint(result),
+  };
+}
+
+function toVercelRunStatus(
+  trial: TrialResult,
+  result: EvaluationResult,
+): VercelRunResultArtifact['status'] {
+  if (trial.executionStatus === 'execution_error' || result.executionStatus === 'execution_error') {
+    return 'error';
+  }
+  return trial.verdict === 'pass' ? 'passed' : 'failed';
+}
+
+function toFilePathList(entries: readonly unknown[]): readonly string[] {
+  return entries
+    .map((entry) => (isRecord(entry) && typeof entry.path === 'string' ? entry.path : undefined))
+    .filter((entry): entry is string => entry !== undefined);
+}
+
+function buildVercelRunResultArtifact(params: {
+  readonly trial: TrialResult;
+  readonly result: EvaluationResult;
+  readonly metricsArtifact: ReturnType<typeof buildMetricsArtifact>;
+  readonly hasTranscript: boolean;
+  readonly hasOutput: boolean;
+}): VercelRunResultArtifact {
+  const metrics = params.metricsArtifact.metrics;
+  return dropUndefined({
+    status: toVercelRunStatus(params.trial, params.result),
+    duration: resultDurationSeconds(params.result),
+    model: params.result.target ?? 'unknown',
+    transcriptPath: params.hasTranscript ? './transcript.json' : undefined,
+    transcriptRawPath: params.hasTranscript ? './transcript-raw.jsonl' : undefined,
+    o11y: {
+      totalTurns: metrics.total_turns,
+      toolCalls: metrics.tool_calls,
+      totalToolCalls: metrics.total_tool_calls,
+      webFetches: metrics.web_fetches,
+      filesRead: toFilePathList(metrics.files_read),
+      filesModified: toFilePathList(metrics.files_modified),
+      shellCommands: metrics.shell_commands,
+      errors: metrics.errors,
+      thinkingBlocks: metrics.thinking_blocks,
+    },
+    outputPaths: params.hasOutput ? { eval: './outputs/eval.txt' } : undefined,
+  }) as unknown as VercelRunResultArtifact;
+}
+
 async function writeTrialRunArtifacts(params: {
   readonly trial: TrialResult;
   readonly parentTestDir: string;
@@ -566,13 +759,9 @@ async function writeTrialRunArtifacts(params: {
   const runDir = path.join(params.parentTestDir, runDirName);
   const grading = buildGradingArtifact(result);
   const timing = buildTimingArtifact([result]);
-  const input = extractInput(result);
   const gradingPath = path.join(runDir, 'grading.json');
-  const timingPath = path.join(runDir, 'timing.json');
-  const inputPath = input ? path.join(runDir, 'input.md') : undefined;
   const outputsDir = path.join(runDir, 'outputs');
-  const answerPath = result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined;
-  const responsePath = result.output.length > 0 ? path.join(outputsDir, 'response.md') : undefined;
+  const evalOutputPath = result.output.length > 0 ? path.join(outputsDir, 'eval.txt') : undefined;
   const attemptRunId = params.runId
     ? `${params.runId}:${runDirName}`
     : `${result.testId}:${result.target}:${runDirName}`;
@@ -585,96 +774,46 @@ async function writeTrialRunArtifacts(params: {
     runId: attemptRunId,
     duplicatePolicy: params.duplicatePolicy,
   });
-  const tracePath = path.join(runDir, CANONICAL_TRACE_ARTIFACT_PATH);
-  const transcriptPath = hasTranscriptProjection(result, envelope)
-    ? path.join(runDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH)
-    : undefined;
+  const hasTranscript = hasTranscriptProjection(result, envelope);
+  const transcriptPath = hasTranscript ? path.join(runDir, 'transcript.json') : undefined;
+  const transcriptRawPath = hasTranscript ? path.join(runDir, 'transcript-raw.jsonl') : undefined;
   const metricsPath = path.join(runDir, CANONICAL_METRICS_ARTIFACT_PATH);
-  const rawProviderLogSource = rawProviderLogSourcePath(result);
-  const rawProviderLogPath = rawProviderLogSource ? rawProviderLogArtifactPath(runDir) : undefined;
 
   await mkdir(runDir, { recursive: true });
   await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}\n`, 'utf8');
-  await writeFile(timingPath, `${JSON.stringify(timing, null, 2)}\n`, 'utf8');
-
-  if (inputPath && input) {
-    await writeFile(inputPath, input, 'utf8');
-  }
 
   await mkdir(outputsDir, { recursive: true });
-  if (answerPath && responsePath) {
-    await writeFile(answerPath, result.output, 'utf8');
-    await writeFile(responsePath, result.output, 'utf8');
-  }
-  if (rawProviderLogSource) {
-    await copyRawProviderLogArtifact(rawProviderLogSource, runDir);
+  if (evalOutputPath) {
+    await writeFile(evalOutputPath, result.output, 'utf8');
   }
-  await writeFile(tracePath, `${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}\n`, 'utf8');
-  if (transcriptPath) {
-    await writeTranscriptJsonl(transcriptPath, result, envelope);
+  if (transcriptPath && transcriptRawPath) {
     await writeFile(
-      path.join(runDir, 'transcript-raw.jsonl'),
-      await readFile(transcriptPath, 'utf8'),
+      transcriptPath,
+      `${JSON.stringify(traceEnvelopeToTranscriptMessages(envelope), null, 2)}\n`,
       'utf8',
     );
+    await writeTranscriptJsonl(transcriptRawPath, result, envelope);
   }
-  await writeMetricsArtifact({
+  const metricsArtifact = await writeMetricsArtifact({
     filePath: metricsPath,
     result,
     envelope,
-    transcriptPath,
+    traceArtifactPath: 'transcript.json',
+    transcriptArtifactPath: transcriptRawPath ? 'transcript-raw.jsonl' : undefined,
+    timingArtifactPath: null,
+    timing,
   });
 
-  const artifactPointers = await buildArtifactPointers({
-    outputDir: params.outputDir,
-    tracePath,
-    transcriptPath,
-  });
-  const resultArtifact = {
-    ...buildIndexArtifactEntry(result, {
-      outputDir: params.outputDir,
-      artifactDir: runDir,
-      gradingPath,
-      timingPath,
-      outputPath: answerPath,
-      answerPath,
-      tracePath,
-      transcriptPath,
-      metricsPath,
-      artifactPointers,
-      rawProviderLogPath,
-      inputPath,
-      responsePath,
-      projectionIdentity: envelope.projectionIdentity,
-      duplicatePolicy: params.duplicatePolicy,
-    }),
-    attempt: params.trial.attempt,
-    run: params.trial.attempt + 1,
-    run_id: runDirName,
-    experiment: params.experiment,
-  };
   await writeFile(
     path.join(runDir, 'result.json'),
-    `${JSON.stringify(resultArtifact, null, 2)}\n`,
-    'utf8',
-  );
-  await writeFile(
-    path.join(runDir, 'summary.json'),
     `${JSON.stringify(
-      {
-        attempt: params.trial.attempt,
-        run: params.trial.attempt + 1,
-        run_id: runDirName,
-        test_id: result.testId,
-        target: result.target,
-        score: result.score,
-        verdict: params.trial.verdict,
-        execution_status: result.executionStatus,
-        error: result.error,
-        cost_usd: result.costUsd,
-        duration_ms: result.durationMs,
-        token_usage: result.tokenUsage,
-      },
+      buildVercelRunResultArtifact({
+        trial: params.trial,
+        result,
+        metricsArtifact,
+        hasTranscript,
+        hasOutput: result.output.length > 0,
+      }),
       null,
       2,
     )}\n`,
@@ -1270,8 +1409,10 @@ export function buildIndexArtifactEntry(
   options: {
     outputDir: string;
     artifactDir?: string;
-    gradingPath: string;
-    timingPath: string;
+    gradingPath?: string;
+    timingPath?: string;
+    benchmarkPath?: string;
+    summaryPath?: string;
     outputPath?: string;
     answerPath?: string;
     tracePath?: string;
@@ -1310,8 +1451,18 @@ export function buildIndexArtifactEntry(
     artifact_dir: options.artifactDir
       ? toRelativeArtifactPath(options.outputDir, options.artifactDir)
       : undefined,
-    grading_path: toRelativeArtifactPath(options.outputDir, options.gradingPath),
-    timing_path: toRelativeArtifactPath(options.outputDir, options.timingPath),
+    grading_path: options.gradingPath
+      ? toRelativeArtifactPath(options.outputDir, options.gradingPath)
+      : undefined,
+    timing_path: options.timingPath
+      ? toRelativeArtifactPath(options.outputDir, options.timingPath)
+      : undefined,
+    benchmark_path: options.benchmarkPath
+      ? toRelativeArtifactPath(options.outputDir, options.benchmarkPath)
+      : undefined,
+    summary_path: options.summaryPath
+      ? toRelativeArtifactPath(options.outputDir, options.summaryPath)
+      : undefined,
     output_path: options.outputPath
       ? toRelativeArtifactPath(options.outputDir, options.outputPath)
       : undefined,
@@ -1440,14 +1591,24 @@ async function writeMetricsArtifact(params: {
   readonly result: EvaluationResult;
   readonly envelope: TraceEnvelope;
   readonly transcriptPath?: string;
-}): Promise<void> {
+  readonly traceArtifactPath?: string;
+  readonly transcriptArtifactPath?: string;
+  readonly gradingArtifactPath?: string;
+  readonly timingArtifactPath?: string | null;
+  readonly timing?: TimingArtifact;
+}): Promise<ReturnType<typeof buildMetricsArtifact> & { readonly timing?: TimingArtifact }> {
   const artifact = buildMetricsArtifact(params.result, params.envelope, {
-    tracePath: CANONICAL_TRACE_ARTIFACT_PATH,
-    transcriptPath: params.transcriptPath ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined,
-    gradingPath: 'grading.json',
-    timingPath: 'timing.json',
+    tracePath: params.traceArtifactPath ?? CANONICAL_TRACE_ARTIFACT_PATH,
+    transcriptPath:
+      params.transcriptArtifactPath ??
+      (params.transcriptPath ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined),
+    gradingPath: params.gradingArtifactPath ?? 'grading.json',
+    timingPath:
+      params.timingArtifactPath === null ? undefined : (params.timingArtifactPath ?? 'timing.json'),
   });
-  await writeFile(params.filePath, `${JSON.stringify(artifact, null, 2)}\n`, 'utf8');
+  const artifactWithTiming = params.timing ? { ...artifact, timing: params.timing } : artifact;
+  await writeFile(params.filePath, `${JSON.stringify(artifactWithTiming, null, 2)}\n`, 'utf8');
+  return artifactWithTiming;
 }
 
 function indexRecordKey(record: unknown): string | undefined {
@@ -1974,6 +2135,60 @@ export async function writeArtifactsFromResults(
     }
 
     await mkdir(plan.testDir, { recursive: true });
+
+    if (hasPersistedTrialRuns(result)) {
+      const aggregateTiming = buildRepeatAggregateTimingArtifact(result);
+      const summaryPath = path.join(plan.testDir, 'summary.json');
+      const summary = buildVercelEvalSummaryArtifact(
+        result,
+        options?.experimentMetadata?.fingerprint ?? plan.projectionIdentity.id,
+      );
+      await writeFile(plan.gradingPath, `${JSON.stringify(plan.grading, null, 2)}\n`, 'utf8');
+      await writeFile(
+        plan.perTestTimingPath,
+        `${JSON.stringify(aggregateTiming, null, 2)}\n`,
+        'utf8',
+      );
+      await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+      for (const trial of result.trials ?? []) {
+        await writeTrialRunArtifacts({
+          trial,
+          parentTestDir: plan.testDir,
+          outputDir,
+          evalFile: options?.evalFile,
+          experiment: options?.experiment,
+          runId: options?.runId,
+          duplicatePolicy,
+          testByTestId,
+        });
+      }
+
+      const nextRecord = {
+        ...buildIndexArtifactEntry(result, {
+          outputDir,
+          artifactDir: plan.testDir,
+          gradingPath: plan.gradingPath,
+          timingPath: plan.perTestTimingPath,
+          summaryPath,
+          projectionIdentity: plan.projectionIdentity,
+          duplicatePolicy,
+        }),
+        experiment: options?.experiment,
+      };
+      if (duplicatePolicy === 'update' && emittedIdentityIds.has(identityId)) {
+        const existingIndex = indexRecords.findIndex(
+          (record) => projectionIdentityRecordKey(record) === identityId,
+        );
+        if (existingIndex >= 0) {
+          indexRecords[existingIndex] = nextRecord;
+        }
+      } else {
+        indexRecords.push(nextRecord);
+      }
+      emittedIdentityIds.add(identityId);
+      continue;
+    }
+
     await writeFile(plan.gradingPath, `${JSON.stringify(plan.grading, null, 2)}\n`, 'utf8');
     await writeFile(plan.perTestTimingPath, `${JSON.stringify(plan.timing, null, 2)}\n`, 'utf8');
 

From cb5bff099b89d1ea9c688f1dabe0a2649998e385 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 04:46:55 +0200
Subject: [PATCH 13/18] feat: flatten repeat case summary timing

---
 .agents/verification.md                       |  2 +-
 apps/cli/src/commands/results/validate.ts     | 11 ++++
 .../commands/eval/artifact-writer.test.ts     | 58 +++++++++---------
 .../docs/docs/evaluation/experiments.mdx      | 11 ++--
 .../src/content/docs/docs/tools/results.mdx   |  5 +-
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |  5 +-
 packages/core/src/evaluation/run-artifacts.ts | 59 +++++++++++--------
 7 files changed, 84 insertions(+), 67 deletions(-)

diff --git a/.agents/verification.md b/.agents/verification.md
index 9b4091cf2..e54e4b7e9 100644
--- a/.agents/verification.md
+++ b/.agents/verification.md
@@ -127,7 +127,7 @@ Use live dogfood before marking PRs ready when they affect eval execution, exper
 - Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
-- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect root `index.jsonl`, root `benchmark.json`, and the repeated case folder. The repeated case folder should carry Vercel-compatible `summary.json` plus AgentV aggregate `grading.json` and `timing.json`; attempt-specific outputs, metrics, and transcripts live under `run-N/`. Each `run-N/` folder should contain Vercel-compatible `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/`, plus AgentV `grading.json` and `metrics.json` with embedded `timing`.
+- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect root `index.jsonl`, root `benchmark.json`, and the repeated case folder. The repeated case folder should carry aggregate `summary.json` with flattened snake_case timing fields plus AgentV aggregate `grading.json`; attempt-specific outputs, metrics, and transcripts live under `run-N/`. Each `run-N/` folder should contain Vercel-compatible `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/`, plus AgentV `grading.json` and `metrics.json` with embedded `timing`.
 - For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
 - Preserve review evidence in `agentv-private` on an `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, and screenshots when folder structure or UI behavior is under review.
 - If comparing against an external convention such as Vercel `agent-eval`, verify both semantic provenance and the physical `run-N` artifact layout for repeat runs.
diff --git a/apps/cli/src/commands/results/validate.ts b/apps/cli/src/commands/results/validate.ts
index e0641b39e..680f5b0b4 100644
--- a/apps/cli/src/commands/results/validate.ts
+++ b/apps/cli/src/commands/results/validate.ts
@@ -35,6 +35,7 @@ interface IndexEntry {
   readonly scores?: unknown[];
   readonly execution_status?: string;
   readonly benchmark_path?: string;
+  readonly summary_path?: string;
   readonly grading_path?: string;
   readonly timing_path?: string;
   readonly [key: string]: unknown;
@@ -204,6 +205,16 @@ function checkArtifactFiles(runDir: string, entries: IndexEntry[]): Diagnostic[]
   for (const entry of entries) {
     const testId = entry.test_id ?? '?';
 
+    if (entry.summary_path) {
+      const summaryPath = path.join(runDir, entry.summary_path);
+      if (!existsSync(summaryPath)) {
+        diagnostics.push({
+          severity: 'error',
+          message: `${testId}: summary.json not found at '${entry.summary_path}'`,
+        });
+      }
+    }
+
     if (entry.benchmark_path) {
       const benchmarkPath = path.join(runDir, entry.benchmark_path);
       if (!existsSync(benchmarkPath)) {
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index f6c9ea056..289fd2c64 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -964,28 +964,43 @@ describe('writeArtifactsFromResults', () => {
     expect(indexEntry?.summary_path).toBe('repeat-case/summary.json');
     expect(indexEntry?.benchmark_path).toBeUndefined();
     expect(indexEntry?.grading_path).toBe('repeat-case/grading.json');
-    expect(indexEntry?.timing_path).toBe('repeat-case/timing.json');
+    expect(indexEntry?.timing_path).toBeUndefined();
     expect(indexEntry?.metrics_path).toBeUndefined();
 
     const repeatEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case'));
-    expect(repeatEntries.sort()).toEqual([
-      'grading.json',
-      'run-1',
-      'run-2',
-      'summary.json',
-      'timing.json',
-    ]);
+    expect(repeatEntries.sort()).toEqual(['grading.json', 'run-1', 'run-2', 'summary.json']);
 
     const caseSummary = JSON.parse(
       await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'summary.json'), 'utf8'),
     ) as Record<string, unknown>;
-    expect(caseSummary).toEqual({
-      totalRuns: 2,
-      passedRuns: 1,
-      passRate: '50%',
-      meanDuration: 3,
-      fingerprint: expect.any(String),
+    expect(caseSummary).toMatchObject({
+      total_runs: 2,
+      passed_runs: 1,
+      pass_rate: '50%',
+      mean_duration_ms: 3000,
+      mean_duration_seconds: 3,
+      duration_ms: 6000,
+      total_duration_seconds: 6,
+      duration_stats: {
+        count: 2,
+        mean_ms: 3000,
+        mean_seconds: 3,
+        stddev_ms: 1000,
+        stddev_seconds: 1,
+        min_ms: 2000,
+        max_ms: 4000,
+      },
+      total_tokens: 0,
+      cost_usd: null,
+      token_usage: { input: 0, output: 0, reasoning: 0 },
+      usage_sources: {
+        token_usage: 'unavailable',
+        total_tokens: 'unavailable',
+        duration: 'aggregate',
+        cost: 'unavailable',
+      },
     });
+    expect(typeof caseSummary.fingerprint).toBe('string');
 
     const aggregateGrading: GradingArtifact = JSON.parse(
       await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'grading.json'), 'utf8'),
@@ -993,21 +1008,6 @@ describe('writeArtifactsFromResults', () => {
     expect(aggregateGrading.trials).toEqual(indexEntry?.trials);
     expect(aggregateGrading.aggregation).toEqual(indexEntry?.aggregation);
 
-    const aggregateTiming = JSON.parse(
-      await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'timing.json'), 'utf8'),
-    ) as TimingArtifact;
-    expect(aggregateTiming.duration_ms).toBe(6000);
-    expect(aggregateTiming.mean_duration_seconds).toBe(3);
-    expect(aggregateTiming.duration_stats).toEqual({
-      count: 2,
-      mean_ms: 3000,
-      mean_seconds: 3,
-      stddev_ms: 1000,
-      stddev_seconds: 1,
-      min_ms: 2000,
-      max_ms: 4000,
-    });
-
     for (const runDir of ['run-1', 'run-2']) {
       const runEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case', runDir));
       expect(runEntries.sort()).toEqual([
diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
index 667ad8156..cd8e40b70 100644
--- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
@@ -84,7 +84,6 @@ provenance:
 <run-dir>/benchmark.json
 <run-dir>/<suite>/<case-id>/summary.json
 <run-dir>/<suite>/<case-id>/grading.json
-<run-dir>/<suite>/<case-id>/timing.json
 <run-dir>/<suite>/<case-id>/run-1/result.json
 <run-dir>/<suite>/<case-id>/run-1/transcript.json
 <run-dir>/<suite>/<case-id>/run-1/transcript-raw.jsonl
@@ -93,11 +92,11 @@ provenance:
 <run-dir>/<suite>/<case-id>/run-1/metrics.json
 ```
 
-The repeated case aggregate folder uses `summary.json` for run-count and mean
-duration, `grading.json` for compact trial/aggregation verdicts, and
-`timing.json` for aggregate timing totals plus mean attempt duration. Root
-`index.jsonl` and root `benchmark.json` remain stable for existing CI summary
-scripts and uploaded artifact consumers.
+The repeated case aggregate folder uses `summary.json` for run-count, pass-rate,
+fingerprint, and flattened snake_case timing fields such as
+`mean_duration_ms`, and `grading.json` for compact trial/aggregation verdicts.
+Root `index.jsonl` and root `benchmark.json` remain stable for existing CI
+summary scripts and uploaded artifact consumers.
 
 ## Targets and setup
 
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index a06674cd4..b715ab95f 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -119,8 +119,9 @@ derived projection over `trace.json`, the result row, and
 comparison exports, and metric-style graders; it is not canonical trace storage
 and does not carry token/cost usage.
 
-Repeat-enabled cases use aggregate `summary.json`, `grading.json`, and
-`timing.json`, then store attempt details under `run-N/`. Each `run-N/` contains Vercel-compatible
+Repeat-enabled cases use aggregate `summary.json` with flattened snake_case
+timing fields plus aggregate `grading.json`, then store attempt details under
+`run-N/`. Each `run-N/` contains Vercel-compatible
 `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/eval.txt`,
 plus AgentV `grading.json` and `metrics.json` with embedded timing.
 
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index 2e28d717f..9375e99bb 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -150,9 +150,9 @@ Public docs and implementation notes must not reference non-public sources. If a
 
 - KTD1. The repeat config attaches to the **experiment** surface, not to `eval.yaml` `execution`, per the experiments-separation decision (epic `av-991`, recorded on `av-991.1`). This aligns with Vercel agent-eval, where `runs`/`earlyExit` are experiment-level. This epic (`av-i0l`) owns the repeat **mechanics** (schema shape, gate policies, attempt aggregation, flake classification, and the run-N artifact layout); `av-991` owns **placement** (the experiment contract the repeat block lives on). The existing `execution.trials` code path is **hard-removed** (no compatibility alias) because usage is rare; its behavior is replaced by the experiment-level repeat block. Because the experiment surface is delivered by `av-991`, the schema work in `av-i0l.1` depends on that contract landing.
 - KTD2. Keep one-run CI as the default. Repeat runs are for reliability evidence unless `repeat.gate` says they are a CI gate.
-- KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json`, `grading.json`, `timing.json`, and `run-N/` directories so existing aggregate consumers do not inflate case counts.
+- KTD3. Store aggregate rows in the top-level `index.jsonl`, not one row per attempt. Attempt details live in case-local `summary.json`, `grading.json`, and `run-N/` directories so existing aggregate consumers do not inflate case counts.
 - KTD4. Single-run cases keep direct case-local files instead of always nesting under `run-1`. This preserves the simple default artifact shape and makes `.agentv/results/<experiment>/<timestamp>/<case-id>/grading.json` easy to inspect. Repeat-enabled cases use `run-1/`, `run-2/`, and so on under the case directory.
-- KTD5. Root run aggregates keep the existing AgentV `benchmark.json` for compatibility. Repeat case aggregates use Vercel-compatible `summary.json` plus AgentV aggregate `grading.json` and `timing.json`; repeat attempts use `run-N/` children.
+- KTD5. Root run aggregates keep the existing AgentV `benchmark.json` for compatibility. Repeat case aggregates use `summary.json` with flattened snake_case timing fields plus AgentV aggregate `grading.json`; repeat attempts use `run-N/` children.
 - KTD6. `pass_at_k` keeps the existing AgentV/Vercel ergonomics: early exit is enabled unless explicitly disabled. Full reliability sampling requires `early_exit: false` on the experiment and should be recorded because it changes cost and statistics.
 - KTD7. Do not inherit Vercel's implicit CI ambiguity. All policies that can make one failed plus one passed attempt count as passing must be visible in config and artifacts.
 - KTD8. Reuse current failure classification fields before adding new enums. Add aggregate classification fields only after mapping from `execution_status`, `failure_stage`, and `failure_reason_code` proves insufficient.
@@ -261,7 +261,6 @@ Repeat-run cases use attempt directories:
 ```text
 .agentv/results/<experiment>/<timestamp>/<case-id>/summary.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/grading.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/timing.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/grading.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/metrics.json
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 9e324a1c4..37e8f53cd 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -374,12 +374,20 @@ export interface VercelRunResultArtifact {
   };
 }
 
-export interface VercelEvalSummaryArtifact {
-  readonly totalRuns: number;
-  readonly passedRuns: number;
-  readonly passRate: string;
-  readonly meanDuration: number;
+export interface RepeatCaseSummaryArtifact {
+  readonly total_runs: number;
+  readonly passed_runs: number;
+  readonly pass_rate: string;
+  readonly mean_duration_ms: number;
+  readonly mean_duration_seconds: number;
   readonly fingerprint: string;
+  readonly total_tokens: number;
+  readonly duration_ms: number;
+  readonly total_duration_seconds: number;
+  readonly duration_stats?: TimingArtifact['duration_stats'];
+  readonly cost_usd: number | null;
+  readonly token_usage: TimingArtifact['token_usage'];
+  readonly usage_sources: TimingArtifact['usage_sources'];
 }
 
 export type AdditionalResultArtifactsWriter = (
@@ -650,7 +658,7 @@ function buildRepeatAggregateTimingArtifact(result: EvaluationResult): TimingArt
   };
 }
 
-function formatVercelPassRate(passedRuns: number, totalRuns: number): string {
+function formatRepeatPassRate(passedRuns: number, totalRuns: number): string {
   if (totalRuns === 0) {
     return '0%';
   }
@@ -671,27 +679,31 @@ function fallbackRepeatFingerprint(result: EvaluationResult): string {
     .digest('hex');
 }
 
-function buildVercelEvalSummaryArtifact(
+function buildRepeatCaseSummaryArtifact(
   result: EvaluationResult,
+  timing: TimingArtifact,
   fingerprint?: string,
-): VercelEvalSummaryArtifact {
+): RepeatCaseSummaryArtifact {
   const trials = result.trials ?? [];
   const totalRuns = trials.length;
   const passedRuns = trials.filter((trial) => trial.verdict === 'pass').length;
-  const durations = repeatAttemptResults(result).map(resultDurationSeconds);
-  const meanDuration =
-    durations.length === 0
-      ? 0
-      : Math.round(
-          (durations.reduce((sum, duration) => sum + duration, 0) / durations.length) * 1000,
-        ) / 1000;
+  const fallbackMeanMs = totalRuns > 0 ? roundMillis(timing.duration_ms / totalRuns) : 0;
+  const meanDurationMs = timing.mean_duration_ms ?? fallbackMeanMs;
 
   return {
-    totalRuns,
-    passedRuns,
-    passRate: formatVercelPassRate(passedRuns, totalRuns),
-    meanDuration,
+    total_runs: totalRuns,
+    passed_runs: passedRuns,
+    pass_rate: formatRepeatPassRate(passedRuns, totalRuns),
+    mean_duration_ms: meanDurationMs,
+    mean_duration_seconds: timing.mean_duration_seconds ?? roundSecondsFromMs(meanDurationMs),
     fingerprint: fingerprint ?? fallbackRepeatFingerprint(result),
+    total_tokens: timing.total_tokens,
+    duration_ms: timing.duration_ms,
+    total_duration_seconds: timing.total_duration_seconds,
+    duration_stats: timing.duration_stats,
+    cost_usd: timing.cost_usd,
+    token_usage: timing.token_usage,
+    usage_sources: timing.usage_sources,
   };
 }
 
@@ -2139,16 +2151,12 @@ export async function writeArtifactsFromResults(
     if (hasPersistedTrialRuns(result)) {
       const aggregateTiming = buildRepeatAggregateTimingArtifact(result);
       const summaryPath = path.join(plan.testDir, 'summary.json');
-      const summary = buildVercelEvalSummaryArtifact(
+      const summary = buildRepeatCaseSummaryArtifact(
         result,
+        aggregateTiming,
         options?.experimentMetadata?.fingerprint ?? plan.projectionIdentity.id,
       );
       await writeFile(plan.gradingPath, `${JSON.stringify(plan.grading, null, 2)}\n`, 'utf8');
-      await writeFile(
-        plan.perTestTimingPath,
-        `${JSON.stringify(aggregateTiming, null, 2)}\n`,
-        'utf8',
-      );
       await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
       for (const trial of result.trials ?? []) {
         await writeTrialRunArtifacts({
@@ -2168,7 +2176,6 @@ export async function writeArtifactsFromResults(
           outputDir,
           artifactDir: plan.testDir,
           gradingPath: plan.gradingPath,
-          timingPath: plan.perTestTimingPath,
           summaryPath,
           projectionIdentity: plan.projectionIdentity,
           duplicatePolicy,

From 7910e080f8e89f58d8df253e05b8c3d22586673c Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 04:58:45 +0200
Subject: [PATCH 14/18] feat: remove execution metrics from grading artifacts

---
 apps/cli/src/commands/pipeline/bench.ts       |  1 -
 .../commands/eval/artifact-writer.test.ts     |  6 +++---
 .../results/export-e2e-providers.test.ts      | 21 ++++++++++++-------
 apps/cli/test/commands/results/export.test.ts |  4 ++--
 .../src/content/docs/docs/tools/results.mdx   |  4 ++--
 packages/core/src/evaluation/run-artifacts.ts | 13 ------------
 6 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/apps/cli/src/commands/pipeline/bench.ts b/apps/cli/src/commands/pipeline/bench.ts
index 7fe4db49f..f7466f0ed 100644
--- a/apps/cli/src/commands/pipeline/bench.ts
+++ b/apps/cli/src/commands/pipeline/bench.ts
@@ -132,7 +132,6 @@ export const evalBenchCommand = command({
       const grading = {
         assertions: allAssertions,
         summary: { passed, failed, total: allAssertions.length, pass_rate: passRate },
-        execution_metrics: { tool_calls: {}, total_tool_calls: 0, errors_encountered: 0 },
         graders: evaluators.map((e) => ({
           name: e.name,
           type: e.type,
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 289fd2c64..be38cdf26 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -226,10 +226,10 @@ describe('buildGradingArtifact', () => {
     expect(grading.graders?.[1].score).toBe(0.7);
   });
 
-  it('records error as errors_encountered', () => {
+  it('keeps grading.json focused on grading evidence', () => {
     const result = makeResult({ error: 'Timeout exceeded' });
     const grading = buildGradingArtifact(result);
-    expect(grading.execution_metrics.errors_encountered).toBe(1);
+    expect(grading).not.toHaveProperty('execution_metrics');
   });
 
   it('handles result with no assertions or scores', () => {
@@ -875,7 +875,7 @@ describe('writeArtifactsFromResults', () => {
       await readFile(path.join(paths.testArtifactDir, 'alpha', 'grading.json'), 'utf8'),
     );
     expect(alphaGrading.summary).toBeDefined();
-    expect(alphaGrading.execution_metrics).toBeDefined();
+    expect(alphaGrading).not.toHaveProperty('execution_metrics');
 
     const alphaTiming: TimingArtifact = JSON.parse(
       await readFile(path.join(paths.testArtifactDir, 'alpha', 'timing.json'), 'utf8'),
diff --git a/apps/cli/test/commands/results/export-e2e-providers.test.ts b/apps/cli/test/commands/results/export-e2e-providers.test.ts
index 19c0e4be4..cb1b2eee6 100644
--- a/apps/cli/test/commands/results/export-e2e-providers.test.ts
+++ b/apps/cli/test/commands/results/export-e2e-providers.test.ts
@@ -466,10 +466,12 @@ describe('export e2e — multi-provider metrics verification', () => {
       expect(grading.summary.failed).toBe(0);
       expect(grading.summary.pass_rate).toBe(1.0);
 
-      // Tool calls from trace
-      expect(grading.execution_metrics.total_tool_calls).toBe(3);
-      expect(grading.execution_metrics.tool_calls.Read).toBe(2);
-      expect(grading.execution_metrics.tool_calls.Write).toBe(1);
+      const metrics = JSON.parse(
+        readFileSync(path.join(artifactDir(outputDir, CLAUDE_CLI_RESULT), 'metrics.json'), 'utf8'),
+      );
+      expect(metrics.metrics.total_tool_calls).toBe(3);
+      expect(metrics.metrics.tool_call_counts.Read).toBe(2);
+      expect(metrics.metrics.tool_call_counts.Write).toBe(1);
 
       // Graders
       expect(grading.graders).toHaveLength(1);
@@ -490,8 +492,10 @@ describe('export e2e — multi-provider metrics verification', () => {
       expect(grading.summary.failed).toBe(1);
       expect(grading.summary.pass_rate).toBe(0.5);
 
-      // No trace means no tool calls
-      expect(grading.execution_metrics.total_tool_calls).toBe(0);
+      const metrics = JSON.parse(
+        readFileSync(path.join(artifactDir(outputDir, COPILOT_RESULT), 'metrics.json'), 'utf8'),
+      );
+      expect(metrics.metrics.total_tool_calls).toBe(0);
     });
 
     it('should handle error result in grading', async () => {
@@ -507,7 +511,10 @@ describe('export e2e — multi-provider metrics verification', () => {
       // Error result has empty assertions
       expect(grading.summary.total).toBe(0);
       expect(grading.summary.pass_rate).toBe(0);
-      expect(grading.execution_metrics.errors_encountered).toBe(1);
+      const metrics = JSON.parse(
+        readFileSync(path.join(artifactDir(outputDir, ERROR_RESULT), 'metrics.json'), 'utf8'),
+      );
+      expect(metrics.metrics.errors_encountered).toBe(1);
     });
 
     it('should produce grading files for all test IDs in multi-target run', async () => {
diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts
index 2056c16d7..80af19437 100644
--- a/apps/cli/test/commands/results/export.test.ts
+++ b/apps/cli/test/commands/results/export.test.ts
@@ -589,8 +589,8 @@ describe('results export', () => {
     expect(grading.summary).toHaveProperty('total');
     expect(grading.summary).toHaveProperty('pass_rate');
 
-    // Has execution_metrics
-    expect(grading.execution_metrics).toBeDefined();
+    // Grading artifacts stay focused on assertion evidence; execution data lives in metrics.json.
+    expect(grading).not.toHaveProperty('execution_metrics');
 
     // Has evaluators
     expect(grading.graders).toBeDefined();
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index b715ab95f..e1a56ccbb 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -149,11 +149,11 @@ Vercel `@vercel/agent-eval` `results.o11y` maps into AgentV like this:
 | `shellCommands` | `metrics.shell_commands` | `metrics.json` |
 | `filesRead` | `metrics.files_read` | `metrics.json` |
 | `filesModified` | `metrics.files_modified` | `metrics.json` |
-| `toolCalls` | `metrics.tool_call_events`, `metrics.tool_calls`, and `metrics.tool_call_counts` | `metrics.json`; compact counts can also appear in `grading.json.execution_metrics` and `benchmark.json.run_summary[*].tool_calls` |
+| `toolCalls` | `metrics.tool_call_events`, `metrics.tool_calls`, and `metrics.tool_call_counts` | `metrics.json`; compact counts can also appear in `benchmark.json.run_summary[*].tool_calls` |
 | `totalToolCalls` | `metrics.total_tool_calls` | `metrics.json` |
 | `webFetches` | `metrics.web_fetches` | `metrics.json` |
 | `totalTurns` | `metrics.total_turns` | `metrics.json`; conversational rows remain in `transcript.jsonl` |
-| `errors` | `metrics.errors` | `metrics.json`; compact count remains in `grading.json.execution_metrics.errors_encountered` |
+| `errors` | `metrics.errors` | `metrics.json` |
 | `thinkingBlocks` | `metrics.reasoning_blocks` and `thinking_blocks` | `metrics.json` |
 
 Agent Skills eval artifacts map into AgentV like this:
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 37e8f53cd..9abc161d3 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -152,11 +152,6 @@ export interface GradingArtifact {
     readonly total: number;
     readonly pass_rate: number;
   };
-  readonly execution_metrics: {
-    readonly tool_calls: Record<string, number>;
-    readonly total_tool_calls: number;
-    readonly errors_encountered: number;
-  };
   readonly graders?: readonly {
     readonly name: string;
     readonly type: string;
@@ -907,9 +902,6 @@ export function buildGradingArtifact(result: EvaluationResult): GradingArtifact
   const failed = assertions.filter((e) => !e.passed).length;
   const total = assertions.length;
 
-  const { toolCalls, total: totalToolCalls } = countToolCalls(result);
-  const errorsEncountered = result.error ? 1 : 0;
-
   return {
     assertions,
     summary: {
@@ -918,11 +910,6 @@ export function buildGradingArtifact(result: EvaluationResult): GradingArtifact
       total,
       pass_rate: total > 0 ? Math.round((passed / total) * 1000) / 1000 : 0,
     },
-    execution_metrics: {
-      tool_calls: toolCalls,
-      total_tool_calls: totalToolCalls,
-      errors_encountered: errorsEncountered,
-    },
     graders: buildEvaluators(result.scores),
     workspace_changes: parseWorkspaceChanges(result.fileChanges),
     conversation: result.conversationId

From 3726b7630ecb70ec6c565325eed1f7c499c1b4b7 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 05:20:25 +0200
Subject: [PATCH 15/18] feat: use answer artifact for run outputs

---
 apps/cli/src/commands/results/export.ts       |  2 +-
 .../src/commands/results/projection-bundle.ts |  3 --
 .../commands/eval/artifact-writer.test.ts     | 10 ++++---
 apps/cli/test/commands/results/export.test.ts |  5 +---
 .../docs/docs/evaluation/experiments.mdx      |  2 +-
 .../docs/docs/evaluation/running-evals.mdx    |  2 +-
 .../src/content/docs/docs/tools/results.mdx   |  6 ++--
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |  6 ++--
 packages/core/src/evaluation/run-artifacts.ts | 28 +++++--------------
 9 files changed, 23 insertions(+), 41 deletions(-)

diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts
index 0df4c7678..59dafe02d 100644
--- a/apps/cli/src/commands/results/export.ts
+++ b/apps/cli/src/commands/results/export.ts
@@ -10,7 +10,7 @@
  *       grading.json           — per-test grading artifact (assertions, graders)
  *       timing.json            — per-test timing artifact
  *       outputs/
- *         response.md          — human-readable agent response for this test
+ *         answer.md            — human-readable agent response for this test
  *       input.md               — human-readable input messages for this test
  *
  * This module delegates artifact building to the shared artifact-writer so
diff --git a/apps/cli/src/commands/results/projection-bundle.ts b/apps/cli/src/commands/results/projection-bundle.ts
index 1f03a9b7c..e6b69c6d5 100644
--- a/apps/cli/src/commands/results/projection-bundle.ts
+++ b/apps/cli/src/commands/results/projection-bundle.ts
@@ -94,7 +94,6 @@ export type ProjectionBundleArtifactRefs = Partial<
     | 'input_path'
     | 'output_path'
     | 'answer_path'
-    | 'response_path'
     | 'transcript_path'
     | 'metrics_path'
     | 'task_dir'
@@ -180,7 +179,6 @@ function artifactRefs(
     input_path: indexEntry.input_path,
     output_path: indexEntry.output_path,
     answer_path: indexEntry.answer_path,
-    response_path: indexEntry.response_path,
     transcript_path: indexEntry.transcript_path,
     metrics_path: indexEntry.metrics_path,
     trace_path: tracePathFor(indexEntry),
@@ -284,7 +282,6 @@ function buildEntry(
     artifacts: {
       trace_path: tracePathFor(indexRecord ?? plannedIndexEntry),
       answer_path: result.output.length > 0 ? 'outputs/answer.md' : undefined,
-      response_path: result.output.length > 0 ? 'outputs/response.md' : undefined,
     },
     duplicatePolicy: options.duplicatePolicy,
     capture: captureOptions(includeRawContent),
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index be38cdf26..7ef2c7c28 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -578,7 +578,8 @@ describe('buildIndexArtifactEntry', () => {
         outputDir: '/tmp/artifacts',
         gradingPath: '/tmp/artifacts/alpha/grading.json',
         timingPath: '/tmp/artifacts/alpha/timing.json',
-        outputPath: '/tmp/artifacts/alpha/outputs/response.md',
+        outputPath: '/tmp/artifacts/alpha/outputs/answer.md',
+        answerPath: '/tmp/artifacts/alpha/outputs/answer.md',
         inputPath: '/tmp/artifacts/alpha/input.md',
       },
     );
@@ -609,7 +610,8 @@ describe('buildIndexArtifactEntry', () => {
       error: 'model drift',
       grading_path: 'alpha/grading.json',
       timing_path: 'alpha/timing.json',
-      output_path: 'alpha/outputs/response.md',
+      output_path: 'alpha/outputs/answer.md',
+      answer_path: 'alpha/outputs/answer.md',
       input_path: 'alpha/input.md',
     });
   });
@@ -1032,11 +1034,11 @@ describe('writeArtifactsFromResults', () => {
       model: 'test-target',
       transcriptPath: './transcript.json',
       transcriptRawPath: './transcript-raw.jsonl',
-      outputPaths: { eval: './outputs/eval.txt' },
+      outputPaths: { answer: './outputs/answer.md' },
     });
 
     const runTwoAnswer = await readFile(
-      path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'outputs', 'eval.txt'),
+      path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'outputs', 'answer.md'),
       'utf8',
     );
     expect(runTwoAnswer).toBe('second attempt');
diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts
index 80af19437..53f7a1cfa 100644
--- a/apps/cli/test/commands/results/export.test.ts
+++ b/apps/cli/test/commands/results/export.test.ts
@@ -352,7 +352,6 @@ describe('results export', () => {
       input_path: 'privacy/test-private/input.md',
       output_path: 'privacy/test-private/outputs/answer.md',
       answer_path: 'privacy/test-private/outputs/answer.md',
-      response_path: 'privacy/test-private/outputs/response.md',
       trace_path: 'privacy/test-private/trace.json',
     });
     expect(bundle.entries[0].trace.envelope_ref).toBe('privacy/test-private/trace.json');
@@ -417,7 +416,6 @@ describe('results export', () => {
       timing_path: 'demo/test-greeting/timing.json',
       output_path: 'demo/test-greeting/outputs/answer.md',
       answer_path: 'demo/test-greeting/outputs/answer.md',
-      response_path: 'demo/test-greeting/outputs/response.md',
       transcript_path: 'demo/test-greeting/transcript.jsonl',
       input_path: 'demo/test-greeting/input.md',
     });
@@ -613,8 +611,7 @@ describe('results export', () => {
     expect(readFileSync(answerPath, 'utf8')).toBe('Hello, Alice!');
 
     const responsePath = path.join(artifactDir(outputDir, RESULT_FULL), 'outputs', 'response.md');
-    expect(existsSync(responsePath)).toBe(true);
-    expect(readFileSync(responsePath, 'utf8')).toBe('Hello, Alice!');
+    expect(existsSync(responsePath)).toBe(false);
   });
 
   it('should group results by target in benchmark.json', async () => {
diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
index cd8e40b70..ecf1c7cac 100644
--- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
@@ -87,7 +87,7 @@ provenance:
 <run-dir>/<suite>/<case-id>/run-1/result.json
 <run-dir>/<suite>/<case-id>/run-1/transcript.json
 <run-dir>/<suite>/<case-id>/run-1/transcript-raw.jsonl
-<run-dir>/<suite>/<case-id>/run-1/outputs/eval.txt
+<run-dir>/<suite>/<case-id>/run-1/outputs/answer.md
 <run-dir>/<suite>/<case-id>/run-1/grading.json
 <run-dir>/<suite>/<case-id>/run-1/metrics.json
 ```
diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
index 22efc61e1..18d05cedd 100644
--- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
@@ -115,7 +115,7 @@ my-results/
     grading.json
     timing.json
     input.md
-    outputs/response.md
+    outputs/answer.md
     task/
       EVAL.yaml
       targets.yaml
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index e1a56ccbb..bfc1dd4db 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -122,7 +122,7 @@ and does not carry token/cost usage.
 Repeat-enabled cases use aggregate `summary.json` with flattened snake_case
 timing fields plus aggregate `grading.json`, then store attempt details under
 `run-N/`. Each `run-N/` contains Vercel-compatible
-`result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/eval.txt`,
+`result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md`,
 plus AgentV `grading.json` and `metrics.json` with embedded timing.
 
 `transcript.jsonl` remains the ordered conversational/log compatibility
@@ -161,7 +161,7 @@ Agent Skills eval artifacts map into AgentV like this:
 | Agent Skills pattern | AgentV field | Artifact location |
 |----------------------|--------------|-------------------|
 | Authored `evals/evals.json` cases | AgentV eval cases and task bundle paths | Eval source plus optional `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `index.jsonl` |
-| Per-case `outputs/` | Generated target output artifacts | `outputs/answer.md`, `outputs/response.md` |
+| Per-case `outputs/` | Generated target output artifacts | `outputs/answer.md` |
 | Per-case sidecars | Trace, transcript, metrics, and raw provider evidence | `trace.json`, `transcript.jsonl`, `metrics.json`, `provider.log` |
 | Per-case `timing.json` | Duration, token totals, cost, and usage source labels | `timing.json` |
 | Per-case `grading.json` | Assertions, graders, execution metrics, workspace changes | `grading.json`; summary fields can reference the same trace/result facts |
@@ -204,7 +204,7 @@ export `index.jsonl` and use `artifact_refs.status: "emitted"`.
 
 Raw prompt text, final output, and tool arguments/results are excluded by
 default, and raw-bearing artifact refs such as `grading_path`, `input_path`,
-`answer_path`, `response_path`, `transcript_path`, and `trace_path` are omitted from
+`answer_path`, `transcript_path`, and `trace_path` are omitted from
 metadata-only bundles. To include raw payloads and raw-bearing refs in the
 bundle, opt in explicitly:
 
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index 9375e99bb..a462a249c 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -249,7 +249,7 @@ Single-run cases keep direct case-local files:
 .agentv/results/<experiment>/<timestamp>/<case-id>/input.md
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/trace.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/transcript.jsonl
-.agentv/results/<experiment>/<timestamp>/<case-id>/outputs/response.md
+.agentv/results/<experiment>/<timestamp>/<case-id>/outputs/answer.md
 ```
 
 Rationale: the common path stays readable, old mental models stay close, and no user pays a `run-1/` nesting tax for default CI.
@@ -266,13 +266,13 @@ Repeat-run cases use attempt directories:
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/metrics.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript-raw.jsonl
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/eval.txt
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/answer.md
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/grading.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/metrics.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript-raw.jsonl
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/eval.txt
+.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/answer.md
 ```
 
 `<case-id>` should reuse the sanitized artifact key produced by the current artifact writer, including suite disambiguation where needed. Attempt directories are one-indexed because users naturally inspect `run-1`, `run-2`, and this matches the Vercel comparison.
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 9abc161d3..6b4a47e8e 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -308,7 +308,6 @@ export interface IndexArtifactEntry {
   readonly artifact_pointers?: ResultArtifactPointersWire;
   readonly raw_provider_log_path?: string;
   readonly input_path?: string;
-  readonly response_path?: string;
   readonly task_dir?: string;
   readonly eval_path?: string;
   readonly targets_path?: string;
@@ -364,7 +363,7 @@ export interface VercelRunResultArtifact {
     readonly thinkingBlocks: number;
   };
   readonly outputPaths?: {
-    readonly eval?: string;
+    readonly answer?: string;
     readonly scripts?: Record<string, string>;
   };
 }
@@ -743,7 +742,7 @@ function buildVercelRunResultArtifact(params: {
       errors: metrics.errors,
       thinkingBlocks: metrics.thinking_blocks,
     },
-    outputPaths: params.hasOutput ? { eval: './outputs/eval.txt' } : undefined,
+    outputPaths: params.hasOutput ? { answer: './outputs/answer.md' } : undefined,
   }) as unknown as VercelRunResultArtifact;
 }
 
@@ -768,7 +767,8 @@ async function writeTrialRunArtifacts(params: {
   const timing = buildTimingArtifact([result]);
   const gradingPath = path.join(runDir, 'grading.json');
   const outputsDir = path.join(runDir, 'outputs');
-  const evalOutputPath = result.output.length > 0 ? path.join(outputsDir, 'eval.txt') : undefined;
+  const answerOutputPath =
+    result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined;
   const attemptRunId = params.runId
     ? `${params.runId}:${runDirName}`
     : `${result.testId}:${result.target}:${runDirName}`;
@@ -790,8 +790,8 @@ async function writeTrialRunArtifacts(params: {
   await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}\n`, 'utf8');
 
   await mkdir(outputsDir, { recursive: true });
-  if (evalOutputPath) {
-    await writeFile(evalOutputPath, result.output, 'utf8');
+  if (answerOutputPath) {
+    await writeFile(answerOutputPath, result.output, 'utf8');
   }
   if (transcriptPath && transcriptRawPath) {
     await writeFile(
@@ -1316,7 +1316,6 @@ function buildTraceEnvelopeSidecar(params: TraceEnvelopeSidecarParams): TraceEnv
     artifacts: {
       trace_path: CANONICAL_TRACE_ARTIFACT_PATH,
       answer_path: params.result.output.length > 0 ? 'outputs/answer.md' : undefined,
-      response_path: params.result.output.length > 0 ? 'outputs/response.md' : undefined,
       transcript_path: hasTranscript ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined,
       metrics_path: CANONICAL_METRICS_ARTIFACT_PATH,
       raw_provider_log_path: rawProviderLogSourcePath(params.result) ? 'provider.log' : undefined,
@@ -1420,7 +1419,6 @@ export function buildIndexArtifactEntry(
     artifactPointers?: ResultArtifactPointersWire;
     rawProviderLogPath?: string;
     inputPath?: string;
-    responsePath?: string;
     extraIndexFields?: AdditionalResultIndexFields;
     projectionIdentity?: ProjectionIdentity;
     duplicatePolicy?: ExportDuplicatePolicy;
@@ -1484,9 +1482,6 @@ export function buildIndexArtifactEntry(
     input_path: options.inputPath
       ? toRelativeArtifactPath(options.outputDir, options.inputPath)
       : undefined,
-    response_path: options.responsePath
-      ? toRelativeArtifactPath(options.outputDir, options.responsePath)
-      : undefined,
     ...options.extraIndexFields,
     external_trace: toIndexExternalTrace(result, options.projectionIdentity?.dimensions.runId),
     projection_identity: options.projectionIdentity
@@ -1548,9 +1543,6 @@ export function buildResultIndexArtifact(
       ? path.posix.join(artifactSubdir, 'provider.log')
       : undefined,
     artifact_pointers: options?.artifactPointers,
-    response_path: hasAnswer
-      ? path.posix.join(artifactSubdir, 'outputs', 'response.md')
-      : undefined,
     ...extraIndexFields,
     external_trace: toIndexExternalTrace(result, options?.projectionIdentity?.dimensions.runId),
     projection_identity: options?.projectionIdentity
@@ -1959,7 +1951,6 @@ export async function writePerTestArtifacts(
     await mkdir(outputsDir, { recursive: true });
     if (result.output.length > 0) {
       await writeFile(path.join(outputsDir, 'answer.md'), result.output, 'utf8');
-      await writeFile(path.join(outputsDir, 'response.md'), result.output, 'utf8');
     }
     const rawProviderLogSource = rawProviderLogSourcePath(result);
     if (rawProviderLogSource) {
@@ -2057,8 +2048,6 @@ export async function writeArtifactsFromResults(
     const inputPath = input ? path.join(testDir, 'input.md') : undefined;
     const outputsDir = path.join(testDir, 'outputs');
     const answerPath = result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined;
-    const responsePath =
-      result.output.length > 0 ? path.join(outputsDir, 'response.md') : undefined;
     const envelope = buildTraceEnvelopeSidecar({
       result,
       outputDir,
@@ -2093,7 +2082,6 @@ export async function writeArtifactsFromResults(
       inputPath,
       outputsDir,
       answerPath,
-      responsePath,
       tracePath,
       metricsPath,
       envelope,
@@ -2191,9 +2179,8 @@ export async function writeArtifactsFromResults(
     }
 
     await mkdir(plan.outputsDir, { recursive: true });
-    if (plan.answerPath && plan.responsePath) {
+    if (plan.answerPath) {
       await writeFile(plan.answerPath, result.output, 'utf8');
-      await writeFile(plan.responsePath, result.output, 'utf8');
     }
     if (plan.rawProviderLogSource) {
       await copyRawProviderLogArtifact(plan.rawProviderLogSource, plan.testDir);
@@ -2252,7 +2239,6 @@ export async function writeArtifactsFromResults(
         artifactPointers,
         rawProviderLogPath: plan.rawProviderLogPath,
         inputPath: plan.inputPath,
-        responsePath: plan.responsePath,
         extraIndexFields,
         projectionIdentity: plan.projectionIdentity,
         duplicatePolicy,

From e5607f819a7256b22927677e6fd929537da19925 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 06:47:48 +0200
Subject: [PATCH 16/18] feat: align repeat artifacts with task prompts

---
 apps/cli/src/commands/results/export.ts       |   5 +-
 .../commands/eval/artifact-writer.test.ts     |  35 +++++-
 apps/cli/test/commands/results/export.test.ts |  14 +--
 apps/cli/test/commands/results/serve.test.ts  |   4 +-
 .../__fixtures__/structured-transcript.ts     |   2 +-
 .../docs/docs/evaluation/eval-files.mdx       |  29 +++++
 .../docs/docs/evaluation/running-evals.mdx    |   7 +-
 .../src/content/docs/docs/tools/results.mdx   |   2 +-
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |   2 +-
 .../trace-envelope-implementation-spec.md     |   2 +-
 .../input-files-shorthand/evals/PROMPT.md     |   3 +
 .../evals/dataset.eval.yaml                   |  14 +++
 .../evaluation/loaders/prompt-md-fallback.ts  | 113 ++++++++++++++++++
 packages/core/src/evaluation/run-artifacts.ts |  73 +++++++++--
 .../evaluation/validation/eval-validator.ts   |  21 +++-
 packages/core/src/evaluation/yaml-parser.ts   |  38 +++++-
 .../evaluation/input-files-shorthand.test.ts  |  49 ++++++++
 .../validation/eval-validator.test.ts         |  40 +++++++
 18 files changed, 418 insertions(+), 35 deletions(-)
 create mode 100644 examples/features/input-files-shorthand/evals/PROMPT.md
 create mode 100644 packages/core/src/evaluation/loaders/prompt-md-fallback.ts

diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts
index 59dafe02d..bfd4e51df 100644
--- a/apps/cli/src/commands/results/export.ts
+++ b/apps/cli/src/commands/results/export.ts
@@ -9,9 +9,8 @@
  *     <test-id>/
  *       grading.json           — per-test grading artifact (assertions, graders)
  *       timing.json            — per-test timing artifact
- *       outputs/
- *         answer.md            — human-readable agent response for this test
- *       input.md               — human-readable input messages for this test
+ *       outputs/answer.md              — human-readable agent response for this test
+ *       task/PROMPT.md               — human-readable input messages for this test
  *
  * This module delegates artifact building to the shared artifact-writer so
  * that benchmark/grading/timing schemas stay aligned with `agentv eval`.
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index 7ef2c7c28..d9a89ee4f 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -580,7 +580,7 @@ describe('buildIndexArtifactEntry', () => {
         timingPath: '/tmp/artifacts/alpha/timing.json',
         outputPath: '/tmp/artifacts/alpha/outputs/answer.md',
         answerPath: '/tmp/artifacts/alpha/outputs/answer.md',
-        inputPath: '/tmp/artifacts/alpha/input.md',
+        inputPath: '/tmp/artifacts/alpha/task/PROMPT.md',
       },
     );
 
@@ -612,7 +612,7 @@ describe('buildIndexArtifactEntry', () => {
       timing_path: 'alpha/timing.json',
       output_path: 'alpha/outputs/answer.md',
       answer_path: 'alpha/outputs/answer.md',
-      input_path: 'alpha/input.md',
+      input_path: 'alpha/task/PROMPT.md',
     });
   });
 
@@ -945,7 +945,20 @@ describe('writeArtifactsFromResults', () => {
       }),
     ];
 
-    const paths = await writeArtifactsFromResults(results, testDir);
+    const sourceTests = [
+      {
+        id: 'repeat-case',
+        input: [{ role: 'user', content: 'Repeat this task prompt.' }],
+        expected_output: [],
+        reference_answer: '',
+        file_paths: [],
+        criteria: 'Repeats the task prompt',
+        evaluator: 'llm-grader',
+        assertions: [],
+      } as unknown as EvalTest,
+    ];
+
+    const paths = await writeArtifactsFromResults(results, testDir, { sourceTests });
 
     const [indexEntry] = (await readFile(paths.indexPath, 'utf8'))
       .trim()
@@ -964,13 +977,27 @@ describe('writeArtifactsFromResults', () => {
     });
     expect(indexEntry?.artifact_dir).toBe('repeat-case');
     expect(indexEntry?.summary_path).toBe('repeat-case/summary.json');
+    expect(indexEntry?.task_dir).toBe('repeat-case/task');
+    expect(indexEntry?.input_path).toBe('repeat-case/task/PROMPT.md');
     expect(indexEntry?.benchmark_path).toBeUndefined();
     expect(indexEntry?.grading_path).toBe('repeat-case/grading.json');
     expect(indexEntry?.timing_path).toBeUndefined();
     expect(indexEntry?.metrics_path).toBeUndefined();
 
     const repeatEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case'));
-    expect(repeatEntries.sort()).toEqual(['grading.json', 'run-1', 'run-2', 'summary.json']);
+    expect(repeatEntries.sort()).toEqual([
+      'grading.json',
+      'run-1',
+      'run-2',
+      'summary.json',
+      'task',
+    ]);
+
+    const prompt = await readFile(
+      path.join(paths.testArtifactDir, 'repeat-case', 'task', 'PROMPT.md'),
+      'utf8',
+    );
+    expect(prompt).toBe('@[user]:\nRepeat this task prompt.');
 
     const caseSummary = JSON.parse(
       await readFile(path.join(paths.testArtifactDir, 'repeat-case', 'summary.json'), 'utf8'),
diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts
index 53f7a1cfa..9bb6b283a 100644
--- a/apps/cli/test/commands/results/export.test.ts
+++ b/apps/cli/test/commands/results/export.test.ts
@@ -349,7 +349,7 @@ describe('results export', () => {
     });
     expect(bundle.entries[0].artifact_refs).toMatchObject({
       status: 'planned_export',
-      input_path: 'privacy/test-private/input.md',
+      input_path: 'privacy/test-private/task/PROMPT.md',
       output_path: 'privacy/test-private/outputs/answer.md',
       answer_path: 'privacy/test-private/outputs/answer.md',
       trace_path: 'privacy/test-private/trace.json',
@@ -417,7 +417,7 @@ describe('results export', () => {
       output_path: 'demo/test-greeting/outputs/answer.md',
       answer_path: 'demo/test-greeting/outputs/answer.md',
       transcript_path: 'demo/test-greeting/transcript.jsonl',
-      input_path: 'demo/test-greeting/input.md',
+      input_path: 'demo/test-greeting/task/PROMPT.md',
     });
     expect(entries[0].projection_identity).toMatchObject({
       schema_version: 'agentv.projection_identity.v1',
@@ -712,7 +712,7 @@ describe('results export', () => {
     expect(grading.summary.total).toBe(0);
   });
 
-  it('should write string input to <test-id>/input.md', async () => {
+  it('should write string input to <test-id>/task/PROMPT.md', async () => {
     const outputDir = path.join(tempDir, 'output');
     const resultWithInput = {
       ...RESULT_FULL,
@@ -722,12 +722,12 @@ describe('results export', () => {
 
     await exportResults('test.jsonl', content, outputDir);
 
-    const inputPath = path.join(artifactDir(outputDir, resultWithInput), 'input.md');
+    const inputPath = path.join(artifactDir(outputDir, resultWithInput), 'task', 'PROMPT.md');
     expect(existsSync(inputPath)).toBe(true);
     expect(readFileSync(inputPath, 'utf8')).toBe('What is the capital of France?');
   });
 
-  it('should write Message[] input to <test-id>/input.md as markdown', async () => {
+  it('should write Message[] input to <test-id>/task/PROMPT.md as markdown', async () => {
     const outputDir = path.join(tempDir, 'output');
     const resultWithMessages = {
       ...RESULT_FULL,
@@ -740,7 +740,7 @@ describe('results export', () => {
 
     await exportResults('test.jsonl', content, outputDir);
 
-    const inputPath = path.join(artifactDir(outputDir, resultWithMessages), 'input.md');
+    const inputPath = path.join(artifactDir(outputDir, resultWithMessages), 'task', 'PROMPT.md');
     expect(existsSync(inputPath)).toBe(true);
     expect(readFileSync(inputPath, 'utf8')).toBe('@[user]:\nHello\n\n@[assistant]:\nHi there!');
   });
@@ -751,7 +751,7 @@ describe('results export', () => {
 
     await exportResults('test.jsonl', content, outputDir);
 
-    const inputPath = path.join(artifactDir(outputDir, RESULT_FULL), 'input.md');
+    const inputPath = path.join(artifactDir(outputDir, RESULT_FULL), 'task', 'PROMPT.md');
     expect(existsSync(inputPath)).toBe(false);
   });
 
diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts
index 439b10ab2..8a6c5ef62 100644
--- a/apps/cli/test/commands/results/serve.test.ts
+++ b/apps/cli/test/commands/results/serve.test.ts
@@ -3193,7 +3193,7 @@ describe('serve app', () => {
       const runId = 'with-transcript::2026-03-25T10-00-00-000Z';
       const timestampDir = path.join(runsDir, '2026-03-25T10-00-00-000Z');
       const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl';
-      const answerArtifactPath = 'demo/test-greeting/outputs/answer.md';
+      const answerArtifactPath = 'demo/test-greeting/answer.md';
       const transcriptPath = path.join(timestampDir, transcriptArtifactPath);
       const answerPath = path.join(timestampDir, answerArtifactPath);
       const transcriptJsonl = `${JSON.stringify({
@@ -3558,7 +3558,7 @@ describe('serve app', () => {
       const runId = 'escaped-answer::2026-03-25T13-45-00-000Z';
       const timestampDir = path.join(runsDir, '2026-03-25T13-45-00-000Z');
       const transcriptArtifactPath = 'demo/test-greeting/transcript.jsonl';
-      const answerArtifactPath = 'demo/test-greeting/outputs/answer.md';
+      const answerArtifactPath = 'demo/test-greeting/answer.md';
       const transcriptPath = path.join(timestampDir, transcriptArtifactPath);
       const answerPath = path.join(timestampDir, answerArtifactPath);
       const transcriptJsonl = `${JSON.stringify({
diff --git a/apps/dashboard/src/components/__fixtures__/structured-transcript.ts b/apps/dashboard/src/components/__fixtures__/structured-transcript.ts
index 93f37e21b..0a51654de 100644
--- a/apps/dashboard/src/components/__fixtures__/structured-transcript.ts
+++ b/apps/dashboard/src/components/__fixtures__/structured-transcript.ts
@@ -86,7 +86,7 @@ export const structuredTranscriptFiles: FileNode[] = [
     type: 'dir',
     children: [
       { name: 'grading.json', path: 'final-json-answer__codex/grading.json', type: 'file' },
-      { name: 'answer.md', path: 'final-json-answer__codex/answer.md', type: 'file' },
+      { name: 'answer.md', path: 'final-json-answer__codex/outputs/answer.md', type: 'file' },
       {
         name: 'outputs',
         path: 'final-json-answer__codex/outputs',
diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
index 65d0942da..e4d932da0 100644
--- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx
@@ -186,6 +186,35 @@ Each test's effective input becomes a single user message with `[file blocks...,
 
 Per-test `input_files` overrides the suite-level value (it does not merge). To opt out, set `execution.skip_defaults: true` on the test.
 
+### PROMPT.md Fallback
+
+For Vercel-style eval directories, a test may omit `input` and keep the task
+prompt in Markdown instead. AgentV resolves the prompt in this order:
+
+1. If the effective `input_files` contains a file named exactly `PROMPT.md`, that file becomes the test prompt.
+2. Otherwise, if a `PROMPT.md` exists beside the `EVAL.yaml`, that file becomes the test prompt.
+3. Other `input_files` remain attachments. `PROMPT.md` is removed from the attachment list so the prompt is not duplicated.
+
+```text
+agent-001-fix-bug/
+  EVAL.yaml
+  PROMPT.md
+  fixtures/
+    failing-test.log
+```
+
+```yaml
+tests:
+  - id: fix-bug
+    criteria: Fixes the regression described in the prompt
+    input_files:
+      - ./fixtures/failing-test.log
+```
+
+Use explicit `input` when the prompt is short or generated from YAML variables.
+Use `PROMPT.md` when the task text is long enough that duplicating it inside
+YAML would make the eval hard to review.
+
 ### Tests as String Path
 
 Instead of inlining tests in the same file, you can point `tests` to an external YAML or JSONL file. This is the inverse of the sidecar pattern — the metadata file references the test data:
diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
index 18d05cedd..460a43f7b 100644
--- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx
@@ -114,7 +114,7 @@ my-results/
   <test-id>/
     grading.json
     timing.json
-    input.md
+    task/PROMPT.md
     outputs/answer.md
     task/
       EVAL.yaml
@@ -132,6 +132,11 @@ case directories are still useful for organizing bulky prompts, fixtures, or
 tests while authoring an eval, but they are optional input organization rather
 than a separate artifact schema.
 
+If the source eval uses the `PROMPT.md` fallback instead of inline `input`,
+AgentV still writes the resolved task prompt to the generated
+`task/PROMPT.md`. This keeps the run artifact self-contained without requiring
+the same long prompt to be duplicated in both Markdown and YAML.
+
 ### Manual or External-Agent Attempts
 
 Use `agentv prepare` when you want AgentV to set up one eval case but a human,
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index bfc1dd4db..4f0ece01f 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -161,7 +161,7 @@ Agent Skills eval artifacts map into AgentV like this:
 | Agent Skills pattern | AgentV field | Artifact location |
 |----------------------|--------------|-------------------|
 | Authored `evals/evals.json` cases | AgentV eval cases and task bundle paths | Eval source plus optional `task_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `index.jsonl` |
-| Per-case `outputs/` | Generated target output artifacts | `outputs/answer.md` |
+| Per-case answer | Generated target output artifact | `outputs/answer.md` |
 | Per-case sidecars | Trace, transcript, metrics, and raw provider evidence | `trace.json`, `transcript.jsonl`, `metrics.json`, `provider.log` |
 | Per-case `timing.json` | Duration, token totals, cost, and usage source labels | `timing.json` |
 | Per-case `grading.json` | Assertions, graders, execution metrics, workspace changes | `grading.json`; summary fields can reference the same trace/result facts |
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index a462a249c..ae6a5dfb2 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -246,7 +246,7 @@ Single-run cases keep direct case-local files:
 .agentv/results/<experiment>/<timestamp>/benchmark.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/grading.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/timing.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/input.md
+.agentv/results/<experiment>/<timestamp>/<case-id>/task/PROMPT.md
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/trace.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/transcript.jsonl
 .agentv/results/<experiment>/<timestamp>/<case-id>/outputs/answer.md
diff --git a/docs/plans/trace-envelope-implementation-spec.md b/docs/plans/trace-envelope-implementation-spec.md
index 4422e1a0e..1d1c800bb 100644
--- a/docs/plans/trace-envelope-implementation-spec.md
+++ b/docs/plans/trace-envelope-implementation-spec.md
@@ -241,7 +241,7 @@ explicit known-field conversion plus Zod validation. It should not look like
 | Model/chat span | Child span named `chat <model>` for each model turn. Parent is root agent span or the source parent span if importing external OTel. | `gen_ai.operation.name=chat`; `gen_ai.provider.name`; `gen_ai.request.model`; `gen_ai.response.model`; `gen_ai.response.id`; `gen_ai.response.finish_reasons`; `gen_ai.input.messages` opt-in; `gen_ai.output.messages` opt-in; `openinference.span.kind=LLM`; `llm.system`; `llm.provider`; `llm.model_name`; `input.value`; `output.value`; `input.mime_type`; `output.mime_type`. | `agentv.turn_index`, `agentv.message_index`, `agentv.source_event_id` only when standards do not carry the identity. | Content attributes are sensitive and must follow `capture.content`. Preserve message payload keys when storing structured content internally or in raw evidence. |
 | Tool execution span | Span named `execute_tool <tool_name>`. For AgentV-generated traces, parent it to the chat span that requested the tool when known; otherwise parent it to the root agent span. | `gen_ai.operation.name=execute_tool`; `gen_ai.tool.name`; `gen_ai.tool.call.id`; `gen_ai.tool.type`; `gen_ai.tool.description`; `gen_ai.tool.call.arguments` opt-in; `gen_ai.tool.call.result` opt-in; `openinference.span.kind=TOOL`; `tool.name`; `tool.id`; `tool.description`; `tool.json_schema`; `input.value`; `output.value`. | `agentv.tool.index`, `agentv.generated_tool_call_id=true`, `agentv.source_event_id`, and warning `missing_tool_call_id` when AgentV generated an ID. | OTel and OpenInference both have tool-call identifiers but use different names. Emit both when useful and unambiguous. |
 | Tool result/event | Prefer result data on the `execute_tool` span (`gen_ai.tool.call.result` and/or OpenInference `output.value`) when capture policy allows. | `gen_ai.tool.call.result`; `output.value`; `output.mime_type`. | If result content is large or redacted, put an artifact pointer in `artifacts.raw_evidence_dir` and set `agentv.tool.result_ref` on the span. | Do not create a separate canonical `tool_result` span unless the source emitted one. Derived `Message.toolCalls[].output` and transcript rows can still expose a paired result. |
-| Final answer | The final assistant answer is the last relevant LLM output plus an envelope artifact pointer. | `gen_ai.output.messages` opt-in on the final LLM/root span; OpenInference `output.value`/`output.mime_type`. | `artifacts.answer_path`; optional root event `agentv.final_answer` with `agentv.artifact_path`. | Do not make `outputs/answer.md` canonical. It is derived from the final answer in spans or from replay projection. |
+| Final answer | The final assistant answer is the last relevant LLM output plus an envelope artifact pointer. | `gen_ai.output.messages` opt-in on the final LLM/root span; OpenInference `output.value`/`output.mime_type`. | `artifacts.answer_path`; optional root event `agentv.final_answer` with `agentv.artifact_path`. | `outputs/answer.md` is the generated human-readable projection; spans remain the trace source of truth. |
 | Provider error | Set `status.code=ERROR` and message on the failed span and root span. Add exception event when details are available. | OTel exception event attributes `exception.type`, `exception.message`, `exception.stacktrace`; OpenInference reserves `exception.message`, `exception.stacktrace`, `exception.escaped`; span status `ERROR`. | `agentv.failure_stage`, `agentv.failure_reason_code`, envelope `conversion_warnings` when import is lossy rather than execution-failed. | A grader failure is score provenance, not provider execution error. Keep these separate. |
 | Subagent/nested tool evidence | Nested root-like span under the calling tool span, e.g. `execute_tool runSubagent` -> `invoke_agent <subagent>`. Preserve imported parentage. | `gen_ai.operation.name=invoke_agent`; `gen_ai.agent.name`; `gen_ai.provider.name`; `openinference.span.kind=AGENT`; `session.id`. | `agentv.parent_tool_call_id`, `agentv.subagent=true` if needed for derived grader views. | VS Code/Copilot documents subagent invocations as nested `invoke_agent` spans under the parent `execute_tool runSubagent`; use that pattern when AgentV has enough evidence. |
 | Score/evaluator provenance | Keep v1 score provenance in envelope `scores[]`. For OTLP export, also emit either root events or evaluator spans. | OTel GenAI evaluation event: `gen_ai.evaluation.name`, `gen_ai.evaluation.score.value`, `gen_ai.evaluation.score.label`, `gen_ai.evaluation.explanation`, `gen_ai.response.id`; OpenInference `openinference.span.kind=EVALUATOR`. | Existing `agentv.score` and `agentv.grader.*` root events remain compatibility output. Envelope `scores[].target_span_id`, `scores[].evidence.span_ids`, and `scores[].evidence.tool_call_ids` are the AgentV provenance contract. | Do not require `agentv.score` to import or score an external trace. Existing `inspect` OTLP import currently does; `.6` should remove that limitation for trace-only scoring. |
diff --git a/examples/features/input-files-shorthand/evals/PROMPT.md b/examples/features/input-files-shorthand/evals/PROMPT.md
new file mode 100644
index 000000000..1b76d8d2f
--- /dev/null
+++ b/examples/features/input-files-shorthand/evals/PROMPT.md
@@ -0,0 +1,3 @@
+Summarize the monthly revenue trends in the attached CSV.
+
+Call out which product appears to be growing faster and mention one caveat about the sample size.
diff --git a/examples/features/input-files-shorthand/evals/dataset.eval.yaml b/examples/features/input-files-shorthand/evals/dataset.eval.yaml
index e763bc669..058f94a1e 100644
--- a/examples/features/input-files-shorthand/evals/dataset.eval.yaml
+++ b/examples/features/input-files-shorthand/evals/dataset.eval.yaml
@@ -24,6 +24,8 @@
 #   - File blocks come first, text block last
 #   - Only supported with a string `input` (not multi-turn arrays)
 #   - Paths are resolved the same way as explicit type:file blocks
+#   - If `input` is omitted and input_files includes PROMPT.md, that file becomes
+#     the prompt and is not duplicated as an attachment
 
 description: Demonstrates input_files shorthand for attaching files to test inputs
 
@@ -74,3 +76,15 @@ tests:
       - ../fixtures/sales.csv
       - ../fixtures/sales.csv
     input: "Compare these two data files and describe any differences."
+
+  # ==========================================
+  # Example 4: PROMPT.md prompt plus attached data
+  # ==========================================
+  - id: prompt-md-with-attachment
+    criteria: >
+      Agent follows the task prompt from PROMPT.md and uses the attached CSV
+      data when answering.
+
+    input_files:
+      - ./PROMPT.md
+      - ../fixtures/sales.csv
diff --git a/packages/core/src/evaluation/loaders/prompt-md-fallback.ts b/packages/core/src/evaluation/loaders/prompt-md-fallback.ts
new file mode 100644
index 000000000..e10ce8394
--- /dev/null
+++ b/packages/core/src/evaluation/loaders/prompt-md-fallback.ts
@@ -0,0 +1,113 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import type { JsonValue } from '../types.js';
+import { fileExists, resolveFileReference } from './file-resolver.js';
+
+export const PROMPT_MD_FILENAME = 'PROMPT.md';
+
+export type PromptMdInputFilesSource = 'test' | 'suite';
+
+export type PromptMdFallbackResult = {
+  readonly promptText: string;
+  readonly promptPath: string;
+  readonly source: 'input_files' | 'sibling';
+  readonly inputFilesSource?: PromptMdInputFilesSource;
+  readonly remainingInputFiles?: readonly string[];
+};
+
+type PromptMdFallbackOptions = {
+  readonly evalFilePath: string;
+  readonly searchRoots: readonly string[];
+  readonly testInputFiles?: JsonValue;
+  readonly suiteInputFiles?: JsonValue;
+};
+
+type InputFilesSplit = {
+  readonly files?: readonly string[];
+  readonly promptFile?: string;
+  readonly remainingInputFiles?: readonly string[];
+};
+
+function asStringArray(value: JsonValue | undefined): readonly string[] | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+  return value.filter((entry): entry is string => typeof entry === 'string');
+}
+
+function basenameForPortablePath(value: string): string {
+  return value.split(/[\\/]/).at(-1) ?? value;
+}
+
+function isPromptMdPath(value: string): boolean {
+  return basenameForPortablePath(value) === PROMPT_MD_FILENAME;
+}
+
+function splitInputFiles(value: JsonValue | undefined): InputFilesSplit {
+  const files = asStringArray(value);
+  if (!files) {
+    return {};
+  }
+  const promptFile = files.find(isPromptMdPath);
+  const remaining = promptFile ? files.filter((file) => !isPromptMdPath(file)) : files;
+  return {
+    files,
+    ...(promptFile ? { promptFile } : {}),
+    ...(remaining.length > 0 ? { remainingInputFiles: remaining } : {}),
+  };
+}
+
+async function readPromptFile(promptPath: string): Promise<string> {
+  return (await readFile(promptPath, 'utf8')).replace(/\r\n/g, '\n');
+}
+
+/**
+ * Resolve Vercel-style task prompt fallback for evals that omit `input`.
+ *
+ * `input_files` keeps its existing attachment meaning. Only a file named exactly
+ * `PROMPT.md` is promoted to task prompt text, and only when `input` is absent.
+ * If no explicit PROMPT.md is listed, the loader falls back to a sibling
+ * `PROMPT.md` beside the EVAL.yaml file.
+ */
+export async function loadPromptMdFallback(
+  options: PromptMdFallbackOptions,
+): Promise<PromptMdFallbackResult | undefined> {
+  const testSplit = splitInputFiles(options.testInputFiles);
+  const suiteSplit = splitInputFiles(options.suiteInputFiles);
+  const inputFilesSource: PromptMdInputFilesSource | undefined = testSplit.files
+    ? 'test'
+    : suiteSplit.files
+      ? 'suite'
+      : undefined;
+  const activeSplit = inputFilesSource === 'test' ? testSplit : suiteSplit;
+
+  if (activeSplit.promptFile) {
+    const resolved = await resolveFileReference(activeSplit.promptFile, options.searchRoots);
+    if (resolved.resolvedPath) {
+      return {
+        promptText: await readPromptFile(resolved.resolvedPath),
+        promptPath: path.resolve(resolved.resolvedPath),
+        source: 'input_files',
+        ...(inputFilesSource ? { inputFilesSource } : {}),
+        ...(activeSplit.remainingInputFiles
+          ? { remainingInputFiles: activeSplit.remainingInputFiles }
+          : {}),
+      };
+    }
+  }
+
+  const siblingPromptPath = path.join(
+    path.dirname(path.resolve(options.evalFilePath)),
+    PROMPT_MD_FILENAME,
+  );
+  if (await fileExists(siblingPromptPath)) {
+    return {
+      promptText: await readPromptFile(siblingPromptPath),
+      promptPath: siblingPromptPath,
+      source: 'sibling',
+    };
+  }
+
+  return undefined;
+}
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 6b4a47e8e..87533e09e 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -1241,8 +1241,7 @@ function formatOutputMarkdown(output: readonly { role: string; content?: unknown
   return output.map((msg) => `@[${msg.role}]:\n${String(msg.content ?? '')}`).join('\n\n');
 }
 
-function extractInput(result: EvaluationResult): string | null {
-  const input = (result as unknown as Record<string, unknown>).input;
+function formatInputValue(input: unknown): string | null {
   if (!input) return null;
   if (typeof input === 'string') return input;
   if (Array.isArray(input) && input.length > 0) {
@@ -1251,6 +1250,43 @@ function extractInput(result: EvaluationResult): string | null {
   return null;
 }
 
+function extractInput(result: EvaluationResult): string | null {
+  return formatInputValue((result as unknown as Record<string, unknown>).input);
+}
+
+function extractTracePrompt(result: EvaluationResult): string | null {
+  const trace = (result as unknown as { trace?: { messages?: unknown } }).trace;
+  const messages = Array.isArray(trace?.messages) ? trace.messages : [];
+  const promptMessages: { role: string; content?: unknown }[] = [];
+  for (const message of messages) {
+    if (!isRecord(message) || typeof message.role !== 'string') continue;
+    if (message.role === 'assistant') break;
+    if (message.role === 'system' || message.role === 'user' || message.role === 'developer') {
+      promptMessages.push({ role: message.role, content: message.content });
+    }
+  }
+  return promptMessages.length > 0 ? formatOutputMarkdown(promptMessages) : null;
+}
+
+function extractPrompt(result: EvaluationResult, sourceTest?: EvalTest): string | null {
+  const input = extractInput(result);
+  if (input) return input;
+  const traceInput = extractTracePrompt(result);
+  if (traceInput) return traceInput;
+  for (const trial of result.trials ?? []) {
+    if (!trial.result) continue;
+    const trialInput = extractInput(trial.result);
+    if (trialInput) return trialInput;
+    const trialTraceInput = extractTracePrompt(trial.result);
+    if (trialTraceInput) return trialTraceInput;
+  }
+  const sourceInput = sourceTest
+    ? formatInputValue((sourceTest as unknown as Record<string, unknown>).input)
+    : null;
+  if (sourceInput) return sourceInput;
+  return null;
+}
+
 function toRelativeArtifactPath(outputDir: string, filePath: string): string {
   return path.relative(outputDir, filePath).split(path.sep).join('/');
 }
@@ -1502,7 +1538,7 @@ export function buildResultIndexArtifact(
   },
 ): ResultIndexArtifact {
   const artifactSubdir = buildArtifactSubdir(result);
-  const input = extractInput(result);
+  const input = extractPrompt(result);
   const hasAnswer = result.output.length > 0;
   const hasTranscript = resultHasExecutionTraceTranscript(result);
   const hasRawProviderLog = rawProviderLogSourcePath(result) !== undefined;
@@ -1529,9 +1565,10 @@ export function buildResultIndexArtifact(
     failure_reason_code: result.failureReasonCode,
     workspace_path: result.workspacePath,
     artifact_dir: artifactSubdir,
+    task_dir: input ? path.posix.join(artifactSubdir, 'task') : undefined,
     grading_path: path.posix.join(artifactSubdir, 'grading.json'),
     timing_path: path.posix.join(artifactSubdir, 'timing.json'),
-    input_path: input ? path.posix.join(artifactSubdir, 'input.md') : undefined,
+    input_path: input ? path.posix.join(artifactSubdir, 'task', 'PROMPT.md') : undefined,
     output_path: hasAnswer ? path.posix.join(artifactSubdir, 'outputs', 'answer.md') : undefined,
     answer_path: hasAnswer ? path.posix.join(artifactSubdir, 'outputs', 'answer.md') : undefined,
     trace_path: path.posix.join(artifactSubdir, CANONICAL_TRACE_ARTIFACT_PATH),
@@ -1943,9 +1980,11 @@ export async function writePerTestArtifacts(
       'utf8',
     );
 
-    const input = extractInput(result);
+    const input = extractPrompt(result, testByTestId.get(result.testId ?? ''));
     if (input) {
-      await writeFile(path.join(testDir, 'input.md'), input, 'utf8');
+      const promptPath = path.join(testDir, 'task', 'PROMPT.md');
+      await mkdir(path.dirname(promptPath), { recursive: true });
+      await writeFile(promptPath, input, 'utf8');
     }
     const outputsDir = path.join(testDir, 'outputs');
     await mkdir(outputsDir, { recursive: true });
@@ -2044,8 +2083,8 @@ export async function writeArtifactsFromResults(
     const testDir = path.join(outputDir, artifactSubdir);
     const gradingPath = path.join(testDir, 'grading.json');
     const perTestTimingPath = path.join(testDir, 'timing.json');
-    const input = extractInput(result);
-    const inputPath = input ? path.join(testDir, 'input.md') : undefined;
+    const input = extractPrompt(result, testByTestId.get(result.testId ?? ''));
+    const inputPath = input ? path.join(testDir, 'task', 'PROMPT.md') : undefined;
     const outputsDir = path.join(testDir, 'outputs');
     const answerPath = result.output.length > 0 ? path.join(outputsDir, 'answer.md') : undefined;
     const envelope = buildTraceEnvelopeSidecar({
@@ -2133,6 +2172,10 @@ export async function writeArtifactsFromResults(
       );
       await writeFile(plan.gradingPath, `${JSON.stringify(plan.grading, null, 2)}\n`, 'utf8');
       await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+      if (plan.inputPath && plan.input) {
+        await mkdir(path.dirname(plan.inputPath), { recursive: true });
+        await writeFile(plan.inputPath, plan.input, 'utf8');
+      }
       for (const trial of result.trials ?? []) {
         await writeTrialRunArtifacts({
           trial,
@@ -2152,6 +2195,10 @@ export async function writeArtifactsFromResults(
           artifactDir: plan.testDir,
           gradingPath: plan.gradingPath,
           summaryPath,
+          inputPath: plan.inputPath,
+          extraIndexFields: plan.inputPath
+            ? { task_dir: toRelativeArtifactPath(outputDir, path.dirname(plan.inputPath)) }
+            : undefined,
           projectionIdentity: plan.projectionIdentity,
           duplicatePolicy,
         }),
@@ -2175,6 +2222,7 @@ export async function writeArtifactsFromResults(
     await writeFile(plan.perTestTimingPath, `${JSON.stringify(plan.timing, null, 2)}\n`, 'utf8');
 
     if (plan.inputPath && plan.input) {
+      await mkdir(path.dirname(plan.inputPath), { recursive: true });
       await writeFile(plan.inputPath, plan.input, 'utf8');
     }
 
@@ -2225,6 +2273,13 @@ export async function writeArtifactsFromResults(
       options?.additionalArtifacts,
     );
 
+    const indexExtraFields = {
+      ...(plan.inputPath
+        ? { task_dir: toRelativeArtifactPath(outputDir, path.dirname(plan.inputPath)) }
+        : {}),
+      ...extraIndexFields,
+    };
+
     const nextRecord = {
       ...buildIndexArtifactEntry(result, {
         outputDir,
@@ -2239,7 +2294,7 @@ export async function writeArtifactsFromResults(
         artifactPointers,
         rawProviderLogPath: plan.rawProviderLogPath,
         inputPath: plan.inputPath,
-        extraIndexFields,
+        extraIndexFields: indexExtraFields,
         projectionIdentity: plan.projectionIdentity,
         duplicatePolicy,
       }),
diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts
index 19aa9284e..32f9120ab 100644
--- a/packages/core/src/evaluation/validation/eval-validator.ts
+++ b/packages/core/src/evaluation/validation/eval-validator.ts
@@ -3,6 +3,8 @@ import path from 'node:path';
 
 import { interpolateEnv } from '../interpolation.js';
 import { loadCasesFromDirectory, loadCasesFromFile } from '../loaders/case-file-loader.js';
+import { buildSearchRoots } from '../loaders/file-resolver.js';
+import { loadPromptMdFallback } from '../loaders/prompt-md-fallback.js';
 import { isGraderKind } from '../types.js';
 import { parseYamlValue } from '../yaml-loader.js';
 import type { ValidationError, ValidationResult } from './types.js';
@@ -381,9 +383,21 @@ export async function validateEvalFile(filePath: string): Promise<ValidationResu
       });
     }
 
-    // input field (string/object shorthand or message array)
+    // input field (string/object shorthand or message array). When omitted,
+    // AgentV accepts Vercel-style PROMPT.md fallback beside EVAL.yaml or in input_files.
+    const caseExecution = isObject(evalCase.execution) ? evalCase.execution : undefined;
+    const skipDefaults = caseExecution?.skip_defaults === true;
+    const hasPromptMdFallback =
+      evalCase.input === undefined
+        ? (await loadPromptMdFallback({
+            evalFilePath: absolutePath,
+            searchRoots: buildSearchRoots(absolutePath, process.cwd()),
+            testInputFiles: evalCase.input_files,
+            suiteInputFiles: skipDefaults ? undefined : parsed.input_files,
+          })) !== undefined
+        : false;
     validateInputField(evalCase.input, `${location}.input`, absolutePath, errors, {
-      required: true,
+      required: !hasPromptMdFallback,
     });
 
     // expected_output field (string/object shorthand or message array)
@@ -683,7 +697,8 @@ function validateInputField(
         severity: 'error',
         filePath,
         location,
-        message: "Missing 'input' field (must be a string, object, or array of messages)",
+        message:
+          "Missing 'input' field (provide a string, object, message array, or PROMPT.md next to EVAL.yaml / referenced in input_files)",
       });
     }
     return;
diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts
index ba07f57a6..0f074b18d 100644
--- a/packages/core/src/evaluation/yaml-parser.ts
+++ b/packages/core/src/evaluation/yaml-parser.ts
@@ -34,6 +34,7 @@ import {
 } from './loaders/grader-parser.js';
 import { detectFormat, loadTestsFromJsonl } from './loaders/jsonl-parser.js';
 import { processExpectedMessages, processMessages } from './loaders/message-processor.js';
+import { loadPromptMdFallback } from './loaders/prompt-md-fallback.js';
 import {
   expandInputShorthand,
   resolveExpectedMessages,
@@ -504,7 +505,40 @@ async function loadTestsFromParsedYamlValue(
       rawSuiteInputFiles && !skipDefaults
         ? interpolateCaseField(rawSuiteInputFiles, caseVars)
         : undefined;
-    const testInputMessages = resolveInputMessages(renderedCase, effectiveSuiteInputFiles);
+    let inputCase = renderedCase;
+    let inputSuiteFiles = effectiveSuiteInputFiles;
+    if (renderedCase.input === undefined) {
+      const promptFallback = await loadPromptMdFallback({
+        evalFilePath: absoluteTestPath,
+        searchRoots,
+        testInputFiles: renderedCase.input_files,
+        suiteInputFiles: effectiveSuiteInputFiles,
+      });
+      if (promptFallback) {
+        if (promptFallback.inputFilesSource === 'test') {
+          const { input_files: _inputFiles, ...caseWithoutInputFiles } = renderedCase;
+          inputCase = {
+            ...caseWithoutInputFiles,
+            input: promptFallback.promptText,
+            ...(promptFallback.remainingInputFiles
+              ? { input_files: [...promptFallback.remainingInputFiles] }
+              : {}),
+          };
+          inputSuiteFiles = undefined;
+        } else {
+          inputCase = {
+            ...renderedCase,
+            input: promptFallback.promptText,
+          };
+          if (promptFallback.inputFilesSource === 'suite') {
+            inputSuiteFiles = promptFallback.remainingInputFiles
+              ? [...promptFallback.remainingInputFiles]
+              : undefined;
+          }
+        }
+      }
+    }
+    const testInputMessages = resolveInputMessages(inputCase, inputSuiteFiles);
     // Resolve expected_output with shorthand support
     const expectedMessages = resolveExpectedMessages(renderedCase) ?? [];
 
@@ -517,7 +551,7 @@ async function loadTestsFromParsedYamlValue(
       (Array.isArray(renderedCase.turns) && renderedCase.turns.length > 0);
     if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) {
       logError(
-        `Skipping incomplete test: ${id ?? 'unknown'}. Missing required fields: id, input, and at least one of criteria/expected_output/assertions/turns`,
+        `Skipping incomplete test: ${id ?? 'unknown'}. Missing required fields: id, input or PROMPT.md, and at least one of criteria/expected_output/assertions/turns`,
       );
       continue;
     }
diff --git a/packages/core/test/evaluation/input-files-shorthand.test.ts b/packages/core/test/evaluation/input-files-shorthand.test.ts
index 78eb52e2a..215e35ce5 100644
--- a/packages/core/test/evaluation/input-files-shorthand.test.ts
+++ b/packages/core/test/evaluation/input-files-shorthand.test.ts
@@ -266,4 +266,53 @@ tests:
     expect(tests).toHaveLength(1);
     expect(tests[0].input[0]).toEqual({ role: 'user', content: 'What is 2+2?' });
   });
+
+  it('uses a sibling PROMPT.md when input is omitted', async () => {
+    const evalDir = path.join(tempDir, 'sibling-prompt');
+    await mkdir(evalDir, { recursive: true });
+    await writeFile(path.join(evalDir, 'PROMPT.md'), 'Use the sibling prompt.\n');
+    await writeFile(
+      path.join(evalDir, 'EVAL.yaml'),
+      `tests:
+  - id: sibling-prompt
+    criteria: "Uses the prompt file"
+`,
+    );
+
+    const tests = await loadTests(path.join(evalDir, 'EVAL.yaml'), tempDir);
+
+    expect(tests).toHaveLength(1);
+    expect(tests[0].input[0]).toEqual({ role: 'user', content: 'Use the sibling prompt.\n' });
+    expect(tests[0].question).toBe('Use the sibling prompt.');
+  });
+
+  it('uses PROMPT.md from input_files without duplicating it as an attachment', async () => {
+    const evalDir = path.join(tempDir, 'input-files-prompt');
+    await mkdir(path.join(evalDir, 'task'), { recursive: true });
+    await writeFile(
+      path.join(evalDir, 'task', 'PROMPT.md'),
+      'Summarize the attached sales data.\n',
+    );
+    await writeFile(path.join(evalDir, 'sales.csv'), 'month,revenue\nApr,400\n');
+    await writeFile(
+      path.join(evalDir, 'EVAL.yaml'),
+      `tests:
+  - id: prompt-file-with-attachment
+    criteria: "Uses the prompt and attached data"
+    input_files:
+      - ./task/PROMPT.md
+      - ./sales.csv
+`,
+    );
+
+    const tests = await loadTests(path.join(evalDir, 'EVAL.yaml'), tempDir);
+
+    expect(tests).toHaveLength(1);
+    const content = tests[0].input[0].content as Array<{ type: string; value: string }>;
+    expect(content).toHaveLength(2);
+    expect(content[0]).toMatchObject({ type: 'file', value: './sales.csv' });
+    expect(content[1]).toEqual({ type: 'text', value: 'Summarize the attached sales data.\n' });
+    expect(tests[0].file_paths).toHaveLength(1);
+    expect(tests[0].file_paths[0]).toContain('sales.csv');
+  });
 });
diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts
index 6be2fa31e..dd285c692 100644
--- a/packages/core/test/evaluation/validation/eval-validator.test.ts
+++ b/packages/core/test/evaluation/validation/eval-validator.test.ts
@@ -34,6 +34,46 @@ describe('validateEvalFile', () => {
     expect(result.errors).toHaveLength(0);
   });
 
+  it('validates eval file that omits input when sibling PROMPT.md exists', async () => {
+    const evalDir = path.join(tempDir, 'prompt-md-fallback');
+    await mkdir(evalDir, { recursive: true });
+    await writeFile(path.join(evalDir, 'PROMPT.md'), 'Use the prompt fallback.');
+    const filePath = path.join(evalDir, 'EVAL.yaml');
+    await writeFile(
+      filePath,
+      `tests:
+  - id: test-1
+    criteria: Goal
+`,
+    );
+
+    const result = await validateEvalFile(filePath);
+
+    expect(result.valid).toBe(true);
+    expect(result.errors).toHaveLength(0);
+  });
+
+  it('validates eval file that omits input when input_files references PROMPT.md', async () => {
+    const evalDir = path.join(tempDir, 'prompt-md-input-files');
+    await mkdir(path.join(evalDir, 'task'), { recursive: true });
+    await writeFile(path.join(evalDir, 'task', 'PROMPT.md'), 'Use the referenced prompt.');
+    const filePath = path.join(evalDir, 'EVAL.yaml');
+    await writeFile(
+      filePath,
+      `tests:
+  - id: test-1
+    criteria: Goal
+    input_files:
+      - ./task/PROMPT.md
+`,
+    );
+
+    const result = await validateEvalFile(filePath);
+
+    expect(result.valid).toBe(true);
+    expect(result.errors).toHaveLength(0);
+  });
+
   it('validates eval file with suite-level input block shorthand', async () => {
     const filePath = path.join(tempDir, 'suite-input-block.yaml');
     await writeFile(

From 407e40996adb89629f0357aab55f3f448ae0fc6c Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 06:51:34 +0200
Subject: [PATCH 17/18] test: update rerun fixtures for answer artifacts

---
 apps/cli/test/commands/eval/aggregate.test.ts |  6 +++---
 apps/cli/test/commands/runs/rerun.test.ts     | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apps/cli/test/commands/eval/aggregate.test.ts b/apps/cli/test/commands/eval/aggregate.test.ts
index 91200aa61..40b00d9a1 100644
--- a/apps/cli/test/commands/eval/aggregate.test.ts
+++ b/apps/cli/test/commands/eval/aggregate.test.ts
@@ -191,12 +191,12 @@ describe('writePerTestArtifacts', () => {
     expect(grading2.assertions).toHaveLength(1);
   });
 
-  it('writes response.md for results with output', async () => {
+  it('writes outputs/answer.md for results with output', async () => {
     const results = [makeResult({ testId: 'test-1', output: 'hello' })];
 
     await writePerTestArtifacts(results, tmpDir);
 
-    const response = readFileSync(path.join(tmpDir, 'test-1', 'outputs', 'response.md'), 'utf8');
-    expect(response).toContain('hello');
+    const answer = readFileSync(path.join(tmpDir, 'test-1', 'outputs', 'answer.md'), 'utf8');
+    expect(answer).toContain('hello');
   });
 });
diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts
index 9a6e39292..1ca5bdae6 100644
--- a/apps/cli/test/commands/runs/rerun.test.ts
+++ b/apps/cli/test/commands/runs/rerun.test.ts
@@ -61,7 +61,7 @@ tests:
   await writeFile(path.join(taskDir, 'targets.yaml'), options.targetsYaml, 'utf8');
   await writeFile(path.join(artifactDir, 'grading.json'), '{"assertions":[]}\n', 'utf8');
   await writeFile(path.join(artifactDir, 'timing.json'), '{"duration_ms":1}\n', 'utf8');
-  await writeFile(path.join(outputsDir, 'response.md'), '@[assistant]:\nCaptured answer\n', 'utf8');
+  await writeFile(path.join(outputsDir, 'answer.md'), '@[assistant]:\nCaptured answer\n', 'utf8');
 
   return {
     timestamp: '2024-01-01T00:00:00.000Z',
@@ -71,8 +71,8 @@ tests:
     artifact_dir: options.testId,
     grading_path: `${options.testId}/grading.json`,
     timing_path: `${options.testId}/timing.json`,
-    output_path: `${options.testId}/outputs/response.md`,
-    response_path: `${options.testId}/outputs/response.md`,
+    output_path: `${options.testId}/outputs/answer.md`,
+    answer_path: `${options.testId}/outputs/answer.md`,
     task_dir: `${options.testId}/task`,
     eval_path: `${options.testId}/task/EVAL.yaml`,
     targets_path: `${options.testId}/task/targets.yaml`,
@@ -197,10 +197,10 @@ describe('agentv runs rerun', () => {
       },
     });
 
-    const responsePath = path.join(created.outputDir, String(rows[0].response_path));
-    const response = await readFile(responsePath, 'utf8');
-    expect(response).toContain('Alpha answer');
-    expect(response).not.toContain('Captured answer');
+    const answerPath = path.join(created.outputDir, String(rows[0].answer_path));
+    const answer = await readFile(answerPath, 'utf8');
+    expect(answer).toContain('Alpha answer');
+    expect(answer).not.toContain('Captured answer');
   }, 30_000);
 
   it('fails clearly for missing env and accepts an explicit env file', async () => {

From 159c0b6de2cef937f9c8586d2cfeadf568ed1853 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Wed, 24 Jun 2026 08:40:44 +0200
Subject: [PATCH 18/18] feat: embed repeat run metrics in result artifacts

---
 .agents/verification.md                       |  2 +-
 AGENTS.md                                     |  2 +-
 .../commands/eval/artifact-writer.test.ts     | 29 +++----
 .../docs/docs/evaluation/experiments.mdx      |  3 +-
 .../src/content/docs/docs/tools/results.mdx   |  6 +-
 ...3-001-feat-repeat-runs-flaky-evals-plan.md |  7 +-
 packages/core/src/evaluation/run-artifacts.ts | 83 ++++++++++++-------
 7 files changed, 79 insertions(+), 53 deletions(-)

diff --git a/.agents/verification.md b/.agents/verification.md
index e54e4b7e9..0f02e1237 100644
--- a/.agents/verification.md
+++ b/.agents/verification.md
@@ -127,7 +127,7 @@ Use live dogfood before marking PRs ready when they affect eval execution, exper
 - Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
-- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect root `index.jsonl`, root `benchmark.json`, and the repeated case folder. The repeated case folder should carry aggregate `summary.json` with flattened snake_case timing fields plus AgentV aggregate `grading.json`; attempt-specific outputs, metrics, and transcripts live under `run-N/`. Each `run-N/` folder should contain Vercel-compatible `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/`, plus AgentV `grading.json` and `metrics.json` with embedded `timing`.
+- For repeat-run changes, use an experiment-level repeat config with `count >= 2`, `early_exit: false` when validating all attempts are persisted. Inspect root `index.jsonl`, root `benchmark.json`, and the repeated case folder. The repeated case folder should carry aggregate `summary.json` with flattened snake_case timing fields plus AgentV aggregate `grading.json`; attempt-specific outputs and transcripts live under `run-N/`. Each `run-N/` folder should contain `result.json`, `grading.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md`. Do not write per-run `metrics.json`; timing and o11y fields belong in `result.json`, and `result.json` points at `./grading.json` through `grading_path`.
 - For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
 - Preserve review evidence in `agentv-private` on an `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, and screenshots when folder structure or UI behavior is under review.
 - If comparing against an external convention such as Vercel `agent-eval`, verify both semantic provenance and the physical `run-N` artifact layout for repeat runs.
diff --git a/AGENTS.md b/AGENTS.md
index 8f71e9619..da2e54680 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -43,7 +43,7 @@ Read the full rationale and examples in [.agents/product-boundary.md](.agents/pr
 - Non-trivial work needs a plan or task list. If the implementation surface starts to balloon, stop and re-plan.
 - Large or high-risk PRs need meaningful, reviewable commits for each coherent change. Rewrite only the PR branch with `git push --force-with-lease` when needed to replace WIP or accidental squashed history before review.
 - Manual red/green UAT is blocking before a branch is ready for review. GitHub Actions is the authoritative merge gate.
-- For eval, experiment, provider, or grader changes, dogfood with a live agent target and a live grader target before calling the PR ready. Mock graders, dry-run, and deterministic-only smoke tests are useful plumbing checks, but they are not live dogfood.
+- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. Mock graders, dry-run, and deterministic-only smoke tests are useful plumbing checks, but they are not live dogfood. Use canonical `.agentv/results/<experiment>/<timestamp>` output and publish private evidence. See [.agents/verification.md](.agents/verification.md).
 - For browser or screenshot UAT, keep evidence out of the public repo and publish reviewable artifacts to an `agentv-private` evidence branch. See [.agents/verification.md](.agents/verification.md).
 - Wire formats are `snake_case`; internal TypeScript is `camelCase`. Translate only at the boundary.
 - In AgentV, a `project` holds runs, traces, and experiments; a `benchmark` is a curated eval suite. Do not collapse those terms.
diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts
index d9a89ee4f..3e2feac68 100644
--- a/apps/cli/test/commands/eval/artifact-writer.test.ts
+++ b/apps/cli/test/commands/eval/artifact-writer.test.ts
@@ -1041,7 +1041,6 @@ describe('writeArtifactsFromResults', () => {
       const runEntries = await readdir(path.join(paths.testArtifactDir, 'repeat-case', runDir));
       expect(runEntries.sort()).toEqual([
         'grading.json',
-        'metrics.json',
         'outputs',
         'result.json',
         'transcript-raw.jsonl',
@@ -1057,11 +1056,16 @@ describe('writeArtifactsFromResults', () => {
     ) as Record<string, unknown>;
     expect(runOneResult).toMatchObject({
       status: 'failed',
-      duration: 2,
+      duration_ms: 2000,
+      duration_seconds: 2,
       model: 'test-target',
-      transcriptPath: './transcript.json',
-      transcriptRawPath: './transcript-raw.jsonl',
-      outputPaths: { answer: './outputs/answer.md' },
+      grading_path: './grading.json',
+      transcript_path: './transcript.json',
+      transcript_raw_path: './transcript-raw.jsonl',
+      output_paths: { answer: './outputs/answer.md' },
+      timing: {
+        duration_ms: 2000,
+      },
     });
 
     const runTwoAnswer = await readFile(
@@ -1070,23 +1074,20 @@ describe('writeArtifactsFromResults', () => {
     );
     expect(runTwoAnswer).toBe('second attempt');
 
-    const runTwoMetrics = JSON.parse(
+    const runTwoResult = JSON.parse(
       await readFile(
-        path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'metrics.json'),
+        path.join(paths.testArtifactDir, 'repeat-case', 'run-2', 'result.json'),
         'utf8',
       ),
     ) as Record<string, unknown>;
-    expect(runTwoMetrics).toMatchObject({
-      source_artifacts: {
-        trace_path: 'transcript.json',
-        transcript_path: 'transcript-raw.jsonl',
-        grading_path: 'grading.json',
-      },
+    expect(runTwoResult).toMatchObject({
+      grading_path: './grading.json',
+      transcript_path: './transcript.json',
+      transcript_raw_path: './transcript-raw.jsonl',
       timing: {
         duration_ms: 4000,
       },
     });
-    expect((runTwoMetrics.source_artifacts as Record<string, unknown>).timing_path).toBeUndefined();
   });
 
   it('handles empty results array', async () => {
diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
index ecf1c7cac..5e6f6a63b 100644
--- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx
+++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx
@@ -89,12 +89,13 @@ provenance:
 <run-dir>/<suite>/<case-id>/run-1/transcript-raw.jsonl
 <run-dir>/<suite>/<case-id>/run-1/outputs/answer.md
 <run-dir>/<suite>/<case-id>/run-1/grading.json
-<run-dir>/<suite>/<case-id>/run-1/metrics.json
 ```
 
 The repeated case aggregate folder uses `summary.json` for run-count, pass-rate,
 fingerprint, and flattened snake_case timing fields such as
 `mean_duration_ms`, and `grading.json` for compact trial/aggregation verdicts.
+Each `run-N/result.json` is the per-attempt manifest and includes
+`grading_path`, transcript/output paths, and embedded timing/o11y metrics.
 Root `index.jsonl` and root `benchmark.json` remain stable for existing CI
 summary scripts and uploaded artifact consumers.
 
diff --git a/apps/web/src/content/docs/docs/tools/results.mdx b/apps/web/src/content/docs/docs/tools/results.mdx
index 4f0ece01f..63eee4867 100644
--- a/apps/web/src/content/docs/docs/tools/results.mdx
+++ b/apps/web/src/content/docs/docs/tools/results.mdx
@@ -121,9 +121,11 @@ and does not carry token/cost usage.
 
 Repeat-enabled cases use aggregate `summary.json` with flattened snake_case
 timing fields plus aggregate `grading.json`, then store attempt details under
-`run-N/`. Each `run-N/` contains Vercel-compatible
+`run-N/`. Each `run-N/` contains a compact per-attempt manifest
 `result.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md`,
-plus AgentV `grading.json` and `metrics.json` with embedded timing.
+plus AgentV `grading.json`. The `result.json` file carries `grading_path`,
+transcript/output paths, and embedded timing/o11y metrics; repeat attempts do
+not write a separate `metrics.json` sidecar.
 
 `transcript.jsonl` remains the ordered conversational/log compatibility
 projection. Full trace detail stays in `trace.json`
diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
index ae6a5dfb2..8aafcc5a6 100644
--- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
+++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md
@@ -263,18 +263,21 @@ Repeat-run cases use attempt directories:
 .agentv/results/<experiment>/<timestamp>/<case-id>/grading.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/grading.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-1/metrics.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/transcript-raw.jsonl
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-1/outputs/answer.md
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/result.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/grading.json
-.agentv/results/<experiment>/<timestamp>/<case-id>/run-2/metrics.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript.json
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/transcript-raw.jsonl
 .agentv/results/<experiment>/<timestamp>/<case-id>/run-2/outputs/answer.md
 ```
 
+Each `run-N/result.json` is the per-attempt manifest. It carries paths such as
+`grading_path`, `transcript_path`, `transcript_raw_path`, `output_paths.answer`,
+plus embedded timing/o11y metrics so repeat attempts do not need a separate
+`metrics.json` sidecar.
+
 `<case-id>` should reuse the sanitized artifact key produced by the current artifact writer, including suite disambiguation where needed. Attempt directories are one-indexed because users naturally inspect `run-1`, `run-2`, and this matches the Vercel comparison.
 
 ---
diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts
index 87533e09e..c28cf6b11 100644
--- a/packages/core/src/evaluation/run-artifacts.ts
+++ b/packages/core/src/evaluation/run-artifacts.ts
@@ -347,25 +347,28 @@ export interface AdditionalResultArtifactsContext {
 
 export interface VercelRunResultArtifact {
   readonly status: 'passed' | 'failed' | 'error';
-  readonly duration: number;
+  readonly duration_ms?: number;
+  readonly duration_seconds: number;
   readonly model: string;
-  readonly transcriptPath?: string;
-  readonly transcriptRawPath?: string;
+  readonly grading_path: string;
+  readonly transcript_path?: string;
+  readonly transcript_raw_path?: string;
   readonly o11y: {
-    readonly totalTurns: number;
-    readonly toolCalls: Record<string, number>;
-    readonly totalToolCalls: number;
-    readonly webFetches: readonly unknown[];
-    readonly filesRead: readonly string[];
-    readonly filesModified: readonly string[];
-    readonly shellCommands: readonly unknown[];
+    readonly total_turns: number;
+    readonly tool_calls: Record<string, number>;
+    readonly total_tool_calls: number;
+    readonly web_fetches: readonly unknown[];
+    readonly files_read: readonly string[];
+    readonly files_modified: readonly string[];
+    readonly shell_commands: readonly unknown[];
     readonly errors: readonly unknown[];
-    readonly thinkingBlocks: number;
+    readonly thinking_blocks: number;
   };
-  readonly outputPaths?: {
+  readonly output_paths?: {
     readonly answer?: string;
     readonly scripts?: Record<string, string>;
   };
+  readonly timing?: TimingArtifact;
 }
 
 export interface RepeatCaseSummaryArtifact {
@@ -720,29 +723,34 @@ function toFilePathList(entries: readonly unknown[]): readonly string[] {
 function buildVercelRunResultArtifact(params: {
   readonly trial: TrialResult;
   readonly result: EvaluationResult;
-  readonly metricsArtifact: ReturnType<typeof buildMetricsArtifact>;
+  readonly metricsArtifact: ReturnType<typeof buildMetricsArtifact> & {
+    readonly timing?: TimingArtifact;
+  };
   readonly hasTranscript: boolean;
   readonly hasOutput: boolean;
 }): VercelRunResultArtifact {
   const metrics = params.metricsArtifact.metrics;
   return dropUndefined({
     status: toVercelRunStatus(params.trial, params.result),
-    duration: resultDurationSeconds(params.result),
+    duration_ms: resultDurationMs(params.result),
+    duration_seconds: resultDurationSeconds(params.result),
     model: params.result.target ?? 'unknown',
-    transcriptPath: params.hasTranscript ? './transcript.json' : undefined,
-    transcriptRawPath: params.hasTranscript ? './transcript-raw.jsonl' : undefined,
+    grading_path: './grading.json',
+    transcript_path: params.hasTranscript ? './transcript.json' : undefined,
+    transcript_raw_path: params.hasTranscript ? './transcript-raw.jsonl' : undefined,
     o11y: {
-      totalTurns: metrics.total_turns,
-      toolCalls: metrics.tool_calls,
-      totalToolCalls: metrics.total_tool_calls,
-      webFetches: metrics.web_fetches,
-      filesRead: toFilePathList(metrics.files_read),
-      filesModified: toFilePathList(metrics.files_modified),
-      shellCommands: metrics.shell_commands,
+      total_turns: metrics.total_turns,
+      tool_calls: metrics.tool_calls,
+      total_tool_calls: metrics.total_tool_calls,
+      web_fetches: metrics.web_fetches,
+      files_read: toFilePathList(metrics.files_read),
+      files_modified: toFilePathList(metrics.files_modified),
+      shell_commands: metrics.shell_commands,
       errors: metrics.errors,
-      thinkingBlocks: metrics.thinking_blocks,
+      thinking_blocks: metrics.thinking_blocks,
     },
-    outputPaths: params.hasOutput ? { answer: './outputs/answer.md' } : undefined,
+    output_paths: params.hasOutput ? { answer: './outputs/answer.md' } : undefined,
+    timing: params.metricsArtifact.timing,
   }) as unknown as VercelRunResultArtifact;
 }
 
@@ -784,7 +792,6 @@ async function writeTrialRunArtifacts(params: {
   const hasTranscript = hasTranscriptProjection(result, envelope);
   const transcriptPath = hasTranscript ? path.join(runDir, 'transcript.json') : undefined;
   const transcriptRawPath = hasTranscript ? path.join(runDir, 'transcript-raw.jsonl') : undefined;
-  const metricsPath = path.join(runDir, CANONICAL_METRICS_ARTIFACT_PATH);
 
   await mkdir(runDir, { recursive: true });
   await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}\n`, 'utf8');
@@ -801,8 +808,7 @@ async function writeTrialRunArtifacts(params: {
     );
     await writeTranscriptJsonl(transcriptRawPath, result, envelope);
   }
-  const metricsArtifact = await writeMetricsArtifact({
-    filePath: metricsPath,
+  const metricsArtifact = buildMetricsArtifactPayload({
     result,
     envelope,
     traceArtifactPath: 'transcript.json',
@@ -1614,8 +1620,7 @@ async function writeTranscriptJsonl(
   await writeFile(filePath, content, 'utf8');
 }
 
-async function writeMetricsArtifact(params: {
-  readonly filePath: string;
+function buildMetricsArtifactPayload(params: {
   readonly result: EvaluationResult;
   readonly envelope: TraceEnvelope;
   readonly transcriptPath?: string;
@@ -1624,7 +1629,7 @@ async function writeMetricsArtifact(params: {
   readonly gradingArtifactPath?: string;
   readonly timingArtifactPath?: string | null;
   readonly timing?: TimingArtifact;
-}): Promise<ReturnType<typeof buildMetricsArtifact> & { readonly timing?: TimingArtifact }> {
+}): ReturnType<typeof buildMetricsArtifact> & { readonly timing?: TimingArtifact } {
   const artifact = buildMetricsArtifact(params.result, params.envelope, {
     tracePath: params.traceArtifactPath ?? CANONICAL_TRACE_ARTIFACT_PATH,
     transcriptPath:
@@ -1634,7 +1639,21 @@ async function writeMetricsArtifact(params: {
     timingPath:
       params.timingArtifactPath === null ? undefined : (params.timingArtifactPath ?? 'timing.json'),
   });
-  const artifactWithTiming = params.timing ? { ...artifact, timing: params.timing } : artifact;
+  return params.timing ? { ...artifact, timing: params.timing } : artifact;
+}
+
+async function writeMetricsArtifact(params: {
+  readonly filePath: string;
+  readonly result: EvaluationResult;
+  readonly envelope: TraceEnvelope;
+  readonly transcriptPath?: string;
+  readonly traceArtifactPath?: string;
+  readonly transcriptArtifactPath?: string;
+  readonly gradingArtifactPath?: string;
+  readonly timingArtifactPath?: string | null;
+  readonly timing?: TimingArtifact;
+}): Promise<ReturnType<typeof buildMetricsArtifact> & { readonly timing?: TimingArtifact }> {
+  const artifactWithTiming = buildMetricsArtifactPayload(params);
   await writeFile(params.filePath, `${JSON.stringify(artifactWithTiming, null, 2)}\n`, 'utf8');
   return artifactWithTiming;
 }