fix(eval): remove dry-run mock execution

.agents/verification.md

@@ -127,13 +127,13 @@ bun apps/cli/src/cli.ts eval examples/features/rubric/evals/dataset.eval.yaml --
 - the `assertions` array reflects the evaluation logic
 
 4. Update baseline files if output format changes. Baselines live next to eval YAML files as `*.baseline.jsonl`.
-5. `--dry-run` returns schema-valid mock responses, but the scores are not meaningful. Use it only for plumbing and harness checks.
+5. `agentv validate` is the cheap schema/config check. For no-live-provider quality validation, run graders against a real reference/oracle target or frozen transcript/replay fixture.
 
 ## Live Dogfood for Eval and Experiment Changes
 
 Use live dogfood before marking PRs ready when they affect eval execution, experiments, repeat runs, targets, providers, graders, or artifact provenance.
 
-- Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes.
+- Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, replay/frozen transcript runs, or deterministic-only assertions as dogfood for these changes.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For artifact/result contract changes, prefer letting AgentV choose the canonical run directory and capture the printed `Artifact workspace written to:` and `Results written to:` paths for evidence. Do not precompute `--output` unless the test specifically needs a fixed path.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.

AGENTS.md

@@ -43,7 +43,7 @@ Read the full rationale and examples in [.agents/product-boundary.md](.agents/pr
 - Non-trivial work needs a plan or task list. If the implementation surface starts to balloon, stop and re-plan.
 - Large or high-risk PRs need meaningful, reviewable commits for each coherent change. Rewrite only the PR branch with `git push --force-with-lease` when needed to replace WIP or accidental squashed history before review.
 - Manual red/green UAT is blocking before a branch is ready for review. GitHub Actions is the authoritative merge gate.
-- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. Mock graders, dry-run, and deterministic-only smoke tests are useful plumbing checks, but they are not live dogfood. Use canonical `.agentv/results/<experiment>/<timestamp>` output and publish private evidence. See [.agents/verification.md](.agents/verification.md).
+- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. `agentv validate`, mock targets, replay/frozen transcript runs, and deterministic-only smoke tests are useful checks, but they are not live dogfood. Use canonical `.agentv/results/<experiment>/<timestamp>` output and publish private evidence. See [.agents/verification.md](.agents/verification.md).
 - For browser or screenshot UAT, keep evidence out of the public repo and publish reviewable artifacts to an `agentv-private` evidence branch. See [.agents/verification.md](.agents/verification.md).
 - When dogfood or review reveals a durable workflow lesson, capture it in this guide or the relevant `.agents/*.md` guide before merge; do not leave durable agent instructions only in PR comments, Bead comments, or private evidence. Use `docs/solutions/` for fuller reusable writeups.
 - Research-only workers must not run `bun install`, `bun run build`, tests, or evals unless the assigned work explicitly needs that command and the worker records why.

apps/cli/src/commands/eval/commands/run.ts

@@ -92,29 +92,6 @@ export const evalRunCommand = command({
       long: 'results-require-push',
       description: 'Fail the eval command if the completed results branch cannot be pushed',
     }),
-    dryRun: flag({
-      long: 'dry-run',
-      description: 'Use mock provider responses instead of real LLM calls',
-    }),
-    dryRunDelay: option({
-      type: number,
-      long: 'dry-run-delay',
-      description:
-        'Fixed delay in milliseconds for dry-run mode (overridden by delay range if specified)',
-      defaultValue: () => 0,
-    }),
-    dryRunDelayMin: option({
-      type: number,
-      long: 'dry-run-delay-min',
-      description: 'Minimum delay in milliseconds for dry-run mode (requires --dry-run-delay-max)',
-      defaultValue: () => 0,
-    }),
-    dryRunDelayMax: option({
-      type: number,
-      long: 'dry-run-delay-max',
-      description: 'Maximum delay in milliseconds for dry-run mode (requires --dry-run-delay-min)',
-      defaultValue: () => 0,
-    }),
     agentTimeout: option({
       type: optional(number),
       long: 'agent-timeout',
@@ -286,10 +263,6 @@ export const evalRunCommand = command({
       resultsPush: args.resultsPush,
       noResultsPush: args.noResultsPush,
       resultsRequirePush: args.resultsRequirePush,
-      dryRun: args.dryRun,
-      dryRunDelay: args.dryRunDelay,
-      dryRunDelayMin: args.dryRunDelayMin,
-      dryRunDelayMax: args.dryRunDelayMax,
       agentTimeout: args.agentTimeout,
       maxRetries: args.maxRetries,
       cache: args.cache,

apps/cli/src/commands/eval/interactive.ts

@@ -24,7 +24,6 @@ export interface InteractiveConfig {
   readonly evalPaths: readonly string[];
   readonly target: string;
   readonly workers: number;
-  readonly dryRun: boolean;
   readonly cache: boolean;
 }
 
@@ -51,7 +50,6 @@ export async function launchInteractiveWizard(): Promise<void> {
         evalPaths: lastConfig.evalPaths,
         target: lastConfig.target,
         workers: lastConfig.workers,
-        dryRun: lastConfig.dryRun,
         cache: lastConfig.cache,
       },
       { resumeOutputDir: lastConfig.outputDir },
@@ -65,7 +63,6 @@ export async function launchInteractiveWizard(): Promise<void> {
       evalPaths: lastConfig.evalPaths,
       target: lastConfig.target,
       workers: lastConfig.workers,
-      dryRun: lastConfig.dryRun,
       cache: lastConfig.cache,
     });
     return;
@@ -278,7 +275,6 @@ async function findTargetsFile(
 
 async function promptAdvancedOptions(): Promise<{
   workers: number;
-  dryRun: boolean;
   cache: boolean;
 }> {
   const customize = await confirm({
@@ -287,7 +283,7 @@ async function promptAdvancedOptions(): Promise<{
   });
 
   if (!customize) {
-    return { workers: 3, dryRun: false, cache: false };
+    return { workers: 3, cache: false };
   }
 
   const workers =
@@ -298,17 +294,12 @@ async function promptAdvancedOptions(): Promise<{
       max: 50,
     })) ?? 3;
 
-  const dryRun = await confirm({
-    message: 'Enable dry-run mode (mock responses)?',
-    default: false,
-  });
-
   const cache = await confirm({
     message: 'Enable response cache?',
     default: false,
   });
 
-  return { workers, dryRun, cache };
+  return { workers, cache };
 }
 
 async function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): Promise<boolean> {
@@ -324,7 +315,6 @@ async function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): P
   console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}\n${evalDisplay}`);
   console.log(`${ANSI_GREEN}Target:${ANSI_RESET}    ${config.target}`);
   console.log(`${ANSI_GREEN}Workers:${ANSI_RESET}   ${config.workers}`);
-  console.log(`${ANSI_GREEN}Dry run:${ANSI_RESET}   ${config.dryRun ? 'yes' : 'no'}`);
   console.log(`${ANSI_GREEN}Cache:${ANSI_RESET}     ${config.cache ? 'yes' : 'no'}`);
   console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);
 
@@ -342,12 +332,8 @@ async function executeConfig(
   const rawOptions: Record<string, unknown> = {
     target: config.target,
     workers: config.workers,
-    dryRun: config.dryRun,
     cache: config.cache,
     ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}),
-    dryRunDelay: 0,
-    dryRunDelayMin: 0,
-    dryRunDelayMax: 0,
     agentTimeout: 120,
     maxRetries: 2,
     verbose: false,
@@ -371,7 +357,6 @@ async function executeConfig(
       evalPaths: config.evalPaths,
       target: config.target,
       workers: config.workers,
-      dryRun: config.dryRun,
       cache: config.cache,
       outputDir: path.dirname(result.outputPath),
     });
@@ -398,13 +383,9 @@ async function promptRetryErrors(config: InteractiveConfig, outputPath: string):
   const rawOptions: Record<string, unknown> = {
     target: config.target,
     workers: config.workers,
-    dryRun: config.dryRun,
     cache: config.cache,
     retryErrors: outputPath,
     out: outputPath,
-    dryRunDelay: 0,
-    dryRunDelayMin: 0,
-    dryRunDelayMax: 0,
     agentTimeout: 120,
     maxRetries: 2,
     verbose: false,

apps/cli/src/commands/eval/last-config.ts

@@ -11,7 +11,6 @@ export interface LastConfig {
   readonly evalPaths: readonly string[];
   readonly target: string;
   readonly workers: number;
-  readonly dryRun: boolean;
   readonly cache: boolean;
   /**
    * Resolved artifact directory of the last completed wizard run. Used to

apps/cli/src/commands/eval/run-eval.ts

@@ -114,10 +114,6 @@ interface NormalizedOptions {
   readonly outputDir?: string;
   /** Removed: use --output for run directories */
   readonly removedOut?: string;
-  readonly dryRun: boolean;
-  readonly dryRunDelay: number;
-  readonly dryRunDelayMin: number;
-  readonly dryRunDelayMax: number;
   readonly agentTimeoutSeconds?: number;
   readonly cliAgentTimeoutSeconds?: number;
   readonly maxRetries: number;
@@ -204,19 +200,6 @@ export function resolveTimestampPlaceholder(value: string): string {
   return value.replaceAll('{timestamp}', timestamp);
 }
 
-function normalizeNumber(value: unknown, fallback: number): number {
-  if (typeof value === 'number' && Number.isFinite(value)) {
-    return value;
-  }
-  if (typeof value === 'string') {
-    const parsed = Number.parseInt(value, 10);
-    if (!Number.isNaN(parsed)) {
-      return parsed;
-    }
-  }
-  return fallback;
-}
-
 function normalizeOptionalNumber(value: unknown): number | undefined {
   if (typeof value === 'number' && Number.isFinite(value)) {
     return value;
@@ -473,10 +456,6 @@ function normalizeOptions(
     workers: workers > 0 ? workers : undefined,
     outputDir: cliOutputDir ?? configOutputDir,
     removedOut: cliOut,
-    dryRun: normalizeBoolean(rawOptions.dryRun),
-    dryRunDelay: normalizeNumber(rawOptions.dryRunDelay, 0),
-    dryRunDelayMin: normalizeNumber(rawOptions.dryRunDelayMin, 0),
-    dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0),
     agentTimeoutSeconds: cliAgentTimeout ?? configAgentTimeoutSeconds,
     cliAgentTimeoutSeconds: cliAgentTimeout,
     maxRetries: cliMaxRetries ?? configMaxRetries ?? 2,
@@ -983,23 +962,9 @@ async function prepareFileMetadata(params: {
     ];
   } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) {
     const targetDefinition = suite.inlineTarget;
-    const resolvedTarget = options.dryRun
-      ? ({
-          kind: 'mock',
-          name: `${targetDefinition.name}-dry-run`,
-          graderTarget: undefined,
-          config: {
-            // Schema-valid grader response so --dry-run works end-to-end with LLM graders.
-            // Satisfies freeform (score), rubric (checks, overall_reasoning), and score-range (checks) without real LLM calls.
-            response: '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}',
-            delayMs: options.dryRunDelay,
-            delayMinMs: options.dryRunDelayMin,
-            delayMaxMs: options.dryRunDelayMax,
-          },
-        } satisfies ResolvedTarget)
-      : resolveTargetDefinition(targetDefinition, process.env, testFilePath, {
-          emitDeprecationWarnings: false,
-        });
+    const resolvedTarget = resolveTargetDefinition(targetDefinition, process.env, testFilePath, {
+      emitDeprecationWarnings: false,
+    });
     selections = [
       {
         selection: {
@@ -1059,10 +1024,6 @@ async function prepareFileMetadata(params: {
         repoRoot,
         cwd,
         explicitTargetsPath: effectiveOptions.targetsPath,
-        dryRun: effectiveOptions.dryRun,
-        dryRunDelay: effectiveOptions.dryRunDelay,
-        dryRunDelayMin: effectiveOptions.dryRunDelayMin,
-        dryRunDelayMax: effectiveOptions.dryRunDelayMax,
         env: process.env,
         targetNames,
         targetRefs,
@@ -1080,10 +1041,6 @@ async function prepareFileMetadata(params: {
         cwd,
         explicitTargetsPath: effectiveOptions.targetsPath,
         cliTargetName: targetNames.length === 1 ? targetNames[0] : effectiveOptions.target,
-        dryRun: effectiveOptions.dryRun,
-        dryRunDelay: effectiveOptions.dryRunDelay,
-        dryRunDelayMin: effectiveOptions.dryRunDelayMin,
-        dryRunDelayMax: effectiveOptions.dryRunDelayMax,
         env: process.env,
       });
 
@@ -1211,9 +1168,7 @@ async function runSingleEvalFile(params: {
 
   // CLI provider verbose logging should only be enabled when --verbose flag is passed
   const resolvedTargetSelection = applyVerboseOverride(selection, options.verbose);
-  const providerLabel = options.dryRun
-    ? `${resolvedTargetSelection.resolvedTarget.kind} (dry-run)`
-    : resolvedTargetSelection.resolvedTarget.kind;
+  const providerLabel = resolvedTargetSelection.resolvedTarget.kind;
   const targetMessage = options.verbose
     ? `Using target (${resolvedTargetSelection.targetSource}): ${resolvedTargetSelection.targetName} ${buildTargetLabelSuffix(providerLabel, resolvedTargetSelection.resolvedTarget)} via ${resolvedTargetSelection.targetsFilePath}`
     : `Using target: ${inlineTargetLabel}`;
@@ -1223,7 +1178,7 @@ async function runSingleEvalFile(params: {
 
   // Hint about pipeline for CLI agent targets
   const targetKind = resolvedTargetSelection.resolvedTarget.kind;
-  if ((targetKind === 'claude-cli' || targetKind === 'copilot-cli') && !options.dryRun) {
+  if (targetKind === 'claude-cli' || targetKind === 'copilot-cli') {
     console.log('');
     console.log('  TIP: For subagent-mode evals, use `agentv pipeline` instead of `eval run`.');
     console.log('  The agent orchestrates executor + grader subagents directly.');
@@ -1257,7 +1212,7 @@ async function runSingleEvalFile(params: {
   }
 
   // Auto-provision subagents for VSCode targets
-  if (isVSCodeProvider && !options.dryRun) {
+  if (isVSCodeProvider) {
     const vsConfig = resolvedTargetSelection.resolvedTarget.config as { executable?: string };
     await ensureVSCodeSubagents({
       kind: resolvedTargetSelection.resolvedTarget.kind as 'vscode' | 'vscode-insiders',