From d79c18b64529481f5e03297b2ae99522d0b6b9d6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 28 Jun 2026 03:26:11 +0200 Subject: [PATCH] fix(eval): remove dry-run mock execution --- .agents/verification.md | 4 +- AGENTS.md | 2 +- apps/cli/src/commands/eval/commands/run.ts | 27 ----- apps/cli/src/commands/eval/interactive.ts | 23 +--- apps/cli/src/commands/eval/last-config.ts | 1 - apps/cli/src/commands/eval/run-eval.ts | 57 +--------- apps/cli/src/commands/eval/targets.ts | 103 ++---------------- apps/cli/src/commands/grade/index.ts | 4 - apps/cli/src/commands/pipeline/input.ts | 4 - apps/cli/src/commands/pipeline/run.ts | 4 - apps/cli/src/commands/prepare/index.ts | 4 - apps/cli/src/commands/results/eval-runner.ts | 6 - apps/cli/test/commands/results/serve.test.ts | 15 +++ apps/cli/test/eval.integration.test.ts | 55 ++++++++++ .../dashboard/src/components/RunEvalModal.tsx | 15 --- .../src/components/run-eval-threshold.test.ts | 3 - .../src/components/run-eval-threshold.ts | 3 - apps/dashboard/src/lib/types.ts | 1 - .../docs/docs/evaluation/running-evals.mdx | 20 +++- .../docs/docs/targets/cli-provider.mdx | 5 + .../docs/docs/targets/configuration.mdx | 2 +- .../src/content/docs/docs/tools/validate.mdx | 6 + ...0010-remove-eval-dry-run-mock-execution.md | 85 +++++++++++++++ .../features/agent-skills-evals/README.md | 2 +- examples/features/composite/README.md | 4 +- .../docker-workspace/.agentv/targets.yaml | 3 + .../evals/docker-example.EVAL.yaml | 2 +- .../execution-metrics/evals/dataset.eval.yaml | 4 +- .../features/latency-assertions/README.md | 8 +- .../evals/dataset.eval.yaml | 4 +- .../features/prompt-template-sdk/README.md | 2 +- .../suite-level-input/.agentv/targets.yaml | 3 + .../evals/dataset.eval.baseline.jsonl | 6 +- .../.agentv/targets.yaml | 3 + .../evals/dataset.eval.baseline.jsonl | 6 +- .../evals/dataset.eval.yaml | 4 +- examples/features/trace-evaluation/README.md | 4 +- .../trace-evaluation/evals/dataset.eval.yaml | 4 +- .../trial-output-consistency/README.md | 8 +- .../evals/dataset.eval.yaml | 4 +- .../showcase/grader-conformance/EVAL.yaml | 4 +- .../graders/dry-run-mock-response.test.ts | 52 --------- skills-data/agentv-eval-writer/SKILL.md | 2 +- 43 files changed, 248 insertions(+), 330 deletions(-) create mode 100644 docs/adr/0010-remove-eval-dry-run-mock-execution.md create mode 100644 examples/features/docker-workspace/.agentv/targets.yaml create mode 100644 examples/features/suite-level-input/.agentv/targets.yaml create mode 100644 examples/features/tool-evaluation-plugins/.agentv/targets.yaml delete mode 100644 packages/core/test/evaluation/graders/dry-run-mock-response.test.ts diff --git a/.agents/verification.md b/.agents/verification.md index 48a6f9e59..376d5b1c9 100644 --- a/.agents/verification.md +++ b/.agents/verification.md @@ -127,13 +127,13 @@ bun apps/cli/src/cli.ts eval examples/features/rubric/evals/dataset.eval.yaml -- - the `assertions` array reflects the evaluation logic 4. Update baseline files if output format changes. Baselines live next to eval YAML files as `*.baseline.jsonl`. -5. `--dry-run` returns schema-valid mock responses, but the scores are not meaningful. Use it only for plumbing and harness checks. +5. `agentv validate` is the cheap schema/config check. For no-live-provider quality validation, run graders against a real reference/oracle target or frozen transcript/replay fixture. ## Live Dogfood for Eval and Experiment Changes Use live dogfood before marking PRs ready when they affect eval execution, experiments, repeat runs, targets, providers, graders, or artifact provenance. -- Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, `--dry-run`, or deterministic-only assertions as dogfood for these changes. +- Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, replay/frozen transcript runs, or deterministic-only assertions as dogfood for these changes. - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers. - For artifact/result contract changes, prefer letting AgentV choose the canonical run directory and capture the printed `Artifact workspace written to:` and `Results written to:` paths for evidence. Do not precompute `--output` unless the test specifically needs a fixed path. - For native experiment changes, run through `agentv eval run ... --experiment ` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together. diff --git a/AGENTS.md b/AGENTS.md index 8549f6606..9e02890e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ Read the full rationale and examples in [.agents/product-boundary.md](.agents/pr - Non-trivial work needs a plan or task list. If the implementation surface starts to balloon, stop and re-plan. - Large or high-risk PRs need meaningful, reviewable commits for each coherent change. Rewrite only the PR branch with `git push --force-with-lease` when needed to replace WIP or accidental squashed history before review. - Manual red/green UAT is blocking before a branch is ready for review. GitHub Actions is the authoritative merge gate. -- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. Mock graders, dry-run, and deterministic-only smoke tests are useful plumbing checks, but they are not live dogfood. Use canonical `.agentv/results//` output and publish private evidence. See [.agents/verification.md](.agents/verification.md). +- For eval execution, experiments, repeat runs, providers, graders, or artifact-layout changes, dogfood with a live provider and a real LLM grader before marking ready. `agentv validate`, mock targets, replay/frozen transcript runs, and deterministic-only smoke tests are useful checks, but they are not live dogfood. Use canonical `.agentv/results//` output and publish private evidence. See [.agents/verification.md](.agents/verification.md). - For browser or screenshot UAT, keep evidence out of the public repo and publish reviewable artifacts to an `agentv-private` evidence branch. See [.agents/verification.md](.agents/verification.md). - When dogfood or review reveals a durable workflow lesson, capture it in this guide or the relevant `.agents/*.md` guide before merge; do not leave durable agent instructions only in PR comments, Bead comments, or private evidence. Use `docs/solutions/` for fuller reusable writeups. - Research-only workers must not run `bun install`, `bun run build`, tests, or evals unless the assigned work explicitly needs that command and the worker records why. diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 3b392ce7c..b577f6e59 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -92,29 +92,6 @@ export const evalRunCommand = command({ long: 'results-require-push', description: 'Fail the eval command if the completed results branch cannot be pushed', }), - dryRun: flag({ - long: 'dry-run', - description: 'Use mock provider responses instead of real LLM calls', - }), - dryRunDelay: option({ - type: number, - long: 'dry-run-delay', - description: - 'Fixed delay in milliseconds for dry-run mode (overridden by delay range if specified)', - defaultValue: () => 0, - }), - dryRunDelayMin: option({ - type: number, - long: 'dry-run-delay-min', - description: 'Minimum delay in milliseconds for dry-run mode (requires --dry-run-delay-max)', - defaultValue: () => 0, - }), - dryRunDelayMax: option({ - type: number, - long: 'dry-run-delay-max', - description: 'Maximum delay in milliseconds for dry-run mode (requires --dry-run-delay-min)', - defaultValue: () => 0, - }), agentTimeout: option({ type: optional(number), long: 'agent-timeout', @@ -286,10 +263,6 @@ export const evalRunCommand = command({ resultsPush: args.resultsPush, noResultsPush: args.noResultsPush, resultsRequirePush: args.resultsRequirePush, - dryRun: args.dryRun, - dryRunDelay: args.dryRunDelay, - dryRunDelayMin: args.dryRunDelayMin, - dryRunDelayMax: args.dryRunDelayMax, agentTimeout: args.agentTimeout, maxRetries: args.maxRetries, cache: args.cache, diff --git a/apps/cli/src/commands/eval/interactive.ts b/apps/cli/src/commands/eval/interactive.ts index dad5efa9b..268fb231b 100644 --- a/apps/cli/src/commands/eval/interactive.ts +++ b/apps/cli/src/commands/eval/interactive.ts @@ -24,7 +24,6 @@ export interface InteractiveConfig { readonly evalPaths: readonly string[]; readonly target: string; readonly workers: number; - readonly dryRun: boolean; readonly cache: boolean; } @@ -51,7 +50,6 @@ export async function launchInteractiveWizard(): Promise { evalPaths: lastConfig.evalPaths, target: lastConfig.target, workers: lastConfig.workers, - dryRun: lastConfig.dryRun, cache: lastConfig.cache, }, { resumeOutputDir: lastConfig.outputDir }, @@ -65,7 +63,6 @@ export async function launchInteractiveWizard(): Promise { evalPaths: lastConfig.evalPaths, target: lastConfig.target, workers: lastConfig.workers, - dryRun: lastConfig.dryRun, cache: lastConfig.cache, }); return; @@ -278,7 +275,6 @@ async function findTargetsFile( async function promptAdvancedOptions(): Promise<{ workers: number; - dryRun: boolean; cache: boolean; }> { const customize = await confirm({ @@ -287,7 +283,7 @@ async function promptAdvancedOptions(): Promise<{ }); if (!customize) { - return { workers: 3, dryRun: false, cache: false }; + return { workers: 3, cache: false }; } const workers = @@ -298,17 +294,12 @@ async function promptAdvancedOptions(): Promise<{ max: 50, })) ?? 3; - const dryRun = await confirm({ - message: 'Enable dry-run mode (mock responses)?', - default: false, - }); - const cache = await confirm({ message: 'Enable response cache?', default: false, }); - return { workers, dryRun, cache }; + return { workers, cache }; } async function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): Promise { @@ -324,7 +315,6 @@ async function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): P console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}\n${evalDisplay}`); console.log(`${ANSI_GREEN}Target:${ANSI_RESET} ${config.target}`); console.log(`${ANSI_GREEN}Workers:${ANSI_RESET} ${config.workers}`); - console.log(`${ANSI_GREEN}Dry run:${ANSI_RESET} ${config.dryRun ? 'yes' : 'no'}`); console.log(`${ANSI_GREEN}Cache:${ANSI_RESET} ${config.cache ? 'yes' : 'no'}`); console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`); @@ -342,12 +332,8 @@ async function executeConfig( const rawOptions: Record = { target: config.target, workers: config.workers, - dryRun: config.dryRun, cache: config.cache, ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}), - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, agentTimeout: 120, maxRetries: 2, verbose: false, @@ -371,7 +357,6 @@ async function executeConfig( evalPaths: config.evalPaths, target: config.target, workers: config.workers, - dryRun: config.dryRun, cache: config.cache, outputDir: path.dirname(result.outputPath), }); @@ -398,13 +383,9 @@ async function promptRetryErrors(config: InteractiveConfig, outputPath: string): const rawOptions: Record = { target: config.target, workers: config.workers, - dryRun: config.dryRun, cache: config.cache, retryErrors: outputPath, out: outputPath, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, agentTimeout: 120, maxRetries: 2, verbose: false, diff --git a/apps/cli/src/commands/eval/last-config.ts b/apps/cli/src/commands/eval/last-config.ts index fca7a8f15..a08a284b2 100644 --- a/apps/cli/src/commands/eval/last-config.ts +++ b/apps/cli/src/commands/eval/last-config.ts @@ -11,7 +11,6 @@ export interface LastConfig { readonly evalPaths: readonly string[]; readonly target: string; readonly workers: number; - readonly dryRun: boolean; readonly cache: boolean; /** * Resolved artifact directory of the last completed wizard run. Used to diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 182a3a3ed..c044f19e7 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -114,10 +114,6 @@ interface NormalizedOptions { readonly outputDir?: string; /** Removed: use --output for run directories */ readonly removedOut?: string; - readonly dryRun: boolean; - readonly dryRunDelay: number; - readonly dryRunDelayMin: number; - readonly dryRunDelayMax: number; readonly agentTimeoutSeconds?: number; readonly cliAgentTimeoutSeconds?: number; readonly maxRetries: number; @@ -204,19 +200,6 @@ export function resolveTimestampPlaceholder(value: string): string { return value.replaceAll('{timestamp}', timestamp); } -function normalizeNumber(value: unknown, fallback: number): number { - if (typeof value === 'number' && Number.isFinite(value)) { - return value; - } - if (typeof value === 'string') { - const parsed = Number.parseInt(value, 10); - if (!Number.isNaN(parsed)) { - return parsed; - } - } - return fallback; -} - function normalizeOptionalNumber(value: unknown): number | undefined { if (typeof value === 'number' && Number.isFinite(value)) { return value; @@ -473,10 +456,6 @@ function normalizeOptions( workers: workers > 0 ? workers : undefined, outputDir: cliOutputDir ?? configOutputDir, removedOut: cliOut, - dryRun: normalizeBoolean(rawOptions.dryRun), - dryRunDelay: normalizeNumber(rawOptions.dryRunDelay, 0), - dryRunDelayMin: normalizeNumber(rawOptions.dryRunDelayMin, 0), - dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0), agentTimeoutSeconds: cliAgentTimeout ?? configAgentTimeoutSeconds, cliAgentTimeoutSeconds: cliAgentTimeout, maxRetries: cliMaxRetries ?? configMaxRetries ?? 2, @@ -983,23 +962,9 @@ async function prepareFileMetadata(params: { ]; } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) { const targetDefinition = suite.inlineTarget; - const resolvedTarget = options.dryRun - ? ({ - kind: 'mock', - name: `${targetDefinition.name}-dry-run`, - graderTarget: undefined, - config: { - // Schema-valid grader response so --dry-run works end-to-end with LLM graders. - // Satisfies freeform (score), rubric (checks, overall_reasoning), and score-range (checks) without real LLM calls. - response: '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}', - delayMs: options.dryRunDelay, - delayMinMs: options.dryRunDelayMin, - delayMaxMs: options.dryRunDelayMax, - }, - } satisfies ResolvedTarget) - : resolveTargetDefinition(targetDefinition, process.env, testFilePath, { - emitDeprecationWarnings: false, - }); + const resolvedTarget = resolveTargetDefinition(targetDefinition, process.env, testFilePath, { + emitDeprecationWarnings: false, + }); selections = [ { selection: { @@ -1059,10 +1024,6 @@ async function prepareFileMetadata(params: { repoRoot, cwd, explicitTargetsPath: effectiveOptions.targetsPath, - dryRun: effectiveOptions.dryRun, - dryRunDelay: effectiveOptions.dryRunDelay, - dryRunDelayMin: effectiveOptions.dryRunDelayMin, - dryRunDelayMax: effectiveOptions.dryRunDelayMax, env: process.env, targetNames, targetRefs, @@ -1080,10 +1041,6 @@ async function prepareFileMetadata(params: { cwd, explicitTargetsPath: effectiveOptions.targetsPath, cliTargetName: targetNames.length === 1 ? targetNames[0] : effectiveOptions.target, - dryRun: effectiveOptions.dryRun, - dryRunDelay: effectiveOptions.dryRunDelay, - dryRunDelayMin: effectiveOptions.dryRunDelayMin, - dryRunDelayMax: effectiveOptions.dryRunDelayMax, env: process.env, }); @@ -1211,9 +1168,7 @@ async function runSingleEvalFile(params: { // CLI provider verbose logging should only be enabled when --verbose flag is passed const resolvedTargetSelection = applyVerboseOverride(selection, options.verbose); - const providerLabel = options.dryRun - ? `${resolvedTargetSelection.resolvedTarget.kind} (dry-run)` - : resolvedTargetSelection.resolvedTarget.kind; + const providerLabel = resolvedTargetSelection.resolvedTarget.kind; const targetMessage = options.verbose ? `Using target (${resolvedTargetSelection.targetSource}): ${resolvedTargetSelection.targetName} ${buildTargetLabelSuffix(providerLabel, resolvedTargetSelection.resolvedTarget)} via ${resolvedTargetSelection.targetsFilePath}` : `Using target: ${inlineTargetLabel}`; @@ -1223,7 +1178,7 @@ async function runSingleEvalFile(params: { // Hint about pipeline for CLI agent targets const targetKind = resolvedTargetSelection.resolvedTarget.kind; - if ((targetKind === 'claude-cli' || targetKind === 'copilot-cli') && !options.dryRun) { + if (targetKind === 'claude-cli' || targetKind === 'copilot-cli') { console.log(''); console.log(' TIP: For subagent-mode evals, use `agentv pipeline` instead of `eval run`.'); console.log(' The agent orchestrates executor + grader subagents directly.'); @@ -1257,7 +1212,7 @@ async function runSingleEvalFile(params: { } // Auto-provision subagents for VSCode targets - if (isVSCodeProvider && !options.dryRun) { + if (isVSCodeProvider) { const vsConfig = resolvedTargetSelection.resolvedTarget.config as { executable?: string }; await ensureVSCodeSubagents({ kind: resolvedTargetSelection.resolvedTarget.kind as 'vscode' | 'vscode-insiders', diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 25730cc61..173c7a66a 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -13,17 +13,6 @@ const ANSI_YELLOW = '\u001b[33m'; const ANSI_RED = '\u001b[31m'; const ANSI_RESET = '\u001b[0m'; -/** - * Dry-run mock response: satisfies all LLM grader schemas (freeform, rubric, score-range) - * so that --dry-run works end-to-end including graders without real LLM calls. - * - * - freeformEvaluationSchema: "score" (required), "assertions" (optional) - * - rubricEvaluationSchema: "checks" (required), "overall_reasoning" (required) - * - scoreRangeEvaluationSchema: "checks" (required), "overall_reasoning" (optional) - */ -const DRY_RUN_MOCK_RESPONSE = - '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}'; - function isTTY(): boolean { return process.stdout.isTTY ?? false; } @@ -108,10 +97,6 @@ export interface TargetSelectionOptions { readonly explicitTargetsPath?: string; readonly cliTargetName?: string; readonly cliTargetNames?: readonly string[]; - readonly dryRun: boolean; - readonly dryRunDelay: number; - readonly dryRunDelayMin: number; - readonly dryRunDelayMax: number; readonly env: NodeJS.ProcessEnv; } @@ -133,18 +118,7 @@ function pickTargetName(options: { } export async function selectTarget(options: TargetSelectionOptions): Promise { - const { - testFilePath, - repoRoot, - cwd, - explicitTargetsPath, - cliTargetName, - dryRun, - dryRunDelay, - dryRunDelayMin, - dryRunDelayMax, - env, - } = options; + const { testFilePath, repoRoot, cwd, explicitTargetsPath, cliTargetName, env } = options; const targetsFilePath = await discoverTargetsFile({ explicitPath: explicitTargetsPath, @@ -188,28 +162,6 @@ export async function selectTarget(options: TargetSelectionOptions): Promise { - const { - testFilePath, - repoRoot, - cwd, - explicitTargetsPath, - dryRun, - dryRunDelay, - dryRunDelayMin, - dryRunDelayMax, - env, - targetNames, - targetRefs, - } = options; + const { testFilePath, repoRoot, cwd, explicitTargetsPath, env, targetNames, targetRefs } = + options; // Build a lookup for target hooks from eval target refs const hooksMap = new Map(); @@ -312,43 +253,21 @@ export async function selectMultipleTargets( const targetDefinition = resolveUseTarget(name, definitions, env, targetsFilePath); const hooks = hooksMap.get(name); - if (dryRun) { - const mockTarget: ResolvedTarget = { - kind: 'mock', - name: `${targetDefinition.name}-dry-run`, - graderTarget: undefined, - config: { - response: DRY_RUN_MOCK_RESPONSE, - delayMs: dryRunDelay, - delayMinMs: dryRunDelayMin, - delayMaxMs: dryRunDelayMax, - }, - }; + try { + const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, { + emitDeprecationWarnings: false, + }); results.push({ definitions, - resolvedTarget: mockTarget, + resolvedTarget, targetName: name, targetSource: 'cli', targetsFilePath, ...(hooks && { targetHooks: hooks }), }); - } else { - try { - const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, { - emitDeprecationWarnings: false, - }); - results.push({ - definitions, - resolvedTarget, - targetName: name, - targetSource: 'cli', - targetsFilePath, - ...(hooks && { targetHooks: hooks }), - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to resolve target '${name}': ${message}`); - } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to resolve target '${name}': ${message}`); } } diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index fcc74d7cc..aeb9b17c4 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -544,10 +544,6 @@ async function gradePreparedAttempt(options: { testFilePath: evalPath, repoRoot, cwd: process.cwd(), - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, env: process.env, targetNames: [manifest.target], targetRefs: suite.targetRefs, diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index 61f46b854..c71f1fb82 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -106,10 +106,6 @@ export const evalInputCommand = command({ cwd: evalDir, cliTargetName: target, explicitTargetsPath: targets, - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, env: process.env, }); diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 3ac31529c..ff0992cb6 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -137,10 +137,6 @@ export const evalRunCommand = command({ cwd: evalDir, cliTargetName: target, explicitTargetsPath: targets, - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, env: process.env, }); targetName = selection.targetName; diff --git a/apps/cli/src/commands/prepare/index.ts b/apps/cli/src/commands/prepare/index.ts index 1240eb2cb..aa162c932 100644 --- a/apps/cli/src/commands/prepare/index.ts +++ b/apps/cli/src/commands/prepare/index.ts @@ -236,10 +236,6 @@ async function selectPrepareTarget(options: { testFilePath: options.evalPath, repoRoot: options.repoRoot, cwd: process.cwd(), - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, env: process.env, targetNames: [options.target], targetRefs: options.targetRefs, diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index eaa8a1535..4be84255e 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -145,7 +145,6 @@ interface RunEvalRequest { tags?: string[]; threshold?: number; workers?: number; - dry_run?: boolean; /** Resume an interrupted run: skip already-completed tests and append results to `output`. */ resume?: boolean; /** Re-run failed/errored tests while keeping passing results. */ @@ -233,11 +232,6 @@ function buildCliArgs(req: RunEvalRequest, experiment?: string): string[] { args.push('--workers', String(req.workers)); } - // Dry run - if (req.dry_run) { - args.push('--dry-run'); - } - // Resume / rerun-failed / retry-errors / output if (req.output?.trim()) { args.push('--output', req.output.trim()); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 35b2c8c38..b228e0aff 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -4397,6 +4397,21 @@ describe('serve app', () => { expect(data.command).toContain('--experiment smoke'); }); + it('does not emit removed eval dry-run flags from legacy dry_run payloads', async () => { + const app = createApp([], tempDir, undefined, undefined, { studioDir }); + const res = await app.request('/api/eval/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + dry_run: true, + }), + }); + expect(res.status).toBe(200); + const data = (await res.json()) as { command: string }; + expect(data.command).not.toContain('--dry-run'); + }); + it('rejects invalid experiment and tag values', async () => { const app = createApp([], tempDir, undefined, undefined, { studioDir }); const badExperiment = await app.request('/api/eval/preview', { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 2f75409e6..9673d38c5 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -883,6 +883,39 @@ describe('agentv eval CLI', () => { expect(helpText).toContain('summary.json'); }, 30_000); + it('omits removed eval dry-run flags from help', async () => { + const result = await execa('bun', ['--no-env-file', CLI_ENTRY, 'eval', 'run', '--help'], { + cwd: projectRoot, + env: { ...process.env, CI: 'true' }, + reject: false, + }); + const helpText = `${result.stdout}\n${result.stderr}`; + expect(helpText).not.toContain('--dry-run'); + expect(helpText).not.toContain('--dry-run-delay'); + expect(helpText).not.toContain('--dry-run-delay-min'); + expect(helpText).not.toContain('--dry-run-delay-max'); + expect(helpText).toContain('--transcript'); + expect(helpText).toContain('--record-replay'); + }, 30_000); + + it('keeps non-eval dry-run flags available', async () => { + const commands = [ + ['results', 'export', '--help'], + ['import', 'promptfoo', '--help'], + ['runs', 'rerun', '--help'], + ] as const; + + for (const args of commands) { + const result = await execa('bun', ['--no-env-file', CLI_ENTRY, ...args], { + cwd: projectRoot, + env: { ...process.env, CI: 'true' }, + reject: false, + }); + const helpText = `${result.stdout}\n${result.stderr}`; + expect(helpText).toContain('--dry-run'); + } + }, 30_000); + it('rejects the removed benchmark JSON export flag as an unknown argument', async () => { const fixture = await createFixture(); try { @@ -901,4 +934,26 @@ describe('agentv eval CLI', () => { await rm(fixture.baseDir, { recursive: true, force: true }); } }, 30_000); + + it('rejects removed eval dry-run flags as unknown arguments', async () => { + const cases = [ + ['--dry-run'], + ['--dry-run-delay', '10'], + ['--dry-run-delay-min', '5'], + ['--dry-run-delay-max', '20'], + ] as const; + + for (const args of cases) { + const fixture = await createFixture(); + try { + const result = await runCli(fixture, ['eval', fixture.testFilePath, ...args]); + expect(result.exitCode).not.toBe(0); + const output = `${result.stdout}\n${result.stderr}`; + expect(output).toContain('Unknown arguments'); + expect(output).toContain(args[0]); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + } + }, 30_000); }); diff --git a/apps/dashboard/src/components/RunEvalModal.tsx b/apps/dashboard/src/components/RunEvalModal.tsx index 7b4b2bcfd..fa5911c83 100644 --- a/apps/dashboard/src/components/RunEvalModal.tsx +++ b/apps/dashboard/src/components/RunEvalModal.tsx @@ -68,7 +68,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal const [threshold, setThreshold] = useState(''); const [thresholdEdited, setThresholdEdited] = useState(false); const [workers, setWorkers] = useState(''); - const [dryRun, setDryRun] = useState(false); const [showAdvanced, setShowAdvanced] = useState(false); // Run state @@ -114,7 +113,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setThreshold(''); setThresholdEdited(false); setWorkers(''); - setDryRun(false); setShowAdvanced(false); setActiveRunId(null); setError(null); @@ -143,10 +141,8 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal thresholdInput: threshold, studioThreshold: studioConfig?.threshold, workers, - dryRun, }); }, [ - dryRun, experiment, studioConfig?.threshold, suiteFilter, @@ -466,17 +462,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" /> -
- -
)} diff --git a/apps/dashboard/src/components/run-eval-threshold.test.ts b/apps/dashboard/src/components/run-eval-threshold.test.ts index 6f36a3cfc..414da34cd 100644 --- a/apps/dashboard/src/components/run-eval-threshold.test.ts +++ b/apps/dashboard/src/components/run-eval-threshold.test.ts @@ -32,7 +32,6 @@ describe('buildRunEvalRequest', () => { thresholdInput: '', studioThreshold: 0.75, workers: '', - dryRun: false, }), ).toEqual({ suite_filter: 'evals/**/*.eval.yaml', @@ -51,7 +50,6 @@ describe('buildRunEvalRequest', () => { thresholdInput: '0.9', studioThreshold: 0.75, workers: '', - dryRun: false, }), ).toEqual({ suite_filter: 'evals/**/*.eval.yaml', @@ -70,7 +68,6 @@ describe('buildRunEvalRequest', () => { thresholdInput: '', studioThreshold: 0.75, workers: '', - dryRun: false, }), ).toEqual({ suite_filter: 'evals/**/*.eval.yaml', diff --git a/apps/dashboard/src/components/run-eval-threshold.ts b/apps/dashboard/src/components/run-eval-threshold.ts index 5bdf44470..aa3ea60e0 100644 --- a/apps/dashboard/src/components/run-eval-threshold.ts +++ b/apps/dashboard/src/components/run-eval-threshold.ts @@ -10,7 +10,6 @@ interface BuildRunEvalRequestOptions { thresholdInput: string; studioThreshold?: number; workers: string; - dryRun: boolean; } export function getThresholdFieldValue( @@ -45,7 +44,6 @@ export function buildRunEvalRequest({ thresholdInput, studioThreshold, workers, - dryRun, }: BuildRunEvalRequestOptions): RunEvalRequest { const req: RunEvalRequest = {}; @@ -59,7 +57,6 @@ export function buildRunEvalRequest({ if (resolvedThreshold) req.threshold = Number.parseFloat(resolvedThreshold); if (workers) req.workers = Number.parseInt(workers, 10); - if (dryRun) req.dry_run = true; return req; } diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index 835504ce3..feced3b24 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -653,7 +653,6 @@ export interface RunEvalRequest { tags?: string[]; threshold?: number; workers?: number; - dry_run?: boolean; /** Resume an interrupted run: skip already-completed tests and append to `output`. */ resume?: boolean; /** Re-run failed/errored tests while keeping passing results. */ diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index f6664b336..00a580ef7 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -76,16 +76,23 @@ Run a single test by ID: agentv eval --test-id case-123 evals/my-eval.yaml ``` -### Dry Run +### Validate Without Running -Test the harness flow with mock responses (does not call real providers): +Use `agentv validate` when you want a cheap schema and config check without +executing targets or graders: ```bash -agentv eval --dry-run evals/my-eval.yaml +agentv validate evals/my-eval.yaml ``` :::note -Dry-run returns mock responses that don't match grader output schemas. Use it only for testing harness flow, not grader logic. +Eval execution no longer has a `--dry-run` mock-target mode. That mode produced +normal quality failures against fake candidate answers, which made cheap +validation look like a grader or agent result. For no-live-LLM quality +validation, run the eval against an oracle/reference target or a replayed/frozen +transcript so graders see real candidate output. Dry-run preview flags on other +commands, such as `agentv results export --dry-run` and import preview flows, +are unchanged. ::: ### Custom Output Directory @@ -414,6 +421,11 @@ Check eval files for schema errors without executing: agentv validate evals/my-eval.yaml ``` +Validation catches schema, target-reference, and grader configuration problems. +It does not produce quality scores. To validate grader quality behavior without +calling a live agent, use a reference target, imported transcript, or replay +fixture so AgentV still runs graders against real or frozen candidate output. + ## Run a Single Assertion Run a code-grader assertion in isolation without executing a full eval suite: diff --git a/apps/web/src/content/docs/docs/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/targets/cli-provider.mdx index f1eb91e65..049952fc3 100644 --- a/apps/web/src/content/docs/docs/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/targets/cli-provider.mdx @@ -135,6 +135,11 @@ A few practical notes: The pattern needs no special config field, no directory convention, and no flag — it's just a second target that happens to know the answer. +Use this pattern instead of eval mock dry-run for grader validation. Mock +execution was removed because fake candidate answers produced misleading +quality failures; a reference target gives deterministic output while exercising +the real grader path. + ## Debugging When a `cli` target misbehaves: diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index 6ff210675..42ed6dd22 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -57,7 +57,7 @@ already-exported secrets into `.env`. | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | | `cli` | Agent | Any CLI command — see [CLI Provider](/docs/targets/cli-provider) | -| `mock` | Testing | Mock provider for dry runs | +| `mock` | Testing | Explicit mock target for examples and tests | ## Referencing Targets in Evals diff --git a/apps/web/src/content/docs/docs/tools/validate.mdx b/apps/web/src/content/docs/docs/tools/validate.mdx index c1ad27644..c7a23393a 100644 --- a/apps/web/src/content/docs/docs/tools/validate.mdx +++ b/apps/web/src/content/docs/docs/tools/validate.mdx @@ -32,3 +32,9 @@ agentv validate evals/**/*.yaml - Before running evaluations to catch config errors early - In CI/CD pipelines as a pre-check - After editing eval files to verify correctness + +`agentv validate` replaces the old eval mock dry-run use case for schema and +configuration checks. It does not execute targets and does not produce quality +scores. When you need no-live-LLM quality validation, run against an +oracle/reference target or use frozen transcript/replay fixtures so graders see +real candidate output. diff --git a/docs/adr/0010-remove-eval-dry-run-mock-execution.md b/docs/adr/0010-remove-eval-dry-run-mock-execution.md new file mode 100644 index 000000000..8dbc9d584 --- /dev/null +++ b/docs/adr/0010-remove-eval-dry-run-mock-execution.md @@ -0,0 +1,85 @@ +# 10. Remove eval dry-run mock execution + +Date: 2026-06-28 + +## Status + +Accepted + +## Context + +`agentv eval --dry-run` replaced the selected eval target with a synthetic mock +target. That avoided live provider spend, but it still ran normal graders and +reported normal quality verdicts against fake candidate output. Deterministic +graders then failed for the right technical reason while presenting a confusing +product signal: users saw quality failures from an execution mode that was not a +real candidate run. + +The useful product needs are still valid: + +- cheap schema and configuration validation before CI or live runs; +- no-live-provider grader validation against known candidate output; +- deterministic replay of expensive target output; +- dry-run or preview behavior for unrelated import/export tooling. + +Those needs do not require a mock candidate hidden behind eval execution. + +## Decision + +Remove eval execution dry-run mock target behavior. `agentv eval run` does not +expose `--dry-run`, `--dry-run-delay`, `--dry-run-delay-min`, or +`--dry-run-delay-max`, and Dashboard eval launch requests do not send an eval +`dry_run` field. + +Use these existing workflows instead: + +- `agentv validate` for YAML/schema/configuration checks without executing + targets or graders. +- Oracle or reference targets, usually implemented with the `cli` provider, for + no-live-provider quality validation with known-good candidate output. +- Imported transcripts and replay fixtures for frozen candidate output where + graders should run fresh against recorded artifacts. + +Unrelated dry-run and preview flags outside eval execution remain valid, such as +results export dry-run, import preview/dry-run flows, package-manager dry-runs, +and target/provider-specific internal testing controls. + +## Consequences + +Positive: + +- Quality verdicts come from real, oracle, reference, or frozen candidate + output, not from a fake mock answer. +- Cheap validation has a clearer command: `agentv validate`. +- The eval target selection path is simpler because selected targets are always + resolved from eval or target configuration. +- Dashboard launch UX no longer offers a control that could produce misleading + quality failures. + +Negative: + +- Users who used `agentv eval --dry-run` for quick plumbing checks must switch + to `agentv validate` or define an explicit mock/reference target. +- Example docs need to avoid using dry-run as a shorthand for no-provider eval + quality checks. + +## Alternatives Considered + +- **Keep dry-run but suppress quality verdicts.** Rejected. A run that executes + graders but hides quality output would introduce another special case in the + artifact contract. +- **Make dry-run return `expected_output`.** Rejected. That would blur candidate + output with reference data and turn a plumbing flag into an implicit oracle + feature. +- **Add a dedicated oracle target feature.** Rejected for now. The `cli` provider + already composes into oracle/reference targets without new core primitives. +- **Keep only delay options for future tests.** Rejected. The delay knobs existed + for the removed mock execution path. + +## Non-Goals + +- Removing dry-run behavior from results export, import, package managers, or + other non-eval-execution tools. +- Removing provider-specific testing controls such as VS Code target + configuration dry-run behavior. +- Adding new oracle-target schema. diff --git a/examples/features/agent-skills-evals/README.md b/examples/features/agent-skills-evals/README.md index 5469aad6a..c0eb32036 100644 --- a/examples/features/agent-skills-evals/README.md +++ b/examples/features/agent-skills-evals/README.md @@ -11,7 +11,7 @@ agent-skills-evals/ ├── csv-analyzer.EVAL.yaml # Universal EVAL.yaml format ├── csv-analyzer.evals.json # Transpiled output from EVAL.yaml ├── .agentv/ -│ └── targets.yaml # Echo provider for dry-run testing +│ └── targets.yaml # Echo provider for no-live-provider testing └── evals/ └── files/ └── sales.csv # Test fixture diff --git a/examples/features/composite/README.md b/examples/features/composite/README.md index 9148fca3f..8991c07f5 100644 --- a/examples/features/composite/README.md +++ b/examples/features/composite/README.md @@ -14,8 +14,8 @@ Demonstrates composite grader patterns for combining multiple evaluation criteri ```bash # From repository root bun agentv eval run examples/features/composite/evals/dataset.eval.yaml -# Run only the strict-or script path example in dry-run (no live LLM targets required) -bun agentv eval run examples/features/composite/evals/dataset.eval.yaml --test-id strict-or-local --dry-run +# Validate the eval file without executing targets +bun agentv validate examples/features/composite/evals/dataset.eval.yaml ``` ## Key Files diff --git a/examples/features/docker-workspace/.agentv/targets.yaml b/examples/features/docker-workspace/.agentv/targets.yaml new file mode 100644 index 000000000..9fb70f658 --- /dev/null +++ b/examples/features/docker-workspace/.agentv/targets.yaml @@ -0,0 +1,3 @@ +targets: + - name: mock_agent + provider: mock diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index 4b840afbb..c43903554 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -19,7 +19,7 @@ workspace: cpus: 1 execution: - target: dry-run + target: mock_agent workers: 1 tests: diff --git a/examples/features/execution-metrics/evals/dataset.eval.yaml b/examples/features/execution-metrics/evals/dataset.eval.yaml index d31b58304..c5dad3af3 100644 --- a/examples/features/execution-metrics/evals/dataset.eval.yaml +++ b/examples/features/execution-metrics/evals/dataset.eval.yaml @@ -12,8 +12,8 @@ # Only specified thresholds are checked; omitted ones are ignored. # Score is proportional: hits / (hits + misses) # -# Run with: -# bun agentv eval examples/features/execution-metrics/evals/dataset.eval.yaml --dry-run +# Run with the included mock metrics target: +# bun agentv eval examples/features/execution-metrics/evals/dataset.eval.yaml --target mock_metrics_agent name: execution-metrics description: Demonstrates the built-in execution_metrics grader diff --git a/examples/features/latency-assertions/README.md b/examples/features/latency-assertions/README.md index de6946bd4..eb0de363e 100644 --- a/examples/features/latency-assertions/README.md +++ b/examples/features/latency-assertions/README.md @@ -50,11 +50,11 @@ For latency assertions to work, providers must include `duration_ms` in tool cal ## Running the Example ```bash -# Dry-run to validate YAML parsing -npx agentv eval examples/features/latency-assertions/evals/dataset.eval.yaml --dry-run +# Validate YAML parsing +npx agentv validate examples/features/latency-assertions/evals/dataset.eval.yaml -# With a real provider that returns duration_ms in tool calls -npx agentv eval examples/features/latency-assertions/evals/dataset.eval.yaml --target +# With the included mock provider or a real provider that returns duration_ms in tool calls +npx agentv eval examples/features/latency-assertions/evals/dataset.eval.yaml --target mock_latency_agent ``` ## Best Practices diff --git a/examples/features/latency-assertions/evals/dataset.eval.yaml b/examples/features/latency-assertions/evals/dataset.eval.yaml index 62e121339..04c04b02e 100644 --- a/examples/features/latency-assertions/evals/dataset.eval.yaml +++ b/examples/features/latency-assertions/evals/dataset.eval.yaml @@ -28,8 +28,8 @@ # }] # } # -# Note: Run with --dry-run to test parsing. For real latency testing, -# use a provider that returns duration_ms in tool calls. +# Note: use `agentv validate` to test parsing. For real latency testing, +# use the included mock latency target or a provider that returns duration_ms in tool calls. name: latency-assertions description: Latency assertions for per-step performance validation diff --git a/examples/features/prompt-template-sdk/README.md b/examples/features/prompt-template-sdk/README.md index 1796de47b..632d61218 100644 --- a/examples/features/prompt-template-sdk/README.md +++ b/examples/features/prompt-template-sdk/README.md @@ -48,7 +48,7 @@ The template receives evaluation context via stdin (JSON) and outputs the prompt ## Running ```bash -bun agentv eval examples/features/prompt-template-sdk/evals/dataset.eval.yaml --dry-run +bun agentv validate examples/features/prompt-template-sdk/evals/dataset.eval.yaml ``` ## File Structure diff --git a/examples/features/suite-level-input/.agentv/targets.yaml b/examples/features/suite-level-input/.agentv/targets.yaml new file mode 100644 index 000000000..81941ba02 --- /dev/null +++ b/examples/features/suite-level-input/.agentv/targets.yaml @@ -0,0 +1,3 @@ +targets: + - name: llm + provider: mock diff --git a/examples/features/suite-level-input/evals/dataset.eval.baseline.jsonl b/examples/features/suite-level-input/evals/dataset.eval.baseline.jsonl index 0cf7b096c..9f123113b 100644 --- a/examples/features/suite-level-input/evals/dataset.eval.baseline.jsonl +++ b/examples/features/suite-level-input/evals/dataset.eval.baseline.jsonl @@ -1,3 +1,3 @@ -{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"japan-spring","suite":"dataset.eval","score":0,"target":"default-dry-run","input":[{"role":"user","content":"\nYou are a knowledgeable travel assistant. When users ask about destinations,\nprovide practical advice about climate, best travel seasons, visa requirements,\nand local customs. Always include a safety tip.\n"},{"role":"user","content":"When is the best time to visit Japan?"}],"assertions":[]} -{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"iceland-northern-lights","suite":"dataset.eval","score":0,"target":"default-dry-run","input":[{"role":"user","content":"\nYou are a knowledgeable travel assistant. When users ask about destinations,\nprovide practical advice about climate, best travel seasons, visa requirements,\nand local customs. Always include a safety tip.\n"},{"role":"user","content":"I want to see the Northern Lights in Iceland. When should I go?"}],"assertions":[]} -{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"skip-suite-input","suite":"dataset.eval","score":0,"target":"default-dry-run","input":"What currency does Thailand use?","assertions":[]} +{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"japan-spring","suite":"dataset.eval","score":0,"target":"llm","input":[{"role":"user","content":"\nYou are a knowledgeable travel assistant. When users ask about destinations,\nprovide practical advice about climate, best travel seasons, visa requirements,\nand local customs. Always include a safety tip.\n"},{"role":"user","content":"When is the best time to visit Japan?"}],"assertions":[]} +{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"iceland-northern-lights","suite":"dataset.eval","score":0,"target":"llm","input":[{"role":"user","content":"\nYou are a knowledgeable travel assistant. When users ask about destinations,\nprovide practical advice about climate, best travel seasons, visa requirements,\nand local customs. Always include a safety tip.\n"},{"role":"user","content":"I want to see the Northern Lights in Iceland. When should I go?"}],"assertions":[]} +{"timestamp":"2026-02-24T05:08:14.524Z","test_id":"skip-suite-input","suite":"dataset.eval","score":0,"target":"llm","input":"What currency does Thailand use?","assertions":[]} diff --git a/examples/features/tool-evaluation-plugins/.agentv/targets.yaml b/examples/features/tool-evaluation-plugins/.agentv/targets.yaml new file mode 100644 index 000000000..81941ba02 --- /dev/null +++ b/examples/features/tool-evaluation-plugins/.agentv/targets.yaml @@ -0,0 +1,3 @@ +targets: + - name: llm + provider: mock diff --git a/examples/features/tool-evaluation-plugins/evals/dataset.eval.baseline.jsonl b/examples/features/tool-evaluation-plugins/evals/dataset.eval.baseline.jsonl index 957439836..f5167f298 100644 --- a/examples/features/tool-evaluation-plugins/evals/dataset.eval.baseline.jsonl +++ b/examples/features/tool-evaluation-plugins/evals/dataset.eval.baseline.jsonl @@ -1,3 +1,3 @@ -{"timestamp":"2026-02-21T04:00:53.322Z","test_id":"weather-lookup-f1","suite":"dataset.eval","score":0,"target":"default-dry-run","scores":[{"name":"tool-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":2},"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | expected=2 actual=0 TP=0 FP=0 FN=2"},{"text":"Expected tool 'fetch' was NOT called","passed":false}]}],"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"tool-f1: precision=0.000 recall=0.000 F1=0.000 | expected=2 actual=0 TP=0 FP=0 FN=2"},{"text":"Expected tool 'fetch' was NOT called","passed":false}]} -{"timestamp":"2026-02-21T04:00:53.328Z","test_id":"data-analysis-combined","suite":"dataset.eval","score":0,"target":"default-dry-run","scores":[{"name":"trajectory-check","type":"tool-trajectory","score":0,"weight":1,"verdict":"fail","assertions":[{"text":"search: called 0 times (required \u22651)","passed":false}]},{"name":"tool-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":3},"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | expected=3 actual=0 TP=0 FP=0 FN=3"},{"text":"Expected tool 'validate' was NOT called","passed":false},{"text":"Expected tool 'process' was NOT called","passed":false}]}],"assertions":[{"text":"search: called 0 times (required \u22651)","passed":false,"evidence":"tool-f1: precision=0.000 recall=0.000 F1=0.000 | expected=3 actual=0 TP=0 FP=0 FN=3"},{"text":"Expected tool 'search' was NOT called","passed":false},{"text":"Expected tool 'validate' was NOT called","passed":false},{"text":"Expected tool 'process' was NOT called","passed":false}]} -{"timestamp":"2026-02-21T04:00:53.337Z","test_id":"weather-lookup-args-f1","suite":"dataset.eval","score":0,"target":"default-dry-run","scores":[{"name":"tool-args-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":2},"assertions":[{"text":"'search' not called with args {\"query\":\"weather tokyo\"}","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | TP=0 FP=0 FN=2"},{"text":"'fetch' not called","passed":false}]}],"assertions":[{"text":"'search' not called with args {\"query\":\"weather tokyo\"}","passed":false,"evidence":"tool-args-f1: precision=0.000 recall=0.000 F1=0.000 | TP=0 FP=0 FN=2"},{"text":"'fetch' not called","passed":false}]} +{"timestamp":"2026-02-21T04:00:53.322Z","test_id":"weather-lookup-f1","suite":"dataset.eval","score":0,"target":"llm","scores":[{"name":"tool-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":2},"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | expected=2 actual=0 TP=0 FP=0 FN=2"},{"text":"Expected tool 'fetch' was NOT called","passed":false}]}],"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"tool-f1: precision=0.000 recall=0.000 F1=0.000 | expected=2 actual=0 TP=0 FP=0 FN=2"},{"text":"Expected tool 'fetch' was NOT called","passed":false}]} +{"timestamp":"2026-02-21T04:00:53.328Z","test_id":"data-analysis-combined","suite":"dataset.eval","score":0,"target":"llm","scores":[{"name":"trajectory-check","type":"tool-trajectory","score":0,"weight":1,"verdict":"fail","assertions":[{"text":"search: called 0 times (required \u22651)","passed":false}]},{"name":"tool-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":3},"assertions":[{"text":"Expected tool 'search' was NOT called","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | expected=3 actual=0 TP=0 FP=0 FN=3"},{"text":"Expected tool 'validate' was NOT called","passed":false},{"text":"Expected tool 'process' was NOT called","passed":false}]}],"assertions":[{"text":"search: called 0 times (required \u22651)","passed":false,"evidence":"tool-f1: precision=0.000 recall=0.000 F1=0.000 | expected=3 actual=0 TP=0 FP=0 FN=3"},{"text":"Expected tool 'search' was NOT called","passed":false},{"text":"Expected tool 'validate' was NOT called","passed":false},{"text":"Expected tool 'process' was NOT called","passed":false}]} +{"timestamp":"2026-02-21T04:00:53.337Z","test_id":"weather-lookup-args-f1","suite":"dataset.eval","score":0,"target":"llm","scores":[{"name":"tool-args-f1","type":"code-grader","score":0,"weight":1,"verdict":"fail","details":{"precision":0,"recall":0,"f1":0,"tp":0,"fp":0,"fn":2},"assertions":[{"text":"'search' not called with args {\"query\":\"weather tokyo\"}","passed":false,"evidence":"precision=0.000 recall=0.000 F1=0.000 | TP=0 FP=0 FN=2"},{"text":"'fetch' not called","passed":false}]}],"assertions":[{"text":"'search' not called with args {\"query\":\"weather tokyo\"}","passed":false,"evidence":"tool-args-f1: precision=0.000 recall=0.000 F1=0.000 | TP=0 FP=0 FN=2"},{"text":"'fetch' not called","passed":false}]} diff --git a/examples/features/tool-evaluation-plugins/evals/dataset.eval.yaml b/examples/features/tool-evaluation-plugins/evals/dataset.eval.yaml index 7d3396091..1b6a5856a 100644 --- a/examples/features/tool-evaluation-plugins/evals/dataset.eval.yaml +++ b/examples/features/tool-evaluation-plugins/evals/dataset.eval.yaml @@ -8,8 +8,8 @@ # cd examples/tool-evaluation-plugins # bun agentv eval evals/dataset.eval.yaml --target # -# Or dry-run (mock responses — useful for testing harness flow only): -# bun agentv eval evals/dataset.eval.yaml --dry-run +# Validate schema/config without executing a target: +# bun agentv validate evals/dataset.eval.yaml description: Tool-call F1 scoring examples diff --git a/examples/features/trace-evaluation/README.md b/examples/features/trace-evaluation/README.md index 4a054a645..ef16c422c 100644 --- a/examples/features/trace-evaluation/README.md +++ b/examples/features/trace-evaluation/README.md @@ -32,8 +32,8 @@ interface TraceSummary { ## Running ```bash -# From the repository root (dry-run mode for testing without a live agent) -bun agentv eval examples/features/trace-evaluation/evals/dataset.eval.yaml --dry-run +# From the repository root +bun agentv validate examples/features/trace-evaluation/evals/dataset.eval.yaml ``` ## Patterns diff --git a/examples/features/trace-evaluation/evals/dataset.eval.yaml b/examples/features/trace-evaluation/evals/dataset.eval.yaml index 5253abe4e..03f2ff6a2 100644 --- a/examples/features/trace-evaluation/evals/dataset.eval.yaml +++ b/examples/features/trace-evaluation/evals/dataset.eval.yaml @@ -3,8 +3,8 @@ # Demonstrates how to evaluate agent internals (LLM calls, tool executions, # errors, and durations) using code graders that inspect trace. # -# Run with: -# bun agentv eval examples/features/trace-evaluation/evals/dataset.eval.yaml --dry-run +# Validate with: +# bun agentv validate examples/features/trace-evaluation/evals/dataset.eval.yaml description: Trace-based evaluation of agent internals using code graders diff --git a/examples/features/trial-output-consistency/README.md b/examples/features/trial-output-consistency/README.md index f5f7d4292..52a262b69 100644 --- a/examples/features/trial-output-consistency/README.md +++ b/examples/features/trial-output-consistency/README.md @@ -61,11 +61,11 @@ assertions: ### Running ```bash -# Run all tests (uses token-overlap fallback for demo) -bun agentv eval examples/features/trial-output-consistency/evals/dataset.eval.yaml --dry-run +# Validate the eval file +bun agentv validate examples/features/trial-output-consistency/evals/dataset.eval.yaml -# Run a specific test -bun agentv eval examples/features/trial-output-consistency/evals/dataset.eval.yaml --test-id high-consistency --dry-run +# Run a specific test with a configured target +bun agentv eval examples/features/trial-output-consistency/evals/dataset.eval.yaml --test-id high-consistency --target ``` ## Extending diff --git a/examples/features/trial-output-consistency/evals/dataset.eval.yaml b/examples/features/trial-output-consistency/evals/dataset.eval.yaml index 1e381938f..027c4f896 100644 --- a/examples/features/trial-output-consistency/evals/dataset.eval.yaml +++ b/examples/features/trial-output-consistency/evals/dataset.eval.yaml @@ -2,8 +2,8 @@ # Measures how consistent an agent's outputs are across repeated trials # using pairwise cosine similarity (embedding-based or token-overlap fallback). # -# Run: -# bun agentv eval examples/features/trial-output-consistency/evals/dataset.eval.yaml --dry-run +# Validate: +# bun agentv validate examples/features/trial-output-consistency/evals/dataset.eval.yaml description: Trial output consistency via embedding similarity diff --git a/examples/showcase/grader-conformance/EVAL.yaml b/examples/showcase/grader-conformance/EVAL.yaml index c2827bd21..5ac6ff88b 100644 --- a/examples/showcase/grader-conformance/EVAL.yaml +++ b/examples/showcase/grader-conformance/EVAL.yaml @@ -1,8 +1,8 @@ # Grader Conformance Demo # Demonstrates using the keyword-grader grader in a standard AgentV eval. # -# Run: cd examples/showcase/grader-conformance -# npx agentv eval EVAL.yaml --dry-run +# Validate: cd examples/showcase/grader-conformance +# npx agentv validate EVAL.yaml # # The conformance harness validates this grader separately: # bun run conformance-check.ts diff --git a/packages/core/test/evaluation/graders/dry-run-mock-response.test.ts b/packages/core/test/evaluation/graders/dry-run-mock-response.test.ts deleted file mode 100644 index 2471d220b..000000000 --- a/packages/core/test/evaluation/graders/dry-run-mock-response.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Regression test: --dry-run mock response satisfies all LLM grader schemas. - * - * Before the fix, dry-run returned '{"answer":"Mock dry-run response"}' which - * caused LLM graders to fail with "Required: score" parse errors. This test - * ensures the mock response string is always schema-compatible. - */ - -import { describe, expect, it } from 'bun:test'; - -import { - freeformEvaluationSchema, - rubricEvaluationSchema, - scoreRangeEvaluationSchema, -} from '../../../src/evaluation/graders/llm-grader.js'; - -const DRY_RUN_MOCK_RESPONSE = - '{"score":1,"assertions":[],"checks":[],"overall_reasoning":"dry-run mock"}'; - -describe('dry-run mock response schema compatibility', () => { - it('is valid JSON', () => { - expect(() => JSON.parse(DRY_RUN_MOCK_RESPONSE)).not.toThrow(); - }); - - it('satisfies freeformEvaluationSchema (requires score)', () => { - const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); - const result = freeformEvaluationSchema.safeParse(parsed); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.score).toBe(1); - } - }); - - it('satisfies rubricEvaluationSchema (requires checks and overall_reasoning)', () => { - const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); - const result = rubricEvaluationSchema.safeParse(parsed); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.checks).toEqual([]); - expect(result.data.overall_reasoning).toBe('dry-run mock'); - } - }); - - it('satisfies scoreRangeEvaluationSchema (requires checks, optional overall_reasoning)', () => { - const parsed = JSON.parse(DRY_RUN_MOCK_RESPONSE); - const result = scoreRangeEvaluationSchema.safeParse(parsed); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.checks).toEqual([]); - } - }); -}); diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 482496c43..a2051b92a 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -592,7 +592,7 @@ The threshold also controls JUnit XML pass/fail: tests with scores below the thr ```bash # Run evaluation (requires API keys) -agentv eval [--test-id ] [--target ] [--dry-run] [--threshold <0-1>] +agentv eval [--test-id ] [--target ] [--threshold <0-1>] # Run with OTLP JSON file (importable by OTel backends) agentv eval --otel-file traces/eval.otlp.json