From ab12bd3189ae3f1f191dce748c95ecfb29138a92 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 26 Jun 2026 09:42:16 +0200 Subject: [PATCH 1/6] feat(evaluation): inline experiment runtime in eval files --- apps/cli/src/commands/eval/artifact-writer.ts | 4 + apps/cli/src/commands/eval/run-eval.ts | 391 +- apps/cli/src/commands/eval/statistics.ts | 11 +- apps/cli/src/commands/eval/task-bundle.ts | 3 + .../commands/eval/artifact-writer.test.ts | 61 + .../test/commands/eval/result-layout.test.ts | 4 +- apps/cli/test/eval.integration.test.ts | 77 +- apps/cli/test/fixtures/mock-run-evaluation.ts | 2 + .../docs/docs/evaluation/eval-files.mdx | 29 +- .../docs/docs/evaluation/experiments.mdx | 272 +- ...6-06-23-002-experiments-separation-plan.md | 407 - ...rate-eval-tasks-from-experiment-runtime.md | 135 - examples/features/trials/README.md | 25 +- .../features/trials/evals/dataset.eval.yaml | 11 +- .../experiments/confidence-interval.yaml | 12 - .../features/trials/experiments/default.yaml | 12 - .../features/trials/experiments/mean.yaml | 12 - .../showcase/multi-model-benchmark/README.md | 57 +- .../evals/benchmark.eval.yaml | 16 +- .../experiments/default.yaml | 18 - packages/core/scripts/generate-eval-schema.ts | 9 - packages/core/src/evaluation/config.ts | 12 - packages/core/src/evaluation/experiment.ts | 209 +- .../src/evaluation/loaders/config-loader.ts | 113 +- packages/core/src/evaluation/run-artifacts.ts | 67 +- packages/core/src/evaluation/trials.ts | 23 + packages/core/src/evaluation/types.ts | 29 +- .../evaluation/validation/eval-file.schema.ts | 166 +- .../evaluation/validation/eval-validator.ts | 534 +- .../validation/experiment-file.schema.ts | 87 - .../core/src/evaluation/validation/index.ts | 1 - packages/core/src/evaluation/yaml-parser.ts | 537 +- packages/core/src/index.ts | 2 - packages/core/test/evaluation/config.test.ts | 9 - .../evaluation/eval-inline-experiment.test.ts | 546 + .../core/test/evaluation/experiment.test.ts | 119 +- .../evaluation/loaders/config-loader.test.ts | 33 +- packages/core/test/evaluation/trials.test.ts | 21 + .../validation/eval-file-schema.test.ts | 66 + .../validation/eval-schema-sync.test.ts | 25 - .../validation/eval-validator.test.ts | 127 + .../evaluation/yaml-parser-metadata.test.ts | 7 +- skills-data/agentv-eval-writer/SKILL.md | 24 +- .../references/eval-schema.json | 24000 +++++++++------- .../references/experiment-schema.json | 278 - 45 files changed, 16136 insertions(+), 12467 deletions(-) delete mode 100644 docs/plans/2026-06-23-002-experiments-separation-plan.md delete mode 100644 docs/solutions/architecture-patterns/separate-eval-tasks-from-experiment-runtime.md delete mode 100644 examples/features/trials/experiments/confidence-interval.yaml delete mode 100644 examples/features/trials/experiments/default.yaml delete mode 100644 examples/features/trials/experiments/mean.yaml delete mode 100644 examples/showcase/multi-model-benchmark/experiments/default.yaml delete mode 100644 packages/core/src/evaluation/validation/experiment-file.schema.ts create mode 100644 packages/core/test/evaluation/eval-inline-experiment.test.ts delete mode 100644 skills-data/agentv-eval-writer/references/experiment-schema.json diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index e1591675b..4671c8e04 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -210,6 +210,7 @@ export async function writePerTestArtifacts( experiment?: string; runId?: string; duplicatePolicy?: ExportDuplicatePolicy; + resultGroup?: string; cwd?: string; repoRoot?: string; sourceTests?: readonly EvalTest[]; @@ -218,6 +219,7 @@ export async function writePerTestArtifacts( ): Promise { await writeCorePerTestArtifacts(results, outputDir, { experiment: options?.experiment, + resultGroup: options?.resultGroup, runId: options?.runId, duplicatePolicy: options?.duplicatePolicy, sourceTests: options?.sourceTests, @@ -235,6 +237,7 @@ export async function writeArtifactsFromResults( plannedTestCount?: number; runId?: string; duplicatePolicy?: ExportDuplicatePolicy; + resultGroup?: string; cwd?: string; repoRoot?: string; sourceTests?: readonly EvalTest[]; @@ -252,6 +255,7 @@ export async function writeArtifactsFromResults( plannedTestCount: options?.plannedTestCount, runId: options?.runId, duplicatePolicy: options?.duplicatePolicy, + resultGroup: options?.resultGroup, sourceTests: options?.sourceTests, additionalArtifacts: createTaskBundleArtifactsWriter(options), }); diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index ccb767b71..af6ed26d2 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -7,6 +7,7 @@ import { pathToFileURL } from 'node:url'; import { DEFAULT_THRESHOLD, + type EvalRunOverride, type EvalTargetRef, type EvalTest, type EvaluationCache, @@ -25,14 +26,10 @@ import { buildTraceFromMessages, runEvaluation as defaultRunEvaluation, deriveCategory, - deriveExperimentNameFromPath, ensureVSCodeSubagents, - isExperimentFileReference, loadConfig, - loadExperimentConfig, loadTestSuite, loadTsConfig, - resolveDefaultExperimentReference, resolveTargetDefinition, shouldEnableCache, shouldSkipCacheForTemperature, @@ -124,6 +121,7 @@ interface NormalizedOptions { readonly dryRunDelayMin: number; readonly dryRunDelayMax: number; readonly agentTimeoutSeconds?: number; + readonly cliAgentTimeoutSeconds?: number; readonly maxRetries: number; readonly cache: boolean; readonly cachePath?: string; @@ -150,6 +148,7 @@ interface NormalizedOptions { readonly model?: string; readonly outputMessages: number | 'all'; readonly threshold?: number; + readonly cliThreshold?: number; readonly tags: readonly string[]; readonly excludeTags: readonly string[]; readonly transcript?: string; @@ -160,8 +159,8 @@ interface NormalizedOptions { readonly experimentMetadata?: ExperimentArtifactMetadata; readonly experimentTargetRefs?: readonly EvalTargetRef[]; readonly experimentTrialsConfig?: TrialsConfig; - readonly suiteFiltersByEvalFile?: ReadonlyMap; readonly budgetUsd?: number; + readonly cliBudgetUsd?: number; readonly sourceMetadataByEvalFile?: ReadonlyMap>; readonly resultsOverrides?: ResultsPublishOverrides; } @@ -422,6 +421,8 @@ function normalizeOptions( } const cliAgentTimeout = normalizeOptionalNumber(rawOptions.agentTimeout); + const cliThreshold = normalizeOptionalNumber(rawOptions.threshold); + const cliBudgetUsd = normalizeOptionalNumber(rawOptions.budgetUsd); const configAgentTimeoutSeconds = config?.execution?.agentTimeoutMs != null ? config.execution.agentTimeoutMs / 1000 : undefined; @@ -479,6 +480,7 @@ function normalizeOptions( dryRunDelayMin: normalizeNumber(rawOptions.dryRunDelayMin, 0), dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0), agentTimeoutSeconds: cliAgentTimeout ?? configAgentTimeoutSeconds, + cliAgentTimeoutSeconds: cliAgentTimeout, maxRetries: cliMaxRetries ?? configMaxRetries ?? 2, cache: cliCache, cachePath: cliCachePath, @@ -523,14 +525,16 @@ function normalizeOptions( graderTarget: normalizeString(rawOptions.graderTarget), model: normalizeString(rawOptions.model), outputMessages: normalizeOutputMessages(normalizeString(rawOptions.outputMessages)), - threshold: normalizeOptionalNumber(rawOptions.threshold), + threshold: cliThreshold, + cliThreshold, tags: normalizeStringArray(rawOptions.tag), excludeTags: normalizeStringArray(rawOptions.excludeTag), transcript: normalizeString(rawOptions.transcript), recordReplay: normalizeString(rawOptions.recordReplay), recordReplayVariant: normalizeString(rawOptions.recordReplayVariant), experiment: normalizeString(rawOptions.experiment), - budgetUsd: normalizeOptionalNumber(rawOptions.budgetUsd), + budgetUsd: cliBudgetUsd, + cliBudgetUsd, sourceMetadataByEvalFile: normalizeSourceMetadataByEvalFile( rawOptions.sourceMetadataByEvalFile, ), @@ -566,69 +570,33 @@ async function ensureFileExists(filePath: string, description: string): Promise< function buildDefaultOutputPathForExperiment( cwd: string, - experiment: string | undefined, + resultGroup: string | undefined, runDirName: string, ): string { - const runDir = buildDefaultRunDirFromName(cwd, experiment, runDirName); + const runDir = buildDefaultRunDirFromName(cwd, resultGroup, runDirName); mkdirSync(runDir, { recursive: true }); return path.join(runDir, 'index.jsonl'); } -function normalizeTsDefaultExperiment( - config: Awaited> | null, -): string | undefined { +function deriveEvalResultGroupName(evalFilePath: string | undefined): string { + if (!evalFilePath) { + return 'eval'; + } return ( - normalizeString(config?.experiments?.default) ?? normalizeString(config?.defaultExperiment) + path + .basename(evalFilePath) + .replace(/\.eval\.ya?ml$/i, '') + .replace(/\.ya?ml$/i, '') + .replace(/[^A-Za-z0-9._-]/g, '-') || 'eval' ); } type ResolvedExperimentForRun = { readonly name?: string; - readonly config?: ExperimentConfig; }; -async function resolveExperimentForRun(params: { - readonly cwd: string; - readonly explicitExperiment?: string; - readonly yamlDefaultExperiment?: string; - readonly tsDefaultExperiment?: string; -}): Promise { - const experimentRef = - params.explicitExperiment ?? params.yamlDefaultExperiment ?? params.tsDefaultExperiment; - if (!experimentRef) { - return {}; - } - - const experimentPath = resolveExperimentFilePath(params.cwd, experimentRef); - if (!experimentPath) { - if (isExperimentFileReference(experimentRef)) { - throw new Error(`Experiment file not found: ${experimentRef}`); - } - return { name: experimentRef }; - } - - const config = await loadExperimentConfig(experimentPath); - return { - name: config.name ?? deriveExperimentNameFromPath(experimentPath), - config, - }; -} - -function resolveExperimentFilePath(cwd: string, experimentRef: string): string | undefined { - if (isExperimentFileReference(experimentRef)) { - const experimentPath = path.isAbsolute(experimentRef) - ? experimentRef - : path.resolve(cwd, experimentRef); - return existsSync(experimentPath) ? experimentPath : undefined; - } - - for (const ext of ['yaml', 'yml', 'ts', 'js', 'mts', 'mjs']) { - const candidate = path.resolve(cwd, 'experiments', `${experimentRef}.${ext}`); - if (existsSync(candidate)) { - return candidate; - } - } - return undefined; +function resolveExperimentForRun(explicitExperiment?: string): ResolvedExperimentForRun { + return explicitExperiment ? { name: explicitExperiment } : {}; } function applyExperimentOptions( @@ -664,6 +632,7 @@ function applyExperimentOptions( workspaceMode: workspacePath ? 'static' : workspaceMode, workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, + threshold: options.threshold ?? experiment.threshold, experimentConfig: experiment, experimentMetadata: buildExperimentArtifactMetadata(experiment), experimentTargetRefs: options.cliTargets.length === 0 ? experimentTargetRefs : undefined, @@ -715,6 +684,89 @@ function buildExperimentTrialsConfig(experiment: ExperimentConfig): TrialsConfig }; } +type EffectiveRunPolicy = { + readonly trialsConfig?: TrialsConfig; + readonly threshold?: number; + readonly timeoutSeconds?: number; + readonly budgetUsd?: number; + readonly hasScopedOverride: boolean; +}; + +function buildRunOverrideTrialsConfig(run: EvalRunOverride | undefined): TrialsConfig | undefined { + const repeat = run?.repeat; + if (!repeat || repeat.count <= 1) { + return undefined; + } + return { + count: repeat.count, + strategy: repeat.strategy, + ...(repeat.costLimitUsd !== undefined && { costLimitUsd: repeat.costLimitUsd }), + ...(repeat.earlyExit !== undefined && { earlyExit: repeat.earlyExit }), + }; +} + +function resolveEffectiveRunPolicy(params: { + readonly test: EvalTest; + readonly options: NormalizedOptions; + readonly defaultTrialsConfig?: TrialsConfig; + readonly defaultThreshold?: number; + readonly defaultTimeoutSeconds?: number; + readonly defaultBudgetUsd?: number; +}): EffectiveRunPolicy { + const { test, options, defaultTrialsConfig, defaultThreshold, defaultTimeoutSeconds } = params; + const run = test.run; + const threshold = options.cliThreshold ?? run?.threshold ?? test.threshold ?? defaultThreshold; + const timeoutSeconds = + options.cliAgentTimeoutSeconds ?? run?.timeoutSeconds ?? defaultTimeoutSeconds; + const budgetUsd = run?.budgetUsd ?? params.defaultBudgetUsd; + const trialsConfig = buildRunOverrideTrialsConfig(run) ?? defaultTrialsConfig; + return { + ...(trialsConfig !== undefined && { trialsConfig }), + ...(threshold !== undefined && { threshold }), + ...(timeoutSeconds !== undefined && { timeoutSeconds }), + ...(budgetUsd !== undefined && { budgetUsd }), + hasScopedOverride: run !== undefined || test.threshold !== undefined, + }; +} + +function runPolicyKey(policy: EffectiveRunPolicy): string { + return JSON.stringify({ + trialsConfig: policy.trialsConfig, + threshold: policy.threshold, + timeoutSeconds: policy.timeoutSeconds, + budgetUsd: policy.budgetUsd, + }); +} + +function groupTestsByRunPolicy(params: { + readonly tests: readonly EvalTest[]; + readonly options: NormalizedOptions; + readonly defaultTrialsConfig?: TrialsConfig; + readonly defaultThreshold?: number; + readonly defaultTimeoutSeconds?: number; + readonly defaultBudgetUsd?: number; +}): readonly { readonly policy: EffectiveRunPolicy; readonly tests: readonly EvalTest[] }[] { + const groups = new Map(); + for (const test of params.tests) { + const policy = resolveEffectiveRunPolicy({ + test, + options: params.options, + defaultTrialsConfig: params.defaultTrialsConfig, + defaultThreshold: params.defaultThreshold, + defaultTimeoutSeconds: params.defaultTimeoutSeconds, + defaultBudgetUsd: params.defaultBudgetUsd, + }); + const key = runPolicyKey(policy); + const existing = groups.get(key); + if (existing) { + existing.tests.push(test); + } else { + groups.set(key, { policy, tests: [test] }); + } + } + return [...groups.values()]; +} + function readExperimentWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined { return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined; } @@ -726,55 +778,12 @@ function readExperimentWorkspacePath( return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; } -type ExperimentSuiteSelection = { - readonly testFiles: readonly string[]; - readonly filtersByEvalFile: ReadonlyMap; -}; - function matchesTestFilter(id: string, filter: string | readonly string[]): boolean { return typeof filter === 'string' ? micromatch.isMatch(id, filter) : filter.some((pattern) => micromatch.isMatch(id, pattern)); } -async function resolveExperimentSuiteSelection( - suites: ExperimentConfig['suites'] | undefined, - cwd: string, -): Promise { - if (!suites || suites.length === 0) { - return undefined; - } - - const testFiles = new Set(); - const selectedTestIdsByEvalFile = new Map(); - - for (const suite of suites) { - const resolvedSuiteFiles = await resolveEvalPaths([suite.ref], cwd); - for (const testFilePath of resolvedSuiteFiles) { - const resolvedPath = path.resolve(testFilePath); - testFiles.add(resolvedPath); - if (suite.select?.testIds && suite.select.testIds.length > 0) { - const existing = selectedTestIdsByEvalFile.get(resolvedPath) ?? []; - selectedTestIdsByEvalFile.set(resolvedPath, [...existing, ...suite.select.testIds]); - } - } - } - - const filtersByEvalFile = new Map(); - for (const [testFilePath, testIds] of selectedTestIdsByEvalFile.entries()) { - const uniqueTestIds = [...new Set(testIds)]; - filtersByEvalFile.set( - testFilePath, - uniqueTestIds.length === 1 ? uniqueTestIds[0] : uniqueTestIds, - ); - } - - return { - testFiles: [...testFiles], - filtersByEvalFile, - }; -} - async function runExperimentSteps(params: { readonly label: 'setup' | 'script'; readonly steps: readonly ExperimentScript[] | undefined; @@ -849,10 +858,10 @@ function shellCommand(script: string): readonly string[] { function resolveExperimentStepCwd( cwd: string, - experimentConfig: ExperimentConfig | undefined, + _experimentConfig: ExperimentConfig | undefined, stepCwd: string | undefined, ): string { - const base = experimentConfig?.sourcePath ? path.dirname(experimentConfig.sourcePath) : cwd; + const base = cwd; if (!stepCwd) { return base; } @@ -1066,9 +1075,12 @@ async function prepareFileMetadata(params: { filter: suiteFilter ?? options.filter, category, }); + const effectiveOptions = applyExperimentOptions(options, suite.experimentConfig); const testCases = - suiteFilter && options.filter - ? suite.tests.filter((testCase) => matchesTestFilter(testCase.id, options.filter ?? '')) + suiteFilter && effectiveOptions.filter + ? suite.tests.filter((testCase) => + matchesTestFilter(testCase.id, effectiveOptions.filter ?? ''), + ) : suite.tests; const testIds = testCases.map((value) => value.id); const suiteTargets = suite.targets; @@ -1078,7 +1090,7 @@ async function prepareFileMetadata(params: { testIds, testCases, selections: [], - trialsConfig: options.experimentTrialsConfig, + trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, @@ -1093,7 +1105,7 @@ async function prepareFileMetadata(params: { let selections: { selection: TargetSelection; inlineTargetLabel: string }[]; - if (options.transcript) { + if (effectiveOptions.transcript) { // --transcript mode: bypass target resolution entirely. // Create a synthetic TargetSelection for the transcript provider. const transcriptSelection: TargetSelection = { @@ -1105,15 +1117,15 @@ async function prepareFileMetadata(params: { }, targetName: 'transcript', targetSource: 'cli', - targetsFilePath: options.transcript, + targetsFilePath: effectiveOptions.transcript, }; selections = [ { selection: transcriptSelection, - inlineTargetLabel: `transcript (${path.basename(options.transcript)})`, + inlineTargetLabel: `transcript (${path.basename(effectiveOptions.transcript)})`, }, ]; - } else if (suite.inlineTarget && options.cliTargets.length === 0) { + } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) { const targetDefinition = suite.inlineTarget; const resolvedTarget = options.dryRun ? ({ @@ -1144,7 +1156,7 @@ async function prepareFileMetadata(params: { inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedTarget.name), }, ]; - } else if (suite.providerFactory && options.cliTargets.length === 0) { + } else if (suite.providerFactory && effectiveOptions.cliTargets.length === 0) { const taskTarget: ResolvedTarget = { kind: 'mock', name: 'custom-task', @@ -1165,10 +1177,10 @@ async function prepareFileMetadata(params: { ]; } else { // Determine target names: CLI --target flags override YAML - const cliTargets = options.cliTargets; + const cliTargets = effectiveOptions.cliTargets; const suiteTargets = suite.targets; const suiteTargetRefs = suite.targetRefs; - const experimentTargetRefs = options.experimentTargetRefs; + const experimentTargetRefs = effectiveOptions.experimentTargetRefs; // Resolve which target names to use (precedence: CLI/experiment > suite YAML targets > default) let targetNames: readonly string[]; @@ -1190,11 +1202,11 @@ async function prepareFileMetadata(params: { testFilePath, repoRoot, cwd, - explicitTargetsPath: options.targetsPath, - dryRun: options.dryRun, - dryRunDelay: options.dryRunDelay, - dryRunDelayMin: options.dryRunDelayMin, - dryRunDelayMax: options.dryRunDelayMax, + explicitTargetsPath: effectiveOptions.targetsPath, + dryRun: effectiveOptions.dryRun, + dryRunDelay: effectiveOptions.dryRunDelay, + dryRunDelayMin: effectiveOptions.dryRunDelayMin, + dryRunDelayMax: effectiveOptions.dryRunDelayMax, env: process.env, targetNames, targetRefs, @@ -1210,12 +1222,12 @@ async function prepareFileMetadata(params: { testFilePath, repoRoot, cwd, - explicitTargetsPath: options.targetsPath, - cliTargetName: targetNames.length === 1 ? targetNames[0] : options.target, - dryRun: options.dryRun, - dryRunDelay: options.dryRunDelay, - dryRunDelayMin: options.dryRunDelayMin, - dryRunDelayMax: options.dryRunDelayMax, + explicitTargetsPath: effectiveOptions.targetsPath, + cliTargetName: targetNames.length === 1 ? targetNames[0] : effectiveOptions.target, + dryRun: effectiveOptions.dryRun, + dryRunDelay: effectiveOptions.dryRunDelay, + dryRunDelayMin: effectiveOptions.dryRunDelayMin, + dryRunDelayMax: effectiveOptions.dryRunDelayMax, env: process.env, }); @@ -1241,7 +1253,7 @@ async function prepareFileMetadata(params: { testIds, testCases, selections, - trialsConfig: options.experimentTrialsConfig, + trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, @@ -1293,6 +1305,7 @@ async function runSingleEvalFile(params: { readonly inlineTargetLabel: string; readonly testCases: readonly EvalTest[]; readonly trialsConfig?: TrialsConfig; + readonly agentTimeoutSeconds?: number; readonly matrixMode?: boolean; readonly budgetUsd?: number; readonly runBudgetTracker?: RunBudgetTracker; @@ -1320,6 +1333,7 @@ async function runSingleEvalFile(params: { inlineTargetLabel, testCases, trialsConfig, + agentTimeoutSeconds, matrixMode, budgetUsd, runBudgetTracker, @@ -1361,9 +1375,7 @@ async function runSingleEvalFile(params: { } const agentTimeoutMs = - options.agentTimeoutSeconds != null - ? Math.max(0, options.agentTimeoutSeconds) * 1000 - : undefined; + agentTimeoutSeconds != null ? Math.max(0, agentTimeoutSeconds) * 1000 : undefined; // Resolve workers: CLI flag > eval YAML execution.workers > target setting > default const workerPreference = workersOverride ?? options.workers; @@ -1440,7 +1452,7 @@ async function runSingleEvalFile(params: { failOnError, graderTarget: options.graderTarget, model: options.model, - threshold: options.threshold, + threshold: params.threshold, targetHooks: resolvedTargetSelection.targetHooks, replayRecording, providerFactory, @@ -1562,38 +1574,31 @@ export async function runEvalCommand( } let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); - const resolvedExperiment = await resolveExperimentForRun({ - cwd, - explicitExperiment: options.experiment, - yamlDefaultExperiment: resolveDefaultExperimentReference(yamlConfig), - tsDefaultExperiment: normalizeTsDefaultExperiment(config), - }); - options = { - ...applyExperimentOptions(options, resolvedExperiment.config), - experiment: resolvedExperiment.name, - }; - - const suiteSelection = await resolveExperimentSuiteSelection( - options.experimentConfig?.suites, - cwd, - ); - const evalPathInputs = - input.testFiles.length > 0 - ? [...input.testFiles] - : suiteSelection - ? [...suiteSelection.testFiles] - : []; + const resolvedExperiment = resolveExperimentForRun(options.experiment); + const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : []; if (evalPathInputs.length === 0 && process.stdin.isTTY) { const { launchInteractiveWizard } = await import('./interactive.js'); await launchInteractiveWizard(); return undefined; } const resolvedTestFiles = await resolveEvalPaths(evalPathInputs, cwd); + const fallbackResultGroupName = + resolvedTestFiles.length === 1 ? deriveEvalResultGroupName(resolvedTestFiles[0]) : 'multi-eval'; + const primarySuite = + resolvedTestFiles.length > 0 + ? await loadTestSuite(resolvedTestFiles[0], repoRoot, { + verbose: options.verbose, + filter: options.filter, + category: deriveCategory(path.relative(cwd, resolvedTestFiles[0])), + }) + : undefined; + const resultGroupName = + resolvedTestFiles.length === 1 + ? (primarySuite?.metadata?.name ?? fallbackResultGroupName) + : fallbackResultGroupName; options = { - ...options, - ...(suiteSelection !== undefined && { - suiteFiltersByEvalFile: suiteSelection.filtersByEvalFile, - }), + ...applyExperimentOptions(options, primarySuite?.experimentConfig), + experiment: resolvedExperiment.name ?? resultGroupName, }; if (!process.env.AGENTV_EXPERIMENT) { @@ -1732,8 +1737,8 @@ export async function runEvalCommand( mkdirSync(runDir, { recursive: true }); outputPath = path.join(runDir, 'index.jsonl'); } else { - // Default: .agentv/results///, using "default" when unspecified. - outputPath = buildDefaultOutputPathForExperiment(cwd, options.experiment, runDirName); + // Default: .agentv/results///. + outputPath = buildDefaultOutputPathForExperiment(cwd, resultGroupName, runDirName); runDir = path.dirname(outputPath); } if (!process.env.AGENTV_RUN_TIMESTAMP) { @@ -1867,7 +1872,7 @@ export async function runEvalCommand( repoRoot, cwd, options, - suiteFilter: options.suiteFiltersByEvalFile?.get(path.resolve(testFilePath)), + suiteFilter: undefined, }); fileMetadata.set(testFilePath, meta); } @@ -2092,6 +2097,7 @@ export async function runEvalCommand( // Eval files run sequentially; within each file, --workers N test cases run in parallel. // This matches industry practice (promptfoo, deepeval, OpenAI Evals) and avoids cross-file // workspace races without any grouping complexity. + let hasScopedRunPolicies = false; try { for (const testFilePath of activeTestFiles) { // Run-level budget check: skip remaining files if budget exceeded @@ -2166,45 +2172,59 @@ export async function runEvalCommand( } try { - const result = await runSingleEvalFile({ - testFilePath, - cwd, - repoRoot, + const runGroups = groupTestsByRunPolicy({ + tests: filteredTestCases, options, - outputWriter, - otelExporter, - cache, - evaluationRunner, - workersOverride: perFileWorkers, - yamlWorkers: targetPrep.yamlWorkers, - progressReporter, - seenTestCases, - displayIdTracker, - selection, - inlineTargetLabel, - testCases: filteredTestCases, - trialsConfig: options.transcript ? undefined : targetPrep.trialsConfig, - matrixMode: targetPrep.selections.length > 1, - budgetUsd: targetPrep.budgetUsd, - runBudgetTracker, - failOnError: targetPrep.failOnError, - threshold: resolvedThreshold, - providerFactory: transcriptProviderFactory ?? targetPrep.providerFactory, + defaultTrialsConfig: options.transcript ? undefined : targetPrep.trialsConfig, + defaultThreshold: resolvedThreshold, + defaultTimeoutSeconds: options.agentTimeoutSeconds, + defaultBudgetUsd: targetPrep.budgetUsd, }); + const groupResults: EvaluationResult[] = []; + for (const group of runGroups) { + hasScopedRunPolicies ||= group.policy.hasScopedOverride; + const result = await runSingleEvalFile({ + testFilePath, + cwd, + repoRoot, + options, + outputWriter, + otelExporter, + cache, + evaluationRunner, + workersOverride: perFileWorkers, + yamlWorkers: targetPrep.yamlWorkers, + progressReporter, + seenTestCases, + displayIdTracker, + selection, + inlineTargetLabel, + testCases: group.tests, + trialsConfig: options.transcript ? undefined : group.policy.trialsConfig, + agentTimeoutSeconds: group.policy.timeoutSeconds, + matrixMode: targetPrep.selections.length > 1, + budgetUsd: group.policy.budgetUsd, + runBudgetTracker, + failOnError: targetPrep.failOnError, + threshold: group.policy.threshold, + providerFactory: transcriptProviderFactory ?? targetPrep.providerFactory, + }); + groupResults.push(...result.results); + } const evalFile = path.relative(cwd, testFilePath); const existingSummary = remoteEvalSummaries.find( (summary) => summary.evalFile === evalFile, ); if (existingSummary) { - existingSummary.results.push(...result.results); + existingSummary.results.push(...groupResults); } else { remoteEvalSummaries.push({ evalFile, - results: [...result.results], + results: [...groupResults], }); } - return result.results; + return groupResults; } catch (fileError) { // before_all or other setup failures should not abort the entire run. // Mark all tests in this file as errors and continue with other files. @@ -2277,8 +2297,11 @@ export async function runEvalCommand( summaryResults = deduplicateByTestIdTarget(parseJsonlResults(content)); } - const thresholdOpts = - resolvedThreshold !== undefined ? { threshold: resolvedThreshold } : undefined; + const thresholdOpts = hasScopedRunPolicies + ? { thresholdLabel: 'configured threshold(s)', useExecutionStatus: true } + : resolvedThreshold !== undefined + ? { threshold: resolvedThreshold } + : undefined; const summary = calculateEvaluationSummary(summaryResults, thresholdOpts); console.log(formatEvaluationSummary(summary, thresholdOpts)); if ( @@ -2312,6 +2335,7 @@ export async function runEvalCommand( const { writePerTestArtifacts } = await import('./artifact-writer.js'); await writePerTestArtifacts(allResults, runDir, { experiment: normalizeExperimentName(options.experiment), + resultGroup: resultGroupName, cwd, repoRoot, sourceTests, @@ -2335,6 +2359,7 @@ export async function runEvalCommand( evalFile, experiment: normalizeExperimentName(options.experiment), experimentMetadata: options.experimentMetadata, + resultGroup: resultGroupName, cwd, repoRoot, sourceTests, diff --git a/apps/cli/src/commands/eval/statistics.ts b/apps/cli/src/commands/eval/statistics.ts index 54d6d373c..13d64f508 100644 --- a/apps/cli/src/commands/eval/statistics.ts +++ b/apps/cli/src/commands/eval/statistics.ts @@ -84,7 +84,7 @@ function buildHistogram(values: readonly number[]): readonly HistogramBin[] { export function calculateEvaluationSummary( results: readonly EvaluationResult[], - options?: { threshold?: number }, + options?: { threshold?: number; thresholdLabel?: string; useExecutionStatus?: boolean }, ): EvaluationSummary { const total = results.length; @@ -139,11 +139,11 @@ export function calculateEvaluationSummary( const executionErrorCount = executionErrors.length; const scoreThreshold = options?.threshold; const passedCount = - scoreThreshold !== undefined + scoreThreshold !== undefined && options?.useExecutionStatus !== true ? qualityResults.filter((r) => r.score >= scoreThreshold).length : results.filter((r) => r.executionStatus === 'ok').length; const qualityFailureCount = - scoreThreshold !== undefined + scoreThreshold !== undefined && options?.useExecutionStatus !== true ? qualityResults.filter((r) => r.score < scoreThreshold).length : results.filter((r) => r.executionStatus === 'quality_failure').length; @@ -186,7 +186,7 @@ function formatScore(value: number): string { export function formatEvaluationSummary( summary: EvaluationSummary, - options?: { threshold?: number }, + options?: { threshold?: number; thresholdLabel?: string; useExecutionStatus?: boolean }, ): string { if (summary.total === 0) { return '\nNo results to summarize'; @@ -209,6 +209,7 @@ export function formatEvaluationSummary( // Overall verdict: all non-error cases must score >= per-test threshold. const gradedCount = summary.total - summary.executionErrorCount; const threshold = options?.threshold ?? 0.8; + const thresholdText = options?.thresholdLabel ?? `${Math.round(threshold * 100)}%`; const allExecutionErrors = summary.total > 0 && summary.executionErrorCount === summary.total; const overallPassed = !allExecutionErrors && @@ -226,7 +227,7 @@ export function formatEvaluationSummary( } else { overallVerdict = overallPassed ? 'PASS' : 'FAIL'; verdictColor = overallPassed ? '\x1b[32m' : '\x1b[31m'; - verdictText = `RESULT: ${overallVerdict} (${summary.passedCount}/${summary.total} scored >= ${Math.round(threshold * 100)}%, mean: ${formatScore(summary.mean)})`; + verdictText = `RESULT: ${overallVerdict} (${summary.passedCount}/${summary.total} scored >= ${thresholdText}, mean: ${formatScore(summary.mean)})`; } lines.push('\n=================================================='); diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index a716eeda7..ae81fb721 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -736,6 +736,9 @@ function buildPortableEvalCase( if (test.metadata && Object.keys(test.metadata).length > 0) { testCase.metadata = rewritePathsDeep(test.metadata, rewrites); } + if (test.run && Object.keys(test.run).length > 0) { + testCase.run = rewritePathsDeep(test.run, rewrites); + } if (test.conversation_id) { testCase.conversation_id = test.conversation_id; } diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index bf30802d4..61310efae 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -190,6 +190,23 @@ describe('buildGradingArtifact', () => { passed_attempts: 1, total_attempts: 2, }); + + const passAll = buildGradingArtifact( + makeResult({ + aggregation: { + strategy: 'pass_all', + passedAttempts: 1, + totalAttempts: 2, + min: 0.4, + }, + }), + ); + expect(passAll.aggregation).toEqual({ + strategy: 'pass_all', + passed_attempts: 1, + total_attempts: 2, + min: 0.4, + }); }); it('uses top-level assertions when no grader scores', () => { @@ -1719,6 +1736,50 @@ describe('writeArtifactsFromResults', () => { expect(indexLine.grading_path).toBe('eval-top-months-chart/shared-id/run-1/grading.json'); }); + it('does not prefix artifact paths with suite when it matches the result group', async () => { + const paths = await writeArtifactsFromResults( + [makeResult({ suite: 'eval-top-months-chart', testId: 'shared-id', target: 'baseline' })], + testDir, + { resultGroup: 'eval-top-months-chart' }, + ); + + const [indexLine] = (await readFile(paths.indexPath, 'utf8')) + .trim() + .split('\n') + .map(JSON.parse); + expect(indexLine.suite).toBe('eval-top-months-chart'); + expect(indexLine.grading_path).toBe('shared-id/run-1/grading.json'); + }); + + it('prefixes imported suite artifacts even when the suite matches the result group', async () => { + const sourceTests = [ + { + id: 'shared-id', + suite: 'eval-top-months-chart', + source: { + evalFilePath: 'evals/imported.eval.yaml', + evalFileAbsolutePath: path.join(testDir, 'evals/imported.eval.yaml'), + importedSuiteName: 'eval-top-months-chart', + testId: 'shared-id', + testSnapshotYaml: 'id: shared-id', + graderDefinitions: [], + references: [], + }, + } as EvalTest, + ]; + const paths = await writeArtifactsFromResults( + [makeResult({ suite: 'eval-top-months-chart', testId: 'shared-id', target: 'baseline' })], + testDir, + { resultGroup: 'eval-top-months-chart', sourceTests }, + ); + + const [indexLine] = (await readFile(paths.indexPath, 'utf8')) + .trim() + .split('\n') + .map(JSON.parse); + expect(indexLine.grading_path).toBe('eval-top-months-chart/shared-id/run-1/grading.json'); + }); + it('writes task bundle artifacts with local source paths when source metadata is provided', async () => { const sourceRoot = path.join(testDir, 'src'); await mkdir(sourceRoot, { recursive: true }); diff --git a/apps/cli/test/commands/eval/result-layout.test.ts b/apps/cli/test/commands/eval/result-layout.test.ts index 97424c4c1..79dfd805d 100644 --- a/apps/cli/test/commands/eval/result-layout.test.ts +++ b/apps/cli/test/commands/eval/result-layout.test.ts @@ -9,7 +9,7 @@ import { } from '../../../src/commands/eval/result-layout.js'; describe('result layout', () => { - it('groups default run directories under the default experiment', () => { + it('groups default run directories under the default result group', () => { const cwd = '/repo'; const timestamp = new Date('2026-06-22T12:34:56.789Z'); @@ -18,7 +18,7 @@ describe('result layout', () => { ); }); - it('groups named experiment run directories under the experiment', () => { + it('groups named run directories under the result group', () => { expect(buildDefaultRunDirFromName('/repo', 'with-skills', '2026-run')).toBe( path.join('/repo', '.agentv', 'results', 'with-skills', '2026-run'), ); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index ce53172ff..629d12763 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -519,11 +519,9 @@ describe('agentv eval CLI', () => { } }, 30_000); - it('runs a native experiment file with suite test selection and run knobs', async () => { + it('runs inline experiment config with suite test selection and run knobs', async () => { const fixture = await createFixture(); try { - const experimentsDir = path.join(fixture.suiteDir, 'experiments'); - await mkdir(experimentsDir, { recursive: true }); await writeFile( path.join(fixture.suiteDir, '.agentv', 'config.yaml'), 'eval_patterns:\n - sample.test.yaml\n - unused.test.yaml\n', @@ -543,38 +541,44 @@ describe('agentv eval CLI', () => { ].join('\n'), 'utf8', ); - const experimentPath = path.join(experimentsDir, 'default.yaml'); + const wrapperPath = path.join(fixture.suiteDir, 'native-exp.eval.yaml'); await writeFile( - experimentPath, + wrapperPath, [ 'name: native-exp', - 'target: cli-target', - 'suites:', - ' - ref: sample.test.yaml', - ' select:', - ' test_ids:', - ' - case-alpha', - 'timeout_seconds: 12', - 'workers: 4', - 'repeat:', - ' count: 2', - ' strategy: mean', - ' cost_limit_usd: 1.25', - 'early_exit: false', - 'setup:', - ' - script: "printf setup > ../experiment-setup.txt"', - 'scripts:', - ' - script: "printf script > ../experiment-script.txt"', + 'experiment:', + ' name: native-exp', + ' target: cli-target', + ' timeout_seconds: 12', + ' workers: 4', + ' threshold: 0.8', + ' budget_usd: 3', + ' repeat:', + ' count: 2', + ' strategy: mean', + ' cost_limit_usd: 1.25', + ' early_exit: false', + ' setup:', + ' - script: "printf setup > experiment-setup.txt"', + ' scripts:', + ' - script: "printf script > experiment-script.txt"', + 'tests:', + ' - include: sample.test.yaml', + ' type: suite', + ' select: case-alpha', + ' run:', + ' threshold: 1.0', + ' timeout_seconds: 5', + ' budget_usd: 0.75', + ' repeat:', + ' count: 3', + ' strategy: pass_all', '', ].join('\n'), 'utf8', ); - const { stdout, exitCode } = await runCli(fixture, [ - 'eval', - '--experiment', - 'experiments/default.yaml', - ]); + const { stdout, exitCode } = await runCli(fixture, ['eval', wrapperPath]); expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); @@ -583,14 +587,14 @@ describe('agentv eval CLI', () => { const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ target: 'cli-target', - agentTimeoutMs: 12000, + agentTimeoutMs: 5000, maxConcurrency: 4, evalCaseIds: ['case-alpha'], + budgetUsd: 0.75, + threshold: 1, trials: { - count: 2, - strategy: 'mean', - costLimitUsd: 1.25, - earlyExit: false, + count: 3, + strategy: 'pass_all', }, }); @@ -603,16 +607,7 @@ describe('agentv eval CLI', () => { expect(benchmark.metadata?.experiment).toBe('native-exp'); expect(benchmark.metadata?.experiment_config).toMatchObject({ name: 'native-exp', - source_path: experimentPath, target: 'cli-target', - suites: [ - { - ref: 'sample.test.yaml', - select: { - test_ids: ['case-alpha'], - }, - }, - ], repeat: { count: 2, strategy: 'mean', diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index 32162888e..f8f06e34a 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -25,6 +25,7 @@ interface RunEvaluationOptionsLike { readonly costLimitUsd?: number; readonly earlyExit?: boolean; }; + readonly threshold?: number; readonly budgetUsd?: number; readonly runBudgetTracker?: { readonly budgetCapUsd?: number; @@ -181,6 +182,7 @@ async function maybeWriteDiagnostics( budgetUsd: options.budgetUsd ?? null, maxConcurrency: options.maxConcurrency ?? null, trials: options.trials ?? null, + threshold: options.threshold ?? null, hasRunBudgetTracker: options.runBudgetTracker !== undefined, runBudgetCapUsd: options.runBudgetTracker?.budgetCapUsd ?? null, replayRecording: options.replayRecording ?? null, diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx index e4d932da0..60b96f164 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx @@ -5,21 +5,21 @@ sidebar: order: 1 --- -Evaluation files define the test cases and graders for an evaluation run. Runtime choices such as target matrices, setup, scripts, and repeat runs belong in [experiments](/docs/evaluation/experiments/). AgentV supports two eval formats: YAML and JSONL. +Evaluation files define the test cases, graders, and inline runtime block for an evaluation run. Runtime choices such as target matrices, setup, scripts, and repeat runs belong under top-level [`experiment:`](/docs/evaluation/experiments/). AgentV supports two eval formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. ## Suites -An eval file is a **suite**: it binds test cases to task context, assertions, and reusable fixtures. Runtime choices such as target matrices, setup, and run counts belong in experiments. Test cases can be inline or loaded from an external file via `tests: ./cases.yaml` for reuse across suites. +An eval file is a **suite**: it binds test cases to task context, assertions, reusable fixtures, and the inline runtime block. Test cases can be inline, loaded from an external file via `tests: ./cases.yaml`, or imported with `tests[].include`. ## YAML Format -The primary format. A single file contains metadata, execution config, and tests: +The primary format. A single file contains metadata, inline runtime config, and tests: ```yaml description: Math problem solving evaluation -execution: +experiment: target: default assertions: @@ -40,9 +40,9 @@ tests: |-------|-------------| | `description` | Human-readable description of the evaluation | | `suite` | Optional suite identifier | -| `execution` | Default execution config (`target`, `fail_on_error`, `threshold`, etc.) | +| `experiment` | Runtime config (`target`, `targets`, `workers`, `repeat`, `setup`, `scripts`, `threshold`, etc.) | | `workspace` | Suite-level workspace config — inline object or string path to an [external workspace file](/docs/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | -| `tests` | Array of individual tests, or a string path to an external file or directory | +| `tests` | Array of individual tests, include entries, or a string path to an external file or directory. Tests and include entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. | | `assertions` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test | | `input` | Suite-level input messages prepended to each test's input unless `execution.skip_defaults: true` is set on the test | @@ -222,12 +222,25 @@ Instead of inlining tests in the same file, you can point `tests` to an external ```yaml name: my-eval description: My evaluation suite -execution: +experiment: target: default tests: ./cases.yaml ``` -The path is resolved relative to the eval file's directory. The external file should contain a YAML array of test objects or a JSONL file with one test per line. +The path is resolved relative to the eval file's directory. The external file +should contain a YAML array of test objects or a JSONL file with one test per +line. String entries inside a `tests:` list work the same way and may use direct +paths, directories, or globs: + +```yaml +tests: + - ./cases/*.cases.yaml + - include: ./suites/*.eval.yaml + type: suite +``` + +String shorthand is raw-case-only. Import eval suites with object entries using +`include:` and `type: suite`. ### Tests as Directory Path diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 23a9f5cd2..2d034045a 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -5,175 +5,185 @@ sidebar: order: 2 --- -Experiments define **how** eval cases run: target or target matrix, setup, -scripts, timeout, sandbox, case filters, and repeat-run policy. Eval files stay -focused on **what** is tested: prompts, datasets, assertions, and task fixtures. - -## Experiment YAML - -Committed experiments conventionally live under `experiments/`: +AgentV eval files are the only runnable authoring artifact. Use top-level +`experiment:` inside `eval.yaml` for runtime choices: targets, workers, setup, +scripts, timeout, sandbox/workspace runtime knobs, budgets, thresholds, and +repeat-run policy. ```yaml -name: baseline -target: codex-gpt5 -suites: - - ref: evals/support-regression.eval.yaml - select: - test_ids: - - refund-eligibility - - missing-order-date -timeout_seconds: 720 -repeat: - count: 4 - strategy: pass_at_k - cost_limit_usd: 2.00 -setup: - - script: bun install -scripts: - - build +name: support-regression + +experiment: + targets: [codex-gpt5, claude-sonnet] + workers: 2 + timeout_seconds: 720 + repeat: + count: 4 + strategy: pass_at_k + cost_limit_usd: 2.00 + setup: + - script: bun install + scripts: + - build + +tests: + - id: refund-eligibility + input: Can this customer get a refund? + criteria: Applies the refund policy correctly ``` -Wire fields use `snake_case`. AgentV translates to internal `camelCase` when it -loads the file. +`execution:` is accepted only as a legacy top-level alias for existing eval +files. Do not use both `experiment:` and `execution:` in the same eval. -## Suites and test selection +## Tests Imports -Eval files keep `tests[]` as the canonical atomic test definition. Experiments -reference one or more reusable eval suites through `suites[]`: +Use `tests[]` for composition, imports, and selection. ```yaml -suites: - - ref: evals/support-regression.eval.yaml - - ref: evals/billing-*.eval.yaml -``` - -Use suite-local `select.test_ids[]` to run only specific tests from a suite. The -values match `tests[].id` inside that suite and use the same glob semantics as -`--test-id`: - -```yaml -suites: - - ref: evals/support-regression.eval.yaml +tests: + - include: evals/support/*.eval.yaml + type: suite select: test_ids: - refund-* - missing-order-date + tags: regression + metadata: + priority: high + run: + threshold: 1.0 + repeat: + count: 2 + strategy: pass_all + - include: cases/*.cases.yaml + type: tests + - include: cases/regression.jsonl + type: tests + - cases/smoke/*.cases.yaml ``` -## Repeat runs +`type: suite` preserves the imported suite's task contract: metadata, +`workspace`, shared `input`, shared `assertions`, and tests. The child suite's +`experiment:` or legacy `execution:` runtime block is ignored; the parent eval's +runtime block controls the run. -`repeat` is the full AgentV replacement for the old eval-level -`execution.trials` shape. It supports the same core strategies: +`type: tests` imports only raw test entries. It intentionally drops shared +context from an imported eval suite, so parent suite fields apply to those raw +cases. -```yaml -repeat: - count: 3 - strategy: mean - cost_limit_usd: 1.50 -``` +`tests[].select.test_ids` filters imported test IDs with glob patterns. +`tests[].select.tags` filters each imported case's effective `metadata.tags`. +Effective case tags are suite-first and deduped: +`suite.tags + suite.metadata.tags + test.metadata.tags`. Top-level suite `tags` +still remain suite identity metadata for discovery and reporting; selection reads +the merged case metadata view. `tests[].select.metadata` filters case metadata by +key/value, where selector values may be scalars or lists. Globbed include paths +are resolved in deterministic path order, then test order. -Supported strategies: +String-valued `tests` and string entries inside `tests[]` are raw-case import +shorthand. They are equivalent to `include` with `type: tests` and may point at +raw case files, directories, or globs. Importing another eval suite must use +object form with `include:` and `type: suite`. -| Strategy | Behavior | -| --- | --- | -| `pass_at_k` | Uses the best passing attempt; early-exits by default unless the experiment sets `early_exit: false` | -| `mean` | Aggregates repeated attempt scores by mean | -| `confidence_interval` | Uses the lower bound of a 95% confidence interval as the conservative score | +Suite imports are resolved as a deterministic include graph. Circular `type: +suite` imports fail validation with the import chain; raw-case shorthand does +not recursively load suite runtime blocks. -`repeat.cost_limit_usd` caps repeat-run spend. `repeat.costLimitUsd` is also -accepted for prerelease trial-schema parity, but new YAML should use -`cost_limit_usd`. +Imported suite artifacts are nested under the source suite name inside a wrapper +eval result directory, for example +`.agentv/results/////...`. +Direct tests owned by the wrapper eval and raw case imports live directly under +`/...`. -## Vercel-compatible shorthand +## Scoped Run Overrides -AgentV also accepts Vercel-style top-level `runs` and `early_exit`: +Use scoped `run:` blocks for result interpretation and scheduling policies that +vary by include group or test case. Precedence is: -```yaml -runs: 4 -early_exit: true +```text +test.run > tests[].run > experiment ``` -This is shorthand for a `pass_at_k` repeat run. Use `repeat` when you need -AgentV-specific strategy or cost-limit fields. - -Do not set both `repeat` and `runs` in the same experiment. `repeat` is the -canonical AgentV shape; `runs` exists only for Vercel-compatible shorthand. - -Vercel defines the requested run count at the experiment level. Some result -summaries show fewer actual runs for a case because `earlyExit: true` stops -remaining attempts after the first pass; smoke runs can also force one run. -AgentV follows the same experiment-level placement while keeping the richer -`repeat` block for AgentV strategies. - -Repeat-enabled cases use a Vercel-style physical layout with AgentV aggregate -provenance: - -```text -/index.jsonl -/summary.json -///summary.json -///run-1/result.json -///run-1/grading.json -///run-1/metrics.json -///run-1/timing.json -///run-1/transcript.json -///run-1/transcript-raw.jsonl -///run-1/outputs/answer.md +```yaml +experiment: + target: agent + threshold: 0.8 + repeat: + count: 3 + strategy: pass_at_k + +tests: + - include: ./evals/flaky-agentic/**/*.eval.yaml + type: suite + select: + tags: [agentic] + run: + repeat: + count: 3 + strategy: pass_at_k + + - include: ./evals/regression/**/*.eval.yaml + type: suite + select: + tags: [must-pass] + run: + threshold: 1.0 + repeat: + count: 2 + strategy: pass_all + + - id: critical-case + input: "..." + criteria: Must pass exactly + run: + threshold: 1.0 + repeat: + count: 1 ``` -The repeated case aggregate folder uses `summary.json` for run-count, pass-rate, -fingerprint, and flattened snake_case timing fields such as -`mean_duration_ms`. -Each `run-N/result.json` is the per-attempt manifest and includes -`grading_path`, transcript/output paths, and embedded timing/o11y metrics. Each -attempt also keeps AgentV `grading.json`, `metrics.json`, and `timing.json` -sidecars for detailed inspection. -Root `index.jsonl` and root `summary.json` remain stable for existing CI -summary scripts and uploaded artifact consumers. +Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and +`budget_usd`. Candidate-changing fields such as `target`, `targets`, setup +scripts, and workspace mutation stay parent-level under `experiment:` for now. -## Targets and setup +## Repeat Runs -Experiments reuse targets from `.agentv/targets.yaml`; they do not define a new -provider registry. +`repeat` supports the same core strategies as repeated attempts: ```yaml -targets: - - copilot - - claude - - name: gemini-with-hooks - use_target: gemini +experiment: + repeat: + count: 3 + strategy: mean + cost_limit_usd: 1.50 ``` -Setup and scripts belong on the experiment because they are often the A/B -variable: - -```yaml -setup: - - script: cp skills/with-docs/AGENTS.md AGENTS.md -scripts: - - script: bun test - timeout_seconds: 120 -``` +Supported strategies: -## Running experiments +| Strategy | Behavior | +| --- | --- | +| `pass_at_k` | Uses the best passing attempt; early-exits by default unless `early_exit: false` is set | +| `pass_all` | Uses the weakest attempt score, so every repeated attempt must meet the threshold | +| `mean` | Aggregates repeated attempt scores by mean | +| `confidence_interval` | Uses the lower bound of a 95% confidence interval as the conservative score | -Run a specific experiment: +AgentV also accepts Vercel-style `runs` and `early_exit` under `experiment:`: -```bash -bun agentv eval --experiment experiments/default.yaml +```yaml +experiment: + runs: 4 + early_exit: true ``` -If no experiment is passed, AgentV checks `.agentv/config.yaml` for a default: +Do not set both `repeat` and `runs` in the same runtime block. -```yaml -experiments: - default: experiments/default.yaml -``` +## Result Layout -If no default is configured, AgentV keeps the old behavior and uses the -`default` experiment label. +Default eval runs write to: -## Schema +```text +.agentv/results/// +``` -The generated JSON Schema is available at -`skills-data/agentv-eval-writer/references/experiment-schema.json`. +Imported source suite metadata appears in `index.jsonl` rows and manifests. +AgentV does not add a redundant suite directory when the result group is already +the eval name. diff --git a/docs/plans/2026-06-23-002-experiments-separation-plan.md b/docs/plans/2026-06-23-002-experiments-separation-plan.md deleted file mode 100644 index a2e22f6fe..000000000 --- a/docs/plans/2026-06-23-002-experiments-separation-plan.md +++ /dev/null @@ -1,407 +0,0 @@ ---- -title: "feat: Separate experiments from eval definitions" -type: feat -date: 2026-06-23 -origin: docs/adr/0006-separate-experiments-from-eval-definitions.md ---- - -# feat: Separate experiments from eval definitions - -## Summary - -AgentV should separate eval task definitions from experiment run definitions. -Eval YAML stays the canonical authoring layer for prompts, datasets, assertions, -and task fixtures. Experiments become first-class committed files that select the -agent or target under test, model, harness options, setup injection, run knobs, -and case filter. - -This should ship in phases. Phase 1 adds the non-breaking foundation: -experiment contract types, default experiment resolution, and artifact -attribution by resolved experiment name. Later phases move runtime controls out -of `eval.yaml execution`, teach the CLI to run experiment matrices, and record -full experiment provenance and fingerprints in run bundles. - -## Problem Frame - -Today `experiment` is a string label passed through -`packages/core/src/evaluation/evaluate.ts`, `packages/core/src/evaluation/run-artifacts.ts`, -`packages/core/src/evaluation/results-repo.ts`, and -`packages/core/src/evaluation/trace-envelope.ts`. Runtime choices are still -scattered across CLI flags, TypeScript config, `.agentv/config.yaml`, and -`eval.yaml execution`. - -That makes it hard to review A/B variants such as `baseline` versus -`agents-md`, because the variable under test can be hidden inside the eval -definition. The desired model is: - -- Eval equals what is tested. -- Experiment equals how and with what it is run. -- Setup that changes the agent's environment belongs to the experiment. -- Existing eval-only repositories keep working through a default experiment - fallback. - -## Requirements - -- R1. Existing `eval.yaml` files validate and run without modification. -- R2. Experiment wire config uses `snake_case`; TypeScript types use - `camelCase`. -- R3. `config.yaml` can point at a default experiment, with no pointer falling - back to the current `default` experiment label. -- R4. `agentv eval --experiment