From 35ae2ef38c515e9a04fceb2d536ab5ad412cb654 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 13:56:31 +0200 Subject: [PATCH 1/3] fix(eval): require numeric repeat authoring --- apps/cli/src/commands/eval/run-eval.ts | 6 -- apps/cli/test/eval.integration.test.ts | 13 +-- apps/cli/test/fixtures/mock-run-evaluation.ts | 2 - packages/core/src/evaluation/experiment.ts | 93 ++----------------- .../src/evaluation/loaders/ts-eval-loader.ts | 14 ++- .../evaluation/validation/eval-file.schema.ts | 15 +-- .../evaluation/validation/eval-validator.ts | 71 ++------------ packages/core/src/evaluation/yaml-parser.ts | 4 +- .../evaluation/eval-inline-experiment.test.ts | 71 ++++++++------ .../core/test/evaluation/experiment.test.ts | 54 +++-------- .../loaders/fixtures/relative-import.eval.ts | 5 +- .../evaluation/loaders/jsonl-parser.test.ts | 9 +- .../validation/eval-file-schema.test.ts | 22 +++-- .../validation/eval-validator.test.ts | 44 ++++++--- packages/sdk/src/eval.ts | 33 ++++--- packages/sdk/src/index.ts | 2 +- packages/sdk/test/eval-authoring.test.ts | 29 ++++-- 17 files changed, 186 insertions(+), 301 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 8ce164699..a3a1d9237 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1049,10 +1049,6 @@ function buildExperimentTrialsConfig(experiment: ExperimentConfig): TrialsConfig return { count: experiment.repeat.count, strategy: experiment.repeat.strategy, - ...(experiment.repeat.costLimitUsd !== undefined && { - costLimitUsd: experiment.repeat.costLimitUsd, - }), - ...(experiment.repeat.earlyExit !== undefined && { earlyExit: experiment.repeat.earlyExit }), }; } return undefined; @@ -1074,8 +1070,6 @@ function buildRunOverrideTrialsConfig(run: EvalRunOverride | undefined): TrialsC return { count: repeat.count, strategy: repeat.strategy, - ...(repeat.costLimitUsd !== undefined && { costLimitUsd: repeat.costLimitUsd }), - ...(repeat.earlyExit !== undefined && { earlyExit: repeat.earlyExit }), }; } diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 8051c94b5..d40552a2f 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -725,10 +725,7 @@ describe('agentv eval CLI', () => { 'threshold: 0.8', 'evaluate_options:', ' budget_usd: 3', - ' repeat:', - ' count: 2', - ' strategy: pass_any', - ' early_exit: true', + ' repeat: 2', 'tests:', ' - include: sample.test.yaml', ' type: suite', @@ -737,10 +734,7 @@ describe('agentv eval CLI', () => { ' threshold: 1.0', ' timeout_seconds: 5', ' budget_usd: 0.75', - ' repeat:', - ' count: 3', - ' strategy: pass_all', - ' early_exit: true', + ' repeat: 3', '', ].join('\n'), 'utf8', @@ -765,8 +759,7 @@ describe('agentv eval CLI', () => { threshold: 1, trials: { count: 3, - strategy: 'pass_all', - earlyExit: true, + strategy: 'pass_any', }, }); diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index 0e96f3d01..0b85b9d40 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -24,8 +24,6 @@ interface RunEvaluationOptionsLike { readonly trials?: { readonly count: number; readonly strategy: string; - readonly costLimitUsd?: number; - readonly earlyExit?: boolean; }; readonly threshold?: number; readonly budgetUsd?: number; diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index 6d179b729..f6d8547e0 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -18,20 +18,11 @@ export type ExperimentTargetRef = readonly hooks?: Record; }; -export type ExperimentRepeatWire = { - readonly count?: number; - readonly strategy?: TrialStrategy; - readonly early_exit?: boolean; - readonly cost_limit_usd?: number; -}; - -export type ExperimentRepeatInput = number | ExperimentRepeatWire; +export type ExperimentRepeatInput = number; export type ExperimentRepeat = { readonly count: number; readonly strategy: TrialStrategy; - readonly earlyExit?: boolean; - readonly costLimitUsd?: number; }; export type ExperimentConfigWire = { @@ -72,21 +63,12 @@ export type ExperimentArtifactMetadata = { readonly repeat?: { readonly count: number; readonly strategy: TrialStrategy; - readonly early_exit?: boolean; - readonly cost_limit_usd?: number; }; readonly timeout_seconds?: number; readonly threshold?: number; readonly budget_usd?: number; }; -const VALID_REPEAT_STRATEGIES: ReadonlySet = new Set([ - 'pass_any', - 'pass_all', - 'mean', - 'confidence_interval', -]); - const RUN_OVERRIDE_FIELDS: ReadonlySet = new Set([ 'threshold', 'repeat', @@ -96,13 +78,6 @@ const RUN_OVERRIDE_FIELDS: ReadonlySet = new Set([ 'budgetUsd', ]); -const REPEAT_FIELDS: ReadonlySet = new Set([ - 'count', - 'strategy', - 'early_exit', - 'cost_limit_usd', -]); - export function normalizeExperimentConfig(rawConfig: unknown): ExperimentConfig { if (!isRecord(rawConfig)) { throw new Error('Experiment config must be an object.'); @@ -204,12 +179,6 @@ export function buildExperimentArtifactMetadata( repeat: { count: config.repeat.count, strategy: config.repeat.strategy, - ...(config.repeat.earlyExit !== undefined && { - early_exit: config.repeat.earlyExit, - }), - ...(config.repeat.costLimitUsd !== undefined && { - cost_limit_usd: config.repeat.costLimitUsd, - }), }, }), ...(config.timeoutSeconds !== undefined && { timeout_seconds: config.timeoutSeconds }), @@ -228,27 +197,9 @@ function readRepeat(raw: unknown): ExperimentRepeat | undefined { strategy: 'pass_any', }; } - if (!isRecord(raw)) { - throw new Error('Experiment repeat must be a positive integer or object.'); - } - for (const key of Object.keys(raw)) { - if (!REPEAT_FIELDS.has(key)) { - throw new Error( - `Experiment repeat.${key} is not supported. Use count, strategy, early_exit, and cost_limit_usd.`, - ); - } - } - const count = readRequiredPositiveInteger(raw.count, 'repeat.count'); - const strategy = readOptionalRepeatStrategy(raw.strategy); - const earlyExit = readOptionalBoolean(raw.early_exit, 'repeat.early_exit'); - const costLimitUsd = readOptionalNonNegativeNumber(raw.cost_limit_usd, 'repeat.cost_limit_usd'); - - return { - count, - strategy: strategy ?? 'pass_any', - ...(earlyExit !== undefined && { earlyExit }), - ...(costLimitUsd !== undefined && { costLimitUsd }), - }; + throw new Error( + 'Experiment repeat must be a positive integer. The repeat object shape has been removed; use evaluate_options.repeat: 2.', + ); } function readTargets(raw: unknown): readonly ExperimentTargetRef[] | undefined { @@ -305,37 +256,15 @@ function readRequiredPositiveInteger(raw: unknown, location: string): number { return value; } -function readOptionalBoolean(raw: unknown, location: string): boolean | undefined { - if (raw === undefined) { - return undefined; - } - if (typeof raw !== 'boolean') { - throw new Error(`Experiment ${location} must be a boolean.`); - } - return raw; -} - -function readOptionalRepeatStrategy(raw: unknown): TrialStrategy | undefined { - if (raw === undefined) { - return undefined; - } - if (typeof raw !== 'string' || !VALID_REPEAT_STRATEGIES.has(raw)) { - throw new Error( - "Experiment repeat.strategy must be one of 'pass_any', 'pass_all', 'mean', or 'confidence_interval'. 'pass_at_k' has been removed; use 'pass_any' instead.", - ); - } - return raw as TrialStrategy; -} - function rejectLegacyTopLevelRepeatFields(rawConfig: Record): void { if (rawConfig.runs !== undefined) { throw new Error( - "Experiment top-level 'runs' has been removed. Use evaluate_options.repeat.count and evaluate_options.repeat.strategy instead.", + "Experiment top-level 'runs' has been removed. Use a positive integer evaluate_options.repeat instead.", ); } if (rawConfig.early_exit !== undefined || rawConfig.earlyExit !== undefined) { throw new Error( - "Experiment top-level 'early_exit' has been removed. Use evaluate_options.repeat.early_exit instead.", + "Experiment top-level 'early_exit' has been removed. Use a positive integer evaluate_options.repeat instead.", ); } } @@ -350,16 +279,6 @@ function readOptionalPositiveInteger(raw: unknown, location: string): number | u return raw; } -function readOptionalNonNegativeNumber(raw: unknown, location: string): number | undefined { - if (raw === undefined) { - return undefined; - } - if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) { - throw new Error(`Experiment ${location} must be a non-negative number.`); - } - return raw; -} - function readOptionalThreshold(raw: unknown): number | undefined { if (raw === undefined) { return undefined; diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index 587b98c0a..c6d6e862d 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -29,10 +29,8 @@ const KNOWN_SNAKE_CASE_KEYS = { cachePath: 'cache_path', budgetUsd: 'budget_usd', conversationId: 'conversation_id', - costLimitUsd: 'cost_limit_usd', defaultTest: 'default_test', dependsOn: 'depends_on', - earlyExit: 'early_exit', expectedOutput: 'expected_output', failOnError: 'fail_on_error', inputFiles: 'input_files', @@ -196,6 +194,7 @@ function isProgrammaticEvalConfig(value: unknown): value is ProgrammaticEvalConf function lowerTypeScriptEvalConfig(config: Record): Record { const lowered = lowerEvalYamlValue(config) as Record; const { budget_usd: budgetUsd, repeat, ...withoutRuntimeAliases } = lowered; + validateTypeScriptRepeat(repeat, 'repeat'); if (budgetUsd === undefined && repeat === undefined) { return withoutRuntimeAliases; } @@ -220,6 +219,17 @@ function lowerTypeScriptEvalConfig(config: Record): Record lowerEvalYamlValue(item)); diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 290fb6b9a..ea6ac79d5 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -721,7 +721,7 @@ const ExecutionSchema = z.object({ assert: z.array(AssertionItemSchema).optional(), skip_defaults: z.boolean().optional(), cache: z.boolean().optional(), - /** Removed before stable release. Repeat counts belong under evaluate_options.repeat.count. */ + /** Removed before stable release. Repeat counts belong under evaluate_options.repeat. */ trials: z.never().optional(), budget_usd: z.number().min(0).optional(), budgetUsd: z.number().min(0).optional(), @@ -730,19 +730,10 @@ const ExecutionSchema = z.object({ threshold: z.number().min(0).max(1).optional(), }); -const ExperimentRepeatSchema = z - .object({ - count: z.number().int().min(1), - strategy: z.enum(['pass_any', 'pass_all', 'mean', 'confidence_interval']).optional(), - early_exit: z.boolean().optional(), - cost_limit_usd: z.number().min(0).optional(), - }) - .strict(); - const RunOverrideSchema = z .object({ threshold: z.number().min(0).max(1).optional(), - repeat: ExperimentRepeatSchema.optional(), + repeat: z.number().int().min(1).optional(), timeout_seconds: z.number().gt(0).optional(), budget_usd: z.number().gt(0).optional(), }) @@ -774,7 +765,7 @@ const EvaluateOptionsSchema = z cache: z.union([z.boolean(), JsonObjectSchema]).optional(), delay: z.number().min(0).optional(), generate_suggestions: z.boolean().optional(), - repeat: z.union([z.number().int().min(1), ExperimentRepeatSchema]).optional(), + repeat: z.number().int().min(1).optional(), timeout_ms: z.number().gt(0).optional(), max_eval_time_ms: z.number().gt(0).optional(), filter_range: z.union([z.tuple([z.number(), z.number()]), z.string()]).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 8b12cadb2..949250fe5 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -275,8 +275,6 @@ const KNOWN_EVALUATE_OPTION_FIELDS = new Set([ 'max_eval_time_ms', 'filter_range', ]); -const KNOWN_REPEAT_FIELDS = new Set(['count', 'strategy', 'early_exit', 'cost_limit_usd']); -const KNOWN_REPEAT_STRATEGIES = new Set(['pass_any', 'pass_all', 'mean', 'confidence_interval']); const KNOWN_TEST_EXECUTION_FIELDS = new Set([ 'assert', 'skip_defaults', @@ -327,10 +325,10 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ "Top-level 'evalcases' has been removed from authored eval YAML. Use 'tests' instead.", ], ['repeat', "Top-level 'repeat' has been removed. Use evaluate_options.repeat instead."], - ['runs', "Top-level 'runs' has been removed. Use evaluate_options.repeat.count instead."], + ['runs', "Top-level 'runs' has been removed. Use a positive integer evaluate_options.repeat."], [ 'early_exit', - "Top-level 'early_exit' has been removed. Use evaluate_options.repeat.early_exit instead.", + "Top-level 'early_exit' has been removed. Use a positive integer evaluate_options.repeat.", ], ['budget_usd', "Top-level 'budget_usd' has been removed. Use evaluate_options.budget_usd."], [ @@ -1880,7 +1878,12 @@ function validateEvaluateOptionsRepeat( } return; } - validateRepeatOverride(repeat, location, filePath, errors); + errors.push({ + severity: 'error', + filePath, + location, + message: `Invalid '${location}' field (must be a positive integer). The repeat object shape has been removed; use ${location}: 2.`, + }); } function validateFilterRange( @@ -1915,67 +1918,13 @@ function validateRepeatOverride( if (repeat === undefined) { return; } - if (!isObject(repeat)) { + if (typeof repeat !== 'number' || !Number.isInteger(repeat) || repeat < 1) { errors.push({ severity: 'error', filePath, location, - message: "Invalid 'repeat' field (must be an object)", - }); - return; - } - - for (const key of Object.keys(repeat)) { - if (!KNOWN_REPEAT_FIELDS.has(key)) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.${key}`, - message: - 'Invalid repeat field. Supported fields: count, strategy, early_exit, cost_limit_usd.', - }); - } - } - - if (typeof repeat.count !== 'number' || !Number.isInteger(repeat.count) || repeat.count < 1) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.count`, - message: "Invalid 'count' field (must be a positive integer)", - }); - } - - if ( - repeat.strategy !== undefined && - (typeof repeat.strategy !== 'string' || !KNOWN_REPEAT_STRATEGIES.has(repeat.strategy)) - ) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.strategy`, message: - "Invalid 'strategy' field (must be pass_any, pass_all, mean, or confidence_interval; use pass_any instead of removed pass_at_k)", - }); - } - - const earlyExit = repeat.early_exit; - if (earlyExit !== undefined && typeof earlyExit !== 'boolean') { - errors.push({ - severity: 'error', - filePath, - location: `${location}.early_exit`, - message: "Invalid 'early_exit' field (must be a boolean)", - }); - } - - const costLimit = repeat.cost_limit_usd; - if (costLimit !== undefined && (typeof costLimit !== 'number' || costLimit < 0)) { - errors.push({ - severity: 'error', - filePath, - location: `${location}.cost_limit_usd`, - message: "Invalid 'cost_limit_usd' field (must be a non-negative number)", + "Invalid 'repeat' field (must be a positive integer). The repeat object shape has been removed; use repeat: 2.", }); } } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index dd770ef0c..5e7b3d303 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -2663,12 +2663,12 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO } if (suite.runs !== undefined) { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'runs' has been removed. Use evaluate_options.repeat.count instead.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'runs' has been removed. Use a positive integer evaluate_options.repeat instead.`, ); } if (suite.early_exit !== undefined) { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'early_exit' has been removed. Use evaluate_options.repeat.early_exit instead.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'early_exit' has been removed. Use a positive integer evaluate_options.repeat instead.`, ); } if (suite.repeat !== undefined) { diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 20ccc8801..8db718cc9 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -17,7 +17,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('parses evaluate_options.repeat object as the canonical runtime block', async () => { + it('parses evaluate_options.repeat number as the canonical runtime block', async () => { const evalPath = path.join(tempDir, 'runtime.eval.yaml'); await writeFile( evalPath, @@ -32,10 +32,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' reasoning_effort: high', 'threshold: 0.7', 'evaluate_options:', - ' repeat:', - ' count: 2', - ' strategy: pass_any', - ' early_exit: true', + ' repeat: 2', ' budget_usd: 1.5', 'timeout_seconds: 30', 'prompts:', @@ -54,7 +51,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { target: 'codex', name: 'release-gate', threshold: 0.7, - repeat: { count: 2, strategy: 'pass_any', earlyExit: true }, + repeat: { count: 2, strategy: 'pass_any' }, timeoutSeconds: 30, budgetUsd: 1.5, }); @@ -1082,9 +1079,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' - parent-target', 'threshold: 0.8', 'evaluate_options:', - ' repeat:', - ' count: 3', - ' strategy: pass_any', + ' repeat: 3', ' budget_usd: 1.5', 'timeout_seconds: 30', 'assert:', @@ -1115,7 +1110,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(test.assertions?.[0]).toMatchObject({ value: 'child' }); }); - it('applies tests[].options.repeat over the global repeat object', async () => { + it('applies numeric tests[].options.repeat over the global repeat count', async () => { const evalPath = path.join(tempDir, 'test-options-repeat.eval.yaml'); await writeFile( evalPath, @@ -1124,9 +1119,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { 'providers:', ' - codex', 'evaluate_options:', - ' repeat:', - ' count: 4', - ' strategy: pass_all', + ' repeat: 4', 'prompts:', ' - "{{ input }}"', 'tests:', @@ -1140,15 +1133,13 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' repeat: 2', ' vars:', ' input: hello', - ' - id: case-repeat-object', + ' - id: case-repeat-run', ' criteria: ok', ' run:', ' threshold: 0.9', + ' repeat: 3', ' options:', - ' repeat:', - ' count: 3', - ' strategy: mean', - ' early_exit: false', + ' repeat: 2', ' vars:', ' input: hello', ].join('\n'), @@ -1157,18 +1148,47 @@ describe('eval.yaml flat runtime controls and tests imports', () => { const suite = await loadTestSuite(evalPath, tempDir); const byId = new Map(suite.tests.map((test) => [test.id, test])); - expect(suite.experimentConfig?.repeat).toEqual({ count: 4, strategy: 'pass_all' }); + expect(suite.experimentConfig?.repeat).toEqual({ count: 4, strategy: 'pass_any' }); expect(byId.get('global-repeat')?.run).toBeUndefined(); expect(byId.get('case-repeat-count')?.run?.repeat).toEqual({ count: 2, strategy: 'pass_any', }); - expect(byId.get('case-repeat-object')?.run).toMatchObject({ + expect(byId.get('case-repeat-run')?.run).toMatchObject({ threshold: 0.9, - repeat: { count: 3, strategy: 'mean', earlyExit: false }, + repeat: { count: 2, strategy: 'pass_any' }, }); }); + it('rejects object-shaped repeat under evaluate_options and scoped run overrides', async () => { + const evalPath = path.join(tempDir, 'test-options-repeat-object.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: test-options-repeat-object', + 'providers:', + ' - codex', + 'evaluate_options:', + ' repeat:', + ' count: 4', + ' strategy: pass_all', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: one', + ' criteria: ok', + ' options:', + ' repeat:', + ' count: 3', + ' early_exit: false', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/positive integer/); + }); + it('rejects parent environment when importing eval suites with type: suite', async () => { await writeFile( path.join(tempDir, 'child.eval.yaml'), @@ -1365,8 +1385,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' criteria: ok', ' run:', ' threshold: 1', - ' repeat:', - ' count: 1', + ' repeat: 1', ' vars:', ' input: critical', ].join('\n'), @@ -1381,9 +1400,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' type: suite', ' run:', ' threshold: 0.9', - ' repeat:', - ' count: 2', - ' strategy: pass_all', + ' repeat: 2', ' timeout_seconds: 30', ' budget_usd: 1.25', '', @@ -1396,7 +1413,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig).toBeUndefined(); expect(byId.get('child-default')?.run).toMatchObject({ threshold: 0.9, - repeat: { count: 2, strategy: 'pass_all' }, + repeat: { count: 2, strategy: 'pass_any' }, timeoutSeconds: 30, budgetUsd: 1.25, }); diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts index 3bdb8a2a2..9808af8af 100644 --- a/packages/core/test/evaluation/experiment.test.ts +++ b/packages/core/test/evaluation/experiment.test.ts @@ -14,7 +14,7 @@ describe('inline experiment config', () => { agent: 'codex', model: 'openai/gpt-5.5', agent_options: { reasoning_effort: 'high' }, - repeat: { count: 3, strategy: 'pass_any', early_exit: false }, + repeat: 3, timeout_seconds: 900, threshold: 0.8, budget_usd: 1.25, @@ -27,42 +27,13 @@ describe('inline experiment config', () => { agent: 'codex', model: 'openai/gpt-5.5', agentOptions: { reasoning_effort: 'high' }, - repeat: { count: 3, strategy: 'pass_any', earlyExit: false }, + repeat: { count: 3, strategy: 'pass_any' }, timeoutSeconds: 900, budgetUsd: 1.25, }); expect(config.fingerprint).toMatch(/^[a-f0-9]{64}$/); }); - it('normalizes repeat config', () => { - const config = normalizeExperimentConfig({ - repeat: { - count: 4, - strategy: 'confidence_interval', - cost_limit_usd: 0, - }, - }); - - expect(config.repeat).toEqual({ - count: 4, - strategy: 'confidence_interval', - costLimitUsd: 0, - }); - }); - - it('defaults repeat strategy to pass_any', () => { - const config = normalizeExperimentConfig({ - repeat: { - count: 2, - }, - }); - - expect(config.repeat).toEqual({ - count: 2, - strategy: 'pass_any', - }); - }); - it('normalizes repeat number shorthand', () => { const config = normalizeExperimentConfig({ repeat: 3, @@ -75,20 +46,21 @@ describe('inline experiment config', () => { }); it('rejects invalid run counts', () => { - expect(() => normalizeExperimentConfig({ runs: 3 })).toThrow(/repeat.count/); - expect(() => normalizeExperimentConfig({ early_exit: true })).toThrow(/repeat.early_exit/); + expect(() => normalizeExperimentConfig({ runs: 3 })).toThrow(/positive integer/); + expect(() => normalizeExperimentConfig({ early_exit: true })).toThrow(/positive integer/); + expect(() => normalizeExperimentConfig({ repeat: { count: 2 } })).toThrow(/positive integer/); expect(() => normalizeExperimentConfig({ repeat: { count: 2, strategy: 'pass_at_k' } }), - ).toThrow(/pass_at_k.*removed/); - expect(() => normalizeExperimentConfig({ repeat: {} })).toThrow(/repeat.count/); + ).toThrow(/positive integer/); + expect(() => normalizeExperimentConfig({ repeat: {} })).toThrow(/positive integer/); expect(() => normalizeExperimentConfig({ repeat: { count: 2, strategy: 'median' } })).toThrow( - /repeat.strategy/, + /positive integer/, ); expect(() => normalizeExperimentConfig({ repeat: { count: 2, cost_limit_usd: -1 } })).toThrow( - /repeat.cost_limit_usd/, + /positive integer/, ); expect(() => normalizeExperimentConfig({ repeat: { count: 2, costLimitUsd: 1 } })).toThrow( - /repeat.costLimitUsd/, + /positive integer/, ); expect(() => normalizeExperimentConfig({ setup: [{ command: 'bun install' }] })).toThrow( /setup is not supported/, @@ -112,7 +84,7 @@ describe('inline experiment config', () => { name: 'baseline', target: 'codex', agent_options: { secret: 'not persisted' }, - repeat: { count: 2, strategy: 'mean', early_exit: true, cost_limit_usd: 0.5 }, + repeat: 2, timeout_seconds: 120, }); @@ -123,9 +95,7 @@ describe('inline experiment config', () => { target: 'codex', repeat: { count: 2, - strategy: 'mean', - early_exit: true, - cost_limit_usd: 0.5, + strategy: 'pass_any', }, timeout_seconds: 120, }); diff --git a/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts index 59bf88c9d..9e9319df5 100644 --- a/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts @@ -6,10 +6,7 @@ const config = { tags: { experiment: 'ts-config', group: 'loader' }, prompts: [relativePrompt], budgetUsd: 1, - repeat: { - count: 2, - strategy: 'pass_any', - }, + repeat: 2, tests: [ { id: 'relative-import', diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index c85f361a1..7839ecef1 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -976,8 +976,7 @@ default_test: shared: default default_only: base options: - repeat: - count: 2 + repeat: 2 assert: - type: contains value: default assertion @@ -990,8 +989,7 @@ scenarios: owner: config source: scenario-config options: - repeat: - count: 3 + repeat: 3 run: timeout_seconds: 10 assert: @@ -1004,8 +1002,7 @@ scenarios: metadata: owner: test options: - repeat: - count: 4 + repeat: 4 run: threshold: 0.8 assert: diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 685a068ab..b1e4f0755 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -300,11 +300,7 @@ describe('EvalFileSchema input shorthand', () => { evaluate_options: { budget_usd: 2, max_concurrency: 3, - repeat: { - count: 2, - strategy: 'pass_any', - early_exit: true, - }, + repeat: 2, }, tests: [ { @@ -326,7 +322,7 @@ describe('EvalFileSchema input shorthand', () => { }, run: { threshold: 1, - repeat: { count: 2, strategy: 'pass_all', early_exit: true }, + repeat: 2, timeout_seconds: 120, budget_usd: 2, }, @@ -750,7 +746,7 @@ describe('EvalFileSchema input shorthand', () => { }, run: { threshold: 1, - repeat: { count: 2, strategy: 'pass_all' }, + repeat: 2, timeout_seconds: 120, budget_usd: 2, }, @@ -820,6 +816,18 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); + it('rejects object-shaped public repeat fields', () => { + const result = EvalFileSchema.safeParse({ + providers: ['mock-target'], + evaluate_options: { + repeat: { count: 2, strategy: 'pass_any', early_exit: true, cost_limit_usd: 1 }, + }, + tests: [baseTest], + }); + + expect(result.success).toBe(false); + }); + it('rejects camelCase fields under authored policy blocks', () => { const result = EvalFileSchema.safeParse({ target: 'codex', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index a754b77a9..1400545f0 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -108,10 +108,7 @@ threshold: 0.8 evaluate_options: budget_usd: 2 max_concurrency: 3 - repeat: - count: 2 - strategy: pass_any - early_exit: true + repeat: 2 tests: - id: local-case vars: @@ -460,9 +457,7 @@ tests: vars: diff: change options: - repeat: - count: 3 - strategy: mean + repeat: 3 assert: - type: contains value: safe @@ -878,6 +873,35 @@ evalcases: expect(result.errors.some((error) => error.severity === 'warning')).toBe(false); }); + it('rejects public repeat objects with migration guidance', async () => { + const filePath = path.join(tempDir, 'removed-repeat-object.yaml'); + await writeFile( + filePath, + `target: codex +evaluate_options: + repeat: + count: 2 + strategy: pass_any + early_exit: true + cost_limit_usd: 1 +tests: + - id: local-case + input: "Hello" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.location === 'evaluate_options.repeat' && + error.message.includes('use evaluate_options.repeat: 2'), + ), + ).toBe(true); + }); + it('rejects removed top-level repeat controls with migration guidance', async () => { const filePath = path.join(tempDir, 'removed-repeat-fields.yaml'); await writeFile( @@ -900,12 +924,10 @@ tests: result.errors.some((error) => error.message.includes('Use evaluate_options.repeat')), ).toBe(true); expect( - result.errors.some((error) => error.message.includes('Use evaluate_options.repeat.count')), + result.errors.some((error) => error.message.includes('Use evaluate_options.repeat')), ).toBe(true); expect( - result.errors.some((error) => - error.message.includes('Use evaluate_options.repeat.early_exit'), - ), + result.errors.some((error) => error.message.includes('Use evaluate_options.repeat')), ).toBe(true); }); diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 3f1766478..8878599b8 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -11,9 +11,7 @@ const KNOWN_SNAKE_CASE_KEYS = { beforeEach: 'before_each', budgetUsd: 'budget_usd', conversationId: 'conversation_id', - costLimitUsd: 'cost_limit_usd', dependsOn: 'depends_on', - earlyExit: 'early_exit', expectedOutput: 'expected_output', explorationTolerance: 'exploration_tolerance', failOnError: 'fail_on_error', @@ -190,14 +188,7 @@ export interface EvalDefaultTest { readonly [key: string]: unknown; } -export interface EvalTrials { - readonly count: number; - readonly strategy?: 'pass_any' | 'pass_all' | 'mean' | 'confidence_interval'; - readonly earlyExit?: boolean; - readonly costLimitUsd?: number; -} - -export type EvalRepeat = EvalTrials; +export type EvalRepeat = number; export interface EvalExecution { readonly provider?: string; @@ -205,7 +196,7 @@ export interface EvalExecution { readonly assert?: readonly EvalAssertionConfig[]; readonly skipDefaults?: boolean; readonly cache?: boolean; - readonly trials?: EvalTrials; + readonly trials?: never; readonly budgetUsd?: number; readonly failOnError?: boolean; readonly threshold?: number; @@ -306,6 +297,7 @@ function lowerEvalYamlValue(value: unknown): unknown { function lowerEvalConfig(config: unknown): Record { const lowered = lowerEvalYamlValue(config) as Record; const { budget_usd: budgetUsd, repeat, ...loweredWithoutRuntimeOptions } = lowered; + validateRepeatValue(repeat, 'repeat'); if (budgetUsd === undefined && repeat === undefined) { return lowered; } @@ -329,6 +321,17 @@ function lowerEvalConfig(config: unknown): Record { }; } +function validateRepeatValue(value: unknown, location: string): void { + if (value === undefined) { + return; + } + if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) { + throw new Error( + `defineEval() expects ${location} to be a positive integer; object-shaped repeat authoring has been removed.`, + ); + } +} + function attachEvalSuiteBrand(definition: T): T & DefinedEvalSuite { validateTopLevelRuntimeFields(definition); const branded = definition as T & Partial; @@ -385,6 +388,7 @@ function validateTopLevelRuntimeFields(definition: EvalConfig): void { ); } } + validateRepeatValue(rawDefinition.repeat, 'repeat'); if (Array.isArray(rawDefinition.tests)) { rawDefinition.tests.forEach((test, index) => { if (test && typeof test === 'object' && Object.prototype.hasOwnProperty.call(test, 'input')) { @@ -392,6 +396,13 @@ function validateTopLevelRuntimeFields(definition: EvalConfig): void { `defineEval() does not accept tests[${index}].input. Use prompts with tests[].vars instead.`, ); } + const options = (test as { readonly options?: unknown }).options; + if (options && typeof options === 'object' && !Array.isArray(options)) { + validateRepeatValue( + (options as { readonly repeat?: unknown }).repeat, + `tests[${index}].options.repeat`, + ); + } }); } validateProviderSurface(definition); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2adfa0e80..691b53bcf 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -160,10 +160,10 @@ export { type EvalProviderEntry, type EvalProviderMap, type EvalProviderRef, + type EvalRepeat, type EvalRequires, type EvalTest, type EvalTestOptions, - type EvalTrials, type EvalTurn, type LowerEvalYamlValue, } from './eval.js'; diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 70edd8dad..d983724fb 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -34,11 +34,7 @@ describe('YAML-aligned eval authoring helpers', () => { provider: 'grader-gpt5-mini', }, }, - repeat: { - count: 3, - strategy: 'pass_any', - earlyExit: false, - }, + repeat: 3, timeoutSeconds: 600, threshold: 0.8, budgetUsd: 1.5, @@ -132,11 +128,7 @@ describe('YAML-aligned eval authoring helpers', () => { threshold: 0.8, prompts: ['{{ input }}'], evaluate_options: { - repeat: { - count: 3, - strategy: 'pass_any', - early_exit: false, - }, + repeat: 3, budget_usd: 1.5, }, assert: [ @@ -405,6 +397,23 @@ describe('YAML-aligned eval authoring helpers', () => { ).toThrow(/top-level 'runs'/); }); + it('rejects object-shaped repeat authoring', () => { + expect(() => + defineEval({ + name: 'removed-repeat-object', + repeat: { count: 3, strategy: 'pass_any' }, + prompts: ['{{ input }}'], + tests: [ + { + id: 'hello', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], + } as never), + ).toThrow(/repeat.*positive integer/); + }); + it('rejects removed target-shaped authoring', () => { expect(() => defineEval({ From 313d9ab6bfe4c4d529ae1974d0c94469c89d9fb3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 13:52:37 +0200 Subject: [PATCH 2/3] docs(eval): document numeric repeat authoring --- .agents/verification.md | 2 +- CONCEPTS.md | 10 ++--- README.md | 12 ++---- .../docs/docs/next/evaluation/eval-files.mdx | 6 +-- .../docs/docs/next/evaluation/experiments.mdx | 37 ++++--------------- .../docs/v4.42.4/evaluation/eval-files.mdx | 8 ++-- ...arate-experiments-from-eval-definitions.md | 14 +++---- docs/adr/0012-finalize-run-artifact-layout.md | 14 +++---- .../0013-stabilize-eval-authoring-contract.md | 7 ++-- ...mptfoo-superset-eval-authoring-contract.md | 8 +++- ...3-001-feat-repeat-runs-flaky-evals-plan.md | 6 +++ .../promptfoo-aligned-eval-restructure.md | 6 +++ examples/features/README.md | 2 +- examples/features/trials/README.md | 9 ++--- examples/features/trials/evals/suite.yaml | 5 +-- 15 files changed, 62 insertions(+), 84 deletions(-) diff --git a/.agents/verification.md b/.agents/verification.md index 29eec2386..0a2b9bb12 100644 --- a/.agents/verification.md +++ b/.agents/verification.md @@ -153,7 +153,7 @@ Use live dogfood before marking PRs ready when they affect eval execution, exper - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers. - For artifact/result contract changes, prefer letting AgentV choose the canonical run directory and capture the printed `Artifact workspace written to:` and `Results written to:` paths for evidence. Do not precompute `--output` unless the test specifically needs a fixed path. - For native experiment changes, run through `agentv eval run ... --experiment ` so resolution, setup, scripts, provider selection, run knobs, and artifact metadata are exercised together. -- For repeat-run changes, use `evaluate_options.repeat.count >= 2` when validating repeated executions. Inspect `.internal/index.jsonl`, root `summary.json`, and the repeated case folder. Use `repeat` for authored configuration and `sample_index`/`retry_index` for produced executions. The repeated case folder should carry aggregate `summary.json`; sample-specific outputs, transcripts, grading, and metrics live under `sample-N/`. Each `sample-N/` folder should contain `result.json`, `grading.json`, `metrics.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md` when answer output is available. `result.json` should point at `./grading.json`, `./metrics.json`, `./transcript.json`, and `./transcript-raw.jsonl` through the corresponding path fields. +- For repeat-run changes, use `evaluate_options.repeat: 2` or higher when validating repeated executions. Inspect `.internal/index.jsonl`, root `summary.json`, and the repeated case folder. Use numeric `repeat` for authored configuration and `sample_index`/`retry_index` for produced executions. The repeated case folder should carry aggregate `summary.json`; sample-specific outputs, transcripts, grading, and metrics live under `sample-N/`. Each `sample-N/` folder should contain `result.json`, `grading.json`, `metrics.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md` when answer output is available. `result.json` should point at `./grading.json`, `./metrics.json`, `./transcript.json`, and `./transcript-raw.jsonl` through the corresponding path fields. - For local OpenAI-compatible grading through the OAuth proxy, use `base_url: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `{{ env.LOCAL_OPENAI_PROXY_API_KEY }}` and `{{ env.LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by provider validation unless a resolver explicitly allows them. - For `codex`/Codex SDK live dogfood through the same local proxy, configure the agent provider with `id`, `label`, `runtime`, and `config`, and put backend settings such as `base_url`, `api_key`, `model`, and `api_format` under `config`. Configure the reusable grader in the same `providers` catalog, then select it with `defaults.grader` or assertion-level `provider`; do not put a grader selector on the system-under-test provider. A minimal run should use `bun apps/cli/src/cli.ts eval run --providers --provider --workers 1`. - If the local proxy returns `401 token_expired`, the blocker is stale Codex OAuth, not AgentV target configuration. Refresh from a trusted local terminal with `codex logout`, `codex login --device-auth`, then restart `openai-oauth` and rerun the same eval command. diff --git a/CONCEPTS.md b/CONCEPTS.md index cad8b6392..5c5a8f838 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -102,15 +102,15 @@ env: **Repeat run** — A configured request to execute the same eval case and target more than once in the same run bundle. Repeat runs measure stochastic reliability, verifier stability, and drift; they are not the default CI path. -**Attempt** — One concrete execution inside a repeat run. Attempts keep their own score, status, metrics, trace, transcript, logs, and artifacts so aggregate results never hide individual evidence. +**Repeat sample** — One concrete execution inside a repeat run. Samples keep their own score, status, metrics, trace, transcript, logs, and artifacts so aggregate results never hide individual evidence. -**Pass rate** — Assertion or expectation pass rate inside a grading result: passed assertions or expectations divided by total assertions or expectations. AgentV does not use `pass_rate` for repeat-attempt success frequency. +**Pass rate** — Assertion or expectation pass rate inside a grading result: passed assertions or expectations divided by total assertions or expectations. AgentV does not use `pass_rate` for repeat-sample success frequency. -**Attempt success rate** — Repeat-run reliability metric equal to successful counted attempts divided by counted attempts. This is distinct from `pass_rate`, which is reserved for assertion or expectation pass rate within a grading result. +**Sample success rate** — Repeat-run reliability metric equal to successful counted samples divided by counted samples. This is distinct from `pass_rate`, which is reserved for assertion or expectation pass rate within a grading result. -**Gate policy** — The explicit rule that decides whether repeated attempts pass CI, such as `all_attempts_successful`, `any_attempt_successful`, `attempt_success_rate_at_least`, or `mean_pass_rate_at_least`. Without a repeat-run gate policy, AgentV preserves the normal single-run gate behavior and treats repeat statistics as report data. +**Aggregate gate policy** — Future fatal post-run policy surface for deciding whether repeated samples pass CI. Until AgentV deliberately designs that surface, repeat configuration only sets the numeric sample count and users can parse run bundles externally for aggregate gating. -**Flaky eval outcome** — A repeat-run aggregate whose attempts disagree, or whose failure classification points at verifier, infrastructure, or timeout instability rather than a stable model-quality failure. +**Flaky eval outcome** — A repeat-run aggregate whose samples disagree, or whose failure classification points at verifier, infrastructure, or timeout instability rather than a stable model-quality failure. ## Release Channels diff --git a/README.md b/README.md index 3ab24fd32..9646b3efb 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Test AI providers on real repo tasks and measure what actually works. - **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts. - **Provider** is the configured system under test: an agent, model provider, gateway, replay provider, CLI wrapper, transcript provider, or future app/service wrapper. Each provider entry uses `id` for the backend/spec and optional `label` for the stable AgentV selection and result identity. - **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and provider/model names out of that tag. -- **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets. +- **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat sample count, and budgets. - **Default test** configures inherited per-test defaults such as score `threshold`. - **Run** is one concrete execution of a tagged eval against a resolved provider that writes portable artifacts for readers such as Dashboard, compare, and trend. @@ -133,9 +133,7 @@ providers: api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: gpt-5.4-mini evaluate_options: - repeat: - count: 2 - strategy: pass_any + repeat: 2 default_test: threshold: 0.85 @@ -253,11 +251,7 @@ const config: EvalConfig = { extends: 'copilot-sdk', model: 'claude-sonnet-4.6', }, - repeat: { - count: 3, - strategy: 'pass_any', - earlyExit: false, - }, + repeat: 3, threshold: 0.8, prompts: ['{{ input }}'], environment: { diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index 4364b8788..d478335f1 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -119,9 +119,7 @@ providers: - codex-gpt5 evaluate_options: max_concurrency: 3 - repeat: - count: 2 - strategy: pass_any + repeat: 2 tests: - file://../evals/cases/refund-smoke.cases.yaml @@ -179,7 +177,7 @@ tests: | `tags` | Optional metadata map. Use `tags.experiment` as the run/result grouping label. | | `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions rendered with `tests[].vars` and `default_test.vars`. | | `providers` | System-under-test matrix. Entries can be Promptfoo-shaped provider strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects, or provider maps; `id` names the backend/spec and `label` is the stable AgentV identity. | -| `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` | +| `evaluate_options.repeat` | Optional positive integer sample count for repeat runs | | `evaluate_options` | Optional evaluation runtime options such as `budget_usd`, `repeat`, and `max_concurrency` | | `timeout_seconds` | Optional per-case timeout | | `threshold` | Optional suite quality threshold | diff --git a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx index 507748bf2..ed0491bf1 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx @@ -27,9 +27,7 @@ providers: reasoning_effort: high timeout_seconds: 720 evaluate_options: - repeat: - count: 4 - strategy: pass_any + repeat: 4 budget_usd: 2.00 max_concurrency: 3 @@ -120,9 +118,7 @@ providers: - agent threshold: 0.8 evaluate_options: - repeat: - count: 3 - strategy: pass_any + repeat: 3 timeout_seconds: 300 tags: area: agentic @@ -161,7 +157,7 @@ target-specific runner state. | Configure an agent runner or provider variant | `providers` entry or `providers.yaml` | | Choose the provider | top-level `providers` or CLI `--provider` | | Override the provider's default model | `providers[].config.model` | -| Configure repeat policy, budget, concurrency, timeout, threshold | `evaluate_options.repeat`, `evaluate_options.budget_usd`, `evaluate_options.max_concurrency`, `timeout_seconds`, `threshold` | +| Configure repeat sample count, budget, concurrency, timeout, threshold | `evaluate_options.repeat`, `evaluate_options.budget_usd`, `evaluate_options.max_concurrency`, `timeout_seconds`, `threshold` | | Bind an existing local workspace directory | `--workspace-path` or `.agentv/config.local.yaml` | ```yaml @@ -176,9 +172,7 @@ providers: before_each: command: ["sh", "-c", "cp -R skills \"{{workspace_path}}/.codex/skills\""] evaluate_options: - repeat: - count: 3 - strategy: pass_any + repeat: 3 ``` Existing local workspace paths are machine-local bindings: pass @@ -197,25 +191,10 @@ evaluate_options: repeat: 3 ``` -Use object form when you need richer AgentV behavior: - -```yaml -evaluate_options: - repeat: - count: 3 - strategy: pass_any - early_exit: true - cost_limit_usd: 1.00 -``` - -`evaluate_options.repeat.strategy` controls sample aggregation. `pass_any` -treats the case as successful when any completed sample passes; `pass_all` -requires every completed sample to pass. `mean` and `confidence_interval` -aggregate scores where supported today. `evaluate_options.repeat.early_exit` is -only a scheduling and cost optimization: `pass_any` may stop at the first pass, -and `pass_all` may stop at the first fail. Leave it unset or `false` when you -want complete variance data. Per-case `tests[].options.repeat` overrides the -global repeat count or object for that case. +Repeat authoring only sets the sample count. A fatal post-run aggregate policy +surface is future work; until that exists, parse the run bundle externally when +CI needs aggregate gating across samples. Per-case `tests[].options.repeat` +overrides the global repeat count for that case. ## Result Layout diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx index 405981fd6..4b9502966 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx @@ -13,7 +13,7 @@ controls for an evaluation run. The reserved `tags.experiment` key is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated -attempts and gates. Coding-agent testbeds, workdirs, Docker config, repository +samples and gates. Coding-agent testbeds, workdirs, Docker config, repository materialization, setup, and reset policy belong in `environment`. Provider environment-variable overrides belong in top-level `env`. Lifecycle hooks belong in `extensions`; runner-specific setup belongs in the `target` object, in @@ -85,9 +85,7 @@ name: refunds-codex target: codex-gpt5 evaluate_options: max_concurrency: 3 - repeat: - count: 2 - strategy: pass_any + repeat: 2 tests: - file://../evals/cases/refund-smoke.cases.yaml @@ -133,7 +131,7 @@ tests: | `tags` | Optional promptfoo-style metadata map. Use `tags.experiment` as the run/result grouping label. | | `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions. | | `targets` | Optional target matrix. Entries reference target ids or inline target objects. | -| `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` | +| `evaluate_options.repeat` | Optional positive integer sample count for repeat runs | | `timeout_seconds` | Optional per-case timeout | | `evaluate_options` | Optional evaluation runtime options such as `budget_usd`, `repeat`, and `max_concurrency` | | `threshold` | Optional suite quality threshold | diff --git a/docs/adr/0006-separate-experiments-from-eval-definitions.md b/docs/adr/0006-separate-experiments-from-eval-definitions.md index e31ad2509..b87c909d2 100644 --- a/docs/adr/0006-separate-experiments-from-eval-definitions.md +++ b/docs/adr/0006-separate-experiments-from-eval-definitions.md @@ -21,6 +21,9 @@ Superseded for the current eval authoring contract by [ADR 0016](0016-promptfoo-superset-eval-authoring-contract.md): `tests`, `prompts`/`vars`, `assert`, direct `input`, `target`/`targets`, and `evaluate_options` are the current promptfoo-aligned authoring surface. +The repeat object examples in this ADR are also superseded by Bead `av-s96i`: +current public authoring uses Promptfoo-style numeric `evaluate_options.repeat` +only. The "Suite And Test Import Surface" section below is historical. Its `imports.suites` / `imports.tests` design is superseded by av-kfik.43 and the @@ -85,10 +88,7 @@ target: timeout_seconds: 900 threshold: 0.8 evaluate_options: - repeat: - count: 3 - strategy: pass_any - early_exit: false + repeat: 3 budget_usd: 2.00 tests: @@ -122,7 +122,7 @@ The old experiment runtime fields are ported into the parent eval file: - target or target matrix - thresholds -- repeated run count through `evaluate_options.repeat.count` +- repeated sample count through numeric `evaluate_options.repeat` - timeout - budget - other run-time controls that do not define the task itself @@ -166,8 +166,8 @@ must stay with the lifecycle surface that actually owns that work: discovery files, provider-specific config, and target-specific harness setup belong here. - Top-level `target` selects the system under test. Top-level `policy` selects - runtime and gating controls: repeat strategy, threshold, timeout, budget, and - early-exit behavior. Workspace lifetime stays under + runtime and gating controls such as repeat count, threshold, timeout, and + budget. Workspace lifetime stays under `workspace.scope`, and Docker/container binding stays under `workspace.docker`. diff --git a/docs/adr/0012-finalize-run-artifact-layout.md b/docs/adr/0012-finalize-run-artifact-layout.md index 98620cb1a..5b4f5dac3 100644 --- a/docs/adr/0012-finalize-run-artifact-layout.md +++ b/docs/adr/0012-finalize-run-artifact-layout.md @@ -38,10 +38,10 @@ summary should carry that metadata explicitly. Artifact-format v2 phase 1 removes that path dependency. The active schema direction also treats `experiment` as a string metadata/run-grouping label, not -as an object wrapper for runtime policy. Runtime fields such as `target`, -`runs`, `early_exit`, `timeout_seconds`, `budget_usd`, and `threshold` belong at -the eval root or target object as the schema defines them; this ADR does not -duplicate that schema migration. +as an object wrapper for runtime policy. Runtime fields such as provider +selection, numeric repeat count, `timeout_seconds`, `budget_usd`, and +`threshold` belong at the eval root or target object as the schema defines them; +this ADR does not duplicate that schema migration. ## Decision @@ -163,9 +163,9 @@ local namespaces for rebuildable state. ## Non-Goals - Flattening or renaming per-case `run-N/` attempt folders. -- Completing the schema-v2 repeat naming migration. User-facing docs should - prefer `pass_any` and `pass_all` when they mention repeat strategies, but that - schema migration is tracked separately. +- Completing the schema-v2 repeat naming migration beyond `sample-N/` folders. + Current public authoring uses numeric `evaluate_options.repeat` only; fatal + post-run aggregate policy is future work. - Moving Dashboard/search indexes into a committed run bundle. - Projecting AgentV-owned runs, transcripts, datasets, experiments, or indexes into Phoenix. diff --git a/docs/adr/0013-stabilize-eval-authoring-contract.md b/docs/adr/0013-stabilize-eval-authoring-contract.md index 0fbc601c0..7aecc2d7f 100644 --- a/docs/adr/0013-stabilize-eval-authoring-contract.md +++ b/docs/adr/0013-stabilize-eval-authoring-contract.md @@ -5,6 +5,8 @@ Date: 2026-07-01 ## Status Accepted, then **superseded** (eval-authoring portions) by [ADR 0016](0016-promptfoo-superset-eval-authoring-contract.md) as part of the promptfoo-superset restructure (2026-07-02). +The repeat object shown below is also superseded by Bead `av-s96i`; current +public authoring uses Promptfoo-style numeric `evaluate_options.repeat` only. Supersedes the eval-authoring placement portions of [ADR 0002](0002-keep-harbor-benchmark-execution-behind-runner-boundary.md), @@ -56,10 +58,7 @@ name: code-generation-quality experiment: with-skills target: copilot-sdk evaluate_options: - repeat: - count: 3 - strategy: pass_any - early_exit: false + repeat: 3 default_test: threshold: 0.8 gate: diff --git a/docs/adr/0016-promptfoo-superset-eval-authoring-contract.md b/docs/adr/0016-promptfoo-superset-eval-authoring-contract.md index bd4c0d07e..bff321205 100644 --- a/docs/adr/0016-promptfoo-superset-eval-authoring-contract.md +++ b/docs/adr/0016-promptfoo-superset-eval-authoring-contract.md @@ -47,6 +47,11 @@ Promptfoo are limited to `environment`, AgentV refs, and built-in AgentV providers; full Promptfoo compatibility for those differences is by `agentv export promptfoo`. +Status note (2026-07-07): Bead `av-s96i` supersedes this ADR's earlier repeat +object decision. Public repeat authoring now follows Promptfoo's numeric +`evaluate_options.repeat: N` shape. Fatal post-run aggregate policy is future +work, not part of the repeat authoring surface. + ## Context AgentV's eval-authoring surface diverged from industry primitives. We are re-basing @@ -126,8 +131,7 @@ keep AgentV's only where its semantics are genuinely better.** unless the author separately places it in `vars`). A specific grader may use it as a strict target, semantic reference, structured expected object, or supporting context, but the field itself is not an active assertion. - `repeat: { count, strategy, early_exit }` (map promptfoo - `repeat:int` → `count`+`pass_all`); executable `gate` release policy (alongside per-test + numeric `repeat` sample counts; executable `gate` release policy (alongside per-test `threshold`); `imports`/`select`; `depends_on`. `experiment` is authored as `tags.experiment` — a plain tag with **no structural privilege** (not a bucket/field/storage path; not a privileged grouping key; tags alphabetical; default compare key is a user preference). `--experiment X` = sugar for `--tag experiment=X`. Its **value** is auto-defaulted to the eval/suite name when unset so runs are always groupable (ADR-0009 derivation) — a default value, not a privileged key (ADR-0017). 10. **Coding-agent testbed setup is a declarative `environment`, not a lifecycle extension and not target identity.** AgentV remains diff --git a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md index e8f446f36..b772986ab 100644 --- a/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md +++ b/docs/plans/2026-06-23-001-feat-repeat-runs-flaky-evals-plan.md @@ -13,6 +13,12 @@ child_beads: # feat: Add repeat runs and flaky eval handling +Supersession note (2026-07-07): Bead `av-s96i` supersedes this plan's public +repeat policy authoring. Current public docs and examples must use +Promptfoo-style numeric `evaluate_options.repeat: N` only. The strategy, +early-exit, cost-limit, and fatal aggregate gate-policy surfaces described here +are historical context or future work, not current public authoring guidance. + ## Summary AgentV should make repeat runs a first-class reliability primitive for stochastic model behavior, flaky verifier or infrastructure outcomes, benchmark reporting, and drift analysis. The default CI path stays simple: one run per case, threshold/pass-fail gating as today. When users opt into repeated attempts, AgentV records every attempt, reports aggregate reliability statistics, and changes CI behavior only when an explicit repeat-run gate policy is configured. diff --git a/docs/plans/promptfoo-aligned-eval-restructure.md b/docs/plans/promptfoo-aligned-eval-restructure.md index 2e0f58905..c3e89a2bc 100644 --- a/docs/plans/promptfoo-aligned-eval-restructure.md +++ b/docs/plans/promptfoo-aligned-eval-restructure.md @@ -18,6 +18,12 @@ language is also stale unless it refers to an explicit sampling metric with a real `k`; use `pass_rate`, `pass_count`, `sample_count`, and `passed`/`pass_any` otherwise. +Supersession note (2026-07-07): Bead `av-s96i` supersedes this plan's decision +to prefer AgentV's repeat object over Promptfoo's integer repeat. Current public +authoring uses numeric `evaluate_options.repeat: N` only. Strategy, early-exit, +cost-limit, and fatal aggregate policy surfaces are historical or future-work +discussion, not current implementation guidance. + Sources analyzed (all cloned locally, read-only): - promptfoo v0.121.17 — `/home/christso/projects/promptfoo-clone` (authoring format — the thing we clone) - Margin-Lab/evals — `/home/christso/projects/margin-lab-evals` (runner, I/O contracts, workspace, analytics) diff --git a/examples/features/README.md b/examples/features/README.md index 3c68c7bfc..ed4d7159a 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -83,7 +83,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | Example | Description | |---------|-------------| | [benchmark-tooling](benchmark-tooling/) | N-way benchmarking with `agentv results compare` over completed runs | -| [trials](trials/) | Configure repeated attempts with `evaluate_options.repeat` | +| [trials](trials/) | Configure repeated samples with `evaluate_options.repeat` | | [trial-output-consistency](trial-output-consistency/) | Measure output consistency across trials using pairwise cosine similarity | | [compare](compare/) | Compare a run against a stored baseline | diff --git a/examples/features/trials/README.md b/examples/features/trials/README.md index 0f541073f..22a2e0760 100644 --- a/examples/features/trials/README.md +++ b/examples/features/trials/README.md @@ -14,14 +14,11 @@ samples. bun agentv eval examples/features/trials/evals/suite.yaml ``` -Edit `evaluate_options.repeat.count` to change how many samples AgentV records -for each case: +Edit `evaluate_options.repeat` to change how many samples AgentV records for +each case: ```yaml evaluate_options: - repeat: - count: 2 - strategy: pass_any - early_exit: false + repeat: 2 budget_usd: 1.00 ``` diff --git a/examples/features/trials/evals/suite.yaml b/examples/features/trials/evals/suite.yaml index ee9bf8fee..cb3ad2b74 100644 --- a/examples/features/trials/evals/suite.yaml +++ b/examples/features/trials/evals/suite.yaml @@ -3,10 +3,7 @@ description: Repeat runs example with 2 samples configured inline providers: - llm evaluate_options: - repeat: - count: 2 - strategy: pass_any - early_exit: false + repeat: 2 budget_usd: 1 prompts: - "{{ input }}" From 14a60e9cadcef6a289d823df8da32469fabb14ad Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 14:11:08 +0200 Subject: [PATCH 3/3] chore(eval): sync numeric repeat schema --- .../references/eval.schema.json | 118 ++---------------- 1 file changed, 10 insertions(+), 108 deletions(-) diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index d7289fe5b..526d83ea2 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -1508,26 +1508,8 @@ "maximum": 1 }, "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false + "type": "integer", + "minimum": 1 }, "timeout_seconds": { "type": "number", @@ -1961,26 +1943,8 @@ "maximum": 1 }, "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false + "type": "integer", + "minimum": 1 }, "timeout_seconds": { "type": "number", @@ -3186,34 +3150,8 @@ "type": "boolean" }, "repeat": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false - } - ] + "type": "integer", + "minimum": 1 }, "timeout_ms": { "type": "number", @@ -5977,26 +5915,8 @@ "maximum": 1 }, "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false + "type": "integer", + "minimum": 1 }, "timeout_seconds": { "type": "number", @@ -7627,26 +7547,8 @@ "maximum": 1 }, "repeat": { - "type": "object", - "properties": { - "count": { - "type": "integer", - "minimum": 1 - }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] - }, - "early_exit": { - "type": "boolean" - }, - "cost_limit_usd": { - "type": "number", - "minimum": 0 - } - }, - "required": ["count"], - "additionalProperties": false + "type": "integer", + "minimum": 1 }, "timeout_seconds": { "type": "number",