diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx index 8fe6335c8..dcdb95114 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx @@ -12,7 +12,7 @@ Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. The reserved `tags.experiment` key is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, -`evaluate_options.budget_usd`, and `execution.max_concurrency` control repeated +`evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated attempts and gates. Workspace lifetime belongs under `workspace.scope`; repository provenance belongs under `workspace.repos`; Docker/container binding belongs under `workspace.docker`. Non-provisioning setup commands belong in @@ -23,7 +23,7 @@ data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. Eval files describe the task, target binding, and run controls. Use -`execution.max_concurrency` for authored suite concurrency. Operators can still +`evaluate_options.max_concurrency` for authored suite concurrency. Operators can still override concurrency with `agentv eval --workers N`; do not author legacy `workers` fields in eval YAML. @@ -83,10 +83,8 @@ A wrapper eval stays ordinary eval YAML while choosing a target and run controls # experiments/refunds-codex.eval.yaml name: refunds-codex target: codex-gpt5 -execution: - max_concurrency: 3 - evaluate_options: + max_concurrency: 3 repeat: count: 2 strategy: pass_any @@ -144,8 +142,7 @@ tests: | `targets` | Optional target matrix. Entries reference target ids or inline target objects. | | `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` | | `timeout_seconds` | Optional per-case timeout | -| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `repeat` | -| `execution.max_concurrency` | Optional general eval parallelism for this suite | +| `evaluate_options` | Optional evaluation runtime options such as `budget_usd`, `repeat`, and `max_concurrency` | | `threshold` | Optional suite quality threshold | | `workspace` | Suite-level task environment — inline object or string path to an external workspace file. Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | | `extensions` | Promptfoo-style lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. Hooks run after `workspace.repos` materializes. | diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx index 64504d34a..3ef800c34 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx @@ -524,9 +524,9 @@ Do not wrap referenced field files in another object. For example, `targets: file://targets.yaml` expects `targets.yaml` to contain a bare array, not `{ targets: [...] }`. -`execution.max_concurrency` is AgentV's general eval parallelism field for this -config graph. It is AgentV's run-policy shape, aligned with the general -max-concurrency concept in eval runners, not a copied Promptfoo YAML path. +In authored eval YAML, use `evaluate_options.max_concurrency` for suite +parallelism. In `.agentv/config.yaml`, operators can set +`execution.max_concurrency` as project-level default run policy. Other project defaults can live beside the graph: diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index 60e76ef33..ca5f8e44d 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -404,7 +404,7 @@ function rejectExperimentWorkers(raw: unknown): void { return; } throw new Error( - 'Experiment workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.', + 'Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.', ); } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 83f01b861..28e8cf7d9 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -359,33 +359,27 @@ function rejectAuthoredRuntimeContainers(suite: JsonObject): void { throw new Error("Top-level 'budget_usd' has been removed. Use evaluate_options.budget_usd."); } if (suite.execution !== undefined) { - assertAllowedSuiteExecution(suite.execution); + rejectAuthoredSuiteExecution(suite.execution); } } -function assertAllowedSuiteExecution(rawExecution: JsonValue): void { +function rejectAuthoredSuiteExecution(rawExecution: JsonValue): void { if (!isJsonObject(rawExecution)) { throw new Error("Invalid top-level 'execution': expected an object."); } for (const key of Object.keys(rawExecution)) { - if (key !== 'max_concurrency') { + if (key === 'max_concurrency') { throw new Error( - `Top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for AgentV eval parallelism; keep target and other run controls at their supported top-level or evaluate_options fields.`, + "Top-level 'execution.max_concurrency' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.", ); } - } - const maxConcurrency = rawExecution.max_concurrency; - if ( - maxConcurrency !== undefined && - (typeof maxConcurrency !== 'number' || - !Number.isInteger(maxConcurrency) || - maxConcurrency < 1 || - maxConcurrency > 50) - ) { throw new Error( - "Invalid top-level 'execution.max_concurrency': expected an integer between 1 and 50.", + `Top-level 'execution.${key}' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`, ); } + throw new Error( + "Top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.", + ); } function getSuiteTopLevelNumber( @@ -604,15 +598,12 @@ export function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined { /** * Extract suite-level max concurrency from eval YAML. * - * AgentV eval YAML accepts execution.max_concurrency as the config-graph field - * and evaluate_options.max_concurrency for promptfoo-shaped eval options. The - * runner still receives the resolved value through its historical workers slot. + * AgentV eval YAML accepts promptfoo-shaped evaluate_options.max_concurrency. + * The runner still receives the resolved value through its historical workers + * slot. */ export function extractWorkersFromSuite(suite: JsonObject): number | undefined { rejectAuthoredRuntimeContainers(suite); - if (isJsonObject(suite.execution) && typeof suite.execution.max_concurrency === 'number') { - return suite.execution.max_concurrency; - } return getSuiteEvaluateOptionsNumber( suite, 'max_concurrency', diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3da85a2f0..ef57ce676 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -679,12 +679,6 @@ const ConfigDefaultsSchema = z }) .strict(); -const ConfigExecutionSchema = z - .object({ - max_concurrency: z.number().int().min(1).max(50).optional(), - }) - .strict(); - const ScenarioConfigSchema = z .object({ vars: JsonObjectSchema.optional(), @@ -775,7 +769,7 @@ export const EvalFileSchema: z.ZodType = z extensions: z.array(ExtensionSchema).optional(), on_run_complete: z.never().optional(), policy: z.never().optional(), - execution: z.union([ConfigExecutionSchema, z.string().min(1)]).optional(), + execution: z.never().optional(), // Suite-level assert entries assert: z.array(AssertionItemSchema).optional(), // Suite-level content preprocessors shared by evaluators diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 06c4a9cdf..0e20985fc 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -168,7 +168,7 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ const REMOVED_TOP_LEVEL_FIELDS = new Map([ [ 'workers', - "'workers' has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.", + "'workers' has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.", ], ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."], [ @@ -658,7 +658,7 @@ function validateTestExecutionFields( filePath, location: `${location}.execution.workers`, message: - 'tests[].execution.workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.', + 'tests[].execution.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.', }); continue; } @@ -708,29 +708,33 @@ function validateExecutionPolicy( }); return; } - for (const key of Object.keys(execution)) { - if (key !== 'max_concurrency') { + const keys = Object.keys(execution); + if (keys.length === 0) { + errors.push({ + severity: 'error', + filePath, + location, + message: + "Top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.", + }); + return; + } + for (const key of keys) { + if (key === 'max_concurrency') { errors.push({ severity: 'error', filePath, location: `${location}.${key}`, - message: `Unsupported execution field '${key}'. Use execution.max_concurrency for eval parallelism.`, + message: + "Top-level 'execution.max_concurrency' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.", }); + continue; } - } - const maxConcurrency = execution.max_concurrency; - if ( - maxConcurrency !== undefined && - (typeof maxConcurrency !== 'number' || - !Number.isInteger(maxConcurrency) || - maxConcurrency < 1 || - maxConcurrency > 50) - ) { errors.push({ severity: 'error', filePath, - location: `${location}.max_concurrency`, - message: "Invalid 'execution.max_concurrency' field (must be an integer between 1 and 50)", + location: `${location}.${key}`, + message: `Unsupported execution field '${key}'. Use supported top-level fields or evaluate_options for authored run controls.`, }); } } @@ -749,7 +753,7 @@ function rejectWorkersField( severity: 'error', filePath, location: `${location}.workers`, - message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, + message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`, }); } rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors); @@ -772,7 +776,7 @@ function rejectTargetWorkers( severity: 'error', filePath, location: `${location}[${index}].workers`, - message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, + message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`, }); }); } diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 47fd27972..487133083 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -454,7 +454,7 @@ export async function prepareSharedWorkspaceSetup( [ `Warning: This eval uses a shared workspace with ${workers} workers.`, 'If the agent under test makes file edits, concurrent runs may corrupt each other.', - 'To limit concurrency, pass --workers 1 on the command line or set execution.max_concurrency in eval YAML or .agentv/config.yaml.', + 'To limit concurrency, pass --workers 1 on the command line, set evaluate_options.max_concurrency in eval YAML, or set execution.max_concurrency in .agentv/config.yaml.', ].join('\n'), ); } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 2a83a721c..f526225d3 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -1624,7 +1624,7 @@ function rejectAuthoredWorkers(parsed: JsonObject): void { } throw new Error( - `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, + `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`, ); } @@ -2184,16 +2184,22 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO if (suite.execution !== undefined) { if (!isJsonObject(suite.execution)) { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' must be an object with max_concurrency.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`, ); } for (const key of Object.keys(suite.execution)) { - if (key !== 'max_concurrency') { + if (key === 'max_concurrency') { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for eval parallelism.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.max_concurrency' has been removed. Use evaluate_options.max_concurrency for authored suite concurrency.`, ); } + throw new Error( + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.${key}' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`, + ); } + throw new Error( + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`, + ); } if (suite.providers !== undefined) { throw new Error( diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 7f22de031..e5ac5cdda 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -990,9 +990,9 @@ describe('extractWorkersFromSuite', () => { expect(extractWorkersFromSuite(suite)).toBe(5); }); - it('parses valid execution.max_concurrency', () => { + it('rejects authored execution.max_concurrency', () => { const suite: JsonObject = { execution: { max_concurrency: 3 } }; - expect(extractWorkersFromSuite(suite)).toBe(3); + expect(() => extractWorkersFromSuite(suite)).toThrow(/evaluate_options\.max_concurrency/); }); it('returns undefined for invalid max_concurrency', () => { diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 162f0c20d..79c929cf2 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -50,7 +50,7 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); - it('accepts eval-level execution.max_concurrency', () => { + it('rejects eval-level execution.max_concurrency', () => { const result = EvalFileSchema.safeParse({ execution: { max_concurrency: 2, @@ -58,7 +58,7 @@ describe('EvalFileSchema input shorthand', () => { tests: [baseTest], }); - expect(result.success).toBe(true); + expect(result.success).toBe(false); }); it('rejects removed eval-level execution fields', () => { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index c2d39a5d8..9f467bc1f 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -104,7 +104,7 @@ tests: ).toBe(true); }); - it('validates composable execution.max_concurrency and defaults in eval YAML', async () => { + it('rejects authored execution.max_concurrency in eval YAML', async () => { const filePath = path.join(tempDir, 'composable-eval-graph.yaml'); await writeFile( filePath, @@ -132,8 +132,14 @@ tests: const result = await validateEvalFile(filePath); - expect(result.valid).toBe(true); - expect(result.errors).toHaveLength(0); + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'execution.max_concurrency', + message: expect.stringContaining('evaluate_options.max_concurrency'), + }), + ); }); it('rejects removed top-level execution fields in eval YAML', async () => { @@ -163,7 +169,7 @@ tests: expect.objectContaining({ severity: 'error', location: 'execution.workers', - message: expect.stringContaining("Unsupported execution field 'workers'"), + message: expect.stringContaining('authored run controls'), }), ); }); diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index 0f0f697d6..b918f347f 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -36,9 +36,9 @@ For a v4.42.4-era eval: `evaluate_options.repeat`. 8. Move suite budget from `execution.budget_usd` to `evaluate_options.budget_usd`. -9. Move authored suite concurrency from `execution.workers` to - `evaluate_options.max_concurrency`, or leave it to `--workers` / - project config if it is operator policy. +9. Move authored suite concurrency from `execution.workers` or + `execution.max_concurrency` to `evaluate_options.max_concurrency`, or leave + it to `--workers` / project config if it is operator policy. 10. Remove top-level `execution`; current eval YAML rejects it. 11. Replace `workspace.isolation: shared|per_test` with `workspace.scope: suite|attempt`. @@ -247,10 +247,10 @@ assert: - `execution.targets` -> top-level `targets`. - `execution.threshold` -> top-level `threshold`. - `execution.budget_usd` -> `evaluate_options.budget_usd`. -- `execution.workers` -> `evaluate_options.max_concurrency` when authored - suite concurrency is part of the eval. If it is operator policy, use - `--workers` or `.agentv/config.yaml` / `agentv.config.*` `execution.workers` - instead. +- `execution.workers` and `execution.max_concurrency` -> + `evaluate_options.max_concurrency` when authored suite concurrency is part of + the eval. If it is operator policy, use `--workers` or `.agentv/config.yaml` + / `agentv.config.*` `execution.max_concurrency` instead. - `execution.fail_on_error` has no current eval-YAML home. Treat it as operational policy; do not commit it into migrated eval YAML. - `execution.cache` has no current eval-YAML home. Use project/operator config diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 831284516..03c7e3df6 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -10781,23 +10781,7 @@ "not": {} }, "execution": { - "anyOf": [ - { - "type": "object", - "properties": { - "max_concurrency": { - "type": "integer", - "minimum": 1, - "maximum": 50 - } - }, - "additionalProperties": false - }, - { - "type": "string", - "minLength": 1 - } - ] + "not": {} }, "assert": { "type": "array",