fix(eval): reject authored execution concurrency

apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx

@@ -12,7 +12,7 @@ Evaluation files define the test cases, graders, workspace lifecycle, and run
 controls for an evaluation run. The reserved `tags.experiment` key is the
 run/result grouping label, top-level `target` identifies the system under test,
 and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`,
-`evaluate_options.budget_usd`, and `execution.max_concurrency` control repeated
+`evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated
 attempts and gates. Workspace lifetime belongs under `workspace.scope`;
 repository provenance belongs under `workspace.repos`; Docker/container binding
 belongs under `workspace.docker`. Non-provisioning setup commands belong in
@@ -23,7 +23,7 @@ data formats: YAML and JSONL.
 
 YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract.
 Eval files describe the task, target binding, and run controls. Use
-`execution.max_concurrency` for authored suite concurrency. Operators can still
+`evaluate_options.max_concurrency` for authored suite concurrency. Operators can still
 override concurrency with `agentv eval --workers N`; do not author legacy
 `workers` fields in eval YAML.
 
@@ -83,10 +83,8 @@ A wrapper eval stays ordinary eval YAML while choosing a target and run controls
 # experiments/refunds-codex.eval.yaml
 name: refunds-codex
 target: codex-gpt5
-execution:
-  max_concurrency: 3
-
 evaluate_options:
+  max_concurrency: 3
   repeat:
     count: 2
     strategy: pass_any
@@ -144,8 +142,7 @@ tests:
 | `targets` | Optional target matrix. Entries reference target ids or inline target objects. |
 | `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` |
 | `timeout_seconds` | Optional per-case timeout |
-| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `repeat` |
-| `execution.max_concurrency` | Optional general eval parallelism for this suite |
+| `evaluate_options` | Optional evaluation runtime options such as `budget_usd`, `repeat`, and `max_concurrency` |
 | `threshold` | Optional suite quality threshold |
 | `workspace` | Suite-level task environment — inline object or string path to an external workspace file. Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). |
 | `extensions` | Promptfoo-style lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. Hooks run after `workspace.repos` materializes. |

apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx

@@ -524,9 +524,9 @@ Do not wrap referenced field files in another object. For example,
 `targets: file://targets.yaml` expects `targets.yaml` to contain a bare array,
 not `{ targets: [...] }`.
 
-`execution.max_concurrency` is AgentV's general eval parallelism field for this
-config graph. It is AgentV's run-policy shape, aligned with the general
-max-concurrency concept in eval runners, not a copied Promptfoo YAML path.
+In authored eval YAML, use `evaluate_options.max_concurrency` for suite
+parallelism. In `.agentv/config.yaml`, operators can set
+`execution.max_concurrency` as project-level default run policy.
 
 Other project defaults can live beside the graph:
 

packages/core/src/evaluation/experiment.ts

@@ -404,7 +404,7 @@ function rejectExperimentWorkers(raw: unknown): void {
     return;
   }
   throw new Error(
-    'Experiment workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.',
+    'Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.',
   );
 }
 

packages/core/src/evaluation/loaders/config-loader.ts

@@ -359,33 +359,27 @@ function rejectAuthoredRuntimeContainers(suite: JsonObject): void {
     throw new Error("Top-level 'budget_usd' has been removed. Use evaluate_options.budget_usd.");
   }
   if (suite.execution !== undefined) {
-    assertAllowedSuiteExecution(suite.execution);
+    rejectAuthoredSuiteExecution(suite.execution);
   }
 }
 
-function assertAllowedSuiteExecution(rawExecution: JsonValue): void {
+function rejectAuthoredSuiteExecution(rawExecution: JsonValue): void {
   if (!isJsonObject(rawExecution)) {
     throw new Error("Invalid top-level 'execution': expected an object.");
   }
   for (const key of Object.keys(rawExecution)) {
-    if (key !== 'max_concurrency') {
+    if (key === 'max_concurrency') {
       throw new Error(
-        `Top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for AgentV eval parallelism; keep target and other run controls at their supported top-level or evaluate_options fields.`,
+        "Top-level 'execution.max_concurrency' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.",
       );
     }
-  }
-  const maxConcurrency = rawExecution.max_concurrency;
-  if (
-    maxConcurrency !== undefined &&
-    (typeof maxConcurrency !== 'number' ||
-      !Number.isInteger(maxConcurrency) ||
-      maxConcurrency < 1 ||
-      maxConcurrency > 50)
-  ) {
     throw new Error(
-      "Invalid top-level 'execution.max_concurrency': expected an integer between 1 and 50.",
+      `Top-level 'execution.${key}' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`,
     );
   }
+  throw new Error(
+    "Top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.",
+  );
 }
 
 function getSuiteTopLevelNumber(
@@ -604,15 +598,12 @@ export function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined {
 /**
  * Extract suite-level max concurrency from eval YAML.
  *
- * AgentV eval YAML accepts execution.max_concurrency as the config-graph field
- * and evaluate_options.max_concurrency for promptfoo-shaped eval options. The
- * runner still receives the resolved value through its historical workers slot.
+ * AgentV eval YAML accepts promptfoo-shaped evaluate_options.max_concurrency.
+ * The runner still receives the resolved value through its historical workers
+ * slot.
  */
 export function extractWorkersFromSuite(suite: JsonObject): number | undefined {
   rejectAuthoredRuntimeContainers(suite);
-  if (isJsonObject(suite.execution) && typeof suite.execution.max_concurrency === 'number') {
-    return suite.execution.max_concurrency;
-  }
   return getSuiteEvaluateOptionsNumber(
     suite,
     'max_concurrency',

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -679,12 +679,6 @@ const ConfigDefaultsSchema = z
   })
   .strict();
 
-const ConfigExecutionSchema = z
-  .object({
-    max_concurrency: z.number().int().min(1).max(50).optional(),
-  })
-  .strict();
-
 const ScenarioConfigSchema = z
   .object({
     vars: JsonObjectSchema.optional(),
@@ -775,7 +769,7 @@ export const EvalFileSchema: z.ZodType = z
     extensions: z.array(ExtensionSchema).optional(),
     on_run_complete: z.never().optional(),
     policy: z.never().optional(),
-    execution: z.union([ConfigExecutionSchema, z.string().min(1)]).optional(),
+    execution: z.never().optional(),
     // Suite-level assert entries
     assert: z.array(AssertionItemSchema).optional(),
     // Suite-level content preprocessors shared by evaluators

packages/core/src/evaluation/validation/eval-validator.ts

@@ -168,7 +168,7 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([
 const REMOVED_TOP_LEVEL_FIELDS = new Map<string, string>([
   [
     'workers',
-    "'workers' has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.",
+    "'workers' has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.",
   ],
   ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."],
   [
@@ -658,7 +658,7 @@ function validateTestExecutionFields(
         filePath,
         location: `${location}.execution.workers`,
         message:
-          'tests[].execution.workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.',
+          'tests[].execution.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.',
       });
       continue;
     }
@@ -708,29 +708,33 @@ function validateExecutionPolicy(
     });
     return;
   }
-  for (const key of Object.keys(execution)) {
-    if (key !== 'max_concurrency') {
+  const keys = Object.keys(execution);
+  if (keys.length === 0) {
+    errors.push({
+      severity: 'error',
+      filePath,
+      location,
+      message:
+        "Top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.",
+    });
+    return;
+  }
+  for (const key of keys) {
+    if (key === 'max_concurrency') {
       errors.push({
         severity: 'error',
         filePath,
         location: `${location}.${key}`,
-        message: `Unsupported execution field '${key}'. Use execution.max_concurrency for eval parallelism.`,
+        message:
+          "Top-level 'execution.max_concurrency' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.",
       });
+      continue;
     }
-  }
-  const maxConcurrency = execution.max_concurrency;
-  if (
-    maxConcurrency !== undefined &&
-    (typeof maxConcurrency !== 'number' ||
-      !Number.isInteger(maxConcurrency) ||
-      maxConcurrency < 1 ||
-      maxConcurrency > 50)
-  ) {
     errors.push({
       severity: 'error',
       filePath,
-      location: `${location}.max_concurrency`,
-      message: "Invalid 'execution.max_concurrency' field (must be an integer between 1 and 50)",
+      location: `${location}.${key}`,
+      message: `Unsupported execution field '${key}'. Use supported top-level fields or evaluate_options for authored run controls.`,
     });
   }
 }
@@ -749,7 +753,7 @@ function rejectWorkersField(
       severity: 'error',
       filePath,
       location: `${location}.workers`,
-      message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`,
+      message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`,
     });
   }
   rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors);
@@ -772,7 +776,7 @@ function rejectTargetWorkers(
       severity: 'error',
       filePath,
       location: `${location}[${index}].workers`,
-      message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`,
+      message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`,
     });
   });
 }

packages/core/src/evaluation/workspace/setup.ts

@@ -454,7 +454,7 @@ export async function prepareSharedWorkspaceSetup(
       [
         `Warning: This eval uses a shared workspace with ${workers} workers.`,
         'If the agent under test makes file edits, concurrent runs may corrupt each other.',
-        'To limit concurrency, pass --workers 1 on the command line or set execution.max_concurrency in eval YAML or .agentv/config.yaml.',
+        'To limit concurrency, pass --workers 1 on the command line, set evaluate_options.max_concurrency in eval YAML, or set execution.max_concurrency in .agentv/config.yaml.',
       ].join('\n'),
     );
   }

packages/core/src/evaluation/yaml-parser.ts

@@ -1624,7 +1624,7 @@ function rejectAuthoredWorkers(parsed: JsonObject): void {
   }
 
   throw new Error(
-    `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`,
+    `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.`,
   );
 }
 
@@ -2184,16 +2184,22 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO
   if (suite.execution !== undefined) {
     if (!isJsonObject(suite.execution)) {
       throw new Error(
-        `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' must be an object with max_concurrency.`,
+        `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`,
       );
     }
     for (const key of Object.keys(suite.execution)) {
-      if (key !== 'max_concurrency') {
+      if (key === 'max_concurrency') {
         throw new Error(
-          `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for eval parallelism.`,
+          `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.max_concurrency' has been removed. Use evaluate_options.max_concurrency for authored suite concurrency.`,
         );
       }
+      throw new Error(
+        `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.${key}' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`,
+      );
     }
+    throw new Error(
+      `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`,
+    );
   }
   if (suite.providers !== undefined) {
     throw new Error(

packages/core/test/evaluation/loaders/config-loader.test.ts

@@ -990,9 +990,9 @@ describe('extractWorkersFromSuite', () => {
     expect(extractWorkersFromSuite(suite)).toBe(5);
   });
 
-  it('parses valid execution.max_concurrency', () => {
+  it('rejects authored execution.max_concurrency', () => {
     const suite: JsonObject = { execution: { max_concurrency: 3 } };
-    expect(extractWorkersFromSuite(suite)).toBe(3);
+    expect(() => extractWorkersFromSuite(suite)).toThrow(/evaluate_options\.max_concurrency/);
   });
 
   it('returns undefined for invalid max_concurrency', () => {

packages/core/test/evaluation/validation/eval-file-schema.test.ts

@@ -50,15 +50,15 @@ describe('EvalFileSchema input shorthand', () => {
     expect(result.success).toBe(false);
   });
 
-  it('accepts eval-level execution.max_concurrency', () => {
+  it('rejects eval-level execution.max_concurrency', () => {
     const result = EvalFileSchema.safeParse({
       execution: {
         max_concurrency: 2,
       },
       tests: [baseTest],
     });
 
-    expect(result.success).toBe(true);
+    expect(result.success).toBe(false);
   });
 
   it('rejects removed eval-level execution fields', () => {

packages/core/test/evaluation/validation/eval-validator.test.ts

@@ -104,7 +104,7 @@ tests:
     ).toBe(true);
   });
 
-  it('validates composable execution.max_concurrency and defaults in eval YAML', async () => {
+  it('rejects authored execution.max_concurrency in eval YAML', async () => {
     const filePath = path.join(tempDir, 'composable-eval-graph.yaml');
     await writeFile(
       filePath,
@@ -132,8 +132,14 @@ tests:
 
     const result = await validateEvalFile(filePath);
 
-    expect(result.valid).toBe(true);
-    expect(result.errors).toHaveLength(0);
+    expect(result.valid).toBe(false);
+    expect(result.errors).toContainEqual(
+      expect.objectContaining({
+        severity: 'error',
+        location: 'execution.max_concurrency',
+        message: expect.stringContaining('evaluate_options.max_concurrency'),
+      }),
+    );
   });
 
   it('rejects removed top-level execution fields in eval YAML', async () => {
@@ -163,7 +169,7 @@ tests:
       expect.objectContaining({
         severity: 'error',
         location: 'execution.workers',
-        message: expect.stringContaining("Unsupported execution field 'workers'"),
+        message: expect.stringContaining('authored run controls'),
       }),
     );
   });

skills-data/agentv-eval-migrations/references/breaking-changes.md

@@ -36,9 +36,9 @@ For a v4.42.4-era eval:
    `evaluate_options.repeat`.
 8. Move suite budget from `execution.budget_usd` to
    `evaluate_options.budget_usd`.
-9. Move authored suite concurrency from `execution.workers` to
-   `evaluate_options.max_concurrency`, or leave it to `--workers` /
-   project config if it is operator policy.
+9. Move authored suite concurrency from `execution.workers` or
+   `execution.max_concurrency` to `evaluate_options.max_concurrency`, or leave
+   it to `--workers` / project config if it is operator policy.
 10. Remove top-level `execution`; current eval YAML rejects it.
 11. Replace `workspace.isolation: shared|per_test` with
     `workspace.scope: suite|attempt`.
@@ -247,10 +247,10 @@ assert:
 - `execution.targets` -> top-level `targets`.
 - `execution.threshold` -> top-level `threshold`.
 - `execution.budget_usd` -> `evaluate_options.budget_usd`.
-- `execution.workers` -> `evaluate_options.max_concurrency` when authored
-  suite concurrency is part of the eval. If it is operator policy, use
-  `--workers` or `.agentv/config.yaml` / `agentv.config.*` `execution.workers`
-  instead.
+- `execution.workers` and `execution.max_concurrency` ->
+  `evaluate_options.max_concurrency` when authored suite concurrency is part of
+  the eval. If it is operator policy, use `--workers` or `.agentv/config.yaml`
+  / `agentv.config.*` `execution.max_concurrency` instead.
 - `execution.fail_on_error` has no current eval-YAML home. Treat it as
   operational policy; do not commit it into migrated eval YAML.
 - `execution.cache` has no current eval-YAML home. Use project/operator config

skills-data/agentv-eval-writer/references/eval.schema.json

@@ -10781,23 +10781,7 @@
           "not": {}
         },
         "execution": {
-          "anyOf": [
-            {
-              "type": "object",
-              "properties": {
-                "max_concurrency": {
-                  "type": "integer",
-                  "minimum": 1,
-                  "maximum": 50
-                }
-              },
-              "additionalProperties": false
-            },
-            {
-              "type": "string",
-              "minLength": 1
-            }
-          ]
+          "not": {}
         },
         "assert": {
           "type": "array",