diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index c0f544784..49e824ce5 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -100,7 +100,6 @@ function definitionsWithEvalTargetRefs( function buildBundleExecution(options: { readonly targetNames: readonly string[]; readonly targetRefs?: readonly EvalTargetRef[]; - readonly workers?: number; readonly cache?: boolean; readonly cachePath?: string; readonly budgetUsd?: number; @@ -121,9 +120,6 @@ function buildBundleExecution(options: { targets: options.targetNames.map((name) => serializeTargetRef(name)), }; - if (options.workers !== undefined) { - execution.workers = options.workers; - } if (options.cache !== undefined) { execution.cache = options.cache; } @@ -224,7 +220,6 @@ export const evalBundleCommand = command({ execution: buildBundleExecution({ targetNames, targetRefs: suite.targetRefs, - workers: suite.workers, cache: suite.cacheConfig?.enabled, cachePath: suite.cacheConfig?.cachePath, budgetUsd: suite.budgetUsd, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index c6d4b599f..77d99201d 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -387,7 +387,7 @@ function normalizeOptions( yamlExecution?: ExecutionDefaults, ): NormalizedOptions { const cliWorkers = normalizeOptionalNumber(rawOptions.workers); - const configWorkers = config?.execution?.workers; + const configWorkers = config?.execution?.workers ?? yamlExecution?.workers; const workers = cliWorkers ?? configWorkers ?? 0; const cliOutputDir = normalizeString(rawOptions.output); @@ -759,7 +759,6 @@ function applyExperimentOptions( ...options, target: options.target, agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds, - workers: options.workers ?? experiment.workers, workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, @@ -1110,7 +1109,6 @@ async function prepareFileMetadata(params: { readonly selections: readonly { selection: TargetSelection; inlineTargetLabel: string }[]; readonly trialsConfig?: TrialsConfig; readonly suiteTargets?: readonly string[]; - readonly yamlWorkers?: number; readonly yamlCache?: boolean; readonly yamlCachePath?: string; readonly budgetUsd?: number; @@ -1160,7 +1158,6 @@ async function prepareFileMetadata(params: { selections: [], trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, - yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, @@ -1318,7 +1315,6 @@ async function prepareFileMetadata(params: { selections, trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, - yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, @@ -1360,7 +1356,6 @@ async function runSingleEvalFile(params: { readonly cache?: EvaluationCache; readonly evaluationRunner: typeof defaultRunEvaluation; readonly workersOverride?: number; - readonly yamlWorkers?: number; readonly progressReporter: ProgressReporter; readonly seenTestCases: Set; readonly displayIdTracker: { getOrAssign(testCaseKey: string): number }; @@ -1388,7 +1383,6 @@ async function runSingleEvalFile(params: { cache, evaluationRunner, workersOverride, - yamlWorkers, progressReporter, seenTestCases, displayIdTracker, @@ -1439,13 +1433,10 @@ async function runSingleEvalFile(params: { const agentTimeoutMs = agentTimeoutSeconds != null ? Math.max(0, agentTimeoutSeconds) * 1000 : undefined; - // Resolve workers: CLI flag > eval YAML execution.workers > target setting > default + // Resolve workers: CLI/config > target setting > default const workerPreference = workersOverride ?? options.workers; let resolvedWorkers = - workerPreference ?? - yamlWorkers ?? - resolvedTargetSelection.resolvedTarget.workers ?? - DEFAULT_WORKERS; + workerPreference ?? resolvedTargetSelection.resolvedTarget.workers ?? DEFAULT_WORKERS; if (resolvedWorkers < 1 || resolvedWorkers > 50) { throw new Error(`Workers must be between 1 and 50, got: ${resolvedWorkers}`); } @@ -1931,7 +1922,6 @@ export async function runEvalCommand( }[]; readonly trialsConfig?: TrialsConfig; readonly suiteTargets?: readonly string[]; - readonly yamlWorkers?: number; readonly yamlCache?: boolean; readonly yamlCachePath?: string; readonly budgetUsd?: number; @@ -2301,7 +2291,6 @@ export async function runEvalCommand( cache, evaluationRunner, workersOverride: fileOptions.workers, - yamlWorkers: targetPrep.yamlWorkers, progressReporter, seenTestCases, displayIdTracker, diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 96487ad93..5faae713b 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -616,8 +616,6 @@ describe('agentv eval CLI', () => { 'name: native-exp', 'target: codex-target', 'model: gpt-5-codex', - 'execution:', - ' workers: 4', 'policy:', ' timeout_seconds: 12', ' threshold: 0.8', @@ -639,7 +637,7 @@ describe('agentv eval CLI', () => { 'utf8', ); - const { stdout, exitCode } = await runCli(fixture, ['eval', wrapperPath]); + const { stdout, exitCode } = await runCli(fixture, ['eval', wrapperPath, '--workers', '4']); expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); @@ -669,7 +667,6 @@ describe('agentv eval CLI', () => { model: 'gpt-5-codex', runs: 2, timeout_seconds: 12, - workers: 4, }); expect( (benchmark.metadata?.experiment_config as Record).fingerprint, @@ -677,7 +674,7 @@ describe('agentv eval CLI', () => { expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', kind: 'wrapper_eval', - config_source: 'inline_experiment', + config_source: 'mixed', experiment_namespace: 'native-exp', experiment_namespace_source: 'eval_metadata', eval_files: ['native-exp.eval.yaml'], @@ -689,7 +686,7 @@ describe('agentv eval CLI', () => { } }, 30_000); - it('keeps inline runtime policy isolated across multiple eval files', async () => { + it('keeps non-concurrency runtime policy isolated across multiple eval files', async () => { const fixture = await createFixture(); try { const firstPath = path.join(fixture.suiteDir, 'first.eval.yaml'); @@ -699,8 +696,6 @@ describe('agentv eval CLI', () => { [ 'name: first', 'target: cli-target', - 'execution:', - ' workers: 1', 'policy:', ' timeout_seconds: 11', ' budget_usd: 0.11', @@ -717,8 +712,6 @@ describe('agentv eval CLI', () => { [ 'name: second', 'target: file-target', - 'execution:', - ' workers: 2', 'policy:', ' timeout_seconds: 22', ' budget_usd: 0.22', @@ -731,7 +724,13 @@ describe('agentv eval CLI', () => { 'utf8', ); - const { stdout, exitCode } = await runCli(fixture, ['eval', firstPath, secondPath]); + const { stdout, exitCode } = await runCli(fixture, [ + 'eval', + firstPath, + secondPath, + '--workers', + '2', + ]); expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); @@ -743,7 +742,7 @@ describe('agentv eval CLI', () => { expect(calls[0]).toMatchObject({ target: 'cli-target', agentTimeoutMs: 11_000, - maxConcurrency: 1, + maxConcurrency: 2, budgetUsd: 0.11, runBudgetCapUsd: 0.11, evalCaseIds: ['first-case'], diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx index 2a473dbb2..791b013c5 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx @@ -8,6 +8,7 @@ sidebar: Evaluation files define the test cases, graders, workspace lifecycle, and run policy for an evaluation run. Top-level `name` is the experiment/result namespace, top-level `target` identifies the system under test, and top-level [`policy`](/docs/evaluation/experiments/) owns thresholds, budgets, timeouts, and repeat runs. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs under `targets[].hooks`. AgentV supports two eval data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. +Eval files describe the task, target binding, and run policy. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML. ## Authoring Shapes diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 6c88d809a..dd8c19787 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -11,6 +11,9 @@ top-level `target` for the system under test, and top-level `policy` for runtime and gating controls such as run count, timeout, budgets, and thresholds. AgentV does not have a separate `experiment.yaml` file, top-level `run_group`, or schema-significant `experiments/` directory. +Concurrency is intentionally outside eval YAML. Use `agentv eval --workers N` +or project config defaults such as `agentv.config.*` / `.agentv/config.yaml` +`execution.workers` for operator-side parallelism. ```yaml name: support-regression @@ -37,7 +40,8 @@ tests: `model`, and runtime controls to top-level `policy` with `runs`, `timeout_seconds`, `threshold`, and `budget_usd`. `execution:` is accepted only as a legacy top-level runtime alias for existing -eval files and target matrices. +eval files and target matrices. It does not accept `workers`; use `--workers` +or project config for concurrency. ## Layout Conventions diff --git a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx index 1bfdcd739..338417962 100644 --- a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx @@ -29,7 +29,7 @@ Use this split when deciding where a benchmark key belongs: | `workspace.template` | Yes | Copies a workspace template into the run workspace. | | `workspace.hooks` | Yes | Runs lifecycle commands with workspace and case context on stdin. | | `workspace.isolation` | Yes | Controls shared vs per-case folder isolation. Runtime workspace paths are machine-local config/CLI bindings, not benchmark provenance. | -| `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, workers, and default grader behavior. | +| `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, and default grader behavior. Concurrency is an operator/run setting from `--workers` or project config. | | `input`, `input_files`, `expected_output` | Yes | Builds the target prompt and passive reference answer. | | `assertions` | Yes | Runs deterministic, LLM, composite, or code graders. | | Top-level `name`, `version`, `tags`, `license`, `requires` | Informational | Identifies and categorizes the suite. | diff --git a/docs/adr/0006-separate-experiments-from-eval-definitions.md b/docs/adr/0006-separate-experiments-from-eval-definitions.md index de53e15d2..abb68ba46 100644 --- a/docs/adr/0006-separate-experiments-from-eval-definitions.md +++ b/docs/adr/0006-separate-experiments-from-eval-definitions.md @@ -70,8 +70,6 @@ fields: name: cargowise-sql-migration-codex target: agent model: gpt-5-codex -execution: - workers: 4 policy: threshold: 0.8 runs: 3 @@ -109,7 +107,6 @@ already-existing eval files and target matrices. New surfaces should not teach The old experiment runtime fields are ported into the parent eval file: - target or target matrix -- workers - thresholds - repeated run count through `policy.runs` - timeout @@ -129,7 +126,8 @@ Parent-versus-child is not the main composition rule. Contract ownership is: | Task prompt | `input`, `input_files`, shared prompt defaults | Imported child suite | | Task environment | `workspace`, `workspace.repos[]`, templates, workspace hooks | Imported child suite | | Scoring | `assertions`, graders, expected references | Imported child suite | -| Run policy | `experiment`, CLI target flags, workers, repeat, gates, budget | Parent wrapper eval or CLI | +| Run policy | `experiment`, CLI target flags, repeat, gates, budget | Parent wrapper eval or CLI | +| Run concurrency | `--workers`, project config defaults, target/provider caps | Operator or selected target | | Target runtime | selected target config and `targets[].hooks` | Selected target | `workspace` can influence what an agent perceives through tools, but it is not diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/code-grader-contract.eval.yaml index 4a1145599..7bfae4ea1 100644 --- a/examples/contract/evals/code-grader-contract.eval.yaml +++ b/examples/contract/evals/code-grader-contract.eval.yaml @@ -1,9 +1,7 @@ name: code-grader-contract description: Release gate verifying the code-grader stdin payload contract. -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: code-grader-stdin-payload diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 228b361f4..c740d2e20 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -4,9 +4,7 @@ description: Lightweight release gate for npm latest promotion. workspace: template: ../workspace-template -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: json-contract diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 033bad68d..c8c6f8320 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -7,9 +7,7 @@ workspace: repo: EntityProcess/agentv-contract-fixture commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: repo-materializes-previous-commit diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index c43903554..80d7e333a 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -10,6 +10,7 @@ name: docker-workspace-example description: Example eval using Docker workspace for grading +target: mock_agent workspace: docker: @@ -18,10 +19,6 @@ workspace: memory: 2g cpus: 1 -execution: - target: mock_agent - workers: 1 - tests: - id: hello-world input: "Write a Python function that returns 'hello world'" diff --git a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml index d9ab18042..75470d9ac 100644 --- a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml +++ b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml @@ -1,6 +1,7 @@ description: >- E2E test for workspace pooling. No pool config in YAML — pooling is enabled by default for shared workspaces with repos. + Run with --workers 2 to exercise multiple pool slots. workspace: repos: @@ -8,9 +9,6 @@ workspace: repo: https://github.com/EntityProcess/agentv.git commit: main -execution: - workers: 2 - tags: [agent] tests: diff --git a/packages/core/scripts/generate-eval-schema.ts b/packages/core/scripts/generate-eval-schema.ts index 71d323b01..90fe690e4 100644 --- a/packages/core/scripts/generate-eval-schema.ts +++ b/packages/core/scripts/generate-eval-schema.ts @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { spawn } from 'node:child_process'; /** * Generates AgentV JSON schemas from Zod schemas. * Run: bun run generate:schema (from packages/core) @@ -9,6 +10,24 @@ import path from 'node:path'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { EvalFileSchema } from '../src/evaluation/validation/eval-file.schema.js'; +async function formatWithBiome(filePath: string): Promise { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['x', 'biome', 'format', '--write', filePath], { + stdio: 'inherit', + }); + + child.on('error', reject); + child.on('exit', (code) => { + if (code === 0) { + resolve(); + return; + } + + reject(new Error(`Biome exited with code ${code}`)); + }); + }); +} + async function writeSchema(options: { readonly schema: Parameters[0]; readonly name: string; @@ -36,6 +55,7 @@ async function writeSchema(options: { ); await writeFile(outputPath, `${JSON.stringify(schema, null, 2)}\n`); + await formatWithBiome(outputPath); console.log(`Generated: ${outputPath}`); } @@ -44,5 +64,5 @@ await writeSchema({ name: 'EvalFile', title: 'AgentV Eval File', description: 'Schema for AgentV evaluation YAML files (.eval.yaml)', - outputFile: 'eval-schema.json', + outputFile: 'eval.schema.json', }); diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index adeb2f204..ea28b1157 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -425,7 +425,7 @@ export async function materializeEvalConfig( return { testFilePath, tests, - workers: config.workers ?? suite.workers, + workers: config.workers, cache: config.cache ?? suite.cacheConfig?.enabled, cachePath: config.cachePath ?? suite.cacheConfig?.cachePath, budgetUsd: config.budgetUsd ?? suite.budgetUsd, diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index 03a423be8..0f2c7514f 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -42,7 +42,6 @@ export type ExperimentConfigWire = { readonly runs?: number; readonly early_exit?: boolean; readonly timeout_seconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budget_usd?: number; readonly workspace?: never; @@ -59,7 +58,6 @@ export type ExperimentConfig = { readonly runs?: number; readonly earlyExit?: boolean; readonly timeoutSeconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budgetUsd?: number; readonly fingerprint?: string; @@ -80,7 +78,6 @@ export type ExperimentArtifactMetadata = { readonly runs?: number; readonly early_exit?: boolean; readonly timeout_seconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budget_usd?: number; }; @@ -123,7 +120,7 @@ export function normalizeExperimentConfig(rawConfig: unknown): ExperimentConfig rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds, 'timeout_seconds', ); - const workers = readOptionalPositiveInteger(rawConfig.workers, 'workers'); + rejectExperimentWorkers(rawConfig.workers); const threshold = readOptionalThreshold(rawConfig.threshold); const budgetUsd = readOptionalPositiveNumber( rawConfig.budget_usd ?? rawConfig.budgetUsd, @@ -142,7 +139,6 @@ export function normalizeExperimentConfig(rawConfig: unknown): ExperimentConfig ...(runs !== undefined && { runs }), ...(earlyExit !== undefined && { earlyExit }), ...(timeoutSeconds !== undefined && { timeoutSeconds }), - ...(workers !== undefined && { workers }), ...(threshold !== undefined && { threshold }), ...(budgetUsd !== undefined && { budgetUsd }), }; @@ -217,7 +213,6 @@ export function buildExperimentArtifactMetadata( ...(config.runs !== undefined && { runs: config.runs }), ...(config.earlyExit !== undefined && { early_exit: config.earlyExit }), ...(config.timeoutSeconds !== undefined && { timeout_seconds: config.timeoutSeconds }), - ...(config.workers !== undefined && { workers: config.workers }), ...(config.threshold !== undefined && { threshold: config.threshold }), ...(config.budgetUsd !== undefined && { budget_usd: config.budgetUsd }), }; @@ -379,6 +374,15 @@ function rejectExperimentWorkspace(raw: unknown): void { ); } +function rejectExperimentWorkers(raw: unknown): void { + if (raw === undefined) { + return; + } + throw new Error( + 'Experiment workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', + ); +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 4ad3b3310..39863f2e7 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -32,6 +32,7 @@ export const DEFAULT_EVAL_PATTERNS: readonly string[] = [ ]; export type ExecutionDefaults = { + readonly workers?: number; readonly verbose?: boolean; readonly keep_workspaces?: boolean; readonly workspace_mode?: 'pooled' | 'temp' | 'static'; @@ -407,23 +408,6 @@ function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined { }; } -/** - * Extract workers count from suite-level execution block. - */ -export function extractWorkersFromSuite(suite: JsonObject): number | undefined { - const runtime = getSuiteRuntimeBlock(suite); - if (!runtime) { - return undefined; - } - - const workers = runtime.workers; - if (typeof workers === 'number' && Number.isInteger(workers) && workers >= 1 && workers <= 50) { - return workers; - } - - return undefined; -} - /** * Cache configuration parsed from execution block. */ @@ -578,6 +562,13 @@ export function parseExecutionDefaults( const obj = raw as Record; const result: Record = {}; + const workers = obj.workers; + if (typeof workers === 'number' && Number.isInteger(workers) && workers >= 1 && workers <= 50) { + result.workers = workers; + } else if (workers !== undefined) { + logWarning(`Invalid execution.workers in ${configPath}, expected integer 1-50`); + } + if (typeof obj.verbose === 'boolean') { result.verbose = obj.verbose; } else if (obj.verbose !== undefined) { diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index b5e20e119..8a1e8e529 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -92,7 +92,6 @@ export async function loadTsEvalSuite( return { tests: materialized.tests, - ...(materialized.workers !== undefined && { workers: materialized.workers }), ...(materialized.cache !== undefined && { cacheConfig: { enabled: materialized.cache, diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 038eacc3b..a2aeab493 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -1,10 +1,10 @@ /** * Zod schema for eval YAML file format. - * Used to generate eval-schema.json for AI agent reference. + * Used to generate eval.schema.json for AI agent reference. * * IMPORTANT: This schema describes the YAML input format, not the parsed runtime types. * When adding new eval features, update this schema AND run `bun run generate:schema` - * to regenerate eval-schema.json. The sync test will fail if they diverge. + * to regenerate eval.schema.json. The sync test will fail if they diverge. */ import { z } from 'zod'; @@ -360,7 +360,7 @@ const FailOnErrorSchema = z.boolean(); const ExecutionSchema = z.object({ target: z.string().optional(), targets: z.array(z.union([z.string(), EvalTargetRefSchema])).optional(), - workers: z.number().int().min(1).max(50).optional(), + workers: z.never().optional(), assertions: z.array(EvaluatorSchema).optional(), evaluators: z.array(EvaluatorSchema).optional(), skip_defaults: z.boolean().optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index ebdf46fe7..114e4d85d 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -82,7 +82,6 @@ const KNOWN_IMPORT_FIELDS = new Set(['path', 'select', 'run']); const KNOWN_RUN_OVERRIDE_FIELDS = new Set(['threshold', 'repeat', 'timeout_seconds', 'budget_usd']); const KNOWN_REPEAT_STRATEGIES = new Set(['pass_at_k', 'pass_all', 'mean', 'confidence_interval']); const KNOWN_TEST_EXECUTION_FIELDS = new Set([ - 'workers', 'assertions', 'evaluators', 'skip_defaults', @@ -99,6 +98,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ /** Removed top-level fields with migration hints. */ const REMOVED_TOP_LEVEL_FIELDS = new Map([ ['assert', "'assert' has been removed. Use 'assertions' instead."], + [ + 'workers', + "'workers' has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.", + ], [ 'experiment', "Top-level 'experiment' has been removed. Move experiment.target to top-level 'target', experiment.model to top-level 'model', and runtime controls to top-level 'policy' with runs, timeout_seconds, threshold, and budget_usd.", @@ -305,6 +308,7 @@ export async function validateEvalFile(filePath: string): Promise { + if (!isObject(entry)) { + return; + } + rejectWorkersField(entry.execution, `tests[${index}].execution`, filePath, errors); + }); + } +} + +function rejectWorkersField( + raw: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (!isObject(raw)) { + return; + } + if (raw.workers !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.workers`, + message: `${location}.workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + }); + } + rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors); +} + +function rejectTargetWorkers( + rawTargets: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (!Array.isArray(rawTargets)) { + return; + } + rawTargets.forEach((target, index) => { + if (!isObject(target) || target.workers === undefined) { + return; + } + errors.push({ + severity: 'error', + filePath, + location: `${location}[${index}].workers`, + message: `${location}[${index}].workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + }); + }); +} + function rejectRuntimeWorkspaceConfig( workspace: JsonValue | undefined, filePath: string, diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 660db6a84..d7f385921 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -468,12 +468,7 @@ export async function prepareSharedWorkspaceSetup( [ `Warning: This eval uses a shared workspace with ${workers} workers.`, 'If the agent under test makes file edits, concurrent runs may corrupt each other.', - 'To limit concurrency, add this to your eval YAML:', - '', - ' execution:', - ' workers: 1', - '', - 'Or pass --workers 1 on the command line.', + 'To limit concurrency, pass --workers 1 on the command line or set execution.workers in agentv.config.* / .agentv/config.yaml.', ].join('\n'), ); } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 73c2c1418..1f3b56783 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -26,7 +26,6 @@ import { extractTargetRefsFromSuite, extractTargetsFromSuite, extractThreshold, - extractWorkersFromSuite, loadConfig, } from './loaders/config-loader.js'; import { buildSearchRoots, resolveToAbsolutePath } from './loaders/file-resolver.js'; @@ -84,7 +83,6 @@ export { extractTargetRefsFromSuite, extractTargetsFromSuite, extractThreshold, - extractWorkersFromSuite, loadConfig, } from './loaders/config-loader.js'; export type { AgentVConfig, CacheConfig, ExecutionDefaults } from './loaders/config-loader.js'; @@ -111,7 +109,6 @@ type SuiteImportStackEntry = { }; const KNOWN_TEST_EXECUTION_FIELDS = new Set([ - 'workers', 'assertions', 'evaluators', 'skip_defaults', @@ -331,8 +328,6 @@ export type EvalSuiteResult = { readonly targets?: readonly string[]; /** Suite-level target refs with hooks from execution.targets (object form) */ readonly targetRefs?: readonly import('./types.js').EvalTargetRef[]; - /** Suite-level workers from execution.workers */ - readonly workers?: number; /** Suite-level cache config from execution.cache */ readonly cacheConfig?: import('./loaders/config-loader.js').CacheConfig; /** Suite-level metadata (name, description, version, etc.) */ @@ -464,6 +459,7 @@ async function loadTestsFromParsedYamlValue( if (!isJsonObject(interpolated)) { throw new Error(`Invalid test file format: ${evalFilePath}`); } + rejectAuthoredWorkers(interpolated); const suite = interpolated as RawTestSuite; const suiteNameFromFile = asString(suite.name)?.trim(); @@ -849,6 +845,7 @@ async function loadTestsFromParsedYamlValue( } function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): EvalSuiteResult { + rejectAuthoredWorkers(parsed); const metadata = parseMetadata(parsed); const failOnError = extractFailOnError(parsed); const threshold = extractThreshold(parsed); @@ -858,7 +855,6 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E tests, targets: extractTargetsFromSuite(parsed), targetRefs: extractTargetRefsFromSuite(parsed), - workers: extractWorkersFromSuite(parsed), cacheConfig: extractCacheConfig(parsed), budgetUsd: extractBudgetUsd(parsed), ...(metadata !== undefined && { metadata }), @@ -868,6 +864,56 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E }; } +function rejectAuthoredWorkers(parsed: JsonObject): void { + const locations: string[] = []; + if (parsed.workers !== undefined) { + locations.push('workers'); + } + collectWorkersLocations(parsed.execution, 'execution', locations); + collectWorkersLocations(parsed.experiment, 'experiment', locations); + if (Array.isArray(parsed.tests)) { + parsed.tests.forEach((entry, index) => { + if (!isJsonObject(entry)) { + return; + } + collectWorkersLocations(entry.execution, `tests[${index}].execution`, locations); + }); + } + + if (locations.length === 0) { + return; + } + + throw new Error( + `${locations[0]} has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + ); +} + +function collectWorkersLocations(raw: unknown, location: string, locations: string[]): void { + if (!isJsonObject(raw)) { + return; + } + if (raw.workers !== undefined) { + locations.push(`${location}.workers`); + } + collectTargetWorkersLocations(raw.targets, `${location}.targets`, locations); +} + +function collectTargetWorkersLocations( + rawTargets: unknown, + location: string, + locations: string[], +): void { + if (!Array.isArray(rawTargets)) { + return; + } + rawTargets.forEach((target, index) => { + if (isJsonObject(target) && target.workers !== undefined) { + locations.push(`${location}[${index}].workers`); + } + }); +} + type IncludeEntryType = 'suite' | 'tests'; type ExpandedInlineTestEntries = { diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 17e708485..3bef0aa67 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -49,7 +49,72 @@ describe('eval.yaml runtime policy and tests imports', () => { budgetUsd: 1.5, }); expect(suite.targets).toBeUndefined(); - expect(suite.workers).toBeUndefined(); + }); + + it('rejects authored workers in eval YAML runtime blocks', async () => { + const cases = [ + { + file: 'top-level.eval.yaml', + body: ['workers: 2', 'tests:', ' - id: one', ' input: hello', ' criteria: ok'], + message: /workers has been removed from eval YAML/, + }, + { + file: 'execution.eval.yaml', + body: [ + 'execution:', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /execution\.workers has been removed from eval YAML/, + }, + { + file: 'experiment.eval.yaml', + body: [ + 'experiment:', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /experiment\.workers has been removed from eval YAML/, + }, + { + file: 'target-ref.eval.yaml', + body: [ + 'execution:', + ' targets:', + ' - name: codex', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /execution\.targets\[0\]\.workers has been removed from eval YAML/, + }, + { + file: 'test-execution.eval.yaml', + body: [ + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ' execution:', + ' workers: 2', + ], + message: /tests\[0\]\.execution\.workers has been removed from eval YAML/, + }, + ]; + + for (const testCase of cases) { + const evalPath = path.join(tempDir, testCase.file); + await writeFile(evalPath, `${testCase.body.join('\n')}\n`); + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(testCase.message); + } }); it('rejects repeat strategy config under top-level policy', async () => { @@ -471,7 +536,6 @@ describe('eval.yaml runtime policy and tests imports', () => { 'name: child-suite', 'execution:', ' target: child-target', - ' workers: 1', ' threshold: 0.2', ' repeat:', ' count: 5', @@ -771,7 +835,7 @@ describe('eval.yaml runtime policy and tests imports', () => { [ 'name: child-a', 'execution:', - ' workers: 2', + ' threshold: 0.2', 'tests:', ' - id: a', ' input: a', @@ -784,7 +848,7 @@ describe('eval.yaml runtime policy and tests imports', () => { [ 'name: child-b', 'execution:', - ' workers: 4', + ' threshold: 0.4', 'tests:', ' - id: b', ' input: b', diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts index b2334b113..763931772 100644 --- a/packages/core/test/evaluation/experiment.test.ts +++ b/packages/core/test/evaluation/experiment.test.ts @@ -17,7 +17,6 @@ describe('inline experiment config', () => { runs: 3, early_exit: false, timeout_seconds: 900, - workers: 4, threshold: 0.8, budget_usd: 1.25, }); @@ -32,7 +31,6 @@ describe('inline experiment config', () => { runs: 3, earlyExit: false, timeoutSeconds: 900, - workers: 4, budgetUsd: 1.25, }); expect(config.fingerprint).toMatch(/^[a-f0-9]{64}$/); @@ -93,6 +91,9 @@ describe('inline experiment config', () => { expect(() => normalizeExperimentConfig({ workspace: { repos: [{ repo: 'acme/support-app' }] } }), ).toThrow(/Experiment workspace has been removed from eval YAML/); + expect(() => normalizeExperimentConfig({ workers: 3 })).toThrow( + /Experiment workers has been removed from eval YAML/, + ); }); it('builds safe snake_case artifact metadata without agent options', () => { @@ -103,7 +104,6 @@ describe('inline experiment config', () => { repeat: { count: 2, strategy: 'mean', cost_limit_usd: 0.5 }, early_exit: true, timeout_seconds: 120, - workers: 3, }); const metadata = buildExperimentArtifactMetadata(config); @@ -118,7 +118,6 @@ describe('inline experiment config', () => { }, early_exit: true, timeout_seconds: 120, - workers: 3, }); expect(metadata).not.toHaveProperty('agent_options'); expect(metadata).not.toHaveProperty('setup'); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 9c4dc46b2..686dc15b1 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -837,6 +837,22 @@ describe('parseExecutionDefaults', () => { expect(result?.verbose).toBe(true); }); + it('parses workers as an operator-side execution default', () => { + const result = parseExecutionDefaults({ workers: 4 }, '/test/config.yaml'); + expect(result?.workers).toBe(4); + }); + + it('ignores invalid workers defaults', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = parseExecutionDefaults({ workers: 0 }, '/test/config.yaml'); + expect(result).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('execution.workers')); + } finally { + warnSpy.mockRestore(); + } + }); + it('parses keep_workspaces boolean', () => { const result = parseExecutionDefaults({ keep_workspaces: true }, '/test/config.yaml'); expect(result?.keep_workspaces).toBe(true); diff --git a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts index ca732b547..bcf339591 100644 --- a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts @@ -12,7 +12,6 @@ const config: EvalConfig = { assertions: [{ type: 'contains', value: 'hello' }], }, ], - workers: 2, cache: false, cachePath: '.agentv/ts-eval-cache', budgetUsd: 1.5, diff --git a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts index ab6a7f0ab..d89394160 100644 --- a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts @@ -7,7 +7,6 @@ const suite = { tags: ['sdk', 'typescript', 'yaml'], execution: { targets: ['mock-target'], - workers: 2, skipDefaults: true, budgetUsd: 2, threshold: 0.75, @@ -49,7 +48,6 @@ export default Object.defineProperties(suite, { tags: suite.tags, execution: { targets: ['mock-target'], - workers: 2, skip_defaults: true, budget_usd: 2, threshold: 0.75, diff --git a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts index 2298ec0f7..e19710c22 100644 --- a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts +++ b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts @@ -56,7 +56,6 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].suite).toBe('default-export-suite'); expect(suite.tests[0].category).toBe('sdk'); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript']); - expect(suite.workers).toBe(2); expect(suite.cacheConfig?.enabled).toBe(false); expect(suite.cacheConfig?.cachePath).toBe('.agentv/ts-eval-cache'); expect(suite.budgetUsd).toBe(1.5); @@ -79,7 +78,6 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].workspace?.hooks?.before_each?.command).toEqual(['echo', 'case-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.timeout_ms).toBe(1_000); expect(suite.targets).toEqual(['mock-target']); - expect(suite.workers).toBe(2); expect(suite.budgetUsd).toBe(2); expect(suite.threshold).toBe(0.75); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript', 'yaml']); diff --git a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts index 89e410861..bfffc49cf 100644 --- a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts +++ b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts @@ -5,11 +5,11 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; import { EvalFileSchema } from '../../../src/evaluation/validation/eval-file.schema.js'; describe('generated schema sync', () => { - it('keeps eval-schema.json synced with the Zod schema', async () => { + it('keeps eval.schema.json synced with the Zod schema', async () => { const repoRoot = path.resolve(import.meta.dirname, '../../../../..'); const schemaPath = path.join( repoRoot, - 'skills-data/agentv-eval-writer/references/eval-schema.json', + 'skills-data/agentv-eval-writer/references/eval.schema.json', ); // Read committed schema diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 3ba05bb1c..76df2203e 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -358,12 +358,50 @@ tests: ).toBe(true); }); + it('rejects authored workers in eval YAML', async () => { + const filePath = path.join(tempDir, 'authored-workers.eval.yaml'); + await writeFile( + filePath, + `workers: 2 +execution: + workers: 3 + targets: + - name: codex + workers: 4 +tests: + - id: test-1 + criteria: Goal + input: Query + execution: + workers: 5 +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + [ + 'workers', + 'execution.workers', + 'execution.targets[0].workers', + 'tests[0].execution.workers', + ].every((location) => + result.errors.some( + (error) => + error.severity === 'error' && + error.location === location && + error.message.includes('has been removed from eval YAML'), + ), + ), + ).toBe(true); + }); + it('warns that imported child legacy execution is ignored by wrapper composition', async () => { await writeFile( path.join(tempDir, 'composition-child-experiment.eval.yaml'), `execution: target: child-target - workers: 2 threshold: 0.9 tests: - id: child-case diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 06abfe49e..f0064828f 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -147,7 +147,6 @@ export interface EvalPolicy { export interface EvalExecution { readonly target?: string; readonly targets?: readonly (string | EvalTargetRef)[]; - readonly workers?: number; readonly assertions?: readonly EvalAssertionConfig[]; readonly skipDefaults?: boolean; readonly cache?: boolean; diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 051c9253b..2be1525a5 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -28,7 +28,6 @@ describe('YAML-aligned eval authoring helpers', () => { }, }, ], - workers: 2, skipDefaults: true, budgetUsd: 1.5, failOnError: true, @@ -119,7 +118,6 @@ describe('YAML-aligned eval authoring helpers', () => { }, }, ], - workers: 2, skip_defaults: true, budget_usd: 1.5, fail_on_error: true, diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index c1ec9580c..183434bf3 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -812,21 +812,21 @@ Do not invent a separate Opik-specific eval surface. Keep the eval definition in ## Schemas -- Eval file: `references/eval-schema.json` -- Config: `references/config-schema.json` +- Eval file: `references/eval.schema.json` +- Config: `references/config.schema.json` ## Accessing reference files To load a specific reference without pulling the entire skill into context: ```bash -agentv skills get agentv-eval-writer --ref eval-schema.json +agentv skills get agentv-eval-writer --ref eval.schema.json ``` Or resolve the skill directory and read files directly: ```bash -cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json +cat $(agentv skills path agentv-eval-writer)/references/eval.schema.json ``` Use `--full` to retrieve every file in the skill at once. diff --git a/skills-data/agentv-eval-writer/references/config-schema.json b/skills-data/agentv-eval-writer/references/config.schema.json similarity index 100% rename from skills-data/agentv-eval-writer/references/config-schema.json rename to skills-data/agentv-eval-writer/references/config.schema.json diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json similarity index 99% rename from skills-data/agentv-eval-writer/references/eval-schema.json rename to skills-data/agentv-eval-writer/references/eval.schema.json index 047712c35..4c852e7b5 100644 --- a/skills-data/agentv-eval-writer/references/eval-schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -3232,9 +3232,7 @@ "type": "object", "properties": { "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 + "not": {} }, "assertions": { "type": "array", @@ -9862,9 +9860,7 @@ "type": "object", "properties": { "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 + "not": {} }, "assertions": { "type": "array", @@ -14071,9 +14067,7 @@ "minItems": 1 }, "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 + "not": {} }, "assertions": { "type": "array",