diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 2f6196bfa..919ccbb36 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1904,19 +1904,14 @@ export async function runEvalCommand( // Detect matrix mode: multiple targets for any file const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1); - // In matrix mode, total eval count is tests × targets (accounting for per-test target overrides) + // In matrix mode, total eval count is tests × selected targets. // When resuming, subtract tests that will be skipped let totalEvalCount = 0; let resumeSkippedCount = 0; for (const meta of fileMetadata.values()) { const suiteTargetNames = meta.selections.map((s) => s.selection.targetName); for (const test of meta.testCases) { - // Per-test targets override suite-level targets. - const testTargetNames = - test.targets && test.targets.length > 0 - ? test.targets.filter((t) => suiteTargetNames.includes(t)) - : suiteTargetNames; - const effectiveTargets = testTargetNames.length > 0 ? testTargetNames : ['unknown']; + const effectiveTargets = suiteTargetNames.length > 0 ? suiteTargetNames : ['unknown']; for (const tn of effectiveTargets) { const key = `${test.id}::${tn}`; if (resumeSkipKeys?.has(key)) { @@ -2140,17 +2135,10 @@ export async function runEvalCommand( // Run all targets concurrently (each target has its own worker limit) const targetResults = await Promise.all( targetPrep.selections.map(async ({ selection, inlineTargetLabel }) => { - // Filter test cases to those applicable to this target. + // Target selection is suite/experiment/CLI runtime policy; every selected + // target runs every filtered test case for this eval file. const targetName = selection.targetName; - const applicableTestCases = - targetPrep.selections.length > 1 - ? targetPrep.testCases.filter((test) => { - if (test.targets && test.targets.length > 0) { - return test.targets.includes(targetName); - } - return true; - }) - : targetPrep.testCases; + const applicableTestCases = targetPrep.testCases; // --resume / --rerun-failed: skip tests that are already completed const filteredTestCases = resumeSkipKeys diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 22a22f7a6..80287621b 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -735,10 +735,6 @@ function buildPortableEvalCase( if (test.conversation_id) { testCase.conversation_id = test.conversation_id; } - if (test.targets && test.targets.length > 0) { - const existingExecution = isRecord(testCase.execution) ? testCase.execution : {}; - testCase.execution = { ...existingExecution, targets: test.targets }; - } if (test.threshold !== undefined) { const existingExecution = isRecord(testCase.execution) ? testCase.execution : {}; testCase.execution = { ...existingExecution, threshold: test.threshold }; diff --git a/apps/cli/src/commands/import/promptfoo.test.ts b/apps/cli/src/commands/import/promptfoo.test.ts index 39b1e8892..b49f0f12c 100644 --- a/apps/cli/src/commands/import/promptfoo.test.ts +++ b/apps/cli/src/commands/import/promptfoo.test.ts @@ -101,6 +101,36 @@ tests: file://./tests.jsonl expect(yaml).toContain('type: equals'); }); + it('rejects promptfoo test provider filters instead of emitting per-case targets', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-promptfoo-')); + tempDirs.push(dir); + + const configPath = path.join(dir, 'promptfooconfig.yaml'); + await writeFile( + configPath, + ` +prompts: + - "Answer {{question}}" +providers: + - openai:gpt-5-mini + - anthropic:claude-sonnet +tests: + - id: codex-only + provider: openai:gpt-5-mini + vars: + question: What is 2+2? + assert: + - type: equals + value: "4" +`, + 'utf8', + ); + + await expect(convertPromptfooToAgentvSuite({ inputPath: configPath })).rejects.toThrow( + 'unsupported per-case target selection', + ); + }); + it('imports promptfoo CSV datasets with __expected columns', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-promptfoo-')); tempDirs.push(dir); diff --git a/apps/cli/src/commands/import/promptfoo.ts b/apps/cli/src/commands/import/promptfoo.ts index db7b45d28..4bbbfe248 100644 --- a/apps/cli/src/commands/import/promptfoo.ts +++ b/apps/cli/src/commands/import/promptfoo.ts @@ -116,17 +116,20 @@ export async function convertPromptfooToAgentvSuite( readAssertionList(defaultTest.assert), absoluteInputPath, ); + const suiteTargetNames = + filterProviders(providers, defaultTest.providers ?? defaultTest.provider) ?? + providers.map((provider) => provider.targetName); const convertedTests = await buildAgentvTests({ inputPath: absoluteInputPath, prompts, - providers, defaultTest, rawTests: testCases, + suiteTargetNames, }); const execution: Record = {}; - if (providers.length > 0) { - execution.targets = providers.map((provider) => provider.targetName); + if (suiteTargetNames.length > 0) { + execution.targets = suiteTargetNames; } const suite: AgentvSuite = { @@ -778,11 +781,11 @@ function parseCsvScalarValue(value: string): JsonValue { async function buildAgentvTests(options: { readonly inputPath: string; readonly prompts: readonly PromptfooPrompt[]; - readonly providers: readonly PromptfooProvider[]; readonly defaultTest: PromptfooTestCase; readonly rawTests: readonly PromptfooTestCase[]; + readonly suiteTargetNames: readonly string[]; }) { - const { inputPath, prompts, providers, defaultTest, rawTests } = options; + const { inputPath, prompts, defaultTest, rawTests, suiteTargetNames } = options; const tests: AgentvTest[] = []; for (let index = 0; index < rawTests.length; index++) { @@ -801,12 +804,11 @@ async function buildAgentvTests(options: { throw new Error(`Test '${baseId}' matches no prompts after prompt filters`); } - const defaultTargets = filterProviders( - providers, - defaultTest.providers ?? defaultTest.provider, - ); - const caseTargets = filterProviders(providers, rawTest.providers ?? rawTest.provider); - const effectiveTargets = caseTargets ?? defaultTargets; + if (rawTest.providers !== undefined || rawTest.provider !== undefined) { + throw new Error( + `Promptfoo test '${baseId}' uses provider filters, which require unsupported per-case target selection. Split provider-specific cases before importing or use defaultTest provider filters for suite-level target selection.`, + ); + } const convertedCaseAssertions = await convertPromptfooAssertions( readAssertionList(rawTest.assert), inputPath, @@ -830,11 +832,10 @@ async function buildAgentvTests(options: { const templatedInput = buildPromptTemplate(prompt, testOptions); const promptSuffix = promptSelection.length > 1 ? `--${sanitizeName(prompt.key || prompt.label)}` : ''; - const metadata = buildPromptfooMetadata(rawTest, effectiveVars, prompt, effectiveTargets); + const metadata = buildPromptfooMetadata(rawTest, effectiveVars, prompt); const execution = buildCaseExecution({ defaultAssertionsEnabled: !testOptions.disableDefaultAsserts, threshold: asNumber(rawTest.threshold), - effectiveTargets, }); const test: AgentvTest = { @@ -1003,14 +1004,12 @@ function buildPromptfooMetadata( rawTest: PromptfooTestCase, vars: Record, prompt: PromptfooPrompt, - effectiveTargets: readonly string[] | undefined, ) { const rawMetadata = isJsonObject(rawTest.metadata) ? rawTest.metadata : undefined; const promptfooMetadata: Record = { vars, prompt_label: prompt.label, prompt_source: prompt.source, - ...(effectiveTargets && effectiveTargets.length > 0 ? { targets: [...effectiveTargets] } : {}), ...(typeof rawTest.description === 'string' ? { description: rawTest.description } : {}), }; @@ -1023,7 +1022,6 @@ function buildPromptfooMetadata( function buildCaseExecution(options: { readonly defaultAssertionsEnabled: boolean; readonly threshold?: number; - readonly effectiveTargets?: readonly string[]; }) { const execution: Record = {}; if (!options.defaultAssertionsEnabled) { @@ -1032,9 +1030,6 @@ function buildCaseExecution(options: { if (options.threshold !== undefined) { execution.threshold = options.threshold; } - if (options.effectiveTargets && options.effectiveTargets.length > 0) { - execution.targets = [...options.effectiveTargets]; - } return Object.keys(execution).length > 0 ? execution : undefined; } diff --git a/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx index 8f19727c5..a18c3bded 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx @@ -27,7 +27,7 @@ tests: | `criteria` | Conditional | Description of what a correct response should contain. Required only when the case has no `expected_output` or `assertions` | | `input` | Yes | Input sent to the target (string, object, or message array) | | `expected_output` | No | Expected response for comparison (string, object, or message array) | -| `execution` | No | Per-case execution overrides (for example `target`, `skip_defaults`) | +| `execution` | No | Per-case execution overrides such as `skip_defaults` or `threshold`; target selection belongs in `experiment.target(s)` or CLI `--target` | | `workspace` | No | Per-case workspace config (overrides suite-level) | | `metadata` | No | Arbitrary key-value pairs passed to graders and workspace scripts | | `rubrics` | No | Structured evaluation criteria | @@ -91,7 +91,9 @@ expected_output: ## Per-Case Execution Overrides -Override the default target or graders for specific tests: +Override graders or local scoring settings for specific tests. Do not put +target selection in cases; use `experiment.target(s)`, CLI `--target`, separate +eval suites, or tags/filters for target-specific cases. ```yaml tests: @@ -99,8 +101,6 @@ tests: criteria: Provides detailed explanation input: Explain quicksort algorithm - execution: - target: gpt4_target assertions: - name: depth_check type: llm-grader diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index 610eefc85..f171595c8 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -61,20 +61,18 @@ already-exported secrets into `.env`. ## Referencing Targets in Evals -Set the default target at the top level or override per case: +Select targets at the eval runtime level with `experiment.target`, +`experiment.targets`, legacy suite-level `execution.target(s)`, or CLI +`--target`. Test cases do not choose targets; split target-specific cases into +separate eval suites or select them with tags/filters. ```yaml -# Top-level default -execution: - target: azure-base +experiment: + targets: [azure-base, vscode_dev] tests: - id: test-1 - # Uses azure-base - - id: test-2 - execution: - target: vscode_dev # Override for this case ``` ## Grader Target diff --git a/examples/features/basic-jsonl/evals/dataset.jsonl b/examples/features/basic-jsonl/evals/dataset.jsonl index 1b788f43a..9e61a2a5f 100644 --- a/examples/features/basic-jsonl/evals/dataset.jsonl +++ b/examples/features/basic-jsonl/evals/dataset.jsonl @@ -1,5 +1,5 @@ {"id": "code-review-javascript", "criteria": "Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT", "input": [{"role": "system", "content": "You are an expert software developer who provides clear, concise code reviews."}, {"role": "user", "content": [{"type": "text", "value": "Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n let total = 0;\n for (let i = 0; i < 0; i++) {\n total += items[i].price * items[i].quantity;\n }\n return total;\n}\n```"}, {"type": "file", "value": "../basic/evals/javascript.instructions.md"}]}], "expected_output": [{"role": "assistant", "content": "The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}]} -{"id": "code-gen-python", "conversation_id": "python-code-generation", "criteria": "AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON", "input": [{"role": "system", "content": "You are a code generator that follows specifications exactly."}, {"role": "user", "content": [{"type": "text", "value": "Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"}, {"type": "file", "value": "../basic/evals/python.instructions.md"}]}], "execution": {"target": "azure-llm"}} +{"id": "code-gen-python", "conversation_id": "python-code-generation", "criteria": "AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON", "input": [{"role": "system", "content": "You are a code generator that follows specifications exactly."}, {"role": "user", "content": [{"type": "text", "value": "Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"}, {"type": "file", "value": "../basic/evals/python.instructions.md"}]}]} {"id": "feature-proposal-brainstorm", "criteria": "Assistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)", "input": [{"role": "system", "content": "You are a product strategist specializing in mobile health and fitness applications."}, {"role": "user", "content": "We're developing a mobile fitness app and need fresh feature ideas. Please brainstorm 3-5 innovative features."}]} {"id": "multiturn-debug-session", "criteria": "Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix.", "input": [{"role": "system", "content": "You are an expert debugging assistant."}, {"role": "user", "content": "I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n result = []\n for i in range(len(items) - 1):\n result.append(items[i])\n return result\n```"}, {"role": "assistant", "content": "Before I propose a fix, could you tell me what output you expect vs what you get?"}, {"role": "user", "content": "For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}], "expected_output": [{"role": "assistant", "content": "You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}]} {"id": "shorthand-string-example", "criteria": "Assistant correctly answers the math question", "input": "What is 2+2?", "expected_output": "The answer is 4."} diff --git a/examples/features/basic/evals/dataset.eval.yaml b/examples/features/basic/evals/dataset.eval.yaml index 7309596c9..a7a106080 100644 --- a/examples/features/basic/evals/dataset.eval.yaml +++ b/examples/features/basic/evals/dataset.eval.yaml @@ -57,7 +57,7 @@ tests: # ========================================== # Example 2: Advanced features - conversation_id, multiple graders - # Demonstrates: conversation threading, execution config, target override, graders + # Demonstrates: conversation threading and per-test graders # Note: Optimization (ACE, etc.) is configured separately in opts/*.yaml files # ========================================== - id: code-gen-python-comprehensive @@ -69,9 +69,6 @@ tests: criteria: AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON - execution: - target: llm - # Multiple graders - supports both code-based and LLM graders assertions: - name: keyword_check diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index d31921a78..278874ce1 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -426,24 +426,6 @@ export function extractWorkersFromSuite(suite: JsonObject): number | undefined { return undefined; } -/** - * Extract per-test targets array from a raw test case object. - */ -export function extractTargetsFromTestCase(testCase: JsonObject): readonly string[] | undefined { - const execution = testCase.execution; - if (!execution || typeof execution !== 'object' || Array.isArray(execution)) { - return undefined; - } - - const targets = (execution as Record).targets; - if (Array.isArray(targets)) { - const valid = targets.filter((t): t is string => typeof t === 'string' && t.trim().length > 0); - return valid.length > 0 ? valid.map((t) => t.trim()) : undefined; - } - - return undefined; -} - /** * Cache configuration parsed from execution block. */ diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 27c564f82..7d93c1762 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1007,8 +1007,6 @@ export interface EvalTest { readonly workspace?: WorkspaceConfig; /** Arbitrary metadata passed to workspace scripts via stdin */ readonly metadata?: Record; - /** Per-test target override (matrix evaluation) */ - readonly targets?: readonly string[]; /** Per-test score threshold override (0-1). Resolution: CLI > test > suite > DEFAULT_THRESHOLD. */ readonly threshold?: number; /** Scoped runtime interpretation/scheduling overrides. */ diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index d6aafbab3..51995be4c 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -434,6 +434,8 @@ const ConversationTurnSchema = z.object({ // Test case // --------------------------------------------------------------------------- +const TestExecutionSchema = ExecutionSchema.omit({ target: true, targets: true }).strict(); + const EvalTestSchema = z.object({ id: z.string().min(1), vars: JsonObjectSchema.optional(), @@ -443,7 +445,7 @@ const EvalTestSchema = z.object({ expected_output: ExpectedOutputSchema.optional(), assertions: z.array(EvaluatorSchema).optional(), evaluators: z.array(EvaluatorSchema).optional(), - execution: ExecutionSchema.optional(), + execution: TestExecutionSchema.optional(), run: RunOverrideSchema.optional(), workspace: WorkspaceSchema.optional(), metadata: z.record(z.unknown()).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index afa311471..2b742b358 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -68,6 +68,20 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ const KNOWN_INCLUDE_FIELDS = new Set(['include', 'type', 'select', 'run']); const KNOWN_RUN_OVERRIDE_FIELDS = new Set(['threshold', 'repeat', 'timeout_seconds', 'budget_usd']); const KNOWN_REPEAT_STRATEGIES = new Set(['pass_at_k', 'pass_all', 'mean', 'confidence_interval']); +const KNOWN_TEST_EXECUTION_FIELDS = new Set([ + 'workers', + 'assertions', + 'evaluators', + 'skip_defaults', + 'cache', + 'trials', + 'budget_usd', + 'budgetUsd', + 'fail_on_error', + 'failOnError', + 'threshold', + 'workspace', +]); /** * Deprecated top-level fields with migration hints. @@ -373,6 +387,7 @@ export async function validateEvalFile(filePath: string): Promise) : undefined; const metadata = mergeSuiteMetadataPayload(rawCaseMetadata, suiteMetadataPayload); - // Extract per-test targets override (matrix evaluation) - const caseTargets = extractTargetsFromTestCase(renderedCase as JsonObject); // Extract dependency fields const dependsOn = Array.isArray(renderedCase.depends_on) @@ -793,7 +805,6 @@ async function loadTestsFromParsedYamlValue( ...(suitePreprocessors ? { preprocessors: suitePreprocessors } : {}), workspace: mergedWorkspace, metadata, - targets: caseTargets, ...(caseRun?.threshold !== undefined ? { threshold: caseRun.threshold } : {}), ...(caseRun !== undefined ? { run: caseRun } : {}), ...(mode ? { mode } : {}), @@ -858,6 +869,18 @@ type IncludeSelect = { readonly metadata?: Record; }; +function rejectUnsupportedTestExecutionFields( + caseExecution: JsonObject | undefined, + testId: string | undefined, +): void { + if (!caseExecution) return; + for (const key of Object.keys(caseExecution)) { + if (!KNOWN_TEST_EXECUTION_FIELDS.has(key)) { + throw new Error(`test '${testId ?? 'unknown'}'.execution.${key} is not supported.`); + } + } +} + function normalizeRunOverride(value: unknown, label: string): EvalRunOverride | undefined { if (value === undefined) { return undefined; diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 99477ed06..0db4e66b2 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -112,6 +112,26 @@ describe('eval.yaml inline experiment and tests imports', () => { ); }); + it('rejects unsupported per-test execution target blocks', async () => { + const evalPath = path.join(tempDir, 'test-execution-target.eval.yaml'); + await writeFile( + evalPath, + [ + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ' execution:', + ' target: codex', + '', + ].join('\n'), + ); + + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow( + "test 'one'.execution.target is not supported.", + ); + }); + it('globs raw case files through tests[].include with deterministic ordering and select filters', async () => { const casesDir = path.join(tempDir, 'cases'); await mkdir(casesDir, { recursive: true }); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 3fdd76e39..dbf5cc615 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -9,7 +9,6 @@ import { extractTargetFromSuite, extractTargetRefsFromSuite, extractTargetsFromSuite, - extractTargetsFromTestCase, extractThreshold, loadConfig, parseExecutionDefaults, @@ -802,29 +801,6 @@ describe('extractTargetRefsFromSuite', () => { }); }); -describe('extractTargetsFromTestCase', () => { - it('returns undefined when no execution block', () => { - const testCase: JsonObject = { id: 'test-1' }; - expect(extractTargetsFromTestCase(testCase)).toBeUndefined(); - }); - - it('extracts targets from test case execution.targets', () => { - const testCase: JsonObject = { - id: 'test-1', - execution: { targets: ['copilot'] }, - }; - expect(extractTargetsFromTestCase(testCase)).toEqual(['copilot']); - }); - - it('returns undefined when targets is empty', () => { - const testCase: JsonObject = { - id: 'test-1', - execution: { targets: [] }, - }; - expect(extractTargetsFromTestCase(testCase)).toBeUndefined(); - }); -}); - describe('extractBudgetUsd', () => { it('returns undefined when no execution block', () => { const suite: JsonObject = { tests: [] }; diff --git a/packages/core/test/evaluation/matrix-targets.test.ts b/packages/core/test/evaluation/matrix-targets.test.ts index 1f8be37ed..1484186e2 100644 --- a/packages/core/test/evaluation/matrix-targets.test.ts +++ b/packages/core/test/evaluation/matrix-targets.test.ts @@ -43,7 +43,7 @@ tests: expect(suite.targets).toBeUndefined(); }); - it('populates per-test targets from test-level execution.targets', async () => { + it('rejects unsupported test-level execution.targets', async () => { const { filePath, dir } = createTempYaml(` execution: targets: @@ -61,14 +61,8 @@ tests: - copilot `); - const suite = await loadTestSuite(filePath, dir); - expect(suite.targets).toEqual(['copilot', 'claude']); - expect(suite.tests.length).toBe(2); - - const generalTest = suite.tests.find((t) => t.id === 'general-test'); - expect(generalTest?.targets).toBeUndefined(); - - const copilotOnly = suite.tests.find((t) => t.id === 'copilot-only'); - expect(copilotOnly?.targets).toEqual(['copilot']); + await expect(loadTestSuite(filePath, dir)).rejects.toThrow( + "test 'copilot-only'.execution.targets is not supported.", + ); }); }); diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 51bf2839a..3b400c51c 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -1,7 +1,21 @@ import { describe, expect, it } from 'bun:test'; +import type { ZodIssue } from 'zod'; import { EvalFileSchema } from '../../../src/evaluation/validation/eval-file.schema.js'; +function collectIssueMessages(issues: readonly ZodIssue[]): string[] { + const messages: string[] = []; + for (const issue of issues) { + messages.push(issue.message); + if (issue.code === 'invalid_union') { + for (const unionError of issue.unionErrors) { + messages.push(...collectIssueMessages(unionError.issues)); + } + } + } + return messages; +} + describe('EvalFileSchema input shorthand', () => { const baseTest = { id: 'test-1', @@ -141,4 +155,42 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); + + it('does not accept test-level execution.targets', () => { + const result = EvalFileSchema.safeParse({ + tests: [ + { + ...baseTest, + execution: { + targets: ['codex'], + }, + }, + ], + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error('Expected test-level execution.targets to be rejected'); + expect(collectIssueMessages(result.error.issues)).toContain( + "Unrecognized key(s) in object: 'targets'", + ); + }); + + it('does not accept test-level execution.target', () => { + const result = EvalFileSchema.safeParse({ + tests: [ + { + ...baseTest, + execution: { + target: 'codex', + }, + }, + ], + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error('Expected test-level execution.target to be rejected'); + expect(collectIssueMessages(result.error.issues)).toContain( + "Unrecognized key(s) in object: 'target'", + ); + }); }); diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index f0a18c0e7..eaaa8ac24 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -69,6 +69,58 @@ tests: expect(result.errors).toHaveLength(0); }); + it('rejects unsupported test-level execution.targets', async () => { + const filePath = path.join(tempDir, 'test-level-targets.yaml'); + await writeFile( + filePath, + `tests: + - id: target-specific + input: "Hello" + criteria: "Greet" + execution: + targets: [codex] +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'tests[0].execution.targets' && + error.message === "Unsupported test execution field 'targets'.", + ), + ).toBe(true); + }); + + it('rejects unsupported test-level execution.target', async () => { + const filePath = path.join(tempDir, 'test-level-target.yaml'); + await writeFile( + filePath, + `tests: + - id: target-specific + input: "Hello" + criteria: "Greet" + execution: + target: codex +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'tests[0].execution.target' && + error.message === "Unsupported test execution field 'target'.", + ), + ).toBe(true); + }); + it('rejects include entries without type', async () => { const filePath = path.join(tempDir, 'include-missing-type.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 4e84d5cd6..0941e7e59 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -133,7 +133,7 @@ tests: | `input` | yes | Input to the agent (string/object shorthand or full message array) | | `expected_output` | no | Gold-standard reference answer (string shorthand or full message array) | | `assertions` | no | Graders: deterministic checks, rubrics, and LLM/code graders | -| `execution` | no | Per-case execution overrides | +| `execution` | no | Per-case non-target execution overrides such as `skip_defaults` or `threshold`; target selection belongs in `experiment.target(s)` or CLI `--target` | | `workspace` | no | Per-case workspace config (overrides suite-level) | | `metadata` | no | Arbitrary key-value pairs passed to setup/teardown scripts | | `conversation_id` | no | Thread grouping | diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval-schema.json index 86f227700..6ccc7d1ef 100644 --- a/skills-data/agentv-eval-writer/references/eval-schema.json +++ b/skills-data/agentv-eval-writer/references/eval-schema.json @@ -2627,219 +2627,6 @@ "execution": { "type": "object", "properties": { - "target": { - "type": "string" - }, - "targets": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "use_target": { - "type": "string" - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "required": ["name"], - "additionalProperties": false - } - ] - } - }, "workers": { "type": "integer", "minimum": 1, @@ -9430,219 +9217,6 @@ "execution": { "type": "object", "properties": { - "target": { - "type": "string" - }, - "targets": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "use_target": { - "type": "string" - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "script": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "required": ["name"], - "additionalProperties": false - } - ] - } - }, "workers": { "type": "integer", "minimum": 1,