fix(eval): reject per-case target selection

apps/cli/src/commands/eval/run-eval.ts

@@ -1904,19 +1904,14 @@ export async function runEvalCommand(
   // Detect matrix mode: multiple targets for any file
   const isMatrixMode = Array.from(fileMetadata.values()).some((meta) => meta.selections.length > 1);
 
-  // In matrix mode, total eval count is tests × targets (accounting for per-test target overrides)
+  // In matrix mode, total eval count is tests × selected targets.
   // When resuming, subtract tests that will be skipped
   let totalEvalCount = 0;
   let resumeSkippedCount = 0;
   for (const meta of fileMetadata.values()) {
     const suiteTargetNames = meta.selections.map((s) => s.selection.targetName);
     for (const test of meta.testCases) {
-      // Per-test targets override suite-level targets.
-      const testTargetNames =
-        test.targets && test.targets.length > 0
-          ? test.targets.filter((t) => suiteTargetNames.includes(t))
-          : suiteTargetNames;
-      const effectiveTargets = testTargetNames.length > 0 ? testTargetNames : ['unknown'];
+      const effectiveTargets = suiteTargetNames.length > 0 ? suiteTargetNames : ['unknown'];
       for (const tn of effectiveTargets) {
         const key = `${test.id}::${tn}`;
         if (resumeSkipKeys?.has(key)) {
@@ -2140,17 +2135,10 @@ export async function runEvalCommand(
       // Run all targets concurrently (each target has its own worker limit)
       const targetResults = await Promise.all(
         targetPrep.selections.map(async ({ selection, inlineTargetLabel }) => {
-          // Filter test cases to those applicable to this target.
+          // Target selection is suite/experiment/CLI runtime policy; every selected
+          // target runs every filtered test case for this eval file.
           const targetName = selection.targetName;
-          const applicableTestCases =
-            targetPrep.selections.length > 1
-              ? targetPrep.testCases.filter((test) => {
-                  if (test.targets && test.targets.length > 0) {
-                    return test.targets.includes(targetName);
-                  }
-                  return true;
-                })
-              : targetPrep.testCases;
+          const applicableTestCases = targetPrep.testCases;
 
           // --resume / --rerun-failed: skip tests that are already completed
           const filteredTestCases = resumeSkipKeys

apps/cli/src/commands/eval/task-bundle.ts

@@ -735,10 +735,6 @@ function buildPortableEvalCase(
   if (test.conversation_id) {
     testCase.conversation_id = test.conversation_id;
   }
-  if (test.targets && test.targets.length > 0) {
-    const existingExecution = isRecord(testCase.execution) ? testCase.execution : {};
-    testCase.execution = { ...existingExecution, targets: test.targets };
-  }
   if (test.threshold !== undefined) {
     const existingExecution = isRecord(testCase.execution) ? testCase.execution : {};
     testCase.execution = { ...existingExecution, threshold: test.threshold };

apps/cli/src/commands/import/promptfoo.test.ts

@@ -101,6 +101,36 @@ tests: file://./tests.jsonl
     expect(yaml).toContain('type: equals');
   });
 
+  it('rejects promptfoo test provider filters instead of emitting per-case targets', async () => {
+    const dir = await mkdtemp(path.join(tmpdir(), 'agentv-promptfoo-'));
+    tempDirs.push(dir);
+
+    const configPath = path.join(dir, 'promptfooconfig.yaml');
+    await writeFile(
+      configPath,
+      `
+prompts:
+  - "Answer {{question}}"
+providers:
+  - openai:gpt-5-mini
+  - anthropic:claude-sonnet
+tests:
+  - id: codex-only
+    provider: openai:gpt-5-mini
+    vars:
+      question: What is 2+2?
+    assert:
+      - type: equals
+        value: "4"
+`,
+      'utf8',
+    );
+
+    await expect(convertPromptfooToAgentvSuite({ inputPath: configPath })).rejects.toThrow(
+      'unsupported per-case target selection',
+    );
+  });
+
   it('imports promptfoo CSV datasets with __expected columns', async () => {
     const dir = await mkdtemp(path.join(tmpdir(), 'agentv-promptfoo-'));
     tempDirs.push(dir);

apps/cli/src/commands/import/promptfoo.ts

@@ -116,17 +116,20 @@ export async function convertPromptfooToAgentvSuite(
     readAssertionList(defaultTest.assert),
     absoluteInputPath,
   );
+  const suiteTargetNames =
+    filterProviders(providers, defaultTest.providers ?? defaultTest.provider) ??
+    providers.map((provider) => provider.targetName);
   const convertedTests = await buildAgentvTests({
     inputPath: absoluteInputPath,
     prompts,
-    providers,
     defaultTest,
     rawTests: testCases,
+    suiteTargetNames,
   });
 
   const execution: Record<string, unknown> = {};
-  if (providers.length > 0) {
-    execution.targets = providers.map((provider) => provider.targetName);
+  if (suiteTargetNames.length > 0) {
+    execution.targets = suiteTargetNames;
   }
 
   const suite: AgentvSuite = {
@@ -778,11 +781,11 @@ function parseCsvScalarValue(value: string): JsonValue {
 async function buildAgentvTests(options: {
   readonly inputPath: string;
   readonly prompts: readonly PromptfooPrompt[];
-  readonly providers: readonly PromptfooProvider[];
   readonly defaultTest: PromptfooTestCase;
   readonly rawTests: readonly PromptfooTestCase[];
+  readonly suiteTargetNames: readonly string[];
 }) {
-  const { inputPath, prompts, providers, defaultTest, rawTests } = options;
+  const { inputPath, prompts, defaultTest, rawTests, suiteTargetNames } = options;
   const tests: AgentvTest[] = [];
 
   for (let index = 0; index < rawTests.length; index++) {
@@ -801,12 +804,11 @@ async function buildAgentvTests(options: {
       throw new Error(`Test '${baseId}' matches no prompts after prompt filters`);
     }
 
-    const defaultTargets = filterProviders(
-      providers,
-      defaultTest.providers ?? defaultTest.provider,
-    );
-    const caseTargets = filterProviders(providers, rawTest.providers ?? rawTest.provider);
-    const effectiveTargets = caseTargets ?? defaultTargets;
+    if (rawTest.providers !== undefined || rawTest.provider !== undefined) {
+      throw new Error(
+        `Promptfoo test '${baseId}' uses provider filters, which require unsupported per-case target selection. Split provider-specific cases before importing or use defaultTest provider filters for suite-level target selection.`,
+      );
+    }
     const convertedCaseAssertions = await convertPromptfooAssertions(
       readAssertionList(rawTest.assert),
       inputPath,
@@ -830,11 +832,10 @@ async function buildAgentvTests(options: {
       const templatedInput = buildPromptTemplate(prompt, testOptions);
       const promptSuffix =
         promptSelection.length > 1 ? `--${sanitizeName(prompt.key || prompt.label)}` : '';
-      const metadata = buildPromptfooMetadata(rawTest, effectiveVars, prompt, effectiveTargets);
+      const metadata = buildPromptfooMetadata(rawTest, effectiveVars, prompt);
       const execution = buildCaseExecution({
         defaultAssertionsEnabled: !testOptions.disableDefaultAsserts,
         threshold: asNumber(rawTest.threshold),
-        effectiveTargets,
       });
 
       const test: AgentvTest = {
@@ -1003,14 +1004,12 @@ function buildPromptfooMetadata(
   rawTest: PromptfooTestCase,
   vars: Record<string, JsonValue>,
   prompt: PromptfooPrompt,
-  effectiveTargets: readonly string[] | undefined,
 ) {
   const rawMetadata = isJsonObject(rawTest.metadata) ? rawTest.metadata : undefined;
   const promptfooMetadata: Record<string, unknown> = {
     vars,
     prompt_label: prompt.label,
     prompt_source: prompt.source,
-    ...(effectiveTargets && effectiveTargets.length > 0 ? { targets: [...effectiveTargets] } : {}),
     ...(typeof rawTest.description === 'string' ? { description: rawTest.description } : {}),
   };
 
@@ -1023,7 +1022,6 @@ function buildPromptfooMetadata(
 function buildCaseExecution(options: {
   readonly defaultAssertionsEnabled: boolean;
   readonly threshold?: number;
-  readonly effectiveTargets?: readonly string[];
 }) {
   const execution: Record<string, unknown> = {};
   if (!options.defaultAssertionsEnabled) {
@@ -1032,9 +1030,6 @@ function buildCaseExecution(options: {
   if (options.threshold !== undefined) {
     execution.threshold = options.threshold;
   }
-  if (options.effectiveTargets && options.effectiveTargets.length > 0) {
-    execution.targets = [...options.effectiveTargets];
-  }
   return Object.keys(execution).length > 0 ? execution : undefined;
 }
 

apps/web/src/content/docs/docs/evaluation/eval-cases.mdx

@@ -27,7 +27,7 @@ tests:
 | `criteria` | Conditional | Description of what a correct response should contain. Required only when the case has no `expected_output` or `assertions` |
 | `input` | Yes | Input sent to the target (string, object, or message array) |
 | `expected_output` | No | Expected response for comparison (string, object, or message array) |
-| `execution` | No | Per-case execution overrides (for example `target`, `skip_defaults`) |
+| `execution` | No | Per-case execution overrides such as `skip_defaults` or `threshold`; target selection belongs in `experiment.target(s)` or CLI `--target` |
 | `workspace` | No | Per-case workspace config (overrides suite-level) |
 | `metadata` | No | Arbitrary key-value pairs passed to graders and workspace scripts |
 | `rubrics` | No | Structured evaluation criteria |
@@ -91,16 +91,16 @@ expected_output:
 
 ## Per-Case Execution Overrides
 
-Override the default target or graders for specific tests:
+Override graders or local scoring settings for specific tests. Do not put
+target selection in cases; use `experiment.target(s)`, CLI `--target`, separate
+eval suites, or tags/filters for target-specific cases.
 
 ```yaml
 tests:
   - id: complex-case
     criteria: Provides detailed explanation
     input: Explain quicksort algorithm
 
-    execution:
-      target: gpt4_target
     assertions:
       - name: depth_check
         type: llm-grader

apps/web/src/content/docs/docs/targets/configuration.mdx

@@ -61,20 +61,18 @@ already-exported secrets into `.env`.
 
 ## Referencing Targets in Evals
 
-Set the default target at the top level or override per case:
+Select targets at the eval runtime level with `experiment.target`,
+`experiment.targets`, legacy suite-level `execution.target(s)`, or CLI
+`--target`. Test cases do not choose targets; split target-specific cases into
+separate eval suites or select them with tags/filters.
 
 ```yaml
-# Top-level default
-execution:
-  target: azure-base
+experiment:
+  targets: [azure-base, vscode_dev]
 
 tests:
   - id: test-1
-    # Uses azure-base
-
   - id: test-2
-    execution:
-      target: vscode_dev  # Override for this case
 ```
 
 ## Grader Target

examples/features/basic-jsonl/evals/dataset.jsonl

@@ -1,5 +1,5 @@
 {"id": "code-review-javascript", "criteria": "Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT", "input": [{"role": "system", "content": "You are an expert software developer who provides clear, concise code reviews."}, {"role": "user", "content": [{"type": "text", "value": "Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n  let total = 0;\n  for (let i = 0; i < 0; i++) {\n    total += items[i].price * items[i].quantity;\n  }\n  return total;\n}\n```"}, {"type": "file", "value": "../basic/evals/javascript.instructions.md"}]}], "expected_output": [{"role": "assistant", "content": "The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}]}
-{"id": "code-gen-python", "conversation_id": "python-code-generation", "criteria": "AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON", "input": [{"role": "system", "content": "You are a code generator that follows specifications exactly."}, {"role": "user", "content": [{"type": "text", "value": "Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"}, {"type": "file", "value": "../basic/evals/python.instructions.md"}]}], "execution": {"target": "azure-llm"}}
+{"id": "code-gen-python", "conversation_id": "python-code-generation", "criteria": "AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON", "input": [{"role": "system", "content": "You are a code generator that follows specifications exactly."}, {"role": "user", "content": [{"type": "text", "value": "Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"}, {"type": "file", "value": "../basic/evals/python.instructions.md"}]}]}
 {"id": "feature-proposal-brainstorm", "criteria": "Assistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)", "input": [{"role": "system", "content": "You are a product strategist specializing in mobile health and fitness applications."}, {"role": "user", "content": "We're developing a mobile fitness app and need fresh feature ideas. Please brainstorm 3-5 innovative features."}]}
 {"id": "multiturn-debug-session", "criteria": "Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix.", "input": [{"role": "system", "content": "You are an expert debugging assistant."}, {"role": "user", "content": "I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n    result = []\n    for i in range(len(items) - 1):\n        result.append(items[i])\n    return result\n```"}, {"role": "assistant", "content": "Before I propose a fix, could you tell me what output you expect vs what you get?"}, {"role": "user", "content": "For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}], "expected_output": [{"role": "assistant", "content": "You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}]}
 {"id": "shorthand-string-example", "criteria": "Assistant correctly answers the math question", "input": "What is 2+2?", "expected_output": "The answer is 4."}

examples/features/basic/evals/dataset.eval.yaml

@@ -57,7 +57,7 @@ tests:
 
   # ==========================================
   # Example 2: Advanced features - conversation_id, multiple graders
-  # Demonstrates: conversation threading, execution config, target override, graders
+  # Demonstrates: conversation threading and per-test graders
   # Note: Optimization (ACE, etc.) is configured separately in opts/*.yaml files
   # ==========================================
   - id: code-gen-python-comprehensive
@@ -69,9 +69,6 @@ tests:
 
     criteria: AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON
 
-    execution:
-      target: llm
-
     # Multiple graders - supports both code-based and LLM graders
     assertions:
       - name: keyword_check

packages/core/src/evaluation/loaders/config-loader.ts

@@ -426,24 +426,6 @@ export function extractWorkersFromSuite(suite: JsonObject): number | undefined {
   return undefined;
 }
 
-/**
- * Extract per-test targets array from a raw test case object.
- */
-export function extractTargetsFromTestCase(testCase: JsonObject): readonly string[] | undefined {
-  const execution = testCase.execution;
-  if (!execution || typeof execution !== 'object' || Array.isArray(execution)) {
-    return undefined;
-  }
-
-  const targets = (execution as Record<string, unknown>).targets;
-  if (Array.isArray(targets)) {
-    const valid = targets.filter((t): t is string => typeof t === 'string' && t.trim().length > 0);
-    return valid.length > 0 ? valid.map((t) => t.trim()) : undefined;
-  }
-
-  return undefined;
-}
-
 /**
  * Cache configuration parsed from execution block.
  */

packages/core/src/evaluation/types.ts

@@ -1007,8 +1007,6 @@ export interface EvalTest {
   readonly workspace?: WorkspaceConfig;
   /** Arbitrary metadata passed to workspace scripts via stdin */
   readonly metadata?: Record<string, unknown>;
-  /** Per-test target override (matrix evaluation) */
-  readonly targets?: readonly string[];
   /** Per-test score threshold override (0-1). Resolution: CLI > test > suite > DEFAULT_THRESHOLD. */
   readonly threshold?: number;
   /** Scoped runtime interpretation/scheduling overrides. */

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -434,6 +434,8 @@ const ConversationTurnSchema = z.object({
 // Test case
 // ---------------------------------------------------------------------------
 
+const TestExecutionSchema = ExecutionSchema.omit({ target: true, targets: true }).strict();
+
 const EvalTestSchema = z.object({
   id: z.string().min(1),
   vars: JsonObjectSchema.optional(),
@@ -443,7 +445,7 @@ const EvalTestSchema = z.object({
   expected_output: ExpectedOutputSchema.optional(),
   assertions: z.array(EvaluatorSchema).optional(),
   evaluators: z.array(EvaluatorSchema).optional(),
-  execution: ExecutionSchema.optional(),
+  execution: TestExecutionSchema.optional(),
   run: RunOverrideSchema.optional(),
   workspace: WorkspaceSchema.optional(),
   metadata: z.record(z.unknown()).optional(),

packages/core/src/evaluation/validation/eval-validator.ts

@@ -68,6 +68,20 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([
 const KNOWN_INCLUDE_FIELDS = new Set(['include', 'type', 'select', 'run']);
 const KNOWN_RUN_OVERRIDE_FIELDS = new Set(['threshold', 'repeat', 'timeout_seconds', 'budget_usd']);
 const KNOWN_REPEAT_STRATEGIES = new Set(['pass_at_k', 'pass_all', 'mean', 'confidence_interval']);
+const KNOWN_TEST_EXECUTION_FIELDS = new Set([
+  'workers',
+  'assertions',
+  'evaluators',
+  'skip_defaults',
+  'cache',
+  'trials',
+  'budget_usd',
+  'budgetUsd',
+  'fail_on_error',
+  'failOnError',
+  'threshold',
+  'workspace',
+]);
 
 /**
  * Deprecated top-level fields with migration hints.
@@ -373,6 +387,7 @@ export async function validateEvalFile(filePath: string): Promise<ValidationResu
     // AgentV accepts Vercel-style PROMPT.md fallback beside EVAL.yaml or in input_files.
     const caseExecution = isObject(evalCase.execution) ? evalCase.execution : undefined;
     if (caseExecution) {
+      validateTestExecutionFields(caseExecution, absolutePath, errors, location);
       rejectRuntimeWorkspaceConfig(
         caseExecution.workspace,
         absolutePath,
@@ -480,6 +495,24 @@ async function validateSuiteWorkspaceConfigs(
   }
 }
 
+function validateTestExecutionFields(
+  caseExecution: JsonObject,
+  filePath: string,
+  errors: ValidationError[],
+  location: string,
+): void {
+  for (const key of Object.keys(caseExecution)) {
+    if (!KNOWN_TEST_EXECUTION_FIELDS.has(key)) {
+      errors.push({
+        severity: 'error',
+        filePath,
+        location: `${location}.execution.${key}`,
+        message: `Unsupported test execution field '${key}'.`,
+      });
+    }
+  }
+}
+
 function rejectRuntimeWorkspaceConfig(
   workspace: JsonValue | undefined,
   filePath: string,