feat(evaluation): support scenario file references

apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx

@@ -185,6 +185,7 @@ tests:
 | `env` | Promptfoo-compatible provider/eval environment-variable overrides and load-time template inputs. |
 | `extensions` | Lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. |
 | `tests` | Inline raw tests, a string path to an external raw-case file or directory, or a list mixing inline tests with raw-case file refs. |
+| `scenarios` | Promptfoo-style scenario matrix rows. Each inline scenario crosses `config` rows with `tests` rows; entries can also be `file://` refs or globs that load scenario arrays. |
 | `assert` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test |
 
 `environment` is what the agent can inspect or modify through tools, not prompt
@@ -282,6 +283,36 @@ External raw-case files referenced through `tests: file://...` may still contain
 raw internal `input` rows for compatibility. Keep that compatibility out of
 normal eval YAML authoring.
 
+### Scenarios
+
+Use `scenarios` when a small set of configuration rows should cross with a set
+of test rows. AgentV flattens inline scenario objects and `file://` scenario
+refs or globs into one scenario list, then lowers each scenario as
+`scenarios[].config x scenarios[].tests` before prompt expansion.
+
+```yaml
+prompts:
+  - "Translate '{{ phrase }}' to {{ language }}."
+
+scenarios:
+  - config:
+      - vars:
+          language: Spanish
+    tests:
+      - id: spanish-hello-world
+        vars:
+          phrase: hello world
+          expected_translation: hola mundo
+        assert:
+          - type: equals
+            value: "{{ expected_translation }}"
+  - file://scenarios/*.yaml
+```
+
+External scenario files must contain a YAML or JSON scenario array, or an
+object with a `scenarios` array. Scenario file refs are field-local to
+`scenarios`; AgentV does not expand arbitrary YAML fields as files.
+
 ### Lifecycle Extensions
 
 `extensions` uses AgentV lifecycle names. File hooks are local
@@ -535,11 +566,13 @@ mini-DSL (`contains:*`, `icontains:*`, `contains-any:*`, `contains-all:*`,
 `file://*.py`; file paths inside CSV cells are resolved relative to the CSV
 file). Unsupported assertion forms such as `similar:*` are rejected during
 validation instead of being skipped at runtime.
-`__provider_output` becomes first-class `expected_output` reference data,
-`__metric` names the generated assertions, `__threshold` sets the test threshold,
-`__metadata:<key>` adds metadata, and `__config:__expectedN:threshold` sets an
-assertion `min_score`. Ordinary columns become `vars`, so CSV rows can rely on
-suite-level `prompts` that interpolate those variables.
+`__provider_output` is rejected; use an explicit deterministic target such as
+`provider: cli` for fixed outputs, or a replay/fixture target for captured
+provider responses. `__metric` names the generated assertions, `__threshold`
+sets the test threshold, `__metadata:<key>` adds metadata, and
+`__config:__expectedN:threshold` sets an assertion `min_score`. Ordinary
+columns become `vars`, so CSV rows can rely on suite-level `prompts` that
+interpolate those variables.
 
 String shorthand is raw-case-only. Use a direct raw-case path or file ref when
 you want case data to run in the parent suite context:

examples/features/README.md

@@ -70,6 +70,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | Example | Description |
 |---------|-------------|
 | [external-datasets](external-datasets/) | Load test cases from YAML/JSONL files using `file://` references and globs |
+| [scenarios](scenarios/) | Build Promptfoo-style `scenarios[].config x scenarios[].tests` matrices with inline and file-backed scenarios |
 | [input-files-shorthand](input-files-shorthand/) | Attach files to every test using a compact shorthand |
 | [suite-level-input](suite-level-input/) | Prepend a shared system prompt to every test in the suite |
 | [suite-level-input-files](suite-level-input-files/) | Share file attachments across every test in the suite |
@@ -163,6 +164,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK |
 | [repo-lifecycle](repo-lifecycle/) | Workspace & targets |
 | [rubric](rubric/) | LLM grading |
+| [scenarios](scenarios/) | Dataset & prompt templates |
 | [sdk-config-file](sdk-config-file/) | TypeScript SDK |
 | [sdk-custom-assertion](sdk-custom-assertion/) | TypeScript SDK |
 | [sdk-eval-authoring](sdk-eval-authoring/) | TypeScript SDK |

examples/features/external-datasets/README.md

@@ -50,15 +50,16 @@ One JSON test object per line:
 CSV files use ordinary columns for `id`, `input`, and `vars`, plus promptfoo-style magic columns for assertions and metadata:
 
 ```csv
-id,input,__expected,__provider_output,__metric,__threshold,__metadata:source,locale
-csv-test,Reply with a greeting,icontains:hello,Hello there,greeting,0.8,csv,en-US
+id,input,__expected,__metric,__threshold,__metadata:source,locale
+csv-test,Reply with a greeting,icontains:hello,greeting,0.8,csv,en-US
 ```
 
 `__expected` and `__expectedN` become AgentV assertions for the supported CSV
 mini-DSL. `latency(<ms>)`, `cost(<usd>)`, and `file://*.py` map to runnable
 AgentV graders, with CSV file paths resolved relative to the CSV file;
 unsupported promptfoo forms such as `similar:*` are rejected during validation.
-`__provider_output` becomes AgentV `expected_output`; ordinary non-magic
+Use an explicit deterministic target such as `provider: cli` for fixed outputs,
+or a replay/fixture target for captured provider responses. Ordinary non-magic
 columns such as `locale` become `vars` and can be interpolated by suite-level
 `input`.
 

examples/features/external-datasets/evals/cases/magic.csv

@@ -1,2 +1,2 @@
-id,vars.input,__expected,__provider_output,__metric,__threshold,__metadata:source,locale
-csv-magic-greeting,Reply with a short greeting,icontains:hello,Hello there,greeting,0.8,csv,en-US
+id,vars.input,__expected,__metric,__threshold,__metadata:source,locale
+csv-magic-greeting,Reply with a short greeting,icontains:hello,greeting,0.8,csv,en-US

examples/features/scenarios/README.md

@@ -0,0 +1,79 @@
+# Scenarios Example
+
+Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current contract:
+top-level `prompts`, inline `scenarios`, `scenarios[].config`,
+`scenarios[].tests`, reference answers in `vars`, and explicit assertions.
+
+## What This Shows
+
+- Crossing each scenario `config` row with each scenario `tests` row
+- Mixing inline scenario objects with `file://` scenario refs
+- Loading scenario files through a glob
+- Keeping reference answers in `vars.expected_translation`
+- Consuming reference answers with explicit `equals` assertions
+- Running against a deterministic local CLI target
+
+## Expansion
+
+The main eval contains one inline Portuguese scenario and one file glob:
+
+```yaml
+scenarios:
+  - description: Inline Portuguese scenario
+    config:
+      - vars:
+          language: Portuguese
+    tests:
+      - id: inline-portuguese-hello
+        vars:
+          phrase: hello
+          expected_translation: ola
+        assert:
+          - type: equals
+            value: "{{ expected_translation }}"
+  - file://scenarios/*.yaml
+```
+
+The glob loads `scenarios/french.yaml` and `scenarios/spanish.yaml`. AgentV
+flattens those files into the top-level scenario list before lowering each
+scenario as `config x tests`.
+
+For example, the Spanish scenario has one config row and two tests:
+
+```yaml
+config:
+  - vars:
+      language: Spanish
+tests:
+  - id: spanish-hello-world
+    vars:
+      phrase: hello world
+      expected_translation: hola mundo
+```
+
+That row renders the prompt:
+
+```text
+Translate 'hello world' to Spanish.
+```
+
+The deterministic local CLI target returns `hola mundo`, and the assertion
+compares it to the reference answer from `vars.expected_translation`.
+
+## Running
+
+```bash
+# From repository root
+bun apps/cli/src/cli.ts validate examples/features/scenarios/evals/suite.yaml
+bun apps/cli/src/cli.ts eval run examples/features/scenarios/evals/suite.yaml \
+  --targets examples/features/scenarios/targets.yaml \
+  --target translation-cli
+```
+
+## Key Files
+
+- `evals/suite.yaml` - Main eval with inline and file-backed scenarios
+- `evals/scenarios/french.yaml` - Scenario file loaded by glob
+- `evals/scenarios/spanish.yaml` - Scenario file loaded by glob
+- `targets.yaml` - Deterministic CLI target for running the example
+- `scripts/translation-target.mjs` - Prompt-to-translation target script

examples/features/scenarios/evals/scenarios/french.yaml

@@ -0,0 +1,12 @@
+- description: French translation scenario
+  config:
+    - vars:
+        language: French
+  tests:
+    - id: french-hello
+      vars:
+        phrase: hello
+        expected_translation: bonjour
+      assert:
+        - type: equals
+          value: "{{ expected_translation }}"

examples/features/scenarios/evals/scenarios/spanish.yaml

@@ -0,0 +1,19 @@
+- description: Spanish translation scenario
+  config:
+    - vars:
+        language: Spanish
+  tests:
+    - id: spanish-hello-world
+      vars:
+        phrase: hello world
+        expected_translation: hola mundo
+      assert:
+        - type: equals
+          value: "{{ expected_translation }}"
+    - id: spanish-thank-you
+      vars:
+        phrase: thank you
+        expected_translation: gracias
+      assert:
+        - type: equals
+          value: "{{ expected_translation }}"

examples/features/scenarios/evals/suite.yaml

@@ -0,0 +1,19 @@
+name: scenarios-demo
+description: Demonstrates Promptfoo-style scenario matrices with inline and file-backed scenarios.
+version: "1.0"
+prompts:
+  - "Translate '{{ phrase }}' to {{ language }}."
+scenarios:
+  - description: Inline Portuguese scenario
+    config:
+      - vars:
+          language: Portuguese
+    tests:
+      - id: inline-portuguese-hello
+        vars:
+          phrase: hello
+          expected_translation: ola
+        assert:
+          - type: equals
+            value: "{{ expected_translation }}"
+  - file://scenarios/*.yaml

examples/features/scenarios/scripts/translation-target.mjs

@@ -0,0 +1,16 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const promptFile = process.argv[2];
+const outputFile = process.argv[3];
+
+const translations = new Map([
+  ["Translate 'hello' to Portuguese.", 'ola'],
+  ["Translate 'hello' to French.", 'bonjour'],
+  ["Translate 'hello world' to Spanish.", 'hola mundo'],
+  ["Translate 'thank you' to Spanish.", 'gracias'],
+]);
+
+const prompt = readFileSync(promptFile, 'utf8').trim();
+const text = translations.get(prompt) ?? `unexpected prompt: ${prompt}`;
+
+writeFileSync(outputFile, JSON.stringify({ text }));

examples/features/scenarios/targets.yaml

@@ -0,0 +1,4 @@
+targets:
+  - id: translation-cli
+    provider: cli
+    command: "node ../scripts/translation-target.mjs {PROMPT_FILE} {OUTPUT_FILE}"

packages/core/src/evaluation/loaders/case-file-loader.ts

@@ -357,7 +357,6 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] {
     let prefix = '';
     let suffix = '';
     let criteria: string | undefined;
-    let expectedOutput: string | undefined;
     let metric: string | undefined;
     let threshold: number | undefined;
 
@@ -379,7 +378,9 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] {
       } else if (key === '__description') {
         criteria = value;
       } else if (key === '__provider_output' || key === '__providerOutput') {
-        expectedOutput = value;
+        throw new Error(
+          `${key} has been removed from CSV imports. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.`,
+        );
       } else if (key === '__metric') {
         metric = value;
       } else if (key === '__threshold') {
@@ -435,7 +436,6 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] {
       id: id && id.trim() !== '' ? id : `row-${rowIndex + 1}`,
       ...(caseInput !== undefined ? { input: caseInput } : {}),
       ...(criteria ? { criteria } : {}),
-      ...(expectedOutput ? { expected_output: expectedOutput } : {}),
       ...(assertions.length > 0 ? { assert: assertions } : {}),
       ...(threshold !== undefined ? { threshold } : {}),
       ...(threshold !== undefined ? { execution: { threshold } } : {}),