docs(examples): clarify scenarios matrix use case

.agents/verification.md

@@ -200,6 +200,7 @@ grader dogfood is unavailable.
 - Red: run the scenario on `main` or the pre-change state and confirm the bug or missing feature is observable.
 - Green: run the identical scenario on your branch and confirm the fix or feature works from the end user's perspective.
 - Document both red and green evidence in the PR description or comments.
+- For feature examples, the green path must prove the intended use case, not just a smoke path. The example should make the feature's advantage visible compared with the simpler existing authoring pattern; if it only proves that syntax parses or a deterministic target returns output, improve the example before claiming dogfood evidence.
 
 5. Verify no regressions in adjacent areas.
 6. For scoring, threshold, or grader changes, run at least one real eval with a live provider and verify the output JSONL.

AGENTS.md

@@ -99,6 +99,7 @@ These baseline rules apply to every repo change. They summarize the most common
 - Prefer commit-addressed CI build artifacts over copying mutable main-tree build output. A prebuilt artifact is valid only when its manifest commit SHA, `bun.lock` hash, runner platform, Bun version expectation, and included output paths match the consuming checkout.
 - Implementation workers must rebuild any package whose source they changed; never trust a prebuilt artifact for a touched package, and never publish `node_modules`, Bun caches, `.turbo`, `.cache`, or `.tsbuildinfo` as the build artifact.
 - Public docs and examples should describe the current user-facing contract directly. Reserve historical context for files that are explicitly migration guides, changelogs, or ADRs. Do not reassure users that current fields are "not deprecated" or explain abandoned intermediate names; state the supported field and how to use it.
+- Feature examples and dogfood should demonstrate the real user-facing use case, not merely prove that a parser path or runtime smoke test works. If an example does not show why the feature is better than the simpler existing pattern, revise the example before merge.
 - Wire formats are `snake_case`; internal TypeScript is `camelCase`. Translate only at the boundary.
 - In AgentV, a `project` holds runs, traces, and experiments; a `benchmark` is a curated eval suite. Do not collapse those terms.
 - `artifact_pointers` are an offload indirection for large detached payload bytes, such as transcript artifacts. Do not use them as the discovery path for ordinary per-case sidecars; expose those with explicit index/manifest path fields such as `metrics_path`.

apps/cli/src/commands/eval/run-eval.ts

@@ -1260,6 +1260,18 @@ async function readExistingResultsFromRunDir(runDir: string): Promise<Evaluation
   return results;
 }
 
+export async function collectTerminalSummaryResults(params: {
+  readonly allResults: readonly EvaluationResult[];
+  readonly isResumeAppend: boolean;
+  readonly runDir: string;
+  readonly readExistingResults?: (runDir: string) => Promise<readonly EvaluationResult[]>;
+}): Promise<readonly EvaluationResult[]> {
+  const rawResults = params.isResumeAppend
+    ? await (params.readExistingResults ?? readExistingResultsFromRunDir)(params.runDir)
+    : params.allResults;
+  return deduplicateByTestIdTarget(rawResults);
+}
+
 async function resolveRerunFailedRunDir(cwd: string, source: string): Promise<string> {
   const trimmed = source.trim();
   if (!trimmed) {
@@ -2568,9 +2580,14 @@ export async function runEvalCommand(
     // Flush the output writer so all results are on disk before we read back.
     await outputWriter.close().catch(() => undefined);
 
-    // Compute summary from the persisted bundle indexes so resume includes old
-    // rows and normal runs reflect the same manifests Dashboard will read.
-    const summaryResults = deduplicateByTestIdTarget(await readExistingResultsFromRunDir(runDir));
+    // Normal runs summarize the completed in-memory results; the final artifact
+    // writer rewrites the bundle from this same set. Resume/append runs read the
+    // persisted bundle so the terminal summary includes old rows too.
+    const summaryResults = await collectTerminalSummaryResults({
+      allResults,
+      isResumeAppend,
+      runDir,
+    });
 
     const thresholdOpts =
       hasScopedRunPolicies || hasPerFileRuntimeThresholds

apps/cli/test/commands/eval/aggregate.test.ts

@@ -21,6 +21,7 @@ import {
   parseJsonlResults,
   writePerTestArtifacts,
 } from '../../../src/commands/eval/artifact-writer.js';
+import { collectTerminalSummaryResults } from '../../../src/commands/eval/run-eval.js';
 
 function makeResult(overrides: Partial<EvaluationResult> = {}): EvaluationResult {
   const result = {
@@ -194,6 +195,52 @@ describe('deduplicateByTestIdTarget', () => {
   });
 });
 
+describe('collectTerminalSummaryResults', () => {
+  it('uses in-memory results for normal runs so terminal summary matches completed cases', async () => {
+    const allResults = [
+      makeResult({ testId: 'spanish__scenario_test_1' }),
+      makeResult({ testId: 'spanish__scenario_test_2' }),
+      makeResult({ testId: 'french__scenario_test_1' }),
+      makeResult({ testId: 'french__scenario_test_2' }),
+      makeResult({ testId: 'portuguese__scenario_test_1' }),
+      makeResult({ testId: 'portuguese__scenario_test_2' }),
+    ];
+
+    const summaryResults = await collectTerminalSummaryResults({
+      allResults,
+      isResumeAppend: false,
+      runDir: '/tmp/incomplete-index',
+      readExistingResults: async () => allResults.slice(0, -1),
+    });
+
+    expect(summaryResults.map((result) => result.testId)).toEqual([
+      'spanish__scenario_test_1',
+      'spanish__scenario_test_2',
+      'french__scenario_test_1',
+      'french__scenario_test_2',
+      'portuguese__scenario_test_1',
+      'portuguese__scenario_test_2',
+    ]);
+  });
+
+  it('reads persisted results for resume append runs', async () => {
+    const allResults = [makeResult({ testId: 'new-case' })];
+    const persistedResults = [
+      makeResult({ testId: 'old-case' }),
+      makeResult({ testId: 'new-case' }),
+    ];
+
+    const summaryResults = await collectTerminalSummaryResults({
+      allResults,
+      isResumeAppend: true,
+      runDir: '/tmp/resume-run',
+      readExistingResults: async () => persistedResults,
+    });
+
+    expect(summaryResults.map((result) => result.testId)).toEqual(['old-case', 'new-case']);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // aggregateRunDir
 // ---------------------------------------------------------------------------

apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx

@@ -295,17 +295,31 @@ prompts:
   - "Translate '{{ phrase }}' to {{ language }}."
 
 scenarios:
-  - config:
-      - vars:
+  - description: Translation matrix
+    config:
+      - id: spanish
+        vars:
           language: Spanish
+          expected_hello: hola
+          expected_thank_you: gracias
+      - id: french
+        vars:
+          language: French
+          expected_hello: bonjour
+          expected_thank_you: merci
     tests:
-      - id: spanish-hello-world
+      - description: translates a greeting
+        vars:
+          phrase: hello
+        assert:
+          - type: equals
+            value: "{{ expected_hello }}"
+      - description: translates a courtesy phrase
         vars:
-          phrase: hello world
-          expected_translation: hola mundo
+          phrase: thank you
         assert:
           - type: equals
-            value: "{{ expected_translation }}"
+            value: "{{ expected_thank_you }}"
   - file://scenarios/*.yaml
 ```
 

examples/features/scenarios/README.md

@@ -1,64 +1,79 @@
 # Scenarios Example
 
-Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current contract:
-top-level `prompts`, inline `scenarios`, `scenarios[].config`,
-`scenarios[].tests`, reference answers in `vars`, and explicit assertions.
+Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current
+contract: top-level `prompts`, file-backed `scenarios`, `scenarios[].config`,
+`scenarios[].tests`, config-owned reference answers in `vars`, and explicit
+assertions.
 
 ## What This Shows
 
-- Crossing each scenario `config` row with each scenario `tests` row
-- Mixing inline scenario objects with `file://` scenario refs
-- Loading scenario files through a glob
-- Keeping reference answers in `vars.expected_translation`
-- Consuming reference answers with explicit `equals` assertions
+- Reusing one shared set of phrase tests across multiple language configs
+- Crossing three `config` rows with two `tests` rows to produce six cases
+- Loading the scenario matrix through a `file://` glob
+- Keeping per-language reference answers in config vars
+- Consuming config vars with explicit `equals` assertions
 - Running against a deterministic local CLI target
 
 ## Expansion
 
-The main eval contains one inline Portuguese scenario and one file glob:
+The main eval defines one prompt template and points `scenarios` at a file glob:
 
 ```yaml
+prompts:
+  - "Translate '{{ phrase }}' to {{ language }}."
 scenarios:
-  - description: Inline Portuguese scenario
-    config:
-      - vars:
-          language: Portuguese
-    tests:
-      - id: inline-portuguese-hello
-        vars:
-          phrase: hello
-          expected_translation: ola
-        assert:
-          - type: equals
-            value: "{{ expected_translation }}"
   - file://scenarios/*.yaml
 ```
 
-The glob loads `scenarios/french.yaml` and `scenarios/spanish.yaml`. AgentV
-flattens those files into the top-level scenario list before lowering each
-scenario as `config x tests`.
+The glob loads `scenarios/translation-matrix.yaml`. AgentV flattens that file
+into the top-level scenario list before lowering each scenario as
+`config x tests`.
 
-For example, the Spanish scenario has one config row and two tests:
+The scenario file keeps the changing data in `config` rows:
 
 ```yaml
 config:
-  - vars:
+  - id: spanish
+    vars:
       language: Spanish
-tests:
-  - id: spanish-hello-world
+      expected_hello: hola
+      expected_thank_you: gracias
+  - id: french
     vars:
-      phrase: hello world
-      expected_translation: hola mundo
+      language: French
+      expected_hello: bonjour
+      expected_thank_you: merci
 ```
 
-That row renders the prompt:
+The shared tests are written once:
 
-```text
-Translate 'hello world' to Spanish.
+```yaml
+tests:
+  - description: translates a greeting
+    vars: { phrase: hello }
+    assert:
+      - type: equals
+        value: "{{ expected_hello }}"
+  - description: translates a courtesy phrase
+    vars: { phrase: thank you }
+    assert:
+      - type: equals
+        value: "{{ expected_thank_you }}"
 ```
 
-The deterministic local CLI target returns `hola mundo`, and the assertion
-compares it to the reference answer from `vars.expected_translation`.
+That produces six concrete cases without duplicating the test content:
+
+| Config row | Test row | Rendered prompt | Expected value |
+| --- | --- | --- | --- |
+| `spanish` | greeting | `Translate 'hello' to Spanish.` | `hola` |
+| `spanish` | courtesy phrase | `Translate 'thank you' to Spanish.` | `gracias` |
+| `french` | greeting | `Translate 'hello' to French.` | `bonjour` |
+| `french` | courtesy phrase | `Translate 'thank you' to French.` | `merci` |
+| `portuguese` | greeting | `Translate 'hello' to Portuguese.` | `ola` |
+| `portuguese` | courtesy phrase | `Translate 'thank you' to Portuguese.` | `obrigado` |
+
+The deterministic local CLI target returns the translation, and the assertion
+compares it to the reference answer from the config row.
 
 ## Running
 
@@ -72,8 +87,7 @@ bun apps/cli/src/cli.ts eval run examples/features/scenarios/evals/suite.yaml \
 
 ## Key Files
 
-- `evals/suite.yaml` - Main eval with inline and file-backed scenarios
-- `evals/scenarios/french.yaml` - Scenario file loaded by glob
-- `evals/scenarios/spanish.yaml` - Scenario file loaded by glob
+- `evals/suite.yaml` - Main eval with a file-backed scenario matrix
+- `evals/scenarios/translation-matrix.yaml` - Scenario file loaded by glob
 - `targets.yaml` - Deterministic CLI target for running the example
 - `scripts/translation-target.mjs` - Prompt-to-translation target script

examples/features/scenarios/evals/scenarios/translation-matrix.yaml

@@ -0,0 +1,30 @@
+- description: Translation languages crossed with shared phrase tests
+  config:
+    - id: spanish
+      vars:
+        language: Spanish
+        expected_hello: hola
+        expected_thank_you: gracias
+    - id: french
+      vars:
+        language: French
+        expected_hello: bonjour
+        expected_thank_you: merci
+    - id: portuguese
+      vars:
+        language: Portuguese
+        expected_hello: ola
+        expected_thank_you: obrigado
+  tests:
+    - description: translates a greeting
+      vars:
+        phrase: hello
+      assert:
+        - type: equals
+          value: "{{ expected_hello }}"
+    - description: translates a courtesy phrase
+      vars:
+        phrase: thank you
+      assert:
+        - type: equals
+          value: "{{ expected_thank_you }}"

examples/features/scenarios/evals/suite.yaml

@@ -1,19 +1,7 @@
 name: scenarios-demo
-description: Demonstrates Promptfoo-style scenario matrices with inline and file-backed scenarios.
+description: Demonstrates Promptfoo-style scenario matrices with shared tests crossed against language config rows.
 version: "1.0"
 prompts:
   - "Translate '{{ phrase }}' to {{ language }}."
 scenarios:
-  - description: Inline Portuguese scenario
-    config:
-      - vars:
-          language: Portuguese
-    tests:
-      - id: inline-portuguese-hello
-        vars:
-          phrase: hello
-          expected_translation: ola
-        assert:
-          - type: equals
-            value: "{{ expected_translation }}"
   - file://scenarios/*.yaml

examples/features/scenarios/scripts/translation-target.mjs

@@ -4,10 +4,12 @@ const promptFile = process.argv[2];
 const outputFile = process.argv[3];
 
 const translations = new Map([
-  ["Translate 'hello' to Portuguese.", 'ola'],
-  ["Translate 'hello' to French.", 'bonjour'],
-  ["Translate 'hello world' to Spanish.", 'hola mundo'],
+  ["Translate 'hello' to Spanish.", 'hola'],
   ["Translate 'thank you' to Spanish.", 'gracias'],
+  ["Translate 'hello' to French.", 'bonjour'],
+  ["Translate 'thank you' to French.", 'merci'],
+  ["Translate 'hello' to Portuguese.", 'ola'],
+  ["Translate 'thank you' to Portuguese.", 'obrigado'],
 ]);
 
 const prompt = readFileSync(promptFile, 'utf8').trim();