From e97018b273122f3f09d29f1702c00a0fe0cabaa6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 11:15:33 +0200 Subject: [PATCH 1/2] docs(examples): clarify scenarios matrix use case --- .agents/verification.md | 1 + AGENTS.md | 1 + .../docs/docs/next/evaluation/eval-files.mdx | 26 ++++-- examples/features/scenarios/README.md | 90 +++++++++++-------- .../scenarios/evals/scenarios/french.yaml | 12 --- .../scenarios/evals/scenarios/spanish.yaml | 19 ---- .../evals/scenarios/translation-matrix.yaml | 30 +++++++ examples/features/scenarios/evals/suite.yaml | 14 +-- .../scenarios/scripts/translation-target.mjs | 8 +- 9 files changed, 110 insertions(+), 91 deletions(-) delete mode 100644 examples/features/scenarios/evals/scenarios/french.yaml delete mode 100644 examples/features/scenarios/evals/scenarios/spanish.yaml create mode 100644 examples/features/scenarios/evals/scenarios/translation-matrix.yaml diff --git a/.agents/verification.md b/.agents/verification.md index 2921a19e6..aa78e9dfd 100644 --- a/.agents/verification.md +++ b/.agents/verification.md @@ -200,6 +200,7 @@ grader dogfood is unavailable. - Red: run the scenario on `main` or the pre-change state and confirm the bug or missing feature is observable. - Green: run the identical scenario on your branch and confirm the fix or feature works from the end user's perspective. - Document both red and green evidence in the PR description or comments. +- For feature examples, the green path must prove the intended use case, not just a smoke path. The example should make the feature's advantage visible compared with the simpler existing authoring pattern; if it only proves that syntax parses or a deterministic target returns output, improve the example before claiming dogfood evidence. 5. Verify no regressions in adjacent areas. 6. For scoring, threshold, or grader changes, run at least one real eval with a live provider and verify the output JSONL. diff --git a/AGENTS.md b/AGENTS.md index cba5b9a32..6a9844970 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,6 +99,7 @@ These baseline rules apply to every repo change. They summarize the most common - Prefer commit-addressed CI build artifacts over copying mutable main-tree build output. A prebuilt artifact is valid only when its manifest commit SHA, `bun.lock` hash, runner platform, Bun version expectation, and included output paths match the consuming checkout. - Implementation workers must rebuild any package whose source they changed; never trust a prebuilt artifact for a touched package, and never publish `node_modules`, Bun caches, `.turbo`, `.cache`, or `.tsbuildinfo` as the build artifact. - Public docs and examples should describe the current user-facing contract directly. Reserve historical context for files that are explicitly migration guides, changelogs, or ADRs. Do not reassure users that current fields are "not deprecated" or explain abandoned intermediate names; state the supported field and how to use it. +- Feature examples and dogfood should demonstrate the real user-facing use case, not merely prove that a parser path or runtime smoke test works. If an example does not show why the feature is better than the simpler existing pattern, revise the example before merge. - Wire formats are `snake_case`; internal TypeScript is `camelCase`. Translate only at the boundary. - In AgentV, a `project` holds runs, traces, and experiments; a `benchmark` is a curated eval suite. Do not collapse those terms. - `artifact_pointers` are an offload indirection for large detached payload bytes, such as transcript artifacts. Do not use them as the discovery path for ordinary per-case sidecars; expose those with explicit index/manifest path fields such as `metrics_path`. diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index 71975aeee..7b7ada926 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -295,17 +295,31 @@ prompts: - "Translate '{{ phrase }}' to {{ language }}." scenarios: - - config: - - vars: + - description: Translation matrix + config: + - id: spanish + vars: language: Spanish + expected_hello: hola + expected_thank_you: gracias + - id: french + vars: + language: French + expected_hello: bonjour + expected_thank_you: merci tests: - - id: spanish-hello-world + - description: translates a greeting + vars: + phrase: hello + assert: + - type: equals + value: "{{ expected_hello }}" + - description: translates a courtesy phrase vars: - phrase: hello world - expected_translation: hola mundo + phrase: thank you assert: - type: equals - value: "{{ expected_translation }}" + value: "{{ expected_thank_you }}" - file://scenarios/*.yaml ``` diff --git a/examples/features/scenarios/README.md b/examples/features/scenarios/README.md index e6bf50057..71da892e6 100644 --- a/examples/features/scenarios/README.md +++ b/examples/features/scenarios/README.md @@ -1,64 +1,79 @@ # Scenarios Example -Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current contract: -top-level `prompts`, inline `scenarios`, `scenarios[].config`, -`scenarios[].tests`, reference answers in `vars`, and explicit assertions. +Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current +contract: top-level `prompts`, file-backed `scenarios`, `scenarios[].config`, +`scenarios[].tests`, config-owned reference answers in `vars`, and explicit +assertions. ## What This Shows -- Crossing each scenario `config` row with each scenario `tests` row -- Mixing inline scenario objects with `file://` scenario refs -- Loading scenario files through a glob -- Keeping reference answers in `vars.expected_translation` -- Consuming reference answers with explicit `equals` assertions +- Reusing one shared set of phrase tests across multiple language configs +- Crossing three `config` rows with two `tests` rows to produce six cases +- Loading the scenario matrix through a `file://` glob +- Keeping per-language reference answers in config vars +- Consuming config vars with explicit `equals` assertions - Running against a deterministic local CLI target ## Expansion -The main eval contains one inline Portuguese scenario and one file glob: +The main eval defines one prompt template and points `scenarios` at a file glob: ```yaml +prompts: + - "Translate '{{ phrase }}' to {{ language }}." scenarios: - - description: Inline Portuguese scenario - config: - - vars: - language: Portuguese - tests: - - id: inline-portuguese-hello - vars: - phrase: hello - expected_translation: ola - assert: - - type: equals - value: "{{ expected_translation }}" - file://scenarios/*.yaml ``` -The glob loads `scenarios/french.yaml` and `scenarios/spanish.yaml`. AgentV -flattens those files into the top-level scenario list before lowering each -scenario as `config x tests`. +The glob loads `scenarios/translation-matrix.yaml`. AgentV flattens that file +into the top-level scenario list before lowering each scenario as +`config x tests`. -For example, the Spanish scenario has one config row and two tests: +The scenario file keeps the changing data in `config` rows: ```yaml config: - - vars: + - id: spanish + vars: language: Spanish -tests: - - id: spanish-hello-world + expected_hello: hola + expected_thank_you: gracias + - id: french vars: - phrase: hello world - expected_translation: hola mundo + language: French + expected_hello: bonjour + expected_thank_you: merci ``` -That row renders the prompt: +The shared tests are written once: -```text -Translate 'hello world' to Spanish. +```yaml +tests: + - description: translates a greeting + vars: { phrase: hello } + assert: + - type: equals + value: "{{ expected_hello }}" + - description: translates a courtesy phrase + vars: { phrase: thank you } + assert: + - type: equals + value: "{{ expected_thank_you }}" ``` -The deterministic local CLI target returns `hola mundo`, and the assertion -compares it to the reference answer from `vars.expected_translation`. +That produces six concrete cases without duplicating the test content: + +| Config row | Test row | Rendered prompt | Expected value | +| --- | --- | --- | --- | +| `spanish` | greeting | `Translate 'hello' to Spanish.` | `hola` | +| `spanish` | courtesy phrase | `Translate 'thank you' to Spanish.` | `gracias` | +| `french` | greeting | `Translate 'hello' to French.` | `bonjour` | +| `french` | courtesy phrase | `Translate 'thank you' to French.` | `merci` | +| `portuguese` | greeting | `Translate 'hello' to Portuguese.` | `ola` | +| `portuguese` | courtesy phrase | `Translate 'thank you' to Portuguese.` | `obrigado` | + +The deterministic local CLI target returns the translation, and the assertion +compares it to the reference answer from the config row. ## Running @@ -72,8 +87,7 @@ bun apps/cli/src/cli.ts eval run examples/features/scenarios/evals/suite.yaml \ ## Key Files -- `evals/suite.yaml` - Main eval with inline and file-backed scenarios -- `evals/scenarios/french.yaml` - Scenario file loaded by glob -- `evals/scenarios/spanish.yaml` - Scenario file loaded by glob +- `evals/suite.yaml` - Main eval with a file-backed scenario matrix +- `evals/scenarios/translation-matrix.yaml` - Scenario file loaded by glob - `targets.yaml` - Deterministic CLI target for running the example - `scripts/translation-target.mjs` - Prompt-to-translation target script diff --git a/examples/features/scenarios/evals/scenarios/french.yaml b/examples/features/scenarios/evals/scenarios/french.yaml deleted file mode 100644 index 6a4f1bb3f..000000000 --- a/examples/features/scenarios/evals/scenarios/french.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- description: French translation scenario - config: - - vars: - language: French - tests: - - id: french-hello - vars: - phrase: hello - expected_translation: bonjour - assert: - - type: equals - value: "{{ expected_translation }}" diff --git a/examples/features/scenarios/evals/scenarios/spanish.yaml b/examples/features/scenarios/evals/scenarios/spanish.yaml deleted file mode 100644 index 71fd22f1f..000000000 --- a/examples/features/scenarios/evals/scenarios/spanish.yaml +++ /dev/null @@ -1,19 +0,0 @@ -- description: Spanish translation scenario - config: - - vars: - language: Spanish - tests: - - id: spanish-hello-world - vars: - phrase: hello world - expected_translation: hola mundo - assert: - - type: equals - value: "{{ expected_translation }}" - - id: spanish-thank-you - vars: - phrase: thank you - expected_translation: gracias - assert: - - type: equals - value: "{{ expected_translation }}" diff --git a/examples/features/scenarios/evals/scenarios/translation-matrix.yaml b/examples/features/scenarios/evals/scenarios/translation-matrix.yaml new file mode 100644 index 000000000..4134ca5a2 --- /dev/null +++ b/examples/features/scenarios/evals/scenarios/translation-matrix.yaml @@ -0,0 +1,30 @@ +- description: Translation languages crossed with shared phrase tests + config: + - id: spanish + vars: + language: Spanish + expected_hello: hola + expected_thank_you: gracias + - id: french + vars: + language: French + expected_hello: bonjour + expected_thank_you: merci + - id: portuguese + vars: + language: Portuguese + expected_hello: ola + expected_thank_you: obrigado + tests: + - description: translates a greeting + vars: + phrase: hello + assert: + - type: equals + value: "{{ expected_hello }}" + - description: translates a courtesy phrase + vars: + phrase: thank you + assert: + - type: equals + value: "{{ expected_thank_you }}" diff --git a/examples/features/scenarios/evals/suite.yaml b/examples/features/scenarios/evals/suite.yaml index 60b8d568d..e32223279 100644 --- a/examples/features/scenarios/evals/suite.yaml +++ b/examples/features/scenarios/evals/suite.yaml @@ -1,19 +1,7 @@ name: scenarios-demo -description: Demonstrates Promptfoo-style scenario matrices with inline and file-backed scenarios. +description: Demonstrates Promptfoo-style scenario matrices with shared tests crossed against language config rows. version: "1.0" prompts: - "Translate '{{ phrase }}' to {{ language }}." scenarios: - - description: Inline Portuguese scenario - config: - - vars: - language: Portuguese - tests: - - id: inline-portuguese-hello - vars: - phrase: hello - expected_translation: ola - assert: - - type: equals - value: "{{ expected_translation }}" - file://scenarios/*.yaml diff --git a/examples/features/scenarios/scripts/translation-target.mjs b/examples/features/scenarios/scripts/translation-target.mjs index 8b944d686..ce0b8c844 100644 --- a/examples/features/scenarios/scripts/translation-target.mjs +++ b/examples/features/scenarios/scripts/translation-target.mjs @@ -4,10 +4,12 @@ const promptFile = process.argv[2]; const outputFile = process.argv[3]; const translations = new Map([ - ["Translate 'hello' to Portuguese.", 'ola'], - ["Translate 'hello' to French.", 'bonjour'], - ["Translate 'hello world' to Spanish.", 'hola mundo'], + ["Translate 'hello' to Spanish.", 'hola'], ["Translate 'thank you' to Spanish.", 'gracias'], + ["Translate 'hello' to French.", 'bonjour'], + ["Translate 'thank you' to French.", 'merci'], + ["Translate 'hello' to Portuguese.", 'ola'], + ["Translate 'thank you' to Portuguese.", 'obrigado'], ]); const prompt = readFileSync(promptFile, 'utf8').trim(); From ea2cb75418c2cc606c0379ad55d8f086aa124b67 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 11:15:43 +0200 Subject: [PATCH 2/2] fix(cli): summarize completed normal-run results --- apps/cli/src/commands/eval/run-eval.ts | 23 +++++++-- apps/cli/test/commands/eval/aggregate.test.ts | 47 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 9f054b727..b93d8819e 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1260,6 +1260,18 @@ async function readExistingResultsFromRunDir(runDir: string): Promise Promise; +}): Promise { + const rawResults = params.isResumeAppend + ? await (params.readExistingResults ?? readExistingResultsFromRunDir)(params.runDir) + : params.allResults; + return deduplicateByTestIdTarget(rawResults); +} + async function resolveRerunFailedRunDir(cwd: string, source: string): Promise { const trimmed = source.trim(); if (!trimmed) { @@ -2568,9 +2580,14 @@ export async function runEvalCommand( // Flush the output writer so all results are on disk before we read back. await outputWriter.close().catch(() => undefined); - // Compute summary from the persisted bundle indexes so resume includes old - // rows and normal runs reflect the same manifests Dashboard will read. - const summaryResults = deduplicateByTestIdTarget(await readExistingResultsFromRunDir(runDir)); + // Normal runs summarize the completed in-memory results; the final artifact + // writer rewrites the bundle from this same set. Resume/append runs read the + // persisted bundle so the terminal summary includes old rows too. + const summaryResults = await collectTerminalSummaryResults({ + allResults, + isResumeAppend, + runDir, + }); const thresholdOpts = hasScopedRunPolicies || hasPerFileRuntimeThresholds diff --git a/apps/cli/test/commands/eval/aggregate.test.ts b/apps/cli/test/commands/eval/aggregate.test.ts index e0543c781..500212640 100644 --- a/apps/cli/test/commands/eval/aggregate.test.ts +++ b/apps/cli/test/commands/eval/aggregate.test.ts @@ -21,6 +21,7 @@ import { parseJsonlResults, writePerTestArtifacts, } from '../../../src/commands/eval/artifact-writer.js'; +import { collectTerminalSummaryResults } from '../../../src/commands/eval/run-eval.js'; function makeResult(overrides: Partial = {}): EvaluationResult { const result = { @@ -194,6 +195,52 @@ describe('deduplicateByTestIdTarget', () => { }); }); +describe('collectTerminalSummaryResults', () => { + it('uses in-memory results for normal runs so terminal summary matches completed cases', async () => { + const allResults = [ + makeResult({ testId: 'spanish__scenario_test_1' }), + makeResult({ testId: 'spanish__scenario_test_2' }), + makeResult({ testId: 'french__scenario_test_1' }), + makeResult({ testId: 'french__scenario_test_2' }), + makeResult({ testId: 'portuguese__scenario_test_1' }), + makeResult({ testId: 'portuguese__scenario_test_2' }), + ]; + + const summaryResults = await collectTerminalSummaryResults({ + allResults, + isResumeAppend: false, + runDir: '/tmp/incomplete-index', + readExistingResults: async () => allResults.slice(0, -1), + }); + + expect(summaryResults.map((result) => result.testId)).toEqual([ + 'spanish__scenario_test_1', + 'spanish__scenario_test_2', + 'french__scenario_test_1', + 'french__scenario_test_2', + 'portuguese__scenario_test_1', + 'portuguese__scenario_test_2', + ]); + }); + + it('reads persisted results for resume append runs', async () => { + const allResults = [makeResult({ testId: 'new-case' })]; + const persistedResults = [ + makeResult({ testId: 'old-case' }), + makeResult({ testId: 'new-case' }), + ]; + + const summaryResults = await collectTerminalSummaryResults({ + allResults, + isResumeAppend: true, + runDir: '/tmp/resume-run', + readExistingResults: async () => persistedResults, + }); + + expect(summaryResults.map((result) => result.testId)).toEqual(['old-case', 'new-case']); + }); +}); + // --------------------------------------------------------------------------- // aggregateRunDir // ---------------------------------------------------------------------------