fix(core): inherit default_test vars for prompt expansion

apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx

@@ -136,7 +136,7 @@ tests:
 | `category` | Optional slash-delimited analytics taxonomy path. Overrides the category derived from the eval file path. |
 | `target` | System under test by configured target `id` or inline target object |
 | `tags` | Optional metadata map. Use `tags.experiment` as the run/result grouping label. |
-| `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions. |
+| `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions rendered with `tests[].vars` and `default_test.vars`. |
 | `targets` | Optional target matrix. Entries reference target ids or inline target objects. |
 | `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` |
 | `evaluate_options` | Optional evaluation runtime options such as `budget_usd`, `repeat`, and `max_concurrency` |
@@ -160,11 +160,12 @@ context, but it does not materialize a repo for the agent to inspect.
 
 ### Prompts, Vars, and Target Expansion
 
-Use top-level `prompts` when you want a prompt matrix. AgentV
-renders each prompt with each test's `vars`, then expands the run as
-`prompts x targets x tests x repeat` before execution. Each expanded row keeps
-the original `test_id` plus prompt and target identity for Dashboard filtering,
-reruns, and comparisons.
+Use top-level `prompts` when you want the Promptfoo-compatible authoring shape:
+prompt templates at the top level, test data in `tests[].vars`, and shared
+test-data defaults in `default_test.vars`. AgentV renders each prompt with the
+merged vars for each test, then expands the run as `prompts x targets x tests x
+repeat` before execution. Each expanded row keeps the original `test_id` plus
+prompt and target identity for Dashboard filtering, reruns, and comparisons.
 
 ```yaml
 description: Release-note summarization
@@ -174,10 +175,14 @@ tags:
 prompts:
   - id: direct
     label: Direct
-    prompt: "Summarize {{ vars.topic }}."
+    prompt: "Summarize {{ topic }} for {{ audience }}."
   - id: terse
     label: Terse
-    prompt: "In one sentence, summarize {{ vars.topic }}."
+    prompt: "In one sentence, summarize {{ topic }} for {{ audience }}."
+
+default_test:
+  vars:
+    audience: engineers
 
 targets:
   - id: local-mini
@@ -200,12 +205,49 @@ tests:
     assert:
       - Identifies the most important change
       - Avoids unsupported details
+  - id: roadmap
+    vars:
+      audience: executives
+      topic: the next roadmap phase
+    expected_output: concise roadmap summary
+    assert:
+      - Identifies the main product direction
+      - Uses the requested audience framing
 ```
 
-If `prompts` is present, put per-case data in `tests[].vars` rather than
-`tests[].input`. For direct task suites, `input` remains the supported shorthand
-for the target task and can be a string, object, or message array. Use
-`prompts` only when you want a prompt matrix rendered from `tests[].vars`.
+For prompt matrices, put per-case data in `tests[].vars` and shared defaults in
+`default_test.vars`. `tests[].vars` overrides `default_test.vars` by key.
+Prompt templates can use either `{{ name }}` or `{{ vars.name }}` placeholders;
+the top-level form matches Promptfoo-style prompt templates, while the
+`vars.*` namespace is explicit and useful when a key might collide with other
+template context.
+
+Do not mix top-level `prompts` with direct input fields. `tests[].input`,
+`tests[].input_files`, top-level `input`, and top-level `input_files` are
+AgentV direct-input conveniences for suites that already know the target task
+input. They cannot be combined with top-level `prompts`; use `tests[].vars` for
+prompt-matrix data, or remove `prompts` for a direct-input suite.
+
+For simple direct-input text, this:
+
+```yaml
+tests:
+  - id: direct
+    input: Summarize the July release notes.
+```
+
+is conceptually equivalent to a prompt template such as
+`"{{ input }}"` or `"{{ vars.input }}"` with:
+
+```yaml
+tests:
+  - id: direct
+    vars:
+      input: Summarize the July release notes.
+```
+
+Use the direct shorthand for compact AgentV-native suites. Use top-level
+`prompts` when you want Promptfoo-compatible prompt and vars expansion.
 
 ### Lifecycle Extensions
 
@@ -380,7 +422,10 @@ To opt out for a specific test, set `execution.skip_defaults: true` (same flag t
 
 ### Suite-level Input Files
 
-The `input_files` field provides a shorthand for attaching shared file references to every test. When a test has a string `input`, the suite-level files are prepended as `type: file` content blocks in a single user message — the same shape produced by per-test `input_files`.
+The `input_files` field is an AgentV direct-input convenience for attaching
+shared file references to every test. When a test has a string `input`, the
+suite-level files are prepended as `type: file` content blocks in a single user
+message — the same shape produced by per-test `input_files`.
 
 ```yaml
 description: Schema review evaluation
@@ -401,6 +446,12 @@ Each test's effective input becomes a single user message with `[file blocks...,
 
 Per-test `input_files` overrides the suite-level value (it does not merge). To opt out, set `execution.skip_defaults: true` on the test.
 
+`input_files` cannot be mixed with top-level `prompts`. In Promptfoo-style
+prompt authoring, model file-backed context as vars whose values are file paths
+or `file://` references, then render those vars from the prompt template next to
+the test input. AgentV `input_files` is shorthand for the direct-input version
+of that pattern.
+
 ### PROMPT.md Fallback
 
 For directory-style evals, a test may omit `input` and keep the task prompt in
@@ -575,7 +626,10 @@ MY_REPO_COMMIT=main
 
 ## Per-Test Template Variables
 
-Eval YAML also supports per-test `vars` for data-driven prompt templates. Use `{{ vars.name }}` placeholders in test-facing text fields, and AgentV resolves them when the suite loads.
+Eval YAML also supports per-test `vars` for data-driven direct-input suites.
+Use `{{ vars.name }}` placeholders in test-facing text fields, and AgentV
+resolves them when the suite loads. Shared defaults can live in
+`default_test.vars`; per-test `vars` override those defaults by key.
 
 ```yaml
 input: "Answer clearly: {{ vars.question }}"
@@ -594,7 +648,7 @@ tests:
 
 ### Behavior
 
-- `vars` is defined per test as an object
+- `vars` is defined per test as an object, with optional defaults from `default_test.vars`
 - `{{ vars.name }}` and dotted paths like `{{ vars.user.name }}` are supported
 - Substitution applies to suite-level `input`, test `input`, `input_files`, `criteria`, `expected_output`, assertion values/metrics, and conversation turn `input` / `expected_output` / assertions
 - When the whole string is a single placeholder, the original JSON value is preserved

examples/features/README.md

@@ -74,7 +74,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [suite-level-input](suite-level-input/) | Prepend a shared system prompt to every test in the suite |
 | [suite-level-input-files](suite-level-input-files/) | Share file attachments across every test in the suite |
 | [env-interpolation](env-interpolation/) | Inject environment variables into eval config with `{{ env.VAR }}` |
-| [test-vars-templating](test-vars-templating/) | Inject per-test `vars` into `{{ vars.name }}` templates in eval fields |
+| [test-vars-templating](test-vars-templating/) | Render prompt templates and chat prompt files from `default_test.vars` and per-test `vars` |
 
 ---
 
@@ -169,7 +169,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the
 | [sdk-programmatic-api](sdk-programmatic-api/) | TypeScript SDK |
 | [suite-level-input](suite-level-input/) | Dataset & input |
 | [suite-level-input-files](suite-level-input-files/) | Dataset & input |
-| [test-vars-templating](test-vars-templating/) | Dataset & input |
+| [test-vars-templating](test-vars-templating/) | Dataset & prompt templates |
 | [threshold-grader](threshold-grader/) | LLM grading |
 | [tool-evaluation-plugins](tool-evaluation-plugins/) | Tool & agent evaluation |
 | [tool-trajectory-advanced](tool-trajectory-advanced/) | Tool & agent evaluation |

examples/features/test-vars-templating/README.md

@@ -1,16 +1,21 @@
-# Per-Test Vars Templating
+# Prompt and Vars Templating
 
-Demonstrates `tests[].vars` with `{{ vars.name }}` placeholders in eval files.
+Demonstrates prompt templates rendered from `default_test.vars` and
+`tests[].vars`, including a chat prompt file. The companion direct-input suite
+shows the AgentV shorthand forms for suites that do not need top-level prompts.
 
 ## Usage
 
 ```bash
 agentv eval examples/features/test-vars-templating/evals/suite.yaml
+agentv eval examples/features/test-vars-templating/evals/direct-input.eval.yaml
 ```
 
 ## Features
 
-- **Per-test data**: each test defines its own `vars` object
-- **Template substitution**: `{{ vars.question }}` and dotted paths like `{{ vars.expected.answer }}`
-- **Suite-level templates**: shared `input` can reference per-test vars too
+- **Prompt matrix data**: top-level `prompts` render with shared `default_test.vars` plus per-test `vars`
+- **Chat prompt files**: prompt files can contain role/content message arrays with `{{ name }}` placeholders
+- **Per-test overrides**: `tests[].vars` overrides default vars by key
+- **Template substitution**: `{{ question }}`, `{{ vars.question }}`, and dotted paths like `{{ vars.expected.answer }}`
+- **Direct input convenience**: direct suites can use string `input` or role/content message arrays without top-level prompts
 - **Separate from env interpolation**: `{{ vars.question }}` uses test data, `{{ env.VAR }}` uses environment variables

examples/features/test-vars-templating/evals/direct-input.eval.yaml

@@ -0,0 +1,28 @@
+# Direct-input convenience example
+#
+# Direct input suites do not use top-level prompts. Use string input shorthand
+# for compact single-message tasks, or role/content message arrays when the
+# target needs an explicit conversation shape.
+
+description: Demonstrates AgentV direct-input shorthand and role/content messages
+
+target: llm
+
+tests:
+  - id: direct-string
+    input: "Summarize the onboarding checklist in one sentence."
+    expected_output: concise onboarding summary
+    assert:
+      - Summarizes the request concisely
+
+  - id: direct-chat-messages
+    vars:
+      product: AgentV
+    input:
+      - role: system
+        content: "You answer product questions precisely."
+      - role: user
+        content: "Explain what {{ vars.product }} evaluates."
+    expected_output: AgentV evaluates agent workflows
+    assert:
+      - Explains that AgentV evaluates agent workflows

examples/features/test-vars-templating/evals/prompts/support-chat.json

@@ -0,0 +1,10 @@
+[
+  {
+    "role": "system",
+    "content": "You are a concise assistant answering {{ category }} questions for {{ audience }}."
+  },
+  {
+    "role": "user",
+    "content": "{{ question }}"
+  }
+]

examples/features/test-vars-templating/evals/suite.yaml

@@ -1,18 +1,28 @@
-# Per-test vars templating example
+# Prompt and vars templating example
 #
-# tests[].vars provides per-test data for {{ vars.name }} placeholders in eval fields.
-# Placeholders support dotted paths like {{ vars.expected.answer }}.
+# Top-level prompts define the prompt templates. tests[].vars provides per-test
+# data, and default_test.vars provides shared defaults. Prompt templates can use
+# Promptfoo-style {{ name }} placeholders or explicit {{ vars.name }} placeholders.
 #
 # Usage:
 #   agentv eval examples/features/test-vars-templating/evals/suite.yaml
 
-description: Demonstrates tests[].vars templating in eval fields
+description: Demonstrates prompt templates rendered from default_test.vars and tests[].vars
 
 target: llm
 
-input:
-  - role: system
-    content: "You are a concise assistant answering {{ vars.category }} questions."
+prompts:
+  - id: support-chat
+    label: Support chat
+    file: ./prompts/support-chat.json
+  - id: terse
+    label: Terse answer
+    prompt: "Answer for {{ audience }} in one sentence: {{ question }}"
+
+default_test:
+  vars:
+    audience: users
+    category: general
 
 tests:
   - id: capital-france
@@ -23,19 +33,17 @@ tests:
         answer: Paris
     assert:
       - "Answers {{ vars.question }} correctly"
-    input: "Question: {{ vars.question }}"
     expected_output: "{{ vars.expected.answer }}"
 
   - id: greet-ada
     vars:
-      category: etiquette
+      audience: new teammates
+      category: onboarding
       person:
         name: Ada
+      question: "How should I greet {{ vars.person.name }}?"
       expected:
         answer: Hello, Ada!
     assert:
       - "Greets {{ vars.person.name }} warmly"
-    input:
-      - role: user
-        content: "Say hello to {{ vars.person.name }}."
     expected_output: "{{ vars.expected.answer }}"

packages/core/src/evaluation/yaml-parser.ts

@@ -510,6 +510,40 @@ function combineInheritedAssertions(
   return parts.length > 0 ? parts : undefined;
 }
 
+function readDefaultTestVars(defaultTest: JsonValue | undefined): JsonObject | undefined {
+  if (!isJsonObject(defaultTest) || !isJsonObject(defaultTest.vars)) {
+    return undefined;
+  }
+  return defaultTest.vars;
+}
+
+function mergeDefaultTestVarsIntoCases(
+  rawCases: readonly JsonValue[],
+  defaultTest: JsonValue | undefined,
+): readonly JsonValue[] {
+  const defaultVars = readDefaultTestVars(defaultTest);
+  if (!defaultVars || Object.keys(defaultVars).length === 0) {
+    return rawCases;
+  }
+
+  return rawCases.map((rawCase) => {
+    if (!isJsonObject(rawCase)) {
+      return rawCase;
+    }
+    if (rawCase.vars !== undefined && !isJsonObject(rawCase.vars)) {
+      return rawCase;
+    }
+    const caseVars = isJsonObject(rawCase.vars) ? rawCase.vars : {};
+    return {
+      ...rawCase,
+      vars: {
+        ...defaultVars,
+        ...caseVars,
+      },
+    };
+  });
+}
+
 function isChatPromptArray(value: readonly JsonValue[]): boolean {
   return value.length > 0 && value.every((entry) => isJsonObject(entry) && isTestMessage(entry));
 }
@@ -1190,6 +1224,8 @@ async function loadTestsFromParsedYamlValue(
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'tests' field`);
   }
 
+  expandedTestCases = mergeDefaultTestVarsIntoCases(expandedTestCases, suite.default_test);
+
   const promptDefinitions = await parseSuitePrompts(suite.prompts, searchRoots);
   const promptExpansion = expandPromptMatrix(expandedTestCases, promptDefinitions, suite);
   expandedTestCases = promptExpansion.rawCases;