feat(eval): wire defaults.grader as a config-level grader fallback

apps/cli/src/commands/eval/run-eval.ts

@@ -298,6 +298,8 @@ interface NormalizedOptions {
   /** Removed: the run directory always uses index.jsonl */
   readonly outputFormat?: string;
   readonly graderTarget?: string;
+  /** Config-level fallback grader target name, from `.agentv/config.yaml`'s `defaults.grader`. */
+  readonly defaultGraderTarget?: string;
   readonly model?: string;
   readonly outputMessages: number | 'all';
   readonly threshold?: number;
@@ -1710,6 +1712,7 @@ async function runSingleEvalFile(params: {
     runBudgetTracker,
     failOnError,
     graderTarget: options.graderTarget,
+    defaultGraderTarget: options.defaultGraderTarget,
     model: options.model,
     threshold: params.threshold,
     targetHooks: resolvedTargetSelection.targetHooks,
@@ -1816,6 +1819,9 @@ export async function runEvalCommand(
   }
 
   let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  if (yamlConfig?.defaults?.grader) {
+    options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader };
+  }
   const resolvedExperiment = resolveExperimentForRun(options.experiment);
   const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : [];
   if (evalPathInputs.length === 0 && process.stdin.isTTY) {

packages/core/src/evaluation/loaders/config-graph.ts

@@ -315,16 +315,22 @@ function parseExecution(
 }
 
 function validateDefaultSelections(graph: ComposableConfigGraph, configPath: string): void {
-  if (graph.defaults?.target !== undefined) {
-    const targetIds = new Set((graph.targets ?? []).map((target) => target.id));
+  // Only validated against targets/graders defined inline in this same config
+  // document. `defaults.target`/`defaults.grader` may instead name a target
+  // defined in a separately-discovered `.agentv/targets.yaml`, which this
+  // graph has no visibility into — that case is resolved (and, on an unknown
+  // name, reported) lazily at eval-run time, the same way CLI
+  // `--grader-target` already is.
+  if (graph.defaults?.target !== undefined && graph.targets && graph.targets.length > 0) {
+    const targetIds = new Set(graph.targets.map((target) => target.id));
     if (!targetIds.has(graph.defaults.target)) {
       throw new Error(
         `Invalid defaults.target in ${configPath}: '${graph.defaults.target}' does not match a configured target id.`,
       );
     }
   }
-  if (graph.defaults?.grader !== undefined) {
-    const graderIds = new Set((graph.graders ?? []).map((grader) => grader.id));
+  if (graph.defaults?.grader !== undefined && graph.graders && graph.graders.length > 0) {
+    const graderIds = new Set(graph.graders.map((grader) => grader.id));
     if (!graderIds.has(graph.defaults.grader)) {
       throw new Error(
         `Invalid defaults.grader in ${configPath}: '${graph.defaults.grader}' does not match a configured grader id.`,

packages/core/src/evaluation/orchestrator.ts

@@ -185,6 +185,8 @@ interface EvaluationRuntimeOptions {
   readonly providerFactory?: (target: ResolvedTarget) => Provider;
   readonly evalFilePath?: string;
   readonly graderTarget?: string;
+  /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */
+  readonly defaultGraderTarget?: string;
   readonly model?: string;
 }
 
@@ -203,6 +205,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR
     providerFactory,
     evalFilePath,
     graderTarget: cliGraderTarget,
+    defaultGraderTarget,
     model: cliModel,
   } = options;
   const resolvedTargetsByName = new Map<string, ResolvedTarget>();
@@ -262,7 +265,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR
     // TODO: When --model is provided without --grader-target, override the model of
     // whichever grader target is resolved. For now, --model only works with --grader-target agentv.
 
-    const graderName = targetContext.graderTarget ?? targetContext.name;
+    const graderName = targetContext.graderTarget ?? defaultGraderTarget ?? targetContext.name;
     const resolvedGrader = resolveTargetByName(graderName);
     if (!resolvedGrader) {
       // Only use the eval target as its own grader if it can return structured JSON.
@@ -542,6 +545,8 @@ export interface RunEvaluationOptions {
   readonly retainOnFailure?: 'keep' | 'cleanup';
   /** CLI override: grader target name (e.g., "agentv" or a target from targets.yaml) */
   readonly graderTarget?: string;
+  /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */
+  readonly defaultGraderTarget?: string;
   /** CLI override: model for grader target (e.g., "openai:gpt-5-mini") */
   readonly model?: string;
   /** Per-test score threshold for pass/fail (default: 0.8) */
@@ -831,6 +836,7 @@ export async function runEvaluation(
     retainOnSuccess,
     retainOnFailure,
     graderTarget: cliGraderTarget,
+    defaultGraderTarget,
     model: cliModel,
     threshold: scoreThreshold,
     replayRecording,
@@ -868,17 +874,24 @@ export async function runEvaluation(
     providerFactory,
     evalFilePath,
     graderTarget: cliGraderTarget,
+    defaultGraderTarget,
     model: cliModel,
   });
   const { getOrCreateProvider, resolveGraderProvider, targetResolver, availableTargets } = runtime;
 
   // Validate grader_target: error if an agent provider would be used as grader.
   // Agent providers can't return structured JSON for grading — they respond with
   // tool calls and markdown, causing silent score-0 failures.
-  // CLI --grader-target override also satisfies this requirement.
-  if (isAgentProvider(getOrCreateProvider(target)) && !target.graderTarget && !cliGraderTarget) {
+  // CLI --grader-target override or config-level `defaults.grader` also satisfy
+  // this requirement.
+  if (
+    isAgentProvider(getOrCreateProvider(target)) &&
+    !target.graderTarget &&
+    !cliGraderTarget &&
+    !defaultGraderTarget
+  ) {
     throw new Error(
-      `Target "${target.name}" is an agent provider ("${target.kind}") with no grader_target — agent providers cannot return structured JSON for grading. Set grader_target to an LLM provider (e.g., azure-llm).`,
+      `Target "${target.name}" is an agent provider ("${target.kind}") with no grader_target — agent providers cannot return structured JSON for grading. Set grader_target on the target, pass --grader-target, or set defaults.grader in .agentv/config.yaml to an LLM provider (e.g., azure-llm).`,
     );
   }
 

packages/core/test/evaluation/loaders/config-loader.test.ts

@@ -357,6 +357,20 @@ describe('loadConfig', () => {
     }
   });
 
+  it('allows defaults.target/defaults.grader to name a target from a separately-discovered targets.yaml (no inline targets/graders block)', async () => {
+    const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-defaults-'));
+    try {
+      const configPath = path.join(tempDir, 'config.yaml');
+      writeFileSync(configPath, ['defaults:', '  target: llm', '  grader: grader', ''].join('\n'));
+
+      const config = await loadComposableConfigGraph(configPath);
+
+      expect(config.defaults).toEqual({ target: 'llm', grader: 'grader' });
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
   it('falls back to AGENTV_HOME/config.yaml when no project-local config exists', async () => {
     const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-global-config-'));
     try {

packages/core/test/evaluation/orchestrator.test.ts

@@ -2985,6 +2985,91 @@ describe('criteria with assertions runs only declared evaluators (#452)', () =>
   });
 });
 
+describe('defaultGraderTarget (config-level defaults.grader fallback)', () => {
+  it('uses defaultGraderTarget when the target has no grader_target of its own', async () => {
+    const answerProvider = new SequenceProvider('answer', {
+      responses: [
+        { output: [{ role: 'assistant', content: 'Logging improved via structured logs.' }] },
+      ],
+    });
+    const graderProvider = new CapturingGraderProvider('grader', {
+      output: [
+        {
+          role: 'assistant',
+          content: JSON.stringify({ score: 1, assertions: [{ text: 'ok', passed: true }] }),
+        },
+      ],
+    });
+
+    const results = await runEvaluation({
+      testFilePath: 'in-memory.yaml',
+      repoRoot: 'in-memory',
+      target: { ...baseTarget, name: 'answer' },
+      targets: [
+        { name: 'answer', provider: 'mock' },
+        { name: 'grader', provider: 'mock' },
+      ],
+      defaultGraderTarget: 'grader',
+      providerFactory: (target) => (target.name === 'grader' ? graderProvider : answerProvider),
+      evaluators: undefined,
+      evalCases: [baseTestCase],
+    });
+
+    expect(results[0]?.score).toBe(1);
+    expect(graderProvider.lastRequest).toBeDefined();
+  });
+
+  it("prefers the target's own grader_target over defaultGraderTarget", async () => {
+    const answerProvider = new SequenceProvider('answer', {
+      responses: [
+        { output: [{ role: 'assistant', content: 'Logging improved via structured logs.' }] },
+      ],
+    });
+    const explicitGrader = new CapturingGraderProvider('explicit-grader', {
+      output: [
+        {
+          role: 'assistant',
+          content: JSON.stringify({ score: 1, assertions: [{ text: 'ok', passed: true }] }),
+        },
+      ],
+    });
+    const fallbackGrader = new CapturingGraderProvider('fallback-grader', {
+      output: [
+        {
+          role: 'assistant',
+          content: JSON.stringify({
+            score: 0,
+            assertions: [{ text: 'should not run', passed: false }],
+          }),
+        },
+      ],
+    });
+
+    const results = await runEvaluation({
+      testFilePath: 'in-memory.yaml',
+      repoRoot: 'in-memory',
+      target: { ...baseTarget, name: 'answer', graderTarget: 'explicit-grader' },
+      targets: [
+        { name: 'answer', provider: 'mock' },
+        { name: 'explicit-grader', provider: 'mock' },
+        { name: 'fallback-grader', provider: 'mock' },
+      ],
+      defaultGraderTarget: 'fallback-grader',
+      providerFactory: (target) => {
+        if (target.name === 'explicit-grader') return explicitGrader;
+        if (target.name === 'fallback-grader') return fallbackGrader;
+        return answerProvider;
+      },
+      evaluators: undefined,
+      evalCases: [baseTestCase],
+    });
+
+    expect(results[0]?.score).toBe(1);
+    expect(explicitGrader.lastRequest).toBeDefined();
+    expect(fallbackGrader.lastRequest).toBeUndefined();
+  });
+});
+
 describe('required gates', () => {
   const assertionTestCase: EvalTest = {
     id: 'required-gate-1',