From c43888ba89e279041f82d7ae90a9803df65bf5ad Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 6 Jul 2026 22:29:00 +1000 Subject: [PATCH] feat(eval): wire defaults.grader as a config-level grader fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promptfoo research (this session) confirmed grading providers there are a single layered override (assertion > test > suite default > env-detected default), fully decoupled from the providers-under-test list — never a per-provider field. AgentV's own design principles already state config-level grader selection through `defaults.grader` should be preferred over target-level `grader_target`, but `defaults.grader` was dead: validated at config-parse time, never consumed at eval-run time. Agent-provider targets (copilot-cli, codex, claude-cli, etc.) hard-required an explicit `grader_target` on every single target definition with no global fallback. - orchestrator.ts: thread a new `defaultGraderTarget` option through `RunEvaluationOptions` / `EvaluationRuntimeOptions`, inserted into the existing grader-resolution chain between a target's own `grader_target` and self-grading (`target.graderTarget ?? defaultGraderTarget ?? target.name`), and into the agent-provider guard so `defaults.grader` alone satisfies the "agent providers need a grader" requirement. - run-eval.ts: thread `.agentv/config.yaml`'s `defaults.grader` into the new option. - config-graph.ts: `defaults.target`/`defaults.grader` validation incorrectly required a match against this same config document's inline `targets:`/ `graders:` arrays — always empty in the common case where targets live in a separately-discovered `.agentv/targets.yaml`. Only validate when those arrays are non-empty; otherwise resolution (and its errors) happens lazily at eval-run time, same as CLI `--grader-target` already does. Verified live: a real copilot-cli agent target with no `grader_target`, `defaults.grader` set to a separate real Azure LLM target — previously threw "agent provider ... with no grader_target", now runs and grades correctly (grading.json shows `"target": "grader-llm"`). Co-Authored-By: Claude Sonnet 5 Entire-Checkpoint: 639d5c677078 --- apps/cli/src/commands/eval/run-eval.ts | 6 ++ .../src/evaluation/loaders/config-graph.ts | 14 ++- packages/core/src/evaluation/orchestrator.ts | 21 ++++- .../evaluation/loaders/config-loader.test.ts | 14 +++ .../core/test/evaluation/orchestrator.test.ts | 85 +++++++++++++++++++ 5 files changed, 132 insertions(+), 8 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 6643fff71..f680ef271 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -298,6 +298,8 @@ interface NormalizedOptions { /** Removed: the run directory always uses index.jsonl */ readonly outputFormat?: string; readonly graderTarget?: string; + /** Config-level fallback grader target name, from `.agentv/config.yaml`'s `defaults.grader`. */ + readonly defaultGraderTarget?: string; readonly model?: string; readonly outputMessages: number | 'all'; readonly threshold?: number; @@ -1710,6 +1712,7 @@ async function runSingleEvalFile(params: { runBudgetTracker, failOnError, graderTarget: options.graderTarget, + defaultGraderTarget: options.defaultGraderTarget, model: options.model, threshold: params.threshold, targetHooks: resolvedTargetSelection.targetHooks, @@ -1816,6 +1819,9 @@ export async function runEvalCommand( } let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); + if (yamlConfig?.defaults?.grader) { + options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader }; + } const resolvedExperiment = resolveExperimentForRun(options.experiment); const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : []; if (evalPathInputs.length === 0 && process.stdin.isTTY) { diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index 448fb875f..50696ac58 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -315,16 +315,22 @@ function parseExecution( } function validateDefaultSelections(graph: ComposableConfigGraph, configPath: string): void { - if (graph.defaults?.target !== undefined) { - const targetIds = new Set((graph.targets ?? []).map((target) => target.id)); + // Only validated against targets/graders defined inline in this same config + // document. `defaults.target`/`defaults.grader` may instead name a target + // defined in a separately-discovered `.agentv/targets.yaml`, which this + // graph has no visibility into — that case is resolved (and, on an unknown + // name, reported) lazily at eval-run time, the same way CLI + // `--grader-target` already is. + if (graph.defaults?.target !== undefined && graph.targets && graph.targets.length > 0) { + const targetIds = new Set(graph.targets.map((target) => target.id)); if (!targetIds.has(graph.defaults.target)) { throw new Error( `Invalid defaults.target in ${configPath}: '${graph.defaults.target}' does not match a configured target id.`, ); } } - if (graph.defaults?.grader !== undefined) { - const graderIds = new Set((graph.graders ?? []).map((grader) => grader.id)); + if (graph.defaults?.grader !== undefined && graph.graders && graph.graders.length > 0) { + const graderIds = new Set(graph.graders.map((grader) => grader.id)); if (!graderIds.has(graph.defaults.grader)) { throw new Error( `Invalid defaults.grader in ${configPath}: '${graph.defaults.grader}' does not match a configured grader id.`, diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index a356d3a1c..ae4e2e940 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -185,6 +185,8 @@ interface EvaluationRuntimeOptions { readonly providerFactory?: (target: ResolvedTarget) => Provider; readonly evalFilePath?: string; readonly graderTarget?: string; + /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */ + readonly defaultGraderTarget?: string; readonly model?: string; } @@ -203,6 +205,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR providerFactory, evalFilePath, graderTarget: cliGraderTarget, + defaultGraderTarget, model: cliModel, } = options; const resolvedTargetsByName = new Map(); @@ -262,7 +265,7 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR // TODO: When --model is provided without --grader-target, override the model of // whichever grader target is resolved. For now, --model only works with --grader-target agentv. - const graderName = targetContext.graderTarget ?? targetContext.name; + const graderName = targetContext.graderTarget ?? defaultGraderTarget ?? targetContext.name; const resolvedGrader = resolveTargetByName(graderName); if (!resolvedGrader) { // Only use the eval target as its own grader if it can return structured JSON. @@ -542,6 +545,8 @@ export interface RunEvaluationOptions { readonly retainOnFailure?: 'keep' | 'cleanup'; /** CLI override: grader target name (e.g., "agentv" or a target from targets.yaml) */ readonly graderTarget?: string; + /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */ + readonly defaultGraderTarget?: string; /** CLI override: model for grader target (e.g., "openai:gpt-5-mini") */ readonly model?: string; /** Per-test score threshold for pass/fail (default: 0.8) */ @@ -831,6 +836,7 @@ export async function runEvaluation( retainOnSuccess, retainOnFailure, graderTarget: cliGraderTarget, + defaultGraderTarget, model: cliModel, threshold: scoreThreshold, replayRecording, @@ -868,6 +874,7 @@ export async function runEvaluation( providerFactory, evalFilePath, graderTarget: cliGraderTarget, + defaultGraderTarget, model: cliModel, }); const { getOrCreateProvider, resolveGraderProvider, targetResolver, availableTargets } = runtime; @@ -875,10 +882,16 @@ export async function runEvaluation( // Validate grader_target: error if an agent provider would be used as grader. // Agent providers can't return structured JSON for grading — they respond with // tool calls and markdown, causing silent score-0 failures. - // CLI --grader-target override also satisfies this requirement. - if (isAgentProvider(getOrCreateProvider(target)) && !target.graderTarget && !cliGraderTarget) { + // CLI --grader-target override or config-level `defaults.grader` also satisfy + // this requirement. + if ( + isAgentProvider(getOrCreateProvider(target)) && + !target.graderTarget && + !cliGraderTarget && + !defaultGraderTarget + ) { throw new Error( - `Target "${target.name}" is an agent provider ("${target.kind}") with no grader_target — agent providers cannot return structured JSON for grading. Set grader_target to an LLM provider (e.g., azure-llm).`, + `Target "${target.name}" is an agent provider ("${target.kind}") with no grader_target — agent providers cannot return structured JSON for grading. Set grader_target on the target, pass --grader-target, or set defaults.grader in .agentv/config.yaml to an LLM provider (e.g., azure-llm).`, ); } diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index a9a814b0b..9eb99adee 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -357,6 +357,20 @@ describe('loadConfig', () => { } }); + it('allows defaults.target/defaults.grader to name a target from a separately-discovered targets.yaml (no inline targets/graders block)', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-defaults-')); + try { + const configPath = path.join(tempDir, 'config.yaml'); + writeFileSync(configPath, ['defaults:', ' target: llm', ' grader: grader', ''].join('\n')); + + const config = await loadComposableConfigGraph(configPath); + + expect(config.defaults).toEqual({ target: 'llm', grader: 'grader' }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('falls back to AGENTV_HOME/config.yaml when no project-local config exists', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-global-config-')); try { diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 2655cb114..3a749b579 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -2985,6 +2985,91 @@ describe('criteria with assertions runs only declared evaluators (#452)', () => }); }); +describe('defaultGraderTarget (config-level defaults.grader fallback)', () => { + it('uses defaultGraderTarget when the target has no grader_target of its own', async () => { + const answerProvider = new SequenceProvider('answer', { + responses: [ + { output: [{ role: 'assistant', content: 'Logging improved via structured logs.' }] }, + ], + }); + const graderProvider = new CapturingGraderProvider('grader', { + output: [ + { + role: 'assistant', + content: JSON.stringify({ score: 1, assertions: [{ text: 'ok', passed: true }] }), + }, + ], + }); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: { ...baseTarget, name: 'answer' }, + targets: [ + { name: 'answer', provider: 'mock' }, + { name: 'grader', provider: 'mock' }, + ], + defaultGraderTarget: 'grader', + providerFactory: (target) => (target.name === 'grader' ? graderProvider : answerProvider), + evaluators: undefined, + evalCases: [baseTestCase], + }); + + expect(results[0]?.score).toBe(1); + expect(graderProvider.lastRequest).toBeDefined(); + }); + + it("prefers the target's own grader_target over defaultGraderTarget", async () => { + const answerProvider = new SequenceProvider('answer', { + responses: [ + { output: [{ role: 'assistant', content: 'Logging improved via structured logs.' }] }, + ], + }); + const explicitGrader = new CapturingGraderProvider('explicit-grader', { + output: [ + { + role: 'assistant', + content: JSON.stringify({ score: 1, assertions: [{ text: 'ok', passed: true }] }), + }, + ], + }); + const fallbackGrader = new CapturingGraderProvider('fallback-grader', { + output: [ + { + role: 'assistant', + content: JSON.stringify({ + score: 0, + assertions: [{ text: 'should not run', passed: false }], + }), + }, + ], + }); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: { ...baseTarget, name: 'answer', graderTarget: 'explicit-grader' }, + targets: [ + { name: 'answer', provider: 'mock' }, + { name: 'explicit-grader', provider: 'mock' }, + { name: 'fallback-grader', provider: 'mock' }, + ], + defaultGraderTarget: 'fallback-grader', + providerFactory: (target) => { + if (target.name === 'explicit-grader') return explicitGrader; + if (target.name === 'fallback-grader') return fallbackGrader; + return answerProvider; + }, + evaluators: undefined, + evalCases: [baseTestCase], + }); + + expect(results[0]?.score).toBe(1); + expect(explicitGrader.lastRequest).toBeDefined(); + expect(fallbackGrader.lastRequest).toBeUndefined(); + }); +}); + describe('required gates', () => { const assertionTestCase: EvalTest = { id: 'required-gate-1',