From ef24b82f187580f4d225768edb518913123b7e05 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 09:51:50 +0200 Subject: [PATCH 1/2] fix(sdk): align eval config providers surface --- .../content/docs/docs/next/evaluation/sdk.mdx | 19 +++- .../src/evaluation/loaders/grader-parser.ts | 35 +++++- .../src/evaluation/loaders/ts-eval-loader.ts | 7 +- packages/core/src/evaluation/yaml-parser.ts | 7 +- .../loaders/fixtures/default-export.eval.ts | 20 ++-- .../fixtures/eval-config-named.eval.ts | 6 +- .../loaders/fixtures/legacy-graders.eval.ts | 13 +++ .../loaders/fixtures/legacy-target.eval.ts | 12 ++ .../loaders/fixtures/legacy-targets.eval.ts | 12 ++ .../loaders/fixtures/module.eval.mts | 2 +- .../loaders/fixtures/named-config.eval.ts | 6 +- .../loaders/fixtures/relative-import.eval.ts | 2 +- .../loaders/fixtures/sdk-define-eval.eval.ts | 58 +++++++++- .../evaluation/loaders/ts-eval-loader.test.ts | 47 ++++++-- packages/sdk/README.md | 23 +++- packages/sdk/src/eval.ts | 39 ++++++- packages/sdk/src/index.ts | 4 + packages/sdk/src/target-client.ts | 8 +- packages/sdk/test/eval-authoring.test.ts | 105 +++++++++++++++++- 19 files changed, 375 insertions(+), 50 deletions(-) create mode 100644 packages/core/test/evaluation/loaders/fixtures/legacy-graders.eval.ts create mode 100644 packages/core/test/evaluation/loaders/fixtures/legacy-target.eval.ts create mode 100644 packages/core/test/evaluation/loaders/fixtures/legacy-targets.eval.ts diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index f8e7dba65..19b6863c2 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -97,13 +97,28 @@ import { graders, type EvalConfig } from '@agentv/sdk'; const config: EvalConfig = { name: 'hello-suite', - providers: ['mock-sdk'], + providers: [ + { id: 'mock', label: 'mock-sdk', config: { response: 'Hello from the mock provider' } }, + { id: 'openai:gpt-5-mini', label: 'grader-provider' }, + ], + defaults: { + provider: 'mock-sdk', + grader: 'grader-provider', + }, + defaultTest: { + options: { + provider: 'grader-provider', + }, + }, prompts: ['{{ task }}'], tests: [ { id: 'hello', vars: { task: 'Say hello' }, inputFiles: ['../fixtures/per-test-note.md'], + options: { + provider: 'grader-provider', + }, assert: [graders.contains('Hello')], }, ], @@ -120,6 +135,8 @@ Useful companion helpers: The durable authored field remains `assert`. TypeScript eval config authoring does not introduce a second YAML vocabulary. +TypeScript eval configs use the same provider surface as YAML: top-level `providers` defines both systems under test and reusable grader providers, `providers[].id` names the backend/spec, `providers[].label` is the stable AgentV identity, and `defaults.provider` / `defaults.grader` select the default candidate and grader. Per-test grader provider selection belongs in `defaultTest.options.provider`, `tests[].options.provider`, or assertion-level `provider`. + ## Built-In Grader Helpers `@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand. diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index d806781c8..116286405 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -238,6 +238,7 @@ export async function parseGraders( ): Promise { const execution = rawEvalCase.execution; const executionObject = isJsonObject(execution) ? execution : undefined; + const inheritedAssertionConfig = inheritedAssertionConfigFromOptions(rawEvalCase.options); // Case-level graders priority: assert > execution assert. const caseEvaluators = @@ -255,7 +256,7 @@ export async function parseGraders( evalId, defaultPreprocessors, defaultRubricPrompt, - undefined, + inheritedAssertionConfig, inheritedGraderTarget, ); // Parse root-level evaluators (appended after case-level) @@ -265,7 +266,7 @@ export async function parseGraders( evalId, defaultPreprocessors, defaultRubricPrompt, - undefined, + inheritedAssertionConfig, inheritedGraderTarget, ); @@ -1872,21 +1873,45 @@ function withInheritedAssertionConfig( inheritedConfig?: JsonObject, ): JsonObject { const ownConfig = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined; - if (!inheritedConfig && !ownConfig) { + const inheritedProvider = + typeof inheritedConfig?.provider === 'string' && inheritedConfig.provider.trim().length > 0 + ? inheritedConfig.provider.trim() + : undefined; + const inheritedConfigWithoutProvider = inheritedConfig + ? Object.fromEntries(Object.entries(inheritedConfig).filter(([key]) => key !== 'provider')) + : undefined; + const inheritedConfigForConfig = + inheritedConfigWithoutProvider && Object.keys(inheritedConfigWithoutProvider).length > 0 + ? inheritedConfigWithoutProvider + : undefined; + if (!inheritedConfigForConfig && !ownConfig && inheritedProvider === undefined) { return rawEvaluator; } const mergedConfig = { - ...(inheritedConfig ?? {}), + ...(inheritedConfigForConfig ?? {}), ...(ownConfig ?? {}), }; return { ...rawEvaluator, - config: mergedConfig, + ...(rawEvaluator.provider === undefined && inheritedProvider !== undefined + ? { provider: inheritedProvider } + : {}), + ...(Object.keys(mergedConfig).length > 0 ? { config: mergedConfig } : {}), }; } +function inheritedAssertionConfigFromOptions( + options: JsonValue | undefined, +): JsonObject | undefined { + if (!isJsonObject(options)) { + return undefined; + } + const provider = typeof options.provider === 'string' ? options.provider.trim() : ''; + return provider.length > 0 ? { provider } : undefined; +} + interface ParsedPromptField { readonly prompt?: string; readonly promptPath?: string; diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index c58ac913f..587b98c0a 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -30,6 +30,7 @@ const KNOWN_SNAKE_CASE_KEYS = { budgetUsd: 'budget_usd', conversationId: 'conversation_id', costLimitUsd: 'cost_limit_usd', + defaultTest: 'default_test', dependsOn: 'depends_on', earlyExit: 'early_exit', expectedOutput: 'expected_output', @@ -45,6 +46,7 @@ const KNOWN_SNAKE_CASE_KEYS = { outputPath: 'output_path', readOnly: 'read_only', reasoningEffort: 'reasoning_effort', + rubricPrompt: 'rubric_prompt', skipDefaults: 'skip_defaults', timeoutMs: 'timeout_ms', timeoutSeconds: 'timeout_seconds', @@ -193,10 +195,7 @@ function isProgrammaticEvalConfig(value: unknown): value is ProgrammaticEvalConf function lowerTypeScriptEvalConfig(config: Record): Record { const lowered = lowerEvalYamlValue(config) as Record; - const { budget_usd: budgetUsd, repeat, target, ...withoutRuntimeAliases } = lowered; - if (target !== undefined && withoutRuntimeAliases.providers === undefined) { - withoutRuntimeAliases.providers = [target]; - } + const { budget_usd: budgetUsd, repeat, ...withoutRuntimeAliases } = lowered; if (budgetUsd === undefined && repeat === undefined) { return withoutRuntimeAliases; } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index b33835873..dd770ef0c 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -2653,7 +2653,12 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO } if (suite.model !== undefined) { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'model' is not part of eval YAML. Put model inside the target object.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'model' is not part of eval YAML. Put model inside the relevant providers[].config object.`, + ); + } + if ((suite as Record).graders !== undefined) { + throw new Error( + `Invalid eval runtime config in ${evalFilePath}: top-level 'graders' has been removed. Put grader providers in 'providers' and select them with defaults.grader, default_test.options.provider, tests[].options.provider, or assertion provider.`, ); } if (suite.runs !== undefined) { diff --git a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts index 09603e881..a4fb171bc 100644 --- a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts @@ -1,11 +1,14 @@ -import type { EvalConfig } from '../../../../src/evaluation/evaluate.js'; - -const config: EvalConfig = { - metadata: { - name: 'default-export-suite', - tags: ['sdk', 'typescript'], - }, +const config = { + name: 'default-export-suite', + tags: ['sdk', 'typescript'], prompts: ['{{ input }}'], + providers: [ + { + id: 'mock', + label: 'inline-provider', + config: { response: 'hello there' }, + }, + ], tests: [ { id: 'greeting', @@ -13,11 +16,8 @@ const config: EvalConfig = { assert: [{ type: 'contains', value: 'hello' }], }, ], - cache: false, - cachePath: '.agentv/ts-eval-cache', budgetUsd: 1.5, threshold: 0.9, - target: { name: 'inline-target', provider: 'mock', response: 'hello there' }, }; export default config; diff --git a/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts b/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts index 845532ed1..60bf18bd8 100644 --- a/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts @@ -1,7 +1,6 @@ -import type { EvalConfig } from '../../../../src/evaluation/evaluate.js'; - -export const evalConfig: EvalConfig = { +export const evalConfig = { prompts: ['{{ input }}'], + providers: ['mock-provider'], tests: [ { id: 'eval-config-named', @@ -9,5 +8,4 @@ export const evalConfig: EvalConfig = { assert: [{ type: 'contains', value: 'hello' }], }, ], - target: { provider: 'mock_agent' }, }; diff --git a/packages/core/test/evaluation/loaders/fixtures/legacy-graders.eval.ts b/packages/core/test/evaluation/loaders/fixtures/legacy-graders.eval.ts new file mode 100644 index 000000000..a796d4cf8 --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/legacy-graders.eval.ts @@ -0,0 +1,13 @@ +export default { + name: 'legacy-graders', + providers: ['mock-provider'], + graders: [{ id: 'mock', label: 'grader-provider' }], + prompts: ['{{ input }}'], + tests: [ + { + id: 'legacy-graders', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; diff --git a/packages/core/test/evaluation/loaders/fixtures/legacy-target.eval.ts b/packages/core/test/evaluation/loaders/fixtures/legacy-target.eval.ts new file mode 100644 index 000000000..ab10e534b --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/legacy-target.eval.ts @@ -0,0 +1,12 @@ +export default { + name: 'legacy-target', + target: 'mock-target', + prompts: ['{{ input }}'], + tests: [ + { + id: 'legacy-target', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; diff --git a/packages/core/test/evaluation/loaders/fixtures/legacy-targets.eval.ts b/packages/core/test/evaluation/loaders/fixtures/legacy-targets.eval.ts new file mode 100644 index 000000000..d6c83faed --- /dev/null +++ b/packages/core/test/evaluation/loaders/fixtures/legacy-targets.eval.ts @@ -0,0 +1,12 @@ +export default { + name: 'legacy-targets', + targets: ['mock-target'], + prompts: ['{{ input }}'], + tests: [ + { + id: 'legacy-targets', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], +}; diff --git a/packages/core/test/evaluation/loaders/fixtures/module.eval.mts b/packages/core/test/evaluation/loaders/fixtures/module.eval.mts index 01c9dc136..e3a5a7779 100644 --- a/packages/core/test/evaluation/loaders/fixtures/module.eval.mts +++ b/packages/core/test/evaluation/loaders/fixtures/module.eval.mts @@ -1,6 +1,6 @@ const config = { name: 'module-mts-config', - target: 'mock-target', + providers: ['mock-provider'], prompts: ['{{ input }}'], tests: [ { diff --git a/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts b/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts index 6b0e9fe52..25920eb65 100644 --- a/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts @@ -1,7 +1,6 @@ -import type { EvalConfig } from '../../../../src/evaluation/evaluate.js'; - -export const config: EvalConfig = { +export const config = { prompts: ['{{ input }}'], + providers: ['mock-provider'], tests: [ { id: 'named-config', @@ -9,5 +8,4 @@ export const config: EvalConfig = { assert: [{ type: 'contains', value: 'hello' }], }, ], - target: { provider: 'mock_agent' }, }; diff --git a/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts index 403991806..59bf88c9d 100644 --- a/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/relative-import.eval.ts @@ -2,7 +2,7 @@ import { relativePrompt } from './relative-prompt.ts'; const config = { name: 'relative-import-ts-config', - target: 'mock-target', + providers: ['mock-provider'], tags: { experiment: 'ts-config', group: 'loader' }, prompts: [relativePrompt], budgetUsd: 1, diff --git a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts index 960f5b6f0..fc156b6a3 100644 --- a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts @@ -5,7 +5,27 @@ const suite = { name: 'sdk-define-eval-suite', description: 'YAML-aligned TypeScript suite authored with @agentv/sdk', tags: ['sdk', 'typescript', 'yaml'], - target: 'mock-target', + providers: [ + { + id: 'mock', + label: 'sdk-provider', + config: { response: 'hello there' }, + }, + { + id: 'openai:gpt-5-mini', + label: 'grader-provider', + config: { api_key: '{{ env.OPENAI_API_KEY }}' }, + }, + ], + defaults: { + provider: 'sdk-provider', + grader: 'grader-provider', + }, + defaultTest: { + options: { + provider: 'grader-provider', + }, + }, budgetUsd: 2, threshold: 0.75, prompts: ['{{ input }}'], @@ -21,7 +41,14 @@ const suite = { id: 'sdk-define-eval', vars: { input: 'Say hello' }, expectedOutput: 'hello there', - assert: [{ type: 'contains', value: 'hello' }], + options: { + provider: 'test-grader', + }, + assert: [ + { type: 'contains', value: 'hello' }, + { type: 'llm-rubric', value: 'Greets the user' }, + { type: 'llm-rubric', value: 'Uses concise language', provider: 'assertion-grader' }, + ], workspace: { hooks: { beforeEach: { @@ -31,6 +58,11 @@ const suite = { }, }, }, + { + id: 'sdk-default-test-provider', + vars: { input: 'Say hello again' }, + assert: [{ type: 'llm-rubric', value: 'Greets the user' }], + }, ], }; @@ -44,7 +76,13 @@ export default Object.defineProperties(suite, { name: suite.name, description: suite.description, tags: suite.tags, - target: suite.target, + providers: suite.providers, + defaults: suite.defaults, + default_test: { + options: { + provider: suite.defaultTest.options.provider, + }, + }, evaluate_options: { budget_usd: suite.budgetUsd, }, @@ -62,7 +100,14 @@ export default Object.defineProperties(suite, { id: 'sdk-define-eval', vars: { input: 'Say hello' }, expected_output: 'hello there', - assert: [{ type: 'contains', value: 'hello' }], + options: { + provider: 'test-grader', + }, + assert: [ + { type: 'contains', value: 'hello' }, + { type: 'llm-rubric', value: 'Greets the user' }, + { type: 'llm-rubric', value: 'Uses concise language', provider: 'assertion-grader' }, + ], workspace: { hooks: { before_each: { @@ -72,6 +117,11 @@ export default Object.defineProperties(suite, { }, }, }, + { + id: 'sdk-default-test-provider', + vars: { input: 'Say hello again' }, + assert: [{ type: 'llm-rubric', value: 'Greets the user' }], + }, ], }), enumerable: false, diff --git a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts index 076965819..89dfde66b 100644 --- a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts +++ b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts @@ -5,6 +5,7 @@ import { isTypeScriptEvalConfigFileName, loadTsEvalFile, } from '../../../src/evaluation/loaders/ts-eval-loader.js'; +import type { LlmRubricGraderConfig } from '../../../src/evaluation/types.js'; import { loadTestSuite, loadTests } from '../../../src/evaluation/yaml-parser.js'; const fixtureDir = path.join(import.meta.dir, 'fixtures'); @@ -54,11 +55,20 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].suite).toBe('default-export-suite'); expect(suite.tests[0].category).toBe('sdk'); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript']); - expect(suite.cacheConfig?.enabled).toBe(false); - expect(suite.cacheConfig?.cachePath).toBe('.agentv/ts-eval-cache'); expect(suite.budgetUsd).toBe(1.5); expect(suite.threshold).toBe(0.9); - expect(suite.inlineTarget?.name).toBe('inline-target'); + expect(suite.targetRefs).toEqual([ + { + name: 'inline-provider', + id: 'mock', + label: 'inline-provider', + definition: expect.objectContaining({ + name: 'inline-provider', + provider: 'mock', + response: 'hello there', + }), + }, + ]); }); it('materializes *.eval.ts default exports with relative imports', async () => { @@ -67,7 +77,7 @@ describe('loadTsEvalFile', () => { expect(suite.tests).toHaveLength(1); expect(suite.tests[0].id).toBe('relative-import'); expect(suite.tests[0].input).toEqual([{ role: 'user', content: 'Say hello' }]); - expect(suite.targetRefs).toEqual([{ name: 'mock-target' }]); + expect(suite.targetRefs).toEqual([{ name: 'mock-provider' }]); expect(suite.budgetUsd).toBe(1); expect(suite.experimentConfig?.repeat?.count).toBe(2); expect(suite.tags).toEqual({ experiment: 'ts-config', group: 'loader' }); @@ -95,17 +105,40 @@ describe('loadTsEvalFile', () => { }, ); - expect(suite.tests).toHaveLength(1); + expect(suite.tests).toHaveLength(2); expect(suite.tests[0].suite).toBe('sdk-define-eval-suite'); expect(suite.tests[0].workspace?.hooks?.before_all?.command).toEqual(['echo', 'suite-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.command).toEqual(['echo', 'case-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.timeout_ms).toBe(1_000); - expect(suite.targetRefs).toEqual([{ name: 'mock-target' }]); - expect(suite.targets).toEqual(['mock-target']); + expect(suite.targetRefs?.map((ref) => ref.name)).toEqual(['sdk-provider', 'grader-provider']); + expect(suite.targets).toEqual(['sdk-provider', 'grader-provider']); expect(suite.workers).toBeUndefined(); expect(suite.budgetUsd).toBe(2); expect(suite.threshold).toBe(0.75); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript', 'yaml']); + const firstCaseRubrics = suite.tests[0].assertions?.filter( + (assertion): assertion is LlmRubricGraderConfig => assertion.type === 'llm-rubric', + ); + expect(firstCaseRubrics?.map((assertion) => assertion.target)).toEqual([ + 'test-grader', + 'assertion-grader', + ]); + const defaultCaseRubrics = suite.tests[1].assertions?.filter( + (assertion): assertion is LlmRubricGraderConfig => assertion.type === 'llm-rubric', + ); + expect(defaultCaseRubrics?.map((assertion) => assertion.target)).toEqual(['grader-provider']); + }); + + it('rejects stale target-era authoring in YAML-aligned TypeScript evals', async () => { + await expect( + loadTestSuite(path.join(fixtureDir, 'legacy-target.eval.ts'), fixtureDir), + ).rejects.toThrow(/Top-level 'target'.*providers/); + await expect( + loadTestSuite(path.join(fixtureDir, 'legacy-targets.eval.ts'), fixtureDir), + ).rejects.toThrow(/Top-level 'targets'.*providers/); + await expect( + loadTestSuite(path.join(fixtureDir, 'legacy-graders.eval.ts'), fixtureDir), + ).rejects.toThrow(/top-level 'graders'.*providers/); }); it('routes TypeScript evals through loadTests', async () => { diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 18a1427e9..89c4b48ed 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -167,13 +167,28 @@ import { graders, type EvalConfig } from '@agentv/sdk'; const config: EvalConfig = { name: 'hello-suite', - target: 'mock-sdk', + providers: [ + { id: 'mock', label: 'mock-sdk', config: { response: 'Hello from the mock provider' } }, + { id: 'openai:gpt-5-mini', label: 'grader-provider' }, + ], + defaults: { + provider: 'mock-sdk', + grader: 'grader-provider', + }, + defaultTest: { + options: { + provider: 'grader-provider', + }, + }, prompts: ['{{ input }}'], tests: [ { id: 'hello', vars: { input: 'Say hello' }, - expectedOutput: 'Hello from the mock target', + expectedOutput: 'Hello from the mock provider', + options: { + provider: 'grader-provider', + }, assert: [graders.contains('Hello')], }, ], @@ -184,6 +199,8 @@ export default config; AgentV loads explicit `*.eval.ts` and `*.eval.mts` files through the same core loader used for YAML evals. The supported TypeScript contract is a default-exported `EvalConfig`. `defineEval(config)` is available as a thin optional helper over the same shape; plain typed default exports are the recommended path. +TypeScript eval configs use the same provider surface as YAML: author systems under test and reusable grader providers in top-level `providers`, use `id` for the backend/spec, use `label` as the stable AgentV identity, and select defaults with `defaults.provider` and `defaults.grader`. Per-test grader provider selection belongs in `defaultTest.options.provider`, `tests[].options.provider`, or assertion-level `provider`. + ### Grader helpers Use the `graders` catalog when you want TypeScript helpers for common AgentV grader configs without creating a new eval vocabulary: @@ -266,7 +283,7 @@ Python workflows should emit canonical YAML/JSONL or implement script graders ov - `AssertionContext`, `AssertionScore` - Assertion types - `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` - Grader types - `TraceSummary`, `Message`, `ToolCall` - Trace data types -- `createTargetClient()` - LLM target proxy for graders +- `createTargetClient()` - Runtime target proxy for script graders that explicitly opt into target access; this is not eval authoring syntax. - `z` - Re-exported Zod for custom config schemas ## Documentation diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 5ecbd6996..3f1766478 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -34,6 +34,7 @@ const KNOWN_SNAKE_CASE_KEYS = { outputPath: 'output_path', readOnly: 'read_only', reasoningEffort: 'reasoning_effort', + rubricPrompt: 'rubric_prompt', scoreRange: 'score_range', scoreRanges: 'score_ranges', skipDefaults: 'skip_defaults', @@ -74,6 +75,7 @@ export interface EvalMessage { export interface EvalAssertionConfig { readonly type: string; + readonly provider?: string | true | object; readonly [key: string]: unknown; } @@ -151,18 +153,43 @@ export interface EvalProviderConfig { readonly prompts?: unknown; readonly transform?: unknown; readonly delay?: number; + readonly inputs?: unknown; readonly env?: Readonly>; readonly reasoningEffort?: string; readonly hooks?: EvalLifecycleHooks; readonly [key: string]: unknown; } +export type EvalProviderMap = Readonly< + Record< + string, + Omit | EvalProviderRef | Readonly> + > +>; + +export type EvalProviderEntry = string | EvalProviderRef | EvalProviderConfig | EvalProviderMap; + export interface EvalDefaultsConfig { readonly provider?: string; readonly grader?: string; readonly [key: string]: unknown; } +export interface EvalTestOptions { + readonly provider?: string; + readonly transform?: unknown; + readonly repeat?: EvalRepeat; + readonly rubricPrompt?: unknown; + readonly [key: string]: unknown; +} + +export interface EvalDefaultTest { + readonly vars?: Readonly>; + readonly assert?: readonly (string | EvalAssertionConfig)[]; + readonly options?: EvalTestOptions; + readonly [key: string]: unknown; +} + export interface EvalTrials { readonly count: number; readonly strategy?: 'pass_any' | 'pass_all' | 'mean' | 'confidence_interval'; @@ -174,7 +201,7 @@ export type EvalRepeat = EvalTrials; export interface EvalExecution { readonly provider?: string; - readonly providers?: readonly (string | EvalProviderRef | EvalProviderConfig)[]; + readonly providers?: readonly EvalProviderEntry[]; readonly assert?: readonly EvalAssertionConfig[]; readonly skipDefaults?: boolean; readonly cache?: boolean; @@ -198,6 +225,7 @@ export interface EvalTest { readonly inputFiles?: readonly string[]; readonly expectedOutput?: string | Readonly> | readonly EvalMessage[]; readonly assert?: readonly EvalAssertionConfig[]; + readonly options?: EvalTestOptions; readonly execution?: EvalExecution; readonly environment?: EvalEnvironment | string; readonly metadata?: Readonly>; @@ -236,9 +264,9 @@ export interface EvalConfig { readonly requires?: EvalRequires; readonly inputFiles?: readonly string[]; readonly prompts?: unknown; - readonly providers?: readonly (string | EvalProviderRef | EvalProviderConfig)[]; + readonly providers?: readonly EvalProviderEntry[]; readonly defaults?: EvalDefaultsConfig; - readonly defaultTest?: Readonly>; + readonly defaultTest?: EvalDefaultTest | string; readonly tests: readonly EvalTest[] | string; /** * @deprecated A top-level `experiment` label no longer sets the run's @@ -339,6 +367,11 @@ function validateTopLevelRuntimeFields(definition: EvalConfig): void { "defineEval() does not accept top-level 'preprocessors'. Use defaultTest.options.transform or assertion-level transform instead.", ); } + if (Object.prototype.hasOwnProperty.call(rawDefinition, 'graders')) { + throw new Error( + "defineEval() no longer accepts top-level 'graders'. Put grader providers in 'providers' and select them with defaults.grader, defaultTest.options.provider, tests[].options.provider, or assertion provider.", + ); + } if ( Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment') && typeof rawDefinition.experiment !== 'string' diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 5b9e89d06..2adfa0e80 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -147,6 +147,7 @@ export { type EvalDockerEnvironment, type EvalDockerEnvironmentMount, type EvalDockerEnvironmentResources, + type EvalDefaultTest, type EvalEnvironment, type EvalEnvironmentSetup, type EvalExecution, @@ -156,9 +157,12 @@ export { type EvalMessage, type EvalMessageContent, type EvalProviderConfig, + type EvalProviderEntry, + type EvalProviderMap, type EvalProviderRef, type EvalRequires, type EvalTest, + type EvalTestOptions, type EvalTrials, type EvalTurn, type LowerEvalYamlValue, diff --git a/packages/sdk/src/target-client.ts b/packages/sdk/src/target-client.ts index a7e3c26ca..f07c4d9a9 100644 --- a/packages/sdk/src/target-client.ts +++ b/packages/sdk/src/target-client.ts @@ -1,5 +1,11 @@ /** - * Client for invoking configured targets from script-grader scripts. + * Runtime-only client for invoking configured providers from script-grader scripts + * through AgentV's target proxy. + * + * This module keeps the historical target-proxy vocabulary because it talks to + * runtime internals. It is not TypeScript eval authoring syntax; eval configs + * should select systems under test and grader providers through `providers`, + * `defaults.provider`, `defaults.grader`, and assertion `provider`. * * Environment variables (set automatically by AgentV when `target` config is present): * - AGENTV_TARGET_PROXY_URL: The URL of the local proxy server diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 8f9308191..70edd8dad 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'bun:test'; -import { defineEval, serializeEvalYaml, toEvalYamlObject } from '../src/eval.js'; +import { type EvalConfig, defineEval, serializeEvalYaml, toEvalYamlObject } from '../src/eval.js'; describe('YAML-aligned eval authoring helpers', () => { it('lowers known AgentV fields to canonical snake_case without broad key rewriting', () => { @@ -230,6 +230,97 @@ describe('YAML-aligned eval authoring helpers', () => { expect(lowered.tags).toEqual({ experiment: 'sdk-baseline', team: 'compliance' }); }); + it('authors the Promptfoo-shaped provider and grader-provider surface', () => { + const config = { + name: 'sdk-provider-surface', + providers: [ + 'openai:gpt-4.1-mini', + { + id: 'agentv:codex-cli', + label: 'codex-host', + config: { model: 'gpt-5-codex' }, + env: { AGENTV_MODE: 'test' }, + inputs: { cwd: './workspace' }, + }, + { + 'openai:gpt-5-mini': { + label: 'grader-provider', + config: { temperature: 0 }, + }, + }, + ], + defaults: { + provider: 'codex-host', + grader: 'grader-provider', + }, + defaultTest: { + vars: { tone: 'brief' }, + options: { + provider: 'grader-provider', + transform: 'output.trim()', + }, + }, + prompts: ['{{ task }}'], + tests: [ + { + id: 'provider-options', + vars: { task: 'Say hello' }, + options: { + provider: 'case-grader', + }, + assert: [ + { type: 'llm-rubric', value: 'Greets the user' }, + { type: 'llm-rubric', value: 'Uses concise language', provider: 'assertion-grader' }, + ], + }, + ], + } satisfies EvalConfig; + + const lowered = toEvalYamlObject(defineEval(config)); + + expect(lowered).toMatchObject({ + providers: [ + 'openai:gpt-4.1-mini', + { + id: 'agentv:codex-cli', + label: 'codex-host', + config: { model: 'gpt-5-codex' }, + env: { AGENTV_MODE: 'test' }, + inputs: { cwd: './workspace' }, + }, + { + 'openai:gpt-5-mini': { + label: 'grader-provider', + config: { temperature: 0 }, + }, + }, + ], + defaults: { + provider: 'codex-host', + grader: 'grader-provider', + }, + default_test: { + vars: { tone: 'brief' }, + options: { + provider: 'grader-provider', + transform: 'output.trim()', + }, + }, + tests: [ + { + id: 'provider-options', + options: { + provider: 'case-grader', + }, + assert: [ + { type: 'llm-rubric', value: 'Greets the user' }, + { type: 'llm-rubric', value: 'Uses concise language', provider: 'assertion-grader' }, + ], + }, + ], + }); + }); + it('keeps the list form of tags for selection', () => { const suite = defineEval({ name: 'sdk-tags-list', @@ -339,4 +430,16 @@ describe('YAML-aligned eval authoring helpers', () => { } as never), ).toThrow(/target.*provider/); }); + + it('rejects removed top-level graders authoring', () => { + expect(() => + defineEval({ + name: 'removed-graders', + providers: ['mock-provider'], + graders: [{ id: 'openai:gpt-5-mini', label: 'grader-provider' }], + prompts: ['{{ input }}'], + tests: [{ id: 'hello', vars: { input: 'Say hello' } }], + } as never), + ).toThrow(/top-level 'graders'.*providers/); + }); }); From 1ab307bf48043fff99d3c8495d8b185150e6d6a7 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 10:06:05 +0200 Subject: [PATCH 2/2] fix(sdk): use inline TS providers during evaluate --- packages/core/src/evaluation/evaluate.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index 00a0ee7a2..87984840b 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -222,6 +222,7 @@ export interface MaterializedEvalConfig { readonly threshold?: number; readonly metadata?: EvalMetadata; readonly target?: ProviderDefinition; + readonly targets?: readonly ProviderDefinition[]; readonly task?: (input: string) => string | Promise; readonly providerFactory?: ProviderFactoryFn; } @@ -362,6 +363,7 @@ export async function evaluate(config: EvalConfig): Promise { testFilePath, repoRoot, target: resolvedTarget, + ...(materialized.targets ? { targets: materialized.targets } : {}), ...(providerFactory ? { providerFactory } : {}), maxRetries: config.maxRetries ?? 2, agentTimeoutMs: config.agentTimeoutMs, @@ -426,6 +428,9 @@ export async function materializeEvalConfig( category: options?.category, }); const tests = applyProgrammaticSuiteOverrides(suite.tests, config); + const suiteTargetDefinitions = suite.targetRefs + ?.map((targetRef) => targetRef.definition) + .filter((definition): definition is ProviderDefinition => definition !== undefined); return { testFilePath, tests, @@ -435,7 +440,10 @@ export async function materializeEvalConfig( budgetUsd: config.budgetUsd ?? suite.budgetUsd, threshold: config.threshold ?? suite.threshold, metadata: config.metadata ?? suite.metadata, - target: config.target ?? suite.inlineTarget, + target: config.target ?? suite.inlineTarget ?? suiteTargetDefinitions?.[0], + ...(suiteTargetDefinitions && suiteTargetDefinitions.length > 0 + ? { targets: suiteTargetDefinitions } + : {}), task: config.task, providerFactory: suite.providerFactory, };