diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 1c7a4cd34..4d6d44bb6 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -1,66 +1,66 @@ -# AgentV Evaluation Targets +# AgentV Evaluation Providers # Consolidated from root, examples/features, and examples/showcase. # Per-eval .agentv folders override these for specific eval cases. # -# "grader" is the LLM used for scoring; agent targets reference it via +# "grader" is the LLM used for scoring; agent providers reference it via # grader_target so eval execution and grading use separate models. # -# default/agent/llm/grader are concrete targets, not env-var-driven aliases: +# default/agent/llm/grader are concrete provider labels, not env-var-driven aliases: # AgentV removed the `use_target` indirection field with no replacement, so a -# target id can no longer redirect to "whichever target another target names". +# provider label can no longer redirect to "whichever provider another provider names". # These four mirror today's AGENT_TARGET/LLM_TARGET/GRADER_TARGET selection -# (copilot-cli / azure / azure) so `target: llm` etc. across examples/ keep +# (copilot-cli / azure / azure) so `providers: [llm]` etc. across examples/ keep # resolving. To switch the active provider, edit the definitions below -# directly (or point examples at a different concrete target id) instead of +# directly (or point examples at a different concrete provider label) instead of # changing an env var. -targets: - # ── Default target ────────────────────────────────────────────────── - # Evals without an explicit target resolve to "default" by convention. - - id: default - provider: copilot-cli +providers: + # ── Default provider ──────────────────────────────────────────────── + # Evals without an explicit provider resolve to "default" by convention. + - id: copilot-cli + label: default model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - - id: agent - provider: copilot-cli + - id: copilot-cli + label: agent model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - # ── LLM target (text generation, no agent binary needed) ──────────── - - id: llm - provider: azure + # ── LLM provider (text generation, no agent binary needed) ────────── + - id: azure + label: llm endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" api_key: "{{ env.AZURE_OPENAI_API_KEY }}" model: "{{ env.AZURE_DEPLOYMENT_NAME }}" version: "{{ env.AZURE_OPENAI_API_VERSION }}" # ── Grader (LLM-as-judge) ────────────────────────────────────────── - # Used by agent targets via grader_target. - - id: grader - provider: azure + # Used by agent providers via grader_target. + - id: azure + label: grader endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" api_key: "{{ env.AZURE_OPENAI_API_KEY }}" model: "{{ env.AZURE_DEPLOYMENT_NAME }}" version: "{{ env.AZURE_OPENAI_API_VERSION }}" - # ── Named agent targets ─────────────────────────────────────────── - - id: copilot - provider: copilot-cli + # ── Named agent providers ───────────────────────────────────────── + - id: copilot-cli + label: copilot model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - id: copilot-sdk - provider: copilot-sdk + label: copilot-sdk model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - - id: copilot-sdk-azure - provider: copilot-sdk + - id: copilot-sdk + label: copilot-sdk-azure model: "{{ env.AZURE_DEPLOYMENT_NAME }}" subprovider: azure base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" @@ -68,23 +68,23 @@ targets: grader_target: grader stream_log: raw - - id: claude - provider: claude-cli + - id: claude-cli + label: claude grader_target: grader - id: claude-sdk - provider: claude-sdk + label: claude-sdk grader_target: grader - - id: pi - provider: pi-cli + - id: pi-cli + label: pi subprovider: openrouter model: "{{ env.OPENROUTER_MODEL }}" api_key: "{{ env.OPENROUTER_API_KEY }}" grader_target: grader - - id: pi-sdk - provider: pi-coding-agent + - id: pi-coding-agent + label: pi-sdk subprovider: openai base_url: "{{ env.OPENAI_ENDPOINT }}" api_key: "{{ env.OPENAI_API_KEY }}" @@ -94,16 +94,16 @@ targets: stream_log: raw - - id: pi-azure - provider: pi-cli + - id: pi-cli + label: pi-azure subprovider: azure base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" model: "{{ env.AZURE_DEPLOYMENT_NAME }}" api_key: "{{ env.AZURE_OPENAI_API_KEY }}" grader_target: grader - - id: pi-sdk-azure - provider: pi-coding-agent + - id: pi-coding-agent + label: pi-sdk-azure subprovider: azure base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" model: gpt-5.5 @@ -112,63 +112,63 @@ targets: thinking: low stream_log: raw - - id: codex - provider: codex - executable: "{{ env.CODEX_EXECUTABLE }}" + - id: agentv:codex-cli + label: codex + command: ["{{ env.CODEX_EXECUTABLE }}"] model: "{{ env.CODEX_MODEL }}" - model_reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" + reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" grader_target: grader cwd: "{{ env.CODEX_WORKSPACE_DIR }}" log_dir: "{{ env.CODEX_LOG_DIR }}" stream_log: raw - # ── LLM targets (direct model access) ───────────────────────────── - - id: gh-models - provider: openai + # ── LLM providers (direct model access) ─────────────────────────── + - id: openai + label: gh-models base_url: https://models.github.ai/inference api_key: "{{ env.GH_MODELS_TOKEN }}" model: "{{ env.GH_MODELS_MODEL }}" - # Single Azure target. Always uses Azure's Responses API + # Single Azure provider. Always uses Azure's Responses API # (`/openai/v1/responses`); the api version defaults to `v1` and can be # overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure # deployments must use `provider: openai` with a deployment-scoped # `base_url` instead. - id: azure - provider: azure + label: azure endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" api_key: "{{ env.AZURE_OPENAI_API_KEY }}" model: "{{ env.AZURE_DEPLOYMENT_NAME }}" version: "{{ env.AZURE_OPENAI_API_VERSION }}" - id: gemini - provider: gemini + label: gemini api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}" model: "{{ env.GEMINI_MODEL_NAME }}" - id: openai - provider: openai + label: openai endpoint: "{{ env.OPENAI_ENDPOINT }}" api_key: "{{ env.OPENAI_API_KEY }}" model: "{{ env.OPENAI_MODEL }}" # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy - # without changing provider-specific target labels. - - id: local-openai - provider: openai + # without changing provider-specific labels. + - id: openai + label: local-openai base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" grader_target: local-openai-grader - - id: local-openai-grader - provider: openai + - id: openai + label: local-openai-grader base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - - id: pi-cli-openai - provider: pi-cli + - id: pi-cli + label: pi-cli-openai subprovider: openai base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" @@ -177,18 +177,18 @@ targets: thinking: low stream_log: raw - - id: codex-sdk-openai - provider: codex + - id: openai:codex-sdk + label: codex-sdk-openai base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" api_format: responses - model_reasoning_effort: low + reasoning_effort: low grader_target: local-openai-grader stream_log: raw - - id: copilot-sdk-openai - provider: copilot-sdk + - id: copilot-sdk + label: copilot-sdk-openai subprovider: openai base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" @@ -197,25 +197,25 @@ targets: stream_log: raw - id: openrouter - provider: openrouter + label: openrouter api_key: "{{ env.OPENROUTER_API_KEY }}" model: "{{ env.OPENROUTER_MODEL }}" # ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── - - id: mimo - provider: openrouter + - id: openrouter + label: mimo api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2.5-pro grader_target: grader - - id: mimo-flash - provider: openrouter + - id: openrouter + label: mimo-flash api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2-flash grader_target: grader - - id: mimo-direct - provider: openai + - id: openai + label: mimo-direct base_url: https://token-plan-sgp.xiaomimimo.com/v1 api_key: "{{ env.XIAOMI_MIMO_API_KEY }}" model: xiaomi/mimo-v2.5-pro diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index 8b0a2d3dc..d1c9e3b95 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -134,7 +134,7 @@ function buildBundleRuntime(options: { readonly threshold?: number; }): Record { const runtime: Record = - options.targetNames.length === 1 ? { target: options.targetNames[0] } : {}; + options.targetNames.length > 0 ? { providers: options.targetNames } : {}; if (options.budgetUsd !== undefined) { runtime.budget_usd = options.budgetUsd; } diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index f680ef271..0a8e13e24 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1819,6 +1819,9 @@ export async function runEvalCommand( } let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); + if (yamlConfig?.defaults?.provider && options.cliTargets.length === 0 && !options.target) { + options = { ...options, target: yamlConfig.defaults.provider }; + } if (yamlConfig?.defaults?.grader) { options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader }; } diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index c0a6d50ff..4fd1f2691 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -48,11 +48,14 @@ const SKIPPED_DIR_NAMES = new Set([ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([ 'label', 'provider', + 'provider_spec', 'prompts', 'transform', 'delay', 'env', 'model', + 'runtime', + 'hooks', 'use_target', 'fallback_targets', 'grader_target', @@ -649,9 +652,17 @@ function uniqueTargetDefinitions( } function serializeTargetDefinition(definition: TargetDefinition): Record { - const target: Record = { id: definition.name }; - if (definition.id !== undefined) { - target.id = definition.id; + const providerSpec = + typeof definition.provider_spec === 'string' && definition.provider_spec.trim().length > 0 + ? definition.provider_spec.trim() + : typeof definition.provider === 'string' && definition.provider.trim().length > 0 + ? definition.provider.trim() + : typeof definition.id === 'string' && definition.id.trim().length > 0 + ? definition.id.trim() + : definition.name; + const provider: Record = { id: providerSpec }; + if (definition.name !== providerSpec) { + provider.label = definition.name; } const config: Record = {}; @@ -660,13 +671,15 @@ function serializeTargetDefinition(definition: TargetDefinition): Record 0) { - target.config = config; + provider.config = config; } - return target; + return provider; } function serializeTargetDefinitions( @@ -1133,11 +1146,11 @@ export async function materializeTaskBundle( const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME); await writeYamlFile(evalPath, { - target: options.targetName, + providers: [options.targetName], prompts: [INPUT_PROMPT], tests: [evalCase], }); - await writeYamlFile(targetsPath, { targets: serializeTargetDefinitions(targetDefinitions) }); + await writeYamlFile(targetsPath, { providers: serializeTargetDefinitions(targetDefinitions) }); return { testDir, @@ -1203,7 +1216,7 @@ export async function materializeEvalBundle( const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME); const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME); const runtime = - options.runtime ?? (targetNames.length === 1 ? { target: targetNames[0] } : undefined); + options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : undefined); await writeYamlFile(evalPath, { ...(runtime ?? {}), @@ -1211,7 +1224,7 @@ export async function materializeEvalBundle( tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), }); await writeYamlFile(targetsPath, { - targets: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), + providers: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), }); const manifest = bundleManifest({ diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index 6fdda50e8..ef3f4620d 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -2,7 +2,7 @@ import { constants } from 'node:fs'; import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; -import { parseYamlValue } from '@agentv/core'; +import { parseYamlValue, readTargetDefinitions as readCoreTargetDefinitions } from '@agentv/core'; import { array, command, @@ -111,14 +111,11 @@ async function readTaskTarget(evalPath: string, fallback: string): Promise[]> { - const parsed = parseYamlValue(await readFile(targetsPath, 'utf8')); - if (!isRecord(parsed) || !Array.isArray(parsed.targets)) { - throw new Error(`Targets file is missing a top-level targets array: ${targetsPath}`); - } - return parsed.targets.filter(isRecord); + const definitions = await readCoreTargetDefinitions(targetsPath); + return definitions.map((definition) => definition as unknown as Record); } function targetName(definition: Record): string | undefined { @@ -198,7 +195,7 @@ async function validateTargetFile( targetNames: readonly string[], label: string, ): Promise { - const definitions = await readTargetDefinitions(targetsPath); + const definitions = await readRerunTargetDefinitions(targetsPath); const byName = new Map>(); for (const definition of definitions) { const name = targetName(definition); diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 9800b377c..25e1271a1 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -2670,7 +2670,7 @@ describe('writeArtifactsFromResults', () => { const parsedEval = parseYamlValue(taskEval) as Record; const [testCase] = parsedEval.tests as Record[]; const [assertion] = testCase.assert as Record[]; - expect(parsedEval.target).toBe('gpt-4o'); + expect(parsedEval.providers).toEqual(['gpt-4o']); expect(parsedEval.prompts).toEqual(['{{ input }}']); expect((testCase.vars as Record).input).toBe('file://files/src/input.txt'); expect(assertion.prompt).toBe('file://graders/src/grader.md'); @@ -2808,7 +2808,7 @@ describe('writeArtifactsFromResults', () => { 'utf8', ); const parsedEval = parseYamlValue(taskEval) as Record; - expect(parsedEval.target).toBe('mock-target'); + expect(parsedEval.providers).toEqual(['mock-target']); }); }); diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 92d1b01f9..0ef298f0e 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -53,13 +53,13 @@ describe('agentv eval bundle', () => { await writeFile( path.join(sourceDir, '.agentv', 'targets.yaml'), - `targets: - - id: inherited - provider: mock + `providers: + - id: mock + label: inherited response: '{"answer":"Mock provider response from inherited target"}' fallback_targets: [backup] - - id: backup - provider: mock + - id: mock + label: backup response: '{"answer":"Backup mock response"}' `, 'utf8', @@ -92,7 +92,8 @@ await Bun.write(\`\${payload.workspace_path}/hook-ran.txt\`, 'ok\\n'); const evalPath = path.join(sourceDir, 'evals', 'demo.eval.yaml'); const sourceEvalBefore = `name: portable-demo -target: inherited +providers: + - inherited environment: type: host workdir: ../workspace-template @@ -141,7 +142,7 @@ tests: ../data/cases.yaml const bundledEvalText = await readFile(path.join(bundleDir, 'evals', 'demo.eval.yaml'), 'utf8'); expect(bundledEvalText).not.toContain(sourceDir); const bundledEval = parseYamlValue(bundledEvalText) as Record; - expect(bundledEval.target).toBe('inherited'); + expect(bundledEval.providers).toEqual(['inherited']); expect(bundledEval.execution).toBeUndefined(); const [testCase] = bundledEval.tests as Record[]; expect(testCase.id).toBe('case-alpha'); @@ -158,8 +159,8 @@ tests: ../data/cases.yaml expect(input[0]?.content[0]).toEqual({ type: 'file', value: 'files/data/input.txt' }); const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('id: inherited'); - expect(bundledTargets).toContain('id: backup'); + expect(bundledTargets).toContain('label: inherited'); + expect(bundledTargets).toContain('label: backup'); await rm(sourceDir, { recursive: true, force: true }); const run = await runCli(bundleDir, [ @@ -179,12 +180,12 @@ tests: ../data/cases.yaml const bundleDir = path.join(tempDir, 'inline-bundle'); await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); - await writeFile(path.join(sourceDir, '.agentv', 'targets.yaml'), 'targets: []\n', 'utf8'); + await writeFile(path.join(sourceDir, '.agentv', 'targets.yaml'), 'providers: []\n', 'utf8'); await writeFile( path.join(sourceDir, 'evals', 'inline.eval.yaml'), - `targets: - - id: candidate - provider: mock + `providers: + - id: mock + label: candidate response: '{"answer":"inline bundled response"}' prompts: - "{{ input }}" @@ -209,8 +210,8 @@ tests: expect(bundle.exitCode).toBe(0); const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('id: candidate'); - expect(bundledTargets).toContain('provider: mock'); + expect(bundledTargets).toContain('id: mock'); + expect(bundledTargets).toContain('label: candidate'); expect(bundledTargets).toContain('inline bundled response'); }, 30_000); @@ -221,9 +222,9 @@ tests: await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); await writeFile( path.join(sourceDir, '.agentv', 'targets.yaml'), - `targets: - - id: default - provider: mock + `providers: + - id: mock + label: default `, 'utf8', ); diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index de98177b7..2f7aa56b4 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -25,9 +25,9 @@ describe('eval target selection', () => { path.join(agentvDir, 'targets.yaml'), [ '$schema: agentv-targets-v2.2', - 'targets:', - ' - id: openai:gpt-5.4-mini', - ' provider: mock', + 'providers:', + ' - id: mock', + ' label: openai:gpt-5.4-mini', '', ].join('\n'), ); @@ -36,8 +36,8 @@ describe('eval target selection', () => { evalPath, [ 'name: target-label-suite', - 'targets:', - ' - id: openai:gpt-5.4-mini', + 'providers:', + ' - openai:gpt-5.4-mini', 'prompts:', ' - "{{ input }}"', 'tests:', diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index 04eb1dcdd..cf3548a61 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -123,7 +123,7 @@ describe('materializeTaskBundle', () => { const [testCase] = parsedEval.tests as Record[]; const [assertion] = testCase.assert as Record[]; - expect(parsedEval.target).toBe('selected'); + expect(parsedEval.providers).toEqual(['selected']); expect(parsedEval.prompts).toEqual(['{{ input }}']); expect(parsedEval.execution).toBeUndefined(); expect(parsedEval.tests as unknown[]).toHaveLength(1); diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index 3b409dad3..a9db768b1 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -82,9 +82,9 @@ console.log(JSON.stringify({ await writeFile( path.join(root, '.agentv', 'targets.yaml'), ` -targets: - - id: codex - provider: cli +providers: + - id: cli + label: codex command: bun ./scripts/target.ts `, 'utf8', diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 1a9a2ed57..745d69912 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -59,9 +59,9 @@ await Bun.write(\`\${payload.workspace_path}/\${step}.txt\`, \`\${payload.test_i await writeFile( path.join(root, '.agentv', 'targets.yaml'), ` -targets: - - id: codex - provider: cli +providers: + - id: cli + label: codex command: bun ./scripts/target.ts `, 'utf8', @@ -71,19 +71,21 @@ targets: `environment: type: host workdir: ../template -target: - extends: codex - hooks: - before_all: - command: - - bun - - ../scripts/hook.ts - - target_before_all - before_each: - command: - - bun - - ../scripts/hook.ts - - target_before_each +providers: + - id: cli + label: codex + command: bun ../scripts/target.ts + hooks: + before_all: + command: + - bun + - ../scripts/hook.ts + - target_before_all + before_each: + command: + - bun + - ../scripts/hook.ts + - target_before_each assertions: - name: secret-grader type: script @@ -250,9 +252,9 @@ describe('agentv prepare', () => { await writeFile( path.join(tempDir, '.agentv', 'targets.yaml'), ` -targets: - - id: codex - provider: cli +providers: + - id: cli + label: codex command: bun ./scripts/target.ts `, 'utf8', diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index f5ea2a044..aca413157 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -29,9 +29,9 @@ interface CliResult { readonly exitCode: number; } -const DEFAULT_TARGETS = `targets: - - id: captured - provider: mock +const DEFAULT_TARGETS = `providers: + - id: mock + label: captured `; async function writeTaskBundle(options: { @@ -49,7 +49,8 @@ async function writeTaskBundle(options: { await writeFile( path.join(bundleDir, 'EVAL.yaml'), - `target: captured + `providers: + - captured prompts: - "{{ input }}" tests: @@ -126,9 +127,9 @@ async function createBundleFixture( const overrideTargetsPath = path.join(baseDir, 'override-targets.yaml'); await writeFile( overrideTargetsPath, - `targets: - - id: local - provider: mock + `providers: + - id: mock + label: local `, 'utf8', ); @@ -280,9 +281,9 @@ describe('agentv runs rerun', () => { }, 30_000); it('fails clearly for missing env and accepts an explicit env file', async () => { - const created = await fixture(`targets: - - id: captured - provider: cli + const created = await fixture(`providers: + - id: cli + label: captured command: "{{ env.LOCAL_AGENT_COMMAND }}" `); diff --git a/apps/cli/test/commands/validate/validate-files.test.ts b/apps/cli/test/commands/validate/validate-files.test.ts index b53c4acab..61341800f 100644 --- a/apps/cli/test/commands/validate/validate-files.test.ts +++ b/apps/cli/test/commands/validate/validate-files.test.ts @@ -56,8 +56,14 @@ describe('validateFiles TypeScript eval configs', () => { configFile, `export default { prompts: ['{{ input }}'], - providers: ['openai:gpt-5'], - tests: [{ id: 'hello', vars: { input: 'Say hello' } }], + providers: [{ id: 'mock', provider: 'mock' }], + tests: [ + { + id: 'hello', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], }; `, ); @@ -65,7 +71,7 @@ describe('validateFiles TypeScript eval configs', () => { const summary = await validateFiles([configFile]); expect(summary.invalidFiles).toBe(1); - expect(summary.results[0].errors[0].message).toContain("top-level 'providers'"); + expect(summary.results[0].errors[0].message).toContain('providers[0].provider'); }); it('expands directories without treating custom assertion files as eval configs', async () => { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index cc88cd5df..0545f4f3e 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -70,22 +70,23 @@ async function createFixture(): Promise { const targetsPath = path.join(agentvDir, 'targets.yaml'); const targetsContent = `$schema: agentv-targets-v2.2 -targets: - - id: default - provider: mock - - id: file-target - provider: mock - - id: cli-target - provider: mock - - id: codex-target - provider: codex-sdk +providers: + - id: mock + label: default + - id: mock + label: file-target + - id: mock + label: cli-target + - id: openai:codex-sdk + label: codex-target model: gpt-5-default `; await writeFile(targetsPath, targetsContent, 'utf8'); const testFilePath = path.join(suiteDir, 'sample.test.yaml'); const testFileContent = `description: CLI integration test -target: file-target +providers: + - file-target prompts: - "{{ input }}" tests: @@ -131,9 +132,9 @@ async function createNestedEnvFixture(): Promise { const targetsPath = path.join(agentvDir, 'targets.yaml'); const targetsContent = `$schema: agentv-targets-v2.2 -targets: - - id: default - provider: mock +providers: + - id: mock + label: default `; await writeFile(targetsPath, targetsContent, 'utf8'); @@ -633,7 +634,8 @@ describe('agentv eval CLI', () => { evalPath, [ 'description: Budget options integration test', - 'target: file-target', + 'providers:', + ' - file-target', 'evaluate_options:', ' budget_usd: 1.25', 'prompts:', @@ -672,7 +674,8 @@ describe('agentv eval CLI', () => { path.join(fixture.suiteDir, 'unused.test.yaml'), [ 'description: unmatched eval file should not resolve targets', - 'target: missing-target', + 'providers:', + ' - missing-target', 'prompts:', ' - "{{ input }}"', 'tests:', @@ -689,9 +692,8 @@ describe('agentv eval CLI', () => { wrapperPath, [ 'name: native-exp', - 'target:', - ' extends: codex-target', - ' model: gpt-5-codex', + 'providers:', + ' - codex-target', 'timeout_seconds: 12', 'threshold: 0.8', 'evaluate_options:', @@ -728,7 +730,7 @@ describe('agentv eval CLI', () => { const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ target: 'codex-target', - targetModel: 'gpt-5-codex', + targetModel: 'gpt-5-default', agentTimeoutMs: 5000, maxConcurrency: 4, evalCaseIds: ['case-alpha'], @@ -772,7 +774,8 @@ describe('agentv eval CLI', () => { evalPath, [ 'name: default-threshold', - 'target: file-target', + 'providers:', + ' - file-target', 'threshold: 0.9', 'prompts:', ' - "{{ input }}"', @@ -826,7 +829,8 @@ describe('agentv eval CLI', () => { firstPath, [ 'name: first-default-threshold', - 'target: file-target', + 'providers:', + ' - file-target', 'prompts:', ' - "{{ input }}"', 'default_test:', @@ -843,7 +847,8 @@ describe('agentv eval CLI', () => { secondPath, [ 'name: second-default-threshold', - 'target: file-target', + 'providers:', + ' - file-target', 'prompts:', ' - "{{ input }}"', 'default_test:', @@ -875,7 +880,8 @@ describe('agentv eval CLI', () => { firstPath, [ 'name: first', - 'target: cli-target', + 'providers:', + ' - cli-target', 'timeout_seconds: 11', 'evaluate_options:', ' budget_usd: 0.11', @@ -893,7 +899,8 @@ describe('agentv eval CLI', () => { secondPath, [ 'name: second', - 'target: file-target', + 'providers:', + ' - file-target', 'timeout_seconds: 22', 'evaluate_options:', ' budget_usd: 0.22', @@ -963,7 +970,8 @@ describe('agentv eval CLI', () => { evalPath, [ 'name: max-concurrency', - 'target: file-target', + 'providers:', + ' - file-target', 'evaluate_options:', ' max_concurrency: 2', 'prompts:', @@ -1027,7 +1035,8 @@ describe('agentv eval CLI', () => { await writeFile( fixture.testFilePath, `description: CLI integration test -target: file-target +providers: + - file-target prompts: - "{{ input }}" tests: @@ -1096,7 +1105,8 @@ tests: const firstEvalPath = path.join(fixture.suiteDir, 'collision-a.eval.yaml'); const secondEvalPath = path.join(fixture.suiteDir, 'collision-b.eval.yaml'); const evalContent = (name: string) => `description: ${name} -target: file-target +providers: + - file-target prompts: - "{{ input }}" tests: @@ -1201,7 +1211,8 @@ tests: evalPath, [ 'name: target-matrix', - 'target: file-target', + 'providers:', + ' - file-target', 'prompts:', ' - "{{ input }}"', 'tests:', diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 58f3fcf2e..103aebc5e 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -3,7 +3,8 @@ description: Lightweight release gate for npm latest promotion. environment: type: host workdir: ../workspace-template -target: github-models-contract +providers: + - github-models-contract prompts: - "{{ input }}" tests: diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 55bec6799..9aa7b1904 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -12,7 +12,8 @@ environment: - EntityProcess/agentv-contract-fixture - 21a34daed7ebcfe36cbed053607622a55e5e94cb cwd: "." -target: github-models-contract +providers: + - github-models-contract prompts: - "{{ input }}" tests: diff --git a/examples/contract/evals/script-grader-contract.eval.yaml b/examples/contract/evals/script-grader-contract.eval.yaml index e25584338..24a5fe0cb 100644 --- a/examples/contract/evals/script-grader-contract.eval.yaml +++ b/examples/contract/evals/script-grader-contract.eval.yaml @@ -1,6 +1,7 @@ name: script-grader-contract description: Release gate verifying the script-grader stdin payload contract. -target: github-models-contract +providers: + - github-models-contract prompts: - "{{ input }}" tests: diff --git a/examples/features/assert-extended/evals/suite.yaml b/examples/features/assert-extended/evals/suite.yaml index 9b9c87539..884d8fa2b 100644 --- a/examples/features/assert-extended/evals/suite.yaml +++ b/examples/features/assert-extended/evals/suite.yaml @@ -1,6 +1,7 @@ name: assert-extended description: Extended deterministic assertions for natural language validation -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/assert-set/evals/suite.yaml b/examples/features/assert-set/evals/suite.yaml index 2d17b908d..cea51377c 100644 --- a/examples/features/assert-set/evals/suite.yaml +++ b/examples/features/assert-set/evals/suite.yaml @@ -1,6 +1,7 @@ name: assert-set-examples description: Demonstrates assert-set patterns for grouping multiple evaluation criteria -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/assert/evals/suite.yaml b/examples/features/assert/evals/suite.yaml index 878923f23..170f6dc04 100644 --- a/examples/features/assert/evals/suite.yaml +++ b/examples/features/assert/evals/suite.yaml @@ -4,7 +4,8 @@ version: "1.0" tags: - demo - assert -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/autoresearch/EVAL.yaml b/examples/features/autoresearch/EVAL.yaml index 99804f5d7..1cdbf530d 100644 --- a/examples/features/autoresearch/EVAL.yaml +++ b/examples/features/autoresearch/EVAL.yaml @@ -3,7 +3,8 @@ description: | Evaluates the incident severity classifier prompt for accuracy, output format, and reasoning quality. Used as an autoresearch demo — the prompt artifact starts weak and the optimization loop iteratively improves it. -target: llm +providers: + - llm defaults: system_message: | Classify the incident into P0, P1, P2, or P3. diff --git a/examples/features/basic-jsonl/evals/suite.yaml b/examples/features/basic-jsonl/evals/suite.yaml index 2fd866580..6e773cf6f 100644 --- a/examples/features/basic-jsonl/evals/suite.yaml +++ b/examples/features/basic-jsonl/evals/suite.yaml @@ -1,7 +1,8 @@ description: JSONL version of the basic example - demonstrates file references, multi-turn, and per-case overrides name: basic-jsonl -target: llm +providers: + - llm prompts: - "{{ input }}" tests: ./cases.jsonl diff --git a/examples/features/basic/evals/suite.yaml b/examples/features/basic/evals/suite.yaml index 8230a0be2..1bd25990e 100644 --- a/examples/features/basic/evals/suite.yaml +++ b/examples/features/basic/evals/suite.yaml @@ -1,6 +1,7 @@ name: basic description: Example showing basic features, conversation threading, multiple graders -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/batch-cli/evals/suite.yaml b/examples/features/batch-cli/evals/suite.yaml index 7eaf206ed..1d591d4c9 100644 --- a/examples/features/batch-cli/evals/suite.yaml +++ b/examples/features/batch-cli/evals/suite.yaml @@ -1,6 +1,7 @@ name: batch-cli description: Batch CLI demo (AML screening) using structured input → CSV → JSONL with trace extraction -target: batch_cli +providers: + - batch_cli tags: - agent prompts: diff --git a/examples/features/benchmark-tooling/evals/benchmark.eval.yaml b/examples/features/benchmark-tooling/evals/benchmark.eval.yaml index 15d6109db..6362c947e 100644 --- a/examples/features/benchmark-tooling/evals/benchmark.eval.yaml +++ b/examples/features/benchmark-tooling/evals/benchmark.eval.yaml @@ -1,6 +1,7 @@ name: multi-model-benchmark description: Compare greeting, code generation, and summarization across three model targets -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/compare/evals/suite.yaml b/examples/features/compare/evals/suite.yaml index 100cb6702..8a7ab97f5 100644 --- a/examples/features/compare/evals/suite.yaml +++ b/examples/features/compare/evals/suite.yaml @@ -1,6 +1,7 @@ name: compare-demo description: Demo eval for generating baseline and candidate results to compare -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/copilot-transcript-replay/evals/skill-use.EVAL.yaml b/examples/features/copilot-transcript-replay/evals/skill-use.EVAL.yaml index 960872ac2..5431d3f1b 100644 --- a/examples/features/copilot-transcript-replay/evals/skill-use.EVAL.yaml +++ b/examples/features/copilot-transcript-replay/evals/skill-use.EVAL.yaml @@ -1,6 +1,7 @@ tags: - agent -target: copilot-transcript-replay +providers: + - copilot-transcript-replay extensions: - file://../scripts/copilot-replay-workspace.mjs:beforeAll environment: diff --git a/examples/features/default-graders/evals/suite.yaml b/examples/features/default-graders/evals/suite.yaml index 7b886a213..d39969aaf 100644 --- a/examples/features/default-graders/evals/suite.yaml +++ b/examples/features/default-graders/evals/suite.yaml @@ -1,6 +1,7 @@ name: default-graders-example description: Root-level graders that automatically apply to every test -target: llm +providers: + - llm assert: - Response has a professional, helpful, and polite tone prompts: diff --git a/examples/features/deterministic-graders/evals/suite.yaml b/examples/features/deterministic-graders/evals/suite.yaml index ef8d37635..d1f30c415 100644 --- a/examples/features/deterministic-graders/evals/suite.yaml +++ b/examples/features/deterministic-graders/evals/suite.yaml @@ -1,6 +1,7 @@ name: deterministic-graders description: Built-in deterministic assertions — contains, regex, JSON validation, equals -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index 330b35fd2..8c1cd85c4 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -1,6 +1,7 @@ name: docker-workspace-example description: Example eval using a Docker environment for grading -target: mock_agent +providers: + - mock_agent environment: type: docker image: python:3.11-slim diff --git a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml index 8dd4814a5..274beee7e 100644 --- a/examples/features/document-extraction/evals/confusion-metrics.eval.yaml +++ b/examples/features/document-extraction/evals/confusion-metrics.eval.yaml @@ -1,5 +1,6 @@ description: Header field confusion metrics (TP/TN/FP/FN aggregation) -target: mock_extractor +providers: + - mock_extractor assert: - metric: header_confusion type: script diff --git a/examples/features/document-extraction/evals/field-accuracy.eval.yaml b/examples/features/document-extraction/evals/field-accuracy.eval.yaml index 616e6f5c6..f0ec72acd 100644 --- a/examples/features/document-extraction/evals/field-accuracy.eval.yaml +++ b/examples/features/document-extraction/evals/field-accuracy.eval.yaml @@ -1,5 +1,6 @@ description: Field accuracy grader patterns (per-test-case scoring) -target: mock_extractor +providers: + - mock_extractor assert: - metric: invoice_field_accuracy type: field-accuracy diff --git a/examples/features/env-interpolation/evals/suite.yaml b/examples/features/env-interpolation/evals/suite.yaml index e8f079fd1..2239edf96 100644 --- a/examples/features/env-interpolation/evals/suite.yaml +++ b/examples/features/env-interpolation/evals/suite.yaml @@ -1,5 +1,6 @@ description: Demonstrates {{ env.VAR }} interpolation in eval fields -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/eval-assert-demo/evals/suite.yaml b/examples/features/eval-assert-demo/evals/suite.yaml index bf83e8a4a..52e89dbb0 100644 --- a/examples/features/eval-assert-demo/evals/suite.yaml +++ b/examples/features/eval-assert-demo/evals/suite.yaml @@ -1,5 +1,6 @@ description: script graders with eval assert CLI integration -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/execution-metrics/evals/suite.yaml b/examples/features/execution-metrics/evals/suite.yaml index 5330a3503..344ea124c 100644 --- a/examples/features/execution-metrics/evals/suite.yaml +++ b/examples/features/execution-metrics/evals/suite.yaml @@ -1,6 +1,7 @@ name: execution-metrics description: Demonstrates the built-in execution_metrics grader -target: mock_metrics_agent +providers: + - mock_metrics_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/experiments/evals/coding-ability.eval.yaml b/examples/features/experiments/evals/coding-ability.eval.yaml index 5fd03ff0c..eabce1ed9 100644 --- a/examples/features/experiments/evals/coding-ability.eval.yaml +++ b/examples/features/experiments/evals/coding-ability.eval.yaml @@ -1,5 +1,6 @@ name: coding-ability -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/external-datasets/evals/suite.yaml b/examples/features/external-datasets/evals/suite.yaml index a914f2706..0746c90d6 100644 --- a/examples/features/external-datasets/evals/suite.yaml +++ b/examples/features/external-datasets/evals/suite.yaml @@ -1,6 +1,7 @@ name: external-datasets-demo version: "1.0" -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/file-changes-graders/evals/suite.yaml b/examples/features/file-changes-graders/evals/suite.yaml index 898961c0d..9653ebd70 100644 --- a/examples/features/file-changes-graders/evals/suite.yaml +++ b/examples/features/file-changes-graders/evals/suite.yaml @@ -2,7 +2,8 @@ description: Verify file_changes diffs are accessible to LLM grader (llm-rubric, environment: type: host workdir: ../workspace-template -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/file-changes-with-repos/evals/suite.yaml b/examples/features/file-changes-with-repos/evals/suite.yaml index 1e183940d..0f3fadc4b 100644 --- a/examples/features/file-changes-with-repos/evals/suite.yaml +++ b/examples/features/file-changes-with-repos/evals/suite.yaml @@ -5,7 +5,8 @@ environment: workdir: ../workspace-template extensions: - file://../scripts/setup-nested-repo.mjs:beforeAll -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/file-changes/evals/suite.yaml b/examples/features/file-changes/evals/suite.yaml index 4da0dd020..76e1573b0 100644 --- a/examples/features/file-changes/evals/suite.yaml +++ b/examples/features/file-changes/evals/suite.yaml @@ -3,7 +3,8 @@ description: Verify file_changes captures edits, creates, and deletes across mul environment: type: host workdir: ../workspace-template -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/functional-grading/evals/suite.yaml b/examples/features/functional-grading/evals/suite.yaml index 371c427d7..a409da978 100644 --- a/examples/features/functional-grading/evals/suite.yaml +++ b/examples/features/functional-grading/evals/suite.yaml @@ -3,7 +3,8 @@ description: Functional grading with workspace_path — deploy-and-test pattern environment: type: host workdir: ../workspace-template -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/import-claude/evals/transcript-check.EVAL.yaml b/examples/features/import-claude/evals/transcript-check.EVAL.yaml index 50be7a29e..260728f02 100644 --- a/examples/features/import-claude/evals/transcript-check.EVAL.yaml +++ b/examples/features/import-claude/evals/transcript-check.EVAL.yaml @@ -1,4 +1,5 @@ -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/input-files-shorthand/evals/suite.yaml b/examples/features/input-files-shorthand/evals/suite.yaml index f460fae36..ca9137b53 100644 --- a/examples/features/input-files-shorthand/evals/suite.yaml +++ b/examples/features/input-files-shorthand/evals/suite.yaml @@ -1,5 +1,6 @@ description: Demonstrates input_files shorthand for attaching files to test inputs -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/latency-assertions/evals/suite.yaml b/examples/features/latency-assertions/evals/suite.yaml index 76356bed8..4d0ab593b 100644 --- a/examples/features/latency-assertions/evals/suite.yaml +++ b/examples/features/latency-assertions/evals/suite.yaml @@ -1,6 +1,7 @@ name: latency-assertions description: Tool sequence checks for the latency demo target -target: mock_latency_agent +providers: + - mock_latency_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/local-cli/evals/suite.yaml b/examples/features/local-cli/evals/suite.yaml index 423b4c93d..42de2c10e 100644 --- a/examples/features/local-cli/evals/suite.yaml +++ b/examples/features/local-cli/evals/suite.yaml @@ -1,5 +1,6 @@ description: Minimal demo showing how to invoke a CLI target with file attachments -target: local_cli +providers: + - local_cli tags: - agent prompts: diff --git a/examples/features/multi-turn-conversation-live/evals/suite.yaml b/examples/features/multi-turn-conversation-live/evals/suite.yaml index e8ff047af..c13bdd81d 100644 --- a/examples/features/multi-turn-conversation-live/evals/suite.yaml +++ b/examples/features/multi-turn-conversation-live/evals/suite.yaml @@ -1,5 +1,6 @@ description: Live multi-turn conversation evaluation with per-turn grading -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/multi-turn-conversation/evals/suite.yaml b/examples/features/multi-turn-conversation/evals/suite.yaml index 7e1808b53..75d638c9b 100644 --- a/examples/features/multi-turn-conversation/evals/suite.yaml +++ b/examples/features/multi-turn-conversation/evals/suite.yaml @@ -1,5 +1,6 @@ description: Multi-turn conversation evaluation with per-turn score breakdown -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/nlp-metrics/evals/suite.yaml b/examples/features/nlp-metrics/evals/suite.yaml index b5e5cfdba..cd32845cd 100644 --- a/examples/features/nlp-metrics/evals/suite.yaml +++ b/examples/features/nlp-metrics/evals/suite.yaml @@ -1,6 +1,7 @@ name: nlp-metrics description: NLP text-quality metrics using script_grader graders -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/prompt-template-sdk/evals/suite.yaml b/examples/features/prompt-template-sdk/evals/suite.yaml index 9646f221c..8bc32ba7f 100644 --- a/examples/features/prompt-template-sdk/evals/suite.yaml +++ b/examples/features/prompt-template-sdk/evals/suite.yaml @@ -1,5 +1,6 @@ description: Demonstrates TypeScript prompt templates for custom LLM grader prompts -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/readme-quickstart/evals/my-eval.eval.yaml b/examples/features/readme-quickstart/evals/my-eval.eval.yaml index af69c2669..22f11cc7d 100644 --- a/examples/features/readme-quickstart/evals/my-eval.eval.yaml +++ b/examples/features/readme-quickstart/evals/my-eval.eval.yaml @@ -1,7 +1,8 @@ description: Code generation quality tags: experiment: with-skills -target: local-openai +providers: + - local-openai evaluate_options: max_concurrency: 2 prompts: diff --git a/examples/features/readme-quickstart/targets.yaml b/examples/features/readme-quickstart/targets.yaml index 38641bdaa..6eb9f60ad 100644 --- a/examples/features/readme-quickstart/targets.yaml +++ b/examples/features/readme-quickstart/targets.yaml @@ -1,6 +1,6 @@ -targets: - - id: local-openai - provider: openai +providers: + - id: openai + label: local-openai runtime: host config: api_format: chat diff --git a/examples/features/rubric/evals/operators.eval.yaml b/examples/features/rubric/evals/operators.eval.yaml index 76d27ac5f..acc299384 100644 --- a/examples/features/rubric/evals/operators.eval.yaml +++ b/examples/features/rubric/evals/operators.eval.yaml @@ -1,6 +1,7 @@ name: rubric-operators description: Focused example showing correctness and contradiction rubric operators -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/rubric/evals/suite.yaml b/examples/features/rubric/evals/suite.yaml index 08fce5db5..be55a6249 100644 --- a/examples/features/rubric/evals/suite.yaml +++ b/examples/features/rubric/evals/suite.yaml @@ -1,6 +1,7 @@ name: rubric description: "Example showing rubric grader - string shorthand and type: llm-rubric" -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/scenarios/targets.yaml b/examples/features/scenarios/targets.yaml index f8b5d7e76..68542ef31 100644 --- a/examples/features/scenarios/targets.yaml +++ b/examples/features/scenarios/targets.yaml @@ -1,4 +1,4 @@ -targets: - - id: translation-cli - provider: cli +providers: + - id: cli + label: translation-cli command: "node ../scripts/translation-target.mjs {PROMPT_FILE} {OUTPUT_FILE}" diff --git a/examples/features/script-grader-sdk/evals/suite.yaml b/examples/features/script-grader-sdk/evals/suite.yaml index 5ebba3712..443c8d492 100644 --- a/examples/features/script-grader-sdk/evals/suite.yaml +++ b/examples/features/script-grader-sdk/evals/suite.yaml @@ -1,5 +1,6 @@ description: Demonstrates TypeScript helpers for script_grader payloads -target: local_cli +providers: + - local_cli tags: - agent prompts: diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml index de252c231..f0e07bcd8 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml @@ -7,7 +7,8 @@ assert: - ../scripts/contextual-precision.ts target: max_calls: 10 -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml index 1241f4d8b..5d778b790 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml @@ -7,7 +7,8 @@ assert: - ../scripts/contextual-recall.ts target: max_calls: 15 -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/sdk-config-file/evals/suite.yaml b/examples/features/sdk-config-file/evals/suite.yaml index 207f8ea01..1dea2af38 100644 --- a/examples/features/sdk-config-file/evals/suite.yaml +++ b/examples/features/sdk-config-file/evals/suite.yaml @@ -1,6 +1,7 @@ name: sdk-config-file description: Demonstrates defineConfig() for typed project configuration -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/sdk-custom-assertion/evals/suite.yaml b/examples/features/sdk-custom-assertion/evals/suite.yaml index 7fcbd81c8..9226ff1c1 100644 --- a/examples/features/sdk-custom-assertion/evals/suite.yaml +++ b/examples/features/sdk-custom-assertion/evals/suite.yaml @@ -1,6 +1,7 @@ name: sdk-custom-assertion description: Demonstrates custom assertions via defineAssertion() and convention discovery -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/sdk-python/evals/suite.yaml b/examples/features/sdk-python/evals/suite.yaml index cb8cd13fd..1027df6c1 100644 --- a/examples/features/sdk-python/evals/suite.yaml +++ b/examples/features/sdk-python/evals/suite.yaml @@ -1,6 +1,7 @@ description: Python helper example that emits canonical AgentV YAML/JSONL. name: python-helper -target: local_cli +providers: + - local_cli tags: - python - sdk diff --git a/examples/features/suite-level-input-files/evals/suite.yaml b/examples/features/suite-level-input-files/evals/suite.yaml index dc46f7dc8..1637843b2 100644 --- a/examples/features/suite-level-input-files/evals/suite.yaml +++ b/examples/features/suite-level-input-files/evals/suite.yaml @@ -1,6 +1,7 @@ name: suite-level-input-files-example description: Suite-level input + input_files shorthands -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/suite-level-input/evals/suite.yaml b/examples/features/suite-level-input/evals/suite.yaml index 8fd3fa733..8b562cd23 100644 --- a/examples/features/suite-level-input/evals/suite.yaml +++ b/examples/features/suite-level-input/evals/suite.yaml @@ -1,6 +1,7 @@ name: suite-level-input-example description: Suite-level input prepended to all tests (like suite-level assertions) -target: llm +providers: + - llm prompts: - "{{ input }}" tests: ./cases.yaml diff --git a/examples/features/test-vars-templating/evals/direct-input.eval.yaml b/examples/features/test-vars-templating/evals/direct-input.eval.yaml index f6876e9b9..71726b8f7 100644 --- a/examples/features/test-vars-templating/evals/direct-input.eval.yaml +++ b/examples/features/test-vars-templating/evals/direct-input.eval.yaml @@ -1,5 +1,6 @@ description: Demonstrates AgentV direct-input shorthand and role/content messages -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/test-vars-templating/evals/suite.yaml b/examples/features/test-vars-templating/evals/suite.yaml index 3c3a65b13..11249c702 100644 --- a/examples/features/test-vars-templating/evals/suite.yaml +++ b/examples/features/test-vars-templating/evals/suite.yaml @@ -1,5 +1,6 @@ description: Demonstrates prompt templates rendered from default_test.vars and tests[].vars -target: llm +providers: + - llm prompts: - id: support-chat label: Support chat diff --git a/examples/features/threshold-grader/evals/suite.yaml b/examples/features/threshold-grader/evals/suite.yaml index a1550c150..0d0c9a4aa 100644 --- a/examples/features/threshold-grader/evals/suite.yaml +++ b/examples/features/threshold-grader/evals/suite.yaml @@ -1,6 +1,7 @@ name: threshold-grader-example description: Demonstrates assert-set threshold grouping -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/tool-evaluation-plugins/evals/suite.yaml b/examples/features/tool-evaluation-plugins/evals/suite.yaml index f34733c18..1b473ef58 100644 --- a/examples/features/tool-evaluation-plugins/evals/suite.yaml +++ b/examples/features/tool-evaluation-plugins/evals/suite.yaml @@ -1,5 +1,6 @@ description: Tool-call F1 scoring examples -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/trace-analysis/evals/suite.yaml b/examples/features/trace-analysis/evals/suite.yaml index 6158c1834..d8d6a6e72 100644 --- a/examples/features/trace-analysis/evals/suite.yaml +++ b/examples/features/trace-analysis/evals/suite.yaml @@ -1,6 +1,7 @@ name: trace-analysis-demo description: Demo eval for generating execution traces to analyze -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/trace-evaluation/evals/suite.yaml b/examples/features/trace-evaluation/evals/suite.yaml index 8838e4b5c..71a421b92 100644 --- a/examples/features/trace-evaluation/evals/suite.yaml +++ b/examples/features/trace-evaluation/evals/suite.yaml @@ -1,5 +1,6 @@ description: Trace-based evaluation of agent internals using script graders -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/trajectory-assertions-advanced/evals/trace-file-demo.eval.yaml b/examples/features/trajectory-assertions-advanced/evals/trace-file-demo.eval.yaml index 0f3366122..c3c3675ed 100644 --- a/examples/features/trajectory-assertions-advanced/evals/trace-file-demo.eval.yaml +++ b/examples/features/trajectory-assertions-advanced/evals/trace-file-demo.eval.yaml @@ -1,5 +1,6 @@ description: Static trace file evaluation demo - Product Research Agent -target: static_trace +providers: + - static_trace prompts: - "{{ input }}" tests: diff --git a/examples/features/trajectory-assertions-simple/evals/suite.yaml b/examples/features/trajectory-assertions-simple/evals/suite.yaml index b5882dd61..e2674d549 100644 --- a/examples/features/trajectory-assertions-simple/evals/suite.yaml +++ b/examples/features/trajectory-assertions-simple/evals/suite.yaml @@ -1,6 +1,7 @@ name: trajectory-assertions-simple description: Promptfoo trajectory assertion examples for agent execution validation -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/trial-output-consistency/evals/suite.yaml b/examples/features/trial-output-consistency/evals/suite.yaml index 4a14d45a8..228d5cb3c 100644 --- a/examples/features/trial-output-consistency/evals/suite.yaml +++ b/examples/features/trial-output-consistency/evals/suite.yaml @@ -1,5 +1,6 @@ description: Trial output consistency via embedding similarity -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/trials/evals/suite.yaml b/examples/features/trials/evals/suite.yaml index 04f4e7206..ee9bf8fee 100644 --- a/examples/features/trials/evals/suite.yaml +++ b/examples/features/trials/evals/suite.yaml @@ -1,6 +1,7 @@ name: trials description: Repeat runs example with 2 samples configured inline -target: llm +providers: + - llm evaluate_options: repeat: count: 2 diff --git a/examples/features/vitest-workspace-grader/evals/suite.yaml b/examples/features/vitest-workspace-grader/evals/suite.yaml index 527363cae..35747024b 100644 --- a/examples/features/vitest-workspace-grader/evals/suite.yaml +++ b/examples/features/vitest-workspace-grader/evals/suite.yaml @@ -3,7 +3,8 @@ description: Deterministic workspace grading with a Vitest verifier file. environment: type: host workdir: ../workspace-template -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/features/weighted-graders/evals/suite.yaml b/examples/features/weighted-graders/evals/suite.yaml index 2ba034239..a94821569 100644 --- a/examples/features/weighted-graders/evals/suite.yaml +++ b/examples/features/weighted-graders/evals/suite.yaml @@ -1,5 +1,6 @@ name: weighted-graders-examples -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/features/workspace-artifact/evals/suite.yaml b/examples/features/workspace-artifact/evals/suite.yaml index ada8130ac..bae74ec7b 100644 --- a/examples/features/workspace-artifact/evals/suite.yaml +++ b/examples/features/workspace-artifact/evals/suite.yaml @@ -3,7 +3,8 @@ description: Verify file_changes captures generated artifacts (CSV) under worksp environment: type: host workdir: ../workspace-template -target: mock_csv_agent +providers: + - mock_csv_agent prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/backdoor-pr.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/backdoor-pr.eval.yaml index 77c09be27..7d9a30aa4 100644 --- a/examples/red-team/archetypes/coding-agent/suites/backdoor-pr.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/backdoor-pr.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/benign-controls.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/benign-controls.eval.yaml index 776eb5031..51d36623b 100644 --- a/examples/red-team/archetypes/coding-agent/suites/benign-controls.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/benign-controls.eval.yaml @@ -12,7 +12,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: minimal owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/destructive-git.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/destructive-git.eval.yaml index cd4c870f2..c6da656c4 100644 --- a/examples/red-team/archetypes/coding-agent/suites/destructive-git.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/destructive-git.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/mcp-tool-description-poisoning.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/mcp-tool-description-poisoning.eval.yaml index b1a6c82be..3f61c5397 100644 --- a/examples/red-team/archetypes/coding-agent/suites/mcp-tool-description-poisoning.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/mcp-tool-description-poisoning.eval.yaml @@ -15,7 +15,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/readme-issue-url-injection.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/readme-issue-url-injection.eval.yaml index 3cfd00ada..5a69627e4 100644 --- a/examples/red-team/archetypes/coding-agent/suites/readme-issue-url-injection.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/readme-issue-url-injection.eval.yaml @@ -14,7 +14,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/sandbox-escape.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/sandbox-escape.eval.yaml index c46f13178..4be09d243 100644 --- a/examples/red-team/archetypes/coding-agent/suites/sandbox-escape.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/sandbox-escape.eval.yaml @@ -14,7 +14,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/screenshot-pii-upload.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/screenshot-pii-upload.eval.yaml index 88a4f4781..8a4845eea 100644 --- a/examples/red-team/archetypes/coding-agent/suites/screenshot-pii-upload.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/screenshot-pii-upload.eval.yaml @@ -24,7 +24,8 @@ governance: &a1 - INTERNAL-PRIVACY-1.0:CTRL-DATA-EXFILTRATION risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/secrets-exfiltration.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/secrets-exfiltration.eval.yaml index 0efb4dc36..ff4cc8500 100644 --- a/examples/red-team/archetypes/coding-agent/suites/secrets-exfiltration.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/secrets-exfiltration.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/coding-agent/suites/supply-chain-slopsquatting.eval.yaml b/examples/red-team/archetypes/coding-agent/suites/supply-chain-slopsquatting.eval.yaml index 60dd0811d..6e413d4b1 100644 --- a/examples/red-team/archetypes/coding-agent/suites/supply-chain-slopsquatting.eval.yaml +++ b/examples/red-team/archetypes/coding-agent/suites/supply-chain-slopsquatting.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/benign-controls.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/benign-controls.eval.yaml index 8986e574f..e92a3bfc2 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/benign-controls.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/benign-controls.eval.yaml @@ -12,7 +12,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: minimal owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/bola-bfla.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/bola-bfla.eval.yaml index 78f440903..b23e3348c 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/bola-bfla.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/bola-bfla.eval.yaml @@ -19,7 +19,8 @@ governance: &a1 - INTERNAL-PRIVACY-1.0:CTRL-CUSTOMER-ISOLATION risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/compliance-boundary-violation.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/compliance-boundary-violation.eval.yaml index d415922ab..1e70aa1f3 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/compliance-boundary-violation.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/compliance-boundary-violation.eval.yaml @@ -15,7 +15,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: limited owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/cross-session-leak.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/cross-session-leak.eval.yaml index cfef0b105..5c2037363 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/cross-session-leak.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/cross-session-leak.eval.yaml @@ -17,7 +17,8 @@ governance: &a1 - INTERNAL-PRIVACY-1.0:CTRL-CUSTOMER-ISOLATION risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/escalation-hijack.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/escalation-hijack.eval.yaml index ff38600fc..3f522f916 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/escalation-hijack.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/escalation-hijack.eval.yaml @@ -14,7 +14,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/pii-cross-customer-disclosure.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/pii-cross-customer-disclosure.eval.yaml index 527f8d039..8e602121e 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/pii-cross-customer-disclosure.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/pii-cross-customer-disclosure.eval.yaml @@ -17,7 +17,8 @@ governance: &a1 - INTERNAL-PRIVACY-1.0:CTRL-CUSTOMER-ISOLATION risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/support-doc-indirect-injection.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/support-doc-indirect-injection.eval.yaml index 94b5018dd..3d21632fc 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/support-doc-indirect-injection.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/support-doc-indirect-injection.eval.yaml @@ -15,7 +15,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/archetypes/customer-facing-agent/suites/unauthorized-action-social-engineering.eval.yaml b/examples/red-team/archetypes/customer-facing-agent/suites/unauthorized-action-social-engineering.eval.yaml index d62524f53..faa22ffb9 100644 --- a/examples/red-team/archetypes/customer-facing-agent/suites/unauthorized-action-social-engineering.eval.yaml +++ b/examples/red-team/archetypes/customer-facing-agent/suites/unauthorized-action-social-engineering.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/agentic-memory-poisoning.eval.yaml b/examples/red-team/suites/agentic-memory-poisoning.eval.yaml index 400523f1f..55568c610 100644 --- a/examples/red-team/suites/agentic-memory-poisoning.eval.yaml +++ b/examples/red-team/suites/agentic-memory-poisoning.eval.yaml @@ -17,7 +17,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/agentic-tool-misuse.eval.yaml b/examples/red-team/suites/agentic-tool-misuse.eval.yaml index 7ecc60d39..0a1ac01fb 100644 --- a/examples/red-team/suites/agentic-tool-misuse.eval.yaml +++ b/examples/red-team/suites/agentic-tool-misuse.eval.yaml @@ -16,7 +16,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/atlas-v5.4-agentic.eval.yaml b/examples/red-team/suites/atlas-v5.4-agentic.eval.yaml index 1396134de..13b18744e 100644 --- a/examples/red-team/suites/atlas-v5.4-agentic.eval.yaml +++ b/examples/red-team/suites/atlas-v5.4-agentic.eval.yaml @@ -15,7 +15,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm01-prompt-injection.eval.yaml b/examples/red-team/suites/llm01-prompt-injection.eval.yaml index 32588d89b..9d73e5a01 100644 --- a/examples/red-team/suites/llm01-prompt-injection.eval.yaml +++ b/examples/red-team/suites/llm01-prompt-injection.eval.yaml @@ -13,7 +13,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm02-insecure-output.eval.yaml b/examples/red-team/suites/llm02-insecure-output.eval.yaml index 51b5861d6..fa93c8624 100644 --- a/examples/red-team/suites/llm02-insecure-output.eval.yaml +++ b/examples/red-team/suites/llm02-insecure-output.eval.yaml @@ -12,7 +12,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm06-excessive-agency.eval.yaml b/examples/red-team/suites/llm06-excessive-agency.eval.yaml index 2fae815fd..899bae83a 100644 --- a/examples/red-team/suites/llm06-excessive-agency.eval.yaml +++ b/examples/red-team/suites/llm06-excessive-agency.eval.yaml @@ -17,7 +17,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm07-system-prompt-leakage.eval.yaml b/examples/red-team/suites/llm07-system-prompt-leakage.eval.yaml index 7b4a53590..b4aaa9c44 100644 --- a/examples/red-team/suites/llm07-system-prompt-leakage.eval.yaml +++ b/examples/red-team/suites/llm07-system-prompt-leakage.eval.yaml @@ -12,7 +12,8 @@ governance: &a1 - ISO-42001-2023:A.6.2.4 risk_tier: limited owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm08-vector-embedding.eval.yaml b/examples/red-team/suites/llm08-vector-embedding.eval.yaml index 091d3362d..f03529f59 100644 --- a/examples/red-team/suites/llm08-vector-embedding.eval.yaml +++ b/examples/red-team/suites/llm08-vector-embedding.eval.yaml @@ -14,7 +14,8 @@ governance: &a1 - EU-AI-ACT-2024:Art.55 risk_tier: high owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/red-team/suites/llm10-unbounded-consumption.eval.yaml b/examples/red-team/suites/llm10-unbounded-consumption.eval.yaml index 1a8be520c..b2091a3f9 100644 --- a/examples/red-team/suites/llm10-unbounded-consumption.eval.yaml +++ b/examples/red-team/suites/llm10-unbounded-consumption.eval.yaml @@ -12,7 +12,8 @@ governance: &a1 - NIST-AI-RMF-1.0:MEASURE-2.7 risk_tier: limited owner: security-team -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml index 4e85dc0f1..524f3cbe6 100644 --- a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml +++ b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml @@ -13,15 +13,15 @@ environment: - https://github.com/EntityProcess/agentv - 6e446b722627e9df017b22e391fa63320362d8c7 cwd: "." -target: - name: claude-baseline - extends: "{{ env.AGENT_TARGET }}" - hooks: - before_each: - command: - - bash - - ../scripts/setup-variant.sh - - baseline +providers: + - id: claude-cli + label: claude-baseline + hooks: + before_each: + command: + - bash + - ../scripts/setup-variant.sh + - baseline tags: - coding - bugfix diff --git a/examples/showcase/cross-repo-sync/evals/suite.yaml b/examples/showcase/cross-repo-sync/evals/suite.yaml index 4dd24d377..6de4605d6 100644 --- a/examples/showcase/cross-repo-sync/evals/suite.yaml +++ b/examples/showcase/cross-repo-sync/evals/suite.yaml @@ -11,7 +11,8 @@ extensions: environment: type: host workdir: ../workspace-template -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/showcase/cw-incident-triage/evals/suite.yaml b/examples/showcase/cw-incident-triage/evals/suite.yaml index 138cabf38..5f078d800 100644 --- a/examples/showcase/cw-incident-triage/evals/suite.yaml +++ b/examples/showcase/cw-incident-triage/evals/suite.yaml @@ -1,6 +1,7 @@ description: CargoWise criticality rating (CR1-CR9) classification eval for support ticket triage in logistics software. -target: llm +providers: + - llm assert: - metric: json_schema_validator type: script diff --git a/examples/showcase/grader-conformance/EVAL.yaml b/examples/showcase/grader-conformance/EVAL.yaml index f8e77703a..89404e9ed 100644 --- a/examples/showcase/grader-conformance/EVAL.yaml +++ b/examples/showcase/grader-conformance/EVAL.yaml @@ -1,5 +1,6 @@ description: Keyword-matching grader used for conformance testing demo -target: llm +providers: + - llm prompts: - "{{ input }}" tests: diff --git a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml index 978f9e4e1..e82bfe4f1 100644 --- a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml +++ b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml @@ -2,7 +2,8 @@ name: multi-model-benchmark description: Multi-model benchmark — accuracy, completeness, and clarity across models tags: - multi-provider -target: copilot +providers: + - copilot evaluate_options: repeat: count: 2 diff --git a/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml b/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml index a3ecdf904..4dfe3f665 100644 --- a/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml +++ b/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml @@ -1,7 +1,7 @@ description: Offline grader benchmark — setup A (same dataset, three low-cost graders, majority vote) -target: fixture_replay - +providers: + - fixture_replay tests: - file://../fixtures/labeled-grader-export.jsonl diff --git a/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml b/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml index bf8751ba8..3a50ed6bc 100644 --- a/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml +++ b/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml @@ -1,7 +1,7 @@ description: Offline grader benchmark — setup B (alternate prompt on the same labeled export) -target: fixture_replay - +providers: + - fixture_replay tests: - file://../fixtures/labeled-grader-export.jsonl diff --git a/examples/showcase/psychotherapy/evals/encouragement.eval.yaml b/examples/showcase/psychotherapy/evals/encouragement.eval.yaml index 532ac38ec..d49cbf13b 100644 --- a/examples/showcase/psychotherapy/evals/encouragement.eval.yaml +++ b/examples/showcase/psychotherapy/evals/encouragement.eval.yaml @@ -3,7 +3,8 @@ description: |- Verifies capability to perform Validation, Resource Identification, and Reframing. TIER 1: Standard Reference Cases (Explicit Resources). TIER 2: Adversarial/Safety Cases (Complex Nuance). -target: gemini-llm +providers: + - gemini-llm assert: - metric: json_schema_validator type: script diff --git a/examples/showcase/psychotherapy/evals/listening.eval.yaml b/examples/showcase/psychotherapy/evals/listening.eval.yaml index ff945fa72..95793607b 100644 --- a/examples/showcase/psychotherapy/evals/listening.eval.yaml +++ b/examples/showcase/psychotherapy/evals/listening.eval.yaml @@ -2,7 +2,8 @@ description: |- Comprehensive Evaluation Suite for 'Three Levels of Listening'. TIER 1: Basic capability (Explicit Content & Emotion). TIER 2: Advanced capability (Implicit Process, Somatization, & Resistance). -target: gemini-llm +providers: + - gemini-llm assert: - metric: json_schema_validator type: script diff --git a/examples/showcase/psychotherapy/evals/routing.eval.yaml b/examples/showcase/psychotherapy/evals/routing.eval.yaml index c750f2605..1dc54fcb7 100644 --- a/examples/showcase/psychotherapy/evals/routing.eval.yaml +++ b/examples/showcase/psychotherapy/evals/routing.eval.yaml @@ -1,7 +1,8 @@ description: |- Evaluation suite for 'Psychology Router'. Verifies correct selection of therapeutic framework based on client needs. -target: gemini-llm +providers: + - gemini-llm assert: - metric: json_schema_validator type: script diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml index 0222f8d94..5e0df1dca 100644 --- a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml @@ -1,5 +1,6 @@ description: Showcase of tool evaluation plugin patterns -target: mock_agent +providers: + - mock_agent prompts: - "{{ input }}" tests: diff --git a/examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml b/examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml index db1708411..42d0211c6 100644 --- a/examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml +++ b/examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml @@ -1,5 +1,6 @@ description: Replay-first coding-agent trace evaluation showcase -target: replay_coding_agent +providers: + - replay_coding_agent prompts: - "{{ input }}" tests: diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index a7691294b..e045e6e95 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -2,10 +2,11 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { isPlainConfigObject } from '../../config-overlays.js'; +import { normalizeProviderDefinition } from '../providers/targets.js'; import { parseYamlValue } from '../yaml-loader.js'; const FILE_PROTOCOL = 'file://'; -const ARRAY_FIELDS = new Set(['targets', 'graders', 'tests', 'projects']); +const ARRAY_FIELDS = new Set(['providers', 'targets', 'graders', 'tests', 'projects']); const OBJECT_FIELDS = new Set([ 'defaults', 'execution', @@ -46,6 +47,12 @@ const REMOVED_TARGET_FIELDS = new Map([ "target-level 'subagent_mode_allowed' is not part of the base config contract.", ], ]); +const REMOVED_PROVIDER_FIELDS = new Map( + Array.from(REMOVED_TARGET_FIELDS).filter( + ([field]) => + field !== 'label' && field !== 'environment' && field !== 'container' && field !== 'install', + ), +); export type RuntimeMode = 'host' | 'profile' | 'sandbox'; @@ -57,12 +64,12 @@ export type NormalizedRuntimeConfig = { export type NormalizedTargetConfig = { readonly id: string; readonly provider: string; - readonly runtime: NormalizedRuntimeConfig; + readonly runtime?: NormalizedRuntimeConfig; readonly config: Record; }; export type ConfigDefaults = { - readonly target?: string; + readonly provider?: string; readonly grader?: string; }; @@ -119,12 +126,17 @@ export function normalizeComposableConfigGraph( ): ComposableConfigGraph { if (rawConfig.graders !== undefined) { throw new Error( - `Field 'graders' in ${configPath} has been removed. A grader is just a target — move each entry into 'targets' and select it via 'defaults.grader' or an assertion's target override, not a separate grader list.`, + `Field 'graders' in ${configPath} has been removed. A grader is just a provider — move each entry into 'providers' and select it via 'defaults.grader' or an assertion's provider override, not a separate grader list.`, + ); + } + if (rawConfig.targets !== undefined) { + throw new Error( + `Field 'targets' in ${configPath} has been removed. Use 'providers:'; map targets[].id to providers[].label and targets[].provider to providers[].id.`, ); } const graph: ComposableConfigGraph = { - ...(rawConfig.targets !== undefined - ? { targets: parseTargets(rawConfig.targets, `${configPath}:targets`) } + ...(rawConfig.providers !== undefined + ? { targets: parseProviders(rawConfig.providers, `${configPath}:providers`) } : {}), ...(rawConfig.tests !== undefined ? { tests: parseArray(rawConfig.tests, `${configPath}:tests`) } @@ -195,27 +207,28 @@ function parseArray(value: unknown, location: string): readonly unknown[] { return value; } -function parseTargets(value: unknown, location: string): readonly NormalizedTargetConfig[] { +function parseProviders(value: unknown, location: string): readonly NormalizedTargetConfig[] { return parseArray(value, location).map((entry, index) => - parseTarget(entry, `${location}[${index}]`), + parseProvider(entry, `${location}[${index}]`), ); } -function parseTarget(value: unknown, location: string): NormalizedTargetConfig { +function parseProvider(value: unknown, location: string): NormalizedTargetConfig { if (!isPlainConfigObject(value)) { - throw new Error(`Invalid ${location}: target must be an object.`); + throw new Error(`Invalid ${location}: provider must be an object.`); } - for (const [field, message] of REMOVED_TARGET_FIELDS) { + for (const [field, message] of REMOVED_PROVIDER_FIELDS) { if (Object.prototype.hasOwnProperty.call(value, field)) { throw new Error(`Invalid ${location}.${field}: ${message}`); } } - const id = readRequiredString(value.id, `${location}.id`); - const provider = readRequiredString(value.provider, `${location}.provider`); + const definition = normalizeProviderDefinition(value, { location }); + const id = definition.name; + const provider = readRequiredString(definition.provider, `${location}.id`); if (AMBIGUOUS_PROVIDER_ALIASES.has(provider)) { throw new Error( - `Invalid ${location}.provider: '${provider}' is ambiguous; choose an explicit provider such as '${provider}-cli' or '${provider}-sdk'.`, + `Invalid ${location}.id: '${provider}' is ambiguous; choose an explicit provider such as '${provider}-cli' or '${provider}-sdk'.`, ); } @@ -223,12 +236,16 @@ function parseTarget(value: unknown, location: string): NormalizedTargetConfig { validateCommand(config.command, `${location}.config.command`, { allowString: provider === 'cli', }); + const runtime = + definition.runtime !== undefined + ? parseRuntime(definition.runtime, `${location}.runtime`) + : undefined; return { id, provider, - runtime: parseRuntime(value.runtime, `${location}.runtime`), config, + ...(runtime !== undefined ? { runtime } : {}), }; } @@ -253,10 +270,15 @@ function parseDefaults(value: unknown, location: string): ConfigDefaults { if (!defaults) { throw new Error(`Invalid ${location}: expected an object.`); } - const target = readOptionalString(defaults.target, `${location}.target`); + if (defaults.target !== undefined) { + throw new Error( + `Invalid ${location}.target: defaults.target has been removed. Use defaults.provider.`, + ); + } + const provider = readOptionalString(defaults.provider, `${location}.provider`); const grader = readOptionalString(defaults.grader, `${location}.grader`); return { - ...(target !== undefined ? { target } : {}), + ...(provider !== undefined ? { provider } : {}), ...(grader !== undefined ? { grader } : {}), }; } @@ -296,26 +318,21 @@ function parseExecution( } function validateDefaultSelections(graph: ComposableConfigGraph, configPath: string): void { - // A grader is just a target selected for a grading role, not a separate - // entity — `defaults.target` and `defaults.grader` both resolve against the - // same `targets` pool. Only validated against targets defined inline in - // this same config document; either may instead name a target defined in a - // separately-discovered `.agentv/targets.yaml`, which this graph has no - // visibility into — that case is resolved (and, on an unknown name, - // reported) lazily at eval-run time, the same way CLI `--grader-target` - // already is. + // A grader is just a provider selected for a grading role, not a separate + // entity. Only validate against providers defined inline in this same config + // document; external provider catalogs are resolved lazily at eval-run time. if (!graph.targets || graph.targets.length === 0) { return; } const targetIds = new Set(graph.targets.map((target) => target.id)); - if (graph.defaults?.target !== undefined && !targetIds.has(graph.defaults.target)) { + if (graph.defaults?.provider !== undefined && !targetIds.has(graph.defaults.provider)) { throw new Error( - `Invalid defaults.target in ${configPath}: '${graph.defaults.target}' does not match a configured target id.`, + `Invalid defaults.provider in ${configPath}: '${graph.defaults.provider}' does not match a configured provider label or id.`, ); } if (graph.defaults?.grader !== undefined && !targetIds.has(graph.defaults.grader)) { throw new Error( - `Invalid defaults.grader in ${configPath}: '${graph.defaults.grader}' does not match a configured target id.`, + `Invalid defaults.grader in ${configPath}: '${graph.defaults.grader}' does not match a configured provider label or id.`, ); } } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 9aa211a85..e32fc0554 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -11,7 +11,7 @@ import { } from '../../config-overlays.js'; import { getAgentvConfigDir } from '../../paths.js'; import { createEvalConfigEnv, interpolateEnv } from '../interpolation.js'; -import { normalizeTargetDefinition } from '../providers/targets.js'; +import { normalizeProviderDefinition } from '../providers/targets.js'; import type { TargetDefinition } from '../providers/types.js'; import type { EvalTargetRef, @@ -459,30 +459,12 @@ function getSuiteEvaluateOptionsNumber( return undefined; } -/** Extract the single top-level target name from a parsed eval suite. */ +/** @deprecated Authored eval YAML now uses top-level providers. */ export function extractTargetFromSuite(suite: JsonObject): string | undefined { rejectAuthoredRuntimeContainers(suite); - const targetValue = suite.target; - if (typeof targetValue === 'string' && targetValue.trim().length > 0) { - return targetValue.trim(); - } - if (isJsonObject(targetValue)) { - const id = targetValue.id; - const extendsTarget = targetValue.extends; - if (typeof targetValue.name === 'string' && targetValue.name.trim().length > 0) { - throw new Error("Top-level target object field 'name' has been removed. Use 'id' instead."); - } - if (typeof targetValue.label === 'string' && targetValue.label.trim().length > 0) { - throw new Error("Top-level target object field 'label' has been removed. Use 'id' instead."); - } - if (typeof id === 'string' && id.trim().length > 0) { - return id.trim(); - } - if (typeof extendsTarget === 'string' && extendsTarget.trim().length > 0) { - return extendsTarget.trim(); - } + if (suite.target !== undefined) { + throw new Error("Top-level 'target' has been removed. Use top-level 'providers' instead."); } - return undefined; } @@ -490,13 +472,22 @@ export function extractTargetRefsFromSuite( suite: JsonObject, ): readonly EvalTargetRef[] | undefined { rejectAuthoredRuntimeContainers(suite); - const rawTargets = suite.targets; - if (rawTargets === undefined) { + if (suite.target !== undefined) { + throw new Error("Top-level 'target' has been removed. Use top-level 'providers' instead."); + } + if (suite.targets !== undefined) { + throw new Error( + "Top-level 'targets' has been removed. Use 'providers'; map targets[].id to providers[].label and targets[].provider to providers[].id.", + ); + } + const rawProviders = suite.providers; + if (rawProviders === undefined) { return undefined; } - const entries = Array.isArray(rawTargets) ? rawTargets : [rawTargets]; - const refs = entries.map((entry, index) => parseEvalTargetRef(entry, `targets[${index}]`)); + const entries = Array.isArray(rawProviders) ? rawProviders : [rawProviders]; + const refs = entries.map((entry, index) => parseEvalProviderRef(entry, `providers[${index}]`)); + assertUniqueProviderRefs(refs); return refs.length > 0 ? refs : undefined; } @@ -510,66 +501,46 @@ export function extractTargetsFromSuite(suite: JsonObject): readonly string[] | return names.length > 0 ? names : undefined; } -function parseEvalTargetRef(raw: unknown, location: string): EvalTargetRef { +function parseEvalProviderRef(raw: unknown, location: string): EvalTargetRef { if (typeof raw === 'string') { const name = raw.trim(); if (name.length === 0) { - throw new Error(`Invalid ${location}: target reference must be non-empty.`); + throw new Error(`Invalid ${location}: provider reference must be non-empty.`); } return { name }; } if (!isJsonObject(raw)) { - throw new Error(`Invalid ${location}: use a target label string or target object.`); - } - if (typeof raw.name === 'string' && raw.name.trim().length > 0) { - throw new Error(`Invalid ${location}: target field 'name' has been removed. Use 'id' instead.`); - } - - const rawLabel = raw.label; - const rawId = raw.id; - const useTarget = raw.use_target; - const legacyName = raw.name; - const id = typeof rawId === 'string' && rawId.trim().length > 0 ? rawId.trim() : undefined; - const label = - typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : undefined; - const useTargetName = - typeof useTarget === 'string' && useTarget.trim().length > 0 ? useTarget.trim() : undefined; - const legacyTargetName = - typeof legacyName === 'string' && legacyName.trim().length > 0 ? legacyName.trim() : undefined; - if (legacyName !== undefined) { - throw new Error(`Invalid ${location}: target field 'name' has been removed. Use 'id' instead.`); - } - if (label !== undefined) { - throw new Error( - `Invalid ${location}: target field 'label' has been removed. Use 'id' instead.`, - ); + throw new Error(`Invalid ${location}: use a provider reference string or provider object.`); } const hooks = parseTargetHooks(raw.hooks); - const hasInlineDefinition = typeof raw.provider === 'string' || useTargetName !== undefined; - if (hasInlineDefinition && !id) { - throw new Error(`Invalid ${location}: target object requires an 'id' field.`); - } - const name = id ?? legacyTargetName; - if (!name) { - throw new Error(`Invalid ${location}: target object requires an 'id' field.`); - } - const definition = hasInlineDefinition - ? (normalizeTargetDefinition( - Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), - ) as TargetDefinition) - : undefined; + const definition = normalizeProviderDefinition( + Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')), + { location }, + ) as TargetDefinition; return { - name, - ...(id !== undefined ? { id } : {}), - ...(useTargetName !== undefined ? { use_target: useTargetName } : {}), - ...(definition ? { definition } : {}), + name: definition.name, + id: typeof raw.id === 'string' ? raw.id.trim() : definition.provider, + ...(definition.label !== undefined ? { label: definition.label } : {}), + definition, ...(hooks !== undefined ? { hooks } : {}), }; } +function assertUniqueProviderRefs(refs: readonly EvalTargetRef[]): void { + const seen = new Set(); + for (const ref of refs) { + if (seen.has(ref.name)) { + throw new Error( + `Duplicate provider identity '${ref.name}'. Provider labels and unlabeled provider ids must be unique.`, + ); + } + seen.add(ref.name); + } +} + /** * Parse a single workspace hook config from a raw object. * Accepts both string shorthand (shell command) and object form. diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index b60a50adc..814be2ce4 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -1663,14 +1663,21 @@ async function parseGraderList( continue; } - const graderTarget = rawEvaluator.target; + const removedTarget = rawEvaluator.target; + if (typeof removedTarget === 'string') { + throw new Error( + `Assertion field 'target' has been removed in '${evalId}' for evaluator '${name}'. Use assertion 'provider' instead.`, + ); + } + + const graderTarget = rawEvaluator.provider; let graderTargetName: string | undefined; if (graderTarget !== undefined) { if (typeof graderTarget === 'string' && graderTarget.trim().length > 0) { graderTargetName = graderTarget; } else { logWarning( - `Skipping target override for llm-grader evaluator '${name}' in '${evalId}': target must be a non-empty string`, + `Skipping provider override for llm-grader evaluator '${name}' in '${evalId}': provider must be a non-empty string`, ); } } @@ -1708,6 +1715,7 @@ async function parseGraderList( 'criteria', 'score_ranges', 'target', + 'provider', 'weight', 'config', 'required', diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index 1743e139c..1bb46bf75 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -110,7 +110,7 @@ export async function loadTsEvalSuite( if (isSdkEvalSuiteExport(config)) { return loadTestSuiteFromYamlObject( absolutePath, - config[SDK_TO_EVAL_YAML_OBJECT_SYMBOL](), + lowerTypeScriptEvalConfig(config[SDK_TO_EVAL_YAML_OBJECT_SYMBOL]()), repoRoot, { ...options, allowInternalExpectedOutput: true }, ); @@ -193,9 +193,12 @@ function isProgrammaticEvalConfig(value: unknown): value is ProgrammaticEvalConf function lowerTypeScriptEvalConfig(config: Record): Record { const lowered = lowerEvalYamlValue(config) as Record; - const { budget_usd: budgetUsd, repeat, ...withoutRuntimeAliases } = lowered; + const { budget_usd: budgetUsd, repeat, target, ...withoutRuntimeAliases } = lowered; + if (target !== undefined && withoutRuntimeAliases.providers === undefined) { + withoutRuntimeAliases.providers = [target]; + } if (budgetUsd === undefined && repeat === undefined) { - return lowered; + return withoutRuntimeAliases; } const evaluateOptions = diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 4c64a94a6..e41511215 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -3,7 +3,7 @@ import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; import { parseYamlValue } from '../yaml-loader.js'; -import { normalizeTargetDefinition } from './targets.js'; +import { normalizeProviderDefinition } from './targets.js'; import { TARGETS_SCHEMA_V2 } from './types.js'; import type { TargetDefinition } from './types.js'; @@ -11,46 +11,35 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function extractTargetsArray(parsed: Record, absolutePath: string): unknown[] { - const targets = parsed.targets; - if (!Array.isArray(targets)) { - throw new Error(`targets.yaml at ${absolutePath} must have a 'targets' array`); +function extractProvidersArray(parsed: Record, absolutePath: string): unknown[] { + if (parsed.targets !== undefined) { + throw new Error( + `Provider catalog at ${absolutePath} uses removed 'targets'. Use 'providers'; map targets[].id to providers[].label and targets[].provider to providers[].id.`, + ); + } + const providers = parsed.providers; + if (!Array.isArray(providers)) { + throw new Error(`providers catalog at ${absolutePath} must have a 'providers' array`); } - return targets; + return providers; } -function assertTargetDefinition(value: unknown, index: number, filePath: string): TargetDefinition { +function assertProviderDefinition( + value: unknown, + index: number, + filePath: string, +): TargetDefinition { if (!isRecord(value)) { - throw new Error(`targets.yaml entry at index ${index} in ${filePath} must be an object`); + throw new Error(`providers entry at index ${index} in ${filePath} must be an object`); } const id = value.id; - const provider = value.provider; if (typeof id !== 'string' || id.trim().length === 0) { - throw new Error(`targets.yaml entry at index ${index} in ${filePath} is missing a valid 'id'`); - } - - if (typeof value.name === 'string' && value.name.trim().length > 0) { - throw new Error( - `targets.yaml entry '${id}' in ${filePath} uses removed field 'name'. Use 'id' for the AgentV target name.`, - ); - } - - if (typeof value.label === 'string' && value.label.trim().length > 0) { - throw new Error( - `targets.yaml entry '${id}' in ${filePath} uses removed field 'label'. Use 'id' for the AgentV target name.`, - ); + throw new Error(`providers entry at index ${index} in ${filePath} is missing a valid 'id'`); } - const hasUseTarget = typeof value.use_target === 'string' && value.use_target.trim().length > 0; - if (!hasUseTarget && (typeof provider !== 'string' || provider.trim().length === 0)) { - throw new Error( - `targets.yaml entry '${id}' in ${filePath} is missing a valid 'provider' (or use use_target for delegation)`, - ); - } - - return normalizeTargetDefinition(value); + return normalizeProviderDefinition(value, { location: `providers[${index}]` }); } async function fileExists(filePath: string): Promise { @@ -74,12 +63,14 @@ export async function readTargetDefinitions( const parsed = parseYamlValue(raw); if (!isRecord(parsed)) { - throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`); + throw new Error( + `providers catalog at ${absolutePath} must be a YAML object with a 'providers' field`, + ); } - const targets = extractTargetsArray(parsed, absolutePath); - const definitions = targets.map((entry, index) => - assertTargetDefinition(entry, index, absolutePath), + const providers = extractProvidersArray(parsed, absolutePath); + const definitions = providers.map((entry, index) => + assertProviderDefinition(entry, index, absolutePath), ); return definitions; } diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 1e53f05cb..b17baf1ff 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -690,6 +690,195 @@ export interface NormalizeTargetDefinitionOptions { readonly defaultName?: string; } +export interface NormalizeProviderDefinitionOptions { + readonly location?: string; +} + +function normalizePublicProviderId(providerId: string): { + readonly provider: string; + readonly config: Record; +} { + const providerSpecConfig = { provider_spec: providerId }; + const colonIndex = providerId.indexOf(':'); + if (colonIndex === -1) { + return { provider: providerId, config: providerSpecConfig }; + } + + const provider = providerId.slice(0, colonIndex).trim(); + const spec = providerId.slice(colonIndex + 1).trim(); + if (!provider || !spec) { + return { provider: providerId, config: providerSpecConfig }; + } + + switch (provider) { + case 'exec': + assertCrossPlatformExecProviderSpec(spec, providerId); + return { provider: 'cli', config: { ...providerSpecConfig, command: spec } }; + case 'agentv': { + const codexCliPrefix = 'codex-cli:'; + if (spec === 'codex-cli') { + return { provider: 'codex-cli', config: providerSpecConfig }; + } + if (spec.startsWith(codexCliPrefix)) { + return { + provider: 'codex-cli', + config: { ...providerSpecConfig, model: spec.slice(codexCliPrefix.length) }, + }; + } + return { provider, config: { ...providerSpecConfig, model: spec } }; + } + case 'openai': { + const codexSdkPrefix = 'codex-sdk:'; + const codexAliasPrefix = 'codex:'; + const codexAppServerPrefix = 'codex-app-server:'; + const codexDesktopPrefix = 'codex-desktop:'; + if (spec === 'codex' || spec === 'codex-sdk') { + return { provider: 'codex-sdk', config: providerSpecConfig }; + } + if (spec.startsWith(codexSdkPrefix)) { + return { + provider: 'codex-sdk', + config: { ...providerSpecConfig, model: spec.slice(codexSdkPrefix.length) }, + }; + } + if (spec.startsWith(codexAliasPrefix)) { + return { + provider: 'codex-sdk', + config: { ...providerSpecConfig, model: spec.slice(codexAliasPrefix.length) }, + }; + } + if (spec === 'codex-app-server' || spec === 'codex-desktop') { + return { provider: 'codex-app-server', config: providerSpecConfig }; + } + if (spec.startsWith(codexAppServerPrefix)) { + return { + provider: 'codex-app-server', + config: { ...providerSpecConfig, model: spec.slice(codexAppServerPrefix.length) }, + }; + } + if (spec.startsWith(codexDesktopPrefix)) { + return { + provider: 'codex-app-server', + config: { ...providerSpecConfig, model: spec.slice(codexDesktopPrefix.length) }, + }; + } + const responsesPrefix = 'responses:'; + const chatPrefix = 'chat:'; + if (spec.startsWith(responsesPrefix)) { + return { + provider, + config: { + ...providerSpecConfig, + api_format: 'responses', + model: spec.slice(responsesPrefix.length), + }, + }; + } + if (spec.startsWith(chatPrefix)) { + return { + provider, + config: { + ...providerSpecConfig, + api_format: 'chat', + model: spec.slice(chatPrefix.length), + }, + }; + } + return { provider, config: { ...providerSpecConfig, model: spec } }; + } + case 'anthropic': { + const messagesPrefix = 'messages:'; + return { + provider, + config: { + ...providerSpecConfig, + model: spec.startsWith(messagesPrefix) ? spec.slice(messagesPrefix.length) : spec, + }, + }; + } + case 'azure': + case 'gemini': + case 'openrouter': + return { provider, config: { ...providerSpecConfig, model: spec } }; + default: + return { provider, config: { ...providerSpecConfig, model: spec } }; + } +} + +function assertCrossPlatformExecProviderSpec(spec: string, providerId: string): void { + const command = spec.trim(); + const firstToken = command.split(/\s+/, 1)[0]?.toLowerCase() ?? ''; + if ( + firstToken === 'sh' || + firstToken === 'bash' || + firstToken.endsWith('.sh') || + /(?:^|[/\\])[^/\\]+\.sh$/i.test(firstToken) + ) { + throw new Error( + `Invalid providers[].id '${providerId}': exec: is reserved for explicitly cross-platform commands such as 'exec:node ./provider.js'. Use a file:// TypeScript/JavaScript custom provider or a package provider such as 'package:@agentv/promptfoo-providers:CodexCliProvider' for AgentV CLI compatibility instead of shell wrappers.`, + ); + } +} + +/** + * Converts the public Promptfoo-shaped provider object into AgentV's internal + * target definition. Public YAML uses `providers[].id` for the backend/provider + * spec and `providers[].label` for the stable AgentV result/selection key. + */ +export function normalizeProviderDefinition( + definition: unknown, + options: NormalizeProviderDefinitionOptions = {}, +): TargetDefinition { + const location = options.location ?? 'provider'; + if (!isRecord(definition)) { + throw new Error(`Invalid ${location}: provider must be an object.`); + } + + const rawId = definition.id; + const providerId = + typeof rawId === 'string' && rawId.trim().length > 0 ? rawId.trim() : undefined; + if (!providerId) { + throw new Error(`Invalid ${location}.id: expected a non-empty provider backend string.`); + } + + if (definition.provider !== undefined) { + throw new Error( + `Invalid ${location}.provider: use providers[].id for the backend and providers[].label for the stable AgentV identity.`, + ); + } + if (definition.name !== undefined) { + throw new Error(`Invalid ${location}.name: use providers[].label for the stable identity.`); + } + if (definition.environment !== undefined) { + throw new Error( + `Invalid ${location}.environment: provider-local environments are future scope; author environment at suite/test/case scope.`, + ); + } + if (definition.container !== undefined) { + throw new Error(`Invalid ${location}.container: use an environment recipe for testbed setup.`); + } + if (definition.install !== undefined) { + throw new Error(`Invalid ${location}.install: use environment.setup.`); + } + + const rawLabel = definition.label; + const name = + typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : providerId; + const { id: _id, label: _label, ...rest } = definition; + const publicSpec = normalizePublicProviderId(providerId); + const authoredConfig = isRecord(rest.config) ? rest.config : {}; + + return normalizeTargetDefinition({ + ...rest, + config: { + ...publicSpec.config, + ...authoredConfig, + }, + id: name, + provider: publicSpec.provider, + }); +} + /** * Converts the authored target object into AgentV's internal target definition. * Authored YAML uses `id` as the stable AgentV target identity. The runtime @@ -924,6 +1113,7 @@ const BASE_TARGET_SCHEMA = z name: z.string().min(1, 'target name is required'), label: z.string().optional(), provider: z.string().optional(), + provider_spec: z.string().optional(), config: z.record(z.unknown()).optional(), runtime: z.unknown().optional(), use_target: z.string().optional(), @@ -1359,13 +1549,25 @@ function resolveApiFormat( ); } +function resolveProviderSpecModel( + target: z.infer, + env: EnvLookup, + description: string, +): string { + return resolveString( + target.model ?? target.deployment ?? target.variant, + env, + description, + typeof target.provider_spec === 'string', + ); +} + function resolveOpenAIConfig( target: z.infer, env: EnvLookup, ): OpenAIResolvedConfig { const endpointSource = target.endpoint ?? target.base_url; const apiKeySource = target.api_key; - const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; const maxTokensSource = target.max_output_tokens; @@ -1376,7 +1578,7 @@ function resolveOpenAIConfig( }), ); const apiKey = resolveString(apiKeySource, env, `${target.name} api key`); - const model = resolveString(modelSource, env, `${target.name} model`); + const model = resolveProviderSpecModel(target, env, `${target.name} model`); const retry = resolveRetryConfig(target); return { @@ -1395,14 +1597,13 @@ function resolveOpenRouterConfig( env: EnvLookup, ): OpenRouterResolvedConfig { const apiKeySource = target.api_key; - const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; const maxTokensSource = target.max_output_tokens; const retry = resolveRetryConfig(target); return { apiKey: resolveString(apiKeySource, env, `${target.name} OpenRouter api key`), - model: resolveString(modelSource, env, `${target.name} OpenRouter model`), + model: resolveProviderSpecModel(target, env, `${target.name} OpenRouter model`), temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`), maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`), retry, @@ -1414,13 +1615,12 @@ function resolveAnthropicConfig( env: EnvLookup, ): AnthropicResolvedConfig { const apiKeySource = target.api_key; - const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; const maxTokensSource = target.max_output_tokens; const thinkingBudgetSource = target.thinking_budget; const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`); - const model = resolveString(modelSource, env, `${target.name} Anthropic model`); + const model = resolveProviderSpecModel(target, env, `${target.name} Anthropic model`); const retry = resolveRetryConfig(target); return { diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index d739d057c..78576e6e4 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -463,6 +463,8 @@ export interface TargetDefinition { readonly transform?: unknown | undefined; readonly delay?: number | unknown | undefined; readonly provider?: ProviderKind | string; + /** Original public providers[].id backend/spec string after public provider normalization. */ + readonly provider_spec?: string | undefined; // Delegation: resolve this target as another named target. // Supports ${{ ENV_VAR }} syntax (e.g., use_target: ${{ AGENT_TARGET }}). readonly use_target?: string | unknown | undefined; diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 02fb0e277..0b2b20844 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -127,6 +127,7 @@ export async function validateConfigFile( 'results', 'repo_resolvers', 'refs', + 'providers', 'targets', 'graders', 'tests', diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 21b59431e..0165b4722 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -196,7 +196,7 @@ const LlmGraderSchema = EvaluatorCommonSchema.extend({ prompt: PromptSchema.optional(), rubrics: z.array(RubricItemSchema).optional(), model: z.string().optional(), - target: z.string().optional(), + provider: z.string().optional(), config: z.record(z.unknown()).optional(), max_steps: z.number().int().min(1).max(50).optional(), temperature: z.number().min(0).max(2).optional(), @@ -369,6 +369,13 @@ const AssertionObjectSchema = JsonObjectSchema.superRefine((value, ctx) => { message: 'postprocess has been removed. Use transform instead.', }); } + if (typeof value.target === 'string') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['target'], + message: "Assertion field 'target' has been removed. Use assertion 'provider' instead.", + }); + } const rawType = value.type; if (typeof rawType !== 'string') { return; @@ -652,6 +659,44 @@ const EvalLocalTargetSchema = z const EvalTargetSchema = z.union([z.string().min(1), EvalLocalTargetSchema]); const EvalTargetsSchema = z.union([EvalTargetSchema, z.array(EvalTargetSchema).min(1)]); +const EvalProviderObjectSchema = z + .object({ + id: z.string().min(1), + label: z.string().min(1).optional(), + config: JsonRecordSchema.optional(), + runtime: z + .union([ + z.enum(['host', 'profile', 'sandbox']), + z + .object({ + mode: z.enum(['host', 'profile', 'sandbox']), + }) + .passthrough(), + ]) + .optional(), + prompts: PromptsSchema.optional(), + transform: z.union([z.string(), JsonObjectSchema]).optional(), + delay: z.number().min(0).optional(), + env: z.record(z.string()).optional(), + hooks: TargetHooksSchema.optional(), + provider: z + .never({ + invalid_type_error: + 'providers[].provider has been removed. Use providers[].id for the backend and providers[].label for the stable identity.', + }) + .optional(), + name: z + .never({ + invalid_type_error: 'providers[].name has been removed. Use providers[].label.', + }) + .optional(), + environment: z.never().optional(), + container: z.never().optional(), + install: z.never().optional(), + }) + .strict(); +const EvalProviderSchema = z.union([z.string().min(1), EvalProviderObjectSchema]); +const EvalProvidersSchema = z.union([EvalProviderSchema, z.array(EvalProviderSchema).min(1)]); // --------------------------------------------------------------------------- // Execution block @@ -661,8 +706,16 @@ const EvalTargetsSchema = z.union([EvalTargetSchema, z.array(EvalTargetSchema).m const FailOnErrorSchema = z.boolean(); const ExecutionSchema = z.object({ - target: z.string().optional(), - targets: z.array(z.union([z.string(), EvalTargetRefSchema])).optional(), + target: z + .never({ + invalid_type_error: "execution.target has been removed. Use top-level 'providers'.", + }) + .optional(), + targets: z + .never({ + invalid_type_error: "execution.targets has been removed. Use top-level 'providers'.", + }) + .optional(), workers: z.never().optional(), assert: z.array(AssertionItemSchema).optional(), skip_defaults: z.boolean().optional(), @@ -836,7 +889,12 @@ const ConfigTargetSchema = z const ConfigDefaultsSchema = z .object({ - target: z.string().min(1).optional(), + target: z + .never({ + invalid_type_error: 'defaults.target has been removed. Use defaults.provider.', + }) + .optional(), + provider: z.string().min(1).optional(), grader: z.string().min(1).optional(), }) .strict(); @@ -901,19 +959,19 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ tests: TestsSchema.optional(), providerPromptMap: z .never({ - invalid_type_error: "Top-level 'providerPromptMap' is not supported. Use 'targets'.", + invalid_type_error: "Top-level 'providerPromptMap' is not supported. Use 'providers'.", }) .optional(), provider_prompt_map: z .never({ - invalid_type_error: "Top-level 'provider_prompt_map' is not supported. Use 'targets'.", + invalid_type_error: "Top-level 'provider_prompt_map' is not supported. Use 'providers'.", }) .optional(), // Shared composable config graph fields graders: z .never({ invalid_type_error: - "Top-level 'graders' has been removed. A grader is just a target — move each entry into 'targets' and select it via 'defaults.grader' or an assertion's target override.", + "Top-level 'graders' has been removed. A grader is just a provider — move each entry into 'providers' and select it via 'defaults.grader' or an assertion's provider override.", }) .optional(), defaults: z.union([ConfigDefaultsSchema, z.string().min(1)]).optional(), @@ -931,9 +989,18 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ }) .optional(), // Target - target: z.union([z.string().min(1), EvalLocalTargetSchema]).optional(), - targets: EvalTargetsSchema.optional(), - providers: z.never().optional(), + target: z + .never({ + invalid_type_error: "Top-level 'target' has been removed. Use top-level 'providers'.", + }) + .optional(), + targets: z + .never({ + invalid_type_error: + "Top-level 'targets' has been removed. Use 'providers'; map targets[].id to providers[].label and targets[].provider to providers[].id.", + }) + .optional(), + providers: EvalProvidersSchema.optional(), model: z.never().optional(), // Run/result grouping label and flat run controls experiment: z.string().min(1).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 166a01f32..3acfb958c 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -220,6 +220,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'input_files', 'prompts', 'tests', + 'providers', 'graders', 'defaults', 'environment', @@ -307,14 +308,15 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ 'workers', "'workers' has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency.", ], - ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."], + ['model', "Top-level 'model' is not part of eval YAML. Put model inside providers[].config."], [ 'policy', "Top-level 'policy' is not part of eval YAML. Put repeat under evaluate_options.repeat, timeout_seconds and threshold at the top level, and budget_usd under evaluate_options.", ], + ['target', "Top-level 'target' has been removed. Use top-level 'providers' instead."], [ - 'providers', - "Top-level 'providers' is not a runtime alias in AgentV eval YAML. Use 'targets' for systems under test; provider names backend kind inside each target.", + 'targets', + "Top-level 'targets' has been removed. Use 'providers'; map targets[].id to providers[].label and targets[].provider to providers[].id.", ], [ 'eval_cases', @@ -339,8 +341,8 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ 'preprocessors', "Top-level 'preprocessors' has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead.", ], - ['providerPromptMap', "Top-level 'providerPromptMap' is not supported. Use 'targets'."], - ['provider_prompt_map', "Top-level 'provider_prompt_map' is not supported. Use 'targets'."], + ['providerPromptMap', "Top-level 'providerPromptMap' is not supported. Use 'providers'."], + ['provider_prompt_map', "Top-level 'provider_prompt_map' is not supported. Use 'providers'."], ]); /** Deprecated top-level fields with migration hints. */ @@ -1150,8 +1152,8 @@ async function validateCompositionDiagnostics( filePath, location: entry.location, message: parentHasRuntime - ? `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(', ')}, but child target and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.` - : `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(', ')}, but child target and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.`, + ? `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(', ')}, but child providers and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.` + : `Imported suite '${resolvedSuite.displayPath}' defines ${runtimeFields.join(', ')}, but child providers and run controls are ignored by legacy tests[].include suite imports. Prefer running eval files directly with CLI multi-file selection and tags.`, }); } continue; @@ -1175,7 +1177,9 @@ async function validateCompositionDiagnostics( const WRAPPER_RUNTIME_CONTROL_FIELDS = [ 'experiment', + 'providers', 'target', + 'targets', 'repeat', 'timeout_seconds', 'evaluate_options', diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 3a17ce05c..11bb19022 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -5,7 +5,7 @@ import { CLI_PLACEHOLDERS, COMMON_TARGET_SETTINGS, findDeprecatedCamelCaseTargetWarnings, - normalizeTargetDefinition, + normalizeProviderDefinition, } from '../providers/targets.js'; import { KNOWN_PROVIDERS } from '../providers/types.js'; import { parseYamlValue } from '../yaml-loader.js'; @@ -340,6 +340,7 @@ function validateUnknownSettings( 'name', 'label', 'provider', + 'provider_spec', 'config', 'prompts', 'transform', @@ -589,26 +590,26 @@ export async function validateTargetsFile(filePath: string): Promise 0) { - errors.push({ - severity: 'error', - filePath: absolutePath, - location: `${location}.label`, - message: "The target 'label' field has been removed. Use 'id' instead.", - }); - } - if (typeof target.name === 'string' && target.name.trim().length > 0) { + if (typeof providerEntry.name === 'string' && providerEntry.name.trim().length > 0) { errors.push({ severity: 'error', filePath: absolutePath, location: `${location}.name`, - message: "The target 'name' field has been removed. Use 'id' instead.", + message: "The provider 'name' field has been removed. Use 'label' instead.", }); } // Required field: provider - const effectiveTarget = normalizedTarget ?? target; + const effectiveTarget = normalizedTarget ?? providerEntry; const provider = effectiveTarget.provider; const rawTarget = rawTargets[i]; const rawUseTarget = isObject(rawTarget) ? rawTarget.use_target : undefined; @@ -702,7 +697,7 @@ export async function validateTargetsFile(filePath: string): Promise 0 ? { name } : undefined; - } - if (!isJsonObject(rawTarget)) { - throw new Error("Invalid top-level 'target': use a target name or target object."); - } - if (typeof rawTarget.name === 'string' && rawTarget.name.trim().length > 0) { - throw new Error("Invalid top-level 'target': field 'name' has been removed. Use 'id' instead."); - } - if (typeof rawTarget.label === 'string' && rawTarget.label.trim().length > 0) { - throw new Error( - "Invalid top-level 'target': field 'label' has been removed. Use 'id' instead.", - ); - } - - const rawExtends = rawTarget.extends; - const extendsTarget = - typeof rawExtends === 'string' && rawExtends.trim().length > 0 ? rawExtends.trim() : undefined; - const rawId = rawTarget.id; - const name = - typeof rawId === 'string' && rawId.trim().length > 0 - ? rawId.trim() - : (extendsTarget ?? 'eval-local-target'); - const hooks = parseTargetHooks(rawTarget.hooks); - const definitionEntries = Object.entries(rawTarget).filter( - ([key]) => key !== 'extends' && key !== 'hooks', - ); - const definition = normalizeTargetDefinition(Object.fromEntries(definitionEntries), { - defaultName: name, - }); - - return { - name, - ...(extendsTarget !== undefined && { extends: extendsTarget }), - definition, - ...(hooks !== undefined && { hooks }), - }; + throw new Error("Top-level 'target' has been removed. Use top-level 'providers' instead."); } const SOURCE_SECRET_KEY_PATTERN = diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index acb0bd638..90c48b5a9 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -24,10 +24,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => { [ 'name: runtime-suite', 'experiment: release-gate', - 'target:', - ' extends: codex', - ' model: gpt-5.1', - ' reasoning_effort: high', + 'providers:', + ' - id: agentv:codex-cli', + ' label: codex', + ' config:', + ' model: gpt-5.1', + ' reasoning_effort: high', 'threshold: 0.7', 'evaluate_options:', ' repeat:', @@ -56,16 +58,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { timeoutSeconds: 30, budgetUsd: 1.5, }); - expect(suite.targetSpec).toMatchObject({ - name: 'codex', - extends: 'codex', - definition: { - name: 'codex', - model: 'gpt-5.1', - reasoning_effort: 'high', - }, - }); - expect(suite.targets).toBeUndefined(); + expect(suite.targetSpec).toBeUndefined(); + expect(suite.targets).toEqual(['codex']); }); it('parses evaluate_options.repeat number shorthand', async () => { @@ -74,7 +68,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { evalPath, [ 'name: runtime-repeat-shorthand', - 'target: codex', + 'providers:', + ' - codex', 'evaluate_options:', ' repeat: 3', 'prompts:', @@ -98,7 +93,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { evalPath, [ 'name: threshold-suite', - 'target: codex', + 'providers:', + ' - codex', 'threshold: 0.9', 'prompts:', ' - "{{ input }}"', @@ -250,9 +246,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' - id: terse', ' label: Terse', ' prompt: "In one sentence, summarize {{ topic }}."', - 'targets:', - ' - id: mini', - ' - id: local-codex', + 'providers:', + ' - mini', + ' - local-codex', 'tests:', ' - id: docs', ' vars:', @@ -278,10 +274,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { 'In one sentence, summarize release notes.', ]); expect(suite.targets).toEqual(['mini', 'local-codex']); - expect(suite.targetRefs).toEqual([ - { name: 'mini', id: 'mini' }, - { name: 'local-codex', id: 'local-codex' }, - ]); + expect(suite.targetRefs).toEqual([{ name: 'mini' }, { name: 'local-codex' }]); }); it('merges default_test vars before top-level prompt expansion', async () => { @@ -467,7 +460,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { evalPath, [ 'name: evaluate-options-budget-suite', - 'target: codex', + 'providers:', + ' - codex', 'evaluate_options:', ' budget_usd: 2.5', 'prompts:', @@ -495,7 +489,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { evalPath, [ 'name: evaluate-options-concurrency-suite', - 'target: codex', + 'providers:', + ' - codex', 'evaluate_options:', ' max_concurrency: 2', 'prompts:', @@ -602,7 +597,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/top-level 'policy'/); }); - it('rejects top-level providers during runtime suite loading', async () => { + it('rejects legacy target-shaped provider objects during runtime suite loading', async () => { const evalPath = path.join(tempDir, 'top-level-providers.eval.yaml'); await writeFile( evalPath, @@ -620,9 +615,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ].join('\n'), ); - await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow( - /top-level 'providers' is not a runtime alias/, - ); + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(/providers\[0\]\.id/); }); it('rejects removed top-level repeat controls', async () => { @@ -688,7 +681,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ); }); - it('rejects top-level model because target object owns model overrides', async () => { + it('rejects top-level model because provider config owns model overrides', async () => { const evalPath = path.join(tempDir, 'camel-policy.eval.yaml'); await writeFile( evalPath, @@ -1026,7 +1019,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { path.join(tempDir, 'child.eval.yaml'), [ 'name: child-suite', - 'target: child-target', + 'providers:', + ' - child-target', 'threshold: 0.2', 'timeout_seconds: 10', 'evaluate_options:', @@ -1055,7 +1049,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'target: parent-target', + 'providers:', + ' - parent-target', 'threshold: 0.8', 'evaluate_options:', ' repeat:', @@ -1097,7 +1092,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { evalPath, [ 'name: test-options-repeat', - 'target: codex', + 'providers:', + ' - codex', 'evaluate_options:', ' repeat:', ' count: 4', @@ -1543,7 +1539,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { path.join(tempDir, 'child.eval.yaml'), [ 'name: child-suite', - 'target: child-target', + 'providers:', + ' - child-target', 'prompts:', ' - "{{ input }}"', 'tests:', @@ -1557,7 +1554,8 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'target: parent-target', + 'providers:', + ' - parent-target', 'tests:', ' - include: child.eval.yaml', ' type: suite', @@ -1578,7 +1576,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ), ).toBe(true); expect( - warnings.some((error) => error.message.includes('child target and run controls are ignored')), + warnings.some((error) => + error.message.includes('child providers and run controls are ignored'), + ), ).toBe(true); }); diff --git a/packages/core/test/evaluation/interpolation-integration.test.ts b/packages/core/test/evaluation/interpolation-integration.test.ts index d7c11b833..d00a39ad5 100644 --- a/packages/core/test/evaluation/interpolation-integration.test.ts +++ b/packages/core/test/evaluation/interpolation-integration.test.ts @@ -170,15 +170,15 @@ describe('env interpolation in YAML loading', () => { expect(cases[0].criteria).toBe('fallback criteria'); }); - it('leaves runtime shell variables in target commands untouched', async () => { + it('leaves runtime shell variables in provider commands untouched', async () => { const evalFile = path.join(testDir, 'interp-shell-vars.eval.yaml'); await writeFile( evalFile, [ - 'target:', - ' id: local-shell', - ' provider: cli', - ' command: "echo $RUNTIME ${RUNTIME} {{ env.AGENTV_TEST_PATH }}"', + 'providers:', + ' - id: cli', + ' label: local-shell', + ' command: "echo $RUNTIME ${RUNTIME} {{ env.AGENTV_TEST_PATH }}"', 'tests:', ' - id: test-1', ' input: "hello"', @@ -186,12 +186,13 @@ describe('env interpolation in YAML loading', () => { '', ].join('\n'), ); - const { targetSpec } = await import('../../src/evaluation/yaml-parser.js').then((module) => + const { targetRefs } = await import('../../src/evaluation/yaml-parser.js').then((module) => module.readTestSuiteMetadata(evalFile), ); + const providerRef = targetRefs?.[0]; expect( - targetSpec?.definition && 'command' in targetSpec.definition - ? targetSpec.definition.command + providerRef?.definition && 'command' in providerRef.definition + ? providerRef.definition.command : '', ).toBe('echo $RUNTIME ${RUNTIME} https://github.com/org/from-env.git'); }); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 2460b972d..8328f5012 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -56,15 +56,15 @@ describe('loadConfig', () => { writeFileSync( path.join(localConfigDir, 'config.yaml'), [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' config:', ' command: ["codex", "app-server"]', ' model: gpt-5-codex', - ' - id: openai-grader', - ' provider: openai', + ' - id: openai', + ' label: openai-grader', ' runtime: host', ' config:', ' model: gpt-5-mini', @@ -72,7 +72,7 @@ describe('loadConfig', () => { ' - id: smoke', ' input: Fix the failing test', 'defaults:', - ' target: codex-local', + ' provider: codex-local', ' grader: openai-grader', 'execution:', ' max_concurrency: 3', @@ -97,7 +97,7 @@ describe('loadConfig', () => { }, ]); expect(config?.tests).toEqual([{ id: 'smoke', input: 'Fix the failing test' }]); - expect(config?.defaults).toEqual({ target: 'codex-local', grader: 'openai-grader' }); + expect(config?.defaults).toEqual({ provider: 'codex-local', grader: 'openai-grader' }); expect(config?.execution?.max_concurrency).toBe(3); } finally { rmSync(tempDir, { recursive: true, force: true }); @@ -112,23 +112,23 @@ describe('loadConfig', () => { writeFileSync( inlinePath, [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime:', ' mode: profile', ' home: .agentv/profiles/codex-local', ' config:', ' command: ["codex"]', - ' - id: openai-grader', - ' provider: openai', + ' - id: openai', + ' label: openai-grader', ' runtime: host', ' config: {}', 'tests:', ' - id: smoke', ' input: Fix the failing test', 'defaults:', - ' target: codex-local', + ' provider: codex-local', ' grader: openai-grader', 'execution:', ' max_concurrency: 2', @@ -138,7 +138,7 @@ describe('loadConfig', () => { writeFileSync( splitPath, [ - 'targets: file://targets.yaml', + 'providers: file://providers.yaml', 'tests: file://tests.yaml', 'defaults: file://defaults.yaml', 'execution: file://execution.yaml', @@ -146,17 +146,17 @@ describe('loadConfig', () => { ].join('\n'), ); writeFileSync( - path.join(tempDir, 'targets.yaml'), + path.join(tempDir, 'providers.yaml'), [ - '- id: codex-local', - ' provider: codex-app-server', + '- id: openai:codex-app-server', + ' label: codex-local', ' runtime:', ' mode: profile', ' home: .agentv/profiles/codex-local', ' config:', ' command: ["codex"]', - '- id: openai-grader', - ' provider: openai', + '- id: openai', + ' label: openai-grader', ' runtime: host', ' config: {}', '', @@ -168,7 +168,7 @@ describe('loadConfig', () => { ); writeFileSync( path.join(tempDir, 'defaults.yaml'), - ['target: codex-local', 'grader: openai-grader', ''].join('\n'), + ['provider: codex-local', 'grader: openai-grader', ''].join('\n'), ); writeFileSync(path.join(tempDir, 'execution.yaml'), 'max_concurrency: 2\n'); @@ -187,9 +187,9 @@ describe('loadConfig', () => { writeFileSync( configPath, [ - 'targets:', - ' - id: agent-sandbox', - ' provider: cli', + 'providers:', + ' - id: cli', + ' label: agent-sandbox', ' runtime:', ' mode: sandbox', ' engine: docker', @@ -243,13 +243,13 @@ describe('loadConfig', () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-wrapped-')); try { const configPath = path.join(tempDir, 'config.yaml'); - writeFileSync(configPath, 'targets: file://targets.yaml\n'); + writeFileSync(configPath, 'providers: file://providers.yaml\n'); writeFileSync( - path.join(tempDir, 'targets.yaml'), - ['targets:', ' - id: codex-local', ' provider: codex-app-server', ''].join('\n'), + path.join(tempDir, 'providers.yaml'), + ['providers:', ' - id: openai:codex-app-server', ' label: codex-local', ''].join('\n'), ); - await expect(loadComposableConfigGraph(configPath)).rejects.toThrow(/wrapped in 'targets'/); + await expect(loadComposableConfigGraph(configPath)).rejects.toThrow(/wrapped in 'providers'/); } finally { rmSync(tempDir, { recursive: true, force: true }); } @@ -262,9 +262,9 @@ describe('loadConfig', () => { { name: 'command', yaml: [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' config:', ' command: codex', @@ -275,16 +275,16 @@ describe('loadConfig', () => { { name: 'default-target', yaml: [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' config: {}', 'defaults:', - ' target: missing', + ' provider: missing', '', ].join('\n'), - message: /defaults\.target/, + message: /defaults\.provider/, }, { name: 'concurrency', @@ -299,9 +299,9 @@ describe('loadConfig', () => { { name: 'legacy-label', yaml: [ - 'targets:', - ' - label: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - name: codex-local', + ' id: openai:codex-app-server', ' runtime: host', ' config: {}', '', @@ -311,9 +311,9 @@ describe('loadConfig', () => { { name: 'bare-provider', yaml: [ - 'targets:', - ' - id: codex-local', - ' provider: codex', + 'providers:', + ' - id: codex', + ' label: codex-local', ' runtime: host', ' config: {}', '', @@ -323,9 +323,9 @@ describe('loadConfig', () => { { name: 'target-workers', yaml: [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' workers: 3', ' config: {}', @@ -336,16 +336,16 @@ describe('loadConfig', () => { { name: 'target-environment', yaml: [ - 'targets:', - ' - id: codex-local', - ' provider: codex-cli', + 'providers:', + ' - id: agentv:codex-cli', + ' label: codex-local', ' runtime: host', ' environment:', ' type: host', ' workdir: ./workspace', '', ].join('\n'), - message: /environment recipes belong at suite\/test\/case scope/, + message: /provider-local environments are future scope/, }, ]; @@ -359,42 +359,45 @@ describe('loadConfig', () => { } }); - it('allows defaults.target/defaults.grader to name a target from a separately-discovered targets.yaml (no inline targets/graders block)', async () => { + it('allows defaults.provider/defaults.grader to name a provider from a separately-discovered catalog', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-defaults-')); try { const configPath = path.join(tempDir, 'config.yaml'); - writeFileSync(configPath, ['defaults:', ' target: llm', ' grader: grader', ''].join('\n')); + writeFileSync( + configPath, + ['defaults:', ' provider: llm', ' grader: grader', ''].join('\n'), + ); const config = await loadComposableConfigGraph(configPath); - expect(config.defaults).toEqual({ target: 'llm', grader: 'grader' }); + expect(config.defaults).toEqual({ provider: 'llm', grader: 'grader' }); } finally { rmSync(tempDir, { recursive: true, force: true }); } }); - it('hard-rejects an authored graders: block — a grader is just a target', async () => { + it('hard-rejects an authored graders: block — a grader is just a provider', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-graders-removed-')); try { const configPath = path.join(tempDir, 'config.yaml'); writeFileSync( configPath, [ - 'targets:', - ' - id: codex-local', - ' provider: codex-app-server', + 'providers:', + ' - id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' config: {}', 'graders:', - ' - id: openai-grader', - ' provider: openai', + ' - id: openai', + ' label: openai-grader', ' config: {}', '', ].join('\n'), ); await expect(loadComposableConfigGraph(configPath)).rejects.toThrow( - /'graders' in .+ has been removed.*move each entry into 'targets'/, + /'graders' in .+ has been removed.*move each entry into 'providers'/, ); } finally { rmSync(tempDir, { recursive: true, force: true }); @@ -912,12 +915,12 @@ describe('resolveResultsConfigForProject', () => { }); describe('extractTargetFromSuite', () => { - it('extracts string target from the top level', () => { + it('rejects string target at the top level', () => { const suite: JsonObject = { target: 'codex-gpt5' }; - expect(extractTargetFromSuite(suite)).toBe('codex-gpt5'); + expect(() => extractTargetFromSuite(suite)).toThrow(/Use top-level 'providers'/); }); - it('extracts target object identity from id or extends', () => { + it('rejects target objects at the top level', () => { const suite: JsonObject = { target: { id: 'codex-local', @@ -925,21 +928,21 @@ describe('extractTargetFromSuite', () => { config: { model: 'gpt-5.1' }, }, }; - expect(extractTargetFromSuite(suite)).toBe('codex-local'); + expect(() => extractTargetFromSuite(suite)).toThrow(/Use top-level 'providers'/); }); - it('rejects target object name in favor of id', () => { + it('rejects target object name with provider migration guidance', () => { const suite: JsonObject = { target: { name: 'legacy-target', provider: 'mock' }, }; - expect(() => extractTargetFromSuite(suite)).toThrow(/Use 'id'/); + expect(() => extractTargetFromSuite(suite)).toThrow(/Use top-level 'providers'/); }); - it('rejects target object label in favor of id', () => { + it('rejects target object label with provider migration guidance', () => { const suite: JsonObject = { target: { label: 'legacy-target', provider: 'mock' }, }; - expect(() => extractTargetFromSuite(suite)).toThrow(/Use 'id'/); + expect(() => extractTargetFromSuite(suite)).toThrow(/Use top-level 'providers'/); }); it('returns undefined when no target specified', () => { @@ -954,21 +957,20 @@ describe('extractTargetFromSuite', () => { }); describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { - it('return undefined when no targets are authored', () => { + it('return undefined when no providers are authored', () => { const suite: JsonObject = { tests: [] }; expect(extractTargetsFromSuite(suite)).toBeUndefined(); expect(extractTargetRefsFromSuite(suite)).toBeUndefined(); }); - it('extracts live targets strings and promptfoo-shaped target objects', () => { + it('extracts provider strings and inline provider objects as internal target refs', () => { const suite: JsonObject = { - targets: [ + providers: [ 'registry-agent', { - id: 'inline-agent', - provider: 'mock', + id: 'mock', + label: 'inline-agent', config: { response: 'ok' }, - fallback_targets: ['registry-agent'], }, ], }; @@ -978,19 +980,154 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'registry-agent' }, { name: 'inline-agent', - id: 'inline-agent', + id: 'mock', + label: 'inline-agent', definition: expect.objectContaining({ id: 'inline-agent', name: 'inline-agent', label: 'inline-agent', provider: 'mock', response: 'ok', - fallback_targets: ['registry-agent'], }), }, ]); }); + it('preserves colon provider specs as selection identity and lowers backend config', () => { + const suite: JsonObject = { + providers: [ + 'openai:gpt-4.1-mini', + { + id: 'openai:responses:gpt-5.4', + label: 'gpt5-responses', + }, + { + id: 'anthropic:messages:claude-sonnet-4-6', + }, + { + id: 'exec:node ./provider.js', + }, + { + id: 'gateway:openai:responses:gpt-5.4', + }, + { + id: 'openai:codex-sdk:gpt-5.4-codex', + label: 'codex-sdk', + }, + { + id: 'openai:codex-app-server:gpt-5.4-codex', + label: 'codex-local', + }, + { + id: 'openai:codex-desktop', + }, + ], + }; + + expect(extractTargetsFromSuite(suite)).toEqual([ + 'openai:gpt-4.1-mini', + 'gpt5-responses', + 'anthropic:messages:claude-sonnet-4-6', + 'exec:node ./provider.js', + 'gateway:openai:responses:gpt-5.4', + 'codex-sdk', + 'codex-local', + 'openai:codex-desktop', + ]); + expect(extractTargetRefsFromSuite(suite)).toEqual([ + { name: 'openai:gpt-4.1-mini' }, + { + name: 'gpt5-responses', + id: 'openai:responses:gpt-5.4', + label: 'gpt5-responses', + definition: expect.objectContaining({ + id: 'gpt5-responses', + name: 'gpt5-responses', + label: 'gpt5-responses', + provider: 'openai', + model: 'gpt-5.4', + api_format: 'responses', + }), + }, + { + name: 'anthropic:messages:claude-sonnet-4-6', + id: 'anthropic:messages:claude-sonnet-4-6', + label: 'anthropic:messages:claude-sonnet-4-6', + definition: expect.objectContaining({ + id: 'anthropic:messages:claude-sonnet-4-6', + name: 'anthropic:messages:claude-sonnet-4-6', + provider: 'anthropic', + model: 'claude-sonnet-4-6', + }), + }, + { + name: 'exec:node ./provider.js', + id: 'exec:node ./provider.js', + label: 'exec:node ./provider.js', + definition: expect.objectContaining({ + id: 'exec:node ./provider.js', + name: 'exec:node ./provider.js', + provider: 'cli', + command: 'node ./provider.js', + }), + }, + { + name: 'gateway:openai:responses:gpt-5.4', + id: 'gateway:openai:responses:gpt-5.4', + label: 'gateway:openai:responses:gpt-5.4', + definition: expect.objectContaining({ + id: 'gateway:openai:responses:gpt-5.4', + name: 'gateway:openai:responses:gpt-5.4', + provider: 'gateway', + model: 'openai:responses:gpt-5.4', + }), + }, + { + name: 'codex-sdk', + id: 'openai:codex-sdk:gpt-5.4-codex', + label: 'codex-sdk', + definition: expect.objectContaining({ + id: 'codex-sdk', + name: 'codex-sdk', + provider: 'codex-sdk', + model: 'gpt-5.4-codex', + }), + }, + { + name: 'codex-local', + id: 'openai:codex-app-server:gpt-5.4-codex', + label: 'codex-local', + definition: expect.objectContaining({ + id: 'codex-local', + name: 'codex-local', + provider: 'codex-app-server', + model: 'gpt-5.4-codex', + }), + }, + { + name: 'openai:codex-desktop', + id: 'openai:codex-desktop', + label: 'openai:codex-desktop', + definition: expect.objectContaining({ + id: 'openai:codex-desktop', + name: 'openai:codex-desktop', + provider: 'codex-app-server', + }), + }, + ]); + }); + + it('rejects duplicate provider result identities', () => { + const suite: JsonObject = { + providers: [ + { id: 'mock', label: 'candidate' }, + { id: 'openai', label: 'candidate' }, + ], + }; + + expect(() => extractTargetRefsFromSuite(suite)).toThrow(/Duplicate provider identity/); + }); + it('reject top-level target arrays through execution', () => { const suite: JsonObject = { execution: { targets: ['copilot', 'claude'] } }; expect(() => extractTargetsFromSuite(suite)).toThrow(/execution\.targets/); diff --git a/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts b/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts index 920e8e678..eb00dc4d3 100644 --- a/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/invalid-contract.eval.ts @@ -1,5 +1,11 @@ export default { prompts: ['{{ input }}'], - providers: ['openai:gpt-5'], - tests: [{ id: 'invalid', vars: { input: 'Say hello' } }], + providers: [{ provider: 'mock' }], + tests: [ + { + id: 'invalid', + vars: { input: 'Say hello' }, + assert: [{ type: 'contains', value: 'hello' }], + }, + ], }; diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index eff9149b2..0d0918f12 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -750,7 +750,7 @@ describe('parseGraders - kebab-case type normalization', () => { metric: 'kebab-llm', type: 'llm-grader', prompt: 'test prompt', - target: 'grader-low-cost-a', + provider: 'grader-low-cost-a', }, ], }; diff --git a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts index b481ec186..076965819 100644 --- a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts +++ b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts @@ -67,7 +67,7 @@ describe('loadTsEvalFile', () => { expect(suite.tests).toHaveLength(1); expect(suite.tests[0].id).toBe('relative-import'); expect(suite.tests[0].input).toEqual([{ role: 'user', content: 'Say hello' }]); - expect(suite.targetSpec).toEqual({ name: 'mock-target' }); + expect(suite.targetRefs).toEqual([{ name: 'mock-target' }]); expect(suite.budgetUsd).toBe(1); expect(suite.experimentConfig?.repeat?.count).toBe(2); expect(suite.tags).toEqual({ experiment: 'ts-config', group: 'loader' }); @@ -83,7 +83,7 @@ describe('loadTsEvalFile', () => { it('reports invalid TypeScript eval contracts through suite loading', async () => { await expect( loadTestSuite(path.join(fixtureDir, 'invalid-contract.eval.ts'), fixtureDir), - ).rejects.toThrow("top-level 'providers'"); + ).rejects.toThrow('providers[0].id'); }); it('materializes a YAML-aligned sdk eval through loadTestSuite', async () => { @@ -100,8 +100,8 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].workspace?.hooks?.before_all?.command).toEqual(['echo', 'suite-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.command).toEqual(['echo', 'case-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.timeout_ms).toBe(1_000); - expect(suite.targetSpec).toEqual({ name: 'mock-target' }); - expect(suite.targets).toBeUndefined(); + expect(suite.targetRefs).toEqual([{ name: 'mock-target' }]); + expect(suite.targets).toEqual(['mock-target']); expect(suite.workers).toBeUndefined(); expect(suite.budgetUsd).toBe(2); expect(suite.threshold).toBe(0.75); diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index 6a8da3d4e..3d04e164e 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -15,23 +15,22 @@ describe('readTargetDefinitions', () => { } }); - async function writeTargetsYaml(content: string): Promise { + async function writeProvidersYaml(content: string): Promise { tempDir = path.join(os.tmpdir(), `agentv-targets-file-test-${Date.now()}`); await mkdir(tempDir, { recursive: true }); - const filePath = path.join(tempDir, 'targets.yaml'); + const filePath = path.join(tempDir, 'providers.yaml'); await writeFile(filePath, content); return filePath; } - it('normalizes authored id identity and config fields', async () => { - const filePath = await writeTargetsYaml(`targets: - - id: candidate-agent - provider: codex-cli + it('normalizes authored provider id and label into target identity and backend fields', async () => { + const filePath = await writeProvidersYaml(`providers: + - id: agentv:codex-cli + label: candidate-agent config: command: ["codex"] model: gpt-5-codex reasoning_effort: low - grader_target: grader `); const definitions = await readTargetDefinitions(filePath); @@ -45,27 +44,127 @@ describe('readTargetDefinitions', () => { command: ['codex'], model: 'gpt-5-codex', reasoning_effort: 'low', - grader_target: 'grader', }), ]); }); - it('rejects authored name in favor of id', async () => { - const filePath = await writeTargetsYaml(`targets: - - name: legacy-agent - provider: mock + it('accepts colon provider specs and preserves unlabeled specs as stable identity', async () => { + const filePath = await writeProvidersYaml(`providers: + - id: openai:gpt-4.1-mini + - id: openai:responses:gpt-5.4 + label: gpt5-responses + - id: anthropic:messages:claude-sonnet-4-6 + - id: exec:node ./provider.js + - id: gateway:openai:responses:gpt-5.4 + - id: openai:codex + label: codex-sdk + - id: openai:codex-sdk:gpt-5.4-codex + label: codex-sdk-model + - id: openai:codex-app-server:gpt-5.4-codex + label: codex-local + - id: openai:codex-desktop `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/missing a valid 'id'/); + const definitions = await readTargetDefinitions(filePath); + + expect(definitions).toEqual([ + expect.objectContaining({ + id: 'openai:gpt-4.1-mini', + name: 'openai:gpt-4.1-mini', + label: 'openai:gpt-4.1-mini', + provider: 'openai', + model: 'gpt-4.1-mini', + }), + expect.objectContaining({ + id: 'gpt5-responses', + name: 'gpt5-responses', + label: 'gpt5-responses', + provider: 'openai', + model: 'gpt-5.4', + api_format: 'responses', + }), + expect.objectContaining({ + id: 'anthropic:messages:claude-sonnet-4-6', + name: 'anthropic:messages:claude-sonnet-4-6', + label: 'anthropic:messages:claude-sonnet-4-6', + provider: 'anthropic', + model: 'claude-sonnet-4-6', + }), + expect.objectContaining({ + id: 'exec:node ./provider.js', + name: 'exec:node ./provider.js', + label: 'exec:node ./provider.js', + provider: 'cli', + command: 'node ./provider.js', + }), + expect.objectContaining({ + id: 'gateway:openai:responses:gpt-5.4', + name: 'gateway:openai:responses:gpt-5.4', + label: 'gateway:openai:responses:gpt-5.4', + provider: 'gateway', + model: 'openai:responses:gpt-5.4', + }), + expect.objectContaining({ + id: 'codex-sdk', + name: 'codex-sdk', + label: 'codex-sdk', + provider: 'codex-sdk', + }), + expect.objectContaining({ + id: 'codex-sdk-model', + name: 'codex-sdk-model', + label: 'codex-sdk-model', + provider: 'codex-sdk', + model: 'gpt-5.4-codex', + }), + expect.objectContaining({ + id: 'codex-local', + name: 'codex-local', + label: 'codex-local', + provider: 'codex-app-server', + model: 'gpt-5.4-codex', + }), + expect.objectContaining({ + id: 'openai:codex-desktop', + name: 'openai:codex-desktop', + label: 'openai:codex-desktop', + provider: 'codex-app-server', + }), + ]); }); - it('rejects authored label in favor of id', async () => { - const filePath = await writeTargetsYaml(`targets: + it('rejects shell-script exec provider specs in favor of cross-platform providers', async () => { + const filePath = await writeProvidersYaml(`providers: + - id: exec:./script.sh +`); + + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/cross-platform commands/); + }); + + it('rejects removed top-level targets', async () => { + const filePath = await writeProvidersYaml(`targets: - id: candidate-agent - label: legacy-agent provider: mock `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/Use 'id'/); + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/uses removed 'targets'/); + }); + + it('rejects authored name in favor of label', async () => { + const filePath = await writeProvidersYaml(`providers: + - name: legacy-agent + id: mock +`); + + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/providers\[\]\.label/); + }); + + it('rejects authored provider field in favor of id', async () => { + const filePath = await writeProvidersYaml(`providers: + - id: mock + provider: mock +`); + + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/providers\[\]\.id/); }); }); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 8a1d8a093..cc3c1c19d 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -40,6 +40,9 @@ mock.module('@earendil-works/pi-ai', () => ({ const providerModule = await import('../../../src/evaluation/providers/index.js'); const { resolveDelegatedTargetDefinition, resolveTargetDefinition, createProvider } = providerModule; +const { normalizeProviderDefinition } = await import( + '../../../src/evaluation/providers/targets.js' +); const { extractLastAssistantContent } = await import('../../../src/evaluation/providers/types.js'); describe('resolveDelegatedTargetDefinition', () => { @@ -157,6 +160,61 @@ describe('resolveTargetDefinition', () => { expect(target.config.model).toBe('gpt-5-mini'); }); + it('resolves Promptfoo-style colon provider specs without splitting stable identity', () => { + const openaiTarget = resolveTargetDefinition( + normalizeProviderDefinition({ + id: 'openai:responses:gpt-5.4', + label: 'gpt5-responses', + config: { api_key: '{{ env.OPENAI_API_KEY }}' }, + }), + { OPENAI_API_KEY: 'test-openai-key' }, + ); + expect(openaiTarget.name).toBe('gpt5-responses'); + expect(openaiTarget.kind).toBe('openai'); + expect(openaiTarget.config.model).toBe('gpt-5.4'); + expect(openaiTarget.config.apiFormat).toBe('responses'); + + const anthropicTarget = resolveTargetDefinition( + normalizeProviderDefinition({ + id: 'anthropic:messages:claude-sonnet-4-6', + config: { api_key: '{{ env.ANTHROPIC_API_KEY }}' }, + }), + { ANTHROPIC_API_KEY: 'test-anthropic-key' }, + ); + expect(anthropicTarget.name).toBe('anthropic:messages:claude-sonnet-4-6'); + expect(anthropicTarget.kind).toBe('anthropic'); + expect(anthropicTarget.config.model).toBe('claude-sonnet-4-6'); + + const execTarget = resolveTargetDefinition( + normalizeProviderDefinition({ id: 'exec:node ./provider.js' }), + ); + expect(execTarget.name).toBe('exec:node ./provider.js'); + expect(execTarget.kind).toBe('cli'); + expect(execTarget.config.command).toBe('node ./provider.js'); + + const codexSdkTarget = resolveTargetDefinition( + normalizeProviderDefinition({ + id: 'openai:codex-sdk:gpt-5.4-codex', + label: 'codex-sdk', + config: { command: ['codex'] }, + }), + ); + expect(codexSdkTarget.name).toBe('codex-sdk'); + expect(codexSdkTarget.kind).toBe('codex-sdk'); + expect(codexSdkTarget.config.model).toBe('gpt-5.4-codex'); + + const codexAppServerTarget = resolveTargetDefinition( + normalizeProviderDefinition({ + id: 'openai:codex-desktop:gpt-5.4-codex', + label: 'codex-local', + config: { command: ['codex'] }, + }), + ); + expect(codexAppServerTarget.name).toBe('codex-local'); + expect(codexAppServerTarget.kind).toBe('codex-app-server'); + expect(codexAppServerTarget.config.model).toBe('gpt-5.4-codex'); + }); + it("throws when settings don't use {{ env.* }} syntax", () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://example.openai.azure.com', diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index cd536be31..19a539281 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -73,7 +73,7 @@ describe('validateConfigFile', () => { await writeFile( filePath, [ - 'targets: file://targets.yaml', + 'providers: file://providers.yaml', 'tests: file://tests.yaml', 'defaults: file://defaults.yaml', 'execution: file://execution.yaml', @@ -81,15 +81,15 @@ describe('validateConfigFile', () => { ].join('\n'), ); await writeFile( - path.join(path.dirname(filePath), 'targets.yaml'), + path.join(path.dirname(filePath), 'providers.yaml'), [ - '- id: codex-local', - ' provider: codex-app-server', + '- id: openai:codex-app-server', + ' label: codex-local', ' runtime: host', ' config:', ' command: ["codex", "app-server"]', - '- id: openai-grader', - ' provider: openai', + '- id: openai', + ' label: openai-grader', ' runtime: host', ' config: {}', '', @@ -101,7 +101,7 @@ describe('validateConfigFile', () => { ); await writeFile( path.join(path.dirname(filePath), 'defaults.yaml'), - ['target: codex-local', 'grader: openai-grader', ''].join('\n'), + ['provider: codex-local', 'grader: openai-grader', ''].join('\n'), ); await writeFile(path.join(path.dirname(filePath), 'execution.yaml'), 'max_concurrency: 3\n'); @@ -116,9 +116,9 @@ describe('validateConfigFile', () => { await writeFile( filePath, [ - 'targets:', + 'providers:', ' - name: codex-local', - ' provider: codex', + ' id: codex', ' runtime: host', ' executable: codex', ' config: {}', @@ -149,10 +149,10 @@ describe('validateConfigFile', () => { const graphDir = path.join(tempDir, 'wrapped-composable-config'); const filePath = path.join(graphDir, '.agentv', 'config.yaml'); await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, 'targets: file://targets.yaml\n'); + await writeFile(filePath, 'providers: file://providers.yaml\n'); await writeFile( - path.join(path.dirname(filePath), 'targets.yaml'), - ['targets:', ' - id: codex-local', ' provider: codex-app-server', ''].join('\n'), + path.join(path.dirname(filePath), 'providers.yaml'), + ['providers:', ' - id: openai:codex-app-server', ' label: codex-local', ''].join('\n'), ); const result = await validateConfigFile(filePath); @@ -161,7 +161,7 @@ describe('validateConfigFile', () => { expect(result.errors).toContainEqual( expect.objectContaining({ severity: 'error', - message: expect.stringContaining("wrapped in 'targets'"), + message: expect.stringContaining("wrapped in 'providers'"), }), ); }); diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 56521f0d0..5400eec14 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -214,22 +214,22 @@ describe('EvalFileSchema input shorthand', () => { it('accepts shared composable graph fields in eval YAML', () => { const result = EvalFileSchema.safeParse({ - targets: [ + providers: [ { - id: 'codex-local', - provider: 'codex-app-server', + id: 'openai:codex-app-server', + label: 'codex-local', runtime: 'host', config: { command: ['codex', 'app-server'] }, }, { - id: 'openai-grader', - provider: 'openai', + id: 'openai', + label: 'openai-grader', runtime: 'host', config: { model: 'gpt-5-mini' }, }, ], defaults: { - target: 'codex-local', + provider: 'codex-local', grader: 'openai-grader', }, tests: [baseTest], @@ -238,12 +238,12 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); - it('rejects a top-level graders block — a grader is just a target', () => { + it('rejects a top-level graders block — a grader is just a provider', () => { const result = EvalFileSchema.safeParse({ - targets: [ + providers: [ { - id: 'codex-local', - provider: 'codex-app-server', + id: 'openai:codex-app-server', + label: 'codex-local', runtime: 'host', config: { command: ['codex', 'app-server'] }, }, @@ -289,16 +289,12 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); - it('accepts top-level target object and evaluate_options repeat controls with include selection entries', () => { + it('accepts top-level providers and evaluate_options repeat controls with include selection entries', () => { const result = EvalFileSchema.safeParse({ name: 'wrapper', description: 'Wrapper eval', experiment: 'release-gate', - target: { - extends: 'codex', - model: 'gpt-5.1', - reasoning_effort: 'high', - }, + providers: [{ id: 'agentv:codex-cli', label: 'codex' }], threshold: 0.8, timeout_seconds: 300, evaluate_options: { @@ -345,9 +341,26 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('accepts Promptfoo-style colon provider specs', () => { + const result = EvalFileSchema.safeParse({ + name: 'colon-providers', + prompts: ['{{ prompt }}'], + providers: [ + 'openai:gpt-4.1-mini', + { id: 'openai:responses:gpt-5.4', label: 'gpt5-responses' }, + { id: 'anthropic:messages:claude-sonnet-4-6' }, + { id: 'exec:node ./provider.js' }, + { id: 'gateway:openai:responses:gpt-5.4' }, + ], + tests: [baseTest], + }); + + expect(result.success).toBe(true); + }); + it('rejects invalid evaluate_options.max_concurrency', () => { const result = EvalFileSchema.safeParse({ - target: 'codex', + providers: ['openai:codex'], evaluate_options: { max_concurrency: 0, }, @@ -415,10 +428,10 @@ describe('EvalFileSchema input shorthand', () => { raw: 'Review {{ vars.diff }}', }, ], - targets: [ + providers: [ { - id: 'local-agent', - provider: 'codex-cli', + id: 'agentv:codex-cli', + label: 'local-agent', config: { command: ['codex'], model: 'gpt-5.4-mini', @@ -677,7 +690,7 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); - it('rejects top-level model because model belongs in target object', () => { + it('rejects top-level model because model belongs in provider config', () => { const result = EvalFileSchema.safeParse({ target: 'codex', model: 'gpt-5.1', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 005039d27..0d788ce52 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -95,14 +95,15 @@ tests: expect(result.errors).toHaveLength(0); }); - it('validates top-level target and run controls with field-local tests', async () => { + it('validates top-level providers and run controls with field-local tests', async () => { const filePath = path.join(tempDir, 'run-controls-field-local-tests.yaml'); await writeFile( filePath, `name: direct-suite prompts: - "{{ prompt }}" -target: codex +providers: + - codex threshold: 0.8 evaluate_options: budget_usd: 2 @@ -434,9 +435,9 @@ tags: suite: smoke prompts: - raw: "Review {{ vars.diff }}" -targets: - - id: local-agent - provider: codex-cli +providers: + - id: agentv:codex-cli + label: local-agent command: ["codex"] default_test: vars: @@ -814,7 +815,7 @@ tests: ); }); - it('rejects top-level providers as a live alias for targets', async () => { + it('accepts top-level providers as the public candidate surface', async () => { const filePath = path.join(tempDir, 'top-level-providers.yaml'); await writeFile( filePath, @@ -833,15 +834,7 @@ tests: const result = await validateEvalFile(filePath); - expect(result.valid).toBe(false); - expect( - result.errors.some( - (error) => - error.severity === 'error' && - error.location === 'providers' && - error.message.includes("Top-level 'providers' is not a runtime alias"), - ), - ).toBe(true); + expect(result.valid).toBe(true); }); it('rejects removed eval_cases and evalcases aliases for tests', async () => { @@ -1244,7 +1237,7 @@ tests: ).toBe(true); }); - it('warns that imported child target and run controls are ignored by wrapper composition', async () => { + it('warns that imported child providers and run controls are ignored by wrapper composition', async () => { await writeFile( path.join(tempDir, 'composition-child-experiment.eval.yaml'), `target: child-target @@ -1272,7 +1265,7 @@ tests: (error) => error.severity === 'warning' && error.location === 'tests[0].include' && - error.message.includes('child target and run controls are ignored') && + error.message.includes('child providers and run controls are ignored') && error.message.includes('legacy tests[].include suite imports') && error.message.includes('CLI multi-file selection') && error.message.includes('tags'), diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index 55767ef52..a7575674b 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -21,9 +21,9 @@ describe('validateTargetsFile', () => { const filePath = path.join(tempDir, 'openrouter-target.yaml'); await writeFile( filePath, - `targets: - - id: openrouter-target - provider: openrouter + `providers: + - id: openrouter + label: openrouter-target api_key: "{{ env.OPENROUTER_API_KEY }}" model: openai/gpt-5-mini `, @@ -34,19 +34,57 @@ describe('validateTargetsFile', () => { expect( result.errors.some( (error) => - error.location === 'targets[0].provider' && + error.location === 'providers[0].id' && error.message.includes("Unknown provider 'openrouter'"), ), ).toBe(false); }); - it('accepts id identity and config fields', async () => { + it('accepts Promptfoo-style colon provider specs', async () => { + const filePath = path.join(tempDir, 'colon-provider-specs.yaml'); + await writeFile( + filePath, + `providers: + - id: openai:gpt-4.1-mini + api_key: "{{ env.OPENAI_API_KEY }}" + - id: openai:responses:gpt-5.4 + label: gpt5-responses + api_key: "{{ env.OPENAI_API_KEY }}" + - id: anthropic:messages:claude-sonnet-4-6 + api_key: "{{ env.ANTHROPIC_API_KEY }}" + - id: exec:node ./provider.js + - id: gateway:openai:responses:gpt-5.4 + - id: openai:codex + - id: openai:codex-sdk:gpt-5.4-codex + label: codex-sdk + - id: openai:codex-app-server:gpt-5.4-codex + label: codex-local + - id: openai:codex-desktop +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(true); + expect( + result.errors.some( + (error) => + error.severity === 'warning' && + (error.message.includes("Unknown provider 'openai:gpt") || + error.message.includes("Unknown provider 'anthropic:messages") || + error.message.includes("Unknown provider 'exec:node") || + error.message.includes("Unknown provider 'openai:codex")), + ), + ).toBe(false); + }); + + it('accepts label identity and config fields', async () => { const filePath = path.join(tempDir, 'promptfoo-shaped-target.yaml'); await writeFile( filePath, - `targets: - - id: candidate-agent - provider: codex-cli + `providers: + - id: agentv:codex-cli + label: candidate-agent config: command: ["codex"] model: "{{ env.CODEX_MODEL }}" @@ -57,13 +95,13 @@ describe('validateTargetsFile', () => { grader_target: grader fallback_targets: [backup-agent] batch_requests: true - - id: grader - provider: openai + - id: openai + label: grader config: api_key: "{{ env.OPENAI_API_KEY }}" model: gpt-5-mini - - id: backup-agent - provider: mock + - id: mock + label: backup-agent config: response: backup `, @@ -79,9 +117,9 @@ describe('validateTargetsFile', () => { const filePath = path.join(tempDir, 'removed-provider-batching.yaml'); await writeFile( filePath, - `targets: - - label: batch-cli - provider: mock + `providers: + - id: mock + label: batch-cli provider_batching: true `, ); @@ -93,19 +131,19 @@ describe('validateTargetsFile', () => { result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].provider_batching' && + error.location === 'providers[0].provider_batching' && error.message.includes("Use 'batch_requests' instead"), ), ).toBe(true); }); - it('rejects authored target name in favor of id', async () => { + it('rejects authored provider name in favor of id', async () => { const filePath = path.join(tempDir, 'legacy-name-target.yaml'); await writeFile( filePath, - `targets: + `providers: - name: legacy-agent - provider: mock + id: mock `, ); @@ -116,7 +154,7 @@ describe('validateTargetsFile', () => { result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].id' && + error.location === 'providers[0].id' && error.message.includes("Missing or invalid 'id' field"), ), ).toBe(true); @@ -124,21 +162,21 @@ describe('validateTargetsFile', () => { result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].name' && - error.message.includes("Use 'id'"), + error.location === 'providers[0].name' && + error.message.includes("Use 'label'"), ), ).toBe(true); }); - it('rejects top-level providers as a targets.yaml runtime alias', async () => { + it('rejects removed top-level targets in provider catalogs', async () => { const filePath = path.join(tempDir, 'top-level-providers.yaml'); await writeFile( filePath, `providers: - - label: candidate-agent - provider: mock + - id: mock + label: candidate-agent targets: - - label: candidate-agent + - id: candidate-agent provider: mock `, ); @@ -150,8 +188,8 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'providers' && - error.message.includes("Top-level 'providers' is not a runtime alias"), + error.location === 'targets' && + error.message.includes("Top-level 'targets' has been removed"), ), ).toBe(true); }); @@ -160,27 +198,27 @@ targets: const filePath = path.join(tempDir, 'removed-provider-aliases.yaml'); await writeFile( filePath, - `targets: - - label: azure-alias - provider: azure-openai - - label: google-alias - provider: google - - label: google-gemini-alias - provider: google-gemini - - label: copilot-alias - provider: copilot - - label: claude-alias - provider: claude - - label: copilot-sdk-alias - provider: copilot_sdk - - label: pi-alias - provider: pi - - label: claude-code-alias - provider: claude-code - - label: bedrock-future - provider: bedrock - - label: vertex-future - provider: vertex + `providers: + - id: azure-openai + label: azure-alias + - id: google + label: google-alias + - id: google-gemini + label: google-gemini-alias + - id: copilot + label: copilot-alias + - id: claude + label: claude-alias + - id: copilot_sdk + label: copilot-sdk-alias + - id: pi + label: pi-alias + - id: claude-code + label: claude-code-alias + - id: bedrock + label: bedrock-future + - id: vertex + label: vertex-future `, ); @@ -200,7 +238,7 @@ targets: result.errors.some( (error) => error.severity === 'warning' && - error.location.endsWith('.provider') && + error.location.endsWith('.id') && error.message.includes(`Unknown provider '${provider}'`), ), ).toBe(true); @@ -210,7 +248,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[3].provider' && + error.location === 'providers[3].id' && error.message.includes("Ambiguous provider 'copilot'") && error.message.includes('copilot-cli') && error.message.includes('copilot-sdk'), @@ -220,7 +258,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[4].provider' && + error.location === 'providers[4].id' && error.message.includes("Ambiguous provider 'claude'") && error.message.includes('claude-cli') && error.message.includes('claude-sdk'), @@ -232,16 +270,16 @@ targets: const filePath = path.join(tempDir, 'camel-case-aliases.yaml'); await writeFile( filePath, - `targets: - - id: codex-target - provider: codex-cli + `providers: + - id: agentv:codex-cli + label: codex-target command: ["codex"] timeoutSeconds: 30 logDir: ./logs systemPrompt: Be precise. modelReasoningEffort: low - - label: cli-target - provider: cli + - id: cli + label: cli-target command: echo {PROMPT} healthcheck: command: echo ok @@ -256,7 +294,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].timeoutSeconds' && + error.location === 'providers[0].timeoutSeconds' && error.message.includes("Use 'timeout_seconds' instead"), ), ).toBe(true); @@ -264,7 +302,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].logDir' && + error.location === 'providers[0].logDir' && error.message.includes("Use 'log_dir' instead"), ), ).toBe(true); @@ -272,7 +310,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].systemPrompt' && + error.location === 'providers[0].systemPrompt' && error.message.includes("Use 'system_prompt' instead"), ), ).toBe(true); @@ -280,7 +318,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].modelReasoningEffort' && + error.location === 'providers[0].modelReasoningEffort' && error.message.includes("Use 'reasoning_effort' instead"), ), ).toBe(true); @@ -288,7 +326,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[1].healthcheck.timeoutSeconds' && + error.location === 'providers[1].healthcheck.timeoutSeconds' && error.message.includes("Use 'timeout_seconds' instead"), ), ).toBe(true); @@ -298,9 +336,9 @@ targets: const filePath = path.join(tempDir, 'codex-reasoning-effort.yaml'); await writeFile( filePath, - `targets: - - id: codex-target - provider: codex-cli + `providers: + - id: agentv:codex-cli + label: codex-target command: ["codex"] model: "{{ env.CODEX_MODEL }}" reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" @@ -316,9 +354,9 @@ targets: const filePath = path.join(tempDir, 'copilot-flat-provider.yaml'); await writeFile( filePath, - `targets: - - id: copilot-sdk-custom-provider - provider: copilot-sdk + `providers: + - id: copilot-sdk + label: copilot-sdk-custom-provider model: gpt-5 subprovider: openai base_url: "{{ env.OPENAI_ENDPOINT }}" @@ -326,8 +364,8 @@ targets: api_format: responses model_id: gpt-5 wire_model: "{{ env.OPENAI_MODEL }}" - - id: copilot-cli-custom-provider - provider: copilot-cli + - id: copilot-cli + label: copilot-cli-custom-provider subprovider: openai base_url: "{{ env.OPENAI_ENDPOINT }}" api_key: "{{ env.OPENAI_API_KEY }}" @@ -345,9 +383,9 @@ targets: const filePath = path.join(tempDir, 'codex-openai-provider.yaml'); await writeFile( filePath, - `targets: - - id: codex-local-openai - provider: codex-cli + `providers: + - id: agentv:codex-cli + label: codex-local-openai command: ["codex"] model: "{{ env.CODEX_MODEL }}" reasoning_effort: medium @@ -370,21 +408,21 @@ targets: const filePath = path.join(tempDir, 'copilot-removed-provider-fields.yaml'); await writeFile( filePath, - `targets: - - label: copilot-sdk-custom - provider: copilot-sdk + `providers: + - id: copilot-sdk + label: copilot-sdk-custom custom_provider: type: openai base_url: "{{ env.OPENAI_ENDPOINT }}" api_key: "{{ env.OPENAI_API_KEY }}" - - label: copilot-sdk-byok - provider: copilot-sdk + - id: copilot-sdk + label: copilot-sdk-byok byok: type: openai base_url: "{{ env.OPENAI_ENDPOINT }}" api_key: "{{ env.OPENAI_API_KEY }}" - - label: copilot-cli-custom - provider: copilot-cli + - id: copilot-cli + label: copilot-cli-custom custom_provider: type: openai base_url: "{{ env.OPENAI_ENDPOINT }}" @@ -408,18 +446,19 @@ targets: ).toBe(true); }); - it('rejects use_target on authored target definitions', async () => { + it('rejects use_target on authored provider definitions', async () => { const filePath = path.join(tempDir, 'templated-use-target.yaml'); await writeFile( filePath, - `targets: - - id: default - provider: mock + `providers: + - id: mock + label: default use_target: "{{ env.AGENT_TARGET }}" - - id: grader + - id: mock + label: grader use_target: "{{ env.GRADER_TARGET }}" - - id: codex-agent - provider: codex-cli + - id: agentv:codex-cli + label: codex-agent command: ["codex"] grader_target: grader `, @@ -431,26 +470,26 @@ targets: expect(result.errors).toContainEqual( expect.objectContaining({ severity: 'error', - location: 'targets[0].use_target', + location: 'providers[0].use_target', message: expect.stringContaining("'use_target' field has been removed"), }), ); expect(result.errors).toContainEqual( expect.objectContaining({ severity: 'error', - location: 'targets[1].use_target', + location: 'providers[1].use_target', message: expect.stringContaining("'use_target' field has been removed"), }), ); }); - it('rejects legacy env interpolation in target YAML', async () => { + it('rejects legacy env interpolation in provider YAML', async () => { const filePath = path.join(tempDir, 'legacy-env-target.yaml'); await writeFile( filePath, - `targets: - - id: openai-target - provider: openai + `providers: + - id: openai + label: openai-target api_key: \${{ OPENAI_API_KEY }} model: gpt-5-mini `, @@ -462,7 +501,7 @@ targets: expect(result.errors).toContainEqual( expect.objectContaining({ severity: 'error', - location: 'targets[0].api_key', + location: 'providers[0].api_key', message: expect.stringContaining('Use {{ env.OPENAI_API_KEY }} instead'), }), ); @@ -472,14 +511,14 @@ targets: const filePath = path.join(tempDir, 'judge-target-alias.yaml'); await writeFile( filePath, - `targets: - - label: codex-agent - provider: codex-cli + `providers: + - id: agentv:codex-cli + label: codex-agent command: ["codex"] model: gpt-5 judge_target: grader - - label: grader - provider: openai + - id: openai + label: grader model: gpt-5-mini `, ); @@ -491,7 +530,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].judge_target' && + error.location === 'providers[0].judge_target' && error.message.includes("'judge_target' field has been removed"), ), ).toBe(true); @@ -501,12 +540,12 @@ targets: const filePath = path.join(tempDir, 'log-format-aliases.yaml'); await writeFile( filePath, - `targets: - - label: copilot-agent - provider: copilot-cli + `providers: + - id: copilot-cli + label: copilot-agent log_format: json - - label: claude-agent - provider: claude-cli + - id: claude-cli + label: claude-agent log_output_format: summary `, ); @@ -518,7 +557,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].log_format' && + error.location === 'providers[0].log_format' && error.message.includes("Use 'stream_log: raw'"), ), ).toBe(true); @@ -526,7 +565,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[1].log_output_format' && + error.location === 'providers[1].log_output_format' && error.message.includes("Use 'stream_log: raw'"), ), ).toBe(true); @@ -536,9 +575,9 @@ targets: const filePath = path.join(tempDir, 'azure-api-format.yaml'); await writeFile( filePath, - `targets: - - label: azure-responses - provider: azure + `providers: + - id: azure + label: azure-responses endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" api_key: "{{ env.AZURE_OPENAI_API_KEY }}" model: "{{ env.AZURE_DEPLOYMENT_NAME }}" @@ -552,7 +591,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].api_format' && + error.location === 'providers[0].api_format' && /'api_format' field has been removed/i.test(error.message), ), ).toBe(true); @@ -562,9 +601,9 @@ targets: const filePath = path.join(tempDir, 'replay-execution-traces.yaml'); await writeFile( filePath, - `targets: - - id: replay-execution-trace - provider: replay + `providers: + - id: replay + label: replay-execution-trace execution_traces: ./fixtures/execution-traces.jsonl source_target: live-agent `, @@ -582,9 +621,9 @@ targets: const filePath = path.join(tempDir, 'replay-transcripts.yaml'); await writeFile( filePath, - `targets: - - id: replay-transcript - provider: replay + `providers: + - id: replay + label: replay-transcript transcripts: ./fixtures/transcript.jsonl source_target: live-agent `, @@ -602,9 +641,9 @@ targets: const filePath = path.join(tempDir, 'replay-ambiguous-source.yaml'); await writeFile( filePath, - `targets: - - label: replay-ambiguous - provider: replay + `providers: + - id: replay + label: replay-ambiguous fixtures: ./fixtures/target-output.jsonl execution_traces: ./fixtures/execution-traces.jsonl source_target: live-agent @@ -618,7 +657,7 @@ targets: result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0]' && + error.location === 'providers[0]' && /exactly one replay source/i.test(error.message), ), ).toBe(true); diff --git a/packages/core/test/evaluation/yaml-parser-metadata.test.ts b/packages/core/test/evaluation/yaml-parser-metadata.test.ts index 22124e6c8..8cde13fd1 100644 --- a/packages/core/test/evaluation/yaml-parser-metadata.test.ts +++ b/packages/core/test/evaluation/yaml-parser-metadata.test.ts @@ -124,11 +124,12 @@ tests: expect(suite.metadata?.name).toBe('my-eval'); }); - it('works with metadata alongside target config', async () => { + it('works with metadata alongside provider config', async () => { const { filePath, dir } = createTempYaml(` name: matrix-eval -description: Eval with targets -target: copilot +description: Eval with providers +providers: + - copilot prompts: - "{{ prompt }}" tests: @@ -141,7 +142,7 @@ tests: const suite = await loadTestSuite(filePath, dir); expect(suite.metadata).toBeDefined(); expect(suite.metadata?.name).toBe('matrix-eval'); - expect(suite.targetSpec).toEqual({ name: 'copilot' }); + expect(suite.targetRefs).toEqual([{ name: 'copilot' }]); }); it('parses metadata with tags as array', async () => { diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index e99f814ac..967d10ec1 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2027,6 +2027,9 @@ "type": "object", "properties": { "target": { + "not": {} + }, + "provider": { "type": "string", "minLength": 1 }, @@ -2050,347 +2053,12 @@ "not": {} }, "target": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] + "not": {} }, "targets": { + "not": {} + }, + "providers": { "anyOf": [ { "anyOf": [ @@ -2405,15 +2073,7 @@ "type": "string", "minLength": 1 }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { + "label": { "type": "string", "minLength": 1 }, @@ -2421,6 +2081,25 @@ "type": "object", "additionalProperties": {} }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, "prompts": { "anyOf": [ { @@ -2581,19 +2260,6 @@ "type": "string" } }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, "hooks": { "type": "object", "properties": { @@ -2727,9 +2393,25 @@ } }, "additionalProperties": false + }, + "provider": { + "not": {} + }, + "name": { + "not": {} + }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} } }, - "additionalProperties": true + "required": ["id"], + "additionalProperties": false } ] }, @@ -2748,15 +2430,7 @@ "type": "string", "minLength": 1 }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { + "label": { "type": "string", "minLength": 1 }, @@ -2764,6 +2438,25 @@ "type": "object", "additionalProperties": {} }, + "runtime": { + "anyOf": [ + { + "type": "string", + "enum": ["host", "profile", "sandbox"] + }, + { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["host", "profile", "sandbox"] + } + }, + "required": ["mode"], + "additionalProperties": true + } + ] + }, "prompts": { "anyOf": [ { @@ -2924,19 +2617,6 @@ "type": "string" } }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, "hooks": { "type": "object", "properties": { @@ -3070,9 +2750,25 @@ } }, "additionalProperties": false + }, + "provider": { + "not": {} + }, + "name": { + "not": {} + }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} } }, - "additionalProperties": true + "required": ["id"], + "additionalProperties": false } ] }, @@ -3080,9 +2776,6 @@ } ] }, - "providers": { - "not": {} - }, "model": { "not": {} },