diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 385421745..6d1176e6d 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -11,36 +11,36 @@ targets: # redirects to a named target, controlled via AGENT_TARGET env var. # One env var switches the entire provider config (auth, model, etc.). # Example: AGENT_TARGET=copilot-cli or AGENT_TARGET=claude - - label: default + - id: default use_target: ${{ AGENT_TARGET }} - - label: agent + - id: agent use_target: ${{ AGENT_TARGET }} # ── LLM target (text generation, no agent binary needed) ──────────── # Delegates to LLM_TARGET — same provider used for grading and LLM evals. - - label: llm + - id: llm use_target: ${{ LLM_TARGET }} # ── Grader (LLM-as-judge) ────────────────────────────────────────── # Used by agent targets via grader_target. Switch provider via GRADER_TARGET. - - label: grader + - id: grader use_target: ${{ GRADER_TARGET }} # ── Named agent targets ─────────────────────────────────────────── - - label: copilot + - id: copilot provider: copilot-cli model: ${{ COPILOT_MODEL }} grader_target: grader stream_log: raw - - label: copilot-sdk + - id: copilot-sdk provider: copilot-sdk model: ${{ COPILOT_MODEL }} grader_target: grader stream_log: raw - - label: copilot-sdk-azure + - id: copilot-sdk-azure provider: copilot-sdk model: ${{ AZURE_DEPLOYMENT_NAME }} subprovider: azure @@ -49,22 +49,22 @@ targets: grader_target: grader stream_log: raw - - label: claude + - id: claude provider: claude-cli grader_target: grader - - label: claude-sdk + - id: claude-sdk provider: claude-sdk grader_target: grader - - label: pi + - id: pi provider: pi-cli subprovider: openrouter model: ${{ OPENROUTER_MODEL }} api_key: ${{ OPENROUTER_API_KEY }} grader_target: grader - - label: pi-sdk + - id: pi-sdk provider: pi-coding-agent subprovider: openai base_url: ${{ OPENAI_ENDPOINT }} @@ -75,7 +75,7 @@ targets: stream_log: raw - - label: pi-azure + - id: pi-azure provider: pi-cli subprovider: azure base_url: ${{ AZURE_OPENAI_ENDPOINT }} @@ -83,7 +83,7 @@ targets: api_key: ${{ AZURE_OPENAI_API_KEY }} grader_target: grader - - label: pi-sdk-azure + - id: pi-sdk-azure provider: pi-coding-agent subprovider: azure base_url: ${{ AZURE_OPENAI_ENDPOINT }} @@ -93,7 +93,7 @@ targets: thinking: low stream_log: raw - - label: codex + - id: codex provider: codex executable: ${{ CODEX_EXECUTABLE }} model: ${{ CODEX_MODEL }} @@ -104,7 +104,7 @@ targets: stream_log: raw # ── LLM targets (direct model access) ───────────────────────────── - - label: gh-models + - id: gh-models provider: openai base_url: https://models.github.ai/inference api_key: ${{ GH_MODELS_TOKEN }} @@ -115,19 +115,19 @@ targets: # overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure # deployments must use `provider: openai` with a deployment-scoped # `base_url` instead. - - label: azure + - id: azure provider: azure endpoint: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} model: ${{ AZURE_DEPLOYMENT_NAME }} version: ${{ AZURE_OPENAI_API_VERSION }} - - label: gemini + - id: gemini provider: gemini api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} model: ${{ GEMINI_MODEL_NAME }} - - label: openai + - id: openai provider: openai endpoint: ${{ OPENAI_ENDPOINT }} api_key: ${{ OPENAI_API_KEY }} @@ -135,20 +135,20 @@ targets: # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy # without changing provider-specific target labels. - - label: local-openai + - id: local-openai provider: openai base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} model: ${{ LOCAL_OPENAI_PROXY_MODEL }} grader_target: local-openai-grader - - label: local-openai-grader + - id: local-openai-grader provider: openai base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} model: ${{ LOCAL_OPENAI_PROXY_MODEL }} - - label: pi-cli-openai + - id: pi-cli-openai provider: pi-cli subprovider: openai base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} @@ -158,7 +158,7 @@ targets: thinking: low stream_log: raw - - label: codex-sdk-openai + - id: codex-sdk-openai provider: codex base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} @@ -168,7 +168,7 @@ targets: grader_target: local-openai-grader stream_log: raw - - label: copilot-sdk-openai + - id: copilot-sdk-openai provider: copilot-sdk subprovider: openai base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} @@ -177,25 +177,25 @@ targets: grader_target: local-openai-grader stream_log: raw - - label: openrouter + - id: openrouter provider: openrouter api_key: ${{ OPENROUTER_API_KEY }} model: ${{ OPENROUTER_MODEL }} # ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── - - label: mimo + - id: mimo provider: openrouter api_key: ${{ OPENROUTER_API_KEY }} model: xiaomi/mimo-v2.5-pro grader_target: grader - - label: mimo-flash + - id: mimo-flash provider: openrouter api_key: ${{ OPENROUTER_API_KEY }} model: xiaomi/mimo-v2-flash grader_target: grader - - label: mimo-direct + - id: mimo-direct provider: openai base_url: https://token-plan-sgp.xiaomimimo.com/v1 api_key: ${{ XIAOMI_MIMO_API_KEY }} diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 28f4187b3..28388308e 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -625,14 +625,20 @@ function uniqueTargetDefinitions( } function serializeTargetDefinition(definition: TargetDefinition): Record { - const target: Record = { label: definition.name }; + const target: Record = { id: definition.name }; if (definition.id !== undefined) { target.id = definition.id; } const config: Record = {}; for (const [key, value] of Object.entries(definition)) { - if (value === undefined || key === 'name' || key === 'id' || key === 'config') { + if ( + value === undefined || + key === 'name' || + key === 'id' || + key === 'label' || + key === 'config' + ) { continue; } if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) { diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index bcf4b3b3f..6fdda50e8 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -122,6 +122,9 @@ async function readTargetDefinitions( } function targetName(definition: Record): string | undefined { + if (typeof definition.id === 'string' && definition.id.trim().length > 0) { + return definition.id.trim(); + } if (typeof definition.label === 'string' && definition.label.trim().length > 0) { return definition.label.trim(); } diff --git a/apps/cli/src/templates/.agentv/targets.yaml b/apps/cli/src/templates/.agentv/targets.yaml index 04743c72f..0d70548ce 100644 --- a/apps/cli/src/templates/.agentv/targets.yaml +++ b/apps/cli/src/templates/.agentv/targets.yaml @@ -4,14 +4,14 @@ # Agent and CLI targets use grader_target to reference an LLM target for scoring. targets: - - label: default + - id: default provider: azure endpoint: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} model: ${{ AZURE_DEPLOYMENT_NAME }} # version: ${{ AZURE_OPENAI_API_VERSION }} # Optional: uncomment to override default (v1) - - label: codex + - id: codex provider: codex grader_target: azure-llm # Uses the Codex CLI (defaults to `codex` on PATH) @@ -28,7 +28,7 @@ targets: stream_log: raw # Optional: 'summary' for consolidated logs or 'raw' for per-event logs # Claude - Anthropic's Claude Agent SDK - - label: claude + - id: claude provider: claude grader_target: azure-llm # Uses the @anthropic-ai/claude-agent-sdk @@ -40,19 +40,19 @@ targets: stream_log: raw # Optional: 'summary' for consolidated logs or 'raw' for per-event logs # system_prompt: optional override (default instructs agent to include code in response) - - label: azure-llm + - id: azure-llm provider: azure endpoint: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} model: ${{ AZURE_DEPLOYMENT_NAME }} version: ${{ AZURE_OPENAI_API_VERSION }} - - label: gemini-llm + - id: gemini-llm provider: gemini api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} model: ${{ GEMINI_MODEL_NAME }} - - label: local_cli + - id: local_cli provider: cli grader_target: azure-llm # Passes the fully rendered prompt and any attached files to a local Python script @@ -75,12 +75,12 @@ targets: # mimo-v2.5 — 1M context, ~131K output, multimodal # mimo-v2-flash — 262K context, 65K output, fast MoE (open-source) # mimo-v2-omni — 262K context, 65K output, omni-modal - - label: mimo + - id: mimo provider: openrouter api_key: ${{ OPENROUTER_API_KEY }} model: xiaomi/mimo-v2.5-pro - - label: mimo-flash + - id: mimo-flash provider: openrouter api_key: ${{ OPENROUTER_API_KEY }} model: xiaomi/mimo-v2-flash diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 9b0291178..7dd2da850 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -54,11 +54,11 @@ describe('agentv eval bundle', () => { await writeFile( path.join(sourceDir, '.agentv', 'targets.yaml'), `targets: - - label: inherited + - id: inherited provider: mock response: '{"answer":"Mock provider response from inherited target"}' fallback_targets: [backup] - - label: backup + - id: backup provider: mock response: '{"answer":"Backup mock response"}' `, @@ -153,8 +153,8 @@ tests: ../data/cases.yaml expect(input[0]?.content[0]).toEqual({ type: 'file', value: 'files/data/input.txt' }); const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('label: inherited'); - expect(bundledTargets).toContain('label: backup'); + expect(bundledTargets).toContain('id: inherited'); + expect(bundledTargets).toContain('id: backup'); await rm(sourceDir, { recursive: true, force: true }); const run = await runCli(bundleDir, [ @@ -178,7 +178,7 @@ tests: ../data/cases.yaml await writeFile( path.join(sourceDir, 'evals', 'inline.eval.yaml'), `targets: - - label: candidate + - id: candidate provider: mock response: '{"answer":"inline bundled response"}' tests: @@ -201,7 +201,7 @@ tests: expect(bundle.exitCode).toBe(0); const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('label: candidate'); + expect(bundledTargets).toContain('id: candidate'); expect(bundledTargets).toContain('provider: mock'); expect(bundledTargets).toContain('inline bundled response'); }, 30_000); @@ -214,7 +214,7 @@ tests: await writeFile( path.join(sourceDir, '.agentv', 'targets.yaml'), `targets: - - label: default + - id: default provider: mock `, 'utf8', diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index f8b1aaa09..06c6239f2 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -18,7 +18,7 @@ describe('eval target selection', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('resolves authored target ids through targets.yaml while keeping labels for display', async () => { + it('resolves authored target ids through targets.yaml', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); await writeFile( @@ -26,7 +26,7 @@ describe('eval target selection', () => { [ '$schema: agentv-targets-v2.2', 'targets:', - ' - label: openai:gpt-5.4-mini', + ' - id: openai:gpt-5.4-mini', ' provider: mock', '', ].join('\n'), @@ -38,7 +38,6 @@ describe('eval target selection', () => { 'name: target-label-suite', 'targets:', ' - id: openai:gpt-5.4-mini', - ' label: mini', 'tests:', ' - id: target-case', ' input: hello', @@ -60,7 +59,7 @@ describe('eval target selection', () => { expect(selections).toHaveLength(1); expect(selections[0]?.targetName).toBe('openai:gpt-5.4-mini'); - expect(selections[0]?.targetLabel).toBe('mini'); + expect(selections[0]?.targetLabel).toBeUndefined(); expect(selections[0]?.resolvedTarget.kind).toBe('mock'); }); }); diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index ad74988ad..df5be4147 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -83,7 +83,7 @@ console.log(JSON.stringify({ path.join(root, '.agentv', 'targets.yaml'), ` targets: - - label: codex + - id: codex provider: cli command: bun ./scripts/target.ts `, diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 17bc4206e..3f08d4b7d 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -60,7 +60,7 @@ await Bun.write(\`\${payload.workspace_path}/\${step}.txt\`, \`\${payload.test_i path.join(root, '.agentv', 'targets.yaml'), ` targets: - - label: codex + - id: codex provider: cli command: bun ./scripts/target.ts `, @@ -255,7 +255,7 @@ describe('agentv prepare', () => { path.join(tempDir, '.agentv', 'targets.yaml'), ` targets: - - label: codex + - id: codex provider: cli command: bun ./scripts/target.ts `, diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index f91963d53..dc54b0d26 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -30,7 +30,7 @@ interface CliResult { } const DEFAULT_TARGETS = `targets: - - label: captured + - id: captured provider: mock `; @@ -121,7 +121,7 @@ async function createBundleFixture( await writeFile( overrideTargetsPath, `targets: - - label: local + - id: local provider: mock `, 'utf8', @@ -275,7 +275,7 @@ describe('agentv runs rerun', () => { it('fails clearly for missing env and accepts an explicit env file', async () => { const created = await fixture(`targets: - - label: captured + - id: captured provider: cli command: \${{ LOCAL_AGENT_COMMAND }} `); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 862147855..04ca8b9be 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -41,13 +41,13 @@ async function createFixture(): Promise { const targetsPath = path.join(agentvDir, 'targets.yaml'); const targetsContent = `$schema: agentv-targets-v2.2 targets: - - label: default + - id: default provider: mock - - label: file-target + - id: file-target provider: mock - - label: cli-target + - id: cli-target provider: mock - - label: codex-target + - id: codex-target provider: codex-sdk model: gpt-5-default `; @@ -99,7 +99,7 @@ async function createNestedEnvFixture(): Promise { const targetsPath = path.join(agentvDir, 'targets.yaml'); const targetsContent = `$schema: agentv-targets-v2.2 targets: - - label: default + - id: default provider: mock `; await writeFile(targetsPath, targetsContent, 'utf8'); diff --git a/examples/contract/.agentv/targets.yaml b/examples/contract/.agentv/targets.yaml index e39eb9d9d..f97d07542 100644 --- a/examples/contract/.agentv/targets.yaml +++ b/examples/contract/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: github-models-contract + - id: github-models-contract provider: openai api_format: chat base_url: https://models.github.ai/inference diff --git a/examples/features/batch-cli/.agentv/targets.yaml b/examples/features/batch-cli/.agentv/targets.yaml index 6c605f452..e6ee7aa53 100644 --- a/examples/features/batch-cli/.agentv/targets.yaml +++ b/examples/features/batch-cli/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: batch_cli + - id: batch_cli provider: cli batch_requests: true verbose: true diff --git a/examples/features/copilot-log-eval/.agentv/targets.yaml b/examples/features/copilot-log-eval/.agentv/targets.yaml index 3b0971ccf..1b42e88db 100644 --- a/examples/features/copilot-log-eval/.agentv/targets.yaml +++ b/examples/features/copilot-log-eval/.agentv/targets.yaml @@ -9,6 +9,6 @@ targets: # discover: latest — auto-discover most recent session # session_id: "" — read a specific session by ID # session_dir: "/path/to/dir" — read from an explicit directory - - label: copilot-log + - id: copilot-log provider: copilot-log discover: latest diff --git a/examples/features/docker-workspace/.agentv/targets.yaml b/examples/features/docker-workspace/.agentv/targets.yaml index 1d3a7d14e..7a41cae75 100644 --- a/examples/features/docker-workspace/.agentv/targets.yaml +++ b/examples/features/docker-workspace/.agentv/targets.yaml @@ -1,3 +1,3 @@ targets: - - label: mock_agent + - id: mock_agent provider: mock diff --git a/examples/features/document-extraction/.agentv/targets.yaml b/examples/features/document-extraction/.agentv/targets.yaml index 5e0ffdaea..a2b1aad9c 100644 --- a/examples/features/document-extraction/.agentv/targets.yaml +++ b/examples/features/document-extraction/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: mock_extractor + - id: mock_extractor provider: cli provider_batching: false verbose: true diff --git a/examples/features/execution-metrics/.agentv/targets.yaml b/examples/features/execution-metrics/.agentv/targets.yaml index c02416086..e9ffc6bc1 100644 --- a/examples/features/execution-metrics/.agentv/targets.yaml +++ b/examples/features/execution-metrics/.agentv/targets.yaml @@ -1,12 +1,12 @@ targets: - - label: azure-llm + - id: azure-llm provider: azure endpoint: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} model: ${{ AZURE_DEPLOYMENT_NAME }} version: ${{ AZURE_OPENAI_API_VERSION }} - - label: mock_metrics_agent + - id: mock_metrics_agent provider: cli grader_target: azure-llm command: bun run ./mock-metrics-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} diff --git a/examples/features/file-changes-graders/.agentv/targets.yaml b/examples/features/file-changes-graders/.agentv/targets.yaml index a39fa1a73..38f7a970a 100644 --- a/examples/features/file-changes-graders/.agentv/targets.yaml +++ b/examples/features/file-changes-graders/.agentv/targets.yaml @@ -1,7 +1,7 @@ targets: # Mock agent that adds a subtract function to calculator.ts. # Each test gets a fresh copy of workspace-template/ as its CWD. - - label: mock_agent + - id: mock_agent provider: cli command: >- bash -c ' @@ -11,6 +11,6 @@ targets: grader_target: grader # Copilot CLI — used as delegated llm-grader target - - label: copilot_grader + - id: copilot_grader provider: copilot-cli model: claude-haiku-4.5 diff --git a/examples/features/file-changes-with-repos/.agentv/targets.yaml b/examples/features/file-changes-with-repos/.agentv/targets.yaml index 7c9909889..55874dcb4 100644 --- a/examples/features/file-changes-with-repos/.agentv/targets.yaml +++ b/examples/features/file-changes-with-repos/.agentv/targets.yaml @@ -1,7 +1,7 @@ targets: # Mock agent that writes to the workspace root AND edits a file inside a nested git repo. # Simulates an agent that produces an artifact alongside making code changes. - - label: mock_agent + - id: mock_agent provider: cli command: >- bash -c ' diff --git a/examples/features/file-changes/.agentv/targets.yaml b/examples/features/file-changes/.agentv/targets.yaml index 0bbcec1c9..917521fe8 100644 --- a/examples/features/file-changes/.agentv/targets.yaml +++ b/examples/features/file-changes/.agentv/targets.yaml @@ -1,7 +1,7 @@ targets: # Mock agent that edits, creates, and deletes files in the workspace. # Each test gets a fresh copy of workspace-template/ as its CWD. - - label: mock_agent + - id: mock_agent provider: cli command: >- bash -c ' diff --git a/examples/features/functional-grading/.agentv/targets.yaml b/examples/features/functional-grading/.agentv/targets.yaml index 834f18287..9e0f6c1c2 100644 --- a/examples/features/functional-grading/.agentv/targets.yaml +++ b/examples/features/functional-grading/.agentv/targets.yaml @@ -1,7 +1,7 @@ targets: # Mock agent that implements the stub functions in src/index.ts. # Each test gets a fresh copy of workspace-template/ as its CWD. - - label: mock_agent + - id: mock_agent provider: cli command: >- bash -c ' diff --git a/examples/features/latency-assertions/.agentv/targets.yaml b/examples/features/latency-assertions/.agentv/targets.yaml index 7828dda70..a952d6f43 100644 --- a/examples/features/latency-assertions/.agentv/targets.yaml +++ b/examples/features/latency-assertions/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: mock_latency_agent + - id: mock_latency_agent provider: cli grader_target: grader command: bun run ./mock-latency-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} diff --git a/examples/features/local-cli/.agentv/targets.yaml b/examples/features/local-cli/.agentv/targets.yaml index cf14d7b58..48daa619b 100644 --- a/examples/features/local-cli/.agentv/targets.yaml +++ b/examples/features/local-cli/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: local_cli + - id: local_cli provider: cli grader_target: grader command: uv run ./mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE} diff --git a/examples/features/preprocessors/.agentv/targets.yaml b/examples/features/preprocessors/.agentv/targets.yaml index 6e35db409..dbce6e98f 100644 --- a/examples/features/preprocessors/.agentv/targets.yaml +++ b/examples/features/preprocessors/.agentv/targets.yaml @@ -1,12 +1,12 @@ $schema: agentv-targets-v2.2 targets: - - label: file_output + - id: file_output provider: file-output command: bun run .agentv/providers/file-output.ts {OUTPUT_FILE} cwd: .. grader_target: grader_check - - label: grader_check + - id: grader_check provider: grader-check command: bun run .agentv/providers/grader-check.ts {PROMPT_FILE} {OUTPUT_FILE} cwd: .. diff --git a/examples/features/script-grader-sdk/.agentv/targets.yaml b/examples/features/script-grader-sdk/.agentv/targets.yaml index fb88cf30c..f19f5281f 100644 --- a/examples/features/script-grader-sdk/.agentv/targets.yaml +++ b/examples/features/script-grader-sdk/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: local_cli + - id: local_cli provider: cli grader_target: grader command: uv run ../local-cli/mock_cli.py --prompt {PROMPT} {FILES} --output {OUTPUT_FILE} diff --git a/examples/features/sdk-eval-authoring/.agentv/targets.yaml b/examples/features/sdk-eval-authoring/.agentv/targets.yaml index 4a5a5bab9..d73ec1087 100644 --- a/examples/features/sdk-eval-authoring/.agentv/targets.yaml +++ b/examples/features/sdk-eval-authoring/.agentv/targets.yaml @@ -1,4 +1,4 @@ targets: - - label: mock-sdk + - id: mock-sdk provider: mock response: Hello from the mock target diff --git a/examples/features/sdk-python/.agentv/targets.yaml b/examples/features/sdk-python/.agentv/targets.yaml index c0a67e355..028cbb9c0 100644 --- a/examples/features/sdk-python/.agentv/targets.yaml +++ b/examples/features/sdk-python/.agentv/targets.yaml @@ -1,4 +1,4 @@ targets: - - label: local_cli + - id: local_cli provider: cli command: python3 -c "import argparse; parser = argparse.ArgumentParser(); parser.add_argument('--prompt', required=True); parser.add_argument('--output', required=True); args, _ = parser.parse_known_args(); open(args.output, 'w', encoding='utf-8').write(args.prompt)" --prompt {PROMPT} --output {OUTPUT_FILE} diff --git a/examples/features/suite-level-input/.agentv/targets.yaml b/examples/features/suite-level-input/.agentv/targets.yaml index c45b96f12..c8b0847a5 100644 --- a/examples/features/suite-level-input/.agentv/targets.yaml +++ b/examples/features/suite-level-input/.agentv/targets.yaml @@ -1,3 +1,3 @@ targets: - - label: llm + - id: llm provider: mock diff --git a/examples/features/tool-evaluation-plugins/.agentv/targets.yaml b/examples/features/tool-evaluation-plugins/.agentv/targets.yaml index c45b96f12..c8b0847a5 100644 --- a/examples/features/tool-evaluation-plugins/.agentv/targets.yaml +++ b/examples/features/tool-evaluation-plugins/.agentv/targets.yaml @@ -1,3 +1,3 @@ targets: - - label: llm + - id: llm provider: mock diff --git a/examples/features/tool-trajectory-advanced/.agentv/targets.yaml b/examples/features/tool-trajectory-advanced/.agentv/targets.yaml index 7bda9cb13..3debcb94e 100644 --- a/examples/features/tool-trajectory-advanced/.agentv/targets.yaml +++ b/examples/features/tool-trajectory-advanced/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: static_trace + - id: static_trace provider: cli grader_target: grader command: bun run ./cat-trace.ts --trace ./static-trace.json --prompt {PROMPT} --output {OUTPUT_FILE} diff --git a/examples/features/tool-trajectory-simple/.agentv/targets.yaml b/examples/features/tool-trajectory-simple/.agentv/targets.yaml index d508a2ca7..ca4672f0d 100644 --- a/examples/features/tool-trajectory-simple/.agentv/targets.yaml +++ b/examples/features/tool-trajectory-simple/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: mock_agent + - id: mock_agent provider: cli grader_target: grader command: bun run ./mock-agent.ts --prompt {PROMPT} --output {OUTPUT_FILE} diff --git a/examples/features/vitest-workspace-grader/.agentv/targets.yaml b/examples/features/vitest-workspace-grader/.agentv/targets.yaml index 8908229c9..8f9d22c43 100644 --- a/examples/features/vitest-workspace-grader/.agentv/targets.yaml +++ b/examples/features/vitest-workspace-grader/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: mock_agent + - id: mock_agent provider: cli command: | bash -c ' diff --git a/examples/features/workspace-artifact/.agentv/targets.yaml b/examples/features/workspace-artifact/.agentv/targets.yaml index 9b5fc8cdc..ca8b6097d 100644 --- a/examples/features/workspace-artifact/.agentv/targets.yaml +++ b/examples/features/workspace-artifact/.agentv/targets.yaml @@ -1,7 +1,7 @@ targets: # Mock CLI agent that writes a CSV report to outputs/report.csv under workspace_path. # Simulates what a real agent (e.g. Copilot) would do when asked to generate a report. - - label: mock_csv_agent + - id: mock_csv_agent provider: cli command: >- bash -c ' diff --git a/examples/showcase/cross-repo-sync/.agentv/targets.yaml b/examples/showcase/cross-repo-sync/.agentv/targets.yaml index f3700f89f..6b573588e 100644 --- a/examples/showcase/cross-repo-sync/.agentv/targets.yaml +++ b/examples/showcase/cross-repo-sync/.agentv/targets.yaml @@ -1,11 +1,11 @@ targets: - - label: mock_agent + - id: mock_agent provider: cli command: bash mock-agent.sh {PROMPT} {OUTPUT_FILE} - - label: claude_agent + - id: claude_agent provider: claude - - label: copilot_agent + - id: copilot_agent provider: copilot-cli model: claude-haiku-4.5 diff --git a/examples/showcase/export-screening/.agentv/targets.yaml b/examples/showcase/export-screening/.agentv/targets.yaml index 365d1a71e..cd54298e2 100644 --- a/examples/showcase/export-screening/.agentv/targets.yaml +++ b/examples/showcase/export-screening/.agentv/targets.yaml @@ -5,6 +5,6 @@ targets: # Mock target for testing eval flow without LLM calls # Default: returns High risk - customize to test other scenarios - - label: default + - id: default provider: mock response: '{"riskLevel": "High", "reasoning": "Mock response for testing"}' diff --git a/examples/showcase/offline-grader-benchmark/.agentv/targets.yaml b/examples/showcase/offline-grader-benchmark/.agentv/targets.yaml index 3457b38ba..93ca8172e 100644 --- a/examples/showcase/offline-grader-benchmark/.agentv/targets.yaml +++ b/examples/showcase/offline-grader-benchmark/.agentv/targets.yaml @@ -1,5 +1,5 @@ targets: - - label: fixture_replay + - id: fixture_replay provider: cli command: bun run ./scripts/replay-fixture-output.ts --prompt {PROMPT} --output {OUTPUT_FILE} cwd: .. diff --git a/examples/showcase/trace-evaluation/.agentv/targets.yaml b/examples/showcase/trace-evaluation/.agentv/targets.yaml index 874bc5516..bf4a069ea 100644 --- a/examples/showcase/trace-evaluation/.agentv/targets.yaml +++ b/examples/showcase/trace-evaluation/.agentv/targets.yaml @@ -1,10 +1,10 @@ targets: - - label: live_coding_agent + - id: live_coding_agent provider: codex model: gpt-5 timeout_seconds: 300 - - label: replay_coding_agent + - id: replay_coding_agent provider: replay fixtures: ../fixtures/replay-target-output.jsonl suite: trace-evaluation-showcase diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 7251e3be2..83f01b861 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -440,15 +440,16 @@ export function extractTargetFromSuite(suite: JsonObject): string | undefined { return targetValue.trim(); } if (isJsonObject(targetValue)) { - const label = targetValue.label; + const id = targetValue.id; const extendsTarget = targetValue.extends; if (typeof targetValue.name === 'string' && targetValue.name.trim().length > 0) { - throw new Error( - "Top-level target object field 'name' has been removed. Use 'label' instead.", - ); + throw new Error("Top-level target object field 'name' has been removed. Use 'id' instead."); + } + if (typeof targetValue.label === 'string' && targetValue.label.trim().length > 0) { + throw new Error("Top-level target object field 'label' has been removed. Use 'id' instead."); } - if (typeof label === 'string' && label.trim().length > 0) { - return label.trim(); + if (typeof id === 'string' && id.trim().length > 0) { + return id.trim(); } if (typeof extendsTarget === 'string' && extendsTarget.trim().length > 0) { return extendsTarget.trim(); @@ -495,9 +496,7 @@ function parseEvalTargetRef(raw: unknown, location: string): EvalTargetRef { throw new Error(`Invalid ${location}: use a target label string or target object.`); } if (typeof raw.name === 'string' && raw.name.trim().length > 0) { - throw new Error( - `Invalid ${location}: target field 'name' has been removed. Use 'label' instead.`, - ); + throw new Error(`Invalid ${location}: target field 'name' has been removed. Use 'id' instead.`); } const rawLabel = raw.label; @@ -512,19 +511,22 @@ function parseEvalTargetRef(raw: unknown, location: string): EvalTargetRef { const legacyTargetName = typeof legacyName === 'string' && legacyName.trim().length > 0 ? legacyName.trim() : undefined; if (legacyName !== undefined) { + throw new Error(`Invalid ${location}: target field 'name' has been removed. Use 'id' instead.`); + } + if (label !== undefined) { throw new Error( - `Invalid ${location}: target field 'name' has been removed. Use 'id' and 'label' instead.`, + `Invalid ${location}: target field 'label' has been removed. Use 'id' instead.`, ); } const hooks = parseTargetHooks(raw.hooks); const hasInlineDefinition = typeof raw.provider === 'string' || useTargetName !== undefined; - if (hasInlineDefinition && !label) { - throw new Error(`Invalid ${location}: target object requires a 'label' field.`); + if (hasInlineDefinition && !id) { + throw new Error(`Invalid ${location}: target object requires an 'id' field.`); } - const name = hasInlineDefinition ? label : (id ?? legacyTargetName ?? label); + const name = id ?? legacyTargetName; if (!name) { - throw new Error(`Invalid ${location}: target object requires an 'id' or 'label' field.`); + throw new Error(`Invalid ${location}: target object requires an 'id' field.`); } const definition = hasInlineDefinition ? (normalizeTargetDefinition( @@ -535,7 +537,6 @@ function parseEvalTargetRef(raw: unknown, location: string): EvalTargetRef { return { name, ...(id !== undefined ? { id } : {}), - ...(label !== undefined ? { label } : {}), ...(useTargetName !== undefined ? { use_target: useTargetName } : {}), ...(definition ? { definition } : {}), ...(hooks !== undefined ? { hooks } : {}), diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index c64a93266..4c64a94a6 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -24,25 +24,29 @@ function assertTargetDefinition(value: unknown, index: number, filePath: string) throw new Error(`targets.yaml entry at index ${index} in ${filePath} must be an object`); } - const label = value.label; + const id = value.id; const provider = value.provider; - if (typeof label !== 'string' || label.trim().length === 0) { + if (typeof id !== 'string' || id.trim().length === 0) { + throw new Error(`targets.yaml entry at index ${index} in ${filePath} is missing a valid 'id'`); + } + + if (typeof value.name === 'string' && value.name.trim().length > 0) { throw new Error( - `targets.yaml entry at index ${index} in ${filePath} is missing a valid 'label'`, + `targets.yaml entry '${id}' in ${filePath} uses removed field 'name'. Use 'id' for the AgentV target name.`, ); } - if (typeof value.name === 'string' && value.name.trim().length > 0) { + if (typeof value.label === 'string' && value.label.trim().length > 0) { throw new Error( - `targets.yaml entry '${label}' in ${filePath} uses removed field 'name'. Use 'label' for the AgentV target name.`, + `targets.yaml entry '${id}' in ${filePath} uses removed field 'label'. Use 'id' for the AgentV target name.`, ); } const hasUseTarget = typeof value.use_target === 'string' && value.use_target.trim().length > 0; if (!hasUseTarget && (typeof provider !== 'string' || provider.trim().length === 0)) { throw new Error( - `targets.yaml entry '${label}' in ${filePath} is missing a valid 'provider' (or use use_target for delegation)`, + `targets.yaml entry '${id}' in ${filePath} is missing a valid 'provider' (or use use_target for delegation)`, ); } diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 171a2b756..18bee5e74 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -702,10 +702,9 @@ export interface NormalizeTargetDefinitionOptions { } /** - * Converts the authored promptfoo-shaped target object into AgentV's internal - * target definition. Authored YAML uses `label` for AgentV's target/comparison - * name and `id` for the promptfoo provider/backend identifier. The runtime - * continues to use `name` as its stable resolver and artifact key. + * Converts the authored target object into AgentV's internal target definition. + * Authored YAML uses `id` as the stable AgentV target identity. The runtime + * continues to use `name` as its resolver and artifact key. */ export function normalizeTargetDefinition( definition: unknown, @@ -723,9 +722,9 @@ export function normalizeTargetDefinition( typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : undefined; const legacyName = typeof rawName === 'string' && rawName.trim().length > 0 ? rawName.trim() : undefined; - const name = label ?? legacyName ?? id ?? options.defaultName; + const name = id ?? legacyName ?? label ?? options.defaultName; if (!name || name.trim().length === 0) { - throw new Error("Target definition is missing a valid 'label' field"); + throw new Error("Target definition is missing a valid 'id' field"); } const config = isRecord(definition.config) ? definition.config : {}; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 541dbc2d2..fd07167ab 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -427,11 +427,11 @@ export interface Provider { export type EnvLookup = Readonly>; export interface TargetDefinition { - /** Internal canonical target identity. Authored YAML uses `label`; loaders map label -> name. */ + /** Internal canonical target identity. Authored YAML uses `id`; loaders map id -> name. */ readonly name: string; - /** Promptfoo provider/backend identifier, not the AgentV target reference key. */ + /** Authored AgentV target identity. */ readonly id?: string | undefined; - /** Authored promptfoo-shaped label; used as AgentV's authored target/comparison name. */ + /** Internal compatibility display label; authored `label` is rejected at the boundary. */ readonly label?: string | undefined; /** Promptfoo-shaped provider options bag. Provider settings are flattened at the boundary. */ readonly config?: unknown | undefined; diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 494d9eb34..3da85a2f0 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -452,8 +452,7 @@ const TargetHooksSchema = z /** Eval target reference: string shorthand or object with hooks */ const EvalTargetRefSchema = z .object({ - label: z.string().min(1), - id: z.string().min(1).optional(), + id: z.string().min(1), use_target: z.string().optional(), hooks: TargetHooksSchema.optional(), }) @@ -462,7 +461,6 @@ const EvalTargetRefSchema = z const EvalLocalTargetSchema = z .object({ id: z.string().min(1).optional(), - label: z.string().min(1).optional(), extends: z.string().min(1).optional(), provider: z.string().min(1).optional(), model: z.string().min(1).optional(), diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index b4e254ed5..1b62084ba 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -622,15 +622,22 @@ export async function validateTargetsFile(filePath: string): Promise 0) { errors.push({ severity: 'error', filePath: absolutePath, location: `${location}.label`, - message: "Missing or invalid 'label' field (must be a non-empty string)", + message: "The target 'label' field has been removed. Use 'id' instead.", }); } if (typeof target.name === 'string' && target.name.trim().length > 0) { @@ -638,8 +645,7 @@ export async function validateTargetsFile(filePath: string): Promise 0) { + throw new Error("Invalid top-level 'target': field 'name' has been removed. Use 'id' instead."); + } + if (typeof rawTarget.label === 'string' && rawTarget.label.trim().length > 0) { throw new Error( - "Invalid top-level 'target': field 'name' has been removed. Use 'label' instead.", + "Invalid top-level 'target': field 'label' has been removed. Use 'id' instead.", ); } const rawExtends = rawTarget.extends; const extendsTarget = typeof rawExtends === 'string' && rawExtends.trim().length > 0 ? rawExtends.trim() : undefined; - const rawLabel = rawTarget.label; + const rawId = rawTarget.id; const name = - typeof rawLabel === 'string' && rawLabel.trim().length > 0 - ? rawLabel.trim() + typeof rawId === 'string' && rawId.trim().length > 0 + ? rawId.trim() : (extendsTarget ?? 'eval-local-target'); const hooks = parseTargetHooks(rawTarget.hooks); const definitionEntries = Object.entries(rawTarget).filter( diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index cb66e3d2d..72e737a19 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -239,8 +239,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ' label: Terse', ' prompt: "In one sentence, summarize {{ topic }}."', 'targets:', - ' - id: openai:gpt-5.4-mini', - ' label: mini', + ' - id: mini', ' - id: local-codex', 'tests:', ' - id: docs', @@ -266,9 +265,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { 'Summarize release notes.', 'In one sentence, summarize release notes.', ]); - expect(suite.targets).toEqual(['openai:gpt-5.4-mini', 'local-codex']); + expect(suite.targets).toEqual(['mini', 'local-codex']); expect(suite.targetRefs).toEqual([ - { name: 'openai:gpt-5.4-mini', id: 'openai:gpt-5.4-mini', label: 'mini' }, + { name: 'mini', id: 'mini' }, { name: 'local-codex', id: 'local-codex' }, ]); }); diff --git a/packages/core/test/evaluation/interpolation-integration.test.ts b/packages/core/test/evaluation/interpolation-integration.test.ts index 031882ed9..a018dbb3f 100644 --- a/packages/core/test/evaluation/interpolation-integration.test.ts +++ b/packages/core/test/evaluation/interpolation-integration.test.ts @@ -164,7 +164,7 @@ describe('env interpolation in YAML loading', () => { evalFile, [ 'target:', - ' label: local-shell', + ' id: local-shell', ' provider: cli', ' command: "echo $RUNTIME ${RUNTIME} {{ env.AGENTV_TEST_PATH }}"', 'tests:', diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 435d45ca0..7f22de031 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -856,11 +856,10 @@ describe('extractTargetFromSuite', () => { expect(extractTargetFromSuite(suite)).toBe('codex-gpt5'); }); - it('extracts target object identity from label or extends', () => { + it('extracts target object identity from id or extends', () => { const suite: JsonObject = { target: { - label: 'codex-local', - id: 'codex:gpt-5.1', + id: 'codex-local', extends: 'codex-gpt5', config: { model: 'gpt-5.1' }, }, @@ -868,11 +867,18 @@ describe('extractTargetFromSuite', () => { expect(extractTargetFromSuite(suite)).toBe('codex-local'); }); - it('rejects target object name in favor of label', () => { + it('rejects target object name in favor of id', () => { const suite: JsonObject = { target: { name: 'legacy-target', provider: 'mock' }, }; - expect(() => extractTargetFromSuite(suite)).toThrow(/Use 'label'/); + expect(() => extractTargetFromSuite(suite)).toThrow(/Use 'id'/); + }); + + it('rejects target object label in favor of id', () => { + const suite: JsonObject = { + target: { label: 'legacy-target', provider: 'mock' }, + }; + expect(() => extractTargetFromSuite(suite)).toThrow(/Use 'id'/); }); it('returns undefined when no target specified', () => { @@ -898,8 +904,7 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { targets: [ 'registry-agent', { - label: 'inline-agent', - id: 'mock', + id: 'inline-agent', provider: 'mock', config: { response: 'ok' }, fallback_targets: ['registry-agent'], @@ -912,10 +917,9 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { { name: 'registry-agent' }, { name: 'inline-agent', - id: 'mock', - label: 'inline-agent', + id: 'inline-agent', definition: expect.objectContaining({ - id: 'mock', + id: 'inline-agent', name: 'inline-agent', label: 'inline-agent', provider: 'mock', diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index 07d56bdb1..6a8da3d4e 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -23,10 +23,9 @@ describe('readTargetDefinitions', () => { return filePath; } - it('normalizes promptfoo-shaped label, backend id, and config fields', async () => { + it('normalizes authored id identity and config fields', async () => { const filePath = await writeTargetsYaml(`targets: - - label: candidate-agent - id: openai:gpt-5-codex + - id: candidate-agent provider: codex-cli config: command: ["codex"] @@ -39,7 +38,7 @@ describe('readTargetDefinitions', () => { expect(definitions).toEqual([ expect.objectContaining({ - id: 'openai:gpt-5-codex', + id: 'candidate-agent', name: 'candidate-agent', label: 'candidate-agent', provider: 'codex-cli', @@ -51,12 +50,22 @@ describe('readTargetDefinitions', () => { ]); }); - it('rejects authored name in favor of label', async () => { + it('rejects authored name in favor of id', async () => { const filePath = await writeTargetsYaml(`targets: - name: legacy-agent provider: mock `); - await expect(readTargetDefinitions(filePath)).rejects.toThrow(/missing a valid 'label'/); + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/missing a valid 'id'/); + }); + + it('rejects authored label in favor of id', async () => { + const filePath = await writeTargetsYaml(`targets: + - id: candidate-agent + label: legacy-agent + provider: mock +`); + + await expect(readTargetDefinitions(filePath)).rejects.toThrow(/Use 'id'/); }); }); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index b5ebf9aaa..0864a5136 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -87,11 +87,10 @@ describe('resolveTargetDefinition', () => { piGetModelMock.mockClear(); }); - it('uses promptfoo-shaped label as AgentV target identity and id as backend metadata', () => { + it('uses authored target id as AgentV target identity', () => { const target = resolveTargetDefinition( { - label: 'primary-sut', - id: 'mock', + id: 'primary-sut', provider: 'mock', config: { response: 'ok', @@ -126,11 +125,10 @@ describe('resolveTargetDefinition', () => { expect(target.config.command).toBe('agent run {PROMPT_FILE} {OUTPUT_FILE}'); }); - it('treats provider as backend kind while id remains a provider locator', () => { + it('treats provider as backend kind while id remains target identity', () => { const target = resolveTargetDefinition( { - label: 'candidate-agent', - id: 'openai:gpt-5-mini', + id: 'candidate-agent', provider: 'openai', config: { api_key: '${{ OPENAI_API_KEY }}', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index e3f2a275c..c2d39a5d8 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -305,7 +305,7 @@ tags: prompts: - raw: "Review {{ vars.diff }}" targets: - - label: local-agent + - id: local-agent provider: codex-cli command: ["codex"] default_test: diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index cc9f21f39..dfcfb7b3a 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -22,7 +22,7 @@ describe('validateTargetsFile', () => { await writeFile( filePath, `targets: - - label: openrouter-target + - id: openrouter-target provider: openrouter api_key: \${{ OPENROUTER_API_KEY }} model: openai/gpt-5-mini @@ -40,13 +40,12 @@ describe('validateTargetsFile', () => { ).toBe(false); }); - it('accepts promptfoo-shaped id, label, and config fields', async () => { + it('accepts id identity and config fields', async () => { const filePath = path.join(tempDir, 'promptfoo-shaped-target.yaml'); await writeFile( filePath, `targets: - - label: candidate-agent - id: openai:gpt-5-codex + - id: candidate-agent provider: codex-cli config: command: ["codex"] @@ -58,12 +57,12 @@ describe('validateTargetsFile', () => { grader_target: grader fallback_targets: [backup-agent] batch_requests: true - - label: grader + - id: grader provider: openai config: api_key: \${{ OPENAI_API_KEY }} model: gpt-5-mini - - label: backup-agent + - id: backup-agent provider: mock config: response: backup @@ -100,7 +99,7 @@ describe('validateTargetsFile', () => { ).toBe(true); }); - it('rejects authored target name in favor of label', async () => { + it('rejects authored target name in favor of id', async () => { const filePath = path.join(tempDir, 'legacy-name-target.yaml'); await writeFile( filePath, @@ -117,8 +116,8 @@ describe('validateTargetsFile', () => { result.errors.some( (error) => error.severity === 'error' && - error.location === 'targets[0].label' && - error.message.includes("Missing or invalid 'label' field"), + error.location === 'targets[0].id' && + error.message.includes("Missing or invalid 'id' field"), ), ).toBe(true); expect( @@ -126,7 +125,7 @@ describe('validateTargetsFile', () => { (error) => error.severity === 'error' && error.location === 'targets[0].name' && - error.message.includes("Use 'label'"), + error.message.includes("Use 'id'"), ), ).toBe(true); }); @@ -234,7 +233,7 @@ targets: await writeFile( filePath, `targets: - - label: codex-target + - id: codex-target provider: codex-cli command: ["codex"] timeoutSeconds: 30 @@ -300,7 +299,7 @@ targets: await writeFile( filePath, `targets: - - label: codex-target + - id: codex-target provider: codex-cli command: ["codex"] model: \${{ CODEX_MODEL }} @@ -318,7 +317,7 @@ targets: await writeFile( filePath, `targets: - - label: copilot-sdk-custom-provider + - id: copilot-sdk-custom-provider provider: copilot-sdk model: gpt-5 subprovider: openai @@ -327,7 +326,7 @@ targets: api_format: responses model_id: gpt-5 wire_model: \${{ OPENAI_MODEL }} - - label: copilot-cli-custom-provider + - id: copilot-cli-custom-provider provider: copilot-cli subprovider: openai base_url: \${{ OPENAI_ENDPOINT }} @@ -347,7 +346,7 @@ targets: await writeFile( filePath, `targets: - - label: codex-local-openai + - id: codex-local-openai provider: codex-cli command: ["codex"] model: \${{ CODEX_MODEL }} @@ -414,11 +413,11 @@ targets: await writeFile( filePath, `targets: - - label: default + - id: default use_target: \${{ AGENT_TARGET }} - - label: grader + - id: grader use_target: \${{ GRADER_TARGET }} - - label: codex-agent + - id: codex-agent provider: codex-cli command: ["codex"] grader_target: grader @@ -550,7 +549,7 @@ targets: await writeFile( filePath, `targets: - - label: replay-execution-trace + - id: replay-execution-trace provider: replay execution_traces: ./fixtures/execution-traces.jsonl source_target: live-agent diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index d776f90ee..831284516 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -915,10 +915,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -1253,10 +1249,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -1591,10 +1583,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -3003,10 +2991,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -3341,10 +3325,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -3679,10 +3659,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -5018,10 +4994,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -5356,10 +5328,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -5694,10 +5662,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -6175,10 +6139,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -6513,10 +6473,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -6851,10 +6807,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -7504,10 +7456,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -7842,10 +7790,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -8180,10 +8124,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -8763,10 +8703,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -9101,10 +9037,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1 @@ -9439,10 +9371,6 @@ "type": "string", "minLength": 1 }, - "label": { - "type": "string", - "minLength": 1 - }, "extends": { "type": "string", "minLength": 1