feat(core): normalize public providers schema

.agentv/targets.yaml

@@ -1,90 +1,90 @@
-# AgentV Evaluation Targets
+# AgentV Evaluation Providers
 # Consolidated from root, examples/features, and examples/showcase.
 # Per-eval .agentv folders override these for specific eval cases.
 #
-# "grader" is the LLM used for scoring; agent targets reference it via
+# "grader" is the LLM used for scoring; agent providers reference it via
 # grader_target so eval execution and grading use separate models.
 #
-# default/agent/llm/grader are concrete targets, not env-var-driven aliases:
+# default/agent/llm/grader are concrete provider labels, not env-var-driven aliases:
 # AgentV removed the `use_target` indirection field with no replacement, so a
-# target id can no longer redirect to "whichever target another target names".
+# provider label can no longer redirect to "whichever provider another provider names".
 # These four mirror today's AGENT_TARGET/LLM_TARGET/GRADER_TARGET selection
-# (copilot-cli / azure / azure) so `target: llm` etc. across examples/ keep
+# (copilot-cli / azure / azure) so `providers: [llm]` etc. across examples/ keep
 # resolving. To switch the active provider, edit the definitions below
-# directly (or point examples at a different concrete target id) instead of
+# directly (or point examples at a different concrete provider label) instead of
 # changing an env var.
 
-targets:
-  # ── Default target ──────────────────────────────────────────────────
-  # Evals without an explicit target resolve to "default" by convention.
-  - id: default
-    provider: copilot-cli
+providers:
+  # ── Default provider ────────────────────────────────────────────────
+  # Evals without an explicit provider resolve to "default" by convention.
+  - id: copilot-cli
+    label: default
     model: "{{ env.COPILOT_MODEL }}"
     grader_target: grader
     stream_log: raw
 
-  - id: agent
-    provider: copilot-cli
+  - id: copilot-cli
+    label: agent
     model: "{{ env.COPILOT_MODEL }}"
     grader_target: grader
     stream_log: raw
 
-  # ── LLM target (text generation, no agent binary needed) ────────────
-  - id: llm
-    provider: azure
+  # ── LLM provider (text generation, no agent binary needed) ──────────
+  - id: azure
+    label: llm
     endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
     model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
     version: "{{ env.AZURE_OPENAI_API_VERSION }}"
 
   # ── Grader (LLM-as-judge) ──────────────────────────────────────────
-  # Used by agent targets via grader_target.
-  - id: grader
-    provider: azure
+  # Used by agent providers via grader_target.
+  - id: azure
+    label: grader
     endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
     model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
     version: "{{ env.AZURE_OPENAI_API_VERSION }}"
 
-  # ── Named agent targets ───────────────────────────────────────────
-  - id: copilot
-    provider: copilot-cli
+  # ── Named agent providers ─────────────────────────────────────────
+  - id: copilot-cli
+    label: copilot
     model: "{{ env.COPILOT_MODEL }}"
     grader_target: grader
     stream_log: raw
 
   - id: copilot-sdk
-    provider: copilot-sdk
+    label: copilot-sdk
     model: "{{ env.COPILOT_MODEL }}"
     grader_target: grader
     stream_log: raw
 
-  - id: copilot-sdk-azure
-    provider: copilot-sdk
+  - id: copilot-sdk
+    label: copilot-sdk-azure
     model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
     subprovider: azure
     base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
     grader_target: grader
     stream_log: raw
 
-  - id: claude
-    provider: claude-cli
+  - id: claude-cli
+    label: claude
     grader_target: grader
 
   - id: claude-sdk
-    provider: claude-sdk
+    label: claude-sdk
     grader_target: grader
 
-  - id: pi
-    provider: pi-cli
+  - id: pi-cli
+    label: pi
     subprovider: openrouter
     model: "{{ env.OPENROUTER_MODEL }}"
     api_key: "{{ env.OPENROUTER_API_KEY }}"
     grader_target: grader
 
-  - id: pi-sdk
-    provider: pi-coding-agent
+  - id: pi-coding-agent
+    label: pi-sdk
     subprovider: openai
     base_url: "{{ env.OPENAI_ENDPOINT }}"
     api_key: "{{ env.OPENAI_API_KEY }}"
@@ -94,16 +94,16 @@ targets:
     stream_log: raw
 
 
-  - id: pi-azure
-    provider: pi-cli
+  - id: pi-cli
+    label: pi-azure
     subprovider: azure
     base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
     api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
     grader_target: grader
 
-  - id: pi-sdk-azure
-    provider: pi-coding-agent
+  - id: pi-coding-agent
+    label: pi-sdk-azure
     subprovider: azure
     base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     model: gpt-5.5
@@ -112,63 +112,63 @@ targets:
     thinking: low
     stream_log: raw
 
-  - id: codex
-    provider: codex
-    executable: "{{ env.CODEX_EXECUTABLE }}"
+  - id: agentv:codex-cli
+    label: codex
+    command: ["{{ env.CODEX_EXECUTABLE }}"]
     model: "{{ env.CODEX_MODEL }}"
-    model_reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}"
+    reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}"
     grader_target: grader
     cwd: "{{ env.CODEX_WORKSPACE_DIR }}"
     log_dir: "{{ env.CODEX_LOG_DIR }}"
     stream_log: raw
 
-  # ── LLM targets (direct model access) ─────────────────────────────
-  - id: gh-models
-    provider: openai
+  # ── LLM providers (direct model access) ───────────────────────────
+  - id: openai
+    label: gh-models
     base_url: https://models.github.ai/inference
     api_key: "{{ env.GH_MODELS_TOKEN }}"
     model: "{{ env.GH_MODELS_MODEL }}"
 
-  # Single Azure target. Always uses Azure's Responses API
+  # Single Azure provider. Always uses Azure's Responses API
   # (`/openai/v1/responses`); the api version defaults to `v1` and can be
   # overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure
   # deployments must use `provider: openai` with a deployment-scoped
   # `base_url` instead.
   - id: azure
-    provider: azure
+    label: azure
     endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}"
     api_key: "{{ env.AZURE_OPENAI_API_KEY }}"
     model: "{{ env.AZURE_DEPLOYMENT_NAME }}"
     version: "{{ env.AZURE_OPENAI_API_VERSION }}"
 
   - id: gemini
-    provider: gemini
+    label: gemini
     api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}"
     model: "{{ env.GEMINI_MODEL_NAME }}"
 
   - id: openai
-    provider: openai
+    label: openai
     endpoint: "{{ env.OPENAI_ENDPOINT }}"
     api_key: "{{ env.OPENAI_API_KEY }}"
     model: "{{ env.OPENAI_MODEL }}"
 
   # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy
-  # without changing provider-specific target labels.
-  - id: local-openai
-    provider: openai
+  # without changing provider-specific labels.
+  - id: openai
+    label: local-openai
     base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
     api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
     model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
     grader_target: local-openai-grader
 
-  - id: local-openai-grader
-    provider: openai
+  - id: openai
+    label: local-openai-grader
     base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
     api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
     model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
-  - id: pi-cli-openai
-    provider: pi-cli
+  - id: pi-cli
+    label: pi-cli-openai
     subprovider: openai
     base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
     api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
@@ -177,18 +177,18 @@ targets:
     thinking: low
     stream_log: raw
 
-  - id: codex-sdk-openai
-    provider: codex
+  - id: openai:codex-sdk
+    label: codex-sdk-openai
     base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
     api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
     model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
     api_format: responses
-    model_reasoning_effort: low
+    reasoning_effort: low
     grader_target: local-openai-grader
     stream_log: raw
 
-  - id: copilot-sdk-openai
-    provider: copilot-sdk
+  - id: copilot-sdk
+    label: copilot-sdk-openai
     subprovider: openai
     base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
     api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
@@ -197,25 +197,25 @@ targets:
     stream_log: raw
 
   - id: openrouter
-    provider: openrouter
+    label: openrouter
     api_key: "{{ env.OPENROUTER_API_KEY }}"
     model: "{{ env.OPENROUTER_MODEL }}"
 
   # ── MiMo (Xiaomi) via OpenRouter ───────────────────────────────────
-  - id: mimo
-    provider: openrouter
+  - id: openrouter
+    label: mimo
     api_key: "{{ env.OPENROUTER_API_KEY }}"
     model: xiaomi/mimo-v2.5-pro
     grader_target: grader
 
-  - id: mimo-flash
-    provider: openrouter
+  - id: openrouter
+    label: mimo-flash
     api_key: "{{ env.OPENROUTER_API_KEY }}"
     model: xiaomi/mimo-v2-flash
     grader_target: grader
 
-  - id: mimo-direct
-    provider: openai
+  - id: openai
+    label: mimo-direct
     base_url: https://token-plan-sgp.xiaomimimo.com/v1
     api_key: "{{ env.XIAOMI_MIMO_API_KEY }}"
     model: xiaomi/mimo-v2.5-pro

apps/cli/src/commands/eval/commands/bundle.ts

@@ -134,7 +134,7 @@ function buildBundleRuntime(options: {
   readonly threshold?: number;
 }): Record<string, unknown> {
   const runtime: Record<string, unknown> =
-    options.targetNames.length === 1 ? { target: options.targetNames[0] } : {};
+    options.targetNames.length > 0 ? { providers: options.targetNames } : {};
   if (options.budgetUsd !== undefined) {
     runtime.budget_usd = options.budgetUsd;
   }

apps/cli/src/commands/eval/run-eval.ts

@@ -1819,6 +1819,9 @@ export async function runEvalCommand(
   }
 
   let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution);
+  if (yamlConfig?.defaults?.provider && options.cliTargets.length === 0 && !options.target) {
+    options = { ...options, target: yamlConfig.defaults.provider };
+  }
   if (yamlConfig?.defaults?.grader) {
     options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader };
   }

apps/cli/src/commands/eval/task-bundle.ts

@@ -48,11 +48,14 @@ const SKIPPED_DIR_NAMES = new Set([
 const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([
   'label',
   'provider',
+  'provider_spec',
   'prompts',
   'transform',
   'delay',
   'env',
   'model',
+  'runtime',
+  'hooks',
   'use_target',
   'fallback_targets',
   'grader_target',
@@ -649,9 +652,17 @@ function uniqueTargetDefinitions(
 }
 
 function serializeTargetDefinition(definition: TargetDefinition): Record<string, unknown> {
-  const target: Record<string, unknown> = { id: definition.name };
-  if (definition.id !== undefined) {
-    target.id = definition.id;
+  const providerSpec =
+    typeof definition.provider_spec === 'string' && definition.provider_spec.trim().length > 0
+      ? definition.provider_spec.trim()
+      : typeof definition.provider === 'string' && definition.provider.trim().length > 0
+        ? definition.provider.trim()
+        : typeof definition.id === 'string' && definition.id.trim().length > 0
+          ? definition.id.trim()
+          : definition.name;
+  const provider: Record<string, unknown> = { id: providerSpec };
+  if (definition.name !== providerSpec) {
+    provider.label = definition.name;
   }
   const config: Record<string, unknown> = {};
 
@@ -660,13 +671,15 @@ function serializeTargetDefinition(definition: TargetDefinition): Record<string,
       value === undefined ||
       key === 'name' ||
       key === 'id' ||
+      key === 'provider' ||
+      key === 'provider_spec' ||
       key === 'label' ||
       key === 'config'
     ) {
       continue;
     }
     if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) {
-      target[key] = value;
+      provider[key] = value;
     } else {
       config[key] = value;
     }
@@ -680,10 +693,10 @@ function serializeTargetDefinition(definition: TargetDefinition): Record<string,
     }
   }
   if (Object.keys(config).length > 0) {
-    target.config = config;
+    provider.config = config;
   }
 
-  return target;
+  return provider;
 }
 
 function serializeTargetDefinitions(
@@ -1133,11 +1146,11 @@ export async function materializeTaskBundle(
   const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME);
 
   await writeYamlFile(evalPath, {
-    target: options.targetName,
+    providers: [options.targetName],
     prompts: [INPUT_PROMPT],
     tests: [evalCase],
   });
-  await writeYamlFile(targetsPath, { targets: serializeTargetDefinitions(targetDefinitions) });
+  await writeYamlFile(targetsPath, { providers: serializeTargetDefinitions(targetDefinitions) });
 
   return {
     testDir,
@@ -1203,15 +1216,15 @@ export async function materializeEvalBundle(
   const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME);
   const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME);
   const runtime =
-    options.runtime ?? (targetNames.length === 1 ? { target: targetNames[0] } : undefined);
+    options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : undefined);
 
   await writeYamlFile(evalPath, {
     ...(runtime ?? {}),
     prompts: [INPUT_PROMPT],
     tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)),
   });
   await writeYamlFile(targetsPath, {
-    targets: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)),
+    providers: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)),
   });
 
   const manifest = bundleManifest({