From 2da0b18d0a8134d4ef575c8a83ab498a9dd76eec Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 05:37:28 +0200 Subject: [PATCH 1/2] feat(cli): migrate provider authoring surface --- README.md | 57 +++--- apps/cli/src/commands/create/commands.ts | 16 +- apps/cli/src/commands/eval/commands/bundle.ts | 38 +++- apps/cli/src/commands/eval/interactive.ts | 18 +- apps/cli/src/commands/eval/run-eval.ts | 10 +- apps/cli/src/commands/eval/targets.ts | 12 +- apps/cli/src/commands/init/index.ts | 2 +- apps/cli/src/commands/pipeline/input.ts | 37 +++- apps/cli/src/commands/pipeline/run.ts | 52 ++++-- apps/cli/src/commands/prepare/index.ts | 22 ++- apps/cli/src/commands/results/eval-runner.ts | 18 +- apps/cli/src/templates/.agentv/providers.yaml | 13 -- apps/cli/src/templates/.env.example | 4 +- apps/cli/src/utils/targets.ts | 89 ++++++++-- apps/cli/test/commands/eval/bundle.test.ts | 56 +++++- apps/cli/test/commands/eval/targets.test.ts | 77 +++++++++ .../commands/grade/grade-prepared.test.ts | 10 +- .../cli/test/commands/prepare/prepare.test.ts | 6 +- apps/cli/test/eval.integration.test.ts | 2 +- .../docs/v4.42.4/evaluation/batch-cli.mdx | 19 +- .../docs/v4.42.4/evaluation/eval-files.mdx | 10 +- .../docs/v4.42.4/evaluation/running-evals.mdx | 54 +++--- .../docs/docs/v4.42.4/evaluation/sdk.mdx | 2 +- .../v4.42.4/getting-started/installation.mdx | 4 +- .../v4.42.4/getting-started/quickstart.mdx | 2 +- .../guides/skill-improvement-workflow.mdx | 14 +- .../v4.42.4/guides/workspace-architecture.mdx | 18 +- .../integrations/agent-skills-evals.mdx | 8 +- .../docs/v4.42.4/targets/cli-provider.mdx | 64 +++---- .../docs/v4.42.4/targets/coding-agents.mdx | 122 ++++++------- .../docs/v4.42.4/targets/configuration.mdx | 163 +++++++++--------- .../docs/v4.42.4/targets/custom-providers.mdx | 26 +-- .../docs/v4.42.4/targets/llm-providers.mdx | 54 +++--- .../docs/docs/v4.42.4/targets/retry.mdx | 6 +- .../docs/docs/v4.42.4/tools/compare.mdx | 4 +- .../docs/docs/v4.42.4/tools/dashboard.mdx | 8 +- .../docs/docs/v4.42.4/tools/import.mdx | 12 +- .../docs/docs/v4.42.4/tools/prepare.mdx | 6 +- examples/README.md | 2 +- .../.agentv/{targets.yaml => providers.yaml} | 6 +- .../features/agent-skills-evals/README.md | 6 +- examples/features/autoresearch/README.md | 2 +- examples/features/basic/README.md | 2 +- .../features/batch-cli/.agentv/providers.yaml | 11 ++ .../features/batch-cli/.agentv/targets.yaml | 13 -- examples/features/batch-cli/README.md | 4 +- examples/features/benchmark-tooling/README.md | 4 +- .../.agentv/providers.yaml | 5 + .../.agentv/targets.yaml | 8 - .../copilot-transcript-replay/README.md | 4 +- .../features/deterministic-graders/README.md | 2 +- .../docker-workspace/.agentv/providers.yaml | 3 + .../docker-workspace/.agentv/targets.yaml | 3 - .../.agentv/providers.yaml | 8 + .../document-extraction/.agentv/targets.yaml | 14 -- .../.agentv/{targets.yaml => providers.yaml} | 10 +- examples/features/execution-metrics/README.md | 2 +- .../.agentv/providers.yaml | 11 ++ .../file-changes-graders/.agentv/targets.yaml | 16 -- .../.agentv/providers.yaml | 7 + .../.agentv/targets.yaml | 11 -- .../file-changes/.agentv/providers.yaml | 10 ++ .../file-changes/.agentv/targets.yaml | 15 -- .../.agentv/{targets.yaml => providers.yaml} | 21 +-- examples/features/file-transforms/README.md | 6 +- .../functional-grading/.agentv/providers.yaml | 10 ++ .../functional-grading/.agentv/targets.yaml | 10 -- .../.agentv/{targets.yaml => providers.yaml} | 6 +- .../features/latency-assertions/README.md | 2 +- .../.agentv/{targets.yaml => providers.yaml} | 6 +- examples/features/local-cli/README.md | 2 +- .../readme-quickstart/.agentv/providers.yaml | 17 ++ .../readme-quickstart/.agentv/targets.yaml | 17 -- examples/features/readme-quickstart/README.md | 4 +- .../{targets.yaml => providers.yaml} | 0 examples/features/rubric/README.md | 2 +- examples/features/scenarios/README.md | 6 +- examples/features/scenarios/providers.yaml | 4 + examples/features/scenarios/targets.yaml | 4 - .../.agentv/{targets.yaml => providers.yaml} | 9 +- examples/features/script-grader-sdk/README.md | 4 +- .../script-grader-with-llm-calls/README.md | 4 +- .../sdk-eval-authoring/.agentv/providers.yaml | 4 + .../sdk-eval-authoring/.agentv/targets.yaml | 4 - .../features/sdk-eval-authoring/README.md | 2 +- .../features/sdk-programmatic-api/evaluate.ts | 2 +- .../sdk-python/.agentv/providers.yaml | 9 + .../features/sdk-python/.agentv/targets.yaml | 4 - examples/features/sdk-python/README.md | 2 +- .../suite-level-input/.agentv/providers.yaml | 3 + .../suite-level-input/.agentv/targets.yaml | 3 - .../.agentv/providers.yaml | 3 + .../.agentv/targets.yaml | 3 - .../tool-evaluation-plugins/README.md | 2 +- .../.agentv/{targets.yaml => providers.yaml} | 9 +- .../trajectory-assertions-advanced/README.md | 2 +- .../.agentv/{targets.yaml => providers.yaml} | 6 +- .../trajectory-assertions-simple/README.md | 2 +- examples/features/trend/README.md | 6 +- .../trial-output-consistency/README.md | 2 +- .../.agentv/{targets.yaml => providers.yaml} | 6 +- .../vitest-workspace-grader/README.md | 4 +- .../workspace-artifact/.agentv/providers.yaml | 7 + .../workspace-artifact/.agentv/targets.yaml | 11 -- examples/red-team/README.md | 2 +- .../archetypes/coding-agent/README.md | 4 +- .../customer-facing-agent/README.md | 4 +- examples/showcase/bug-fix-benchmark/README.md | 4 +- .../cross-repo-sync/.agentv/providers.yaml | 9 + .../cross-repo-sync/.agentv/targets.yaml | 11 -- examples/showcase/cross-repo-sync/README.md | 2 +- .../export-screening/.agentv/providers.yaml | 4 + .../export-screening/.agentv/targets.yaml | 10 -- examples/showcase/export-screening/README.md | 2 +- .../showcase/multi-model-benchmark/README.md | 22 +-- .../.agentv/{targets.yaml => providers.yaml} | 9 +- .../offline-grader-benchmark/README.md | 4 +- .../.agentv/{targets.yaml => providers.yaml} | 16 +- examples/showcase/trace-evaluation/README.md | 6 +- .../trace-evaluation/scripts/prove-replay.ts | 2 +- packages/core/src/evaluation/evaluate.ts | 4 +- 121 files changed, 952 insertions(+), 707 deletions(-) rename examples/contract/.agentv/{targets.yaml => providers.yaml} (76%) create mode 100644 examples/features/batch-cli/.agentv/providers.yaml delete mode 100644 examples/features/batch-cli/.agentv/targets.yaml create mode 100644 examples/features/copilot-transcript-replay/.agentv/providers.yaml delete mode 100644 examples/features/copilot-transcript-replay/.agentv/targets.yaml create mode 100644 examples/features/docker-workspace/.agentv/providers.yaml delete mode 100644 examples/features/docker-workspace/.agentv/targets.yaml create mode 100644 examples/features/document-extraction/.agentv/providers.yaml delete mode 100644 examples/features/document-extraction/.agentv/targets.yaml rename examples/features/execution-metrics/.agentv/{targets.yaml => providers.yaml} (81%) create mode 100644 examples/features/file-changes-graders/.agentv/providers.yaml delete mode 100644 examples/features/file-changes-graders/.agentv/targets.yaml create mode 100644 examples/features/file-changes-with-repos/.agentv/providers.yaml delete mode 100644 examples/features/file-changes-with-repos/.agentv/targets.yaml create mode 100644 examples/features/file-changes/.agentv/providers.yaml delete mode 100644 examples/features/file-changes/.agentv/targets.yaml rename examples/features/file-transforms/.agentv/{targets.yaml => providers.yaml} (77%) create mode 100644 examples/features/functional-grading/.agentv/providers.yaml delete mode 100644 examples/features/functional-grading/.agentv/targets.yaml rename examples/features/latency-assertions/.agentv/{targets.yaml => providers.yaml} (80%) rename examples/features/local-cli/.agentv/{targets.yaml => providers.yaml} (83%) create mode 100644 examples/features/readme-quickstart/.agentv/providers.yaml delete mode 100644 examples/features/readme-quickstart/.agentv/targets.yaml rename examples/features/readme-quickstart/{targets.yaml => providers.yaml} (100%) create mode 100644 examples/features/scenarios/providers.yaml delete mode 100644 examples/features/scenarios/targets.yaml rename examples/features/script-grader-sdk/.agentv/{targets.yaml => providers.yaml} (73%) create mode 100644 examples/features/sdk-eval-authoring/.agentv/providers.yaml delete mode 100644 examples/features/sdk-eval-authoring/.agentv/targets.yaml create mode 100644 examples/features/sdk-python/.agentv/providers.yaml delete mode 100644 examples/features/sdk-python/.agentv/targets.yaml create mode 100644 examples/features/suite-level-input/.agentv/providers.yaml delete mode 100644 examples/features/suite-level-input/.agentv/targets.yaml create mode 100644 examples/features/tool-evaluation-plugins/.agentv/providers.yaml delete mode 100644 examples/features/tool-evaluation-plugins/.agentv/targets.yaml rename examples/features/trajectory-assertions-advanced/.agentv/{targets.yaml => providers.yaml} (66%) rename examples/features/trajectory-assertions-simple/.agentv/{targets.yaml => providers.yaml} (81%) rename examples/features/vitest-workspace-grader/.agentv/{targets.yaml => providers.yaml} (90%) create mode 100644 examples/features/workspace-artifact/.agentv/providers.yaml delete mode 100644 examples/features/workspace-artifact/.agentv/targets.yaml create mode 100644 examples/showcase/cross-repo-sync/.agentv/providers.yaml delete mode 100644 examples/showcase/cross-repo-sync/.agentv/targets.yaml create mode 100644 examples/showcase/export-screening/.agentv/providers.yaml delete mode 100644 examples/showcase/export-screening/.agentv/targets.yaml rename examples/showcase/offline-grader-benchmark/.agentv/{targets.yaml => providers.yaml} (69%) rename examples/showcase/trace-evaluation/.agentv/{targets.yaml => providers.yaml} (59%) diff --git a/README.md b/README.md index a05976d1f..3ab24fd32 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AgentV -Test AI targets on real repo tasks and measure what actually works. +Test AI providers on real repo tasks and measure what actually works. ## Why? @@ -10,18 +10,18 @@ Test AI targets on real repo tasks and measure what actually works. - **Version-controlled** — evals, judges, and results all live in Git - **Hybrid graders** — deterministic code checks + LLM-based subjective scoring - **CI/CD native** — exit codes, JSONL output, threshold flags for pipeline gating -- **Any target** — run against agents, model providers, gateways, replay targets, CLI wrappers, transcript providers, and future app or service wrappers +- **Any provider** — run against agents, model providers, gateways, replay providers, CLI wrappers, transcript providers, and future app or service wrappers ## Core Concepts - **Eval suite / tests** are the task corpus: the prompts, cases, datasets, and reusable field-local files you want to evaluate. - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval. - **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts. -- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object. -- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag. +- **Provider** is the configured system under test: an agent, model provider, gateway, replay provider, CLI wrapper, transcript provider, or future app/service wrapper. Each provider entry uses `id` for the backend/spec and optional `label` for the stable AgentV selection and result identity. +- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and provider/model names out of that tag. - **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets. - **Default test** configures inherited per-test defaults such as score `threshold`. -- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. +- **Run** is one concrete execution of a tagged eval against a resolved provider that writes portable artifacts for readers such as Dashboard, compare, and trend. ## Quick start @@ -31,12 +31,12 @@ npm install -g agentv agentv init ``` -**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags: +**2. Configure providers and graders** in `.agentv/providers.yaml` — point to the system under test and the reusable grader. Provider `id` names the backend/spec; `label` is the stable selection name used by evals and CLI flags: ```yaml -targets: - - id: local-openai - provider: openai +providers: + - id: openai + label: local-openai runtime: host config: api_format: chat @@ -44,8 +44,8 @@ targets: api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - - id: local-openai-grader - provider: openai + - id: openai + label: local-openai-grader runtime: host config: api_format: chat @@ -54,7 +54,7 @@ targets: model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" defaults: - target: local-openai + provider: local-openai grader: local-openai-grader ``` @@ -82,7 +82,8 @@ options: description: Code generation quality tags: experiment: with-skills -target: local-openai +providers: + - local-openai evaluate_options: max_concurrency: 2 @@ -112,25 +113,25 @@ tests: Plain assertion strings are short-form rubric criteria: AgentV groups them into `llm-rubric` and writes grader detail to `grading.json.component_results` for the Dashboard. Use explicit `type: llm-rubric` when you need weights, required -flags, `score_ranges`, a custom grader prompt, a grader target, or output +flags, `score_ranges`, a custom grader prompt, a grader provider, or output transforms; use string `value` for free-form rubric checks. Executable graders use `type: script`. -The target can be an eval-local object when this eval needs target settings of its own: +The provider can be an eval-local object when this eval needs provider settings of its own: ```yaml -description: Code generation quality with eval-local target settings +description: Code generation quality with eval-local provider settings tags: experiment: with-skills -target: - id: local-mini - provider: openai - runtime: host - config: - api_format: chat - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: gpt-5.4-mini +providers: + - id: openai + label: local-mini + runtime: host + config: + api_format: chat + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: gpt-5.4-mini evaluate_options: repeat: count: 2 @@ -148,7 +149,7 @@ tests: input: Write FizzBuzz in Python ``` -`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata. +`providers: [local-openai]` resolves the configured provider label from `.agentv/providers.yaml` and uses its backend, model, hooks, and provider settings. The object form above defines a full eval-local provider and must include enough provider configuration to run. AgentV records the resolved provider information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved provider metadata. Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file: @@ -179,7 +180,7 @@ agentv results compare .agentv/results//.internal/index.jsonl . ## Results -Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run. +Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `providers: [local-openai]` selects the system under test from `.agentv/providers.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and provider configuration used for the run. ```bash agentv eval evals/my-eval.eval.yaml @@ -192,7 +193,7 @@ Run bundle layout: .agentv/results/ ├── 2026-06-30T08-30-00-000Z/ # — one committed run bundle │ ├── summary.json # run rollup: metadata, pass rate, counts, cost -│ ├── fizzbuzz--a1b2c3d4/ # for one test/target row +│ ├── fizzbuzz--a1b2c3d4/ # for one test/provider row │ │ ├── summary.json # optional per-case rollup across samples │ │ ├── test/ # generated test bundle: frozen inputs for reproducibility │ │ │ ├── EVAL.yaml # resolved eval spec diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 02cf21488..8924e200d 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -42,7 +42,8 @@ export default defineAssertion(({ output }) => { const EVAL_TEMPLATES: Record string> = { default: (name: string) => `description: ${name} evaluation suite -target: default +providers: + - default tests: - id: sample-test @@ -54,7 +55,8 @@ tests: value: "well" `, rubric: (name: string) => `description: ${name} evaluation suite -target: default +providers: + - default tests: - id: sample-test @@ -78,11 +80,11 @@ const PROVIDER_TEMPLATE = `#!/usr/bin/env bun /** * Custom provider scaffold. * - * AgentV providers are configured via .agentv/targets.yaml using the CLI provider: + * AgentV providers are configured via .agentv/providers.yaml using the CLI provider: * - * targets: - * - name: my-target - * provider: cli + * providers: + * - id: cli + * label: my-provider * command: "bun run .agentv/providers/.ts {PROMPT}" * * This script receives the prompt as a CLI argument and prints the response to stdout. @@ -168,7 +170,7 @@ export const createProviderCommand = command({ await writeFile(filePath, PROVIDER_TEMPLATE); console.log(`Created ${path.relative(process.cwd(), filePath)} (template: ${templateName})`); console.log( - `\nConfigure in .agentv/targets.yaml:\n targets:\n - name: ${name}\n provider: cli\n command: "bun run .agentv/providers/${name}.ts {PROMPT}"`, + `\nConfigure in .agentv/providers.yaml:\n providers:\n - id: cli\n label: ${name}\n command: "bun run .agentv/providers/${name}.ts {PROMPT}"`, ); }, }); diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index 39816d153..44c43399c 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -67,9 +67,9 @@ function ensureTargetGraph( .map((entry) => entry.name) .sort() .join(', '); - const owner = requestedBy ? ` referenced by target '${requestedBy}'` : ''; + const owner = requestedBy ? ` referenced by provider '${requestedBy}'` : ''; throw new Error( - `Target '${name}'${owner} not found in ${targetsFilePath}. Available targets: ${available}`, + `Provider '${name}'${owner} not found in ${targetsFilePath}. Available providers: ${available}`, ); } seen.add(name); @@ -117,7 +117,7 @@ function definitionsWithEvalTargetSpec( if (!base) { const available = definitions.map((definition) => definition.name).join(', '); throw new Error( - `Target '${targetSpec.extends}' not found for eval-local target '${targetSpec.name}'. Available targets: ${available}`, + `Provider '${targetSpec.extends}' not found for eval-local provider '${targetSpec.name}'. Available providers: ${available}`, ); } const effective = { @@ -161,15 +161,37 @@ export const evalBundleCommand = command({ target: multioption({ type: array(string), long: 'target', - description: 'Target name to bundle (repeatable). Defaults to eval target(s) or default.', + description: '[Removed: use --provider