diff --git a/README.md b/README.md index a05976d1f..3ab24fd32 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AgentV -Test AI targets on real repo tasks and measure what actually works. +Test AI providers on real repo tasks and measure what actually works. ## Why? @@ -10,18 +10,18 @@ Test AI targets on real repo tasks and measure what actually works. - **Version-controlled** — evals, judges, and results all live in Git - **Hybrid graders** — deterministic code checks + LLM-based subjective scoring - **CI/CD native** — exit codes, JSONL output, threshold flags for pipeline gating -- **Any target** — run against agents, model providers, gateways, replay targets, CLI wrappers, transcript providers, and future app or service wrappers +- **Any provider** — run against agents, model providers, gateways, replay providers, CLI wrappers, transcript providers, and future app or service wrappers ## Core Concepts - **Eval suite / tests** are the task corpus: the prompts, cases, datasets, and reusable field-local files you want to evaluate. - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval. - **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts. -- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object. -- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag. +- **Provider** is the configured system under test: an agent, model provider, gateway, replay provider, CLI wrapper, transcript provider, or future app/service wrapper. Each provider entry uses `id` for the backend/spec and optional `label` for the stable AgentV selection and result identity. +- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and provider/model names out of that tag. - **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets. - **Default test** configures inherited per-test defaults such as score `threshold`. -- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. +- **Run** is one concrete execution of a tagged eval against a resolved provider that writes portable artifacts for readers such as Dashboard, compare, and trend. ## Quick start @@ -31,12 +31,12 @@ npm install -g agentv agentv init ``` -**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags: +**2. Configure providers and graders** in `.agentv/providers.yaml` — point to the system under test and the reusable grader. Provider `id` names the backend/spec; `label` is the stable selection name used by evals and CLI flags: ```yaml -targets: - - id: local-openai - provider: openai +providers: + - id: openai + label: local-openai runtime: host config: api_format: chat @@ -44,8 +44,8 @@ targets: api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - - id: local-openai-grader - provider: openai + - id: openai + label: local-openai-grader runtime: host config: api_format: chat @@ -54,7 +54,7 @@ targets: model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" defaults: - target: local-openai + provider: local-openai grader: local-openai-grader ``` @@ -82,7 +82,8 @@ options: description: Code generation quality tags: experiment: with-skills -target: local-openai +providers: + - local-openai evaluate_options: max_concurrency: 2 @@ -112,25 +113,25 @@ tests: Plain assertion strings are short-form rubric criteria: AgentV groups them into `llm-rubric` and writes grader detail to `grading.json.component_results` for the Dashboard. Use explicit `type: llm-rubric` when you need weights, required -flags, `score_ranges`, a custom grader prompt, a grader target, or output +flags, `score_ranges`, a custom grader prompt, a grader provider, or output transforms; use string `value` for free-form rubric checks. Executable graders use `type: script`. -The target can be an eval-local object when this eval needs target settings of its own: +The provider can be an eval-local object when this eval needs provider settings of its own: ```yaml -description: Code generation quality with eval-local target settings +description: Code generation quality with eval-local provider settings tags: experiment: with-skills -target: - id: local-mini - provider: openai - runtime: host - config: - api_format: chat - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: gpt-5.4-mini +providers: + - id: openai + label: local-mini + runtime: host + config: + api_format: chat + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: gpt-5.4-mini evaluate_options: repeat: count: 2 @@ -148,7 +149,7 @@ tests: input: Write FizzBuzz in Python ``` -`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata. +`providers: [local-openai]` resolves the configured provider label from `.agentv/providers.yaml` and uses its backend, model, hooks, and provider settings. The object form above defines a full eval-local provider and must include enough provider configuration to run. AgentV records the resolved provider information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved provider metadata. Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file: @@ -179,7 +180,7 @@ agentv results compare .agentv/results//.internal/index.jsonl . ## Results -Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run. +Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `providers: [local-openai]` selects the system under test from `.agentv/providers.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and provider configuration used for the run. ```bash agentv eval evals/my-eval.eval.yaml @@ -192,7 +193,7 @@ Run bundle layout: .agentv/results/ ├── 2026-06-30T08-30-00-000Z/ # — one committed run bundle │ ├── summary.json # run rollup: metadata, pass rate, counts, cost -│ ├── fizzbuzz--a1b2c3d4/ # for one test/target row +│ ├── fizzbuzz--a1b2c3d4/ # for one test/provider row │ │ ├── summary.json # optional per-case rollup across samples │ │ ├── test/ # generated test bundle: frozen inputs for reproducibility │ │ │ ├── EVAL.yaml # resolved eval spec diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 02cf21488..8924e200d 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -42,7 +42,8 @@ export default defineAssertion(({ output }) => { const EVAL_TEMPLATES: Record string> = { default: (name: string) => `description: ${name} evaluation suite -target: default +providers: + - default tests: - id: sample-test @@ -54,7 +55,8 @@ tests: value: "well" `, rubric: (name: string) => `description: ${name} evaluation suite -target: default +providers: + - default tests: - id: sample-test @@ -78,11 +80,11 @@ const PROVIDER_TEMPLATE = `#!/usr/bin/env bun /** * Custom provider scaffold. * - * AgentV providers are configured via .agentv/targets.yaml using the CLI provider: + * AgentV providers are configured via .agentv/providers.yaml using the CLI provider: * - * targets: - * - name: my-target - * provider: cli + * providers: + * - id: cli + * label: my-provider * command: "bun run .agentv/providers/.ts {PROMPT}" * * This script receives the prompt as a CLI argument and prints the response to stdout. @@ -168,7 +170,7 @@ export const createProviderCommand = command({ await writeFile(filePath, PROVIDER_TEMPLATE); console.log(`Created ${path.relative(process.cwd(), filePath)} (template: ${templateName})`); console.log( - `\nConfigure in .agentv/targets.yaml:\n targets:\n - name: ${name}\n provider: cli\n command: "bun run .agentv/providers/${name}.ts {PROMPT}"`, + `\nConfigure in .agentv/providers.yaml:\n providers:\n - id: cli\n label: ${name}\n command: "bun run .agentv/providers/${name}.ts {PROMPT}"`, ); }, }); diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index 39816d153..44c43399c 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -67,9 +67,9 @@ function ensureTargetGraph( .map((entry) => entry.name) .sort() .join(', '); - const owner = requestedBy ? ` referenced by target '${requestedBy}'` : ''; + const owner = requestedBy ? ` referenced by provider '${requestedBy}'` : ''; throw new Error( - `Target '${name}'${owner} not found in ${targetsFilePath}. Available targets: ${available}`, + `Provider '${name}'${owner} not found in ${targetsFilePath}. Available providers: ${available}`, ); } seen.add(name); @@ -117,7 +117,7 @@ function definitionsWithEvalTargetSpec( if (!base) { const available = definitions.map((definition) => definition.name).join(', '); throw new Error( - `Target '${targetSpec.extends}' not found for eval-local target '${targetSpec.name}'. Available targets: ${available}`, + `Provider '${targetSpec.extends}' not found for eval-local provider '${targetSpec.name}'. Available providers: ${available}`, ); } const effective = { @@ -161,15 +161,37 @@ export const evalBundleCommand = command({ target: multioption({ type: array(string), long: 'target', - description: 'Target name to bundle (repeatable). Defaults to eval target(s) or default.', + description: '[Removed: use --provider