From 0868438c65d9a9498bdbe9319996feabd1f06777 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 00:13:20 +0200 Subject: [PATCH 1/2] docs(environment): teach environment recipe vocabulary --- README.md | 28 +- .../docs/docs/next/evaluation/eval-cases.mdx | 60 ++- .../docs/docs/next/evaluation/eval-files.mdx | 47 +- .../docs/docs/next/evaluation/experiments.mdx | 10 +- .../docs/next/evaluation/running-evals.mdx | 11 +- .../docs/docs/next/graders/script-graders.mdx | 10 +- .../docs/next/guides/benchmark-provenance.mdx | 252 +++++----- .../docs/docs/next/guides/eval-authoring.mdx | 75 +-- .../guides/skill-improvement-workflow.mdx | 2 +- .../next/guides/workspace-architecture.mdx | 436 ++++++------------ .../next/integrations/agent-skills-evals.mdx | 3 +- .../docs/next/reference/promptfoo-parity.mdx | 31 +- .../docs/docs/next/targets/configuration.mdx | 143 +++--- .../content/docs/docs/next/tools/convert.mdx | 2 +- .../content/docs/docs/next/tools/import.mdx | 8 +- .../content/docs/docs/next/tools/prepare.mdx | 2 +- examples/README.md | 2 +- examples/features/README.md | 6 +- .../evals/docker-example.EVAL.yaml | 2 +- .../features/repo-lifecycle/evals/suite.yaml | 4 +- .../scripts/workspace-setup.mjs | 4 +- .../{workspace.yaml => environment.yaml} | 0 .../evals/accuracy/suite.yaml | 4 +- .../evals/regression/suite.yaml | 4 +- skills-data/agentv-bench/SKILL.md | 28 +- .../agentv-bench/references/eval-yaml-spec.md | 25 +- .../migrating-from-skill-creator.md | 2 +- skills-data/agentv-eval-writer/SKILL.md | 45 +- 28 files changed, 570 insertions(+), 676 deletions(-) rename examples/features/workspace-shared-config/{workspace.yaml => environment.yaml} (100%) diff --git a/README.md b/README.md index 262127dc9..806c38cb5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Test AI targets on real repo tasks and measure what actually works. ## Why? - **Local-first** — runs on your machine, no cloud accounts or API keys for eval infrastructure -- **Repo-backed workspaces** — reuse real repos, setup scripts, and existing harnesses instead of rebuilding synthetic tasks +- **Repo-backed environments** — reuse real repos, setup scripts, Docker images, and existing harnesses instead of rebuilding synthetic tasks - **Portable artifacts** — results, traces, and reports are saved in a durable format other tools can consume - **Version-controlled** — evals, judges, and results all live in Git - **Hybrid graders** — deterministic code checks + LLM-based subjective scoring @@ -14,9 +14,9 @@ Test AI targets on real repo tasks and measure what actually works. ## Core Concepts -- **Eval suite / imports / tests** are the task corpus: the prompts, cases, datasets, and imported benchmarks you want to evaluate. +- **Eval suite / tests** are the task corpus: the prompts, cases, datasets, and reusable field-local files you want to evaluate. - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval. -- **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts. +- **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts. - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object. - **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag. - **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets. @@ -259,15 +259,19 @@ export default defineEval({ }, threshold: 0.8, prompts: ['{{ input }}'], - workspace: { - scope: 'attempt', - repos: [ - { - path: './fixture', - repo: 'EntityProcess/agentv-contract-fixture', - commit: '21a34daed7ebcfe36cbed053607622a55e5e94cb', - }, - ], + environment: { + type: 'host', + workdir: './fixture', + setup: { + command: [ + 'bash', + './scripts/materialize-repo.sh', + './fixture', + 'EntityProcess/agentv-contract-fixture', + '21a34daed7ebcfe36cbed053607622a55e5e94cb', + ], + cwd: '.', + }, }, tests: [ { diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx index 079da54ca..2f65beea3 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx @@ -30,8 +30,8 @@ tests: | `expected_output` | No | Passive gold/reference data available to graders (string, object, or message array) | | `assert` | Yes | Per-test graders; plain strings become `llm-rubric` rubric checks | | `execution` | No | Per-case grader/default overrides such as `skip_defaults`; target selection belongs in top-level `target` or CLI `--target` | -| `workspace` | No | Per-case workspace config (overrides suite-level) | -| `metadata` | No | Arbitrary key-value pairs passed to graders and workspace scripts | +| `environment` | No | Per-case environment recipe (overrides suite-level) | +| `metadata` | No | Arbitrary key-value pairs passed to graders, setup commands, and lifecycle extensions | ## Input @@ -145,34 +145,40 @@ tests: # Does NOT get latency_check ``` -## Per-Case Workspace Config +## Per-Case Environment Config -Override the suite-level workspace config for individual tests. Test-level fields replace suite-level fields: +Override the suite-level environment recipe for individual tests. Test-level +environment fields replace suite-level fields: ```yaml -workspace: - hooks: - before_all: - command: ["bun", "run", "default-setup.ts"] +environment: + type: host + workdir: ./workspaces/default + setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." tests: - id: case-1 input: Do something assert: - Completes the requested task - workspace: - hooks: - before_all: - command: ["bun", "run", "custom-setup.ts"] + environment: + type: host + workdir: ./workspaces/case-1 + setup: + command: ["bash", "-lc", "bun install && bun run build && bun run setup:case-1"] + cwd: "." - id: case-2 input: Do something else assert: - Completes the requested task - # Inherits suite-level hooks.before_all + # Inherits suite-level environment ``` -See [Workspace Lifecycle Hooks](/docs/targets/configuration/#workspace-lifecycle-hooks) for the full workspace config reference. +See [Environment Recipes](/docs/guides/workspace-architecture/) for the full +environment config reference. ## Per-Case Metadata @@ -186,17 +192,25 @@ tests: source_repo: sympy/sympy source_commit: "abc123def" test_patch: cases/sympy-20590/test.patch - workspace: - repos: - - path: ./repo - repo: sympy/sympy - commit: "abc123def" - hooks: - before_each: - command: ["python", "apply_test_patch.py"] ``` -The `metadata` field is included in the stdin JSON passed to lifecycle commands as `case_metadata`. +The `metadata` field is included in the stdin JSON passed to lifecycle +extensions as `case_metadata`. Use an `environment` recipe to materialize the +checkout, then use a `beforeEach` extension when per-case metadata needs to +drive patch application or fixture selection: + +```yaml +environment: + type: host + workdir: ./repo + setup: + command: ["bash", "./scripts/materialize-repo.sh", "./repo", "sympy/sympy", "abc123def"] + cwd: "." + +extensions: + - file://scripts/apply-test-patch.py:beforeEach +``` + Operational checkout state belongs in `environment`; matching metadata fields such as `source_commit` are informational only. For historical repo-state evals, pin the checkout in an environment setup recipe instead of only mentioning the diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index bde152e6a..ff94fd061 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -547,11 +547,13 @@ vars: - **Directory name as `id`:** If `case.yaml` doesn't specify an `id`, the directory name is used (e.g., `fix-null-check`) - **Alphabetical ordering:** Subdirectories are sorted alphabetically for deterministic order -- **Per-case workspace:** A `workspace/` subdirectory inside the case directory automatically sets `workspace.template` to that path, unless the case already defines a `workspace` field +- **Per-case files:** Put bulky case-local prompts, patches, fixtures, and oracle files beside `case.yaml`; reference them with `type: file` blocks or read them from lifecycle extensions via `case_metadata` - **Skipped directories:** Subdirectories without `case.yaml` are skipped with a warning -- **Suite-level config applies:** Suite-level `assert`, `prompts`, `workspace`, `target`, and top-level run controls still apply to directory-discovered cases +- **Suite-level config applies:** Suite-level `assert`, `prompts`, `environment`, `target`, and top-level run controls still apply to directory-discovered cases -This pattern is useful for benchmarks with many cases, where each case benefits from its own directory for workspace templates, supporting files, or documentation. +This pattern is useful for benchmarks with many cases, where each case benefits +from its own directory for supporting files, patches, fixtures, or +documentation. For guidance on keeping provenance metadata, patches, oracle files, and generated dataset rows out of oversized inline YAML, see [Benchmark Provenance](/docs/guides/benchmark-provenance/). @@ -560,11 +562,20 @@ dataset rows out of oversized inline YAML, see [Benchmark Provenance](/docs/guid All string fields in eval files support `{{ env.VAR }}` syntax for environment variable interpolation. This enables portable eval configs that work across machines and CI environments without hardcoded paths. ```yaml -workspace: - repos: - - path: ./RepoA - repo: "{{ env.REPO_A_URL }}" - commit: "{{ env.REPO_A_COMMIT }}" +environment: + type: host + workdir: ./workspaces/repo-a + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/repo-a + - "{{ env.REPO_A_URL }}" + - "{{ env.REPO_A_COMMIT }}" + cwd: "." + +env: + OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}" prompts: - "{{ prompt }}" @@ -583,17 +594,23 @@ tests: - **Partial interpolation** is supported: `{{ env.HOME }}/repos/{{ env.PROJECT }}` becomes `/home/user/repos/myproject` - **Non-string values** (numbers, booleans) are not affected - Interpolation is applied recursively to all nested objects and arrays -- Works in YAML eval files, external YAML/JSONL case files, and external workspace config files +- Works in YAML eval files, external YAML/JSONL case files, and external environment recipe files - `.env` files in the directory hierarchy are loaded automatically before interpolation -### Example: Portable Workspace Config +### Example: Portable Environment Recipe ```yaml -# workspace.yaml — works on any machine -repos: - - path: ./my-repo - repo: "{{ env.MY_REPO_URL }}" - commit: "{{ env.MY_REPO_COMMIT }}" +# .agentv/environments/repo.yaml — works on any machine +type: host +workdir: ./workspaces/my-repo +setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/my-repo + - "{{ env.MY_REPO_URL }}" + - "{{ env.MY_REPO_COMMIT }}" + cwd: "." ``` ```bash diff --git a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx index 2d224e8a9..33ff52244 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx @@ -30,10 +30,12 @@ evaluate_options: budget_usd: 2.00 max_concurrency: 3 -workspace: - hooks: - before_all: - command: ["bash", "-lc", "bun install && bun run build"] +environment: + type: host + workdir: ./workspaces/support-codex + setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." tests: - id: refund-eligibility diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 5ce9b9cb7..1ba49e3f4 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -240,11 +240,12 @@ env, Docker, and folder isolation. Use `extensions` for lifecycle function hooks extensions: - file://scripts/setup.mjs:beforeAll -workspace: - scope: suite # suite | attempt - hooks: - after_each: - reset: fast # none | fast | strict +environment: + type: host + workdir: ./workspaces/my-eval + setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." ``` Notes: diff --git a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx index 0524737f0..c1c71dbed 100644 --- a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -385,7 +385,8 @@ Use `expected_output` for reference answers and `output` for the actual final an ## Workspace Access -When `workspace` is configured in the eval YAML (via `workspace.template`, `workspace.repos`, or lifecycle hooks), script graders receive the prepared workspace path in two ways: +When an `environment` recipe prepares a coding-agent testbed, script graders +receive the prepared workspace path in two ways: 1. **JSON payload**: `workspace_path` field in the stdin input 2. **Environment variable**: `AGENTV_WORKSPACE_PATH` @@ -396,7 +397,7 @@ This enables **functional grading** — running commands like `npm test`, `pytes `file_changes` is a unified diff built from two sources, merged in order: -1. **Git baseline**: `git diff` against a baseline commit taken before the agent ran. Captures edits, new files at workspace root, and changes inside any nested git repos materialized via `workspace.repos` or set up via a `before_all` hook. +1. **Git baseline**: `git diff` against a baseline commit taken before the agent ran. Captures edits, new files at workspace root, and changes inside any nested git repos materialized by `environment.setup.command` or set up via a lifecycle hook. 2. **Provider-reported artifacts**: Copilot providers scan their session-state `files/` directory after each run and append those as synthetic diffs. This surfaces files the agent wrote *outside* `workspace_path` entirely (e.g. `~/.copilot/session-state//files/`). ### Example: Deploy-and-Test Pattern @@ -440,8 +441,9 @@ console.log(JSON.stringify({ ```yaml # suite.yaml -workspace: - template: ./workspace-template # copied into a temp dir before each run +environment: + type: host + workdir: ./workspace-template target: my_agent diff --git a/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx index bb3fd1148..e89679257 100644 --- a/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx @@ -1,6 +1,6 @@ --- title: Benchmark Provenance -description: Patterns for source pins, task artifacts, hooks, and generated benchmark metadata. +description: Patterns for source pins, task artifacts, environment recipes, and generated benchmark metadata. sidebar: order: 5 slug: docs/guides/benchmark-provenance @@ -10,37 +10,40 @@ Benchmark suites usually need more than a prompt and a score. They carry source pins, task patches, generated dataset rows, oracle data, setup scripts, and verification commands. AgentV represents that with existing primitives: -- Put runtime behavior in `workspace`, `experiment`, `input`, `expected_output`, - and `assert`. +- Put prompt matrices in `prompts`, `tests[].vars`, `default_test`, and + `assert`. +- Put coding-agent testbed setup in `environment`. +- Put lifecycle callbacks and instrumentation in `extensions`. +- Put provider/eval variables in top-level `env`. - Put provenance and classification in per-case `metadata`. - Put bulky per-case authoring inputs in optional case directories and supporting files. - Use generated run folders, not hand-authored source bundles, as the portable audit artifact. These are documentation patterns, not special runtime schema keys. AgentV does -not interpret keys such as `source_commit`, `test_patch`, or `question_type` -unless your hook or custom assertion reads them. +not interpret metadata keys such as `source_commit`, `test_patch`, or +`question_type` unless your setup command, lifecycle extension, or custom +assertion reads them. -## Operational vs Informational Fields +## Operational Vs Informational Fields Use this split when deciding where a benchmark key belongs: | Field area | Operational? | What AgentV does | -|------------|--------------|------------------| -| `workspace.repos[]` | Yes | Declares repo identity and checkout refs; AgentV resolves acquisition and materializes the checkout. | -| `workspace.template` | Yes | Copies a workspace template into the run workspace. | -| `extensions` | Yes | Runs lifecycle setup after `workspace.template` and `workspace.repos` materialize. | -| `workspace.hooks.after_each.reset` | Yes | Controls workspace reset policy after each case. | -| `workspace.scope` | Yes | Controls suite vs per-attempt workspace lifetime. Runtime workspace paths are machine-local config/CLI bindings, not benchmark provenance. | -| `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, and default grader behavior. Authored concurrency uses `evaluate_options.max_concurrency`; `--workers` is the operator override. | -| `input`, `input_files`, `expected_output` | Yes | Builds the target prompt and carries passive gold/reference data for graders. | +| --- | --- | --- | +| `environment` | Yes | Prepares the host or Docker testbed, including workdir, setup argv, fixtures, services, and repository checkout scripts. | +| `environment: file://...` | Yes | Loads a reusable environment recipe from a field-local file reference. | +| `environment.setup.command` | Yes | Runs the argv command that materializes repo state, installs dependencies, builds fixtures, or prepares services. | +| `extensions` | Yes | Runs Promptfoo-style lifecycle callbacks after the environment is prepared. | +| Top-level `env` | Yes | Supplies provider/eval variables and template inputs. | +| `prompts`, `tests`, `vars`, `default_test` | Yes | Builds the authored matrix of prompts, cases, and row data. | | `assert` | Yes | Runs deterministic, LLM, assert-set, or script graders. | | Top-level `name`, `version`, `tags`, `license`, `requires` | Informational | Identifies and categorizes the suite. | -| `tests[].metadata` | Informational to AgentV | Passes arbitrary case data through to results and extension context; in-process custom assertions can also read it. | +| `tests[].metadata` | Informational to AgentV | Passes arbitrary case data through to results, setup payloads, and extension context. | -`metadata` can still become operational inside your own lifecycle extensions. For -example, a `beforeEach` extension can read `case_metadata.test_patch` and apply that -patch before the agent starts. The distinction is that AgentV itself only passes -the metadata along; the extension owns the behavior. +`metadata` can still become operational inside your own setup. For example, a +`beforeEach` extension can read `case_metadata.test_patch` and apply that patch +before the target starts. The distinction is that AgentV itself only passes the +metadata along; your extension owns the behavior. ## Extension Context @@ -61,8 +64,8 @@ the current test's metadata as `case_metadata`: } ``` -`beforeAll` runs once for the shared workspace after repo materialization, so it -should do suite setup only. Use `beforeEach` when setup depends on per-case +`beforeAll` runs after the authored `environment` is prepared, so it should do +suite lifecycle work only. Use `beforeEach` when setup depends on per-case metadata such as a patch path, source row, or selected test list. ## Task Artifact Anatomy @@ -70,44 +73,47 @@ metadata such as a patch path, source row, or selected test list. Benchmark task packs map cleanly onto AgentV fields at authoring time: | Task artifact | AgentV pattern | -|---------------|----------------| -| Prompt or instruction | `input`, usually with `type: file` blocks for long prompts | -| Source checkout | `workspace.repos[].repo` and `workspace.repos[].commit` | +| --- | --- | +| Prompt or instruction | `prompts` rendered with `tests[].vars` | +| Source checkout | `environment.setup.command` materializes the checkout from argv inputs | | Per-case setup | `extensions: ["file://scripts/setup.mjs:beforeEach"]` reading `case_metadata` | | Gold answer or reference context | `expected_output` when the data is passive grader context | | Active verification | `assert`, especially `script` for commands or artifact checks | | Provenance | `tests[].metadata` with source pins, generator rows, and curation labels | -| Bulky task files | Optional `tests: ./cases/` with per-case directories and supporting files | +| Bulky task files | `tests: file://./cases.yaml` or directory cases with supporting files | Use this separation only when it makes the source eval easier to maintain. It is -not a first-class artifact schema. After an eval runs, AgentV writes the portable -audit surface into the generated run folder: each result can link from +not a first-class artifact schema. After an eval runs, AgentV writes the +portable audit surface into the generated run folder: each result can link from `index.jsonl` to a run-local `test/` bundle containing `EVAL.yaml`, `targets.yaml`, and copied `files/` or `graders/` snapshots where applicable. -Review, Dashboard files views, and rerun workflows should inspect those generated -run artifacts instead of requiring authors to maintain a parallel source-side -bundle layout. See [Generated Test Bundles](/docs/evaluation/running-evals/#generated-test-bundles). +Review, Dashboard files views, and rerun workflows should inspect those +generated run artifacts instead of requiring authors to maintain a parallel +source-side bundle layout. See [Generated Test Bundles](/docs/evaluation/running-evals/#generated-test-bundles). ## SWE-Style Case A SWE-style benchmark usually needs a source repo, a commit pin, a patch that adds or selects tests, and a list of failing tests that should pass after the -agent's fix. Keep the checkout operational under `workspace.repos`; keep the -benchmark provenance and per-case test selectors in `metadata`. +agent's fix. Keep the checkout operational under `environment`; keep benchmark +provenance and per-case test selectors in `metadata`. ```yaml name: swe-style-regression description: Regression tasks against pinned source commits. -workspace: - scope: attempt - repos: - - path: ./repo - repo: https://github.com/example/widget.git - commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 - hooks: - after_each: - reset: strict +environment: + type: host + workdir: ./workspaces/widget + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/widget + - https://github.com/example/widget.git + - 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 + cwd: "." + timeout_ms: 120000 extensions: - file://scripts/apply-test-patch.mjs:beforeEach @@ -118,12 +124,15 @@ assert: command: ["python", "./graders/run-focused-tests.py"] required: true +prompts: + - "{{ task }}" + tests: - id: widget-1234 - criteria: Fix the widget parser regression without breaking existing behavior. - input: | - Work in repo/. Fix the parser regression described by the failing tests. - Do not change unrelated public APIs. + vars: + task: | + Work in the prepared widget checkout. Fix the parser regression + described by the failing tests. Do not change unrelated public APIs. metadata: repo_url: https://github.com/example/widget.git source_commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 @@ -133,51 +142,56 @@ tests: - tests/parser.test.ts::preserves-widget-id ``` -In this example, `workspace.repos[].commit` is the actual checkout. The -matching `metadata.source_commit` is audit data that gets recorded with the case -and is available to extensions. `apply-test-patch.mjs` can read +In this example, the setup argv is the operational checkout instruction. The +matching `metadata.source_commit` is audit data that gets recorded with the +case and is available to extensions. `apply-test-patch.mjs` can read `case_metadata.test_patch` and `case_metadata.fail_to_pass_tests`, then apply -the patch and write the selected test list into the workspace. The script grader -can read that workspace file through its `workspace_path` payload. Repo -acquisition remains outside the eval; use registered projects or -`git_cache.mirrors` when a local machine needs faster large-repo setup. See -[Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). +the patch and write the selected test list into the prepared workdir. The +script grader can read that state through its `workspace_path` payload. See +[Environment Recipes](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). -## Native AgentV vs Harbor-backed Benchmarks +## Native AgentV Vs Harbor-Backed Benchmarks -Use native AgentV workspaces for repo-backed evals where AgentV should own the -run lifecycle: materialize generic repos, run targets, execute hooks and graders, -gate CI, and write AgentV result bundles. This fits custom internal suites, -target comparisons, narrow regression suites, and CI checks built from AgentV -primitives. +Use native AgentV environment recipes for repo-backed evals where AgentV should +own the run lifecycle: prepare host or Docker testbeds, run targets, execute +extensions and graders, gate CI, and write AgentV result bundles. This fits +custom internal suites, target comparisons, narrow regression suites, and CI +checks built from AgentV primitives. ```yaml name: repo-regressions -workspace: - scope: attempt - repos: - - path: ./repo - repo: https://github.com/example/widget.git - commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 +environment: file://.agentv/environments/widget.yaml + +targets: + - id: codex-host + provider: codex-cli + runtime: host + - id: claude-host + provider: claude-cli + runtime: host extensions: - file://scripts/apply-case-fixtures.mjs:beforeEach -target: codex +prompts: + - "{{ task }}" -assert: - - metric: tests-pass - type: script - command: ["python", "./graders/run-tests.py"] - required: true +tests: + - id: parser-regression + vars: + task: Fix the parser regression in the prepared checkout. + assert: + - type: script + command: ["python", "./graders/run-tests.py"] + required: true ``` Use a Harbor-backed runner for standard benchmark suites Harbor owns, such as SWE-Bench Verified, Multi-SWE-Bench, Terminal-Bench, or suites with Harbor-owned Docker and Compose adapters. In that path AgentV should stay at the orchestration boundary: launch or import the Harbor job, apply AgentV gates to -the imported results, and link Opik traces when Harbor uploads them. +the imported results, and link external traces when the runner emits them. ```yaml # Proposed runner boundary, not a current AgentV task schema. @@ -187,39 +201,20 @@ target: codex-gpt5-mini runner: type: harbor options: - opik: - enabled: true + trace_links: true ``` Do not translate Harbor `task.toml`, verifier packaging, or suite-specific Docker/Compose adapter fields into AgentV core eval schema. If the benchmark's runtime contract is already owned by Harbor, keep those details in Harbor and -let AgentV consume the job metadata, rewards, artifacts, and trace links. -Do not add a generic top-level `source` field just to identify Harbor. If a -future Harbor adapter needs suite selection, keep that selector narrow and -adapter-owned instead of making it the AgentV workspace model. - -## Eval Composition - -When one eval references another eval, preserve the task/runtime split: - -- The parent runnable eval owns top-level `target` and run controls. -- Child suite imports preserve task context, while the parent owns the run. -- Child `workspace` setup is preserved for `type: suite` imports. A parent eval - that imports any `type: suite` entry must not define parent `workspace`. - Parent workspace context is for parent-owned raw cases, including raw cases - imported with `type: tests`. -- A tests-only import can drop child workspace context only when the import mode - says so explicitly. -- Workspace path collisions or incompatible isolation settings should fail - loudly if a future explicit remap mode is added. - -That rule keeps imported benchmark cases attached to their setup while still -letting a parent eval compare targets, repeat policy, and gates consistently. +let AgentV consume the job metadata, rewards, artifacts, and trace links. Do +not add a generic top-level `source` field just to identify Harbor. If a future +Harbor adapter needs suite selection, keep that selector narrow and +adapter-owned instead of making it the AgentV environment model. ## Finance-Style Generated Dataset -Generated datasets often need stable row provenance more than workspace setup. +Generated datasets often need stable row provenance more than repository setup. Keep the generated row identity in metadata, use `expected_output` for the gold answer, and score with rubrics or an LLM/script grader. @@ -227,6 +222,9 @@ answer, and score with rubrics or an LLM/script grader. name: finance-research-generated description: Generated finance research cases with row-level provenance. +prompts: + - "{{ question }}" + assert: - metric: answer-quality type: llm-rubric @@ -235,10 +233,10 @@ assert: tests: - id: finance-agent-row-0042 - criteria: Answer the finance question with the correct conclusion and evidence. - input: | - Research the company filing and answer: - What drove the year-over-year change in gross margin? + vars: + question: | + Research the company filing and answer: + What drove the year-over-year change in gross margin? expected_output: - role: assistant content: | @@ -254,9 +252,9 @@ tests: Here, `source_repo`, `source_commit`, `source_file`, `source_row`, and `question_type` are informational metadata. They support audits, slices, and -regeneration checks. If a hook or grader needs the source file at runtime, clone -it through `workspace.repos` or make the generator output available as a normal -fixture file. +regeneration checks. If a hook or grader needs the source file at runtime, +materialize it through `environment.setup.command` or make the generator output +available as a normal fixture file. ## Optional Source-Side Case Directories @@ -268,7 +266,7 @@ benchmark starts accumulating bulky authoring resources: fixture files. - The prompt or expected output is long enough that YAML diffs become hard to review. -- Each task needs a different workspace template or setup files. +- Each task needs different supporting setup files. - A generator emits many rows and reviewers need to inspect individual cases. - Hook and grader scripts need stable file paths for per-case resources. @@ -276,7 +274,7 @@ Use an external YAML or JSONL file for many simple generated rows: ```yaml name: generated-finance -tests: ./cases.jsonl +tests: file://./cases.jsonl ``` Use case directories when each case needs supporting files: @@ -284,35 +282,33 @@ Use case directories when each case needs supporting files: ```text swe-benchmark/ EVAL.yaml + .agentv/ + environments/ + widget.yaml cases/ widget-1234/ case.yaml prompt.md test.patch oracle.json - workspace/ - README.md ``` ```yaml # EVAL.yaml name: swe-benchmark -workspace: - repos: - - path: ./repo - repo: https://github.com/example/widget.git - commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 -tests: ./cases/ +environment: file://.agentv/environments/widget.yaml +tests: file://./cases/ ``` ```yaml # cases/widget-1234/case.yaml -criteria: Fix the widget parser regression. -input: - - role: user - content: - - type: file - value: cases/widget-1234/prompt.md +id: widget-1234 +vars: + task: + - role: user + content: + - type: file + value: cases/widget-1234/prompt.md metadata: repo_url: https://github.com/example/widget.git source_commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 @@ -321,19 +317,17 @@ metadata: ``` When `tests` points to a directory, AgentV discovers each immediate -subdirectory's `case.yaml`, uses the directory name as `id` if no `id` is set, -and automatically uses a `workspace/` subdirectory as that case's -`workspace.template`. File blocks still use the normal eval-file search roots, -so include the case directory in paths such as `cases/widget-1234/prompt.md`. -Metadata paths are not resolved by AgentV; resolve them in your hook or grader -script. +subdirectory's `case.yaml` and uses the directory name as `id` if no `id` is +set. File blocks still use the normal eval-file search roots, so include the +case directory in paths such as `cases/widget-1234/prompt.md`. Metadata paths +are not resolved by AgentV; resolve them in your hook or grader script. ## Authoring Rules -- Do not add benchmark-specific fields when `metadata` plus hooks or custom - assert entries can express the need. +- Do not add benchmark-specific fields when `metadata` plus setup commands, + hooks, or custom assert entries can express the need. - Do not duplicate operational checkout state only in metadata. Put the real - checkout under `workspace.repos`. + checkout under `environment`. - Keep `metadata` snake_case because it crosses process and result boundaries. - Prefer `expected_output` for passive gold answers and `script` for active commands, file checks, or generated artifact validation. diff --git a/apps/web/src/content/docs/docs/next/guides/eval-authoring.mdx b/apps/web/src/content/docs/docs/next/guides/eval-authoring.mdx index bd1d69a3a..d6193945e 100644 --- a/apps/web/src/content/docs/docs/next/guides/eval-authoring.mdx +++ b/apps/web/src/content/docs/docs/next/guides/eval-authoring.mdx @@ -1,6 +1,6 @@ --- title: Eval Authoring Guide -description: Practical guidance for writing workspace-based evals that work reliably across providers. +description: Practical guidance for writing environment-based evals that work reliably across providers. sidebar: order: 3 slug: docs/guides/eval-authoring @@ -10,8 +10,8 @@ slug: docs/guides/eval-authoring Use the built-in `agentv:agent-rules` extension when an eval needs to stage or expose agent-facing rules, skills, hooks, or subagents. It runs after -`workspace.template` and `workspace.repos` materialize, then writes -`agent_rules_paths` into provider context and result metadata. +the authored `environment` recipe is prepared, then writes `agent_rules_paths` +into provider context and result metadata. ```yaml extensions: @@ -22,12 +22,12 @@ extensions: agents: agent-rules/agents rules: agent-rules/AGENTS.md -workspace: - template: ./workspace-template - repos: - - path: ./app - repo: acme/app - commit: main +environment: + type: host + workdir: ./workspace-template + setup: + command: ["bash", "./scripts/materialize-repo.sh", "./app", "acme/app", "main"] + cwd: "." ``` Configured paths are resolved relative to the eval file and staged under the @@ -39,8 +39,8 @@ extensions: - agentv:agent-rules ``` -Do not move repo acquisition into `agentv:agent-rules`. Repositories remain -first-class workspace provenance through `workspace.repos`. +Do not move repo acquisition into `agentv:agent-rules`. Repositories and other +testbed setup belong in the authored `environment` recipe. ## Custom Lifecycle Setup @@ -105,27 +105,31 @@ Agents verify `git diff` against prompt claims. If your prompt says "The PR modi 1. If a prompt references specific code changes, the workspace **must** contain those exact changes 2. Or frame prompts as hypothetical: describe changes inline rather than claiming they exist in the workspace -3. Use `before_each` hooks to set up per-test git state when tests need different diffs +3. Use `beforeEach` lifecycle extensions to set up per-test git state when tests need different diffs ### Example: per-test git state ```yaml -workspace: - template: ./workspace-template - hooks: - before_each: - command: - - node - - ../scripts/apply-test-diff.mjs +environment: + type: host + workdir: ./workspace-template + +extensions: + - file://../scripts/apply-test-diff.mjs:beforeEach + +prompts: + - "{{ task }}" tests: - id: risky-change metadata: diff_file: diffs/risky-timeout-change.patch - input: "Review the current changes and assess risk." + vars: + task: Review the current changes and assess risk. ``` -The `before_each` hook reads `metadata.diff_file` from the AgentV payload and applies the patch to the workspace before each test runs. +The `beforeEach` extension reads `case_metadata.diff_file` from the AgentV +payload and applies the patch to the prepared workdir before each test runs. ### Hypothetical framing pattern @@ -150,24 +154,31 @@ When you don't want to maintain actual diffs, describe the changes inline: This avoids workspace state issues entirely — the agent evaluates the diff as presented without checking `git diff`. -## Historical Repo State: Pin the Checkout +## Historical Repo State: Pin The Checkout If a test asks the agent to inspect how a repository looked at a past commit, -declare that checkout in `workspace.repos[]`. Do not rely on prompt prose that -mentions a SHA without materializing the repo. +materialize that checkout in `environment.setup.command`. Do not rely on prompt +prose that mentions a SHA without preparing the repo. ```yaml -workspace: - repos: - - path: ./agentv - repo: https://github.com/EntityProcess/agentv.git - commit: 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215 +environment: + type: host + workdir: ./workspaces/agentv + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/agentv + - https://github.com/EntityProcess/agentv.git + - 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215 + cwd: "." tests: - id: verification-learning-capture - input: | - The eval harness has prepared ./agentv at the historical commit. - Use that checkout to decide which durable guidance should change. + vars: + input: | + The eval harness has prepared the historical AgentV checkout. + Use that checkout to decide which durable guidance should change. expected_output: | The durable repo change is to update .agents/verification.md with the reusable verification workflow lessons. diff --git a/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx b/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx index 4e4e27049..cd760d76e 100644 --- a/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx +++ b/apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx @@ -293,7 +293,7 @@ agentv pipeline run evals/my-eval.yaml --experiment with_skills Both runs use the same eval file and produce separate run directories. The experiment label is recorded in `manifest.json` and `index.jsonl`, making it easy to filter and compare in dashboards. -This replaces the need for separate `--target baseline` / `--target candidate` configurations when the only difference between runs is the workspace setup (skills, config, etc.) rather than the target harness. +This replaces the need for separate `--target baseline` / `--target candidate` configurations when the only difference between runs is the environment setup (skills, config, etc.) rather than the target harness. ## Baseline Comparison Best Practices diff --git a/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx b/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx index 985b81d20..620f535aa 100644 --- a/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx +++ b/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx @@ -1,346 +1,182 @@ --- -title: Workspace Architecture -description: How AgentV materializes eval workspaces, resolves repo acquisition, and keeps target comparisons fair. +title: Environment Recipes +description: How AgentV prepares coding-agent testbeds with host and Docker environment recipes. sidebar: order: 7 slug: docs/guides/workspace-architecture --- -AgentV workspaces are the shared substrate an eval runs against: templates, -fixtures, repositories, and lifecycle hooks. Targets run inside that substrate. -When `workspace.repos` is present, the eval declares repository identity and -checkout pins; AgentV decides how to acquire the bytes. +AgentV combines Promptfoo-compatible eval authoring with AgentV-owned +`environment` recipes for coding-agent testbeds. Use `prompts`, `tests`, +`vars`, `default_test`, `assert`, `targets`, top-level `env`, and +`extensions` for the Promptfoo-shaped eval matrix. Use `environment` for the +host or Docker state a coding agent should inspect or modify. -By default, repo workspaces are harness-managed temp workspaces. `workspace.scope` -selects the portable lifetime: `suite` creates one workspace for the run, and -`attempt` creates a clean workspace for each resolved execution attempt. Use -`--workspace-path` only when you intentionally want to point a run at an existing -machine-local directory. - -## Eval setup lifecycle - -Each evaluation run proceeds through these phases: - -``` -eval start - | - v -+---------------------------+ -| 1. Workspace setup | Create suite workspace or attempt workspace -+---------------------------+ - | - v -+---------------------------+ -| 2. Template copy | workspace.template dir -> workspace/ -+---------------------------+ - | - v -+---------------------------+ -| 3. Repo materialization | For each workspace.repos entry: -| a. resolve acquisition | - registered project, configured mirror, -| b. materialize bytes | resolver source, AgentV cache, or remote fallback -| c. checkout if Git | - check out commit or HEAD for Git sources -+---------------------------+ - | - v -+---------------------------+ -| 4. beforeAll lifecycle | extensions, then target hook -+---------------------------+ - | - v -+---------------------------+ -| 5. Test loop | For each test case: -| beforeEach -> run -> | extension, target hook, agent, -| afterEach | target hook, extension, reset -+---------------------------+ - | - v -+---------------------------+ -| 6. after_all / cleanup | target hook, workspace hook, cleanup -+---------------------------+ -``` - -With `workspace.scope: suite`, steps 2-4 run once and the test loop shares that -workspace. With `workspace.scope: attempt`, AgentV repeats setup for each -prompt-target-test-repeat attempt so mutating agents do not share filesystem -state. - -## Repo provenance vs acquisition - -A `workspace.repos[]` entry declares **identity**, not acquisition policy: +`environment` is an AgentV extension because Promptfoo does not define a typed +primitive for repo materialization, Docker images, setup commands, fixture +generation, services, and working directories. The recipe can be inline or +loaded from a field-local file reference: ```yaml -workspace: - repos: - - path: ./repo - repo: https://github.com/org/repo.git - commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 - sparse: [packages/core] - ancestor: 0 +environment: file://.agentv/environments/local-python.yaml ``` -Supported repo fields: - -| Field | Meaning | -|-------|---------| -| `path` | Directory inside the workspace where the repo is materialized | -| `repo` | Repository identity: full clone URL or GitHub `org/name` shorthand | -| `commit` | Branch, tag, or SHA to check out after clone | -| `sparse` | Optional sparse-checkout paths | -| `ancestor` | Walk N parents back after resolving `commit` | - -`commit` is the AgentV checkout pin. - -`source`, `type`, `checkout`, `checkout.resolve`, `clone`, `resolve`, and -`resolver` are not part of the repo schema. Acquisition settings are -deliberately outside eval YAML so the same benchmark can run against the same -repository identity on every machine -while each harness uses the fastest safe local source available. - -## Native workspace boundary - -Use native AgentV workspaces when AgentV owns the run lifecycle: custom internal -suites, CI gates, target comparisons, local setup hooks, Docker workspaces, and -generic repository acquisition. In that path, -`workspace.repos` declares the repos and checkout pins while AgentV materializes -the workspace, runs targets and graders, and writes AgentV run bundles. - -Use a Harbor-backed runner boundary for standard benchmark suites whose -acquisition, packaging, verifier layout, Docker or Compose adapters, and trace -export are already owned by Harbor. In that path, AgentV should launch, import, -and gate Harbor jobs and link Opik traces. It should not copy Harbor `task.toml` -or suite-specific adapter fields into AgentV core workspace schema. - -## Acquisition resolver - -AgentV normalizes `repo` identity before acquisition. For example, -`org/repo`, `https://github.com/org/repo.git`, and -`git@github.com:org/repo.git` resolve to the same identity key. - -For each materialized repo, AgentV resolves acquisition in this order: - -| Order | Source | How it is used | -|-------|--------|----------------| -| 1 | Pattern resolver | The first non-`default` `repo_resolvers[]` entry whose `repos` pattern matches the repo URL or identity. If it returns `status: "skip"`, AgentV continues to the default resolver. | -| 2 | Default resolver | The resolver named `default`, if configured. It must not declare `repos`; it is the unconditional project default. If it returns `status: "skip"`, AgentV continues to the built-in git resolver. | -| 3 | Registered project | A project in `$AGENTV_HOME/projects.yaml` whose `origin` matches the repo identity. AgentV seeds its mirror cache from that local checkout, then clones the cache into the workspace and resets `origin` to the declared repo URL. | -| 4 | Configured mirror | A path listed under `git_cache.mirrors`. AgentV seeds its mirror cache from that checkout or bare mirror, then clones the cache into the workspace. | -| 5 | Mirror cache | An AgentV-owned bare cache under `$AGENTV_DATA_DIR/git-cache/`. Cache population is locked, cloned into a temporary path, verified, and atomically renamed before use. | -| 6 | Remote clone | The normalized clone URL from the eval's `repo` field. | - -Workspace clones are independent from user-owned checkouts, configured mirrors, -and resolver source directories. AgentV does not leave Git alternates pointing -back to those sources, so preserved workspaces and pool slots keep working if a -local checkout is moved, deleted, or garbage-collected. - -### Command repo resolvers - -Use `repo_resolvers` when repo bytes come from a project-specific source that -AgentV core should not understand, such as an internal snapshot bundle. Put that -logic in a resolver script and return a local acquisition path: - ```yaml -# .agentv/config.yaml -repo_resolvers: - - name: org_snapshots - repos: - - https://github.com/example/* - command: - - bun - - scripts/eval-config/repo-resolver.ts - config: - release_tag: snapshot/v1.1.0 - - - name: default - command: - - bun - - scripts/eval-config/default-repo-resolver.ts +# .agentv/environments/local-python.yaml +type: host +workdir: ./workspaces/example +setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." + timeout_ms: 120000 ``` -AgentV sends JSON on stdin with `version`, `repo`, `commit`, `path`, `sparse`, -`ancestor`, `cache_dir`, `workspace_path`, and the resolver `config`. The -resolver writes JSON on stdout: - -```json -{ - "status": "handled", - "path": "/tmp/source.git" -} -``` - -Return `{ "status": "skip" }` when the resolver does not own that repo. -`path` can point at a Git source or a plain directory snapshot. AgentV -classifies the path at runtime. - -For Git sources, AgentV clones from `path`, applies `sparse` when present, -resolves `commit`, walks `ancestor`, and checks out the resulting commit. -Resolver stdout does not set or override `origin`; the resolver chooses -acquisition bytes only. - -For plain directory snapshots, AgentV copies the directory contents into -`workspace.repos[].path` without requiring `.git`. Because the snapshot has no -Git history, `commit`, `ancestor`, and `sparse` are ignored for that repo. -Workspace resets restore the snapshot by copying the same resolver directory -source again. Targets and graders see a normal filesystem directory at the -configured workspace path. - -### Configured mirrors - -Use `git_cache.mirrors` when you want AgentV to prefer a known local checkout or -bare mirror for a repository identity: +For containerized testbeds, use a Docker environment recipe: ```yaml -# $AGENTV_HOME/config.yaml -git_cache: - mirrors: - "https://github.com/WiseTechGlobal/CargoWise.git": ~/src/CargoWise - "sympy/sympy": /mnt/git-mirrors/sympy.git +environment: + type: docker + context: ./environment + dockerfile: Dockerfile + workdir: /app ``` -Mirror keys use the same identity normalization as `workspace.repos[].repo`, so -full URLs and GitHub `org/name` shorthand can match the same eval repo. If a -configured mirror path is missing, AgentV warns and continues down the resolver -chain. - -The mirror setting is machine-local configuration. Keep it out of eval YAML so -the eval remains a portable statement of what repository and checkout are being -tested. - -## World vs player boundary +## Vocabulary -The eval workspace is the **world**: the same repos, fixtures, template files, -and workspace hooks are shared by every target in the run. A target is the -**player**: the harness under evaluation, plus provider configuration and -target-specific setup hooks. +| Field | Meaning | +| --- | --- | +| `environment` | The AgentV testbed recipe: host or Docker placement, cwd, setup command, fixtures, services, and provenance. | +| `environment.type` | Current recipe discriminator. Supported values are `host` and `docker`. | +| `environment.workdir` | The cwd inside the prepared testbed. Host paths are local paths; Docker paths are container paths. | +| `environment.setup.command` | A non-empty argv array. Use `["bash", "-lc", "..."]` when shell behavior is required. | +| Top-level `env` | Promptfoo-compatible provider/eval variables and template inputs, such as `OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}"`. | +| `environment.env` | Recipe-scoped process environment for the host or container testbed when a recipe needs it. It is distinct from top-level `env`. | +| `extensions` | Promptfoo-style lifecycle callbacks and instrumentation. They can customize flow, but they are not the canonical testbed materialization contract. | -Targets do **not** declare `repos`. Keeping repo provenance in the shared eval -workspace is what makes multi-target comparison valid: every target sees the -same substrate, and differences in results come from the harness, not from a -different checkout. +## Targets Stay Separate -Use an eval-local target object for per-harness setup: +Targets select the system under test. The environment prepares the world those +targets run against. This separation lets Codex, Claude, Copilot, and Pi see +the same repo state, fixtures, cwd, and setup output. ```yaml -target: - extends: baseline - hooks: - before_each: - command: ["sh", "-c", "cp -R skills \"{{workspace_path}}/.claude/skills\""] -``` - -Workspace hooks run first on setup, then target hooks. Teardown runs in the -opposite order. See [Target Hooks](/docs/targets/configuration/#target-hooks) -for the command schema and full lifecycle order. +environment: file://.agentv/environments/local-python.yaml -## Windows performance guidance +targets: + - id: codex-host + provider: codex-cli + runtime: host + - id: claude-host + provider: claude-cli + runtime: host + - id: copilot-host + provider: copilot-cli + runtime: host + - id: pi-host + provider: pi-cli + runtime: host -### Drive choice affects checkout time +prompts: + - "{{ task }}" -On Windows, the drive type materially affects file-write throughput during checkout: - -| Drive type | Example path | Checkout time (large repo) | Notes | -|------------|-------------|---------------------------|-------| -| Standard NTFS (C:) | `C:\Users\\.agentv` | ~184s | Normal Defender/AV interception | -| Dev Drive (D:) | `D:\Users\\.agentv` | ~119s | ~35% faster, lower AV overhead | - -[Windows Dev Drive](https://learn.microsoft.com/en-us/windows/dev-drive/) uses the Resilient File System (ReFS) with a performance mode that reduces antivirus filter overhead for developer workloads. If you evaluate large repos frequently, relocating `~/.agentv` to a Dev Drive volume can meaningfully reduce per-run setup time. - -To relocate the AgentV home directory, set `HOME` or `USERPROFILE` to point to the Dev Drive path before running `agentv eval`: - -```powershell -$env:USERPROFILE = "D:\Users\$env:USERNAME" -agentv eval evals/my-eval.yaml +tests: + - id: fix-parser + vars: + task: Fix the parser regression in ./src/parser.ts. + assert: + - type: script + command: ["bash", "-lc", "bun test"] ``` -### Long-path support for relocated home directories - -When `HOME` or `USERPROFILE` is redirected to another drive, the Git global config (`~/.gitconfig`) also moves. If `core.longpaths=true` is not set in the new profile location, `git checkout` can fail with: +Target `runtime` describes how the provider is invoked. It does not replace the +authored `environment` recipe. -``` -error: unable to create file : Filename too long -``` +## Eval Setup Lifecycle -Set it globally in the **redirected** home: +Each run follows the same high-level order: -```bash -git config --global core.longpaths true ``` - -Or add it to the repo-level config after clone (this runs automatically if your `before_all` script includes it): - -```bash -git config core.longpaths true +eval start + | + v ++-----------------------------+ +| 1. Resolve environment | Inline object or file:// recipe ++-----------------------------+ + | + v ++-----------------------------+ +| 2. Prepare testbed | Host directory or Docker image/container context ++-----------------------------+ + | + v ++-----------------------------+ +| 3. Run environment setup | setup.command argv in setup.cwd, with timeout_ms ++-----------------------------+ + | + v ++-----------------------------+ +| 4. beforeAll lifecycle | extensions, then target hook ++-----------------------------+ + | + v ++-----------------------------+ +| 5. Test loop | beforeEach -> target -> grading -> afterEach ++-----------------------------+ + | + v ++-----------------------------+ +| 6. afterAll / cleanup | target hook, extensions, cleanup ++-----------------------------+ ``` -## Troubleshooting: eval appears stuck at startup - -Large repo setup is visible now: git clone/fetch progress streams by default, -and long-running git operations emit heartbeat messages. If an acquisition -times out, the error points to the durable fix: register a matching local -checkout, configure `git_cache.mirrors`, or fix network access. +Use `environment.setup` for installs, builds, repo checkout scripts, fixture +generation, and other authored testbed setup that should be visible in config +and provenance. Use `extensions` for lifecycle callbacks after the testbed is +prepared, such as staging agent rules or applying per-case patches from +metadata. -The old symptom where AgentV looked silent while doing a full remote clone has -been fixed. A first run can still take time, especially when a large working -tree is checked out, but the active phase should be visible in the terminal. +## Repo Provenance Vs Acquisition -### Enable verbose logging - -```bash -agentv eval evals/my-eval.yaml --verbose -``` +When a benchmark needs a real repository checkout, make that operational in the +environment recipe. Keep source pins in setup argv or in the setup script's own +inputs, and duplicate them in `tests[].metadata` only for audit and slicing. -Verbose mode logs each setup phase with timestamps. Look for: +```yaml +environment: + type: host + workdir: ./workspaces/widget + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/widget + - https://github.com/example/widget.git + - 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 + cwd: "." + timeout_ms: 120000 +tests: + - id: widget-parser + vars: + task: Fix the widget parser regression in the prepared checkout. + metadata: + repo_url: https://github.com/example/widget.git + source_commit: 4f3e2d19b6e4e8f1c2b7d9a0e5a6b7c8d9e0f123 ``` -[workspace] Creating shared workspace... -[workspace] Materializing repo ./repo... -[repo] materialize start path=./repo repo=https://github.com/org/repo.git acquisition=registered-project ... -Cloning into '.../repo'... -[repo] git clone https://github.com/org/repo.git still running after 30s -[workspace] Repo materialization complete -[workspace] Running before_all script... -[workspace] Setup complete, starting test loop -``` - -If the log shows clone or fetch progress, git is still acquiring objects. If it -shows checkout progress or a long gap after clone completes, the working-tree -write is likely the bottleneck. With pooling enabled, this usually only happens -on the first run for a given repo fingerprint. - -### Speed up large repo acquisition - -The durable fix for large repos is to make the resolver hit a local source: - -1. Register an existing checkout as an AgentV project so its `origin` matches - `workspace.repos[].repo`. -2. Or add a matching entry under `git_cache.mirrors` in - `$AGENTV_HOME/config.yaml`. - -Both paths use local Git objects for speed and full history, then dissociate the -workspace clone from user-owned storage. - -### Common causes and fixes - -| Symptom | Likely cause | Fix | -|---------|-------------|-----| -| Clone progress runs for minutes | Large repo acquired from remote | Register a matching local project or configure `git_cache.mirrors`; use `--workspace-path` only when an existing local workspace is acceptable. | -| Heartbeat ends with a clone/fetch timeout | Remote network or missing local cache | Use the timeout guidance in the error: local checkout, configured mirror, or network fix. | -| Stuck at checkout for 2+ minutes | Large repo file materialization after objects are present | Expected for 100k+ files; use Dev Drive on Windows or a local mirror. | -| `Filename too long` during checkout | Missing `core.longpaths` | `git config --global core.longpaths true` | -| Before_all timeout | Setup script exceeds default 60s | Increase `timeout_ms` in workspace config | -## Existing local workspace +`metadata.source_commit` is audit data. A SHA in metadata or prompt prose does +not give the agent an actual checkout; the environment setup must materialize +the checkout. -`--workspace-path` points AgentV at an existing local directory and skips -harness-managed materialization for that run. It is useful for local debugging, -but it is shared mutable state, not a portable benchmark contract and not a -clean per-attempt workspace. +## Existing Local Directories -To reuse an existing workspace for a run: +`--workspace-path` points AgentV at an existing local mutable directory for a +one-off run. It is useful for local debugging, but it is machine-local state, +not a portable authored testbed recipe. ```bash -agentv eval evals/my-eval.yaml --workspace-path /path/to/workspace +agentv eval evals/my-eval.yaml --workspace-path /path/to/workdir ``` -Do not commit local paths in eval YAML. Use `.agentv/config.local.yaml` when a -workspace path is part of your machine-local setup. +Do not commit local absolute paths in eval YAML. Use an `environment` file for +portable setup and keep machine-local overrides in local config. diff --git a/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx b/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx index c566fbbb7..8213ef6a5 100644 --- a/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx +++ b/apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx @@ -112,7 +112,8 @@ tests: input: "Analyze the sales data." ``` -Use `workspace.repos` when the eval should materialize a repository before those fixture paths are read. +Use `environment.setup.command` when the eval should materialize a repository +before those fixture paths are read. ## Offline Grading diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index 24a2ffc39..14163ce00 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -10,7 +10,8 @@ AgentV uses a similar eval config contract to Promptfoo for ordinary authored evals: prompt matrices, test rows, vars, default test data, assertions, and target matrices all use the same broad shape. AgentV keeps the wire format `snake_case`, keeps target identity separate from provider/backend selection, -and adds repo-native workspace and artifact fields for agent evaluation. +and adds repo-native environment recipes and artifact fields for agent +evaluation. Use this matrix when translating a Promptfoo-style normal eval into AgentV YAML. It documents which surfaces align directly, which AgentV surfaces are cleaner @@ -32,7 +33,7 @@ implements equivalent semantics directly. | Surface | Promptfoo shape | AgentV shape | Decision | Notes | | --- | --- | --- | --- | --- | | Prompt matrix | Top-level `prompts` rendered with each test's `vars`. | Top-level `prompts` rendered with `tests[].vars` and `default_test.vars`. | Align with Promptfoo | This is the canonical Promptfoo-compatible input shape in AgentV. Prompt entries can be inline strings, chat arrays, files, or generated prompt functions. | -| Test rows | `tests` can be inline rows or a case-file reference; rows carry `vars`, `assert`, metadata, prompt/provider filters, and expected data. | `tests` can be inline rows or a raw-case path; rows carry `vars`, `assert`, `expected_output`, metadata, workspace overrides, and run overrides. | Align with Promptfoo | AgentV uses field-local file refs such as `tests: file://...`, `prompts: file://...`, and `default_test: file://...`; coding-agent testbeds use `environment: file://...`. There is no separate imports table. | +| Test rows | `tests` can be inline rows or a case-file reference; rows carry `vars`, `assert`, metadata, prompt/provider filters, and expected data. | `tests` can be inline rows or a raw-case path; rows carry `vars`, `assert`, `expected_output`, metadata, optional environment overrides, and run overrides. | Align with Promptfoo | AgentV uses field-local file refs such as `tests: file://...`, `prompts: file://...`, and `default_test: file://...`; coding-agent testbeds use `environment: file://...`. There is no separate imports table. | | Variables | `tests[].vars` plus `defaultTest.vars`; prompt templates can reference top-level var names. | `tests[].vars` plus `default_test.vars`; templates can use `{{ name }}` or `{{ vars.name }}`. | Align with Promptfoo | Per-test vars override default vars by key. | | Default test | `defaultTest`, inline object or `file://` reference. | `default_test`, inline object or `file://` / `ref://` reference. | Align with Promptfoo | AgentV uses `snake_case` for YAML. Shared prompt matrix defaults belong in `default_test.vars`. | | Output transform | `defaultTest.options.transform`, `tests[].options.transform`, and assertion-level `transform`. | `default_test.options.transform`, `tests[].options.transform`, and assertion-level `transform`. | Align with Promptfoo | Use `transform` to shape provider output before grading, including file-output conversions such as `.xlsx` to text. `tests[].options.transform` overrides the inherited default transform; assertion-level `transform` is scoped to one grader. | @@ -50,10 +51,10 @@ implements equivalent semantics directly. | Script/custom grader terminology | Promptfoo custom code assertions are still assertion types. | `defineScriptGrader()` powers command-backed graders referenced with `type: script` and `command:`. | Keep AgentV divergence | Use script grader wording only for command-backed or LLM-backed scoring components that need explicit score and assertion-result control. | | Tool and trace assertions | Promptfoo includes `trajectory:tool-used`, `trajectory:tool-sequence`, `trajectory:tool-args-match`, `trajectory:step-count`, `trajectory:goal-success`, `tool-call-f1`, `skill-used`, `trace-span-count`, `trace-span-duration`, and `trace-error-spans`. | AgentV rejects those names until their semantics are implemented directly. | Defer/future-scope | These names are not aliases for AgentV's `tool-trajectory` grader. | | Tool trajectory grader | No direct Promptfoo alias for AgentV-normalized transcript semantics. | `type: tool-trajectory`. | Keep AgentV extension | This is AgentV-specific and operates over AgentV-normalized transcripts and trace summaries. | -| Coding-agent testbeds | Promptfoo normal evals do not own AgentV repository or Docker materialization. | `environment`, `environment: file://...`, `env`, and `extensions`. | Keep AgentV extension | AgentV evaluates real repositories and agent testbeds. Use `environment` for workdir/Docker/repository/setup/reset policy, top-level `env` for provider environment overrides, and `extensions` for lifecycle hooks. | +| Coding-agent testbeds | Promptfoo normal evals do not define a typed coding-agent testbed primitive. | `environment`, usually inline or `environment: file://...`, plus distinct top-level `env` and lifecycle `extensions`. | Keep AgentV extension | `environment` is an AgentV extension informed by Margin local-agent ergonomics and Harbor/Terminal-Bench Docker substrate evidence. Use it for host/Docker workdir, setup argv, fixtures, services, and repo materialization. Use top-level `env` for provider/eval variables and `extensions` for lifecycle hooks. | | Run artifacts and inspection | Promptfoo owns its own result viewer and output formats. | AgentV writes `.agentv/results//` bundles with `summary.json`, `.internal/index.jsonl`, sidecars, and local Dashboard support. | Keep AgentV extension | AgentV-owned bundles are the source of truth for compare, Dashboard, CI, and adapters. Phoenix is link-out correlation only through safe external trace metadata. | | Compare command | Promptfoo has its own result comparison surfaces. | `agentv results compare `. | Keep AgentV extension | Compare consumes completed AgentV run indexes such as `.agentv/results//.internal/index.jsonl`. | -| CLI runtime filters | Promptfoo exposes filters such as prompt/provider/test subset flags. | AgentV supports its current CLI filters and selection fields; full Promptfoo runtime-filter parity is future work. | Defer/future-scope | Prefer authored `select`/`imports` or current AgentV CLI flags until runtime-filter parity lands. | +| CLI runtime filters | Promptfoo exposes filters such as prompt/provider/test subset flags. | AgentV supports its current CLI filters and selection fields; full Promptfoo runtime-filter parity is future work. | Defer/future-scope | Prefer current AgentV CLI flags and authored selection fields until runtime-filter parity lands. | | Wire-format casing | Promptfoo config uses camelCase fields such as `defaultTest` and `evaluateOptions`. | AgentV YAML, JSONL, artifacts, and CLI JSON use `snake_case`; internal TypeScript uses `camelCase`. | Keep AgentV divergence | Translate only at process boundaries. New public wire fields should be `snake_case`. | | Hard-rejected stale AgentV fields | Not applicable to Promptfoo. | Removed AgentV-era fields such as top-level `execution`, `execution.target`, `execution.targets`, top-level `budget_usd`, top-level `repeat`/`runs`, and `composite` are rejected. | Keep AgentV divergence | Use top-level `target`/`targets`, `evaluate_options`, `evaluate_options.repeat`, and `assert-set`. Migration guidance lives in the eval migration skill reference. | @@ -96,19 +97,15 @@ tests: ```yaml description: Repo-native direct task suite -target: - id: codex-local - provider: codex-app-server - runtime: host - config: - command: ["codex", "app-server"] - -workspace: - repos: - - path: ./app - repo: acme/support-app - commit: main - scope: attempt +environment: file://.agentv/environments/local-python.yaml + +targets: + - id: codex-host + provider: codex-cli + runtime: host + - id: claude-host + provider: claude-cli + runtime: host prompts: - - role: user diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 9a5aacb55..fdd83fa67 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -247,14 +247,13 @@ target: Use `defaults.grader` for the project default grader. A specific evaluator can still choose its own grader target when the evaluator supports that override. -### Lifecycle Extensions +### Environment And Lifecycle Extensions -Run non-provisioning setup at AgentV lifecycle points using top-level -`extensions`. The harness materializes `workspace.template` and -`workspace.repos` first, then runs `beforeAll` extensions. Use extensions for -dependency installs, builds, fixture generation, and agent-rule staging. Use -target hooks for runner-specific setup. Keep repo identity and checkout pins in -`workspace.repos`; extensions must not become the default repo acquisition path. +Use `environment` for the authored coding-agent testbed recipe: host or Docker +placement, workdir, setup argv, fixtures, services, and repository +materialization scripts. Use top-level `extensions` for Promptfoo-style +lifecycle callbacks and instrumentation that run after the environment is +prepared. Use target hooks only for runner-specific setup. ```yaml extensions: @@ -267,22 +266,29 @@ extensions: skills: agent-rules/skills rules: agent-rules/AGENTS.md -workspace: - template: ./workspace-templates/my-project - hooks: - after_each: - reset: fast +environment: + type: host + workdir: ./workspaces/my-project + setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." + timeout_ms: 120000 ``` | Field | Description | |-------|-------------| -| `template` | Directory to copy as workspace | +| `environment.type` | `host` or `docker` | +| `environment.workdir` | Cwd used by targets and graders inside the prepared testbed | +| `environment.setup.command` | Non-empty argv setup command; use `["bash", "-lc", "..."]` for shell behavior | +| Top-level `env` | Provider/eval variables and template inputs | | `extensions[]` | `file://...:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, or `agentv:agent-rules` | -| `hooks.after_each.reset` | Reset mode: `none`, `fast`, `strict` | - -**Lifecycle order:** template copy → repo materialization → `extensions.beforeAll` → target `hooks.before_all` → git baseline → (`extensions.beforeEach` → target `hooks.before_each` → agent runs → file changes captured → target `hooks.after_each` → `extensions.afterEach` → `workspace.hooks.after_each.reset`) × N tests → target `hooks.after_all` → `extensions.afterAll` → cleanup -**Shared workspace:** The workspace is created once and shared across all tests in a suite. Use `hooks.after_each.reset` to reset state between tests (e.g., `fast`/`strict`). +**Lifecycle order:** resolve environment → prepare host or Docker testbed → +`environment.setup.command` → `extensions.beforeAll` → target +`hooks.before_all` → git baseline → (`extensions.beforeEach` → target +`hooks.before_each` → agent runs → file changes captured → target +`hooks.after_each` → `extensions.afterEach`) × N tests → target +`hooks.after_all` → `extensions.afterAll` → cleanup **Error handling:** - `beforeAll` / `beforeEach` extension failure aborts the affected run with an error result @@ -301,75 +307,44 @@ case context: } ``` -`workspace.hooks` is the reset-policy home for `after_each.reset`. Use -`extensions` for executable setup. +`extensions` are callbacks, not the canonical place to hide repository +materialization, Docker/image selection, cwd, or testbed provenance. Put that +contract in `environment`. -### Repository Lifecycle +### Repository Setup -Materialize git repositories into the shared eval workspace. Repo entries declare provenance only: the repository identity and checkout pin. AgentV resolves acquisition separately using registered projects, configured mirrors, its git cache, and finally remote clone. Define repos at the suite level or per test: +Materialize git repositories through the environment recipe. A setup command can +call your normal clone, checkout, fixture, or build script with explicit argv +inputs: ```yaml -workspace: - repos: - - path: ./my-repo - repo: https://github.com/org/repo.git - commit: main - ancestor: 1 # check out the parent commit - hooks: - after_each: - reset: fast # none | fast | strict - scope: suite # suite (default) | attempt +environment: + type: host + workdir: ./workspaces/my-repo + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./workspaces/my-repo + - https://github.com/org/repo.git + - main + cwd: "." + timeout_ms: 120000 ``` -`repo` declares the repository identity. Acquisition is harness-owned: AgentV first applies configured `repo_resolvers`, then uses the built-in git path of registered projects, configured mirrors, AgentV's git cache, and remote clone. See [Workspace Architecture](/docs/guides/workspace-architecture/#acquisition-resolver) for the resolver order, command resolver protocol, and `git_cache.mirrors` config. +Keep the same repo URL and commit in `tests[].metadata` only when you also need +audit fields in results or extensions. Metadata by itself is not an operational +checkout. | Field | Description | |-------|-------------| -| `repos[].path` | Directory within the workspace to clone into | -| `repos[].repo` | Repository identity: full clone URL or GitHub `org/name` shorthand | -| `repos[].commit` | Branch, tag, or SHA to check out (default: `HEAD`) | -| `repos[].ancestor` | Walk N commits back from the checked-out ref (e.g., `1` for parent) | -| `repos[].sparse` | Sparse checkout paths | -| `hooks.after_each.reset` | Reset policy after each test: `none`, `fast`, `strict` | -| `scope` | `suite` reuses one harness-managed workspace for the suite; `attempt` creates a clean workspace for each resolved execution attempt | -| `hooks.enabled` | Boolean (default: `true`). Set `false` to skip all lifecycle hooks. | - -Use `scope: attempt` when mutating agents need clean filesystem state for every prompt-target-test-repeat execution. Use `scope: suite` when the suite intentionally shares state across tests. +| `setup.command[0]` | Executable | +| `setup.command[1...]` | Arguments passed unchanged to the executable | +| `setup.cwd` | Directory for the setup command | +| `setup.timeout_ms` | Setup timeout in milliseconds | **Existing local workspaces:** do not commit local paths in eval YAML. Use `--workspace-path /path/to/workspace` for a one-off run, or put `execution.workspace_path` in `.agentv/config.local.yaml`. -Workspace command: -- `agentv workspace deps ` — scan eval files and output a JSON manifest of required git repos (for CI pre-cloning) - -**Common patterns:** - -```yaml -# Pinned commit -workspace: - repos: - - path: ./repo - repo: https://github.com/org/repo.git - commit: abc123def - -# Multi-repo shared workspace with reset -workspace: - repos: - - path: ./frontend - repo: https://github.com/org/frontend.git - - path: ./backend - repo: https://github.com/org/backend.git - hooks: - after_each: - reset: fast - -# GitHub shorthand with a pinned commit -workspace: - repos: - - path: ./repo - repo: org/repo - commit: abc123def -``` - ### Cleanup Behavior Default finish behavior: @@ -386,9 +361,14 @@ Use `cwd` on a target to run in an existing directory (shared across tests). If ## Target Hooks -Eval files can define per-target hooks that run setup/teardown scripts to customize the workspace for each target variant. This enables comparing different harness configurations (e.g., baseline vs with-plugins) in a single eval file. +Eval files can define per-target hooks that run setup/teardown scripts to +customize each target variant. This enables comparing different harness +configurations, such as baseline vs with-plugins, in a single eval file. -Targets do not declare `repos`. Repositories belong to the shared eval workspace so every target runs in the same world; target hooks customize the harness under evaluation. Use hooks for per-target setup such as enabling wrappers or changing provider-local config. Keep installs, builds, fixture generation, and case setup in top-level lifecycle `extensions`. +Targets do not declare testbed setup. The shared `environment` prepares the +world so every target runs against the same files, fixtures, cwd, and services. +Target hooks customize the harness under evaluation, such as enabling wrappers +or changing provider-local config. Target hooks can be scoped to an eval-local target object: @@ -402,22 +382,23 @@ target: ### Hook execution order -Target hooks run after workspace hooks on setup, before workspace hooks on teardown: +Target hooks run after environment setup and lifecycle extensions on setup. +Teardown runs target hooks before lifecycle extensions: 1. Extension `beforeAll` 2. **Target `before_all`** 3. For each test: - - Workspace `before_each` + - Extension `beforeEach` - **Target `before_each`** - Test executes - **Target `after_each`** - - Workspace `after_each` + - Extension `afterEach` 4. **Target `after_all`** -5. Workspace `after_all` +5. Extension `afterAll` ### Hook schema -Target hooks follow the same schema as workspace hooks: +Target hooks follow the same command schema as lifecycle hook commands: ```yaml hooks: diff --git a/apps/web/src/content/docs/docs/next/tools/convert.mdx b/apps/web/src/content/docs/docs/next/tools/convert.mdx index 7cd43ecac..c3a014a52 100644 --- a/apps/web/src/content/docs/docs/next/tools/convert.mdx +++ b/apps/web/src/content/docs/docs/next/tools/convert.mdx @@ -39,7 +39,7 @@ Converts an [Agent Skills `evals.json`](/docs/integrations/agent-skills-evals) f - Maps `assertions[]` and `expectations[]` → `llm-rubric` rubric criteria - Maps `files[]` → `input_files` - Maps `skill_name` → `tags.skill` and records adapter provenance metadata -- Adds TODO comments for AgentV-specific features (workspace setup, script graders, rubrics) +- Adds TODO comments for AgentV-specific features (environment setup, script graders, rubrics) AgentV can run detected Agent Skills `evals.json` files directly through the built-in read adapter. Use `convert` to import the definition into an editable diff --git a/apps/web/src/content/docs/docs/next/tools/import.mdx b/apps/web/src/content/docs/docs/next/tools/import.mdx index 6618526ff..2c3e129fc 100644 --- a/apps/web/src/content/docs/docs/next/tools/import.mdx +++ b/apps/web/src/content/docs/docs/next/tools/import.mdx @@ -237,8 +237,8 @@ uv run scripts/import-huggingface.py \ Each instance becomes an EVAL.yaml with: - `input` — the problem statement -- `workspace.docker.image` — the pre-built SWE-bench Docker image (`ghcr.io/epoch-research/swe-bench.eval.x86_64.:latest`) -- `workspace.repos[].commit` — the commit to reset to before the agent runs +- `environment.type: docker` and an image or Dockerfile-backed testbed recipe for the pre-built SWE-bench environment +- `environment.setup.command` argv that records the repository checkout or dataset snapshot setup - `assertions` — `script` tasks that run `FAIL_TO_PASS` and `PASS_TO_PASS` pytest suites inside the container Run an imported SWE-bench eval against any coding agent target: @@ -254,4 +254,6 @@ uv run scripts/import-huggingface.py \ agentv eval /tmp/swebench-eval/*.EVAL.yaml --target codex-cli ``` -The Docker workspace spins up the pre-built SWE-bench image, checks out the imported `commit`, runs the agent to apply a patch, then grades by running the test suite inside the container. +The Docker environment spins up the pre-built SWE-bench image, prepares the +imported checkout through setup argv, runs the agent to apply a patch, then +grades by running the test suite inside the container. diff --git a/apps/web/src/content/docs/docs/next/tools/prepare.mdx b/apps/web/src/content/docs/docs/next/tools/prepare.mdx index d14b1871d..b994c7b95 100644 --- a/apps/web/src/content/docs/docs/next/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/next/tools/prepare.mdx @@ -78,7 +78,7 @@ Use `--response` when the final answer text should be graded independently of th - For Opik, Langfuse, or another export-capable backend, treat their traces as external artifacts that can be imported or projected back into AgentV later. - For Phoenix, use only optional link-out correlation when safe `external_trace` metadata points to spans already emitted independently by Codex, Arize, or another hook. -AgentV remains responsible for eval definitions, workspace setup, grading, result bundles, and CI gates. Live trace storage, dashboards, and provider-specific run monitoring belong in the observability backend or the external harness. +AgentV remains responsible for eval definitions, environment setup, grading, result bundles, and CI gates. Live trace storage, dashboards, and provider-specific run monitoring belong in the observability backend or the external harness. There is no `agentv watch` command. diff --git a/examples/README.md b/examples/README.md index cfd0a6883..e1e6dad62 100644 --- a/examples/README.md +++ b/examples/README.md @@ -54,7 +54,7 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [compare](features/compare/) - Baseline comparison - [deterministic-graders](features/deterministic-graders/) - Deterministic assertions (contains, regex, JSON validation) - [vitest-workspace-grader](features/vitest-workspace-grader/) - Vitest-style deterministic workspace verifiers -- [workspace-setup-script](features/workspace-setup-script/) - Multi-step workspace setup with a `beforeAll` lifecycle extension +- [workspace-setup-script](features/workspace-setup-script/) - Multi-step environment setup with a `beforeAll` lifecycle extension ### SDK diff --git a/examples/features/README.md b/examples/features/README.md index eaa1f5869..a25d2e714 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -96,12 +96,12 @@ Focused examples for specific AgentV capabilities. Find your use case below, the --- -### Workspace and agent setup +### Environment and agent setup | Example | Description | |---------|-------------| | [workspace-setup-script](workspace-setup-script/) | Multi-step setup with a `beforeAll` lifecycle extension | -| [workspace-multi-repo](workspace-multi-repo/) | Multi-repo workspace using a VS Code `.code-workspace` file | -| [workspace-shared-config](workspace-shared-config/) | Define a `workspace.yaml` once and reference it across eval files | +| [workspace-multi-repo](workspace-multi-repo/) | Multi-repo environment using a VS Code `.code-workspace` file | +| [workspace-shared-config](workspace-shared-config/) | Define an environment recipe once and reference it across eval files | | [repo-lifecycle](repo-lifecycle/) | Clone a git repo into the workspace and target the agent at it | | [file-changes](file-changes/) | Capture workspace file changes made by the agent across test runs | | [file-changes-graders](file-changes-graders/) | Grade file diffs with rubrics and LLM graders | diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index ab22ff1a1..330b35fd2 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -1,5 +1,5 @@ name: docker-workspace-example -description: Example eval using Docker workspace for grading +description: Example eval using a Docker environment for grading target: mock_agent environment: type: docker diff --git a/examples/features/repo-lifecycle/evals/suite.yaml b/examples/features/repo-lifecycle/evals/suite.yaml index e4ad125c8..999608b17 100644 --- a/examples/features/repo-lifecycle/evals/suite.yaml +++ b/examples/features/repo-lifecycle/evals/suite.yaml @@ -1,5 +1,5 @@ -description: "Demonstrates workspace repo lifecycle: clone a git repo into the workspace, check out - a specific commit, and have the agent work on it." +description: "Demonstrates environment repo lifecycle: clone a git repo into the prepared workdir, + check out a specific commit, and have the agent work on it." environment: type: host workdir: ./repo diff --git a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs index 0e351e908..ab805655e 100644 --- a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs +++ b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs @@ -2,8 +2,8 @@ // // AgentV beforeAll lifecycle extension for this example. // -// It runs after workspace.template and workspace.repos materialize, then -// refreshes allagents project state inside the prepared workspace. +// It runs after the authored environment recipe is prepared, then refreshes +// allagents project state inside the prepared workspace. import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync, rmSync } from 'node:fs'; diff --git a/examples/features/workspace-shared-config/workspace.yaml b/examples/features/workspace-shared-config/environment.yaml similarity index 100% rename from examples/features/workspace-shared-config/workspace.yaml rename to examples/features/workspace-shared-config/environment.yaml diff --git a/examples/features/workspace-shared-config/evals/accuracy/suite.yaml b/examples/features/workspace-shared-config/evals/accuracy/suite.yaml index 769edfc84..2de0888e6 100644 --- a/examples/features/workspace-shared-config/evals/accuracy/suite.yaml +++ b/examples/features/workspace-shared-config/evals/accuracy/suite.yaml @@ -1,6 +1,6 @@ description: Accuracy eval that references a shared environment recipe file. The environment is - defined once in workspace.yaml and reused across eval files. -environment: file://../../workspace.yaml + defined once in environment.yaml and reused across eval files. +environment: file://../../environment.yaml tags: - agent prompts: diff --git a/examples/features/workspace-shared-config/evals/regression/suite.yaml b/examples/features/workspace-shared-config/evals/regression/suite.yaml index 38c74e265..dc10023b4 100644 --- a/examples/features/workspace-shared-config/evals/regression/suite.yaml +++ b/examples/features/workspace-shared-config/evals/regression/suite.yaml @@ -1,6 +1,6 @@ description: Regression eval that references the same shared environment recipe file. Demonstrates - environment recipe reuse across eval files in different directories. -environment: file://../../workspace.yaml +environment recipe reuse across eval files in different directories. +environment: file://../../environment.yaml tags: - agent prompts: diff --git a/skills-data/agentv-bench/SKILL.md b/skills-data/agentv-bench/SKILL.md index f2a499bf6..486b306d6 100644 --- a/skills-data/agentv-bench/SKILL.md +++ b/skills-data/agentv-bench/SKILL.md @@ -65,24 +65,28 @@ Before running or optimizing, understand what you're working with. AgentV supports two evaluation formats: -**EVAL.yaml** (native, full features) — supports workspaces, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation. +**EVAL.yaml** (native, full features) — supports environment recipes, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, and multi-provider targets. Use this for agent evaluation. ```yaml # example.eval.yaml +prompts: + - "{{ task }}" + +environment: + type: host + workdir: ./workspace-template + setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." + tests: - id: basic-code-review - input: "Review this TypeScript file for bugs and suggest improvements" - criteria: "Identifies the null pointer bug on line 12 and suggests a fix" + vars: + task: "Review this TypeScript file for bugs and suggest improvements" assert: - type: contains value: "null" - Review identifies the null pointer bug and suggests a concrete fix - -workspace: - template: ./workspace-template - hooks: - before_each: - reset: fast ``` Multi-skill evaluation is handled naturally via input messages — describe the task in the test input, and the agent uses whatever skills it needs. @@ -255,11 +259,11 @@ All artifacts use established schemas — see `references/schemas.md` for the fu Write artifacts to `.agentv/artifacts/` or the iteration directory. -### Workspace features (EVAL.yaml only) +### Environment features (EVAL.yaml only) -- **Workspace scope** — clone repos, run setup/teardown hooks (before_all, before_each, after_each, after_all), and choose `suite` or `attempt` lifetime +- **Environment recipes** — prepare host or Docker testbeds with `type`, `workdir`, and argv-only `setup.command` - **Static local override** — use `--workspace-path` only when an existing local directory should be reused as-is -- **Multi-repo** — clone multiple repos with sparse checkout and shallow clone support +- **Multi-repo** — materialize repos from setup scripts when a task needs several checkouts - **File change tracking** — grade by diffing workspace files before/after agent execution --- diff --git a/skills-data/agentv-bench/references/eval-yaml-spec.md b/skills-data/agentv-bench/references/eval-yaml-spec.md index 12c3b9de4..899ac9975 100644 --- a/skills-data/agentv-bench/references/eval-yaml-spec.md +++ b/skills-data/agentv-bench/references/eval-yaml-spec.md @@ -9,32 +9,37 @@ The grader agent uses this to evaluate assertions without the CLI. - `name` (string, optional) — eval name - `description` (string, optional) — description -- `execution` (object, optional) — `target`, `model`, etc. -- `workspace` (object, optional) — workspace config (template, repos, hooks) -- `input` (string | object | Message | Message[], optional) — suite-level input prepended to each test. String/block shorthand expands to a user message. +- `target` or `targets` (string | object | object[], optional) — system under test selection +- `environment` (object | `file://...`, optional) — AgentV coding-agent testbed recipe +- `env` (object, optional) — provider/eval environment variable overrides and template inputs +- `extensions` (array, optional) — lifecycle hooks such as `beforeAll`, `beforeEach`, `afterEach`, `afterAll` +- `prompts` (array, optional) — Promptfoo-compatible prompt matrix entries - `tests` (array, required) — test cases ### Per-test fields - `id` (string, required) — unique test identifier -- `input` (string | object | Message | Message[], required) — task input. String shorthand expands to `[{role: user, content: "..."}]`; object shorthand preserves structured user content when the object has no top-level `role`. Top-level `role` is reserved for message objects. +- `vars` (object, required when prompts need row data) — prompt-template variables for this row - `expected_output` (string | Message[], optional) — passive reference answer. String shorthand expands to `[{role: assistant, content: "..."}]`. It is available to declared graders, but does not add an implicit grader when `assertions` is present. - `criteria` (string, optional) — human-readable success criteria -- `assertions` (array, optional) — grader assertions +- `assert` (array, optional) — grader assertions +- `environment` (object | `file://...`, optional) — per-case testbed override - `conversation_id` (string, optional) — groups related tests - `execution` (object, optional) — per-test execution override -If `assertions` already state the grading contract, omit `criteria` instead of +If `assert` already states the grading contract, omit `criteria` instead of duplicating the same rubric. Prefer plain assertion strings for semantic checks when the default LLM rubric grader can judge them; use multiple named `type: llm-rubric` blocks only for custom prompts, custom grader targets, or intentional grader panels. Write `expected_output` as a golden/reference answer, not as criteria or scoring instructions. -For historical or repo-state evals, materialize the repository under -`workspace.repos[]` and pin `commit` to the commit under test. A SHA in prompt -prose or metadata is context only; it does not give the agent an actual -checkout. +For historical or repo-state evals, materialize the repository through +`environment.setup.command` and pass the repo/ref as argv inputs. A SHA in +prompt prose or metadata is context only; it does not give the agent an actual +checkout. `setup.command` is a non-empty string array. Put the executable at +`command[0]`, put CLI arguments in the remaining array entries, and use +`timeout_ms` for setup timeout. ## 2. Assertion Types and Grading Recipes diff --git a/skills-data/agentv-bench/references/migrating-from-skill-creator.md b/skills-data/agentv-bench/references/migrating-from-skill-creator.md index 3f6410822..95c8fa7f3 100644 --- a/skills-data/agentv-bench/references/migrating-from-skill-creator.md +++ b/skills-data/agentv-bench/references/migrating-from-skill-creator.md @@ -64,7 +64,7 @@ When evals.json becomes limiting, convert to EVAL.yaml for the full feature set: # Convert evals.json to EVAL.yaml agentv convert evals.json -# Edit the generated YAML to add workspace config, script graders, etc. +# Edit the generated YAML to add environment setup, script graders, etc. # Then run with the full lifecycle agentv eval eval.yaml ``` diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index dfe584c7a..235141ad9 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -61,7 +61,7 @@ agentv convert evals.json agentv eval evals.json ``` -The converter maps `prompt` → `input`, `expected_output` → `expected_output`, and Agent Skills `assertions` → AgentV `assert` (`llm-rubric` checks), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (workspace setup, script graders, rubrics, required gates). +The converter maps `prompt` → `input`, `expected_output` → `expected_output`, and Agent Skills `assertions` → AgentV `assert` (`llm-rubric` checks), and resolves `files[]` paths. The generated YAML includes TODO comments for AgentV features to add (environment setup, script graders, rubrics, required gates). After converting, enhance the YAML with AgentV-specific capabilities shown below. @@ -333,19 +333,29 @@ For repo-state evals, combine a pinned checkout, a golden answer, and assertion shorthand: ```yaml -workspace: - repos: - - path: ./agentv - repo: https://github.com/EntityProcess/agentv.git - commit: 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215 +environment: + type: host + workdir: ./agentv + setup: + command: + - bash + - ./scripts/materialize-repo.sh + - ./agentv + - https://github.com/EntityProcess/agentv.git + - 5e3c8f46d80fe66b1a75659e4fd94e38a7e09215 + cwd: "." + +prompts: + - "{{ task }}" tests: - id: verification-learning-capture - input: | - The eval harness has prepared ./agentv at the commit before the - verification guidance was added. + vars: + task: | + The eval harness has prepared ./agentv at the commit before the + verification guidance was added. - Decide what durable repo change should be made and explain why. + Decide what durable repo change should be made and explain why. expected_output: | The durable repo change is to update .agents/verification.md with the reusable verification workflow lessons. AGENTS.md already routes this @@ -482,10 +492,23 @@ setup: timeout_ms: 120000 ``` +```yaml +# .agentv/environments/docker-repo.yaml +type: docker +context: ./environment +dockerfile: Dockerfile +workdir: /app +setup: + command: ["bash", "-lc", "bun install && bun run build"] + cwd: "." + timeout_ms: 120000 +``` + - `type`: `host` or `docker` - `workdir`: path the target and graders should use - `setup`: argv command, optional `cwd`, and optional `timeout_ms` for repository/testbed materialization - Top-level `env`: provider/eval environment overrides +- `environment.env`: recipe-scoped process environment, distinct from top-level `env` - `extensions`: lifecycle hooks such as `beforeAll`, `beforeEach`, `afterEach`, and `afterAll` ## Grader Types @@ -503,7 +526,7 @@ Configure via the `assert` array. Multiple graders produce a weighted average sc Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidence?}], reasoning}` Raw stdin uses snake_case and includes: `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config` SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`. -When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace). +When an environment prepares a workspace directory, `workspace_path` is the absolute path to that directory (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the prepared workdir). For deterministic workspace checks that fit normal Vitest `expect(...)` tests, prefer a plain verifier file and the built-in adapter: ```yaml - name: welcome_banner From 13d882f451012fbeb4759ec2652aa61b2de5ea04 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 00:34:56 +0200 Subject: [PATCH 2/2] fix(examples): repair shared environment YAML --- .../workspace-shared-config/evals/regression/suite.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/features/workspace-shared-config/evals/regression/suite.yaml b/examples/features/workspace-shared-config/evals/regression/suite.yaml index dc10023b4..93d804bdf 100644 --- a/examples/features/workspace-shared-config/evals/regression/suite.yaml +++ b/examples/features/workspace-shared-config/evals/regression/suite.yaml @@ -1,5 +1,7 @@ -description: Regression eval that references the same shared environment recipe file. Demonstrates -environment recipe reuse across eval files in different directories. +description: > + Regression eval that references the same shared environment recipe file. + Demonstrates environment recipe reuse across eval files in different + directories. environment: file://../../environment.yaml tags: - agent