From a01fc740ff902353c609fe7d02bb1293904dffd8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Jul 2026 02:07:57 +0200 Subject: [PATCH 1/4] fix(eval): avoid pooled workspace reuse by default --- apps/cli/src/commands/eval/commands/run.ts | 2 +- .../repo-lifecycle/evals/pool-e2e.eval.yaml | 6 +- packages/core/src/evaluation/orchestrator.ts | 2 +- .../src/evaluation/workspace/repo-manager.ts | 74 ++++++++++++++++++- .../core/src/evaluation/workspace/setup.ts | 4 +- packages/core/src/evaluation/yaml-parser.ts | 2 +- .../evaluation/workspace/repo-manager.test.ts | 28 +++++++ .../test/evaluation/workspace/setup.test.ts | 37 +++++++++- 8 files changed, 144 insertions(+), 11 deletions(-) diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 857383c6c..2129f9c30 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -122,7 +122,7 @@ export const evalRunCommand = command({ workspaceMode: option({ type: optional(string), long: 'workspace-mode', - description: "Workspace mode: 'pooled', 'temp', or 'static'", + description: "Workspace mode: 'temp' (default), 'pooled', or 'static'", }), workspacePath: option({ type: optional(string), diff --git a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml index 75470d9ac..eb2d581c8 100644 --- a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml +++ b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml @@ -1,7 +1,7 @@ description: >- - E2E test for workspace pooling. No pool config in YAML — pooling is - enabled by default for shared workspaces with repos. - Run with --workers 2 to exercise multiple pool slots. + E2E test for workspace pooling. Run with --workspace-mode pooled + --workers 2 to exercise multiple pool slots. Normal shared repo + workspaces use fresh temp materialization by default. workspace: repos: diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 95f1fe38b..dace6158c 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -492,7 +492,7 @@ export interface RunEvaluationOptions { readonly runBudgetTracker?: RunBudgetTracker; /** Execution error tolerance: true halts on first error */ readonly failOnError?: FailOnError; - /** Workspace pooling: true (default) enables pool, false disables, undefined defaults to true */ + /** Legacy workspace pooling toggle. Explicit workspaceMode=pooled opts in to pooled reuse. */ readonly poolWorkspaces?: boolean; /** Maximum number of pool slots on disk (default: 10, max: 50) */ readonly poolMaxSlots?: number; diff --git a/packages/core/src/evaluation/workspace/repo-manager.ts b/packages/core/src/evaluation/workspace/repo-manager.ts index 5ffedb83b..f441a7fb7 100644 --- a/packages/core/src/evaluation/workspace/repo-manager.ts +++ b/packages/core/src/evaluation/workspace/repo-manager.ts @@ -1,7 +1,7 @@ import { execFile, spawn } from 'node:child_process'; import { createHash, randomUUID } from 'node:crypto'; import { existsSync, readFileSync } from 'node:fs'; -import { mkdir, rename, rm } from 'node:fs/promises'; +import { mkdir, rename, rm, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { promisify } from 'node:util'; @@ -40,6 +40,11 @@ interface AcquisitionSource { readonly originUrl: string; } +interface MirrorCacheLockMetadata { + readonly pid: number; + readonly createdAt: string; +} + /** Environment vars to force non-interactive git, stripped of hook-injected vars. */ function gitEnv(): Record { const env = { ...process.env }; @@ -365,14 +370,24 @@ export class RepoManager { private async withMirrorCacheLock(mirrorPath: string, action: () => Promise): Promise { const lockPath = `${mirrorPath}.lock`; const startedAt = Date.now(); + let lastLockHeartbeatAt = 0; while (true) { try { await mkdir(lockPath); + await this.writeMirrorCacheLockMetadata(lockPath); break; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code !== 'EEXIST') throw error; + if (await this.removeStaleMirrorCacheLock(lockPath)) { + continue; + } + if (this.progress && Date.now() - lastLockHeartbeatAt >= this.heartbeatMs) { + lastLockHeartbeatAt = Date.now(); + const elapsed = formatDuration(Date.now() - startedAt); + console.error(`[repo] waiting for git cache lock ${lockPath} after ${elapsed}`); + } if (Date.now() - startedAt > this.timeoutMs) { throw new Error(`Timed out waiting for git cache lock: ${lockPath}`); } @@ -387,6 +402,63 @@ export class RepoManager { } } + private async writeMirrorCacheLockMetadata(lockPath: string): Promise { + const metadata: MirrorCacheLockMetadata = { + pid: process.pid, + createdAt: new Date().toISOString(), + }; + await writeFile(path.join(lockPath, 'owner.json'), JSON.stringify(metadata, null, 2)); + } + + private readMirrorCacheLockMetadata(lockPath: string): MirrorCacheLockMetadata | undefined { + const metadataPath = path.join(lockPath, 'owner.json'); + if (!existsSync(metadataPath)) return undefined; + try { + const parsed = JSON.parse(readFileSync(metadataPath, 'utf-8')) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined; + const metadata = parsed as Record; + if (typeof metadata.pid !== 'number' || typeof metadata.createdAt !== 'string') { + return undefined; + } + return { pid: metadata.pid, createdAt: metadata.createdAt }; + } catch { + return undefined; + } + } + + private isProcessAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } + } + + private async isStaleMirrorCacheLock(lockPath: string): Promise { + const metadata = this.readMirrorCacheLockMetadata(lockPath); + if (metadata) { + return !this.isProcessAlive(metadata.pid); + } + + try { + const lockStat = await stat(lockPath); + return Date.now() - lockStat.mtimeMs > this.timeoutMs; + } catch { + return false; + } + } + + private async removeStaleMirrorCacheLock(lockPath: string): Promise { + if (!(await this.isStaleMirrorCacheLock(lockPath))) { + return false; + } + console.warn(`[repo] removing stale git cache lock: ${lockPath}`); + await rm(lockPath, { recursive: true, force: true }); + return true; + } + private async isValidBareRepo(repoPath: string): Promise { if (!existsSync(path.join(repoPath, 'HEAD'))) return false; try { diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index d7f385921..cd7ccbbb4 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -422,9 +422,7 @@ export async function prepareSharedWorkspaceSetup( if (cliWorkspacePath && workspaceMode && workspaceMode !== 'static') { throw new Error('--workspace-path requires --workspace-mode static when both are provided'); } - let configuredMode: WorkspaceSetupMode = cliWorkspacePath - ? 'static' - : (workspaceMode ?? 'pooled'); + let configuredMode: WorkspaceSetupMode = cliWorkspacePath ? 'static' : (workspaceMode ?? 'temp'); const configuredStaticPath = cliWorkspacePath; if (configuredMode === 'static' && !configuredStaticPath) { diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 6c5f56881..0d9a00781 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -2019,7 +2019,7 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi } if ('pool' in obj) { throw new Error( - 'workspace.pool has been removed from eval YAML. Shared repo workspaces are pooled by default; use --workspace-mode or config.local.yaml execution.workspace_mode for machine-local runtime overrides.', + 'workspace.pool has been removed from eval YAML. Shared repo workspaces use fresh temp materialization by default; use --workspace-mode pooled or config.local.yaml execution.workspace_mode for machine-local pooled reuse.', ); } if ('static' in obj) { diff --git a/packages/core/test/evaluation/workspace/repo-manager.test.ts b/packages/core/test/evaluation/workspace/repo-manager.test.ts index 74369d231..0ce1f12a4 100644 --- a/packages/core/test/evaluation/workspace/repo-manager.test.ts +++ b/packages/core/test/evaluation/workspace/repo-manager.test.ts @@ -7,6 +7,7 @@ import { readFileSync, readdirSync, statSync, + utimesSync, writeFileSync, } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; @@ -688,6 +689,33 @@ describe('RepoManager', () => { expect(gitExec('git rev-parse --is-bare-repository', cachePath)).toBe('true'); }, 30_000); + it('removes stale mirror-cache lock directories before cloning', async () => { + const repoDir = path.join(tmpDir, 'source-repo'); + createTestRepo(repoDir, { 'hello.txt': 'hello world' }); + const remoteDir = path.join(tmpDir, 'remote.git'); + execSync(`git clone --bare "${repoDir}" "${remoteDir}"`, { env: cleanGitEnv() }); + const repo = `file://${remoteDir}`; + const cachePath = cachePathFor(repo); + const lockPath = `${cachePath}.lock`; + mkdirSync(lockPath, { recursive: true }); + const staleTime = new Date(Date.now() - 10_000); + utimesSync(lockPath, staleTime, staleTime); + + const timeoutManager = new RepoManager(false, { progress: false, timeoutMs: 5_000 }); + await timeoutManager.materialize( + { + path: './my-repo', + repo, + }, + workspaceDir, + ); + + const targetDir = path.join(workspaceDir, 'my-repo'); + expect(existsSync(path.join(targetDir, 'hello.txt'))).toBe(true); + expect(existsSync(lockPath)).toBe(false); + expect(gitExec('git rev-parse --is-bare-repository', cachePath)).toBe('true'); + }, 30_000); + it('serializes mirror-cache population for concurrent cold materializations', async () => { const repoDir = path.join(tmpDir, 'source-repo'); createTestRepo(repoDir, { 'hello.txt': 'hello world' }); diff --git a/packages/core/test/evaluation/workspace/setup.test.ts b/packages/core/test/evaluation/workspace/setup.test.ts index 5ebb5a9ed..e6ae5c532 100644 --- a/packages/core/test/evaluation/workspace/setup.test.ts +++ b/packages/core/test/evaluation/workspace/setup.test.ts @@ -121,7 +121,6 @@ describe('prepareSharedWorkspaceSetup', () => { question: 'test', criteria: 'ok', workspace: { - mode: 'pooled', hooks: { after_each: { reset: 'fast' } }, repos, }, @@ -131,6 +130,7 @@ describe('prepareSharedWorkspaceSetup', () => { evalRunId: 'test-pooled-repo-reset', evalCases: [evalCase], evalDir: tmpDir, + workspaceMode: 'pooled', workers: 1, }); @@ -150,6 +150,41 @@ describe('prepareSharedWorkspaceSetup', () => { expect(existsSync(path.join(repoDir, 'stale.txt'))).toBe(false); }, 30_000); + it('uses fresh temp materialization for shared repo workspaces by default', async () => { + const sourceRepo = path.join(tmpDir, 'source-repo'); + const cleanCommit = createTestRepo(sourceRepo, { 'tracked.txt': 'clean\n' }); + const evalCase = testCase('case-1', { + repos: [ + { + path: './repo-a', + repo: `file://${sourceRepo}`, + commit: cleanCommit, + }, + ], + }); + + setup = await prepareSharedWorkspaceSetup({ + evalRunId: 'test-default-temp-repo', + evalCases: [evalCase], + evalDir: tmpDir, + workers: 1, + }); + + expect(setup.configuredMode).toBe('temp'); + expect(setup.poolManager).toBeUndefined(); + expect(setup.poolSlots).toHaveLength(0); + expect(setup.sharedWorkspacePath).toBeDefined(); + if (!setup.sharedWorkspacePath) { + throw new Error('Expected temp setup to include a workspace path'); + } + + const repoDir = path.join(setup.sharedWorkspacePath, 'repo-a'); + expect( + execSync('git rev-parse HEAD', { cwd: repoDir, env: cleanGitEnv() }).toString().trim(), + ).toBe(cleanCommit); + expect(readFileSync(path.join(repoDir, 'tracked.txt'), 'utf8')).toBe('clean\n'); + }, 30_000); + it('uses CLI workspacePath as an existing static workspace without materializing repos', async () => { const existingWorkspace = path.join(tmpDir, 'existing-workspace'); mkdirSync(existingWorkspace, { recursive: true }); From b5f5edbc375ec19b858d172ac60714f2318e876a Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Jul 2026 06:15:58 +0200 Subject: [PATCH 2/4] feat(eval): add command repo resolvers --- README.md | 11 + .../docs/docs/evaluation/running-evals.mdx | 5 +- .../docs/guides/workspace-architecture.mdx | 91 ++++- .../docs/docs/guides/workspace-pool.mdx | 37 +- .../docs/docs/targets/configuration.mdx | 5 +- packages/core/src/evaluation/types.ts | 2 + .../evaluation/validation/config-validator.ts | 103 +++++ .../evaluation/validation/eval-file.schema.ts | 1 + .../workspace/repo-config-parser.ts | 12 +- .../src/evaluation/workspace/repo-manager.ts | 130 +++++- .../src/evaluation/workspace/repo-resolver.ts | 351 ++++++++++++++++ .../validation/config-validator.test.ts | 58 +++ .../workspace-config-parsing.test.ts | 2 + .../evaluation/workspace/repo-manager.test.ts | 375 ++++++++++++++++++ .../references/breaking-changes.md | 6 +- skills-data/agentv-eval-writer/SKILL.md | 6 +- 16 files changed, 1141 insertions(+), 54 deletions(-) create mode 100644 packages/core/src/evaluation/workspace/repo-resolver.ts diff --git a/README.md b/README.md index cf44fb95f..329b84fba 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ default_test: workspace: isolation: per_case + repos: + - path: ./fixture + repo: EntityProcess/agentv-contract-fixture + commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb tests: - id: fizzbuzz @@ -211,6 +215,13 @@ export default defineEval({ threshold: 0.8, workspace: { isolation: 'per_case', + repos: [ + { + path: './fixture', + repo: 'EntityProcess/agentv-contract-fixture', + commit: '21a34daed7ebcfe36cbed053607622a55e5e94cb', + }, + ], }, tests: [ { diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index 02b750eb6..dc881d48b 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -313,7 +313,7 @@ This matches the standard model used by eval frameworks (promptfoo, deepeval, Op Use runtime workspace flags and finish policies instead of multiple conflicting booleans: ```bash -# Mode: pooled | temp | static +# Mode: temp (default) | pooled | static agentv eval evals/my-eval.yaml --workspace-mode pooled # Existing local workspace path for this run @@ -339,7 +339,8 @@ workspace: ``` Notes: -- Pooling is default for shared workspaces with repos. +- Temp workspace materialization is the default for shared workspaces with repos. +- Pooled mode is an explicit machine-local optimization. - `--workspace-path` uses an existing machine-local directory as-is and implies static runtime mode. - Runtime static mode is incompatible with `isolation: per_case`. - `hooks.enabled: false` skips all lifecycle hooks (setup, teardown, reset). diff --git a/apps/web/src/content/docs/docs/guides/workspace-architecture.mdx b/apps/web/src/content/docs/docs/guides/workspace-architecture.mdx index 30d9bcab7..c80c07d3a 100644 --- a/apps/web/src/content/docs/docs/guides/workspace-architecture.mdx +++ b/apps/web/src/content/docs/docs/guides/workspace-architecture.mdx @@ -10,9 +10,9 @@ fixtures, repositories, and lifecycle hooks. Targets run inside that substrate. When `workspace.repos` is present, the eval declares repository identity and checkout pins; AgentV decides how to acquire the bytes. -[Workspace pooling](/docs/guides/workspace-pool/) is enabled by default for -shared repo workspaces, so the first run pays materialization cost and later -runs reset existing pool slots in place. +By default, repo workspaces are materialized into fresh temp workspaces. A +machine-local pooled mode remains available for runs that explicitly opt into +slot reuse. ## Eval setup lifecycle @@ -23,7 +23,7 @@ eval start | v +---------------------------+ -| 1. Pool / workspace setup | Acquire pool slot or create temp workspace +| 1. Workspace setup | Create temp workspace or acquire explicit pool slot +---------------------------+ | v @@ -57,7 +57,7 @@ eval start +---------------------------+ ``` -With workspace pooling (the default), steps 2-3 only happen on the first run. Subsequent runs reset the pool slot in-place, skipping clone and checkout entirely. +With `--workspace-mode pooled`, steps 2-3 only happen on the first run. Subsequent runs reset the pool slot in-place, skipping clone and checkout entirely. The default repo workspace mode is `temp`, which materializes a fresh workspace for each run. ## Repo provenance vs acquisition @@ -83,6 +83,7 @@ Supported repo fields: | `base_commit` | Alias for `commit`, useful for SWE-bench-style datasets | | `sparse` | Optional sparse-checkout paths | | `ancestor` | Walk N parents back after resolving `commit` / `base_commit` | +| `resolver` | Optional `repo_resolvers[].name` override from AgentV config | `commit` is the canonical AgentV checkout pin. `base_commit` exists only as a SWE-Bench-friendly alias for the same value; when both fields are present they @@ -97,8 +98,8 @@ while each harness uses the fastest safe local source available. ## Native workspace boundary Use native AgentV workspaces when AgentV owns the run lifecycle: custom internal -suites, CI gates, target comparisons, pooled workspaces, local setup hooks, -Docker workspaces, and generic repository acquisition. In that path, +suites, CI gates, target comparisons, local setup hooks, Docker workspaces, and +generic repository acquisition. In that path, `workspace.repos` declares the repos and checkout pins while AgentV materializes the workspace, runs targets and graders, and writes AgentV run bundles. @@ -118,17 +119,63 @@ For each materialized repo, AgentV resolves acquisition in this order: | Order | Source | How it is used | |-------|--------|----------------| -| 1 | Registered project | A project in `$AGENTV_HOME/projects.yaml` whose `origin` matches the repo identity. AgentV clones from that local checkout with `--reference --dissociate`, then resets `origin` to the declared repo URL. | -| 2 | Configured mirror | A path listed under `git_cache.mirrors` in `$AGENTV_HOME/config.yaml`. AgentV uses the same `--reference --dissociate` flow. | -| 3 | Mirror cache | An AgentV-owned bare cache under `$AGENTV_DATA_DIR/git-cache/`. Cache population is locked, cloned into a temporary path, verified, and atomically renamed before use. | -| 4 | Remote clone | The normalized clone URL from the eval's `repo` field. | - -`--dissociate` copies the objects needed by the workspace clone and removes the -long-lived alternates dependency on the user-owned checkout or mirror. That -keeps preserved workspaces and pool slots from breaking later if a local -checkout is moved, deleted, or garbage-collected. Local checkouts and mirrors -still provide clone speed, but the resulting workspace has its own required Git -objects and full reachable history for pinned commits and `ancestor` checks. +| 1 | Explicit resolver | `workspace.repos[].resolver` names a configured command resolver. If it returns `handled:false`, AgentV fails clearly instead of guessing. | +| 2 | Pattern resolver | The first non-`default` `repo_resolvers[]` entry whose `repos` pattern matches the repo URL or identity. If it returns `handled:false`, AgentV continues to the default resolver. | +| 3 | Default resolver | The resolver named `default`, if configured. It must not declare `repos`; it is the unconditional project default. If it returns `handled:false`, AgentV continues to the built-in git resolver. | +| 4 | Registered project | A project in `$AGENTV_HOME/projects.yaml` whose `origin` matches the repo identity. AgentV seeds its mirror cache from that local checkout, then clones the cache into the workspace and resets `origin` to the declared repo URL. | +| 5 | Configured mirror | A path listed under `git_cache.mirrors`. AgentV seeds its mirror cache from that checkout or bare mirror, then clones the cache into the workspace. | +| 6 | Mirror cache | An AgentV-owned bare cache under `$AGENTV_DATA_DIR/git-cache/`. Cache population is locked, cloned into a temporary path, verified, and atomically renamed before use. | +| 7 | Remote clone | The normalized clone URL from the eval's `repo` field. | + +Workspace clones are independent from user-owned checkouts, configured mirrors, +and resolver source directories. AgentV does not leave Git alternates pointing +back to those sources, so preserved workspaces and pool slots keep working if a +local checkout is moved, deleted, or garbage-collected. + +### Command repo resolvers + +Use `repo_resolvers` when repo bytes come from a project-specific source that +AgentV core should not understand, such as an internal snapshot bundle. Put that +logic in a resolver script and return a local git source for AgentV to clone and +check out normally: + +```yaml +# .agentv/config.yaml +repo_resolvers: + - name: org_snapshots + repos: + - https://github.com/example/* + command: + - bun + - scripts/eval-config/repo-resolver.ts + config: + release_tag: snapshot/v1.1.0 + + - name: default + command: + - bun + - scripts/eval-config/default-repo-resolver.ts +``` + +AgentV sends JSON on stdin with `version`, `repo`, `commit`, `path`, `sparse`, +`ancestor`, `cache_dir`, `workspace_path`, and the resolver `config`. The +resolver writes JSON on stdout: + +```json +{ + "handled": true, + "source": { + "type": "git", + "path": "/tmp/source.git", + "origin": "https://github.com/example/repo.git" + } +} +``` + +Only `source.type: "git"` is supported. Resolver scripts should prepare or +locate source directories independently from the final workspace; AgentV still +materializes the repo into every shared, per-case, or explicitly pooled +workspace it creates. ### Configured mirrors @@ -269,7 +316,7 @@ workspace clone from user-owned storage. | Symptom | Likely cause | Fix | |---------|-------------|-----| -| Clone progress runs for minutes on first run | Large repo acquired from remote | Register a matching local project or configure `git_cache.mirrors`; subsequent pooled runs skip clone. | +| Clone progress runs for minutes | Large repo acquired from remote | Register a matching local project or configure `git_cache.mirrors`; optionally use `--workspace-mode pooled` for repeated local runs. | | Heartbeat ends with a clone/fetch timeout | Remote network or missing local cache | Use the timeout guidance in the error: local checkout, configured mirror, or network fix. | | Stuck at checkout for 2+ minutes | Large repo file materialization after objects are present | Expected for 100k+ files; use Dev Drive on Windows. Subsequent runs use pool. | | `Filename too long` during checkout | Missing `core.longpaths` | `git config --global core.longpaths true` | @@ -278,12 +325,12 @@ workspace clone from user-owned storage. ## Workspace pooling -Workspace pooling is **enabled by default** for shared workspaces with repos. The first run materializes from scratch. Subsequent runs reset the existing workspace in-place (`git reset --hard` + `git clean -fd`) — typically reducing setup from minutes to seconds. +Workspace pooling is an explicit machine-local optimization for shared workspaces with repos. The first pooled run materializes from scratch. Subsequent pooled runs reset the existing workspace in-place (`git reset --hard` + `git clean -fd`) — typically reducing setup from minutes to seconds. -To disable pooling for a run: +To opt into pooling for a run: ```bash -agentv eval evals/my-eval.yaml --no-pool +agentv eval evals/my-eval.yaml --workspace-mode pooled ``` See the [Workspace Pool](/docs/guides/workspace-pool/) guide for details on pool configuration, clean modes, concurrency, and drift detection. diff --git a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx index 98ba23054..685a1f801 100644 --- a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx +++ b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx @@ -7,7 +7,11 @@ sidebar: Workspace pooling keeps materialized workspaces on disk between eval runs. Instead of cloning repos and checking out files every time, pooled workspaces reset in-place — typically reducing setup from minutes to seconds for large repositories. -**Pooling is enabled by default** for shared workspaces that define `repos`. No extra flags needed. +Pooling is an explicit machine-local runtime mode. The default repo workspace mode is `temp`, which materializes a fresh workspace for each run. + +```bash +agentv eval evals/my-eval.yaml --workspace-mode pooled +``` ## How it works @@ -31,24 +35,22 @@ On subsequent runs: **Keep templates small.** Template files are re-copied into every slot on every run. Use them for lightweight setup — agent skills, configuration files, prompt templates — not large assets. Heavy dependencies belong in repos (pooled and reused) or should be installed by `before_all` hooks (cached across reuse cycles with `fast` reset). -The first run materializes from scratch. Every subsequent run reuses the pool — skipping clone and checkout entirely. +The first pooled run materializes from scratch. Subsequent pooled runs reuse the pool — skipping clone and checkout entirely. -## Disabling pooling +## Enabling pooling -Pooling is on by default. To disable it: - -### CLI mode +Use pooled mode only as a local runtime override: ```bash -agentv eval evals/my-eval.yaml --workspace-mode temp +agentv eval evals/my-eval.yaml --workspace-mode pooled ``` -### Local config +Or set it in local config: ```yaml # .agentv/config.local.yaml execution: - workspace_mode: temp + workspace_mode: pooled ``` `workspace_mode` is a machine-local runtime override. Do not commit it in eval YAML. @@ -202,14 +204,15 @@ CLI flags `--retain-on-success` / `--retain-on-failure` control temporary eval-r | Mode | Setup cost | Persistent | Build artifacts preserved | Concurrent workers | |------|-----------|-----------|--------------------------|-------------------| -| **Pooled** (default) | First run only; reset on reuse | Yes | Yes (`.gitignore`d files) | Yes (slot per worker) | -| **Temp** (`--workspace-mode temp`) | Full clone + checkout every run | No | No | Sequential only | +| **Temp** (default) | Full clone + checkout every run | No | No | Sequential only | +| **Pooled** (`--workspace-mode pooled`) | First run only; reset on reuse | Yes | Yes (`.gitignore`d files) | Yes (slot per worker) | | **Existing path** (`--workspace-path` / `execution.workspace_path`) | Uses the supplied directory as-is | Yes | User-managed | Sequential only | -## When to disable pooling +## When to opt into pooling + +Consider pooled mode when: +- Large repo materialization dominates run time +- You want local cache reuse across repeated development runs +- You understand that ignored build artifacts may survive fast pool resets -**Pooling is typically the right default.** Consider disabling it when: -- You need guaranteed clean-slate isolation between runs -- You're debugging workspace setup issues and want fresh clones each time -- You use `--workspace-path` with a pre-existing local directory (pooling is automatically skipped) -- You need `isolation: per_case` (each test gets its own workspace copy; pooling is automatically skipped) +Prefer the default temp mode when you need clean-slate isolation, are debugging workspace setup, use `--workspace-path`, or run with `isolation: per_case`. diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index d3985231a..88a728c5b 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -165,12 +165,13 @@ workspace: isolation: shared # shared (default) | per_case ``` -`repo` declares the repository identity. Acquisition is harness-owned: AgentV first looks for matching registered projects and configured mirrors, then uses its git cache, then falls back to remote clone. See [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition) for the resolver order and `git_cache.mirrors` config. +`repo` declares the repository identity. Acquisition is harness-owned: AgentV first applies configured `repo_resolvers`, then uses the built-in git path of registered projects, configured mirrors, AgentV's git cache, and remote clone. See [Workspace Architecture](/docs/guides/workspace-architecture/#acquisition-resolver) for the resolver order, command resolver protocol, and `git_cache.mirrors` config. | Field | Description | |-------|-------------| | `repos[].path` | Directory within the workspace to clone into | | `repos[].repo` | Repository identity: full clone URL or GitHub `org/name` shorthand | +| `repos[].resolver` | Optional configured `repo_resolvers[].name` override | | `repos[].commit` | Branch, tag, or SHA to check out (default: `HEAD`) | | `repos[].base_commit` | Alias for `commit`, useful for SWE-bench-style datasets | | `repos[].ancestor` | Walk N commits back from the checked-out ref (e.g., `1` for parent) | @@ -181,7 +182,7 @@ workspace: `isolation: per_case` is the spelling for fresh workspace state per test case. -**Pooling:** shared workspaces with `repos` use pool slots by default. Use `--workspace-mode temp` or `execution.workspace_mode: temp` in `config.local.yaml` to disable pooling for a local run. +**Workspace mode:** shared workspaces with `repos` use fresh temp workspaces by default. Use `--workspace-mode pooled` or `execution.workspace_mode: pooled` in local config only when you explicitly want pool-slot reuse. **Existing local workspaces:** do not commit local paths in eval YAML. Use `--workspace-path /path/to/workspace` for a one-off run, or put `execution.workspace_path` in `.agentv/config.local.yaml`. diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 89ccd31cb..a23acd308 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -248,6 +248,8 @@ export type RepoConfig = { readonly ancestor?: number; /** Optional sparse-checkout paths. */ readonly sparse?: readonly string[]; + /** Optional project-configured repo resolver name. */ + readonly resolver?: string; }; export type WorkspaceHookConfig = { diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 35347d878..ba00ccb33 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -75,6 +75,7 @@ export async function validateConfigFile( } validateResultsConfig(errors, filePath, config.results, 'results'); + validateRepoResolversConfig(errors, filePath, config.repo_resolvers); const projects = config.projects; if (projects !== undefined) { @@ -114,6 +115,7 @@ export async function validateConfigFile( 'required_version', 'execution', 'results', + 'repo_resolvers', 'projects', 'results_by_project', 'dashboard', @@ -145,6 +147,107 @@ export async function validateConfigFile( } } +function validateRepoResolversConfig( + errors: ValidationError[], + filePath: string, + rawResolvers: unknown, +): void { + if (rawResolvers === undefined) { + return; + } + + if (!Array.isArray(rawResolvers)) { + addError(errors, filePath, 'repo_resolvers', "Field 'repo_resolvers' must be an array"); + return; + } + + const seenNames = new Set(); + let defaultCount = 0; + + rawResolvers.forEach((resolver, index) => { + const location = `repo_resolvers[${index}]`; + if (!isPlainObject(resolver)) { + addError(errors, filePath, location, `Field '${location}' must be an object`); + return; + } + + const name = resolver.name; + if (typeof name !== 'string' || name.trim().length === 0) { + addError( + errors, + filePath, + `${location}.name`, + `Field '${location}.name' must be a non-empty string`, + ); + } else { + const trimmedName = name.trim(); + if (seenNames.has(trimmedName)) { + addError( + errors, + filePath, + `${location}.name`, + `Duplicate repo resolver name '${trimmedName}'`, + ); + } + seenNames.add(trimmedName); + + if (trimmedName === 'default') { + defaultCount += 1; + if (resolver.repos !== undefined) { + addError( + errors, + filePath, + `${location}.repos`, + "Repo resolver named 'default' must not declare repos", + ); + } + } + } + + const command = resolver.command; + if ( + !Array.isArray(command) || + !command.every((entry) => typeof entry === 'string') || + command.length === 0 + ) { + addError( + errors, + filePath, + `${location}.command`, + `Field '${location}.command' must be a non-empty string array`, + ); + } + + const repos = resolver.repos; + if ( + repos !== undefined && + (!Array.isArray(repos) || + repos.length === 0 || + !repos.every((entry) => typeof entry === 'string' && entry.trim().length > 0)) + ) { + addError( + errors, + filePath, + `${location}.repos`, + `Field '${location}.repos' must be a non-empty string array when set`, + ); + } + + if (resolver.config !== undefined && !isPlainObject(resolver.config)) { + addError( + errors, + filePath, + `${location}.config`, + `Field '${location}.config' must be an object when set`, + ); + } + }); + + if (defaultCount > 1) { + addError(errors, filePath, 'repo_resolvers', "Duplicate repo resolver named 'default'"); + } +} + function inferConfigScope(filePath: string): 'project' | 'global' { const globalConfigPath = path.resolve(getAgentvConfigDir(), 'config.yaml'); const globalLocalConfigPath = path.resolve(getLocalConfigPath(globalConfigPath)); diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index c7fc5b1f6..7487d693a 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -284,6 +284,7 @@ const RepoSchema = z base_commit: z.string().min(1).optional(), ancestor: z.number().int().min(0).optional(), sparse: z.array(z.string()).optional(), + resolver: z.string().min(1).optional(), }) .strict() .refine((repo) => !repo.commit || !repo.base_commit || repo.commit === repo.base_commit, { diff --git a/packages/core/src/evaluation/workspace/repo-config-parser.ts b/packages/core/src/evaluation/workspace/repo-config-parser.ts index c09f8198d..b24a06509 100644 --- a/packages/core/src/evaluation/workspace/repo-config-parser.ts +++ b/packages/core/src/evaluation/workspace/repo-config-parser.ts @@ -43,12 +43,21 @@ export function parseRepoConfig(raw: unknown): RepoConfig | undefined { const baseCommit = readString(obj, 'base_commit'); const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; const sparse = readStringArray(obj, 'sparse'); + const resolver = readString(obj, 'resolver'); if (commit !== undefined && baseCommit !== undefined && commit !== baseCommit) { throw new Error('workspace.repos[].commit and workspace.repos[].base_commit must match.'); } - if (!repoPath && !repo && !commit && !baseCommit && ancestor === undefined && !sparse) { + if ( + !repoPath && + !repo && + !commit && + !baseCommit && + ancestor === undefined && + !sparse && + !resolver + ) { return undefined; } @@ -59,5 +68,6 @@ export function parseRepoConfig(raw: unknown): RepoConfig | undefined { ...(baseCommit !== undefined && { base_commit: baseCommit }), ...(ancestor !== undefined && { ancestor }), ...(sparse !== undefined && { sparse }), + ...(resolver !== undefined && { resolver }), }; } diff --git a/packages/core/src/evaluation/workspace/repo-manager.ts b/packages/core/src/evaluation/workspace/repo-manager.ts index f441a7fb7..1e142190b 100644 --- a/packages/core/src/evaluation/workspace/repo-manager.ts +++ b/packages/core/src/evaluation/workspace/repo-manager.ts @@ -11,6 +11,13 @@ import type { RepoConfig } from '../types.js'; import { parseYamlValue } from '../yaml-loader.js'; import { getRepoCheckoutRef } from './repo-checkout.js'; import { normalizeRepoIdentity, resolveRepoCloneUrl } from './repo-identity.js'; +import { + type RepoResolverConfig, + parseRepoResolversFromConfig, + runRepoResolverCommand, + selectRepoResolver, + validateRepoResolvers, +} from './repo-resolver.js'; const execFileAsync = promisify(execFile); const DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes @@ -35,7 +42,12 @@ interface RepoManagerOptions { } interface AcquisitionSource { - readonly kind: 'configured-mirror' | 'registered-project' | 'mirror-cache' | 'remote'; + readonly kind: + | 'configured-mirror' + | 'registered-project' + | 'mirror-cache' + | 'remote' + | 'repo-resolver'; readonly sourceUrl: string; readonly originUrl: string; } @@ -249,12 +261,28 @@ export class RepoManager { }); } - private loadConfiguredMirrorsFrom(filePath: string): Record { + private readConfigObjectFrom( + filePath: string, + options: { throwOnError?: boolean } = {}, + ): Record | undefined { if (!existsSync(filePath)) return {}; try { const parsed = parseYamlValue(readFileSync(filePath, 'utf-8')) as unknown; if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}; - const config = parsed as Record; + return parsed as Record; + } catch (error) { + if (options.throwOnError) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Could not read AgentV config at ${filePath}: ${message}`); + } + return {}; + } + } + + private loadConfiguredMirrorsFrom(filePath: string): Record { + try { + const config = this.readConfigObjectFrom(filePath); + if (!config) return {}; const gitCache = config.git_cache; if (!gitCache || typeof gitCache !== 'object' || Array.isArray(gitCache)) return {}; const mirrors = (gitCache as Record).mirrors; @@ -329,6 +357,33 @@ export class RepoManager { ]); } + private loadRepoResolversFrom(filePath: string, cwd?: string): readonly RepoResolverConfig[] { + const config = this.readConfigObjectFrom(filePath, { throwOnError: true }); + if (!config) return []; + return parseRepoResolversFromConfig(config, filePath, cwd); + } + + private loadRepoResolvers(): readonly RepoResolverConfig[] { + const globalResolvers = this.loadRepoResolversFrom(configPath(), getAgentvConfigDir()); + const projectAgentvDir = this.findProjectAgentvDir(); + if (!projectAgentvDir) { + validateRepoResolvers(globalResolvers); + return globalResolvers; + } + + const projectRoot = path.dirname(projectAgentvDir); + const resolvers = [ + ...globalResolvers, + ...this.loadRepoResolversFrom(path.join(projectAgentvDir, 'config.yaml'), projectRoot), + ...this.loadRepoResolversFrom( + path.join(projectAgentvDir, 'config.override.yaml'), + projectRoot, + ), + ]; + validateRepoResolvers(resolvers); + return resolvers; + } + private findConfiguredMirror(repoIdentity: string): string | undefined { const mirrors = this.loadConfiguredMirrors(); for (const [repo, localPath] of Object.entries(mirrors)) { @@ -578,6 +633,64 @@ export class RepoManager { } } + private async resolveConfiguredRepoResolver( + repo: RepoConfig, + workspacePath: string, + ): Promise { + const resolvers = this.loadRepoResolvers(); + if (resolvers.length === 0) { + if (repo.resolver) { + throw new Error(`workspace.repos[].resolver '${repo.resolver}' is not configured.`); + } + return undefined; + } + + const originUrl = resolveRepoCloneUrl(repo.repo ?? ''); + const selection = selectRepoResolver(repo, resolvers); + if (!selection) return undefined; + + const result = await runRepoResolverCommand( + selection.resolver, + repo, + workspacePath, + this.timeoutMs, + ); + if (result.handled) { + return { + kind: 'repo-resolver', + sourceUrl: result.source.path, + originUrl: result.source.origin ?? originUrl, + }; + } + + if (selection.kind === 'explicit') { + throw new Error( + `Repo resolver '${selection.resolver.name}' was selected by workspace.repos[].resolver but returned handled:false.`, + ); + } + + if (selection.kind === 'pattern') { + const defaultResolver = resolvers.find((resolver) => resolver.name === 'default'); + if (defaultResolver) { + const defaultResult = await runRepoResolverCommand( + defaultResolver, + repo, + workspacePath, + this.timeoutMs, + ); + if (defaultResult.handled) { + return { + kind: 'repo-resolver', + sourceUrl: defaultResult.source.path, + originUrl: defaultResult.source.origin ?? originUrl, + }; + } + } + } + + return undefined; + } + private assertNoUserOwnedAlternates(targetDir: string, acquisition: AcquisitionSource): void { const alternatesPath = path.join(targetDir, '.git', 'objects', 'info', 'alternates'); if (!existsSync(alternatesPath)) return; @@ -589,7 +702,10 @@ export class RepoManager { } } - private async resolveAcquisition(repo: RepoConfig): Promise { + private async resolveAcquisition( + repo: RepoConfig, + workspacePath: string, + ): Promise { const declaredRepo = repo.repo; if (!declaredRepo) { throw new Error(`repo is required for workspace repo at path ${repo.path ?? '(none)'}`); @@ -597,6 +713,10 @@ export class RepoManager { const originUrl = resolveRepoCloneUrl(declaredRepo); const repoIdentity = normalizeRepoIdentity(declaredRepo); + const configuredResolver = await this.resolveConfiguredRepoResolver(repo, workspacePath); + if (configuredResolver) { + return configuredResolver; + } const registeredProject = await this.findRegisteredProject(repoIdentity); if (registeredProject) { @@ -647,7 +767,7 @@ export class RepoManager { } const targetDir = path.join(workspacePath, repo.path); - const acquisition = await this.resolveAcquisition(repo); + const acquisition = await this.resolveAcquisition(repo, workspacePath); const startedAt = Date.now(); if (this.verbose) { diff --git a/packages/core/src/evaluation/workspace/repo-resolver.ts b/packages/core/src/evaluation/workspace/repo-resolver.ts new file mode 100644 index 000000000..d2b5ae293 --- /dev/null +++ b/packages/core/src/evaluation/workspace/repo-resolver.ts @@ -0,0 +1,351 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdir } from 'node:fs/promises'; +import path from 'node:path'; + +import micromatch from 'micromatch'; + +import { getAgentvDataDir } from '../../paths.js'; +import type { JsonObject, JsonValue, RepoConfig } from '../types.js'; +import { isJsonObject } from '../types.js'; +import { getRepoCheckoutRef } from './repo-checkout.js'; +import { normalizeRepoIdentity, resolveRepoCloneUrl } from './repo-identity.js'; + +const RESOLVER_OUTPUT_LIMIT = 1024 * 1024; + +export interface RepoResolverConfig { + readonly name: string; + readonly command: readonly string[]; + readonly repos?: readonly string[]; + readonly config: JsonObject; + readonly cwd?: string; + readonly sourcePath: string; +} + +export interface RepoResolverRequest { + readonly version: 1; + readonly repo: string; + readonly commit: string; + readonly path: string; + readonly sparse: readonly string[] | null; + readonly ancestor: number | null; + readonly cache_dir: string; + readonly workspace_path: string; + readonly config: JsonObject; +} + +export interface RepoResolverGitSource { + readonly type: 'git'; + readonly path: string; + readonly origin?: string; +} + +export interface RepoResolverHandledResult { + readonly handled: true; + readonly source: RepoResolverGitSource; +} + +export interface RepoResolverUnhandledResult { + readonly handled: false; +} + +export type RepoResolverResult = RepoResolverHandledResult | RepoResolverUnhandledResult; + +export type RepoResolverSelection = + | { readonly kind: 'explicit'; readonly resolver: RepoResolverConfig } + | { readonly kind: 'pattern'; readonly resolver: RepoResolverConfig } + | { readonly kind: 'default'; readonly resolver: RepoResolverConfig }; + +export function parseRepoResolversFromConfig( + rawConfig: Record, + sourcePath: string, + cwd?: string, +): readonly RepoResolverConfig[] { + const rawResolvers = rawConfig.repo_resolvers; + if (rawResolvers === undefined) return []; + if (!Array.isArray(rawResolvers)) { + throw new Error(`repo_resolvers in ${sourcePath} must be an array.`); + } + + return rawResolvers.map((rawResolver, index) => + parseRepoResolver(rawResolver, `repo_resolvers[${index}]`, sourcePath, cwd), + ); +} + +export function validateRepoResolvers(resolvers: readonly RepoResolverConfig[]): void { + const seenNames = new Set(); + let defaultCount = 0; + + for (const resolver of resolvers) { + if (seenNames.has(resolver.name)) { + throw new Error(`Duplicate repo resolver name '${resolver.name}'.`); + } + seenNames.add(resolver.name); + + if (resolver.name === 'default') { + defaultCount += 1; + if (resolver.repos !== undefined) { + throw new Error("Repo resolver named 'default' must not declare repos."); + } + } + } + + if (defaultCount > 1) { + throw new Error("Duplicate repo resolver named 'default'."); + } +} + +export function selectRepoResolver( + repo: RepoConfig, + resolvers: readonly RepoResolverConfig[], +): RepoResolverSelection | undefined { + if (repo.resolver) { + const resolver = resolvers.find((candidate) => candidate.name === repo.resolver); + if (!resolver) { + throw new Error(`workspace.repos[].resolver '${repo.resolver}' is not configured.`); + } + return { kind: 'explicit', resolver }; + } + + const patternResolver = resolvers.find( + (resolver) => resolver.name !== 'default' && matchesRepoPatterns(repo, resolver.repos), + ); + if (patternResolver) { + return { kind: 'pattern', resolver: patternResolver }; + } + + const defaultResolver = resolvers.find((resolver) => resolver.name === 'default'); + return defaultResolver ? { kind: 'default', resolver: defaultResolver } : undefined; +} + +export async function runRepoResolverCommand( + resolver: RepoResolverConfig, + repo: RepoConfig, + workspacePath: string, + timeoutMs: number, +): Promise { + if (!repo.repo || !repo.path) { + throw new Error('repo resolver requires workspace repo and path.'); + } + + const request = await buildRepoResolverRequest(resolver, repo, workspacePath); + const stdout = await runResolverProcess(resolver, request, timeoutMs); + return parseRepoResolverOutput(stdout, resolver.name); +} + +function parseRepoResolver( + rawResolver: unknown, + location: string, + sourcePath: string, + cwd?: string, +): RepoResolverConfig { + if (!rawResolver || typeof rawResolver !== 'object' || Array.isArray(rawResolver)) { + throw new Error(`${location} in ${sourcePath} must be an object.`); + } + + const resolver = rawResolver as Record; + const name = readNonEmptyString(resolver.name); + if (!name) { + throw new Error(`${location}.name in ${sourcePath} must be a non-empty string.`); + } + + const command = readStringArray(resolver.command); + if (!command) { + throw new Error(`${location}.command in ${sourcePath} must be a non-empty string array.`); + } + + const repos = readStringArray(resolver.repos); + const config = resolver.config === undefined ? {} : resolver.config; + if (!isJsonObject(config)) { + throw new Error(`${location}.config in ${sourcePath} must be a JSON object.`); + } + + return { + name, + command, + config, + sourcePath, + ...(repos !== undefined && { repos }), + ...(cwd !== undefined && { cwd }), + }; +} + +function readNonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function readStringArray(value: unknown): readonly string[] | undefined { + if (!Array.isArray(value)) return undefined; + if (value.length === 0 || !value.every((item) => typeof item === 'string')) return undefined; + return value; +} + +function matchesRepoPatterns(repo: RepoConfig, patterns: readonly string[] | undefined): boolean { + if (!repo.repo || !patterns?.length) return false; + + const cloneUrl = resolveRepoCloneUrl(repo.repo); + const identity = normalizeRepoIdentity(repo.repo); + const candidates = new Set([ + repo.repo, + stripGitSuffix(repo.repo), + cloneUrl, + stripGitSuffix(cloneUrl), + identity, + ]); + + return patterns.some((pattern) => + [...candidates].some((candidate) => micromatch.isMatch(candidate, pattern, { nocase: true })), + ); +} + +function stripGitSuffix(value: string): string { + return value.replace(/\.git$/i, ''); +} + +async function buildRepoResolverRequest( + resolver: RepoResolverConfig, + repo: RepoConfig, + workspacePath: string, +): Promise { + const cacheDir = path.join( + getAgentvDataDir(), + 'cache', + 'repo-resolvers', + cacheKeyForResolver(resolver.name), + ); + await mkdir(cacheDir, { recursive: true }); + + return { + version: 1, + repo: resolveRepoCloneUrl(repo.repo ?? ''), + commit: getRepoCheckoutRef(repo), + path: repo.path ?? '', + sparse: repo.sparse ?? null, + ancestor: repo.ancestor ?? null, + cache_dir: cacheDir, + workspace_path: workspacePath, + config: resolver.config, + }; +} + +function cacheKeyForResolver(name: string): string { + const safePrefix = name.replace(/[^A-Za-z0-9._-]+/g, '_').replace(/^_+|_+$/g, '') || 'resolver'; + const hash = createHash('sha256').update(name).digest('hex').slice(0, 12); + return `${safePrefix}-${hash}`; +} + +function runResolverProcess( + resolver: RepoResolverConfig, + request: RepoResolverRequest, + timeoutMs: number, +): Promise { + return new Promise((resolve, reject) => { + const [command, ...args] = resolver.command; + const child = spawn(command, args, { + cwd: resolver.cwd, + env: process.env, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + let settled = false; + let timedOut = false; + + const finish = (error?: Error): void => { + if (settled) return; + settled = true; + clearTimeout(timeoutHandle); + if (error) reject(error); + else resolve(stdout); + }; + + const timeoutHandle = setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + }, timeoutMs); + + child.stdout.on('data', (chunk: Buffer) => { + stdout = appendLimited(stdout, chunk); + }); + + child.stderr.on('data', (chunk: Buffer) => { + stderr = appendLimited(stderr, chunk); + }); + + child.on('error', (error) => { + finish(new Error(`Repo resolver '${resolver.name}' failed to start: ${error.message}`)); + }); + + child.on('close', (code, signal) => { + if (timedOut) { + finish(new Error(`Repo resolver '${resolver.name}' timed out after ${timeoutMs}ms.`)); + return; + } + if (code !== 0) { + const output = [stderr.trim(), stdout.trim()].filter(Boolean).join('\n'); + finish( + new Error( + `Repo resolver '${resolver.name}' exited with code ${code ?? 'unknown'}${signal ? ` (signal ${signal})` : ''}${output ? `:\n${output}` : ''}`, + ), + ); + return; + } + finish(); + }); + + child.stdin.end(`${JSON.stringify(request)}\n`); + }); +} + +function appendLimited(current: string, chunk: Buffer): string { + if (current.length >= RESOLVER_OUTPUT_LIMIT) return current; + const next = current + chunk.toString(); + return next.length > RESOLVER_OUTPUT_LIMIT ? next.slice(-RESOLVER_OUTPUT_LIMIT) : next; +} + +function parseRepoResolverOutput(stdout: string, resolverName: string): RepoResolverResult { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Repo resolver '${resolverName}' did not write valid JSON stdout: ${message}`); + } + + if (!isJsonObject(parsed)) { + throw new Error(`Repo resolver '${resolverName}' stdout must be a JSON object.`); + } + + const output = parsed as Record; + if (output.handled === false) { + return { handled: false }; + } + if (output.handled !== true) { + throw new Error(`Repo resolver '${resolverName}' stdout must set handled to true or false.`); + } + + if (!isJsonObject(output.source)) { + throw new Error(`Repo resolver '${resolverName}' handled the repo but did not return source.`); + } + + const source = output.source as Record; + if (source.type !== 'git') { + throw new Error(`Repo resolver '${resolverName}' returned unsupported source.type.`); + } + if (typeof source.path !== 'string' || source.path.trim().length === 0) { + throw new Error(`Repo resolver '${resolverName}' source.path must be a non-empty string.`); + } + if (source.origin !== undefined && typeof source.origin !== 'string') { + throw new Error(`Repo resolver '${resolverName}' source.origin must be a string when set.`); + } + + return { + handled: true, + source: { + type: 'git', + path: source.path, + ...(typeof source.origin === 'string' && { origin: source.origin }), + }, + }; +} diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index 3874adca1..68328e50e 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -64,6 +64,64 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); + it('accepts repo_resolvers field without warnings', async () => { + const filePath = path.join(tempDir, 'config-repo-resolvers.yaml'); + await writeFile( + filePath, + `repo_resolvers: + - name: org_snapshots + repos: + - https://github.com/example/* + command: + - bun + - scripts/repo-resolver.ts + config: + release_tag: snapshot/v1 + - name: default + command: + - bun + - scripts/default-repo-resolver.ts +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('errors on invalid repo_resolvers config', async () => { + const filePath = path.join(tempDir, 'config-invalid-repo-resolvers.yaml'); + await writeFile( + filePath, + `repo_resolvers: + - name: duplicate + command: [] + - name: duplicate + command: + - bun + - name: default + repos: + - https://github.com/example/* + command: + - bun + config: [] +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ severity: 'error', location: 'repo_resolvers[0].command' }), + expect.objectContaining({ severity: 'error', location: 'repo_resolvers[1].name' }), + expect.objectContaining({ severity: 'error', location: 'repo_resolvers[2].repos' }), + expect.objectContaining({ severity: 'error', location: 'repo_resolvers[2].config' }), + ]), + ); + }); + it('accepts dashboard field without warnings', async () => { const filePath = path.join(tempDir, 'config-dashboard.yaml'); await writeFile( diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index c10317ebb..72ba310e2 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -306,6 +306,7 @@ workspace: repos: - path: ./repo-a repo: https://github.com/org/repo.git + resolver: custom commit: main ancestor: 1 sparse: @@ -322,6 +323,7 @@ tests: expect(workspace?.repos).toHaveLength(1); expect(workspace?.repos?.[0].path).toBe('./repo-a'); expect(workspace?.repos?.[0].repo).toBe('https://github.com/org/repo.git'); + expect(workspace?.repos?.[0].resolver).toBe('custom'); expect(workspace?.repos?.[0].commit).toBe('main'); expect(workspace?.repos?.[0].ancestor).toBe(1); expect(workspace?.repos?.[0].sparse).toEqual(['src/**']); diff --git a/packages/core/test/evaluation/workspace/repo-manager.test.ts b/packages/core/test/evaluation/workspace/repo-manager.test.ts index 0ce1f12a4..f2facc09f 100644 --- a/packages/core/test/evaluation/workspace/repo-manager.test.ts +++ b/packages/core/test/evaluation/workspace/repo-manager.test.ts @@ -87,6 +87,75 @@ function writeMirrorConfig(configFilePath: string, mirrors: Record; +} + +function writeResolverScript(scriptPath: string): void { + mkdirSync(path.dirname(scriptPath), { recursive: true }); + writeFileSync( + scriptPath, + [ + "import { appendFileSync } from 'node:fs';", + 'const input = await new Response(Bun.stdin.stream()).text();', + 'const request = JSON.parse(input);', + 'if (request.config.request_log) {', + ' appendFileSync(request.config.request_log, `${JSON.stringify(request)}\\n`);', + '}', + 'if (request.config.handled === false) {', + ' process.stdout.write(JSON.stringify({ handled: false }));', + '} else {', + ' process.stdout.write(JSON.stringify({', + ' handled: true,', + ' source: {', + " type: 'git',", + ' path: request.config.source_path,', + ' origin: request.config.origin,', + ' },', + ' }));', + '}', + '', + ].join('\n'), + ); +} + +function writeRepoResolversConfig( + configFilePath: string, + resolvers: readonly ResolverConfigFixture[], +): void { + mkdirSync(path.dirname(configFilePath), { recursive: true }); + if (resolvers.length === 0) { + writeFileSync(configFilePath, 'repo_resolvers: []\n'); + return; + } + writeFileSync( + configFilePath, + [ + 'repo_resolvers:', + ...resolvers.flatMap((resolver) => [ + ` - name: ${JSON.stringify(resolver.name)}`, + ' command:', + ...resolver.command.map((entry) => ` - ${JSON.stringify(entry)}`), + ...(resolver.repos + ? [' repos:', ...resolver.repos.map((entry) => ` - ${JSON.stringify(entry)}`)] + : []), + ...(resolver.config + ? [ + ' config:', + ...Object.entries(resolver.config).map( + ([key, value]) => ` ${key}: ${JSON.stringify(value)}`, + ), + ] + : []), + ]), + '', + ].join('\n'), + ); +} + function findConfiguredMirrorFor(manager: RepoManager, repo: string): string | undefined { const findConfiguredMirror = ( manager as unknown as { @@ -291,6 +360,312 @@ describe('RepoManager', () => { }); }); + describe('repo resolvers', () => { + it('uses the first matching non-default resolver before built-in git acquisition', async () => { + const sourceRepo = path.join(tmpDir, 'resolver-source'); + createTestRepo(sourceRepo, { 'resolved.txt': 'from resolver' }); + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'github_org', + repos: ['https://github.com/example/*'], + command: ['bun', scriptPath], + config: { source_path: sourceRepo }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await projectManager.materialize( + { + path: './resolved', + repo: 'https://github.com/example/unreachable.git', + }, + workspaceDir, + ); + + const targetDir = path.join(workspaceDir, 'resolved'); + expect(readFileSync(path.join(targetDir, 'resolved.txt'), 'utf-8')).toBe('from resolver'); + expect(gitExec('git remote get-url origin', targetDir)).toBe( + 'https://github.com/example/unreachable.git', + ); + }, 30_000); + + it('uses an explicit workspace repo resolver even when the resolver has no repos pattern', async () => { + const sourceRepo = path.join(tmpDir, 'explicit-source'); + createTestRepo(sourceRepo, { 'explicit.txt': 'selected explicitly' }); + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-explicit'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'inline_only', + command: ['bun', scriptPath], + config: { source_path: sourceRepo }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await projectManager.materialize( + { + path: './explicit', + repo: 'https://github.com/other/repo.git', + resolver: 'inline_only', + }, + workspaceDir, + ); + + expect(readFileSync(path.join(workspaceDir, 'explicit', 'explicit.txt'), 'utf-8')).toBe( + 'selected explicitly', + ); + }, 30_000); + + it('sends the stable stdin protocol and clones from resolver stdout git source', async () => { + const sourceRepo = path.join(tmpDir, 'protocol-source'); + const firstCommit = createTestRepo(sourceRepo, { 'src/main.ts': 'first' }); + writeFileSync(path.join(sourceRepo, 'src', 'extra.ts'), 'second'); + execSync('git add -A && git commit -m "second"', { cwd: sourceRepo, ...EXEC_OPTS }); + const secondCommit = gitExec('git rev-parse HEAD', sourceRepo); + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + const requestLog = path.join(tmpDir, 'resolver-requests.jsonl'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-protocol'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'protocol', + repos: ['github.com/example/protocol'], + command: ['bun', scriptPath], + config: { + source_path: sourceRepo, + request_log: requestLog, + custom_value: 'kept', + }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await projectManager.materialize( + { + path: './protocol', + repo: 'https://github.com/example/protocol.git', + commit: secondCommit, + ancestor: 1, + sparse: ['src'], + }, + workspaceDir, + ); + + const request = JSON.parse(readFileSync(requestLog, 'utf-8').trim()); + expect(request).toMatchObject({ + version: 1, + repo: 'https://github.com/example/protocol.git', + commit: secondCommit, + path: './protocol', + sparse: ['src'], + ancestor: 1, + workspace_path: workspaceDir, + config: expect.objectContaining({ custom_value: 'kept' }), + }); + expect(request.cache_dir).toContain(path.join('cache', 'repo-resolvers', 'protocol-')); + expect(existsSync(request.cache_dir)).toBe(true); + + const targetDir = path.join(workspaceDir, 'protocol'); + expect(gitExec('git rev-parse HEAD', targetDir)).toBe(firstCommit); + expect(existsSync(path.join(targetDir, 'src', 'main.ts'))).toBe(true); + }, 30_000); + + it('continues from handled:false pattern resolvers to the default resolver', async () => { + const defaultSource = path.join(tmpDir, 'default-source'); + createTestRepo(defaultSource, { 'default.txt': 'from default' }); + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-default'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'pattern', + repos: ['https://github.com/example/*'], + command: ['bun', scriptPath], + config: { handled: false }, + }, + { + name: 'default', + command: ['bun', scriptPath], + config: { source_path: defaultSource }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await projectManager.materialize( + { + path: './defaulted', + repo: 'https://github.com/example/defaulted.git', + }, + workspaceDir, + ); + + expect(readFileSync(path.join(workspaceDir, 'defaulted', 'default.txt'), 'utf-8')).toBe( + 'from default', + ); + }, 30_000); + + it('falls back to built-in git acquisition when the default resolver returns handled:false', async () => { + const sourceRepo = path.join(tmpDir, 'builtin-source'); + createTestRepo(sourceRepo, { 'builtin.txt': 'from built-in' }); + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-built-in'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'default', + command: ['bun', scriptPath], + config: { handled: false }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await projectManager.materialize( + { + path: './builtin', + repo: `file://${sourceRepo}`, + }, + workspaceDir, + ); + + expect(readFileSync(path.join(workspaceDir, 'builtin', 'builtin.txt'), 'utf-8')).toBe( + 'from built-in', + ); + }, 30_000); + + it('fails when an explicitly selected resolver returns handled:false', async () => { + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-explicit-false'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'explicit_false', + command: ['bun', scriptPath], + config: { handled: false }, + }, + ]); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await expect( + projectManager.materialize( + { + path: './explicit-false', + repo: 'https://github.com/example/explicit-false.git', + resolver: 'explicit_false', + }, + workspaceDir, + ), + ).rejects.toThrow( + "Repo resolver 'explicit_false' was selected by workspace.repos[].resolver but returned handled:false.", + ); + }, 30_000); + + it('fails clearly when inline resolver names are unknown', async () => { + const projectDir = path.join(tmpDir, 'project-missing-resolver'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), []); + + const projectManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await expect( + projectManager.materialize( + { + path: './missing', + repo: 'https://github.com/example/missing.git', + resolver: 'missing', + }, + workspaceDir, + ), + ).rejects.toThrow("workspace.repos[].resolver 'missing' is not configured."); + }, 30_000); + + it('rejects duplicate resolver names and repos on the default resolver', async () => { + const scriptPath = path.join(tmpDir, 'scripts', 'resolver.ts'); + writeResolverScript(scriptPath); + const projectDir = path.join(tmpDir, 'project-invalid-resolvers'); + const evalDir = path.join(projectDir, 'evals'); + mkdirSync(path.join(projectDir, '.git'), { recursive: true }); + mkdirSync(evalDir, { recursive: true }); + + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { name: 'duplicate', command: ['bun', scriptPath] }, + { name: 'duplicate', command: ['bun', scriptPath] }, + ]); + const duplicateManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await expect( + duplicateManager.materialize( + { path: './duplicate', repo: 'https://github.com/example/duplicate.git' }, + workspaceDir, + ), + ).rejects.toThrow("Duplicate repo resolver name 'duplicate'."); + + writeRepoResolversConfig(path.join(projectDir, '.agentv', 'config.yaml'), [ + { + name: 'default', + repos: ['https://github.com/example/*'], + command: ['bun', scriptPath], + }, + ]); + const defaultReposManager = new RepoManager(false, { + progress: false, + projectConfigDir: evalDir, + }); + await expect( + defaultReposManager.materialize( + { path: './default', repo: 'https://github.com/example/default.git' }, + workspaceDir, + ), + ).rejects.toThrow("Repo resolver named 'default' must not declare repos."); + }, 30_000); + }); + describe('materialize', () => { it('clones repo into workspace at specified path', async () => { const repoDir = path.join(tmpDir, 'source-repo'); diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index 38b870848..23af1882e 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -161,9 +161,9 @@ execution: workspace_path: /path/to/local/workspace ``` -Shared repo workspaces are pooled by default. Use -`--workspace-mode temp` or `execution.workspace_mode: temp` in local config to -force fresh temporary materialization for a local run. Use +Shared repo workspaces use fresh temp materialization by default. Use +`--workspace-mode pooled` or `execution.workspace_mode: pooled` in local config +only when pool-slot reuse is intentional. Use `--workspace-path` or `execution.workspace_path` when an existing directory should be used as-is. diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index b1d42da8d..dc15ecd4d 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -373,13 +373,14 @@ metadata without a matching workspace repo pin is not an operational checkout. ### Repository Lifecycle -Materialize repos into the eval workspace automatically. Repo entries declare identity and checkout pins only; AgentV resolves acquisition from registered projects, `git_cache.mirrors`, its mirror cache, then remote clone. `git_cache.mirrors` may be defined in `$AGENTV_HOME/config.yaml`, the project's committed `.agentv/config.yaml`, or a gitignored `.agentv/config.override.yaml` (highest precedence) — use the override for machine-specific local clone paths without editing tracked or user-global config. For shared repo workspaces, pooling is the default: +Materialize repos into the eval workspace automatically. Repo entries declare identity and checkout pins only; AgentV resolves acquisition from configured `repo_resolvers`, then registered projects, `git_cache.mirrors`, its mirror cache, and remote clone. `repo_resolvers` and `git_cache.mirrors` may be defined in `$AGENTV_HOME/config.yaml`, the project's committed `.agentv/config.yaml`, or a gitignored `.agentv/config.override.yaml` (highest precedence) — use the override for machine-specific local bindings without editing tracked or user-global config. Shared repo workspaces use fresh temp materialization by default: ```yaml workspace: repos: - path: ./repo repo: https://github.com/org/repo.git + resolver: org_snapshots # optional repo_resolvers[].name override commit: main ancestor: 1 # parent commit hooks: @@ -389,13 +390,14 @@ workspace: ``` - `repo`: full clone URL or GitHub `org/name` shorthand +- `resolver`: optional configured repo resolver name; omit for pattern/default/built-in selection - `commit`: branch, tag, or SHA to check out - `base_commit`: alias for `commit` for SWE-bench-style datasets - `ancestor`: walk N commits back from the checked-out ref - `sparse`: sparse checkout paths array - Do not use legacy `source`, `type`, `checkout`, `resolve`, or `clone` fields under `workspace.repos[]` - Do not author `workspace.mode`, `workspace.path`, `experiment.workspace`, or `execution.workspace` in eval YAML -- Shared repo workspaces are pooled by default; use `--workspace-mode temp` or `.agentv/config.local.yaml` with `execution.workspace_mode: temp` for a local fresh-clone run +- Shared repo workspaces use fresh temp materialization by default; use `--workspace-mode pooled` or local `execution.workspace_mode: pooled` only when pool-slot reuse is intentional - Existing local workspace directories are machine-local bindings; use `--workspace-path` or `.agentv/config.local.yaml` with `execution.workspace_path` - `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks - Pool reset defaults to `fast` (`git clean -fd`); use `--workspace-clean full` for strict reset (`git clean -fdx`) From ca23a0d2719f4a04a9b4efeb7f4df36e824c8651 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Jul 2026 08:49:34 +0200 Subject: [PATCH 3/4] add evaluate options budget --- apps/cli/test/eval.integration.test.ts | 33 +++++++++++++ .../docs/docs/evaluation/eval-files.mdx | 6 +-- .../docs/docs/evaluation/experiments.mdx | 13 ++++-- examples/features/trials/README.md | 3 +- .../features/trials/evals/dataset.eval.yaml | 3 +- .../showcase/multi-model-benchmark/README.md | 8 ++-- .../evals/benchmark.eval.yaml | 3 +- .../src/evaluation/loaders/config-loader.ts | 43 ++++++++++++++++- .../core/src/evaluation/run-budget-tracker.ts | 6 +-- .../evaluation/validation/eval-file.schema.ts | 7 +++ .../evaluation/validation/eval-validator.ts | 46 ++++++++++++++++++- packages/core/src/evaluation/yaml-parser.ts | 6 ++- .../evaluation/eval-inline-experiment.test.ts | 27 +++++++++++ .../evaluation/loaders/config-loader.test.ts | 12 ++++- .../validation/eval-file-schema.test.ts | 4 +- .../validation/eval-validator.test.ts | 2 + packages/sdk/src/eval.ts | 25 +++++++++- packages/sdk/test/eval-authoring.test.ts | 4 +- .../references/breaking-changes.md | 3 +- skills-data/agentv-eval-writer/SKILL.md | 5 +- .../references/eval.schema.json | 23 ++++++++++ 21 files changed, 253 insertions(+), 29 deletions(-) diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 3908a9e9c..a13d7c381 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -573,6 +573,39 @@ describe('agentv eval CLI', () => { } }, 30_000); + it('lets --budget-usd override evaluate_options.budget_usd', async () => { + const fixture = await createFixture(); + try { + const evalPath = path.join(fixture.suiteDir, 'budget-options.eval.yaml'); + await writeFile( + evalPath, + [ + 'description: Budget options integration test', + 'target: file-target', + 'evaluate_options:', + ' budget_usd: 1.25', + 'tests:', + ' - id: case-alpha', + ' criteria: System responds with alpha', + ' input: alpha', + '', + ].join('\n'), + 'utf8', + ); + + await runCli(fixture, ['eval', evalPath, '--budget-usd', '0.5']); + + const diagnostics = await readDiagnostics(fixture); + expect(diagnostics).toMatchObject({ + budgetUsd: null, + hasRunBudgetTracker: true, + runBudgetCapUsd: 0.5, + }); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + }, 30_000); + it('runs eval-local target config with suite test selection and run knobs', async () => { const fixture = await createFixture(); try { diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx index 67a2175e1..43dd6cae8 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx @@ -5,7 +5,7 @@ sidebar: order: 1 --- -Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, and `budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. +Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, and `evaluate_options.budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. Eval files describe the task, target binding, and run controls. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML. @@ -25,7 +25,7 @@ experiment format. suite context applies because raw cases do not carry their own suite context. - A **wrapper eval** is eval YAML that imports one or more suites with `imports.suites` and binds run controls with top-level `target`, `repeat`, - `threshold`, `timeout_seconds`, and `budget_usd`. + `threshold`, `timeout_seconds`, and `evaluate_options`. Wrapper evals can live anywhere in the repo. A wrapper that imports suites with `imports.suites` must not define parent `workspace`; imported suites own task environment. Machine-local existing workspace paths belong in CLI flags @@ -119,7 +119,7 @@ tests: | `experiment` | Optional run/result grouping label | | `repeat` | Optional repeat policy with `count`, `strategy`, and `early_exit` | | `timeout_seconds` | Optional per-case timeout | -| `budget_usd` | Optional suite budget | +| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` | | `threshold` | Optional suite quality threshold | | `workspace` | Suite-level task environment — inline object or string path to an [external workspace file](/docs/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | | `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. | diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index ce73f8947..3daff07a7 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -8,7 +8,8 @@ sidebar: AgentV eval files are the runnable authoring artifact. Use top-level `description` for display metadata, `experiment` as the run/result grouping label, `target` for the system under test, and flat top-level run controls such -as `repeat`, `timeout_seconds`, `budget_usd`, and `threshold`. +as `repeat`, `timeout_seconds`, and `threshold`. Use `evaluate_options` for +evaluation runtime options such as `budget_usd`. Concurrency is outside eval YAML. Use `agentv eval --workers N` or project config defaults such as `agentv.config.*` / `.agentv/config.yaml` `execution.workers` for operator-side parallelism. @@ -25,7 +26,8 @@ repeat: count: 4 strategy: pass_any timeout_seconds: 720 -budget_usd: 2.00 +evaluate_options: + budget_usd: 2.00 workspace: hooks: @@ -181,8 +183,9 @@ tests: budget_usd: 0.50 ``` -Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and -`budget_usd` for public eval authoring. Candidate-changing fields stay +Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and legacy +per-case `budget_usd` overrides. Parent suite budgets should use +`evaluate_options.budget_usd` for public eval authoring. Candidate-changing fields stay parent-level. Workspace mutation belongs in `workspace.hooks`, and provider-specific setup belongs in target configuration. @@ -198,7 +201,7 @@ target-specific runner state. | Configure an agent runner or provider variant | `target` object or `targets.yaml` | | Choose the target | top-level `target` | | Override the target's default model | `target.model` | -| Configure repeat policy, budget, timeout, threshold | top-level `repeat`, `budget_usd`, `timeout_seconds`, `threshold` | +| Configure repeat policy, budget, timeout, threshold | top-level `repeat`, `evaluate_options.budget_usd`, `timeout_seconds`, `threshold` | | Bind an existing local workspace directory | `--workspace-path` or `.agentv/config.local.yaml` | ```yaml diff --git a/examples/features/trials/README.md b/examples/features/trials/README.md index 4d8ace579..95660ee04 100644 --- a/examples/features/trials/README.md +++ b/examples/features/trials/README.md @@ -20,5 +20,6 @@ repeat: count: 2 strategy: pass_any early_exit: false -budget_usd: 1.00 +evaluate_options: + budget_usd: 1.00 ``` diff --git a/examples/features/trials/evals/dataset.eval.yaml b/examples/features/trials/evals/dataset.eval.yaml index 0fb710021..9f1c38b32 100644 --- a/examples/features/trials/evals/dataset.eval.yaml +++ b/examples/features/trials/evals/dataset.eval.yaml @@ -9,7 +9,8 @@ repeat: count: 2 strategy: pass_any early_exit: false -budget_usd: 1.00 +evaluate_options: + budget_usd: 1.00 tests: - id: math-basics diff --git a/examples/showcase/multi-model-benchmark/README.md b/examples/showcase/multi-model-benchmark/README.md index 6efa05c19..709de04f8 100644 --- a/examples/showcase/multi-model-benchmark/README.md +++ b/examples/showcase/multi-model-benchmark/README.md @@ -46,7 +46,7 @@ bun agentv eval examples/showcase/multi-model-benchmark/evals/benchmark.eval.yam ### Cost & Safety -The eval uses a **low-cost model by default**. For each target, 5 tests × 2 repeat attempts × 3 grader calls is roughly **30 LLM calls**. A `budget_usd: 2.00` cap is set in the eval file. +The eval uses a **low-cost model by default**. For each target, 5 tests × 2 repeat attempts × 3 grader calls is roughly **30 LLM calls**. An `evaluate_options.budget_usd: 2.00` cap is set in the eval file. To run against a single target first: @@ -139,7 +139,8 @@ repeat: count: 2 strategy: pass_any early_exit: false -budget_usd: 2.00 +evaluate_options: + budget_usd: 2.00 ``` This surfaces non-determinism — if a model passes on run 1 but fails on run 2, @@ -210,7 +211,8 @@ repeat: count: 5 strategy: pass_any early_exit: false -budget_usd: 5.00 +evaluate_options: + budget_usd: 5.00 ``` ## See Also diff --git a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml index eaa4de8fc..7f51ddd98 100644 --- a/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml +++ b/examples/showcase/multi-model-benchmark/evals/benchmark.eval.yaml @@ -20,7 +20,8 @@ repeat: count: 2 strategy: pass_any early_exit: false -budget_usd: 2.00 +evaluate_options: + budget_usd: 2.00 assertions: - name: accuracy diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index b3fe62b41..701279a5a 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -265,7 +265,7 @@ function rejectAuthoredRuntimeContainers(suite: JsonObject): void { } if (suite.policy !== undefined) { throw new Error( - "Top-level 'policy' is not part of eval YAML. Put repeat, timeout_seconds, threshold, and budget_usd at the top level.", + "Top-level 'policy' is not part of eval YAML. Put repeat, timeout_seconds, and threshold at the top level, and budget_usd under evaluate_options.", ); } if (suite.execution !== undefined) { @@ -293,6 +293,32 @@ function getSuiteTopLevelNumber( return undefined; } +function getSuiteEvaluateOptionsNumber( + suite: JsonObject, + field: string, + validate: (value: number) => boolean, + label: string, +): number | undefined { + rejectAuthoredRuntimeContainers(suite); + const rawOptions = suite.evaluate_options; + if (rawOptions === undefined || rawOptions === null) { + return undefined; + } + if (!isJsonObject(rawOptions)) { + logWarning('Invalid evaluate_options: expected object. Ignoring.'); + return undefined; + } + const raw = rawOptions[field]; + if (raw === undefined || raw === null) { + return undefined; + } + if (typeof raw === 'number' && validate(raw)) { + return raw; + } + logWarning(`Invalid evaluate_options.${label}: ${raw}. Ignoring.`); + return undefined; +} + /** Extract the single top-level target name from a parsed eval suite. */ export function extractTargetFromSuite(suite: JsonObject): string | undefined { rejectAuthoredRuntimeContainers(suite); @@ -421,10 +447,23 @@ export function extractCacheConfig(suite: JsonObject): CacheConfig | undefined { } /** - * Extract suite-level total budget from top-level eval YAML. + * Extract suite-level total budget from eval YAML. + * + * Preferred authoring uses evaluate_options.budget_usd. Legacy top-level + * budget_usd remains accepted for compatibility, but the nested option wins + * when both are present. * Returns undefined when not specified. */ export function extractBudgetUsd(suite: JsonObject): number | undefined { + const evaluateOptionsBudgetUsd = getSuiteEvaluateOptionsNumber( + suite, + 'budget_usd', + (value) => value > 0, + 'budget_usd. Must be a positive number', + ); + if (evaluateOptionsBudgetUsd !== undefined) { + return evaluateOptionsBudgetUsd; + } return getSuiteTopLevelNumber( suite, 'budget_usd', diff --git a/packages/core/src/evaluation/run-budget-tracker.ts b/packages/core/src/evaluation/run-budget-tracker.ts index 66ec4fdbc..df49621b3 100644 --- a/packages/core/src/evaluation/run-budget-tracker.ts +++ b/packages/core/src/evaluation/run-budget-tracker.ts @@ -1,9 +1,9 @@ /** * Tracks cumulative cost across all eval files in a single CLI run. * - * The per-suite budget (`execution.budget_usd` in YAML) is enforced by the orchestrator - * and caps spend within one eval file. This tracker provides a **run-level** cap that - * spans all files in a single `agentv run` invocation. + * The per-suite budget (`evaluate_options.budget_usd` in YAML) is enforced by the + * orchestrator and caps spend within one eval file. This tracker provides a **run-level** + * cap that spans all files in a single `agentv run` invocation. * * Usage: * 1. Instantiate with the cap from `--budget-usd`. diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 7487d693a..3b364aa44 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -411,6 +411,12 @@ const DefaultTestSchema = z }) .strict(); +const EvaluateOptionsSchema = z + .object({ + budget_usd: z.number().gt(0).optional(), + }) + .strict(); + /** Per-turn assertion: string shorthand (becomes rubric) or full evaluator config */ const TurnAssertionSchema = z.union([z.string(), EvaluatorSchema]); @@ -539,6 +545,7 @@ export const EvalFileSchema = z runs: z.never().optional(), early_exit: z.never().optional(), timeout_seconds: z.number().gt(0).optional(), + evaluate_options: EvaluateOptionsSchema.optional(), budget_usd: z.number().gt(0).optional(), threshold: z.number().min(0).max(1).optional(), default_test: DefaultTestSchema.optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 95e6f8da1..f9a7d9d24 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -72,6 +72,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'runs', 'early_exit', 'timeout_seconds', + 'evaluate_options', 'budget_usd', 'threshold', 'default_test', @@ -114,7 +115,7 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."], [ 'policy', - "Top-level 'policy' is not part of eval YAML. Put repeat, timeout_seconds, threshold, and budget_usd at the top level.", + "Top-level 'policy' is not part of eval YAML. Put repeat, timeout_seconds, and threshold at the top level, and budget_usd under evaluate_options.", ], [ 'execution', @@ -333,6 +334,7 @@ export async function validateEvalFile(filePath: string): Promise { expect(suite.experimentConfig?.threshold).toBe(0.9); }); + it('parses evaluate_options.budget_usd and prefers it over legacy top-level budget_usd', async () => { + const evalPath = path.join(tempDir, 'evaluate-options-budget.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: evaluate-options-budget-suite', + 'target: codex', + 'budget_usd: 99', + 'evaluate_options:', + ' budget_usd: 2.5', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.budgetUsd).toBe(2.5); + expect(suite.experimentConfig).toMatchObject({ + target: 'codex', + budgetUsd: 2.5, + }); + }); + it('rejects authored workers in eval YAML runtime blocks', async () => { const cases = [ { diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index a37456e9b..5c67bf759 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -555,7 +555,17 @@ describe('extractBudgetUsd', () => { expect(extractBudgetUsd(suite)).toBeUndefined(); }); - it('parses valid top-level budget_usd', () => { + it('parses valid evaluate_options.budget_usd', () => { + const suite: JsonObject = { evaluate_options: { budget_usd: 10.0 } }; + expect(extractBudgetUsd(suite)).toBe(10.0); + }); + + it('prefers evaluate_options.budget_usd over legacy top-level budget_usd', () => { + const suite: JsonObject = { evaluate_options: { budget_usd: 2.5 }, budget_usd: 10.0 }; + expect(extractBudgetUsd(suite)).toBe(2.5); + }); + + it('parses legacy top-level budget_usd', () => { const suite: JsonObject = { budget_usd: 10.0 }; expect(extractBudgetUsd(suite)).toBe(10.0); }); diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index f12719062..7da3f1ced 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -106,7 +106,9 @@ describe('EvalFileSchema input shorthand', () => { early_exit: true, }, timeout_seconds: 300, - budget_usd: 2, + evaluate_options: { + budget_usd: 2, + }, tests: [ { include: './evals/**/*.eval.yaml', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 5d11f38f9..9586ad19e 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -41,6 +41,8 @@ describe('validateEvalFile', () => { `name: wrapper target: codex threshold: 0.8 +evaluate_options: + budget_usd: 2 repeat: count: 2 strategy: pass_any diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 7fb17f19d..1a925a8d0 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -243,6 +243,29 @@ function lowerEvalYamlValue(value: unknown): unknown { return value; } +function lowerEvalDefinition(definition: unknown): Record { + const lowered = lowerEvalYamlValue(definition) as Record; + const { budget_usd: budgetUsd, ...loweredWithoutBudget } = lowered; + if (budgetUsd === undefined) { + return lowered; + } + + const evaluateOptions = + lowered.evaluate_options && + typeof lowered.evaluate_options === 'object' && + !Array.isArray(lowered.evaluate_options) + ? { ...(lowered.evaluate_options as Record) } + : {}; + + if (evaluateOptions.budget_usd === undefined) { + evaluateOptions.budget_usd = budgetUsd; + } + return { + ...loweredWithoutBudget, + evaluate_options: evaluateOptions, + }; +} + function attachEvalSuiteBrand(definition: T): T & DefinedEvalSuite { validateTopLevelRuntimeFields(definition); const branded = definition as T & Partial; @@ -314,7 +337,7 @@ export function evalSuite(definition: T): T & DefinedE export function toEvalYamlObject( definition: T, ): LowerEvalYamlValue { - return lowerEvalYamlValue(definition) as LowerEvalYamlValue; + return lowerEvalDefinition(definition) as LowerEvalYamlValue; } /** diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 1b5a52264..4fd98f546 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -110,7 +110,9 @@ describe('YAML-aligned eval authoring helpers', () => { }, timeout_seconds: 600, threshold: 0.8, - budget_usd: 1.5, + evaluate_options: { + budget_usd: 1.5, + }, assertions: [ { type: 'execution-metrics', diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index 23af1882e..509388e3f 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -40,7 +40,8 @@ repeat: timeout_seconds: 600 threshold: 0.8 -budget_usd: 5 +evaluate_options: + budget_usd: 5 ``` ## Repeat Policy Uses `repeat` diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index dc15ecd4d..13afe93e2 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -28,7 +28,8 @@ shorthand for direct paths, directories, and globs. Legacy `tests[].include` entries still load with a migration warning, but new evals should use `imports.suites` or `imports.tests`. Use scoped `run:` on import entries or individual tests only for `threshold`, `repeat`, `timeout_seconds`, and -`budget_usd`; keep target selection at top-level `target` or CLI `--target`, +legacy `budget_usd`; keep target selection at top-level `target` or CLI `--target`, +put suite budget caps under `evaluate_options.budget_usd`, and keep setup and workspace mutation under `workspace`. Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it was a deprecated compatibility package and has been removed from this repository. @@ -124,7 +125,7 @@ tests: ## Eval File Structure **Required:** `tests` (array or string raw-case path) or `imports` -**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `repeat`, `timeout_seconds`, `budget_usd`, `threshold`, `suite`, `workspace`, `assertions`, `input` +**Optional:** `name`, `description`, `experiment`, `version`, `author`, `tags`, `license`, `requires`, `target`, `repeat`, `timeout_seconds`, `evaluate_options`, `threshold`, `suite`, `workspace`, `assertions`, `input` **Test fields:** diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 0eee22f5d..2f743a7d8 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -5668,6 +5668,10 @@ "items": { "type": "string" } + }, + "resolver": { + "type": "string", + "minLength": 1 } }, "additionalProperties": false @@ -12294,6 +12298,10 @@ "items": { "type": "string" } + }, + "resolver": { + "type": "string", + "minLength": 1 } }, "additionalProperties": false @@ -14245,6 +14253,17 @@ "exclusiveMinimum": true, "minimum": 0 }, + "evaluate_options": { + "type": "object", + "properties": { + "budget_usd": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "additionalProperties": false + }, "budget_usd": { "type": "number", "exclusiveMinimum": true, @@ -15510,6 +15529,10 @@ "items": { "type": "string" } + }, + "resolver": { + "type": "string", + "minLength": 1 } }, "additionalProperties": false From 3903bfa2f20941b9439dd5decb1194cd146f4ea5 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 1 Jul 2026 13:48:46 +0200 Subject: [PATCH 4/4] feat(eval): support max concurrency eval option Entire-Checkpoint: 96eca76f596a --- README.md | 3 +- apps/cli/src/commands/eval/run-eval.ts | 8 +++-- apps/cli/test/eval.integration.test.ts | 33 +++++++++++++++++++ .../docs/docs/evaluation/eval-files.mdx | 6 ++-- .../docs/docs/evaluation/experiments.mdx | 14 ++++---- packages/core/src/evaluation/experiment.ts | 2 +- .../src/evaluation/loaders/config-loader.ts | 16 ++++++--- .../evaluation/validation/eval-file.schema.ts | 1 + .../evaluation/validation/eval-validator.ts | 28 ++++++++++++---- packages/core/src/evaluation/yaml-parser.ts | 6 ++-- .../evaluation/eval-inline-experiment.test.ts | 22 +++++++++++++ .../evaluation/loaders/config-loader.test.ts | 23 +++++++++++++ .../validation/eval-file-schema.test.ts | 13 ++++++++ .../validation/eval-validator.test.ts | 26 +++++++++++++++ skills-data/agentv-eval-writer/SKILL.md | 4 ++- .../references/eval.schema.json | 5 +++ 16 files changed, 183 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 329b84fba..62b01b1b2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Test AI targets on real repo tasks and measure what actually works. - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts. - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by name from `targets.yaml` or with an eval-local target object. - **Experiment** is the run/result grouping label being measured over that corpus, such as `with-skills` or `without-skills`. Keep suite/category and target/model names out of this label. -- **Evaluate options** configure runner-level behavior such as repeat policy and optional timeouts under `evaluate_options`. +- **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`. - **Default test** configures inherited per-test defaults such as score `threshold`. - **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. @@ -74,6 +74,7 @@ evaluate_options: count: 3 strategy: pass_any early_exit: false + max_concurrency: 3 default_test: threshold: 0.8 diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 88af2f0be..f15fe9b46 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1093,7 +1093,11 @@ async function prepareFileMetadata(params: { filter: suiteFilter ?? options.filter, category, }); - const effectiveOptions = applyExperimentOptions(options, suite.experimentConfig); + const experimentOptions = applyExperimentOptions(options, suite.experimentConfig); + const effectiveOptions = + experimentOptions.workers === undefined && suite.workers !== undefined + ? { ...experimentOptions, workers: suite.workers } + : experimentOptions; const testCases = suiteFilter && effectiveOptions.filter ? suite.tests.filter((testCase) => @@ -1106,7 +1110,7 @@ async function prepareFileMetadata(params: { const defaultBudgetUsd = effectiveOptions.cliBudgetUsd === undefined ? (effectiveOptions.budgetUsd ?? suite.budgetUsd) - : suite.budgetUsd; + : undefined; const suiteDefaultThreshold = suite.defaultTest?.threshold ?? suite.threshold; if (testCases.length === 0) { diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index a13d7c381..00d49a159 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -898,6 +898,39 @@ describe('agentv eval CLI', () => { } }, 30_000); + it('uses evaluate_options.max_concurrency as the eval-authored concurrency limit', async () => { + const fixture = await createFixture(); + try { + const evalPath = path.join(fixture.suiteDir, 'max-concurrency.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: max-concurrency', + 'target: file-target', + 'evaluate_options:', + ' max_concurrency: 2', + 'tests:', + ' - id: first-case', + ' input: first', + ' criteria: ok', + '', + ].join('\n'), + 'utf8', + ); + + const { exitCode } = await runCli(fixture, ['eval', evalPath]); + + expect(exitCode).toBe(0); + const diagnostics = await readDiagnostics(fixture); + expect(diagnostics).toMatchObject({ + maxConcurrency: 2, + evalCaseIds: ['first-case'], + }); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + }, 30_000); + it('records CLI-named experiment namespace separately from default runtime config', async () => { const fixture = await createFixture(); try { diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx index 43dd6cae8..87ce57f03 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx @@ -5,10 +5,10 @@ sidebar: order: 1 --- -Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, and `evaluate_options.budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. +Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. -Eval files describe the task, target binding, and run controls. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML. +Eval files describe the task, target binding, and run controls. Use `evaluate_options.max_concurrency` for authored suite concurrency. Operators can still override concurrency with `--workers` or set defaults with `execution.workers` in `agentv.config.*` / `.agentv/config.yaml`; do not author legacy `workers` fields in eval YAML. ## Authoring Shapes @@ -119,7 +119,7 @@ tests: | `experiment` | Optional run/result grouping label | | `repeat` | Optional repeat policy with `count`, `strategy`, and `early_exit` | | `timeout_seconds` | Optional per-case timeout | -| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` | +| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `max_concurrency` | | `threshold` | Optional suite quality threshold | | `workspace` | Suite-level task environment — inline object or string path to an [external workspace file](/docs/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | | `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. | diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 3daff07a7..f9be9e8a4 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -9,10 +9,10 @@ AgentV eval files are the runnable authoring artifact. Use top-level `description` for display metadata, `experiment` as the run/result grouping label, `target` for the system under test, and flat top-level run controls such as `repeat`, `timeout_seconds`, and `threshold`. Use `evaluate_options` for -evaluation runtime options such as `budget_usd`. -Concurrency is outside eval YAML. Use `agentv eval --workers N` or project -config defaults such as `agentv.config.*` / `.agentv/config.yaml` -`execution.workers` for operator-side parallelism. +evaluation runtime options such as `budget_usd` and `max_concurrency`. +Use `agentv eval --workers N` or project config defaults such as +`agentv.config.*` / `.agentv/config.yaml` `execution.workers` for operator-side +overrides. ```yaml name: support-regression @@ -28,6 +28,7 @@ repeat: timeout_seconds: 720 evaluate_options: budget_usd: 2.00 + max_concurrency: 3 workspace: hooks: @@ -185,7 +186,8 @@ tests: Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and legacy per-case `budget_usd` overrides. Parent suite budgets should use -`evaluate_options.budget_usd` for public eval authoring. Candidate-changing fields stay +`evaluate_options.budget_usd` for public eval authoring. Use +`evaluate_options.max_concurrency` for authored concurrency. Candidate-changing fields stay parent-level. Workspace mutation belongs in `workspace.hooks`, and provider-specific setup belongs in target configuration. @@ -201,7 +203,7 @@ target-specific runner state. | Configure an agent runner or provider variant | `target` object or `targets.yaml` | | Choose the target | top-level `target` | | Override the target's default model | `target.model` | -| Configure repeat policy, budget, timeout, threshold | top-level `repeat`, `evaluate_options.budget_usd`, `timeout_seconds`, `threshold` | +| Configure repeat policy, budget, concurrency, timeout, threshold | top-level `repeat`, `evaluate_options.budget_usd`, `evaluate_options.max_concurrency`, `timeout_seconds`, `threshold` | | Bind an existing local workspace directory | `--workspace-path` or `.agentv/config.local.yaml` | ```yaml diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index 4dbde224f..e954d3e6e 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -396,7 +396,7 @@ function rejectExperimentWorkers(raw: unknown): void { return; } throw new Error( - 'Experiment workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', + 'Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', ); } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 701279a5a..84038dc4a 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -270,7 +270,7 @@ function rejectAuthoredRuntimeContainers(suite: JsonObject): void { } if (suite.execution !== undefined) { throw new Error( - "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level; configure concurrency with CLI flags or project config.", + "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.", ); } } @@ -422,11 +422,19 @@ export function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined { } /** - * Eval YAML does not own concurrency. + * Extract suite-level max concurrency from eval YAML. + * + * Preferred authoring uses evaluate_options.max_concurrency, matching the + * lowest-common-denominator naming used by other eval runners. Internal + * TypeScript continues to pass this as workers/maxConcurrency at runtime. */ export function extractWorkersFromSuite(suite: JsonObject): number | undefined { - rejectAuthoredRuntimeContainers(suite); - return undefined; + return getSuiteEvaluateOptionsNumber( + suite, + 'max_concurrency', + (value) => Number.isInteger(value) && value >= 1 && value <= 50, + 'max_concurrency. Must be an integer between 1 and 50', + ); } /** diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3b364aa44..2528f1f50 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -414,6 +414,7 @@ const DefaultTestSchema = z const EvaluateOptionsSchema = z .object({ budget_usd: z.number().gt(0).optional(), + max_concurrency: z.number().int().min(1).max(50).optional(), }) .strict(); diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index f9a7d9d24..785dbfc46 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -110,7 +110,7 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ ['assert', "'assert' has been removed. Use 'assertions' instead."], [ 'workers', - "'workers' has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.", + "'workers' has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.", ], ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."], [ @@ -119,7 +119,7 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ ], [ 'execution', - "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level; configure concurrency with CLI flags or project config.", + "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.", ], ['runs', "Top-level 'runs' has been removed. Use repeat.count instead."], ['early_exit', "Top-level 'early_exit' has been removed. Use repeat.early_exit instead."], @@ -578,7 +578,7 @@ function validateTestExecutionFields( filePath, location: `${location}.execution.workers`, message: - 'tests[].execution.workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', + 'tests[].execution.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', }); continue; } @@ -624,7 +624,7 @@ function rejectWorkersField( severity: 'error', filePath, location: `${location}.workers`, - message: `${location}.workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, }); } rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors); @@ -647,7 +647,7 @@ function rejectTargetWorkers( severity: 'error', filePath, location: `${location}[${index}].workers`, - message: `${location}[${index}].workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, }); }); } @@ -1118,7 +1118,7 @@ function validateEvaluateOptions( } for (const key of Object.keys(evaluateOptions)) { - if (key !== 'budget_usd') { + if (key !== 'budget_usd' && key !== 'max_concurrency') { errors.push({ severity: 'warning', filePath, @@ -1137,6 +1137,22 @@ function validateEvaluateOptions( message: "Invalid 'budget_usd' field (must be a positive number)", }); } + + const maxConcurrency = evaluateOptions.max_concurrency; + if ( + maxConcurrency !== undefined && + (typeof maxConcurrency !== 'number' || + !Number.isInteger(maxConcurrency) || + maxConcurrency < 1 || + maxConcurrency > 50) + ) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.max_concurrency`, + message: "Invalid 'max_concurrency' field (must be an integer between 1 and 50)", + }); + } } function validateRepeatOverride( diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index b2dc2a2a5..d086ec80d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -346,7 +346,7 @@ export type EvalSuiteResult = { readonly targetRefs?: readonly import('./types.js').EvalTargetRef[]; /** Single authored target string or eval-local overlay object. */ readonly targetSpec?: EvalTargetSpec; - /** Suite-level workers from project config or CLI, not authored eval YAML. */ + /** Suite-level concurrency from evaluate_options.max_concurrency. */ readonly workers?: number; /** Suite-level cache config from project/CLI runtime surfaces. */ readonly cacheConfig?: import('./loaders/config-loader.js').CacheConfig; @@ -920,7 +920,7 @@ function rejectAuthoredWorkers(parsed: JsonObject): void { } throw new Error( - `${locations[0]} has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, ); } @@ -1460,7 +1460,7 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO } if (suite.execution !== undefined) { throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Put target and run controls at the top level; configure concurrency with CLI flags or project config.`, + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.`, ); } if (suite.model !== undefined) { diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index d7768354f..8027d5f9c 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -117,6 +117,28 @@ describe('eval.yaml flat runtime controls and tests imports', () => { }); }); + it('parses evaluate_options.max_concurrency as suite workers', async () => { + const evalPath = path.join(tempDir, 'evaluate-options-concurrency.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: evaluate-options-concurrency-suite', + 'target: codex', + 'evaluate_options:', + ' max_concurrency: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.workers).toBe(2); + }); + it('rejects authored workers in eval YAML runtime blocks', async () => { const cases = [ { diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 5c67bf759..403afe2d3 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -10,6 +10,7 @@ import { extractTargetRefsFromSuite, extractTargetsFromSuite, extractThreshold, + extractWorkersFromSuite, loadConfig, parseExecutionDefaults, parseResultsConfig, @@ -591,6 +592,28 @@ describe('extractBudgetUsd', () => { }); }); +describe('extractWorkersFromSuite', () => { + it('returns undefined when no max_concurrency', () => { + const suite: JsonObject = { tests: [] }; + expect(extractWorkersFromSuite(suite)).toBeUndefined(); + }); + + it('parses valid evaluate_options.max_concurrency', () => { + const suite: JsonObject = { evaluate_options: { max_concurrency: 5 } }; + expect(extractWorkersFromSuite(suite)).toBe(5); + }); + + it('returns undefined for invalid max_concurrency', () => { + const suite: JsonObject = { evaluate_options: { max_concurrency: 0 } }; + expect(extractWorkersFromSuite(suite)).toBeUndefined(); + }); + + it('rejects authored execution blocks', () => { + const suite: JsonObject = { execution: { workers: 5 } }; + expect(() => extractWorkersFromSuite(suite)).toThrow(/Top-level 'execution'/); + }); +}); + describe('extractFailOnError', () => { it('returns undefined for authored eval YAML', () => { const suite: JsonObject = { tests: [] }; diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 7da3f1ced..c0e4bba2b 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -108,6 +108,7 @@ describe('EvalFileSchema input shorthand', () => { timeout_seconds: 300, evaluate_options: { budget_usd: 2, + max_concurrency: 3, }, tests: [ { @@ -138,6 +139,18 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects invalid evaluate_options.max_concurrency', () => { + const result = EvalFileSchema.safeParse({ + target: 'codex', + evaluate_options: { + max_concurrency: 0, + }, + tests: [baseTest], + }); + + expect(result.success).toBe(false); + }); + it('accepts default_test.threshold as the preferred inherited test threshold', () => { const result = EvalFileSchema.safeParse({ default_test: { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 9586ad19e..b15ee15f6 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -43,6 +43,7 @@ target: codex threshold: 0.8 evaluate_options: budget_usd: 2 + max_concurrency: 3 repeat: count: 2 strategy: pass_any @@ -78,6 +79,31 @@ imports: expect(result.errors).toHaveLength(0); }); + it('rejects invalid evaluate_options.max_concurrency', async () => { + const filePath = path.join(tempDir, 'invalid-max-concurrency.yaml'); + await writeFile( + filePath, + `target: codex +evaluate_options: + max_concurrency: 0 +tests: + - id: local-case + input: "Hello" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.location === 'evaluate_options.max_concurrency' && + error.message.includes('integer between 1 and 50'), + ), + ).toBe(true); + }); + it('validates default_test.threshold', async () => { const filePath = path.join(tempDir, 'default-test-threshold.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 13afe93e2..fcdf0217f 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -29,7 +29,8 @@ entries still load with a migration warning, but new evals should use `imports.suites` or `imports.tests`. Use scoped `run:` on import entries or individual tests only for `threshold`, `repeat`, `timeout_seconds`, and legacy `budget_usd`; keep target selection at top-level `target` or CLI `--target`, -put suite budget caps under `evaluate_options.budget_usd`, +put suite budget caps under `evaluate_options.budget_usd`, authored concurrency +under `evaluate_options.max_concurrency`, and keep setup and workspace mutation under `workspace`. Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for new evals, examples, scaffolds, or skill guidance; it was a deprecated compatibility package and has been removed from this repository. @@ -750,6 +751,7 @@ Programmatic API notes: - Inline programmatic tests use `assert`, not `assertions`. - Use camelCase in TypeScript (`expectedOutput`, `beforeAll`, `budgetUsd`). +- In YAML, use `evaluate_options.max_concurrency` for authored eval concurrency; reserve `workers` for project/target runtime config. - When bridging from Python, generate canonical YAML/JSONL or call the CLI; there is no separate first-party Python authoring SDK. Supports inline tests or file-based usage via `specFile`. diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 2f743a7d8..76d1340cb 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -14260,6 +14260,11 @@ "type": "number", "exclusiveMinimum": true, "minimum": 0 + }, + "max_concurrency": { + "type": "integer", + "minimum": 1, + "maximum": 50 } }, "additionalProperties": false