fix(eval): default repo workspaces to fresh materialization

README.md

@@ -19,7 +19,7 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
 - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by name from `targets.yaml` or with an eval-local target object.
 - **Experiment** is the run/result grouping label being measured over that corpus, such as `with-skills` or `without-skills`. Keep suite/category and target/model names out of this label.
-- **Evaluate options** configure runner-level behavior such as repeat policy and optional timeouts under `evaluate_options`.
+- **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`.
 - **Default test** configures inherited per-test defaults such as score `threshold`.
 - **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
@@ -74,12 +74,17 @@ evaluate_options:
     count: 3
     strategy: pass_any
     early_exit: false
+  max_concurrency: 3
 
 default_test:
   threshold: 0.8
 
 workspace:
   isolation: per_case
+  repos:
+    - path: ./fixture
+      repo: EntityProcess/agentv-contract-fixture
+      commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb
 
 tests:
   - id: fizzbuzz
@@ -211,6 +216,13 @@ export default defineEval({
   threshold: 0.8,
   workspace: {
     isolation: 'per_case',
+    repos: [
+      {
+        path: './fixture',
+        repo: 'EntityProcess/agentv-contract-fixture',
+        commit: '21a34daed7ebcfe36cbed053607622a55e5e94cb',
+      },
+    ],
   },
   tests: [
     {

apps/cli/src/commands/eval/commands/run.ts

@@ -122,7 +122,7 @@ export const evalRunCommand = command({
     workspaceMode: option({
       type: optional(string),
       long: 'workspace-mode',
-      description: "Workspace mode: 'pooled', 'temp', or 'static'",
+      description: "Workspace mode: 'temp' (default), 'pooled', or 'static'",
     }),
     workspacePath: option({
       type: optional(string),

apps/cli/src/commands/eval/run-eval.ts

@@ -1093,7 +1093,11 @@ async function prepareFileMetadata(params: {
     filter: suiteFilter ?? options.filter,
     category,
   });
-  const effectiveOptions = applyExperimentOptions(options, suite.experimentConfig);
+  const experimentOptions = applyExperimentOptions(options, suite.experimentConfig);
+  const effectiveOptions =
+    experimentOptions.workers === undefined && suite.workers !== undefined
+      ? { ...experimentOptions, workers: suite.workers }
+      : experimentOptions;
   const testCases =
     suiteFilter && effectiveOptions.filter
       ? suite.tests.filter((testCase) =>
@@ -1106,7 +1110,7 @@ async function prepareFileMetadata(params: {
   const defaultBudgetUsd =
     effectiveOptions.cliBudgetUsd === undefined
       ? (effectiveOptions.budgetUsd ?? suite.budgetUsd)
-      : suite.budgetUsd;
+      : undefined;
   const suiteDefaultThreshold = suite.defaultTest?.threshold ?? suite.threshold;
 
   if (testCases.length === 0) {

apps/cli/test/eval.integration.test.ts

@@ -573,6 +573,39 @@ describe('agentv eval CLI', () => {
     }
   }, 30_000);
 
+  it('lets --budget-usd override evaluate_options.budget_usd', async () => {
+    const fixture = await createFixture();
+    try {
+      const evalPath = path.join(fixture.suiteDir, 'budget-options.eval.yaml');
+      await writeFile(
+        evalPath,
+        [
+          'description: Budget options integration test',
+          'target: file-target',
+          'evaluate_options:',
+          '  budget_usd: 1.25',
+          'tests:',
+          '  - id: case-alpha',
+          '    criteria: System responds with alpha',
+          '    input: alpha',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+
+      await runCli(fixture, ['eval', evalPath, '--budget-usd', '0.5']);
+
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        budgetUsd: null,
+        hasRunBudgetTracker: true,
+        runBudgetCapUsd: 0.5,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
   it('runs eval-local target config with suite test selection and run knobs', async () => {
     const fixture = await createFixture();
     try {
@@ -865,6 +898,39 @@ describe('agentv eval CLI', () => {
     }
   }, 30_000);
 
+  it('uses evaluate_options.max_concurrency as the eval-authored concurrency limit', async () => {
+    const fixture = await createFixture();
+    try {
+      const evalPath = path.join(fixture.suiteDir, 'max-concurrency.eval.yaml');
+      await writeFile(
+        evalPath,
+        [
+          'name: max-concurrency',
+          'target: file-target',
+          'evaluate_options:',
+          '  max_concurrency: 2',
+          'tests:',
+          '  - id: first-case',
+          '    input: first',
+          '    criteria: ok',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+
+      const { exitCode } = await runCli(fixture, ['eval', evalPath]);
+
+      expect(exitCode).toBe(0);
+      const diagnostics = await readDiagnostics(fixture);
+      expect(diagnostics).toMatchObject({
+        maxConcurrency: 2,
+        evalCaseIds: ['first-case'],
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
   it('records CLI-named experiment namespace separately from default runtime config', async () => {
     const fixture = await createFixture();
     try {

apps/web/src/content/docs/docs/evaluation/eval-files.mdx

@@ -5,10 +5,10 @@ sidebar:
   order: 1
 ---
 
-Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, and `budget_usd` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL.
+Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. Top-level `experiment` is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `repeat`, `threshold`, `timeout_seconds`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated attempts and gates. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL.
 
 YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract.
-Eval files describe the task, target binding, and run controls. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML.
+Eval files describe the task, target binding, and run controls. Use `evaluate_options.max_concurrency` for authored suite concurrency. Operators can still override concurrency with `--workers` or set defaults with `execution.workers` in `agentv.config.*` / `.agentv/config.yaml`; do not author legacy `workers` fields in eval YAML.
 
 ## Authoring Shapes
 
@@ -25,7 +25,7 @@ experiment format.
   suite context applies because raw cases do not carry their own suite context.
 - A **wrapper eval** is eval YAML that imports one or more suites with
   `imports.suites` and binds run controls with top-level `target`, `repeat`,
-  `threshold`, `timeout_seconds`, and `budget_usd`.
+  `threshold`, `timeout_seconds`, and `evaluate_options`.
   Wrapper evals can live anywhere in the repo. A wrapper that imports suites
   with `imports.suites` must not define parent `workspace`; imported suites own
   task environment. Machine-local existing workspace paths belong in CLI flags
@@ -119,7 +119,7 @@ tests:
 | `experiment` | Optional run/result grouping label |
 | `repeat` | Optional repeat policy with `count`, `strategy`, and `early_exit` |
 | `timeout_seconds` | Optional per-case timeout |
-| `budget_usd` | Optional suite budget |
+| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `max_concurrency` |
 | `threshold` | Optional suite quality threshold |
 | `workspace` | Suite-level task environment — inline object or string path to an [external workspace file](/docs/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). |
 | `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. |

apps/web/src/content/docs/docs/evaluation/experiments.mdx

@@ -8,10 +8,11 @@ sidebar:
 AgentV eval files are the runnable authoring artifact. Use top-level
 `description` for display metadata, `experiment` as the run/result grouping
 label, `target` for the system under test, and flat top-level run controls such
-as `repeat`, `timeout_seconds`, `budget_usd`, and `threshold`.
-Concurrency is outside eval YAML. Use `agentv eval --workers N` or project
-config defaults such as `agentv.config.*` / `.agentv/config.yaml`
-`execution.workers` for operator-side parallelism.
+as `repeat`, `timeout_seconds`, and `threshold`. Use `evaluate_options` for
+evaluation runtime options such as `budget_usd` and `max_concurrency`.
+Use `agentv eval --workers N` or project config defaults such as
+`agentv.config.*` / `.agentv/config.yaml` `execution.workers` for operator-side
+overrides.
 
 ```yaml
 name: support-regression
@@ -25,7 +26,9 @@ repeat:
   count: 4
   strategy: pass_any
 timeout_seconds: 720
-budget_usd: 2.00
+evaluate_options:
+  budget_usd: 2.00
+  max_concurrency: 3
 
 workspace:
   hooks:
@@ -181,8 +184,10 @@ tests:
       budget_usd: 0.50
 ```
 
-Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and
-`budget_usd` for public eval authoring. Candidate-changing fields stay
+Scoped `run:` supports `threshold`, `repeat`, `timeout_seconds`, and legacy
+per-case `budget_usd` overrides. Parent suite budgets should use
+`evaluate_options.budget_usd` for public eval authoring. Use
+`evaluate_options.max_concurrency` for authored concurrency. Candidate-changing fields stay
 parent-level. Workspace mutation belongs in `workspace.hooks`, and
 provider-specific setup belongs in target configuration.
 
@@ -198,7 +203,7 @@ target-specific runner state.
 | Configure an agent runner or provider variant | `target` object or `targets.yaml` |
 | Choose the target | top-level `target` |
 | Override the target's default model | `target.model` |
-| Configure repeat policy, budget, timeout, threshold | top-level `repeat`, `budget_usd`, `timeout_seconds`, `threshold` |
+| Configure repeat policy, budget, concurrency, timeout, threshold | top-level `repeat`, `evaluate_options.budget_usd`, `evaluate_options.max_concurrency`, `timeout_seconds`, `threshold` |
 | Bind an existing local workspace directory | `--workspace-path` or `.agentv/config.local.yaml` |
 
 ```yaml

apps/web/src/content/docs/docs/evaluation/running-evals.mdx

@@ -313,7 +313,7 @@ This matches the standard model used by eval frameworks (promptfoo, deepeval, Op
 Use runtime workspace flags and finish policies instead of multiple conflicting booleans:
 
 ```bash
-# Mode: pooled | temp | static
+# Mode: temp (default) | pooled | static
 agentv eval evals/my-eval.yaml --workspace-mode pooled
 
 # Existing local workspace path for this run
@@ -339,7 +339,8 @@ workspace:
 ```
 
 Notes:
-- Pooling is default for shared workspaces with repos.
+- Temp workspace materialization is the default for shared workspaces with repos.
+- Pooled mode is an explicit machine-local optimization.
 - `--workspace-path` uses an existing machine-local directory as-is and implies static runtime mode.
 - Runtime static mode is incompatible with `isolation: per_case`.
 - `hooks.enabled: false` skips all lifecycle hooks (setup, teardown, reset).