Add provider-local environment overlays

apps/cli/src/commands/eval/targets.ts

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import {
   type EvalTargetSpec,
   type ProviderDefinition,
@@ -6,6 +7,7 @@ import {
   readProviderDefinitions,
   readTestSuiteMetadata,
   resolveProviderDefinition,
+  resolveProviderDefinitionEnvironments,
 } from '@agentv/core';
 import { validateTargetsFile } from '@agentv/core/evaluation/validation';
 import { discoverTargetsFile } from '../../utils/targets.js';
@@ -228,6 +230,19 @@ function definitionsWithEffectiveTarget(
   return [effective, ...definitions.filter((definition) => definition.name !== effective.name)];
 }
 
+async function resolveInlineDefinitionEnvironment(
+  definition: ProviderDefinition,
+  testFilePath: string,
+  location: string,
+): Promise<ProviderDefinition> {
+  const [resolved] = await resolveProviderDefinitionEnvironments(
+    [definition],
+    path.dirname(path.resolve(testFilePath)),
+    { location },
+  );
+  return resolved ?? definition;
+}
+
 export async function selectTarget(options: TargetSelectionOptions): Promise<TargetSelection> {
   const {
     testFilePath,
@@ -260,10 +275,13 @@ export async function selectTarget(options: TargetSelectionOptions): Promise<Tar
     options.fileTargetName ?? fileTargetSpec?.name ?? (await readTestSuiteTarget(testFilePath));
   const targetChoice = pickTargetName({ cliTargetName, fileTargetName });
 
-  const overlayDefinition =
+  const rawOverlayDefinition =
     targetChoice.source === 'test-file'
       ? overlayTargetDefinition({ spec: fileTargetSpec, definitions, env, targetsFilePath })
       : undefined;
+  const overlayDefinition = rawOverlayDefinition
+    ? await resolveInlineDefinitionEnvironment(rawOverlayDefinition, testFilePath, 'providers')
+    : undefined;
   const targetDefinition = withModelOverride(
     overlayDefinition ?? resolveUseTarget(targetChoice.name, definitions, env, targetsFilePath),
     modelOverride,
@@ -351,7 +369,9 @@ export async function selectMultipleTargets(
   if (targetRefs) {
     for (const ref of targetRefs) {
       if (ref.definition && !fileDefinitions.some((d) => d.name === ref.name)) {
-        definitions.push(ref.definition);
+        definitions.push(
+          await resolveInlineDefinitionEnvironment(ref.definition, testFilePath, 'providers'),
+        );
       } else if (ref.use_target && !fileDefinitions.some((d) => d.name === ref.name)) {
         definitions.push({ name: ref.name, use_target: ref.use_target } as ProviderDefinition);
       }

apps/cli/src/commands/eval/task-bundle.ts

@@ -51,6 +51,7 @@ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([
   'label',
   'provider',
   'provider_spec',
+  'environment',
   'prompts',
   'transform',
   'delay',
@@ -722,7 +723,10 @@ function selectTaskBundleTargetDefinitions(
   return selected;
 }
 
-function serializeTargetDefinition(definition: ProviderDefinition): Record<string, unknown> {
+function serializeTargetDefinition(
+  definition: ProviderDefinition,
+  rewrites: ReadonlyMap<string, string>,
+): Record<string, unknown> {
   const providerSpec =
     typeof definition.provider_spec === 'string' && definition.provider_spec.trim().length > 0
       ? definition.provider_spec.trim()
@@ -749,7 +753,12 @@ function serializeTargetDefinition(definition: ProviderDefinition): Record<strin
     ) {
       continue;
     }
-    if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) {
+    if (key === 'environment' && definition.environment) {
+      provider.environment = serializeEnvironment(
+        definition.environment as EnvironmentRecipe,
+        rewrites,
+      );
+    } else if (AUTHORING_TOP_LEVEL_TARGET_FIELDS.has(key)) {
       provider[key] = value;
     } else {
       config[key] = value;
@@ -772,8 +781,9 @@ function serializeTargetDefinition(definition: ProviderDefinition): Record<strin
 
 function serializeTargetDefinitions(
   definitions: readonly ProviderDefinition[],
+  rewrites: ReadonlyMap<string, string> = new Map(),
 ): readonly Record<string, unknown>[] {
-  return definitions.map((definition) => serializeTargetDefinition(definition));
+  return definitions.map((definition) => serializeTargetDefinition(definition, rewrites));
 }
 
 function uniqueTargetNames(selections: readonly TaskBundleTargetSelection[]): readonly string[] {
@@ -1057,6 +1067,52 @@ async function collectEnvironmentReferences(
   return references;
 }
 
+async function collectProviderEnvironmentReferences(
+  definitions: readonly ProviderDefinition[],
+  evalFileDir: string,
+): Promise<readonly BundleSourceReference[]> {
+  const references: BundleSourceReference[] = [];
+
+  for (const definition of definitions) {
+    const environment = definition.environment as EnvironmentRecipe | undefined;
+    if (!environment) {
+      continue;
+    }
+    const label = `provider "${definition.name}"`;
+    if (environment.type === 'host') {
+      references.push({
+        kind: 'environment_workdir',
+        displayPath: environment.workdir,
+        resolvedPath: environment.workdir,
+        location: `${label}.environment.workdir`,
+      });
+    }
+
+    const command = environment.setup?.command;
+    if (!command) {
+      continue;
+    }
+    const baseDir = environmentBaseDir(environment, evalFileDir);
+    for (const arg of command) {
+      const reference = await maybeWorkspaceHookCommandReference({
+        arg,
+        baseDir,
+        testId: definition.name,
+        hookName: 'setup',
+      });
+      if (reference) {
+        references.push({
+          ...reference,
+          kind: 'environment_setup_command',
+          location: `${label}.environment.setup.command`,
+        });
+      }
+    }
+  }
+
+  return references;
+}
+
 async function collectWorkspaceReferences(
   tests: readonly EvalTest[],
   evalFileDir: string,
@@ -1214,7 +1270,15 @@ export async function materializeTaskBundle(
   const testDir = path.join(options.outputDir, TEST_BUNDLE_DIRNAME);
   await mkdir(testDir, { recursive: true });
 
-  const copiedReferences = await copyReferences(options.test.source.references, testDir, options);
+  const providerEnvironmentReferences = await collectProviderEnvironmentReferences(
+    targetDefinitions,
+    path.dirname(options.test.source.evalFileAbsolutePath ?? options.test.source.evalFilePath),
+  );
+  const copiedReferences = await copyReferences(
+    [...options.test.source.references, ...providerEnvironmentReferences],
+    testDir,
+    options,
+  );
   const rewrites = buildPathRewrites(copiedReferences);
   const evalCase = buildEvalCase(options.test, rewrites);
   const evalPath = path.join(testDir, TASK_EVAL_FILENAME);
@@ -1225,7 +1289,9 @@ export async function materializeTaskBundle(
     prompts: [INPUT_PROMPT],
     tests: [evalCase],
   });
-  await writeYamlFile(providersPath, { providers: serializeTargetDefinitions(targetDefinitions) });
+  await writeYamlFile(providersPath, {
+    providers: serializeTargetDefinitions(targetDefinitions, rewrites),
+  });
 
   return {
     testDir,
@@ -1270,12 +1336,18 @@ export async function materializeEvalBundle(
   await mkdir(evalsDir, { recursive: true });
 
   const evalFileDir = path.dirname(path.resolve(options.evalFilePath));
+  const targetDefinitions = uniqueTargetDefinitions(options.targetSelections, options.tests);
   const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir);
   const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir);
+  const providerEnvironmentReferences = await collectProviderEnvironmentReferences(
+    targetDefinitions,
+    evalFileDir,
+  );
   const references: BundleSourceReference[] = [
     ...options.tests.flatMap((test) => test.source?.references ?? []),
     ...collectExpectedOutputReferences(options.tests),
     ...environmentReferences,
+    ...providerEnvironmentReferences,
     ...workspaceReferences.references,
   ];
 
@@ -1299,10 +1371,7 @@ export async function materializeEvalBundle(
     prompts: [INPUT_PROMPT],
     tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)),
   });
-  await writeYamlFile(
-    providersPath,
-    serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections, options.tests)),
-  );
+  await writeYamlFile(providersPath, serializeTargetDefinitions(targetDefinitions, rewrites));
   await mkdir(path.dirname(configPath), { recursive: true });
   await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` });
 

apps/cli/test/commands/eval/task-bundle.test.ts

@@ -23,16 +23,22 @@ describe('materializeTaskBundle', () => {
     const fixturePath = path.join(tempDir, 'fixtures', 'input.txt');
     const promptPath = path.join(tempDir, 'graders', 'prompt.md');
     const scriptPath = path.join(tempDir, 'graders', 'check.ts');
+    const providerWorkdir = path.join(tempDir, 'provider-workdir');
+    const providerSetupPath = path.join(tempDir, 'provider-env', 'setup.ts');
     await mkdir(path.dirname(evalFile), { recursive: true });
     await mkdir(path.dirname(fixturePath), { recursive: true });
     await mkdir(path.dirname(promptPath), { recursive: true });
+    await mkdir(providerWorkdir, { recursive: true });
+    await mkdir(path.dirname(providerSetupPath), { recursive: true });
     await writeFile(
       evalFile,
       'tests:\n  - id: direct-case\n    input: file://fixtures/input.txt\n',
     );
     await writeFile(fixturePath, 'fixture text\n');
     await writeFile(promptPath, 'grade carefully\n');
     await writeFile(scriptPath, 'console.log("ok");\n');
+    await writeFile(path.join(providerWorkdir, 'README.md'), 'provider workspace\n');
+    await writeFile(providerSetupPath, 'console.log("provider setup");\n');
 
     const test = {
       id: 'direct-case',
@@ -92,6 +98,12 @@ describe('materializeTaskBundle', () => {
           provider: 'mock',
           api_key: '${{ MOCK_API_KEY }}',
           fallback_targets: ['backup'],
+          environment: {
+            type: 'host',
+            workdir: providerWorkdir,
+            sourceDir: tempDir,
+            setup: { command: ['bun', providerSetupPath, '--api-key', 'literal-secret'] },
+          },
         },
         {
           name: 'backup',
@@ -122,6 +134,9 @@ describe('materializeTaskBundle', () => {
     expect(await readFile(path.join(testBundleDir, 'graders', 'graders', 'check.ts'), 'utf8')).toBe(
       'console.log("ok");\n',
     );
+    expect(
+      await readFile(path.join(testBundleDir, 'scripts', 'provider-env', 'setup.ts'), 'utf8'),
+    ).toBe('console.log("provider setup");\n');
 
     const taskEval = await readFile(paths?.evalPath ?? '', 'utf8');
     const taskProviders = await readFile(paths?.providersPath ?? '', 'utf8');
@@ -144,6 +159,9 @@ describe('materializeTaskBundle', () => {
     expect(taskProviders).toContain('label: judge');
     expect(taskProviders).toContain('api_key: ${{ JUDGE_API_KEY }}');
     expect(taskProviders).toContain('api_key: "[redacted]"');
+    expect(taskProviders).toContain('environment:');
+    expect(taskProviders).toContain('workdir: workspaces/provider-workdir');
+    expect(taskProviders).toContain('scripts/provider-env/setup.ts');
     expect(taskEval).not.toContain('literal-secret');
     expect(taskProviders).not.toContain('literal-secret');
     await expect(readdir(path.join(tempDir, 'out', '.agentv', 'results'))).rejects.toThrow();

apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx

@@ -139,8 +139,9 @@ behavior from the path; files run because they are eval YAML with tests or
 scenarios. Runtime workspace path overrides belong in CLI flags or
 `.agentv/config.local.yaml`; coding-agent testbeds, workdirs, Docker config,
 repository setup, and reset policy belong in top-level or case-level
-`environment`. Provider environment overrides belong in `env`; lifecycle hooks
-belong in `extensions`.
+`environment`. Provider environment variables belong in `env`; provider-specific
+testbed setup can use `providers[].environment` as an overlay on the authored
+environment recipe. Lifecycle hooks belong in `extensions`.
 
 ## YAML Format
 

apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx

@@ -51,6 +51,7 @@ environment:
 | `environment.setup.command` | A non-empty argv array. Use `["bash", "-lc", "..."]` when shell behavior is required. |
 | Top-level `env` | Promptfoo-compatible provider/eval variables and template inputs, such as `OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}"`. |
 | `environment.env` | Recipe-scoped process environment for the host or container testbed when a recipe needs it. It is distinct from top-level `env`. |
+| `providers[].environment` | Optional provider-local overlay, inline or `file://`, that composes with the suite/test/case environment for that provider only. Use it for provider-specific setup while keeping the main testbed in `environment`. |
 | `extensions` | Promptfoo-style lifecycle callbacks and instrumentation. They can customize flow, but they are not the canonical testbed materialization contract. |
 
 ## Providers Stay Separate
@@ -91,6 +92,23 @@ tests:
 Target `runtime` describes how the provider is invoked. It does not replace the
 authored `environment` recipe.
 
+When one provider needs extra local setup, add `environment` to that provider
+entry. AgentV treats it as sugar over the same environment recipe model, composes
+it after the suite/test/case environment for candidate runs, records both layers
+in result provenance, and fails clearly when setup, cwd, or Docker fields cannot
+compose safely. Grader providers use their own provider-local environment only
+for grader invocation.
+
+```yaml
+environment: file://.agentv/environments/local-python.yaml
+
+providers:
+  - id: codex-cli
+    label: codex-with-fixture
+    runtime: host
+    environment: file://.agentv/environments/codex-fixture.yaml
+```
+
 ## Eval Setup Lifecycle
 
 Each run follows the same high-level order:

apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx

@@ -66,7 +66,7 @@ implements equivalent semantics directly.
 | Evaluate options | `evaluateOptions` for runtime controls. | `evaluate_options` for runtime controls. | Align with Promptfoo | AgentV uses `evaluate_options.repeat`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency`. |
 | Authored concurrency | Common Promptfoo usage includes runtime options such as `maxConcurrency`. | `evaluate_options.max_concurrency`. | Keep AgentV divergence | Do not author `execution.max_concurrency` or top-level `workers` in eval YAML. CLI `--workers` remains an operator override. |
 | Provider selection | Promptfoo normal evals use `providers`; `targets` can alias providers in unified config. | Use top-level `providers` for one or more systems under test. | Align with Promptfoo | Old AgentV `target`/`targets` authoring is hard-rejected. Use CLI `--provider`/`--providers` for runtime selection. |
-| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. Built-in AgentV IDs such as `agentv:codex-cli` need export, which lowers them to a generated file provider for validation. |
+| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. AgentV also accepts `providers[].environment` as an inline or `file://` provider-local overlay on the authored environment recipe. | Align with Promptfoo plus AgentV extension | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. `providers[].environment` is AgentV-only environment sugar, not a new canonical testbed contract; Promptfoo export must lower supported host setup through generated extensions or omit unsupported overlays clearly. |
 | Provider-prompt mapping | `providerPromptMap` maps provider identities to prompt subsets. | Rejected. Use explicit AgentV composition: separate eval suites/files for provider-specific prompt subsets, top-level `prompts` plus `tests`/`default_test.vars`, `providers` or CLI `--provider`, and tags/run metadata for grouping. | Removed/rejected surface | Do not author `providerPromptMap` or `provider_prompt_map`. |
 | Direct authored input | Promptfoo prompt authoring normally goes through `prompts` plus vars. | Top-level `input` and inline `tests[].input` are removed from normal authored eval YAML. External raw-case imports may still carry internal input rows for compatibility. | Removed AgentV extension | Author prompt text, chat/system/user messages, and file-backed prompt content as `prompts`; put row data in `tests[].vars` and shared defaults in `default_test.vars`. |
 | Authored preprocessors | Not Promptfoo's canonical output-shaping surface. | Rejected in current authored YAML. | Removed/rejected surface | Use `transform` at `default_test.options`, `tests[].options`, or the assertion that needs the shaped output. Historical versioned docs may still show old preprocessor examples. |