From 1107f586c9e7318da73a933301b3b704d4e2803c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 09:07:11 +0200 Subject: [PATCH 1/2] feat(core): add provider environment overlays --- apps/cli/src/commands/eval/targets.ts | 24 ++- apps/cli/src/commands/eval/task-bundle.ts | 87 +++++++++- .../test/commands/eval/task-bundle.test.ts | 18 ++ .../docs/docs/next/evaluation/eval-files.mdx | 5 +- .../next/guides/workspace-architecture.mdx | 18 ++ .../docs/next/reference/promptfoo-parity.mdx | 2 +- .../src/evaluation/environment/compose.ts | 116 +++++++++++++ .../src/evaluation/environment/provenance.ts | 68 +++++++- .../core/src/evaluation/graders/llm-grader.ts | 5 +- packages/core/src/evaluation/graders/types.ts | 2 + .../src/evaluation/loaders/config-loader.ts | 24 ++- .../evaluation/loaders/environment-recipe.ts | 11 ++ packages/core/src/evaluation/orchestrator.ts | 81 ++++++++- .../core/src/evaluation/providers/index.ts | 8 +- .../src/evaluation/providers/targets-file.ts | 6 +- .../core/src/evaluation/providers/targets.ts | 53 ++++-- .../core/src/evaluation/providers/types.ts | 4 + packages/core/src/evaluation/types.ts | 7 + .../evaluation/validation/eval-file.schema.ts | 2 +- .../evaluation/validation/eval-validator.ts | 34 +++- .../environment-host-runtime.test.ts | 156 ++++++++++++++++++ .../evaluation/loaders/config-loader.test.ts | 43 +++-- .../loaders/environment-recipe.test.ts | 31 ++++ 23 files changed, 741 insertions(+), 64 deletions(-) create mode 100644 packages/core/src/evaluation/environment/compose.ts diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 84c03a0f7..83aa61c06 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import { type EvalTargetSpec, type ProviderDefinition, @@ -6,6 +7,7 @@ import { readProviderDefinitions, readTestSuiteMetadata, resolveProviderDefinition, + resolveProviderDefinitionEnvironments, } from '@agentv/core'; import { validateTargetsFile } from '@agentv/core/evaluation/validation'; import { discoverTargetsFile } from '../../utils/targets.js'; @@ -228,6 +230,19 @@ function definitionsWithEffectiveTarget( return [effective, ...definitions.filter((definition) => definition.name !== effective.name)]; } +async function resolveInlineDefinitionEnvironment( + definition: ProviderDefinition, + testFilePath: string, + location: string, +): Promise { + const [resolved] = await resolveProviderDefinitionEnvironments( + [definition], + path.dirname(path.resolve(testFilePath)), + { location }, + ); + return resolved ?? definition; +} + export async function selectTarget(options: TargetSelectionOptions): Promise { const { testFilePath, @@ -260,10 +275,13 @@ export async function selectTarget(options: TargetSelectionOptions): Promise d.name === ref.name)) { - definitions.push(ref.definition); + definitions.push( + await resolveInlineDefinitionEnvironment(ref.definition, testFilePath, 'providers'), + ); } else if (ref.use_target && !fileDefinitions.some((d) => d.name === ref.name)) { definitions.push({ name: ref.name, use_target: ref.use_target } as ProviderDefinition); } diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 47c3c2852..99b4fc0d6 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -51,6 +51,7 @@ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([ 'label', 'provider', 'provider_spec', + 'environment', 'prompts', 'transform', 'delay', @@ -722,7 +723,10 @@ function selectTaskBundleTargetDefinitions( return selected; } -function serializeTargetDefinition(definition: ProviderDefinition): Record { +function serializeTargetDefinition( + definition: ProviderDefinition, + rewrites: ReadonlyMap, +): Record { const providerSpec = typeof definition.provider_spec === 'string' && definition.provider_spec.trim().length > 0 ? definition.provider_spec.trim() @@ -749,7 +753,12 @@ function serializeTargetDefinition(definition: ProviderDefinition): Record = new Map(), ): readonly Record[] { - return definitions.map((definition) => serializeTargetDefinition(definition)); + return definitions.map((definition) => serializeTargetDefinition(definition, rewrites)); } function uniqueTargetNames(selections: readonly TaskBundleTargetSelection[]): readonly string[] { @@ -1057,6 +1067,52 @@ async function collectEnvironmentReferences( return references; } +async function collectProviderEnvironmentReferences( + definitions: readonly ProviderDefinition[], + evalFileDir: string, +): Promise { + const references: BundleSourceReference[] = []; + + for (const definition of definitions) { + const environment = definition.environment as EnvironmentRecipe | undefined; + if (!environment) { + continue; + } + const label = `provider "${definition.name}"`; + if (environment.type === 'host') { + references.push({ + kind: 'environment_workdir', + displayPath: environment.workdir, + resolvedPath: environment.workdir, + location: `${label}.environment.workdir`, + }); + } + + const command = environment.setup?.command; + if (!command) { + continue; + } + const baseDir = environmentBaseDir(environment, evalFileDir); + for (const arg of command) { + const reference = await maybeWorkspaceHookCommandReference({ + arg, + baseDir, + testId: definition.name, + hookName: 'setup', + }); + if (reference) { + references.push({ + ...reference, + kind: 'environment_setup_command', + location: `${label}.environment.setup.command`, + }); + } + } + } + + return references; +} + async function collectWorkspaceReferences( tests: readonly EvalTest[], evalFileDir: string, @@ -1214,7 +1270,15 @@ export async function materializeTaskBundle( const testDir = path.join(options.outputDir, TEST_BUNDLE_DIRNAME); await mkdir(testDir, { recursive: true }); - const copiedReferences = await copyReferences(options.test.source.references, testDir, options); + const providerEnvironmentReferences = await collectProviderEnvironmentReferences( + targetDefinitions, + path.dirname(options.test.source.evalFileAbsolutePath ?? options.test.source.evalFilePath), + ); + const copiedReferences = await copyReferences( + [...options.test.source.references, ...providerEnvironmentReferences], + testDir, + options, + ); const rewrites = buildPathRewrites(copiedReferences); const evalCase = buildEvalCase(options.test, rewrites); const evalPath = path.join(testDir, TASK_EVAL_FILENAME); @@ -1225,7 +1289,9 @@ export async function materializeTaskBundle( prompts: [INPUT_PROMPT], tests: [evalCase], }); - await writeYamlFile(providersPath, { providers: serializeTargetDefinitions(targetDefinitions) }); + await writeYamlFile(providersPath, { + providers: serializeTargetDefinitions(targetDefinitions, rewrites), + }); return { testDir, @@ -1270,12 +1336,18 @@ export async function materializeEvalBundle( await mkdir(evalsDir, { recursive: true }); const evalFileDir = path.dirname(path.resolve(options.evalFilePath)); + const targetDefinitions = uniqueTargetDefinitions(options.targetSelections, options.tests); const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir); const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir); + const providerEnvironmentReferences = await collectProviderEnvironmentReferences( + targetDefinitions, + evalFileDir, + ); const references: BundleSourceReference[] = [ ...options.tests.flatMap((test) => test.source?.references ?? []), ...collectExpectedOutputReferences(options.tests), ...environmentReferences, + ...providerEnvironmentReferences, ...workspaceReferences.references, ]; @@ -1299,10 +1371,7 @@ export async function materializeEvalBundle( prompts: [INPUT_PROMPT], tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), }); - await writeYamlFile( - providersPath, - serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections, options.tests)), - ); + await writeYamlFile(providersPath, serializeTargetDefinitions(targetDefinitions, rewrites)); await mkdir(path.dirname(configPath), { recursive: true }); await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` }); diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index 670c6b1dd..a8960c45a 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -23,9 +23,13 @@ describe('materializeTaskBundle', () => { const fixturePath = path.join(tempDir, 'fixtures', 'input.txt'); const promptPath = path.join(tempDir, 'graders', 'prompt.md'); const scriptPath = path.join(tempDir, 'graders', 'check.ts'); + const providerWorkdir = path.join(tempDir, 'provider-workdir'); + const providerSetupPath = path.join(tempDir, 'provider-env', 'setup.ts'); await mkdir(path.dirname(evalFile), { recursive: true }); await mkdir(path.dirname(fixturePath), { recursive: true }); await mkdir(path.dirname(promptPath), { recursive: true }); + await mkdir(providerWorkdir, { recursive: true }); + await mkdir(path.dirname(providerSetupPath), { recursive: true }); await writeFile( evalFile, 'tests:\n - id: direct-case\n input: file://fixtures/input.txt\n', @@ -33,6 +37,8 @@ describe('materializeTaskBundle', () => { await writeFile(fixturePath, 'fixture text\n'); await writeFile(promptPath, 'grade carefully\n'); await writeFile(scriptPath, 'console.log("ok");\n'); + await writeFile(path.join(providerWorkdir, 'README.md'), 'provider workspace\n'); + await writeFile(providerSetupPath, 'console.log("provider setup");\n'); const test = { id: 'direct-case', @@ -92,6 +98,12 @@ describe('materializeTaskBundle', () => { provider: 'mock', api_key: '${{ MOCK_API_KEY }}', fallback_targets: ['backup'], + environment: { + type: 'host', + workdir: providerWorkdir, + sourceDir: tempDir, + setup: { command: ['bun', providerSetupPath, '--api-key', 'literal-secret'] }, + }, }, { name: 'backup', @@ -122,6 +134,9 @@ describe('materializeTaskBundle', () => { expect(await readFile(path.join(testBundleDir, 'graders', 'graders', 'check.ts'), 'utf8')).toBe( 'console.log("ok");\n', ); + expect( + await readFile(path.join(testBundleDir, 'scripts', 'provider-env', 'setup.ts'), 'utf8'), + ).toBe('console.log("provider setup");\n'); const taskEval = await readFile(paths?.evalPath ?? '', 'utf8'); const taskProviders = await readFile(paths?.providersPath ?? '', 'utf8'); @@ -144,6 +159,9 @@ describe('materializeTaskBundle', () => { expect(taskProviders).toContain('label: judge'); expect(taskProviders).toContain('api_key: ${{ JUDGE_API_KEY }}'); expect(taskProviders).toContain('api_key: "[redacted]"'); + expect(taskProviders).toContain('environment:'); + expect(taskProviders).toContain('workdir: workspaces/provider-workdir'); + expect(taskProviders).toContain('scripts/provider-env/setup.ts'); expect(taskEval).not.toContain('literal-secret'); expect(taskProviders).not.toContain('literal-secret'); await expect(readdir(path.join(tempDir, 'out', '.agentv', 'results'))).rejects.toThrow(); diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index d32df6d34..4364b8788 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -139,8 +139,9 @@ behavior from the path; files run because they are eval YAML with tests or scenarios. Runtime workspace path overrides belong in CLI flags or `.agentv/config.local.yaml`; coding-agent testbeds, workdirs, Docker config, repository setup, and reset policy belong in top-level or case-level -`environment`. Provider environment overrides belong in `env`; lifecycle hooks -belong in `extensions`. +`environment`. Provider environment variables belong in `env`; provider-specific +testbed setup can use `providers[].environment` as an overlay on the authored +environment recipe. Lifecycle hooks belong in `extensions`. ## YAML Format diff --git a/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx b/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx index 4164258f3..fff16362c 100644 --- a/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx +++ b/apps/web/src/content/docs/docs/next/guides/workspace-architecture.mdx @@ -51,6 +51,7 @@ environment: | `environment.setup.command` | A non-empty argv array. Use `["bash", "-lc", "..."]` when shell behavior is required. | | Top-level `env` | Promptfoo-compatible provider/eval variables and template inputs, such as `OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}"`. | | `environment.env` | Recipe-scoped process environment for the host or container testbed when a recipe needs it. It is distinct from top-level `env`. | +| `providers[].environment` | Optional provider-local overlay, inline or `file://`, that composes with the suite/test/case environment for that provider only. Use it for provider-specific setup while keeping the main testbed in `environment`. | | `extensions` | Promptfoo-style lifecycle callbacks and instrumentation. They can customize flow, but they are not the canonical testbed materialization contract. | ## Providers Stay Separate @@ -91,6 +92,23 @@ tests: Target `runtime` describes how the provider is invoked. It does not replace the authored `environment` recipe. +When one provider needs extra local setup, add `environment` to that provider +entry. AgentV treats it as sugar over the same environment recipe model, composes +it after the suite/test/case environment for candidate runs, records both layers +in result provenance, and fails clearly when setup, cwd, or Docker fields cannot +compose safely. Grader providers use their own provider-local environment only +for grader invocation. + +```yaml +environment: file://.agentv/environments/local-python.yaml + +providers: + - id: codex-cli + label: codex-with-fixture + runtime: host + environment: file://.agentv/environments/codex-fixture.yaml +``` + ## Eval Setup Lifecycle Each run follows the same high-level order: diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index dc5586d7a..ef10edd2d 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -66,7 +66,7 @@ implements equivalent semantics directly. | Evaluate options | `evaluateOptions` for runtime controls. | `evaluate_options` for runtime controls. | Align with Promptfoo | AgentV uses `evaluate_options.repeat`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency`. | | Authored concurrency | Common Promptfoo usage includes runtime options such as `maxConcurrency`. | `evaluate_options.max_concurrency`. | Keep AgentV divergence | Do not author `execution.max_concurrency` or top-level `workers` in eval YAML. CLI `--workers` remains an operator override. | | Provider selection | Promptfoo normal evals use `providers`; `targets` can alias providers in unified config. | Use top-level `providers` for one or more systems under test. | Align with Promptfoo | Old AgentV `target`/`targets` authoring is hard-rejected. Use CLI `--provider`/`--providers` for runtime selection. | -| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. Built-in AgentV IDs such as `agentv:codex-cli` need export, which lowers them to a generated file provider for validation. | +| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. AgentV also accepts `providers[].environment` as an inline or `file://` provider-local overlay on the authored environment recipe. | Align with Promptfoo plus AgentV extension | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. `providers[].environment` is AgentV-only environment sugar, not a new canonical testbed contract; Promptfoo export must lower supported host setup through generated extensions or omit unsupported overlays clearly. | | Provider-prompt mapping | `providerPromptMap` maps provider identities to prompt subsets. | Rejected. Use explicit AgentV composition: separate eval suites/files for provider-specific prompt subsets, top-level `prompts` plus `tests`/`default_test.vars`, `providers` or CLI `--provider`, and tags/run metadata for grouping. | Removed/rejected surface | Do not author `providerPromptMap` or `provider_prompt_map`. | | Direct authored input | Promptfoo prompt authoring normally goes through `prompts` plus vars. | Top-level `input` and inline `tests[].input` are removed from normal authored eval YAML. External raw-case imports may still carry internal input rows for compatibility. | Removed AgentV extension | Author prompt text, chat/system/user messages, and file-backed prompt content as `prompts`; put row data in `tests[].vars` and shared defaults in `default_test.vars`. | | Authored preprocessors | Not Promptfoo's canonical output-shaping surface. | Rejected in current authored YAML. | Removed/rejected surface | Use `transform` at `default_test.options`, `tests[].options`, or the assertion that needs the shaped output. Historical versioned docs may still show old preprocessor examples. | diff --git a/packages/core/src/evaluation/environment/compose.ts b/packages/core/src/evaluation/environment/compose.ts new file mode 100644 index 000000000..ee70e1c90 --- /dev/null +++ b/packages/core/src/evaluation/environment/compose.ts @@ -0,0 +1,116 @@ +import { createHash } from 'node:crypto'; + +import type { EnvironmentRecipe } from '../loaders/environment-recipe.js'; + +export interface EnvironmentCompositionLayer { + readonly scope: 'base' | 'provider'; + readonly providerName?: string; + readonly environment: EnvironmentRecipe; +} + +export type ComposedEnvironmentRecipe = EnvironmentRecipe & { + readonly composition?: { + readonly layers: readonly EnvironmentCompositionLayer[]; + }; +}; + +export function composeProviderEnvironment(params: { + readonly base: EnvironmentRecipe | undefined; + readonly provider: EnvironmentRecipe | undefined; + readonly providerName: string; + readonly role: 'candidate' | 'grader'; +}): ComposedEnvironmentRecipe | undefined { + const { base, provider, providerName, role } = params; + if (!provider) { + return base; + } + if (!base) { + return withComposition(provider, [{ scope: 'provider', providerName, environment: provider }]); + } + + assertComposable(base, provider, providerName, role); + const merged = { + ...base, + ...(base.env || provider.env + ? { + env: { + ...(base.env ?? {}), + ...(provider.env ?? {}), + }, + } + : {}), + ...(provider.setup ? { setup: provider.setup } : {}), + recipeSha256: compositionSha256(base, provider, providerName, role), + } as ComposedEnvironmentRecipe; + return withComposition(merged, [ + { scope: 'base', environment: base }, + { scope: 'provider', providerName, environment: provider }, + ]); +} + +function withComposition( + environment: EnvironmentRecipe, + layers: readonly EnvironmentCompositionLayer[], +): ComposedEnvironmentRecipe { + return { + ...environment, + composition: { layers }, + }; +} + +function assertComposable( + base: EnvironmentRecipe, + provider: EnvironmentRecipe, + providerName: string, + role: 'candidate' | 'grader', +): void { + const prefix = `Provider-local environment for ${role} provider "${providerName}" cannot compose`; + if (base.type !== provider.type) { + throw new Error( + `${prefix}: base environment type is "${base.type}" but provider type is "${provider.type}".`, + ); + } + if (base.workdir !== provider.workdir) { + throw new Error( + `${prefix}: base environment workdir "${base.workdir}" overlaps provider workdir "${provider.workdir}". Use the same workdir or move the provider-specific setup to a separate run.`, + ); + } + if (base.setup && provider.setup) { + throw new Error( + `${prefix}: both base and provider environments define setup commands, and AgentV will not guess setup ordering. Put shared setup in the base environment and provider-only setup in one layer.`, + ); + } + if (base.type === 'docker' && provider.type === 'docker') { + for (const field of ['context', 'dockerfile', 'image'] as const) { + if ( + base[field] !== undefined && + provider[field] !== undefined && + base[field] !== provider[field] + ) { + throw new Error( + `${prefix}: docker ${field} differs between base and provider environments.`, + ); + } + } + } +} + +function compositionSha256( + base: EnvironmentRecipe, + provider: EnvironmentRecipe, + providerName: string, + role: 'candidate' | 'grader', +): string { + return createHash('sha256') + .update( + JSON.stringify({ + role, + providerName, + base: base.recipeSha256, + provider: provider.recipeSha256, + env: { ...(base.env ?? {}), ...(provider.env ?? {}) }, + setup: provider.setup ?? base.setup, + }), + ) + .digest('hex'); +} diff --git a/packages/core/src/evaluation/environment/provenance.ts b/packages/core/src/evaluation/environment/provenance.ts index b2957c2ce..72e7419d6 100644 --- a/packages/core/src/evaluation/environment/provenance.ts +++ b/packages/core/src/evaluation/environment/provenance.ts @@ -24,10 +24,11 @@ export function buildEnvironmentRecipeProvenance(params: { return undefined; } const secretValues = collectSecretValues(environment); - const setupExecutions = params.setupExecutions - ?.filter((execution) => execution.workdir === environment.workdir) - .map((execution) => redactSetupExecution(execution, secretValues)); + const setupExecutions = setupExecutionsForEnvironment(environment, params.setupExecutions).map( + (execution) => redactSetupExecution(execution, secretValues), + ); const repoProvenance = setupExecutions ? extractRepoProvenance(setupExecutions) : undefined; + const composition = buildCompositionProvenance(environment, params.setupExecutions); return { schemaVersion: 'agentv.environment_provenance.v1', authoredKind: environment.authoredReference ? 'file' : 'inline', @@ -39,9 +40,68 @@ export function buildEnvironmentRecipeProvenance(params: { sourceDir: environment.sourceDir, workdir: environment.workdir, ...(environment.setup ? { setup: redactSetupConfig(environment.setup, secretValues) } : {}), - ...(setupExecutions && setupExecutions.length > 0 ? { setupExecutions } : {}), + ...(setupExecutions.length > 0 ? { setupExecutions } : {}), ...(environment.type === 'docker' ? { docker: dockerProvenance(environment) } : {}), ...(repoProvenance !== undefined ? { repoProvenance } : {}), + ...(composition !== undefined ? { composition } : {}), + }; +} + +function setupExecutionsForEnvironment( + environment: EnvironmentRecipe, + setupExecutions: readonly EnvironmentSetupExecution[] | undefined, +): readonly EnvironmentSetupExecution[] { + if (!setupExecutions || setupExecutions.length === 0) { + return []; + } + return setupExecutions.filter((execution) => { + if (execution.workdir !== environment.workdir) { + return false; + } + if (!environment.setup) { + return execution.command === undefined; + } + return commandsEqual(execution.command, environment.setup.command); + }); +} + +function commandsEqual( + left: readonly string[] | undefined, + right: readonly string[] | undefined, +): boolean { + if (!left || !right || left.length !== right.length) { + return false; + } + return left.every((value, index) => value === right[index]); +} + +function buildCompositionProvenance( + environment: EnvironmentRecipe, + setupExecutions: readonly EnvironmentSetupExecution[] | undefined, +): EnvironmentRecipeProvenance['composition'] { + const composition = ( + environment as EnvironmentRecipe & { + readonly composition?: { + readonly layers: readonly { + readonly scope: 'base' | 'provider'; + readonly providerName?: string; + readonly environment: EnvironmentRecipe; + }[]; + }; + } + ).composition; + if (!composition || composition.layers.length === 0) { + return undefined; + } + return { + layers: composition.layers.map((layer) => ({ + scope: layer.scope, + ...(layer.providerName ? { providerName: layer.providerName } : {}), + environment: buildEnvironmentRecipeProvenance({ + environment: layer.environment, + setupExecutions, + }) as EnvironmentRecipeProvenance, + })), }; } diff --git a/packages/core/src/evaluation/graders/llm-grader.ts b/packages/core/src/evaluation/graders/llm-grader.ts index 9151f85bd..967cee79a 100644 --- a/packages/core/src/evaluation/graders/llm-grader.ts +++ b/packages/core/src/evaluation/graders/llm-grader.ts @@ -820,6 +820,7 @@ export class LlmGrader implements Grader { systemPrompt, evalCaseId: context.evalCase.id, attempt: context.attempt, + cwd: context.graderWorkspacePath, temperature: this.temperature ?? 0, tools: fsTools, maxSteps: this.maxSteps, @@ -908,6 +909,7 @@ export class LlmGrader implements Grader { modeLabel: string, ): Promise { const workspacePath = context.workspacePath ?? resolveContentBasePath(context) ?? process.cwd(); + const graderCwd = context.graderWorkspacePath ?? workspacePath; const verdictFile = await createAgentVerdictFile(workspacePath); const prompt = this.buildDelegatedPrompt(context, verdictFile.path); @@ -921,7 +923,7 @@ export class LlmGrader implements Grader { try { const response = await provider.invoke({ question: prompt, - cwd: workspacePath, + cwd: graderCwd, evalCaseId: context.evalCase.id, attempt: context.attempt, }); @@ -1579,6 +1581,7 @@ export class LlmGrader implements Grader { systemPrompt, evalCaseId: context.evalCase.id, attempt: context.attempt, + cwd: context.graderWorkspacePath, maxOutputTokens: this.maxOutputTokens, temperature: this.temperature, ...(images && images.length > 0 ? { images } : {}), diff --git a/packages/core/src/evaluation/graders/types.ts b/packages/core/src/evaluation/graders/types.ts index 26f1af312..fa85f90b2 100644 --- a/packages/core/src/evaluation/graders/types.ts +++ b/packages/core/src/evaluation/graders/types.ts @@ -35,6 +35,8 @@ export interface EvaluationContext { }; readonly now: Date; readonly graderProvider?: Provider; + /** Provider-local environment cwd for grader provider invocation only. */ + readonly graderWorkspacePath?: string; readonly graderTemplateOverride?: string; readonly evaluator?: GraderConfig; /** Output messages from agent execution (primary source for tool trajectory) */ diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 3e694a243..f03ce5bcf 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -11,7 +11,10 @@ import { } from '../../config-overlays.js'; import { getAgentvConfigDir } from '../../paths.js'; import { createEvalConfigEnv, interpolateEnv } from '../interpolation.js'; -import { normalizeProviderDefinition } from '../providers/targets.js'; +import { + normalizeProviderDefinition, + resolveProviderDefinitionEnvironments, +} from '../providers/targets.js'; import type { ProviderDefinition } from '../providers/types.js'; import type { EvalTargetRef, @@ -226,13 +229,13 @@ async function resolveConfigObjectFileReferences( return resolveConfigFieldReferences(rawConfig, configPath); } -function parseConfigObject( +async function parseConfigObject( rawConfig: Record, configPath: string, repoRoot: string, projectDir: string, providerCatalogPath?: string, -): AgentVConfig | null { +): Promise { try { const parsed = interpolateEnv(rawConfig, createEvalConfigEnv(repoRoot)) as unknown; @@ -279,9 +282,10 @@ function parseConfigObject( allowExecutionDefaultFields: true, }); const execution = mergeExecutionConfig(executionDefaults, graph.execution); - const providerDefinitions = parseProviderDefinitions( + const providerDefinitions = await parseProviderDefinitions( (parsed as Record).providers, configPath, + providerCatalogPath ? path.dirname(providerCatalogPath) : path.dirname(configPath), ); return { @@ -314,16 +318,20 @@ function parseConfigObject( function parseProviderDefinitions( rawProviders: unknown, configPath: string, -): readonly ProviderDefinition[] | undefined { + baseDir: string, +): Promise { if (rawProviders === undefined) { - return undefined; + return Promise.resolve(undefined); } if (!Array.isArray(rawProviders)) { - return undefined; + return Promise.resolve(undefined); } - return rawProviders.map((entry, index) => + const definitions = rawProviders.map((entry, index) => normalizeProviderDefinition(entry, { location: `${configPath}:providers[${index}]` }), ); + return resolveProviderDefinitionEnvironments(definitions, baseDir, { + location: `${configPath}:providers`, + }); } function mergeExecutionConfig( diff --git a/packages/core/src/evaluation/loaders/environment-recipe.ts b/packages/core/src/evaluation/loaders/environment-recipe.ts index 3a7081382..934535f66 100644 --- a/packages/core/src/evaluation/loaders/environment-recipe.ts +++ b/packages/core/src/evaluation/loaders/environment-recipe.ts @@ -59,6 +59,17 @@ export type DockerEnvironmentRecipe = { export type EnvironmentRecipe = HostEnvironmentRecipe | DockerEnvironmentRecipe; +export function isResolvedEnvironmentRecipe(value: unknown): value is EnvironmentRecipe { + if (!isJsonObject(value)) { + return false; + } + return ( + (value.type === 'host' || value.type === 'docker') && + typeof value.workdir === 'string' && + typeof value.sourceDir === 'string' + ); +} + export async function resolveEnvironmentRecipe( raw: unknown, evalFileDir: string, diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 123c2462a..260f3495a 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import micromatch from 'micromatch'; import pLimit from 'p-limit'; +import { composeProviderEnvironment } from './environment/compose.js'; import { buildEnvironmentRecipeProvenance } from './environment/provenance.js'; import { runExtensionsForHook } from './extensions/runner.js'; import { readJsonFile } from './file-utils.js'; @@ -245,6 +246,13 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR } const factory = providerFactory ?? createProvider; const instance = factory(resolved); + if (resolved.environment) { + Object.defineProperty(instance, 'environment', { + configurable: true, + enumerable: false, + value: resolved.environment, + }); + } providerCache.set(resolved.name, instance); return instance; }; @@ -961,6 +969,7 @@ export async function runEvaluation( const requiresWorkspaceDispatch = workspacePath !== undefined || legacyWorkspacePath !== undefined || + target.environment !== undefined || filteredEvalCases.some( (evalCase) => evalCase.workspace !== undefined || evalCase.environment !== undefined, ); @@ -1030,9 +1039,12 @@ export async function runEvaluation( const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? 'cleanup' : 'keep'); const workers = options.maxConcurrency ?? target.workers ?? 1; const limit = pLimit(workers); + const sharedSetupEvalCases = target.environment + ? filteredEvalCases.map((evalCase) => ({ ...evalCase, environment: undefined })) + : filteredEvalCases; const sharedSetup = await prepareSharedWorkspaceSetup({ evalRunId, - evalCases: filteredEvalCases, + evalCases: sharedSetupEvalCases, targetHooks: options.targetHooks, evalDir, verbose, @@ -1298,7 +1310,8 @@ export async function runEvaluation( // Attempt-scoped cases and raw/no-workspace cases outside the selected // suite owner prepare without inheriting a child suite's workspace. - const usesSharedWorkspace = caseUsesSharedWorkspaceSetup(evalCase, sharedSetup); + const usesSharedWorkspace = + target.environment === undefined && caseUsesSharedWorkspaceSetup(evalCase, sharedSetup); const testWorkspacePath = usesSharedWorkspace ? sharedWorkspacePath : undefined; const testBaselineCommit = usesSharedWorkspace ? sharedBaselineCommit : undefined; const testExtensionState = usesSharedWorkspace ? sharedExtensionState : undefined; @@ -1935,11 +1948,21 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise new Date()); let afterEachOutput: string | undefined; + let effectiveEvalCase = evalCase; const caseHooksEnabled = hooksEnabled(evalCase.workspace); let workspaceSetup: EvalCaseWorkspaceSetup; try { + const effectiveEnvironment = composeProviderEnvironment({ + base: evalCase.environment, + provider: target.environment, + providerName: target.name, + role: 'candidate', + }); + if (effectiveEnvironment !== evalCase.environment) { + effectiveEvalCase = { ...evalCase, environment: effectiveEnvironment }; + } workspaceSetup = await prepareEvalCaseWorkspace({ - evalCase, + evalCase: effectiveEvalCase, targetName: target.name, evalRunId, sharedWorkspacePath, @@ -1967,7 +1990,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise { + const graderWorkspacePath = graderWorkspaceSetup?.workspacePath; + if ( + graderWorkspacePath && + graderWorkspacePath !== workspacePath && + (forceCleanup || !keepWorkspaces) + ) { + await cleanupWorkspace(graderWorkspacePath).catch(() => {}); + } + }; try { + if (graderEnvironment) { + graderWorkspaceSetup = await prepareEvalCaseWorkspace({ + evalCase: { + ...effectiveEvalCase, + environment: graderEnvironment, + extensions: undefined, + workspace: undefined, + }, + targetName: graderProvider?.targetName ?? 'grader', + evalRunId, + evalDir, + cleanupWorkspaces: forceCleanup, + setupDebug, + }); + } + const result = await evaluateCandidate({ - evalCase, + evalCase: effectiveEvalCase, candidate, target: effectiveTarget, provider: effectiveProvider, @@ -2338,6 +2394,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise {}); const evalRun = { durationMs: Date.now() - caseStartMs }; const errorResultBase = buildErrorResult( @@ -2770,6 +2829,7 @@ async function evaluateCandidate(options: { readonly attempt: number; readonly sampleIndex: number; readonly graderProvider?: Provider; + readonly graderWorkspacePath?: string; readonly agentTimeoutMs?: number; readonly output?: readonly Message[]; readonly trace?: TraceSummary; @@ -2802,6 +2862,7 @@ async function evaluateCandidate(options: { attempt, sampleIndex, graderProvider, + graderWorkspacePath, agentTimeoutMs, output, trace, @@ -2867,6 +2928,7 @@ async function evaluateCandidate(options: { promptInputs, now: gradeTimestamp, graderProvider, + graderWorkspacePath, agentTimeoutMs, output: gradingOutput, trace: evaluationTrace, @@ -2962,6 +3024,7 @@ async function runEvaluatorsForCase(options: { readonly promptInputs: PromptInputs; readonly now: Date; readonly graderProvider?: Provider; + readonly graderWorkspacePath?: string; readonly agentTimeoutMs?: number; readonly output?: readonly Message[]; readonly trace?: Trace; @@ -2992,6 +3055,7 @@ async function runEvaluatorsForCase(options: { promptInputs, now, graderProvider, + graderWorkspacePath, agentTimeoutMs, output, trace, @@ -3028,6 +3092,7 @@ async function runEvaluatorsForCase(options: { promptInputs, now, graderProvider, + graderWorkspacePath, agentTimeoutMs, output, trace, @@ -3084,6 +3149,7 @@ async function runEvaluatorsForCase(options: { promptInputs, now, graderProvider, + graderWorkspacePath, output, trace, tokenUsage, @@ -3172,6 +3238,7 @@ async function runEvaluatorList(options: { readonly promptInputs: PromptInputs; readonly now: Date; readonly graderProvider?: Provider; + readonly graderWorkspacePath?: string; readonly agentTimeoutMs?: number; readonly output?: readonly Message[]; readonly trace?: Trace; @@ -3203,6 +3270,7 @@ async function runEvaluatorList(options: { promptInputs, now, graderProvider, + graderWorkspacePath, agentTimeoutMs, output, trace, @@ -3242,6 +3310,7 @@ async function runEvaluatorList(options: { promptInputs, now, graderProvider, + graderWorkspacePath, output, trace, tokenUsage, diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 6623bca64..288982449 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -21,6 +21,7 @@ import { COMMON_PROVIDER_SETTINGS, resolveDelegatedProviderDefinition, resolveProviderDefinition, + resolveProviderDefinitionEnvironments, } from './targets.js'; import type { EnvLookup, @@ -70,7 +71,12 @@ export type { VSCodeResolvedConfig, } from './targets.js'; -export { COMMON_PROVIDER_SETTINGS, resolveDelegatedProviderDefinition, resolveProviderDefinition }; +export { + COMMON_PROVIDER_SETTINGS, + resolveDelegatedProviderDefinition, + resolveProviderDefinition, + resolveProviderDefinitionEnvironments, +}; export { readProviderDefinitions, listProviderLabels } from './targets-file.js'; export { ensureVSCodeSubagents, diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 885350c34..0910c67df 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -3,7 +3,7 @@ import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; import { parseYamlValue } from '../yaml-loader.js'; -import { normalizeProviderDefinition } from './targets.js'; +import { normalizeProviderDefinition, resolveProviderDefinitionEnvironments } from './targets.js'; import { TARGETS_SCHEMA_V2 } from './types.js'; import type { ProviderDefinition } from './types.js'; @@ -74,7 +74,9 @@ export async function readProviderDefinitions( const definitions = providers.map((entry, index) => assertProviderDefinition(entry, index, absolutePath), ); - return definitions; + return resolveProviderDefinitionEnvironments(definitions, path.dirname(absolutePath), { + location: 'providers', + }); } export function listProviderLabels(definitions: readonly ProviderDefinition[]): readonly string[] { diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 464c34a93..ed9c6ba1e 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -2,6 +2,11 @@ import path from 'node:path'; import { z } from 'zod'; import { renderEnvTemplateString } from '../interpolation.js'; +import { + type EnvironmentRecipe, + isResolvedEnvironmentRecipe, + resolveEnvironmentRecipe, +} from '../loaders/environment-recipe.js'; import type { TargetRuntimeConfig, TargetRuntimeMode } from './sandbox-runner.js'; import type { EnvLookup, ProviderDefinition } from './types.js'; @@ -849,11 +854,6 @@ export function normalizeProviderDefinition( if (definition.name !== undefined) { throw new Error(`Invalid ${location}.name: use providers[].label for the stable identity.`); } - if (definition.environment !== undefined) { - throw new Error( - `Invalid ${location}.environment: provider-local environments are future scope; author environment at suite/test/case scope.`, - ); - } if (definition.container !== undefined) { throw new Error(`Invalid ${location}.container: use an environment recipe for testbed setup.`); } @@ -864,11 +864,11 @@ export function normalizeProviderDefinition( const rawLabel = definition.label; const name = typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : providerId; - const { id: _id, label: _label, ...rest } = definition; + const { id: _id, label: _label, environment, ...rest } = definition; const publicSpec = normalizePublicProviderId(providerId); const authoredConfig = isRecord(rest.config) ? rest.config : {}; - return normalizeInternalProviderDefinition({ + const normalized = normalizeInternalProviderDefinition({ ...rest, config: { ...publicSpec.config, @@ -877,6 +877,10 @@ export function normalizeProviderDefinition( id: name, provider: publicSpec.provider, }); + return { + ...normalized, + ...(environment !== undefined ? { environment } : {}), + }; } /** Normalizes an internal provider definition object for resolver use. */ @@ -913,11 +917,6 @@ export function normalizeInternalProviderDefinition( } function assertNoTargetTestbedFields(definition: Record): void { - if (definition.environment !== undefined) { - throw new Error( - 'Provider definitions cannot include environment; author environment at suite/test/case scope.', - ); - } if (definition.container !== undefined) { throw new Error( 'Provider definitions cannot include container setup; use an environment recipe.', @@ -928,6 +927,32 @@ function assertNoTargetTestbedFields(definition: Record): void } } +export async function resolveProviderDefinitionEnvironments( + definitions: readonly ProviderDefinition[], + baseDir: string, + options: { readonly location?: string } = {}, +): Promise { + const location = options.location ?? 'providers'; + return Promise.all( + definitions.map(async (definition, index) => { + if ( + definition.environment === undefined || + isResolvedEnvironmentRecipe(definition.environment) + ) { + return definition; + } + return { + ...definition, + environment: await resolveEnvironmentRecipe( + definition.environment, + baseDir, + `${location}[${index}].environment`, + ), + }; + }), + ); +} + function collectDeprecatedCamelCaseWarnings( value: unknown, location: string, @@ -1032,6 +1057,7 @@ interface ResolvedProviderBackendBase { readonly name: string; readonly label?: string; readonly runtime?: TargetRuntimeConfig; + readonly environment?: EnvironmentRecipe; readonly graderTarget?: string; readonly workers?: number; readonly providerBatching?: boolean; @@ -1123,6 +1149,7 @@ export type ResolvedProviderBackend = */ export const COMMON_PROVIDER_SETTINGS = [ 'runtime', + 'environment', 'batch_requests', 'subagent_mode_allowed', 'fallback_targets', @@ -1141,6 +1168,7 @@ const BASE_TARGET_SCHEMA = z provider_spec: z.string().optional(), config: z.record(z.unknown()).optional(), runtime: z.unknown().optional(), + environment: z.unknown().optional(), use_target: z.string().optional(), grader_target: z.string().optional(), workers: z.number().int().min(1).optional(), @@ -1345,6 +1373,7 @@ export function resolveProviderDefinition( name: parsed.name, label: parsed.label, runtime: resolveTargetRuntime(parsed.runtime, parsed.name), + environment: isResolvedEnvironmentRecipe(parsed.environment) ? parsed.environment : undefined, graderTarget: parsed.grader_target, workers: parsed.workers, providerBatching, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 7f388addd..dd0de8504 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -1,5 +1,6 @@ import type { Content, ContentImage } from '../content.js'; import { getTextContent, isContentArray } from '../content.js'; +import type { EnvironmentRecipe } from '../loaders/environment-recipe.js'; import type { JsonObject } from '../types.js'; export type ChatMessageRole = 'system' | 'user' | 'assistant' | 'tool' | 'function'; @@ -435,6 +436,7 @@ export interface Provider { readonly id: string; readonly kind: ProviderKind; readonly targetName: string; + readonly environment?: EnvironmentRecipe; invoke(request: ProviderRequest): Promise; /** * Optional capability marker for provider-managed batching (single session handling multiple requests). @@ -459,6 +461,8 @@ export interface ProviderDefinition { /** Promptfoo-shaped provider options bag. Provider settings are flattened at the boundary. */ readonly config?: unknown | undefined; readonly runtime?: unknown | undefined; + /** AgentV-only provider-scoped environment overlay. */ + readonly environment?: EnvironmentRecipe | unknown | undefined; readonly prompts?: unknown | undefined; readonly transform?: unknown | undefined; readonly delay?: number | unknown | undefined; diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index f8c312aae..0a442c15c 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1301,6 +1301,13 @@ export interface EnvironmentRecipeProvenance { readonly buildId?: string; }; readonly repoProvenance?: JsonValue; + readonly composition?: { + readonly layers: readonly { + readonly scope: 'base' | 'provider'; + readonly providerName?: string; + readonly environment: EnvironmentRecipeProvenance; + }[]; + }; } /** diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index ea95d4b9d..290fb6b9a 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -679,6 +679,7 @@ const EvalProviderObjectSchema = z transform: z.union([z.string(), JsonObjectSchema]).optional(), delay: z.number().min(0).optional(), env: z.record(z.string()).optional(), + environment: EnvironmentSchema.optional(), hooks: TargetHooksSchema.optional(), provider: z .never({ @@ -691,7 +692,6 @@ const EvalProviderObjectSchema = z invalid_type_error: 'providers[].name has been removed. Use providers[].label.', }) .optional(), - environment: z.never().optional(), container: z.never().optional(), install: z.never().optional(), }) diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 673e83396..8b12cadb2 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -590,6 +590,7 @@ export async function validateEvalFile(filePath: string): Promise { + const entries = Array.isArray(providers) ? providers : providers === undefined ? [] : [providers]; + for (let index = 0; index < entries.length; index++) { + const entry = entries[index]; + if (!isObject(entry)) { + continue; + } + await validateEnvironmentConfig( + entry.environment, + filePath, + errors, + `${location}[${index}].environment`, + ); + for (const field of ['container', 'install'] as const) { + if (entry[field] !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}[${index}].${field}`, + message: `Provider definitions cannot include ${field}; use an environment recipe for testbed setup.`, + }); + } + } + } +} + function validateTargetsTestbedFields( targets: JsonValue | undefined, location: string, @@ -1724,7 +1756,7 @@ async function validateScenarioTestLikeRow( const targets = row.providers !== undefined ? row.providers : row.provider; if (targets !== undefined) { - validateTargetTestbedFields(targets, `${location}.providers`, filePath, errors); + await validateProviderEnvironmentFields(targets, `${location}.providers`, filePath, errors); } } diff --git a/packages/core/test/evaluation/environment-host-runtime.test.ts b/packages/core/test/evaluation/environment-host-runtime.test.ts index a95024ab9..78ffe8949 100644 --- a/packages/core/test/evaluation/environment-host-runtime.test.ts +++ b/packages/core/test/evaluation/environment-host-runtime.test.ts @@ -177,4 +177,160 @@ describe('host environment runtime', () => { expect(existsSync(path.join(workdir, 'ready.txt'))).toBe(true); expect(existsSync(workdir)).toBe(true); }); + + it('applies provider-local host environment setup as a candidate overlay with provenance', async () => { + const root = tempDir('agentv-provider-env-candidate-'); + const workdir = path.join(root, 'workdir'); + const sourceDir = path.join(root, '.agentv/environments'); + await mkdir(sourceDir, { recursive: true }); + await writeFile( + path.join(sourceDir, 'provider-setup.mjs'), + "import { writeFileSync } from 'node:fs'; writeFileSync(process.env.AGENTV_ENVIRONMENT_WORKDIR + '/provider-ready.txt', 'ready');\n", + ); + + const capturedRequests: ProviderRequest[] = []; + const results = await runEvaluation({ + testFilePath: path.join(root, 'suite.eval.yaml'), + repoRoot: root, + target: { + kind: 'mock', + name: 'candidate', + config: {}, + environment: { + type: 'host', + workdir, + sourceDir, + setup: { command: ['node', 'provider-setup.mjs'] }, + }, + }, + providerFactory: providerFactory(capturedRequests), + evalCases: [ + baseEvalCase({ + environment: { + type: 'host', + workdir, + sourceDir, + env: { BASE_FLAG: '1' }, + }, + }), + ], + maxConcurrency: 1, + }); + + expect(capturedRequests[0].cwd).toBe(workdir); + expect(existsSync(path.join(workdir, 'provider-ready.txt'))).toBe(true); + expect(results[0].environmentProvenance?.composition?.layers).toMatchObject([ + { scope: 'base' }, + { scope: 'provider', providerName: 'candidate' }, + ]); + const [baseLayer, providerLayer] = results[0].environmentProvenance?.composition?.layers ?? []; + expect(baseLayer?.environment.setupExecutions).toBeUndefined(); + expect(providerLayer?.environment.setupExecutions?.[0]?.command).toEqual([ + 'node', + 'provider-setup.mjs', + ]); + }); + + it('returns an explicit conflict when base and provider setup both define commands', async () => { + const root = tempDir('agentv-provider-env-conflict-'); + const workdir = path.join(root, 'workdir'); + + const results = await runEvaluation({ + testFilePath: path.join(root, 'suite.eval.yaml'), + repoRoot: root, + target: { + kind: 'mock', + name: 'candidate', + config: {}, + environment: { + type: 'host', + workdir, + sourceDir: root, + setup: { command: ['node', '-e', ''] }, + }, + }, + providerFactory: providerFactory([]), + evalCases: [ + baseEvalCase({ + environment: { + type: 'host', + workdir, + sourceDir: root, + setup: { command: ['node', '-e', ''] }, + }, + }), + ], + maxConcurrency: 1, + }); + + expect(results[0].executionStatus).toBe('execution_error'); + expect(results[0].error).toContain('both base and provider environments define setup'); + }); + + it('prepares grader provider-local environment only for grader invocation', async () => { + const root = tempDir('agentv-provider-env-grader-'); + const candidateWorkdir = path.join(root, 'candidate'); + const graderWorkdir = path.join(root, 'grader'); + const graderSetupScript = path.join(root, 'grader-setup.ts'); + await writeFile( + graderSetupScript, + 'await Bun.write(`${process.argv[2]}/grader-marker.txt`, "ready");\n', + ); + const captured: Record = { answer: [], grader: [] }; + const answerProvider = providerFactory(captured.answer); + const graderProvider: Provider = { + id: 'mock:grader', + kind: 'mock', + targetName: 'grader', + async invoke(request): Promise { + captured.grader.push(request); + expect(await readFile(path.join(request.cwd ?? '', 'grader-marker.txt'), 'utf8')).toBe( + 'ready', + ); + return { + output: [ + { + role: 'assistant', + content: JSON.stringify({ + score: 1, + assertions: [{ text: 'ok', passed: true }], + }), + }, + ], + }; + }, + }; + + const results = await runEvaluation({ + testFilePath: path.join(root, 'suite.eval.yaml'), + repoRoot: root, + target: { kind: 'mock', name: 'answer', config: {}, graderTarget: 'grader' }, + targets: [ + { name: 'answer', provider: 'mock' }, + { + name: 'grader', + provider: 'mock', + environment: { + type: 'host', + workdir: graderWorkdir, + sourceDir: root, + setup: { command: ['bun', graderSetupScript, graderWorkdir] }, + }, + }, + ], + providerFactory: (target) => + target.name === 'grader' ? graderProvider : answerProvider(target), + evalCases: [ + baseEvalCase({ + environment: { type: 'host', workdir: candidateWorkdir, sourceDir: root }, + assertions: [{ name: 'rubric', type: 'llm-rubric', value: 'Judge it.' }], + }), + ], + maxConcurrency: 1, + }); + + expect(results[0].score).toBe(1); + expect(captured.answer[0].cwd).toBe(candidateWorkdir); + expect(captured.grader[0].cwd).toBe(graderWorkdir); + }); }); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 34da2c8b2..f4705dfc9 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -389,20 +389,6 @@ describe('loadConfig', () => { ].join('\n'), message: /workers/, }, - { - name: 'target-environment', - yaml: [ - 'providers:', - ' - id: agentv:codex-cli', - ' label: codex-local', - ' runtime: host', - ' environment:', - ' type: host', - ' workdir: ./workspace', - '', - ].join('\n'), - message: /provider-local environments are future scope/, - }, ]; for (const testCase of invalidCases) { @@ -415,6 +401,35 @@ describe('loadConfig', () => { } }); + it('accepts provider-local environment in config providers', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-provider-env-')); + try { + const configPath = path.join(tempDir, '.agentv', 'config.yaml'); + mkdirSync(path.dirname(configPath), { recursive: true }); + writeFileSync( + configPath, + [ + 'providers:', + ' - id: mock', + ' label: local-mock', + ' environment:', + ' type: host', + ' workdir: ./workspace', + '', + ].join('\n'), + ); + + const config = await loadConfig(path.join(tempDir, 'suite.eval.yaml'), tempDir); + + expect(config?.providerDefinitions?.[0]?.environment).toMatchObject({ + type: 'host', + workdir: path.join(tempDir, '.agentv', 'workspace'), + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('allows defaults.provider/defaults.grader to name a provider from a separately-discovered catalog', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-defaults-')); try { diff --git a/packages/core/test/evaluation/loaders/environment-recipe.test.ts b/packages/core/test/evaluation/loaders/environment-recipe.test.ts index 9baf4a14c..7ce735cfd 100644 --- a/packages/core/test/evaluation/loaders/environment-recipe.test.ts +++ b/packages/core/test/evaluation/loaders/environment-recipe.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { readProviderDefinitions } from '../../../src/evaluation/providers/targets-file.js'; import { validateEvalFile } from '../../../src/evaluation/validation/eval-validator.js'; import { loadTestSuite, loadTests } from '../../../src/evaluation/yaml-parser.js'; @@ -286,4 +287,34 @@ describe('environment recipe loading', () => { }); }); }); + + it('loads provider-local file:// environment recipes from provider catalogs', async () => { + await withTempDir('agentv-provider-env-file-', async (dir) => { + const environmentDir = path.join(dir, '.agentv/environments'); + await mkdirSync(environmentDir, { recursive: true }); + writeFileSync( + path.join(environmentDir, 'provider.yaml'), + ['type: host', 'workdir: ./provider-workdir', ''].join('\n'), + ); + const providersPath = path.join(dir, 'providers.yaml'); + writeFileSync( + providersPath, + [ + '- id: mock', + ' label: candidate', + ' environment: file://.agentv/environments/provider.yaml', + '', + ].join('\n'), + ); + + const [definition] = await readProviderDefinitions(providersPath); + + expect(definition.environment).toMatchObject({ + type: 'host', + authoredReference: 'file://.agentv/environments/provider.yaml', + recipeFilePath: path.join(environmentDir, 'provider.yaml'), + workdir: path.join(environmentDir, 'provider-workdir'), + }); + }); + }); }); From f0437636446c903fe3be410d824f416bbd8ba6ab Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 10:06:05 +0200 Subject: [PATCH 2/2] chore(schema): sync provider environment schema --- .../references/eval.schema.json | 368 +++++++++++++++++- 1 file changed, 362 insertions(+), 6 deletions(-) diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 967d10ec1..d7289fe5b 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2260,6 +2260,187 @@ "type": "string" } }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, "hooks": { "type": "object", "properties": { @@ -2400,9 +2581,6 @@ "name": { "not": {} }, - "environment": { - "not": {} - }, "container": { "not": {} }, @@ -2617,6 +2795,187 @@ "type": "string" } }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, "hooks": { "type": "object", "properties": { @@ -2757,9 +3116,6 @@ "name": { "not": {} }, - "environment": { - "not": {} - }, "container": { "not": {} },