diff --git a/.agentv/config.yaml b/.agentv/config.yaml index 7ad8b9982..4d1b676fc 100644 --- a/.agentv/config.yaml +++ b/.agentv/config.yaml @@ -2,6 +2,12 @@ $schema: agentv-config-v2 # Repository AgentV configuration. +providers: file://providers.yaml + +defaults: + provider: default + grader: grader + # Discover the current example suite names during interactive eval selection. # Legacy *.eval.yaml fixtures remain discoverable for compatibility coverage. eval_patterns: diff --git a/.agentv/providers.yaml b/.agentv/providers.yaml new file mode 100644 index 000000000..91592ab1e --- /dev/null +++ b/.agentv/providers.yaml @@ -0,0 +1,199 @@ +# AgentV Evaluation Providers +# Consolidated from root, examples/features, and examples/showcase. +# Referenced by .agentv/config.yaml as the project provider catalog. +# +# default/agent/llm/grader are concrete provider labels, not env-var-driven aliases: +# AgentV removed the `use_target` indirection field with no replacement, so a +# provider label can no longer redirect to "whichever provider another provider names". +# These four mirror the local AGENT_TARGET/LLM_TARGET/GRADER_TARGET selection +# (copilot-cli / azure / azure) so `providers: [llm]` etc. across examples/ keep +# resolving. To switch the active provider, edit the definitions below +# directly (or point examples at a different concrete provider label) instead of +# changing an env var. + +# ── Default provider ──────────────────────────────────────────────── +# Evals without an explicit provider resolve to "default" by convention. +- id: copilot-cli + label: default + model: "{{ env.COPILOT_MODEL }}" + stream_log: raw + +- id: copilot-cli + label: agent + model: "{{ env.COPILOT_MODEL }}" + stream_log: raw + +# ── LLM provider (text generation, no agent binary needed) ────────── +- id: azure + label: llm + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" + +# ── Grader (LLM-as-judge) ────────────────────────────────────────── +- id: azure + label: grader + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" + +# ── Named agent providers ───────────────────────────────────────── +- id: copilot-cli + label: copilot + model: "{{ env.COPILOT_MODEL }}" + stream_log: raw + +- id: copilot-sdk + label: copilot-sdk + model: "{{ env.COPILOT_MODEL }}" + stream_log: raw + +- id: copilot-sdk + label: copilot-sdk-azure + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + subprovider: azure + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + stream_log: raw + +- id: claude-cli + label: claude + +- id: claude-sdk + label: claude-sdk + +- id: pi-cli + label: pi + subprovider: openrouter + model: "{{ env.OPENROUTER_MODEL }}" + api_key: "{{ env.OPENROUTER_API_KEY }}" + +- id: pi-coding-agent + label: pi-sdk + subprovider: openai + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" + model: gpt-5.5 + thinking: low + stream_log: raw + + +- id: pi-cli + label: pi-azure + subprovider: azure + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + +- id: pi-coding-agent + label: pi-sdk-azure + subprovider: azure + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" + model: gpt-5.5 + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + thinking: low + stream_log: raw + +- id: agentv:codex-cli + label: codex + command: ["{{ env.CODEX_EXECUTABLE }}"] + model: "{{ env.CODEX_MODEL }}" + reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" + cwd: "{{ env.CODEX_WORKSPACE_DIR }}" + log_dir: "{{ env.CODEX_LOG_DIR }}" + stream_log: raw + +# ── LLM providers (direct model access) ─────────────────────────── +- id: openai + label: gh-models + base_url: https://models.github.ai/inference + api_key: "{{ env.GH_MODELS_TOKEN }}" + model: "{{ env.GH_MODELS_MODEL }}" + +# Single Azure provider. Always uses Azure's Responses API +# (`/openai/v1/responses`); the api version defaults to `v1` and can be +# overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure +# deployments must use `provider: openai` with a deployment-scoped +# `base_url` instead. +- id: azure + label: azure + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" + +- id: gemini + label: gemini + api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}" + model: "{{ env.GEMINI_MODEL_NAME }}" + +- id: openai + label: openai + endpoint: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" + model: "{{ env.OPENAI_MODEL }}" + +# Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy +# without changing provider-specific labels. +- id: openai + label: local-openai + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + +- id: openai + label: local-openai-grader + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + +- id: pi-cli + label: pi-cli-openai + subprovider: openai + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + thinking: low + stream_log: raw + +- id: openai:codex-sdk + label: codex-sdk-openai + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + api_format: responses + reasoning_effort: low + stream_log: raw + +- id: copilot-sdk + label: copilot-sdk-openai + subprovider: openai + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + stream_log: raw + +- id: openrouter + label: openrouter + api_key: "{{ env.OPENROUTER_API_KEY }}" + model: "{{ env.OPENROUTER_MODEL }}" + +# ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── +- id: openrouter + label: mimo + api_key: "{{ env.OPENROUTER_API_KEY }}" + model: xiaomi/mimo-v2.5-pro + +- id: openrouter + label: mimo-flash + api_key: "{{ env.OPENROUTER_API_KEY }}" + model: xiaomi/mimo-v2-flash + +- id: openai + label: mimo-direct + base_url: https://token-plan-sgp.xiaomimimo.com/v1 + api_key: "{{ env.XIAOMI_MIMO_API_KEY }}" + model: xiaomi/mimo-v2.5-pro + max_output_tokens: 131072 diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml deleted file mode 100644 index 4d6d44bb6..000000000 --- a/.agentv/targets.yaml +++ /dev/null @@ -1,223 +0,0 @@ -# AgentV Evaluation Providers -# Consolidated from root, examples/features, and examples/showcase. -# Per-eval .agentv folders override these for specific eval cases. -# -# "grader" is the LLM used for scoring; agent providers reference it via -# grader_target so eval execution and grading use separate models. -# -# default/agent/llm/grader are concrete provider labels, not env-var-driven aliases: -# AgentV removed the `use_target` indirection field with no replacement, so a -# provider label can no longer redirect to "whichever provider another provider names". -# These four mirror today's AGENT_TARGET/LLM_TARGET/GRADER_TARGET selection -# (copilot-cli / azure / azure) so `providers: [llm]` etc. across examples/ keep -# resolving. To switch the active provider, edit the definitions below -# directly (or point examples at a different concrete provider label) instead of -# changing an env var. - -providers: - # ── Default provider ──────────────────────────────────────────────── - # Evals without an explicit provider resolve to "default" by convention. - - id: copilot-cli - label: default - model: "{{ env.COPILOT_MODEL }}" - grader_target: grader - stream_log: raw - - - id: copilot-cli - label: agent - model: "{{ env.COPILOT_MODEL }}" - grader_target: grader - stream_log: raw - - # ── LLM provider (text generation, no agent binary needed) ────────── - - id: azure - label: llm - endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - model: "{{ env.AZURE_DEPLOYMENT_NAME }}" - version: "{{ env.AZURE_OPENAI_API_VERSION }}" - - # ── Grader (LLM-as-judge) ────────────────────────────────────────── - # Used by agent providers via grader_target. - - id: azure - label: grader - endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - model: "{{ env.AZURE_DEPLOYMENT_NAME }}" - version: "{{ env.AZURE_OPENAI_API_VERSION }}" - - # ── Named agent providers ───────────────────────────────────────── - - id: copilot-cli - label: copilot - model: "{{ env.COPILOT_MODEL }}" - grader_target: grader - stream_log: raw - - - id: copilot-sdk - label: copilot-sdk - model: "{{ env.COPILOT_MODEL }}" - grader_target: grader - stream_log: raw - - - id: copilot-sdk - label: copilot-sdk-azure - model: "{{ env.AZURE_DEPLOYMENT_NAME }}" - subprovider: azure - base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - grader_target: grader - stream_log: raw - - - id: claude-cli - label: claude - grader_target: grader - - - id: claude-sdk - label: claude-sdk - grader_target: grader - - - id: pi-cli - label: pi - subprovider: openrouter - model: "{{ env.OPENROUTER_MODEL }}" - api_key: "{{ env.OPENROUTER_API_KEY }}" - grader_target: grader - - - id: pi-coding-agent - label: pi-sdk - subprovider: openai - base_url: "{{ env.OPENAI_ENDPOINT }}" - api_key: "{{ env.OPENAI_API_KEY }}" - model: gpt-5.5 - grader_target: openai - thinking: low - stream_log: raw - - - - id: pi-cli - label: pi-azure - subprovider: azure - base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" - model: "{{ env.AZURE_DEPLOYMENT_NAME }}" - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - grader_target: grader - - - id: pi-coding-agent - label: pi-sdk-azure - subprovider: azure - base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" - model: gpt-5.5 - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - grader_target: grader - thinking: low - stream_log: raw - - - id: agentv:codex-cli - label: codex - command: ["{{ env.CODEX_EXECUTABLE }}"] - model: "{{ env.CODEX_MODEL }}" - reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" - grader_target: grader - cwd: "{{ env.CODEX_WORKSPACE_DIR }}" - log_dir: "{{ env.CODEX_LOG_DIR }}" - stream_log: raw - - # ── LLM providers (direct model access) ─────────────────────────── - - id: openai - label: gh-models - base_url: https://models.github.ai/inference - api_key: "{{ env.GH_MODELS_TOKEN }}" - model: "{{ env.GH_MODELS_MODEL }}" - - # Single Azure provider. Always uses Azure's Responses API - # (`/openai/v1/responses`); the api version defaults to `v1` and can be - # overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure - # deployments must use `provider: openai` with a deployment-scoped - # `base_url` instead. - - id: azure - label: azure - endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" - api_key: "{{ env.AZURE_OPENAI_API_KEY }}" - model: "{{ env.AZURE_DEPLOYMENT_NAME }}" - version: "{{ env.AZURE_OPENAI_API_VERSION }}" - - - id: gemini - label: gemini - api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}" - model: "{{ env.GEMINI_MODEL_NAME }}" - - - id: openai - label: openai - endpoint: "{{ env.OPENAI_ENDPOINT }}" - api_key: "{{ env.OPENAI_API_KEY }}" - model: "{{ env.OPENAI_MODEL }}" - - # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy - # without changing provider-specific labels. - - id: openai - label: local-openai - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - grader_target: local-openai-grader - - - id: openai - label: local-openai-grader - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - - - id: pi-cli - label: pi-cli-openai - subprovider: openai - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - grader_target: local-openai-grader - thinking: low - stream_log: raw - - - id: openai:codex-sdk - label: codex-sdk-openai - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - api_format: responses - reasoning_effort: low - grader_target: local-openai-grader - stream_log: raw - - - id: copilot-sdk - label: copilot-sdk-openai - subprovider: openai - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - grader_target: local-openai-grader - stream_log: raw - - - id: openrouter - label: openrouter - api_key: "{{ env.OPENROUTER_API_KEY }}" - model: "{{ env.OPENROUTER_MODEL }}" - - # ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── - - id: openrouter - label: mimo - api_key: "{{ env.OPENROUTER_API_KEY }}" - model: xiaomi/mimo-v2.5-pro - grader_target: grader - - - id: openrouter - label: mimo-flash - api_key: "{{ env.OPENROUTER_API_KEY }}" - model: xiaomi/mimo-v2-flash - grader_target: grader - - - id: openai - label: mimo-direct - base_url: https://token-plan-sgp.xiaomimimo.com/v1 - api_key: "{{ env.XIAOMI_MIMO_API_KEY }}" - model: xiaomi/mimo-v2.5-pro - max_output_tokens: 131072 - grader_target: grader diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index b1a7e5dc2..2f94a2325 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -20,6 +20,7 @@ import { ResponseCache, RunBudgetTracker, type RunRuntimeSourceMetadata, + type TargetDefinition, type TrialsConfig, buildExperimentArtifactMetadata, buildTraceFromMessages, @@ -274,6 +275,8 @@ interface NormalizedOptions { readonly targetsPath?: string; /** Internal rerun-only carveout for generated test bundle targets.yaml artifacts. */ readonly allowLegacyTargetFiles?: boolean; + /** Internal rerun-only provider catalog paths keyed by captured eval file. */ + readonly providerCatalogPathByEvalFile?: ReadonlyMap; readonly filter?: string | readonly string[]; readonly workers?: number; /** --output : canonical artifact directory */ @@ -421,6 +424,36 @@ function normalizeSourceMetadataByEvalFile( return undefined; } +function normalizeProviderCatalogPathByEvalFile( + value: unknown, +): ReadonlyMap | undefined { + if (value instanceof Map) { + const entries = [...value.entries()].filter( + (entry): entry is [string, string] => + typeof entry[0] === 'string' && typeof entry[1] === 'string' && entry[1].trim().length > 0, + ); + return entries.length > 0 + ? new Map( + entries.map(([key, providerPath]) => [path.resolve(key), path.resolve(providerPath)]), + ) + : undefined; + } + + if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + const entries = Object.entries(value).filter( + (entry): entry is [string, string] => + typeof entry[1] === 'string' && entry[1].trim().length > 0, + ); + return entries.length > 0 + ? new Map( + entries.map(([key, providerPath]) => [path.resolve(key), path.resolve(providerPath)]), + ) + : undefined; + } + + return undefined; +} + const LEGACY_OUTPUT_FILE_EXTENSIONS = new Set([ '.jsonl', '.json', @@ -702,6 +735,9 @@ function normalizeOptions( cliTargets, targetsPath: normalizeString(rawOptions.targets), allowLegacyTargetFiles: normalizeBoolean(rawOptions.allowLegacyTargetFiles), + providerCatalogPathByEvalFile: normalizeProviderCatalogPathByEvalFile( + rawOptions.providerCatalogPathByEvalFile, + ), filter: normalizeFilter(rawOptions.filter), workers: workers > 0 ? workers : undefined, outputDir: cliOutputDir ?? configOutputDir, @@ -1308,6 +1344,9 @@ async function prepareFileMetadata(params: { readonly repoRoot: string; readonly cwd: string; readonly options: NormalizedOptions; + readonly providerCatalogPath?: string; + readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitionsSource?: string; readonly suiteFilter?: string | readonly string[]; }): Promise<{ readonly options: NormalizedOptions; @@ -1326,7 +1365,16 @@ async function prepareFileMetadata(params: { target: import('@agentv/core').ResolvedTarget, ) => import('@agentv/core').Provider; }> { - const { testFilePath, repoRoot, cwd, options, suiteFilter } = params; + const { + testFilePath, + repoRoot, + cwd, + options, + providerCatalogPath, + providerDefinitions, + providerDefinitionsSource, + suiteFilter, + } = params; await ensureFileExists(testFilePath, 'Test file'); await loadEnvFromHierarchy({ @@ -1348,6 +1396,7 @@ async function prepareFileMetadata(params: { experimentOptions.workers === undefined && suite.workers !== undefined ? { ...experimentOptions, workers: suite.workers } : experimentOptions; + const effectiveProviderCatalogPath = effectiveOptions.targetsPath ?? providerCatalogPath; const testCases = suiteFilter && effectiveOptions.filter ? suite.tests.filter((testCase) => @@ -1473,7 +1522,10 @@ async function prepareFileMetadata(params: { testFilePath, repoRoot, cwd, - explicitTargetsPath: effectiveOptions.targetsPath, + explicitTargetsPath: effectiveProviderCatalogPath, + providerDefinitions, + providerDefinitionsSource, + requireExplicitProviderCatalog: true, allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles, env: process.env, targetNames, @@ -1493,7 +1545,10 @@ async function prepareFileMetadata(params: { testFilePath, repoRoot, cwd, - explicitTargetsPath: effectiveOptions.targetsPath, + explicitTargetsPath: effectiveProviderCatalogPath, + providerDefinitions, + providerDefinitionsSource, + requireExplicitProviderCatalog: true, allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles, cliTargetName: targetSource === 'cli' @@ -1830,6 +1885,11 @@ export async function runEvalCommand( if (yamlConfig?.defaults?.grader) { options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader }; } + const providerCatalogPath = yamlConfig?.providerCatalogPath; + const providerDefinitions = yamlConfig?.providerDefinitions; + const providerDefinitionsSource = providerDefinitions + ? (providerCatalogPath ?? '.agentv/config.yaml:providers') + : undefined; const resolvedExperiment = resolveExperimentForRun(options.experiment); const evalPathInputs = input.testFiles.length > 0 ? [...input.testFiles] : []; if (evalPathInputs.length === 0 && process.stdin.isTTY) { @@ -2101,11 +2161,20 @@ export async function runEvalCommand( } >(); for (const testFilePath of resolvedTestFiles) { + const fileProviderCatalogPath = + options.providerCatalogPathByEvalFile?.get(path.resolve(testFilePath)) ?? providerCatalogPath; const meta = await prepareFileMetadata({ testFilePath, repoRoot, cwd, options, + providerCatalogPath: fileProviderCatalogPath, + providerDefinitions, + providerDefinitionsSource: fileProviderCatalogPath + ? fileProviderCatalogPath === providerCatalogPath + ? providerDefinitionsSource + : undefined + : providerDefinitionsSource, suiteFilter: undefined, }); fileMetadata.set(testFilePath, meta); diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index f112c856e..7f109f66a 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -97,6 +97,9 @@ export interface TargetSelectionOptions { readonly repoRoot: string; readonly cwd: string; readonly explicitTargetsPath?: string; + readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitionsSource?: string; + readonly requireExplicitProviderCatalog?: boolean; readonly allowLegacyTargetFiles?: boolean; readonly cliTargetName?: string; readonly cliTargetNames?: readonly string[]; @@ -106,6 +109,70 @@ export interface TargetSelectionOptions { readonly env: NodeJS.ProcessEnv; } +async function readProviderCatalog(options: { + readonly explicitTargetsPath?: string; + readonly providerDefinitions?: readonly TargetDefinition[]; + readonly providerDefinitionsSource?: string; + readonly requireExplicitProviderCatalog?: boolean; + readonly testFilePath: string; + readonly repoRoot: string; + readonly cwd: string; + readonly allowLegacyTargetFiles?: boolean; +}): Promise<{ readonly definitions: readonly TargetDefinition[]; readonly sourcePath: string }> { + if (!options.explicitTargetsPath && options.providerDefinitions) { + return { + definitions: options.providerDefinitions, + sourcePath: options.providerDefinitionsSource ?? '.agentv/config.yaml:providers', + }; + } + if (!options.explicitTargetsPath && options.requireExplicitProviderCatalog) { + throw new Error( + 'No provider catalog configured. Add `providers:` to .agentv/config.yaml, use `providers: file://providers.yaml`, or pass --providers .', + ); + } + const targetsFilePath = await discoverTargetsFile({ + explicitPath: options.explicitTargetsPath, + testFilePath: options.testFilePath, + repoRoot: options.repoRoot, + cwd: options.cwd, + allowLegacyTargetFiles: options.allowLegacyTargetFiles, + }); + await validateProviderCatalogFile(targetsFilePath); + return { + definitions: await readTargetDefinitions(targetsFilePath), + sourcePath: targetsFilePath, + }; +} + +async function validateProviderCatalogFile(targetsFilePath: string): Promise { + const validationResult = await validateTargetsFile(targetsFilePath); + const warnings = validationResult.errors.filter((e) => e.severity === 'warning'); + const useColors = isTTY(); + + if (warnings.length > 0) { + console.warn(`\nWarnings in ${targetsFilePath}:`); + for (const warning of warnings) { + const location = warning.location ? ` [${warning.location}]` : ''; + const prefix = useColors ? `${ANSI_YELLOW} ⚠${ANSI_RESET}` : ' ⚠'; + const message = useColors ? `${ANSI_YELLOW}${warning.message}${ANSI_RESET}` : warning.message; + console.warn(`${prefix}${location} ${message}`); + } + console.warn(''); + } + + const errors = validationResult.errors.filter((e) => e.severity === 'error'); + if (errors.length > 0) { + console.error(`\nErrors in ${targetsFilePath}:`); + for (const error of errors) { + const location = error.location ? ` [${error.location}]` : ''; + const prefix = useColors ? `${ANSI_RED} ✗${ANSI_RESET}` : ' ✗'; + const message = useColors ? `${ANSI_RED}${error.message}${ANSI_RESET}` : error.message; + console.error(`${prefix}${location} ${message}`); + } + throw new Error(`Providers file validation failed with ${errors.length} error(s)`); + } +} + function pickTargetName(options: { readonly cliTargetName?: string; readonly fileTargetName?: string; @@ -165,50 +232,27 @@ export async function selectTarget(options: TargetSelectionOptions): Promise e.severity === 'warning'); - const useColors = isTTY(); - - if (warnings.length > 0) { - console.warn(`\nWarnings in ${targetsFilePath}:`); - for (const warning of warnings) { - const location = warning.location ? ` [${warning.location}]` : ''; - const prefix = useColors ? `${ANSI_YELLOW} ⚠${ANSI_RESET}` : ' ⚠'; - const message = useColors ? `${ANSI_YELLOW}${warning.message}${ANSI_RESET}` : warning.message; - console.warn(`${prefix}${location} ${message}`); - } - console.warn(''); - } - - // Check for errors (should fail if invalid) - const errors = validationResult.errors.filter((e) => e.severity === 'error'); - if (errors.length > 0) { - console.error(`\nErrors in ${targetsFilePath}:`); - for (const error of errors) { - const location = error.location ? ` [${error.location}]` : ''; - const prefix = useColors ? `${ANSI_RED} ✗${ANSI_RESET}` : ' ✗'; - const message = useColors ? `${ANSI_RED}${error.message}${ANSI_RESET}` : error.message; - console.error(`${prefix}${location} ${message}`); - } - throw new Error(`Providers file validation failed with ${errors.length} error(s)`); - } - - const definitions = await readTargetDefinitions(targetsFilePath); + const definitions = providerCatalog.definitions; + const targetsFilePath = providerCatalog.sourcePath; const fileTargetSpec = options.fileTargetSpec; const fileTargetName = options.fileTargetName ?? fileTargetSpec?.name ?? (await readTestSuiteTarget(testFilePath)); @@ -263,6 +307,9 @@ export async function selectMultipleTargets( repoRoot, cwd, explicitTargetsPath, + providerDefinitions, + providerDefinitionsSource, + requireExplicitProviderCatalog, allowLegacyTargetFiles, env, targetNames, @@ -284,41 +331,18 @@ export async function selectMultipleTargets( } } - const targetsFilePath = await discoverTargetsFile({ - explicitPath: explicitTargetsPath, + const providerCatalog = await readProviderCatalog({ + explicitTargetsPath, + providerDefinitions, + providerDefinitionsSource, + requireExplicitProviderCatalog, testFilePath, repoRoot, cwd, allowLegacyTargetFiles, }); - - // Validate targets file once - const validationResult = await validateTargetsFile(targetsFilePath); - const warnings = validationResult.errors.filter((e) => e.severity === 'warning'); - const useColors = isTTY(); - if (warnings.length > 0) { - console.warn(`\nWarnings in ${targetsFilePath}:`); - for (const warning of warnings) { - const location = warning.location ? ` [${warning.location}]` : ''; - const prefix = useColors ? `${ANSI_YELLOW} ⚠${ANSI_RESET}` : ' ⚠'; - const message = useColors ? `${ANSI_YELLOW}${warning.message}${ANSI_RESET}` : warning.message; - console.warn(`${prefix}${location} ${message}`); - } - console.warn(''); - } - const errors = validationResult.errors.filter((e) => e.severity === 'error'); - if (errors.length > 0) { - console.error(`\nErrors in ${targetsFilePath}:`); - for (const error of errors) { - const location = error.location ? ` [${error.location}]` : ''; - const prefix = useColors ? `${ANSI_RED} ✗${ANSI_RESET}` : ' ✗'; - const message = useColors ? `${ANSI_RED}${error.message}${ANSI_RESET}` : error.message; - console.error(`${prefix}${location} ${message}`); - } - throw new Error(`Providers file validation failed with ${errors.length} error(s)`); - } - - const fileDefinitions = await readTargetDefinitions(targetsFilePath); + const targetsFilePath = providerCatalog.sourcePath; + const fileDefinitions = providerCatalog.definitions; // Inject synthetic definitions from eval target refs (for use_target delegation) const definitions = [...fileDefinitions]; diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 4aa257232..1b026dc6b 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -24,6 +24,8 @@ const INPUT_PROMPT = '{{ input }}'; const BUNDLE_EVALS_DIRNAME = 'evals'; const BUNDLE_MANIFEST_FILENAME = 'agentv_bundle.json'; const BUNDLE_PROVIDERS_FILENAME = 'providers.yaml'; +const BUNDLE_CONFIG_DIRNAME = '.agentv'; +const BUNDLE_CONFIG_FILENAME = 'config.yaml'; const BUNDLE_WORKSPACES_DIRNAME = 'workspaces'; const BUNDLE_SCRIPTS_DIRNAME = 'scripts'; const REDACTED_SOURCE_VALUE = '[redacted]'; @@ -1214,6 +1216,7 @@ export async function materializeEvalBundle( const targetNames = uniqueTargetNames(options.targetSelections); const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath)); const providersPath = path.join(outputDir, BUNDLE_PROVIDERS_FILENAME); + const configPath = path.join(outputDir, BUNDLE_CONFIG_DIRNAME, BUNDLE_CONFIG_FILENAME); const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME); const runtime = options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : undefined); @@ -1223,9 +1226,12 @@ export async function materializeEvalBundle( prompts: [INPUT_PROMPT], tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), }); - await writeYamlFile(providersPath, { - providers: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), - }); + await writeYamlFile( + providersPath, + serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), + ); + await mkdir(path.dirname(configPath), { recursive: true }); + await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` }); const manifest = bundleManifest({ outputDir, diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index 7b3fdcf0c..19185c447 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -407,6 +407,20 @@ function buildSourceMetadataByEvalFile( ); } +function buildProviderCatalogPathByEvalFile( + selected: readonly SelectedTaskBundle[], + overrideProvidersPath?: string, +): ReadonlyMap { + return new Map( + selected.map((bundle) => [ + path.resolve(bundle.evalPath), + overrideProvidersPath + ? path.resolve(overrideProvidersPath) + : path.resolve(bundle.providersPath), + ]), + ); +} + export const runsRerunCommand = command({ name: 'rerun', description: 'Rerun captured test bundles with local target environment', @@ -522,6 +536,10 @@ export const runsRerunCommand = command({ verbose: args.verbose, // Legacy generated run bundles stored the captured provider graph as targets.yaml. allowLegacyTargetFiles: true, + providerCatalogPathByEvalFile: buildProviderCatalogPathByEvalFile( + selected, + args.targets ? path.resolve(cwd, args.targets) : undefined, + ), sourceMetadataByEvalFile: buildSourceMetadataByEvalFile(sourceRunDir, indexPath, selected), }, }); diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index 1fb5c21fc..e15528b9a 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -21,16 +21,8 @@ describe('eval target selection', () => { it('resolves authored provider labels through providers.yaml', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); - await writeFile( - path.join(agentvDir, 'providers.yaml'), - [ - '$schema: agentv-targets-v2.2', - 'providers:', - ' - id: mock', - ' label: openai:gpt-5.4-mini', - '', - ].join('\n'), - ); + const providersPath = path.join(agentvDir, 'providers.yaml'); + await writeFile(providersPath, ['- id: mock', ' label: openai:gpt-5.4-mini', ''].join('\n')); const evalPath = path.join(tempDir, 'target-label.eval.yaml'); await writeFile( evalPath, @@ -53,6 +45,7 @@ describe('eval target selection', () => { testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, + explicitTargetsPath: providersPath, env: {}, targetNames: suite.targets ?? [], targetRefs: suite.targetRefs, @@ -65,16 +58,13 @@ describe('eval target selection', () => { expect(selections[0]?.resolvedTarget.kind).toBe('mock'); }); - it('prefers providers.yaml over legacy targets.yaml during provider discovery', async () => { + it('uses an explicit providers.yaml catalog path', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); + const providersPath = path.join(agentvDir, 'providers.yaml'); await writeFile( - path.join(agentvDir, 'providers.yaml'), - ['providers:', ' - id: mock', ' label: modern', ' response: modern', ''].join('\n'), - ); - await writeFile( - path.join(agentvDir, 'targets.yaml'), - ['providers:', ' - id: mock', ' label: legacy', ' response: legacy', ''].join('\n'), + providersPath, + ['- id: mock', ' label: modern', ' response: modern', ''].join('\n'), ); const evalPath = path.join(tempDir, 'provider-discovery.eval.yaml'); await writeFile( @@ -86,6 +76,7 @@ describe('eval target selection', () => { ' - "{{ input }}"', 'tests:', ' - id: provider-case', + ' criteria: ok', ' vars:', ' input: hello', ].join('\n'), @@ -96,6 +87,7 @@ describe('eval target selection', () => { testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, + explicitTargetsPath: providersPath, env: {}, targetNames: suite.targets ?? [], targetRefs: suite.targetRefs, @@ -106,6 +98,88 @@ describe('eval target selection', () => { expect(selections[0]?.resolvedTarget.config.response).toBe('modern'); }); + it('requires config or explicit provider catalog when requested', async () => { + const agentvDir = path.join(tempDir, '.agentv'); + await mkdir(agentvDir, { recursive: true }); + await writeFile( + path.join(agentvDir, 'providers.yaml'), + ['- id: mock', ' label: modern', ''].join('\n'), + ); + const evalPath = path.join(tempDir, 'requires-config.eval.yaml'); + await writeFile( + evalPath, + [ + 'providers:', + ' - modern', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: provider-case', + ' criteria: ok', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + await expect( + selectMultipleTargets({ + testFilePath: evalPath, + repoRoot: tempDir, + cwd: tempDir, + requireExplicitProviderCatalog: true, + env: {}, + targetNames: suite.targets ?? [], + targetRefs: suite.targetRefs, + targetSource: 'test-file', + }), + ).rejects.toThrow(/Add `providers:` to \.agentv\/config\.yaml/); + }); + + it('uses provider definitions loaded from inline project config', async () => { + const evalPath = path.join(tempDir, 'inline-providers.eval.yaml'); + await writeFile( + evalPath, + [ + 'providers:', + ' - inline-modern', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: provider-case', + ' criteria: ok', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + const selections = await selectMultipleTargets({ + testFilePath: evalPath, + repoRoot: tempDir, + cwd: tempDir, + providerDefinitions: [ + { + id: 'inline-modern', + label: 'inline-modern', + name: 'inline-modern', + provider: 'mock', + response: 'inline', + }, + ], + providerDefinitionsSource: '.agentv/config.yaml:providers', + requireExplicitProviderCatalog: true, + env: {}, + targetNames: suite.targets ?? [], + targetRefs: suite.targetRefs, + targetSource: 'test-file', + }); + + expect(selections[0]?.targetName).toBe('inline-modern'); + expect(selections[0]?.targetsFilePath).toBe('.agentv/config.yaml:providers'); + expect(selections[0]?.resolvedTarget.config.response).toBe('inline'); + }); + it('hard-rejects legacy targets.yaml as authored provider config', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); @@ -123,6 +197,7 @@ describe('eval target selection', () => { ' - "{{ input }}"', 'tests:', ' - id: provider-case', + ' criteria: ok', ' vars:', ' input: hello', ].join('\n'), diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index f748167cc..8051c94b5 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -34,6 +34,27 @@ function runDirFromIndexPath(indexPath: string): string { return path.dirname(path.dirname(indexPath)); } +function projectProviderConfigLines(extraLines: readonly string[] = []): string { + return [ + ...extraLines, + 'providers:', + ' - id: mock', + ' label: default', + ' - id: mock', + ' label: file-target', + ' - id: mock', + ' label: cli-target', + ' - id: openai:codex-sdk', + ' label: codex-target', + ' model: gpt-5-default', + '', + ].join('\n'); +} + +function nestedProviderConfigLines(): string { + return ['providers:', ' - id: mock', ' label: default', ''].join('\n'); +} + function rebuildProjectionIdentityWire( wireIdentity: unknown, dimensions: Record, @@ -82,6 +103,7 @@ providers: model: gpt-5-default `; await writeFile(providersPath, providersContent, 'utf8'); + await writeFile(path.join(agentvDir, 'config.yaml'), projectProviderConfigLines(), 'utf8'); const testFilePath = path.join(suiteDir, 'sample.test.yaml'); const testFileContent = `description: CLI integration test @@ -137,6 +159,7 @@ providers: label: default `; await writeFile(providersPath, providersContent, 'utf8'); + await writeFile(path.join(agentvDir, 'config.yaml'), nestedProviderConfigLines(), 'utf8'); const testFilePath = path.join(evalDir, 'sample.test.yaml'); const testFileContent = `description: CLI nested env integration test @@ -271,7 +294,7 @@ async function writeRequiredVersionConfig( ): Promise { await writeFile( path.join(fixture.suiteDir, '.agentv', 'config.yaml'), - `required_version: "${requiredVersion}"\n`, + projectProviderConfigLines([`required_version: "${requiredVersion}"`]), 'utf8', ); } @@ -667,7 +690,11 @@ describe('agentv eval CLI', () => { try { await writeFile( path.join(fixture.suiteDir, '.agentv', 'config.yaml'), - 'eval_patterns:\n - sample.test.yaml\n - unused.test.yaml\n', + projectProviderConfigLines([ + 'eval_patterns:', + ' - sample.test.yaml', + ' - unused.test.yaml', + ]), 'utf8', ); await writeFile( diff --git a/examples/features/readme-quickstart/.agentv/config.yaml b/examples/features/readme-quickstart/.agentv/config.yaml index 477390705..ad60cc870 100644 --- a/examples/features/readme-quickstart/.agentv/config.yaml +++ b/examples/features/readme-quickstart/.agentv/config.yaml @@ -1,3 +1,3 @@ -targets: file://targets.yaml +providers: file://providers.yaml tests: file://tests.yaml defaults: file://defaults.yaml diff --git a/examples/features/readme-quickstart/.agentv/defaults.yaml b/examples/features/readme-quickstart/.agentv/defaults.yaml index 012ed6296..ce5139858 100644 --- a/examples/features/readme-quickstart/.agentv/defaults.yaml +++ b/examples/features/readme-quickstart/.agentv/defaults.yaml @@ -1,2 +1,2 @@ -target: local-openai +provider: local-openai grader: local-openai-grader diff --git a/examples/features/readme-quickstart/.agentv/providers.yaml b/examples/features/readme-quickstart/.agentv/providers.yaml index bc1fa2e62..cbd1b5884 100644 --- a/examples/features/readme-quickstart/.agentv/providers.yaml +++ b/examples/features/readme-quickstart/.agentv/providers.yaml @@ -1,17 +1,16 @@ -providers: - - id: openai - runtime: host - config: - api_format: chat - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - label: local-openai - - id: openai - runtime: host - config: - api_format: chat - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - label: local-openai-grader +- id: openai + runtime: host + config: + api_format: chat + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + label: local-openai +- id: openai + runtime: host + config: + api_format: chat + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + label: local-openai-grader diff --git a/examples/features/readme-quickstart/README.md b/examples/features/readme-quickstart/README.md index 6d76dd243..606e8fa6e 100644 --- a/examples/features/readme-quickstart/README.md +++ b/examples/features/readme-quickstart/README.md @@ -6,15 +6,15 @@ It includes a composable `.agentv/config.yaml` that decomposes the base config graph into direct field refs: ```yaml -targets: file://providers.yaml +providers: file://providers.yaml tests: file://tests.yaml defaults: file://defaults.yaml ``` -Each referenced file contains that field's value directly, such as a bare target +Each referenced file contains that field's value directly, such as a bare provider array in `.agentv/providers.yaml` and a bare defaults object in `.agentv/defaults.yaml`. A grader is not a separate kind of entity — it is a -target listed under `targets` like any other, selected for the grading role +provider listed under `providers` like any other, selected for the grading role via `defaults.grader`. Run it against a local OpenAI-compatible endpoint: diff --git a/examples/features/readme-quickstart/providers.yaml b/examples/features/readme-quickstart/providers.yaml index 6eb9f60ad..986e81c55 100644 --- a/examples/features/readme-quickstart/providers.yaml +++ b/examples/features/readme-quickstart/providers.yaml @@ -1,9 +1,8 @@ -providers: - - id: openai - label: local-openai - runtime: host - config: - api_format: chat - base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" - api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" - model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" +- id: openai + label: local-openai + runtime: host + config: + api_format: chat + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index e045e6e95..5bc206208 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -18,6 +18,12 @@ const OBJECT_FIELDS = new Set([ ]); const SCALAR_OR_ARRAY_FIELDS = new Set(['eval_patterns']); const SCALAR_FIELDS = new Set(['required_version', '$schema']); +const PROJECT_EXECUTION_DEFAULT_FIELDS = new Set([ + 'max_concurrency', + 'verbose', + 'keep_workspaces', + 'workspace_path', +]); const SUPPORTED_FILE_REF_FIELDS = new Set([ ...ARRAY_FIELDS, ...OBJECT_FIELDS, @@ -294,7 +300,7 @@ function parseExecution( } for (const key of Object.keys(execution)) { if (key !== 'max_concurrency') { - if (allowDefaultFields) { + if (allowDefaultFields && PROJECT_EXECUTION_DEFAULT_FIELDS.has(key)) { continue; } throw new Error( diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index e32fc0554..9811f2841 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -103,6 +103,10 @@ export type AgentVConfig = { readonly tags?: Record; /** Project directory containing `.agentv/`, for resolving relative `env_path` entries and `env_from` cwd. */ readonly configDir?: string; + /** Resolved file path when top-level `providers` was authored as a file:// reference. */ + readonly providerCatalogPath?: string; + /** Provider definitions resolved from top-level `providers`, including inline arrays and file refs. */ + readonly providerDefinitions?: readonly TargetDefinition[]; } & ComposableConfigGraph; /** @@ -173,22 +177,43 @@ async function readConfigFilePair( projectDir: string, ): Promise { const localConfigPath = getLocalConfigPath(configPath); + const rawBase = await readConfigObjectFile(configPath); + const rawLocal = await readConfigObjectFile(localConfigPath); + const rawProviderConfig = + rawLocal && Object.prototype.hasOwnProperty.call(rawLocal, 'providers') ? rawLocal : rawBase; + const rawProviderOwnerPath = rawProviderConfig === rawLocal ? localConfigPath : configPath; + const providerCatalogPath = rawProviderConfig + ? resolveProviderCatalogPath(rawProviderConfig.providers, rawProviderOwnerPath) + : undefined; const base = stripLocalOnlyExecutionDefaults( - await resolveConfigObjectFileReferences(await readConfigObjectFile(configPath), configPath), + await resolveConfigObjectFileReferences(rawBase, configPath), configPath, ); const local = stripLocalOnlyExecutionDefaults( - await resolveConfigObjectFileReferences( - await readConfigObjectFile(localConfigPath), - localConfigPath, - ), + await resolveConfigObjectFileReferences(rawLocal, localConfigPath), localConfigPath, ); - const rawMerged = base && local ? mergeConfigObjects(base, local) : (local ?? base); - if (!rawMerged) { + const resolvedMerged = base && local ? mergeConfigObjects(base, local) : (local ?? base); + if (!resolvedMerged) { return null; } - return parseConfigObject(rawMerged, local ? localConfigPath : configPath, repoRoot, projectDir); + return parseConfigObject( + resolvedMerged, + local ? localConfigPath : configPath, + repoRoot, + projectDir, + providerCatalogPath, + ); +} + +function resolveProviderCatalogPath(rawProviders: unknown, ownerPath: string): string | undefined { + if (typeof rawProviders !== 'string' || !rawProviders.startsWith('file://')) { + return undefined; + } + const filePath = rawProviders.slice('file://'.length); + return path.isAbsolute(filePath) + ? filePath + : path.resolve(path.dirname(path.resolve(ownerPath)), filePath); } async function resolveConfigObjectFileReferences( @@ -206,6 +231,7 @@ function parseConfigObject( configPath: string, repoRoot: string, projectDir: string, + providerCatalogPath?: string, ): AgentVConfig | null { try { const parsed = interpolateEnv(rawConfig, createEvalConfigEnv(repoRoot)) as unknown; @@ -253,6 +279,10 @@ function parseConfigObject( allowExecutionDefaultFields: true, }); const execution = mergeExecutionConfig(executionDefaults, graph.execution); + const providerDefinitions = parseProviderDefinitions( + (parsed as Record).providers, + configPath, + ); return { required_version: requiredVersion as string | undefined, @@ -268,6 +298,8 @@ function parseConfigObject( ...(graph.tests && { tests: graph.tests }), ...(graph.defaults && { defaults: graph.defaults }), configDir: projectDir, + ...(providerCatalogPath && { providerCatalogPath }), + ...(providerDefinitions && { providerDefinitions }), }; } catch (error) { const message = (error as Error).message; @@ -279,6 +311,21 @@ function parseConfigObject( } } +function parseProviderDefinitions( + rawProviders: unknown, + configPath: string, +): readonly TargetDefinition[] | undefined { + if (rawProviders === undefined) { + return undefined; + } + if (!Array.isArray(rawProviders)) { + return undefined; + } + return rawProviders.map((entry, index) => + normalizeProviderDefinition(entry, { location: `${configPath}:providers[${index}]` }), + ); +} + function mergeExecutionConfig( defaults: ExecutionDefaults | undefined, graph: ComposableConfigGraph['execution'], diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 8d0b2e18b..0edf80d7d 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -11,7 +11,15 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function extractProvidersArray(parsed: Record, absolutePath: string): unknown[] { +function extractProvidersArray(parsed: unknown, absolutePath: string): unknown[] { + if (Array.isArray(parsed)) { + return parsed; + } + if (!isRecord(parsed)) { + throw new Error( + `providers catalog at ${absolutePath} must be a YAML array or an object with a 'providers' field`, + ); + } if (parsed.targets !== undefined) { throw new Error( `Provider catalog at ${absolutePath} uses removed 'targets'. Use 'providers'; map targets[].id to providers[].label and targets[].provider to providers[].id.`, @@ -62,12 +70,6 @@ export async function readTargetDefinitions( const raw = await readFile(absolutePath, 'utf8'); const parsed = parseYamlValue(raw); - if (!isRecord(parsed)) { - throw new Error( - `providers catalog at ${absolutePath} must be a YAML object with a 'providers' field`, - ); - } - const providers = extractProvidersArray(parsed, absolutePath); const definitions = providers.map((entry, index) => assertProviderDefinition(entry, index, absolutePath), diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 0b2b20844..2faf283dc 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -171,7 +171,7 @@ function validateComposableGraph( config: Record, ): void { try { - normalizeComposableConfigGraph(config, filePath); + normalizeComposableConfigGraph(config, filePath, { allowExecutionDefaultFields: true }); } catch (error) { addError(errors, filePath, undefined, (error as Error).message); } diff --git a/packages/core/src/evaluation/validation/file-type.ts b/packages/core/src/evaluation/validation/file-type.ts index bb2fcc2f2..4aef776a4 100644 --- a/packages/core/src/evaluation/validation/file-type.ts +++ b/packages/core/src/evaluation/validation/file-type.ts @@ -25,8 +25,13 @@ export async function detectFileType(filePath: string): Promise { const content = await readFile(filePath, 'utf8'); const parsed = parseYamlValue(content); - // YAML array root → cases file (array of test case objects) + // YAML array root is usually a cases file, except providers.yaml may be a + // bare array when referenced from .agentv/config.yaml. if (Array.isArray(parsed)) { + const inferred = inferFileTypeFromPath(filePath); + if (inferred === 'targets') { + return inferred; + } return 'cases'; } diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 11bb19022..ec4c79dd1 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -576,11 +576,11 @@ export async function validateTargetsFile(filePath: string): Promise { expect(config?.tests).toEqual([{ id: 'smoke', input: 'Fix the failing test' }]); expect(config?.defaults).toEqual({ provider: 'codex-local', grader: 'openai-grader' }); expect(config?.execution?.max_concurrency).toBe(3); + expect(config?.providerDefinitions?.map((definition) => definition.name)).toEqual([ + 'codex-local', + 'openai-grader', + ]); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('records the provider catalog path for project config file refs', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-provider-path-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const localConfigDir = path.join(projectDir, '.agentv'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(localConfigDir, { recursive: true }); + const providersPath = path.join(localConfigDir, 'providers.yaml'); + writeFileSync( + path.join(localConfigDir, 'config.yaml'), + ['providers: file://providers.yaml', ''].join('\n'), + ); + writeFileSync(providersPath, ['- id: openai', ' label: grader', ''].join('\n')); + + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + + expect(config?.providerCatalogPath).toBe(providersPath); + expect(config?.providerDefinitions?.map((definition) => definition.name)).toEqual(['grader']); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('keeps base provider catalog refs relative to config.yaml when config.local.yaml omits providers', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-provider-path-local-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const localConfigDir = path.join(projectDir, '.agentv'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(localConfigDir, { recursive: true }); + const providersPath = path.join(localConfigDir, 'providers.yaml'); + writeFileSync( + path.join(localConfigDir, 'config.yaml'), + ['providers: file://providers.yaml', ''].join('\n'), + ); + writeFileSync( + path.join(localConfigDir, 'config.local.yaml'), + ['execution:', ' keep_workspaces: true', ''].join('\n'), + ); + writeFileSync(providersPath, ['- id: openai', ' label: grader', ''].join('\n')); + + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + + expect(config?.providerCatalogPath).toBe(providersPath); + expect(config?.execution?.keep_workspaces).toBe(true); } finally { rmSync(tempDir, { recursive: true, force: true }); } diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index 3d04e164e..e03fd3ac2 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -48,6 +48,24 @@ describe('readTargetDefinitions', () => { ]); }); + it('accepts bare provider arrays for config file references', async () => { + const filePath = await writeProvidersYaml(`- id: openai:gpt-4.1-mini + label: mini +`); + + const definitions = await readTargetDefinitions(filePath); + + expect(definitions).toEqual([ + expect.objectContaining({ + id: 'mini', + name: 'mini', + label: 'mini', + provider: 'openai', + model: 'gpt-4.1-mini', + }), + ]); + }); + it('accepts colon provider specs and preserves unlabeled specs as stable identity', async () => { const filePath = await writeProvidersYaml(`providers: - id: openai:gpt-4.1-mini diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index 19a539281..4ba38dd49 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -37,6 +37,8 @@ describe('validateConfigFile', () => { filePath, `execution: max_concurrency: 3 + verbose: true + keep_workspaces: false `, );