From 24aea87f182f24a55968d22344176356bac3fd85 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 05:25:39 +0200 Subject: [PATCH 1/2] fix(artifacts): write provider bundle snapshots --- README.md | 2 +- apps/cli/src/commands/eval/artifact-writer.ts | 6 +-- apps/cli/src/commands/eval/commands/bundle.ts | 4 +- apps/cli/src/commands/eval/interactive.ts | 4 +- apps/cli/src/commands/eval/task-bundle.ts | 28 +++++------ apps/cli/src/commands/results/combine-run.ts | 2 +- apps/cli/src/commands/results/export.ts | 4 +- apps/cli/src/commands/results/manifest.ts | 2 +- .../src/commands/results/projection-bundle.ts | 4 +- apps/cli/src/commands/results/serve.ts | 2 +- apps/cli/src/commands/runs/rerun.ts | 46 +++++++++---------- apps/cli/src/utils/targets.ts | 14 +++--- .../commands/eval/artifact-writer.test.ts | 12 ++--- apps/cli/test/commands/eval/bundle.test.ts | 24 +++++----- apps/cli/test/commands/eval/targets.test.ts | 4 +- .../test/commands/eval/task-bundle.test.ts | 8 ++-- apps/cli/test/commands/results/export.test.ts | 28 ++++++----- apps/cli/test/commands/results/serve.test.ts | 2 +- .../test/commands/results/validate.test.ts | 4 +- apps/cli/test/commands/runs/rerun.test.ts | 24 +++++----- .../docs/next/evaluation/running-evals.mdx | 2 +- .../content/docs/docs/next/tools/results.mdx | 4 +- packages/core/src/evaluation/evaluate.ts | 6 +-- .../src/evaluation/providers/targets-file.ts | 2 +- .../core/src/evaluation/providers/types.ts | 2 +- .../core/src/evaluation/result-row-schema.ts | 2 +- packages/core/src/evaluation/run-artifacts.ts | 4 +- .../src/evaluation/validation/file-type.ts | 4 +- 28 files changed, 129 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index dd171c3d7..a05976d1f 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ Run bundle layout: │ │ ├── summary.json # optional per-case rollup across samples │ │ ├── test/ # generated test bundle: frozen inputs for reproducibility │ │ │ ├── EVAL.yaml # resolved eval spec -│ │ │ ├── targets.yaml # resolved target config +│ │ │ ├── providers.yaml # resolved provider config │ │ │ └── graders/ # grader files used │ │ └── sample-1/ # one materialized sample │ │ ├── result.json # compact sample manifest diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index c4b13301b..8d23ee9b8 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -82,7 +82,7 @@ function buildTaskBundleIndexFields( taskBundle: MaterializedTaskBundlePaths | undefined, ): Pick< IndexArtifactEntry, - 'test_dir' | 'eval_path' | 'targets_path' | 'files_path' | 'graders_path' + 'test_dir' | 'eval_path' | 'providers_path' | 'files_path' | 'graders_path' > { if (!taskBundle) { return {}; @@ -90,7 +90,7 @@ function buildTaskBundleIndexFields( return { test_dir: toRelativeArtifactPath(outputDir, taskBundle.testDir), eval_path: toRelativeArtifactPath(outputDir, taskBundle.evalPath), - targets_path: toRelativeArtifactPath(outputDir, taskBundle.targetsPath), + providers_path: toRelativeArtifactPath(outputDir, taskBundle.providersPath), ...(taskBundle.filesPath ? { files_path: toRelativeArtifactPath(outputDir, taskBundle.filesPath) } : {}), @@ -133,7 +133,7 @@ export function buildResultIndexArtifact( ? { test_dir: path.posix.join(artifactSubdir, 'test'), eval_path: path.posix.join(artifactSubdir, 'test', 'EVAL.yaml'), - targets_path: path.posix.join(artifactSubdir, 'test', 'targets.yaml'), + providers_path: path.posix.join(artifactSubdir, 'test', 'providers.yaml'), ...(taskBundle.filesPath ? { files_path: path.posix.join(artifactSubdir, 'test', 'files') } : {}), diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index d1c9e3b95..39816d153 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -166,7 +166,7 @@ export const evalBundleCommand = command({ targets: option({ type: optional(string), long: 'targets', - description: 'Path to targets.yaml (overrides discovery)', + description: 'Path to providers.yaml (overrides discovery)', }), }, handler: async (args) => { @@ -243,7 +243,7 @@ export const evalBundleCommand = command({ ` Eval: ${path.relative(paths.bundleDir, paths.evalPath).split(path.sep).join('/')}`, ); console.log( - ` Targets: ${path.relative(paths.bundleDir, paths.targetsPath).split(path.sep).join('/')}`, + ` Providers: ${path.relative(paths.bundleDir, paths.providersPath).split(path.sep).join('/')}`, ); console.log( ` Manifest: ${path.relative(paths.bundleDir, paths.manifestPath).split(path.sep).join('/')}`, diff --git a/apps/cli/src/commands/eval/interactive.ts b/apps/cli/src/commands/eval/interactive.ts index 601630114..63a9b48d4 100644 --- a/apps/cli/src/commands/eval/interactive.ts +++ b/apps/cli/src/commands/eval/interactive.ts @@ -201,11 +201,11 @@ async function promptEvalSelection( async function promptTargetSelection(cwd: string, firstEvalPath: string): Promise { const repoRoot = await findRepoRoot(cwd); - // Try to find targets.yaml — search near the eval file first, then cwd/repoRoot + // Try to find providers.yaml near the eval file first, then cwd/repoRoot. const targetsPath = await findTargetsFile(cwd, repoRoot, firstEvalPath); if (!targetsPath) { - console.log(`${ANSI_DIM}No targets.yaml found. Using default target.${ANSI_RESET}`); + console.log(`${ANSI_DIM}No providers.yaml found. Using default provider.${ANSI_RESET}`); return 'default'; } diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 4fd1f2691..4aa257232 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -17,13 +17,13 @@ import { stringify as stringifyYaml } from 'yaml'; const TEST_BUNDLE_DIRNAME = 'test'; const TASK_EVAL_FILENAME = 'EVAL.yaml'; -const TASK_TARGETS_FILENAME = 'targets.yaml'; +const TASK_PROVIDERS_FILENAME = 'providers.yaml'; const TASK_FILES_DIRNAME = 'files'; const TASK_GRADERS_DIRNAME = 'graders'; const INPUT_PROMPT = '{{ input }}'; const BUNDLE_EVALS_DIRNAME = 'evals'; const BUNDLE_MANIFEST_FILENAME = 'agentv_bundle.json'; -const BUNDLE_TARGETS_FILENAME = 'targets.yaml'; +const BUNDLE_PROVIDERS_FILENAME = 'providers.yaml'; const BUNDLE_WORKSPACES_DIRNAME = 'workspaces'; const BUNDLE_SCRIPTS_DIRNAME = 'scripts'; const REDACTED_SOURCE_VALUE = '[redacted]'; @@ -89,7 +89,7 @@ export interface MaterializeTaskBundleOptions { export interface MaterializedTaskBundlePaths { readonly testDir: string; readonly evalPath: string; - readonly targetsPath: string; + readonly providersPath: string; readonly filesPath?: string; readonly gradersPath?: string; } @@ -109,7 +109,7 @@ export interface MaterializedEvalBundlePaths { readonly bundleDir: string; readonly evalsDir: string; readonly evalPath: string; - readonly targetsPath: string; + readonly providersPath: string; readonly manifestPath: string; readonly filesPath?: string; readonly gradersPath?: string; @@ -1088,7 +1088,7 @@ function bundleManifest(options: { readonly outputDir: string; readonly evalFilePath: string; readonly evalPath: string; - readonly targetsPath: string; + readonly providersPath: string; readonly copiedReferences: readonly CopiedReference[]; readonly tests: readonly EvalTest[]; readonly targetNames: readonly string[]; @@ -1101,7 +1101,7 @@ function bundleManifest(options: { created_at: options.createdAt, source_eval: options.evalFilePath, eval_path: relative(options.evalPath), - targets_path: relative(options.targetsPath), + providers_path: relative(options.providersPath), test_count: options.tests.length, targets: options.targetNames, ...(hasCopiedBucket(options.copiedReferences, 'files') ? { files_path: 'evals/files' } : {}), @@ -1120,7 +1120,7 @@ function bundleManifest(options: { /** * Materialize the native AgentV task source for one completed result row. * - * The bundle is intentionally just an eval file, a selected targets file, and + * The bundle is intentionally just an eval file, a selected providers file, and * copied referenced assets. It does not create `.agentv/` under the result * artifact directory, so future reruns can choose their output root explicitly. */ @@ -1143,19 +1143,19 @@ export async function materializeTaskBundle( const rewrites = buildPathRewrites(copiedReferences); const evalCase = buildEvalCase(options.test, rewrites); const evalPath = path.join(testDir, TASK_EVAL_FILENAME); - const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME); + const providersPath = path.join(testDir, TASK_PROVIDERS_FILENAME); await writeYamlFile(evalPath, { providers: [options.targetName], prompts: [INPUT_PROMPT], tests: [evalCase], }); - await writeYamlFile(targetsPath, { providers: serializeTargetDefinitions(targetDefinitions) }); + await writeYamlFile(providersPath, { providers: serializeTargetDefinitions(targetDefinitions) }); return { testDir, evalPath, - targetsPath, + providersPath, ...(hasCopiedBucket(copiedReferences, 'files') ? { filesPath: path.join(testDir, TASK_FILES_DIRNAME) } : {}), @@ -1213,7 +1213,7 @@ export async function materializeEvalBundle( const rewrites = buildPathRewrites(copied); const targetNames = uniqueTargetNames(options.targetSelections); const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath)); - const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME); + const providersPath = path.join(outputDir, BUNDLE_PROVIDERS_FILENAME); const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME); const runtime = options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : undefined); @@ -1223,7 +1223,7 @@ export async function materializeEvalBundle( prompts: [INPUT_PROMPT], tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), }); - await writeYamlFile(targetsPath, { + await writeYamlFile(providersPath, { providers: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), }); @@ -1231,7 +1231,7 @@ export async function materializeEvalBundle( outputDir, evalFilePath: path.resolve(options.evalFilePath), evalPath, - targetsPath, + providersPath, copiedReferences: copied, tests: options.tests, targetNames, @@ -1243,7 +1243,7 @@ export async function materializeEvalBundle( bundleDir: outputDir, evalsDir, evalPath, - targetsPath, + providersPath, manifestPath, ...(hasCopiedBucket(copied, 'files') ? { filesPath: path.join(evalsDir, TASK_FILES_DIRNAME) } diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts index a3b01f522..c2c160d7c 100644 --- a/apps/cli/src/commands/results/combine-run.ts +++ b/apps/cli/src/commands/results/combine-run.ts @@ -366,7 +366,7 @@ const MANIFEST_PATH_FIELDS = [ 'test_dir', 'task_dir', 'eval_path', - 'targets_path', + 'providers_path', 'files_path', 'graders_path', ] as const; diff --git a/apps/cli/src/commands/results/export.ts b/apps/cli/src/commands/results/export.ts index 45b137fe3..5be168d2a 100644 --- a/apps/cli/src/commands/results/export.ts +++ b/apps/cli/src/commands/results/export.ts @@ -200,9 +200,9 @@ function createExportBundleArtifactsWriter(options: { return { test_dir: toRelativeArtifactPath(options.outputDir, testBundlePath), eval_path: toRelativeArtifactPath(options.outputDir, path.join(testBundlePath, 'EVAL.yaml')), - targets_path: toRelativeArtifactPath( + providers_path: toRelativeArtifactPath( options.outputDir, - path.join(testBundlePath, 'targets.yaml'), + path.join(testBundlePath, 'providers.yaml'), ), ...(sourceRecord?.files_path || hasCopiedSubdir(testBundlePath, 'files') ? { diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index 35a8b0aa9..0b4435036 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -99,7 +99,7 @@ export interface ResultManifestRecord { readonly test_dir?: string; readonly task_dir?: string; readonly eval_path?: string; - readonly targets_path?: string; + readonly providers_path?: string; readonly files_path?: string; readonly graders_path?: string; readonly metadata?: Record; diff --git a/apps/cli/src/commands/results/projection-bundle.ts b/apps/cli/src/commands/results/projection-bundle.ts index 91ec2a9d6..d102b1af8 100644 --- a/apps/cli/src/commands/results/projection-bundle.ts +++ b/apps/cli/src/commands/results/projection-bundle.ts @@ -98,7 +98,7 @@ export type ProjectionBundleArtifactRefs = Partial< | 'test_dir' | 'task_dir' | 'eval_path' - | 'targets_path' + | 'providers_path' | 'files_path' | 'graders_path' > @@ -177,7 +177,7 @@ function artifactRefs( test_dir: indexEntry.test_dir, task_dir: indexEntry.task_dir, eval_path: indexEntry.eval_path, - targets_path: indexEntry.targets_path, + providers_path: indexEntry.providers_path, files_path: indexEntry.files_path, graders_path: indexEntry.graders_path, }); diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 5c520cf31..b24391b57 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -839,7 +839,7 @@ function buildResultArtifactCatalog( addDirectArtifactCatalogEntry(entries, seen, record.transcript_raw_path, 'artifact'); addDirectArtifactCatalogEntry(entries, seen, record.trace_path, 'trace'); addDirectArtifactCatalogEntry(entries, seen, record.eval_path, 'artifact'); - addDirectArtifactCatalogEntry(entries, seen, record.targets_path, 'artifact'); + addDirectArtifactCatalogEntry(entries, seen, record.providers_path, 'artifact'); addTrialRunCatalogEntries(entries, seen, record); return entries; diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index ef3f4620d..7471b7718 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -25,7 +25,7 @@ import { runEvalCommand } from '../eval/run-eval.js'; import { type ResultManifestRecord, parseResultManifest } from '../results/manifest.js'; const TASK_EVAL_FILENAME = 'EVAL.yaml'; -const TASK_TARGETS_FILENAME = 'targets.yaml'; +const TASK_PROVIDERS_FILENAME = 'providers.yaml'; const ENV_REF_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; interface SelectedTaskBundle { @@ -35,7 +35,7 @@ interface SelectedTaskBundle { readonly resultDir: string; readonly testDir: string; readonly evalPath: string; - readonly targetsPath: string; + readonly providersPath: string; readonly taskTarget: string; } @@ -112,9 +112,9 @@ async function readTaskTarget(evalPath: string, fallback: string): Promise[]> { - const definitions = await readCoreTargetDefinitions(targetsPath); + const definitions = await readCoreTargetDefinitions(providersPath); return definitions.map((definition) => definition as unknown as Record); } @@ -191,11 +191,11 @@ function collectEnvRefs(value: unknown, names = new Set()): Set } async function validateTargetFile( - targetsPath: string, + providersPath: string, targetNames: readonly string[], label: string, ): Promise { - const definitions = await readRerunTargetDefinitions(targetsPath); + const definitions = await readRerunTargetDefinitions(providersPath); const byName = new Map>(); for (const definition of definitions) { const name = targetName(definition); @@ -207,7 +207,7 @@ async function validateTargetFile( const missingTargets = [...new Set(targetNames)].filter((name) => !byName.has(name)); if (missingTargets.length > 0) { throw new Error( - `${label} is incompatible: ${targetsPath} does not define target(s): ${missingTargets.join( + `${label} is incompatible: ${providersPath} does not define provider(s): ${missingTargets.join( ', ', )}`, ); @@ -239,7 +239,7 @@ async function validateTargetFile( }); if (missingEnv.length > 0) { throw new Error( - `Missing environment variable(s) required by ${targetsPath}: ${missingEnv.join( + `Missing environment variable(s) required by ${providersPath}: ${missingEnv.join( ', ', )}. Provide --env-file or export them before rerun.`, ); @@ -332,11 +332,11 @@ async function loadSelectedTaskBundles(options: { options.sourceRunDir, bundleDir && `${bundleDir}/${TASK_EVAL_FILENAME}`, ); - const targetsPath = - resolveRelativeRunPath(options.sourceRunDir, record.targets_path) ?? + const providersPath = + resolveRelativeRunPath(options.sourceRunDir, record.providers_path) ?? resolveRelativeRunPath( options.sourceRunDir, - bundleDir && `${bundleDir}/${TASK_TARGETS_FILENAME}`, + bundleDir && `${bundleDir}/${TASK_PROVIDERS_FILENAME}`, ); const testDir = resolveRelativeRunPath(options.sourceRunDir, bundleDir) ?? @@ -345,14 +345,14 @@ async function loadSelectedTaskBundles(options: { resolveRelativeRunPath(options.sourceRunDir, record.result_dir) ?? (testDir ? path.dirname(testDir) : undefined); - if (!evalPath || !targetsPath || !testDir || !resultDir) { + if (!evalPath || !providersPath || !testDir || !resultDir) { throw new Error( - `Selected result ${recordLabel} is missing test bundle paths. Re-run requires test/EVAL.yaml and test/targets.yaml.`, + `Selected result ${recordLabel} is missing test bundle paths. Re-run requires test/EVAL.yaml and test/providers.yaml.`, ); } await ensureFile(evalPath, `Test eval for ${recordLabel}`); - await ensureFile(targetsPath, `Test targets for ${recordLabel}`); + await ensureFile(providersPath, `Test providers for ${recordLabel}`); const taskTarget = await readTaskTarget(evalPath, sourceTarget); selected.push({ record, @@ -361,7 +361,7 @@ async function loadSelectedTaskBundles(options: { resultDir, testDir, evalPath, - targetsPath, + providersPath, taskTarget, }); } @@ -425,7 +425,7 @@ export const runsRerunCommand = command({ targets: option({ type: optional(string), long: 'targets', - description: 'Path to replacement targets.yaml for the new eval run', + description: 'Path to replacement providers.yaml for the new eval run', }), envFile: option({ type: optional(string), @@ -478,23 +478,23 @@ export const runsRerunCommand = command({ assertOutputIsSeparate(outputDir, forbiddenOutputRoots(sourceRunDir, selected)); if (args.targets) { - const overrideTargetsPath = path.resolve(cwd, args.targets); - await ensureFile(overrideTargetsPath, 'Target override'); + const overrideProvidersPath = path.resolve(cwd, args.targets); + await ensureFile(overrideProvidersPath, 'Provider override'); const targetNames = targetOverrides.length > 0 ? targetOverrides : selected.map((bundle) => bundle.taskTarget); - await validateTargetFile(overrideTargetsPath, targetNames, 'Target override'); + await validateTargetFile(overrideProvidersPath, targetNames, 'Provider override'); } else { const targetNamesByFile = new Map>(); for (const bundle of selected) { const targetNames = targetOverrides.length > 0 ? targetOverrides : [bundle.taskTarget]; - const names = targetNamesByFile.get(bundle.targetsPath) ?? new Set(); + const names = targetNamesByFile.get(bundle.providersPath) ?? new Set(); for (const targetName of targetNames) { names.add(targetName); } - targetNamesByFile.set(bundle.targetsPath, names); + targetNamesByFile.set(bundle.providersPath, names); } - for (const [targetsPath, names] of targetNamesByFile.entries()) { - await validateTargetFile(targetsPath, [...names], 'Test bundle targets'); + for (const [providersPath, names] of targetNamesByFile.entries()) { + await validateTargetFile(providersPath, [...names], 'Test bundle providers'); } } diff --git a/apps/cli/src/utils/targets.ts b/apps/cli/src/utils/targets.ts index a874cde6f..45502f6b9 100644 --- a/apps/cli/src/utils/targets.ts +++ b/apps/cli/src/utils/targets.ts @@ -4,10 +4,10 @@ import path from 'node:path'; import { buildDirectoryChain } from '@agentv/core'; export const TARGET_FILE_CANDIDATES = [ - 'targets.yaml', - 'targets.yml', - path.join('.agentv', 'targets.yaml'), - path.join('.agentv', 'targets.yml'), + 'providers.yaml', + 'providers.yml', + path.join('.agentv', 'providers.yaml'), + path.join('.agentv', 'providers.yml'), ] as const; export async function fileExists(filePath: string): Promise { @@ -40,7 +40,7 @@ export async function discoverTargetsFile(options: { } } - throw new Error(`targets.yaml not found at provided path: ${resolvedExplicit}`); + throw new Error(`providers.yaml not found at provided path: ${resolvedExplicit}`); } const directories = [...buildDirectoryChain(testFilePath, repoRoot)]; @@ -60,5 +60,7 @@ export async function discoverTargetsFile(options: { } } - throw new Error('Unable to locate targets.yaml. Use --targets to specify the file explicitly.'); + throw new Error( + 'Unable to locate providers.yaml. Use --providers to specify the file explicitly.', + ); } diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 25e1271a1..7356e3772 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -2641,15 +2641,15 @@ describe('writeArtifactsFromResults', () => { const rowDir = expectRowDir(indexLine, 'trace-case'); const testBundleDir = path.join(outputDir, rowDir, 'test'); const evalPath = path.join(testBundleDir, 'EVAL.yaml'); - const targetsPath = path.join(testBundleDir, 'targets.yaml'); + const providersPath = path.join(testBundleDir, 'providers.yaml'); const taskEval = await readFile(evalPath, 'utf8'); - const taskTargets = await readFile(targetsPath, 'utf8'); + const taskProviders = await readFile(providersPath, 'utf8'); expect(indexLine).toMatchObject({ result_dir: rowDir, test_dir: `${rowDir}/test`, eval_path: `${rowDir}/test/EVAL.yaml`, - targets_path: `${rowDir}/test/targets.yaml`, + providers_path: `${rowDir}/test/providers.yaml`, files_path: `${rowDir}/test/files`, graders_path: `${rowDir}/test/graders`, }); @@ -2682,10 +2682,10 @@ describe('writeArtifactsFromResults', () => { '[redacted]', ]); - expect(taskTargets).toContain('api_key: ${{ OPENAI_API_KEY }}'); - expect(taskTargets).toContain('api_key: "[redacted]"'); + expect(taskProviders).toContain('api_key: ${{ OPENAI_API_KEY }}'); + expect(taskProviders).toContain('api_key: "[redacted]"'); expect(taskEval).not.toContain('literal-secret'); - expect(taskTargets).not.toContain('literal-secret'); + expect(taskProviders).not.toContain('literal-secret'); await expect(readdir(path.join(outputDir, rowDir, '.agentv', 'results'))).rejects.toThrow(); await expect(readdir(path.join(testBundleDir, '.agentv', 'results'))).rejects.toThrow(); }); diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 0ef298f0e..4f45384ef 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -52,7 +52,7 @@ describe('agentv eval bundle', () => { await mkdir(path.join(sourceDir, 'workspace-template'), { recursive: true }); await writeFile( - path.join(sourceDir, '.agentv', 'targets.yaml'), + path.join(sourceDir, '.agentv', 'providers.yaml'), `providers: - id: mock label: inherited @@ -127,11 +127,11 @@ tests: ../data/cases.yaml ) as Record; expect(manifest.schema_version).toBe(1); expect(manifest.eval_path).toBe('evals/demo.eval.yaml'); - expect(manifest.targets_path).toBe('targets.yaml'); + expect(manifest.providers_path).toBe('providers.yaml'); expect(manifest.test_count).toBe(1); expect(manifest).not.toHaveProperty('schemaVersion'); - await expectFileExists(path.join(bundleDir, 'targets.yaml')); + await expectFileExists(path.join(bundleDir, 'providers.yaml')); await expectFileExists(path.join(bundleDir, 'evals', 'demo.eval.yaml')); await expectFileExists(path.join(bundleDir, 'evals', 'files', 'data', 'input.txt')); await expectFileExists( @@ -158,9 +158,9 @@ tests: ../data/cases.yaml }>; expect(input[0]?.content[0]).toEqual({ type: 'file', value: 'files/data/input.txt' }); - const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('label: inherited'); - expect(bundledTargets).toContain('label: backup'); + const bundledProviders = await readFile(path.join(bundleDir, 'providers.yaml'), 'utf8'); + expect(bundledProviders).toContain('label: inherited'); + expect(bundledProviders).toContain('label: backup'); await rm(sourceDir, { recursive: true, force: true }); const run = await runCli(bundleDir, [ @@ -180,7 +180,7 @@ tests: ../data/cases.yaml const bundleDir = path.join(tempDir, 'inline-bundle'); await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); - await writeFile(path.join(sourceDir, '.agentv', 'targets.yaml'), 'providers: []\n', 'utf8'); + await writeFile(path.join(sourceDir, '.agentv', 'providers.yaml'), 'providers: []\n', 'utf8'); await writeFile( path.join(sourceDir, 'evals', 'inline.eval.yaml'), `providers: @@ -209,10 +209,10 @@ tests: ]); expect(bundle.exitCode).toBe(0); - const bundledTargets = await readFile(path.join(bundleDir, 'targets.yaml'), 'utf8'); - expect(bundledTargets).toContain('id: mock'); - expect(bundledTargets).toContain('label: candidate'); - expect(bundledTargets).toContain('inline bundled response'); + const bundledProviders = await readFile(path.join(bundleDir, 'providers.yaml'), 'utf8'); + expect(bundledProviders).toContain('id: mock'); + expect(bundledProviders).toContain('label: candidate'); + expect(bundledProviders).toContain('inline bundled response'); }, 30_000); it('reports unbundleable environment references with their eval location', async () => { @@ -221,7 +221,7 @@ tests: await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); await mkdir(path.join(sourceDir, 'evals'), { recursive: true }); await writeFile( - path.join(sourceDir, '.agentv', 'targets.yaml'), + path.join(sourceDir, '.agentv', 'providers.yaml'), `providers: - id: mock label: default diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index 2f7aa56b4..99bc50ad1 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -18,11 +18,11 @@ describe('eval target selection', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('resolves authored target ids through targets.yaml', async () => { + it('resolves authored provider labels through providers.yaml', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); await writeFile( - path.join(agentvDir, 'targets.yaml'), + path.join(agentvDir, 'providers.yaml'), [ '$schema: agentv-targets-v2.2', 'providers:', diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index cf3548a61..19a15a8a9 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -118,7 +118,7 @@ describe('materializeTaskBundle', () => { ); const taskEval = await readFile(paths?.evalPath ?? '', 'utf8'); - const taskTargets = await readFile(paths?.targetsPath ?? '', 'utf8'); + const taskProviders = await readFile(paths?.providersPath ?? '', 'utf8'); const parsedEval = parseYamlValue(taskEval) as Record; const [testCase] = parsedEval.tests as Record[]; const [assertion] = testCase.assert as Record[]; @@ -133,10 +133,10 @@ describe('materializeTaskBundle', () => { ); expect(assertion.prompt).toBe('file://graders/graders/prompt.md'); expect(assertion.command).toEqual(['bun', 'graders/graders/check.ts', '--token', '[redacted]']); - expect(taskTargets).toContain('api_key: ${{ MOCK_API_KEY }}'); - expect(taskTargets).toContain('api_key: "[redacted]"'); + expect(taskProviders).toContain('api_key: ${{ MOCK_API_KEY }}'); + expect(taskProviders).toContain('api_key: "[redacted]"'); expect(taskEval).not.toContain('literal-secret'); - expect(taskTargets).not.toContain('literal-secret'); + expect(taskProviders).not.toContain('literal-secret'); await expect(readdir(path.join(tempDir, 'out', '.agentv', 'results'))).rejects.toThrow(); await expect(readdir(path.join(testBundleDir, '.agentv', 'results'))).rejects.toThrow(); }); diff --git a/apps/cli/test/commands/results/export.test.ts b/apps/cli/test/commands/results/export.test.ts index 38ef78da0..5f319627e 100644 --- a/apps/cli/test/commands/results/export.test.ts +++ b/apps/cli/test/commands/results/export.test.ts @@ -579,7 +579,7 @@ describe('results export', () => { path.join(sourceDir, 'case', 'test', 'EVAL.yaml'), 'tests:\n - id: test-greeting\n', ); - writeFileSync(path.join(sourceDir, 'case', 'test', 'targets.yaml'), 'targets: []\n'); + writeFileSync(path.join(sourceDir, 'case', 'test', 'providers.yaml'), 'providers: []\n'); const sourceFile = path.join(sourceDir, '.internal/index.jsonl'); const outputDir = path.join(tempDir, 'output'); const content = toJsonl({ @@ -587,7 +587,7 @@ describe('results export', () => { result_dir: 'case', test_dir: 'case/test', eval_path: 'case/test/EVAL.yaml', - targets_path: 'case/test/targets.yaml', + providers_path: 'case/test/providers.yaml', }); await exportResults(sourceFile, content, outputDir); @@ -596,10 +596,13 @@ describe('results export', () => { expect(entry.test_dir).toBe(`${entry.result_dir}/test`); expect(entry.task_dir).toBeUndefined(); expect(entry.eval_path).toBe(`${entry.result_dir}/test/EVAL.yaml`); - expect(entry.targets_path).toBe(`${entry.result_dir}/test/targets.yaml`); + expect(entry.providers_path).toBe(`${entry.result_dir}/test/providers.yaml`); expect(readFileSync(path.join(outputDir, entry.eval_path ?? ''), 'utf8')).toContain( 'test-greeting', ); + expect(readFileSync(path.join(outputDir, entry.providers_path ?? ''), 'utf8')).toContain( + 'providers', + ); const bundle = buildProjectionBundleFromExportedIndex({ sourceFile, @@ -611,19 +614,19 @@ describe('results export', () => { status: 'emitted', test_dir: entry.test_dir, eval_path: entry.eval_path, - targets_path: entry.targets_path, + providers_path: entry.providers_path, }); }); - it('exports legacy task_dir bundles as new test_dir artifacts', async () => { - const sourceDir = path.join(tempDir, 'legacy-run'); + it('exports task_dir bundles as new test_dir provider artifacts', async () => { + const sourceDir = path.join(tempDir, 'task-run'); mkdirSync(path.join(sourceDir, 'case', 'task'), { recursive: true }); mkdirSync(path.join(sourceDir, '.internal'), { recursive: true }); writeFileSync( path.join(sourceDir, 'case', 'task', 'EVAL.yaml'), 'tests:\n - id: test-greeting\n', ); - writeFileSync(path.join(sourceDir, 'case', 'task', 'targets.yaml'), 'targets: []\n'); + writeFileSync(path.join(sourceDir, 'case', 'task', 'providers.yaml'), 'providers: []\n'); const sourceFile = path.join(sourceDir, '.internal/index.jsonl'); const outputDir = path.join(tempDir, 'output'); const content = toJsonl({ @@ -631,7 +634,7 @@ describe('results export', () => { result_dir: 'case', task_dir: 'case/task', eval_path: 'case/task/EVAL.yaml', - targets_path: 'case/task/targets.yaml', + providers_path: 'case/task/providers.yaml', }); await exportResults(sourceFile, content, outputDir); @@ -640,10 +643,13 @@ describe('results export', () => { expect(entry.test_dir).toBe(`${entry.result_dir}/test`); expect(entry.task_dir).toBeUndefined(); expect(entry.eval_path).toBe(`${entry.result_dir}/test/EVAL.yaml`); - expect(entry.targets_path).toBe(`${entry.result_dir}/test/targets.yaml`); + expect(entry.providers_path).toBe(`${entry.result_dir}/test/providers.yaml`); expect(readFileSync(path.join(outputDir, entry.eval_path ?? ''), 'utf8')).toContain( 'test-greeting', ); + expect(readFileSync(path.join(outputDir, entry.providers_path ?? ''), 'utf8')).toContain( + 'providers', + ); }); it('preserves source bundle refs in dry-run projection inputs', async () => { @@ -657,7 +663,7 @@ describe('results export', () => { result_dir: 'case', task_dir: 'case/task', eval_path: 'case/task/EVAL.yaml', - targets_path: 'case/task/targets.yaml', + providers_path: 'case/task/providers.yaml', }), ); @@ -674,7 +680,7 @@ describe('results export', () => { status: 'planned_export', task_dir: 'case/task', eval_path: 'case/task/EVAL.yaml', - targets_path: 'case/task/targets.yaml', + providers_path: 'case/task/providers.yaml', }); }); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index b808b8615..422bad10f 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -2481,7 +2481,7 @@ describe('serve app', () => { result_dir: 'demo/test-greeting', test_dir: 'demo/test-greeting/test', eval_path: 'demo/test-greeting/test/EVAL.yaml', - targets_path: 'demo/test-greeting/test/targets.yaml', + providers_path: 'demo/test-greeting/test/providers.yaml', }); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); diff --git a/apps/cli/test/commands/results/validate.test.ts b/apps/cli/test/commands/results/validate.test.ts index e21ee35f9..755e679af 100644 --- a/apps/cli/test/commands/results/validate.test.ts +++ b/apps/cli/test/commands/results/validate.test.ts @@ -319,7 +319,7 @@ describe('results validate', () => { summary_path: 'test-new/summary.json', test_dir: 'test-new/test', eval_path: 'test-new/test/EVAL.yaml', - targets_path: 'test-new/test/targets.yaml', + providers_path: 'test-new/test/providers.yaml', }), JSON.stringify({ timestamp: '2026-03-27T12:42:24.429Z', @@ -331,7 +331,7 @@ describe('results validate', () => { summary_path: 'test-legacy/summary.json', task_dir: 'test-legacy/task', eval_path: 'test-legacy/task/EVAL.yaml', - targets_path: 'test-legacy/task/targets.yaml', + providers_path: 'test-legacy/task/providers.yaml', }), ].join('\n')}\n`, ); diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index aca413157..1580312fd 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -20,7 +20,7 @@ interface BundleFixture { readonly sourceRunDir: string; readonly outputDir: string; readonly envFile: string; - readonly overrideTargetsPath: string; + readonly overrideProvidersPath: string; } interface CliResult { @@ -66,7 +66,7 @@ tests: `, 'utf8', ); - await writeFile(path.join(bundleDir, 'targets.yaml'), options.targetsYaml, 'utf8'); + await writeFile(path.join(bundleDir, 'providers.yaml'), options.targetsYaml, 'utf8'); await writeFile(path.join(artifactDir, 'grading.json'), '{"assertions":[]}\n', 'utf8'); await writeFile(path.join(artifactDir, 'timing.json'), '{"duration_ms":1}\n', 'utf8'); await writeFile(path.join(outputsDir, 'answer.md'), '@[assistant]:\nCaptured answer\n', 'utf8'); @@ -74,7 +74,7 @@ tests: const bundlePaths = { [`${options.legacyTaskDir ? 'task' : 'test'}_dir`]: `${options.testId}/${bundleDirname}`, eval_path: `${options.testId}/${bundleDirname}/EVAL.yaml`, - targets_path: `${options.testId}/${bundleDirname}/targets.yaml`, + providers_path: `${options.testId}/${bundleDirname}/providers.yaml`, }; return { @@ -124,9 +124,9 @@ async function createBundleFixture( const envFile = path.join(baseDir, 'local.env'); await writeFile(envFile, 'LOCAL_AGENT_COMMAND=echo local-agent\n', 'utf8'); - const overrideTargetsPath = path.join(baseDir, 'override-targets.yaml'); + const overrideProvidersPath = path.join(baseDir, 'override-providers.yaml'); await writeFile( - overrideTargetsPath, + overrideProvidersPath, `providers: - id: mock label: local @@ -134,7 +134,7 @@ async function createBundleFixture( 'utf8', ); - return { baseDir, cwd, sourceRunDir, outputDir, envFile, overrideTargetsPath }; + return { baseDir, cwd, sourceRunDir, outputDir, envFile, overrideProvidersPath }; } async function runCli( @@ -257,7 +257,7 @@ describe('agentv runs rerun', () => { expect(answer).not.toContain('Captured answer'); }, 30_000); - it('reruns legacy task_dir bundles for backward compatibility', async () => { + it('reruns task_dir bundles that use providers_path', async () => { const created = await createBundleFixture(DEFAULT_TARGETS, { legacyTaskDir: true }); const result = await runCli(created, [ @@ -321,7 +321,7 @@ describe('agentv runs rerun', () => { it('fails loudly when selected bundle artifacts are missing', async () => { const created = await fixture(); - await rm(path.join(created.sourceRunDir, 'case-beta', 'test', 'targets.yaml')); + await rm(path.join(created.sourceRunDir, 'case-beta', 'test', 'providers.yaml')); const result = await runCli(created, [ 'runs', @@ -334,7 +334,7 @@ describe('agentv runs rerun', () => { ]); expect(result.exitCode).toBe(1); - expect(result.stderr).toContain('Test targets for case-beta@captured not found'); + expect(result.stderr).toContain('Test providers for case-beta@captured not found'); }, 30_000); it('reruns a selected test subset from index.jsonl', async () => { @@ -404,7 +404,7 @@ describe('agentv runs rerun', () => { 'rerun', created.sourceRunDir, '--targets', - created.overrideTargetsPath, + created.overrideProvidersPath, '--target', 'missing', '--output', @@ -413,7 +413,7 @@ describe('agentv runs rerun', () => { ]); expect(result.exitCode).toBe(1); - expect(result.stderr).toContain('Target override is incompatible'); + expect(result.stderr).toContain('Provider override is incompatible'); expect(result.stderr).toContain('missing'); }, 30_000); @@ -425,7 +425,7 @@ describe('agentv runs rerun', () => { 'rerun', created.sourceRunDir, '--targets', - created.overrideTargetsPath, + created.overrideProvidersPath, '--target', 'local', '--output', diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index fed09f967..235918396 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -151,7 +151,7 @@ my-results/ ``` The `index.jsonl` row links to these generated paths with snake_case fields such -as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`, +as `result_dir`, `test_dir`, `eval_path`, `providers_path`, `files_path`, `file_changes_path`, and `graders_path`. Treat those paths as relative to the run directory. When you need a portable artifact for audit, review, Dashboard inspection, or rerun workflows, share the generated run directory and its diff --git a/apps/web/src/content/docs/docs/next/tools/results.mdx b/apps/web/src/content/docs/docs/next/tools/results.mdx index ddd8fd74d..1dd760145 100644 --- a/apps/web/src/content/docs/docs/next/tools/results.mdx +++ b/apps/web/src/content/docs/docs/next/tools/results.mdx @@ -100,7 +100,7 @@ Use `results export` when you need the artifact workspace layout itself rather t agentv results export [--out ] [--duplicate-policy update] ``` -This is useful when a manifest needs to be materialized into a predictable artifact tree for other tooling, review, or archiving. The run workspace is also where generated test bundles live: `.internal/index.jsonl` rows may point to per-result `test_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` entries. Keep those generated artifacts with the run when sharing or auditing results. +This is useful when a manifest needs to be materialized into a predictable artifact tree for other tooling, review, or archiving. The run workspace is also where generated test bundles live: `.internal/index.jsonl` rows may point to per-result `test_dir`, `eval_path`, `providers_path`, `files_path`, and `graders_path` entries. Keep those generated artifacts with the run when sharing or auditing results. The export source is still the canonical run bundle described in the [Result Artifact Contract](/docs/reference/result-artifacts/): `summary.json` @@ -186,7 +186,7 @@ Agent Skills eval artifacts map into AgentV like this: | Agent Skills pattern | AgentV field | Artifact location | |----------------------|--------------|-------------------| -| Converted Agent Skills cases | AgentV eval cases and test bundle paths | Converted EVAL YAML plus optional `test_dir`, `eval_path`, `targets_path`, `files_path`, and `graders_path` in `.internal/index.jsonl` | +| Converted Agent Skills cases | AgentV eval cases and test bundle paths | Converted EVAL YAML plus optional `test_dir`, `eval_path`, `providers_path`, `files_path`, and `graders_path` in `.internal/index.jsonl` | | Per-case answer | Generated target output artifact | `sample-N/outputs/answer.md` | | Per-attempt sidecars | Normalized transcript, metrics, and raw provider evidence | `sample-N/transcript.json`, `sample-N/transcript-raw.jsonl`, `sample-N/metrics.json` | | Per-sample `metrics.json` | Duration, token totals, cost, execution, trajectory, and usage source labels | `sample-N/metrics.json` | diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index ede3b5c26..17b9ad318 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -336,7 +336,7 @@ export async function evaluate(config: EvalConfig): Promise { config: {}, }; } else { - // Resolve target — inline definition or auto-discover from targets.yaml + // Resolve target — inline definition or auto-discover from providers.yaml let targetDef: TargetDefinition; if (config.target) { targetDef = config.target; @@ -743,10 +743,10 @@ function computeSummary( }; } -const TARGET_FILE_CANDIDATES = ['.agentv/targets.yaml', '.agentv/targets.yml'] as const; +const TARGET_FILE_CANDIDATES = ['.agentv/providers.yaml', '.agentv/providers.yml'] as const; /** - * Auto-discover the 'default' target from targets.yaml in the repo tree. + * Auto-discover the 'default' provider from providers.yaml in the repo tree. */ async function discoverDefaultTarget(repoRoot: string): Promise { const cwd = process.cwd(); diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index e41511215..8d0b2e18b 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -56,7 +56,7 @@ export async function readTargetDefinitions( ): Promise { const absolutePath = path.resolve(filePath); if (!(await fileExists(absolutePath))) { - throw new Error(`targets.yaml not found at ${absolutePath}`); + throw new Error(`providers.yaml not found at ${absolutePath}`); } const raw = await readFile(absolutePath, 'utf8'); diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 78576e6e4..db0c53337 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -109,7 +109,7 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ ] as const; /** - * Schema identifier for targets.yaml files (version 2). + * Schema identifier for providers.yaml files (version 2). */ export const TARGETS_SCHEMA_V2 = 'agentv-targets-v2.2'; diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts index 8bb29fab3..723bf3ecd 100644 --- a/packages/core/src/evaluation/result-row-schema.ts +++ b/packages/core/src/evaluation/result-row-schema.ts @@ -42,7 +42,7 @@ const RESULT_ROW_ALIASES = { summaryPath: 'summary_path', targetExecution: 'target_execution', targetExecutionPath: 'target_execution_path', - targetsPath: 'targets_path', + providersPath: 'providers_path', taskDir: 'task_dir', testDir: 'test_dir', testId: 'test_id', diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index de8530208..2a0ce3161 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -634,7 +634,7 @@ export interface IndexArtifactEntry { readonly test_dir?: string; readonly task_dir?: string; readonly eval_path?: string; - readonly targets_path?: string; + readonly providers_path?: string; readonly files_path?: string; readonly graders_path?: string; readonly external_trace?: ExternalTraceMetadataWire; @@ -655,7 +655,7 @@ export type AdditionalResultIndexFields = Partial< | 'test_dir' | 'task_dir' | 'eval_path' - | 'targets_path' + | 'providers_path' | 'files_path' | 'graders_path' | 'raw_provider_log_path' diff --git a/packages/core/src/evaluation/validation/file-type.ts b/packages/core/src/evaluation/validation/file-type.ts index 35f6a0b84..bb2fcc2f2 100644 --- a/packages/core/src/evaluation/validation/file-type.ts +++ b/packages/core/src/evaluation/validation/file-type.ts @@ -17,7 +17,7 @@ const SCHEMA_CONFIG_V2 = 'agentv-config-v2'; * Detect file type by reading $schema field from YAML file. * If $schema is missing, infers type from filename/path: * - config.yaml/config.local.yaml under .agentv folder → 'config' - * - targets.yaml under .agentv folder → 'targets' + * - providers.yaml under .agentv folder → 'targets' * - suite.yaml/suite.yml → 'eval' */ export async function detectFileType(filePath: string): Promise { @@ -77,7 +77,7 @@ function inferFileTypeFromPath(filePath: string): FileType { if (isAgentVConfigFileName(basename)) { return 'config'; } - if (basename === 'targets.yaml' || basename === 'targets.yml') { + if (basename === 'providers.yaml' || basename === 'providers.yml') { return 'targets'; } } From be0e9093365a9dae3203dd5e922ecfedfb4f19d6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 05:57:40 +0200 Subject: [PATCH 2/2] test(cli): migrate eval fixtures to providers yaml --- apps/cli/test/commands/grade/grade-prepared.test.ts | 2 +- apps/cli/test/commands/prepare/prepare.test.ts | 4 ++-- apps/cli/test/eval.integration.test.ts | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index a9db768b1..97e466f75 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -80,7 +80,7 @@ console.log(JSON.stringify({ 'utf8', ); await writeFile( - path.join(root, '.agentv', 'targets.yaml'), + path.join(root, '.agentv', 'providers.yaml'), ` providers: - id: cli diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 745d69912..b7f719d4b 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -57,7 +57,7 @@ await Bun.write(\`\${payload.workspace_path}/\${step}.txt\`, \`\${payload.test_i 'utf8', ); await writeFile( - path.join(root, '.agentv', 'targets.yaml'), + path.join(root, '.agentv', 'providers.yaml'), ` providers: - id: cli @@ -250,7 +250,7 @@ describe('agentv prepare', () => { await writeFile(path.join(tempDir, 'rules', 'AGENTS.md'), '# Rules\n', 'utf8'); await writeFile(path.join(tempDir, 'scripts', 'target.ts'), '', 'utf8'); await writeFile( - path.join(tempDir, '.agentv', 'targets.yaml'), + path.join(tempDir, '.agentv', 'providers.yaml'), ` providers: - id: cli diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 981325837..6777a3495 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -68,8 +68,8 @@ async function createFixture(): Promise { const agentvDir = path.join(suiteDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); - const targetsPath = path.join(agentvDir, 'targets.yaml'); - const targetsContent = `$schema: agentv-targets-v2.2 + const providersPath = path.join(agentvDir, 'providers.yaml'); + const providersContent = `$schema: agentv-targets-v2.2 providers: - id: mock label: default @@ -81,7 +81,7 @@ providers: label: codex-target model: gpt-5-default `; - await writeFile(targetsPath, targetsContent, 'utf8'); + await writeFile(providersPath, providersContent, 'utf8'); const testFilePath = path.join(suiteDir, 'sample.test.yaml'); const testFileContent = `description: CLI integration test @@ -130,13 +130,13 @@ async function createNestedEnvFixture(): Promise { const agentvDir = path.join(suiteDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); - const targetsPath = path.join(agentvDir, 'targets.yaml'); - const targetsContent = `$schema: agentv-targets-v2.2 + const providersPath = path.join(agentvDir, 'providers.yaml'); + const providersContent = `$schema: agentv-targets-v2.2 providers: - id: mock label: default `; - await writeFile(targetsPath, targetsContent, 'utf8'); + await writeFile(providersPath, providersContent, 'utf8'); const testFilePath = path.join(evalDir, 'sample.test.yaml'); const testFileContent = `description: CLI nested env integration test