fix(artifacts): write provider bundle snapshots

README.md

@@ -196,7 +196,7 @@ Run bundle layout:
 │   │   ├── summary.json          # optional per-case rollup across samples
 │   │   ├── test/                 # generated test bundle: frozen inputs for reproducibility
 │   │   │   ├── EVAL.yaml         #   resolved eval spec
-│   │   │   ├── targets.yaml      #   resolved target config
+│   │   │   ├── providers.yaml    #   resolved provider config
 │   │   │   └── graders/          #   grader files used
 │   │   └── sample-1/             # one materialized sample
 │   │       ├── result.json       # compact sample manifest

apps/cli/src/commands/eval/artifact-writer.ts

@@ -82,15 +82,15 @@ function buildTaskBundleIndexFields(
   taskBundle: MaterializedTaskBundlePaths | undefined,
 ): Pick<
   IndexArtifactEntry,
-  'test_dir' | 'eval_path' | 'targets_path' | 'files_path' | 'graders_path'
+  'test_dir' | 'eval_path' | 'providers_path' | 'files_path' | 'graders_path'
 > {
   if (!taskBundle) {
     return {};
   }
   return {
     test_dir: toRelativeArtifactPath(outputDir, taskBundle.testDir),
     eval_path: toRelativeArtifactPath(outputDir, taskBundle.evalPath),
-    targets_path: toRelativeArtifactPath(outputDir, taskBundle.targetsPath),
+    providers_path: toRelativeArtifactPath(outputDir, taskBundle.providersPath),
     ...(taskBundle.filesPath
       ? { files_path: toRelativeArtifactPath(outputDir, taskBundle.filesPath) }
       : {}),
@@ -133,7 +133,7 @@ export function buildResultIndexArtifact(
     ? {
         test_dir: path.posix.join(artifactSubdir, 'test'),
         eval_path: path.posix.join(artifactSubdir, 'test', 'EVAL.yaml'),
-        targets_path: path.posix.join(artifactSubdir, 'test', 'targets.yaml'),
+        providers_path: path.posix.join(artifactSubdir, 'test', 'providers.yaml'),
         ...(taskBundle.filesPath
           ? { files_path: path.posix.join(artifactSubdir, 'test', 'files') }
           : {}),

apps/cli/src/commands/eval/commands/bundle.ts

@@ -166,7 +166,7 @@ export const evalBundleCommand = command({
     targets: option({
       type: optional(string),
       long: 'targets',
-      description: 'Path to targets.yaml (overrides discovery)',
+      description: 'Path to providers.yaml (overrides discovery)',
     }),
   },
   handler: async (args) => {
@@ -243,7 +243,7 @@ export const evalBundleCommand = command({
       `  Eval: ${path.relative(paths.bundleDir, paths.evalPath).split(path.sep).join('/')}`,
     );
     console.log(
-      `  Targets: ${path.relative(paths.bundleDir, paths.targetsPath).split(path.sep).join('/')}`,
+      `  Providers: ${path.relative(paths.bundleDir, paths.providersPath).split(path.sep).join('/')}`,
     );
     console.log(
       `  Manifest: ${path.relative(paths.bundleDir, paths.manifestPath).split(path.sep).join('/')}`,

apps/cli/src/commands/eval/interactive.ts

@@ -201,11 +201,11 @@ async function promptEvalSelection(
 async function promptTargetSelection(cwd: string, firstEvalPath: string): Promise<string> {
   const repoRoot = await findRepoRoot(cwd);
 
-  // Try to find targets.yaml — search near the eval file first, then cwd/repoRoot
+  // Try to find providers.yaml near the eval file first, then cwd/repoRoot.
   const targetsPath = await findTargetsFile(cwd, repoRoot, firstEvalPath);
 
   if (!targetsPath) {
-    console.log(`${ANSI_DIM}No targets.yaml found. Using default target.${ANSI_RESET}`);
+    console.log(`${ANSI_DIM}No providers.yaml found. Using default provider.${ANSI_RESET}`);
     return 'default';
   }
 

apps/cli/src/commands/eval/task-bundle.ts

@@ -17,13 +17,13 @@ import { stringify as stringifyYaml } from 'yaml';
 
 const TEST_BUNDLE_DIRNAME = 'test';
 const TASK_EVAL_FILENAME = 'EVAL.yaml';
-const TASK_TARGETS_FILENAME = 'targets.yaml';
+const TASK_PROVIDERS_FILENAME = 'providers.yaml';
 const TASK_FILES_DIRNAME = 'files';
 const TASK_GRADERS_DIRNAME = 'graders';
 const INPUT_PROMPT = '{{ input }}';
 const BUNDLE_EVALS_DIRNAME = 'evals';
 const BUNDLE_MANIFEST_FILENAME = 'agentv_bundle.json';
-const BUNDLE_TARGETS_FILENAME = 'targets.yaml';
+const BUNDLE_PROVIDERS_FILENAME = 'providers.yaml';
 const BUNDLE_WORKSPACES_DIRNAME = 'workspaces';
 const BUNDLE_SCRIPTS_DIRNAME = 'scripts';
 const REDACTED_SOURCE_VALUE = '[redacted]';
@@ -89,7 +89,7 @@ export interface MaterializeTaskBundleOptions {
 export interface MaterializedTaskBundlePaths {
   readonly testDir: string;
   readonly evalPath: string;
-  readonly targetsPath: string;
+  readonly providersPath: string;
   readonly filesPath?: string;
   readonly gradersPath?: string;
 }
@@ -109,7 +109,7 @@ export interface MaterializedEvalBundlePaths {
   readonly bundleDir: string;
   readonly evalsDir: string;
   readonly evalPath: string;
-  readonly targetsPath: string;
+  readonly providersPath: string;
   readonly manifestPath: string;
   readonly filesPath?: string;
   readonly gradersPath?: string;
@@ -1088,7 +1088,7 @@ function bundleManifest(options: {
   readonly outputDir: string;
   readonly evalFilePath: string;
   readonly evalPath: string;
-  readonly targetsPath: string;
+  readonly providersPath: string;
   readonly copiedReferences: readonly CopiedReference[];
   readonly tests: readonly EvalTest[];
   readonly targetNames: readonly string[];
@@ -1101,7 +1101,7 @@ function bundleManifest(options: {
     created_at: options.createdAt,
     source_eval: options.evalFilePath,
     eval_path: relative(options.evalPath),
-    targets_path: relative(options.targetsPath),
+    providers_path: relative(options.providersPath),
     test_count: options.tests.length,
     targets: options.targetNames,
     ...(hasCopiedBucket(options.copiedReferences, 'files') ? { files_path: 'evals/files' } : {}),
@@ -1120,7 +1120,7 @@ function bundleManifest(options: {
 /**
  * Materialize the native AgentV task source for one completed result row.
  *
- * The bundle is intentionally just an eval file, a selected targets file, and
+ * The bundle is intentionally just an eval file, a selected providers file, and
  * copied referenced assets. It does not create `.agentv/` under the result
  * artifact directory, so future reruns can choose their output root explicitly.
  */
@@ -1143,19 +1143,19 @@ export async function materializeTaskBundle(
   const rewrites = buildPathRewrites(copiedReferences);
   const evalCase = buildEvalCase(options.test, rewrites);
   const evalPath = path.join(testDir, TASK_EVAL_FILENAME);
-  const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME);
+  const providersPath = path.join(testDir, TASK_PROVIDERS_FILENAME);
 
   await writeYamlFile(evalPath, {
     providers: [options.targetName],
     prompts: [INPUT_PROMPT],
     tests: [evalCase],
   });
-  await writeYamlFile(targetsPath, { providers: serializeTargetDefinitions(targetDefinitions) });
+  await writeYamlFile(providersPath, { providers: serializeTargetDefinitions(targetDefinitions) });
 
   return {
     testDir,
     evalPath,
-    targetsPath,
+    providersPath,
     ...(hasCopiedBucket(copiedReferences, 'files')
       ? { filesPath: path.join(testDir, TASK_FILES_DIRNAME) }
       : {}),
@@ -1213,7 +1213,7 @@ export async function materializeEvalBundle(
   const rewrites = buildPathRewrites(copied);
   const targetNames = uniqueTargetNames(options.targetSelections);
   const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath));
-  const targetsPath = path.join(outputDir, BUNDLE_TARGETS_FILENAME);
+  const providersPath = path.join(outputDir, BUNDLE_PROVIDERS_FILENAME);
   const manifestPath = path.join(outputDir, BUNDLE_MANIFEST_FILENAME);
   const runtime =
     options.runtime ?? (targetNames.length > 0 ? { providers: targetNames } : undefined);
@@ -1223,15 +1223,15 @@ export async function materializeEvalBundle(
     prompts: [INPUT_PROMPT],
     tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)),
   });
-  await writeYamlFile(targetsPath, {
+  await writeYamlFile(providersPath, {
     providers: serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)),
   });
 
   const manifest = bundleManifest({
     outputDir,
     evalFilePath: path.resolve(options.evalFilePath),
     evalPath,
-    targetsPath,
+    providersPath,
     copiedReferences: copied,
     tests: options.tests,
     targetNames,
@@ -1243,7 +1243,7 @@ export async function materializeEvalBundle(
     bundleDir: outputDir,
     evalsDir,
     evalPath,
-    targetsPath,
+    providersPath,
     manifestPath,
     ...(hasCopiedBucket(copied, 'files')
       ? { filesPath: path.join(evalsDir, TASK_FILES_DIRNAME) }

apps/cli/src/commands/results/combine-run.ts

@@ -366,7 +366,7 @@ const MANIFEST_PATH_FIELDS = [
   'test_dir',
   'task_dir',
   'eval_path',
-  'targets_path',
+  'providers_path',
   'files_path',
   'graders_path',
 ] as const;

apps/cli/src/commands/results/export.ts

@@ -200,9 +200,9 @@ function createExportBundleArtifactsWriter(options: {
     return {
       test_dir: toRelativeArtifactPath(options.outputDir, testBundlePath),
       eval_path: toRelativeArtifactPath(options.outputDir, path.join(testBundlePath, 'EVAL.yaml')),
-      targets_path: toRelativeArtifactPath(
+      providers_path: toRelativeArtifactPath(
         options.outputDir,
-        path.join(testBundlePath, 'targets.yaml'),
+        path.join(testBundlePath, 'providers.yaml'),
       ),
       ...(sourceRecord?.files_path || hasCopiedSubdir(testBundlePath, 'files')
         ? {

apps/cli/src/commands/results/manifest.ts

@@ -99,7 +99,7 @@ export interface ResultManifestRecord {
   readonly test_dir?: string;
   readonly task_dir?: string;
   readonly eval_path?: string;
-  readonly targets_path?: string;
+  readonly providers_path?: string;
   readonly files_path?: string;
   readonly graders_path?: string;
   readonly metadata?: Record<string, unknown>;

apps/cli/src/commands/results/projection-bundle.ts

@@ -98,7 +98,7 @@ export type ProjectionBundleArtifactRefs = Partial<
     | 'test_dir'
     | 'task_dir'
     | 'eval_path'
-    | 'targets_path'
+    | 'providers_path'
     | 'files_path'
     | 'graders_path'
   >
@@ -177,7 +177,7 @@ function artifactRefs(
     test_dir: indexEntry.test_dir,
     task_dir: indexEntry.task_dir,
     eval_path: indexEntry.eval_path,
-    targets_path: indexEntry.targets_path,
+    providers_path: indexEntry.providers_path,
     files_path: indexEntry.files_path,
     graders_path: indexEntry.graders_path,
   });

apps/cli/src/commands/results/serve.ts

@@ -839,7 +839,7 @@ function buildResultArtifactCatalog(
   addDirectArtifactCatalogEntry(entries, seen, record.transcript_raw_path, 'artifact');
   addDirectArtifactCatalogEntry(entries, seen, record.trace_path, 'trace');
   addDirectArtifactCatalogEntry(entries, seen, record.eval_path, 'artifact');
-  addDirectArtifactCatalogEntry(entries, seen, record.targets_path, 'artifact');
+  addDirectArtifactCatalogEntry(entries, seen, record.providers_path, 'artifact');
   addTrialRunCatalogEntries(entries, seen, record);
 
   return entries;

apps/cli/src/commands/runs/rerun.ts

@@ -25,7 +25,7 @@ import { runEvalCommand } from '../eval/run-eval.js';
 import { type ResultManifestRecord, parseResultManifest } from '../results/manifest.js';
 
 const TASK_EVAL_FILENAME = 'EVAL.yaml';
-const TASK_TARGETS_FILENAME = 'targets.yaml';
+const TASK_PROVIDERS_FILENAME = 'providers.yaml';
 const ENV_REF_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
 
 interface SelectedTaskBundle {
@@ -35,7 +35,7 @@ interface SelectedTaskBundle {
   readonly resultDir: string;
   readonly testDir: string;
   readonly evalPath: string;
-  readonly targetsPath: string;
+  readonly providersPath: string;
   readonly taskTarget: string;
 }
 
@@ -112,9 +112,9 @@ async function readTaskTarget(evalPath: string, fallback: string): Promise<strin
 }
 
 async function readRerunTargetDefinitions(
-  targetsPath: string,
+  providersPath: string,
 ): Promise<readonly Record<string, unknown>[]> {
-  const definitions = await readCoreTargetDefinitions(targetsPath);
+  const definitions = await readCoreTargetDefinitions(providersPath);
   return definitions.map((definition) => definition as unknown as Record<string, unknown>);
 }
 
@@ -191,11 +191,11 @@ function collectEnvRefs(value: unknown, names = new Set<string>()): Set<string>
 }
 
 async function validateTargetFile(
-  targetsPath: string,
+  providersPath: string,
   targetNames: readonly string[],
   label: string,
 ): Promise<void> {
-  const definitions = await readRerunTargetDefinitions(targetsPath);
+  const definitions = await readRerunTargetDefinitions(providersPath);
   const byName = new Map<string, Record<string, unknown>>();
   for (const definition of definitions) {
     const name = targetName(definition);
@@ -207,7 +207,7 @@ async function validateTargetFile(
   const missingTargets = [...new Set(targetNames)].filter((name) => !byName.has(name));
   if (missingTargets.length > 0) {
     throw new Error(
-      `${label} is incompatible: ${targetsPath} does not define target(s): ${missingTargets.join(
+      `${label} is incompatible: ${providersPath} does not define provider(s): ${missingTargets.join(
         ', ',
       )}`,
     );
@@ -239,7 +239,7 @@ async function validateTargetFile(
   });
   if (missingEnv.length > 0) {
     throw new Error(
-      `Missing environment variable(s) required by ${targetsPath}: ${missingEnv.join(
+      `Missing environment variable(s) required by ${providersPath}: ${missingEnv.join(
         ', ',
       )}. Provide --env-file <path> or export them before rerun.`,
     );
@@ -332,11 +332,11 @@ async function loadSelectedTaskBundles(options: {
         options.sourceRunDir,
         bundleDir && `${bundleDir}/${TASK_EVAL_FILENAME}`,
       );
-    const targetsPath =
-      resolveRelativeRunPath(options.sourceRunDir, record.targets_path) ??
+    const providersPath =
+      resolveRelativeRunPath(options.sourceRunDir, record.providers_path) ??
       resolveRelativeRunPath(
         options.sourceRunDir,
-        bundleDir && `${bundleDir}/${TASK_TARGETS_FILENAME}`,
+        bundleDir && `${bundleDir}/${TASK_PROVIDERS_FILENAME}`,
       );
     const testDir =
       resolveRelativeRunPath(options.sourceRunDir, bundleDir) ??
@@ -345,14 +345,14 @@ async function loadSelectedTaskBundles(options: {
       resolveRelativeRunPath(options.sourceRunDir, record.result_dir) ??
       (testDir ? path.dirname(testDir) : undefined);
 
-    if (!evalPath || !targetsPath || !testDir || !resultDir) {
+    if (!evalPath || !providersPath || !testDir || !resultDir) {
       throw new Error(
-        `Selected result ${recordLabel} is missing test bundle paths. Re-run requires test/EVAL.yaml and test/targets.yaml.`,
+        `Selected result ${recordLabel} is missing test bundle paths. Re-run requires test/EVAL.yaml and test/providers.yaml.`,
       );
     }
 
     await ensureFile(evalPath, `Test eval for ${recordLabel}`);
-    await ensureFile(targetsPath, `Test targets for ${recordLabel}`);
+    await ensureFile(providersPath, `Test providers for ${recordLabel}`);
     const taskTarget = await readTaskTarget(evalPath, sourceTarget);
     selected.push({
       record,
@@ -361,7 +361,7 @@ async function loadSelectedTaskBundles(options: {
       resultDir,
       testDir,
       evalPath,
-      targetsPath,
+      providersPath,
       taskTarget,
     });
   }
@@ -425,7 +425,7 @@ export const runsRerunCommand = command({
     targets: option({
       type: optional(string),
       long: 'targets',
-      description: 'Path to replacement targets.yaml for the new eval run',
+      description: 'Path to replacement providers.yaml for the new eval run',
     }),
     envFile: option({
       type: optional(string),
@@ -478,23 +478,23 @@ export const runsRerunCommand = command({
     assertOutputIsSeparate(outputDir, forbiddenOutputRoots(sourceRunDir, selected));
 
     if (args.targets) {
-      const overrideTargetsPath = path.resolve(cwd, args.targets);
-      await ensureFile(overrideTargetsPath, 'Target override');
+      const overrideProvidersPath = path.resolve(cwd, args.targets);
+      await ensureFile(overrideProvidersPath, 'Provider override');
       const targetNames =
         targetOverrides.length > 0 ? targetOverrides : selected.map((bundle) => bundle.taskTarget);
-      await validateTargetFile(overrideTargetsPath, targetNames, 'Target override');
+      await validateTargetFile(overrideProvidersPath, targetNames, 'Provider override');
     } else {
       const targetNamesByFile = new Map<string, Set<string>>();
       for (const bundle of selected) {
         const targetNames = targetOverrides.length > 0 ? targetOverrides : [bundle.taskTarget];
-        const names = targetNamesByFile.get(bundle.targetsPath) ?? new Set<string>();
+        const names = targetNamesByFile.get(bundle.providersPath) ?? new Set<string>();
         for (const targetName of targetNames) {
           names.add(targetName);
         }
-        targetNamesByFile.set(bundle.targetsPath, names);
+        targetNamesByFile.set(bundle.providersPath, names);
       }
-      for (const [targetsPath, names] of targetNamesByFile.entries()) {
-        await validateTargetFile(targetsPath, [...names], 'Test bundle targets');
+      for (const [providersPath, names] of targetNamesByFile.entries()) {
+        await validateTargetFile(providersPath, [...names], 'Test bundle providers');
       }
     }