fix(artifacts): rename generated task bundle to test bundle

CONCEPTS.md

@@ -24,7 +24,7 @@ Shared domain vocabulary for this project — entities, named processes, and sta
 
 **Workspace** — The task environment an eval prepares for the agent: repositories, templates, fixture files, and lifecycle hooks. It is not prompt input; use `input` for instructions and `workspace.repos[]` for multi-repo workspaces the agent can inspect or modify through tools.
 
-**Run manifest** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `task_dir`, `summary_path`, and `grading_path`.
+**Run manifest** — The root `index.jsonl` file in a run bundle. It is the dashboard and tooling loading contract for per-case result rows and artifact locations, including fields such as `result_dir`, `test_dir`, `summary_path`, and `grading_path`.
 
 **Result source identity** — The stable source identity for a result row: repo-relative `eval_path`, `test_id`, and `target`. `suite` and `name` are display metadata, not storage or routing identity.
 

apps/cli/src/commands/eval/artifact-writer.ts

@@ -74,13 +74,13 @@ function buildTaskBundleIndexFields(
   taskBundle: MaterializedTaskBundlePaths | undefined,
 ): Pick<
   IndexArtifactEntry,
-  'task_dir' | 'eval_path' | 'targets_path' | 'files_path' | 'graders_path'
+  'test_dir' | 'eval_path' | 'targets_path' | 'files_path' | 'graders_path'
 > {
   if (!taskBundle) {
     return {};
   }
   return {
-    task_dir: toRelativeArtifactPath(outputDir, taskBundle.taskDir),
+    test_dir: toRelativeArtifactPath(outputDir, taskBundle.testDir),
     eval_path: toRelativeArtifactPath(outputDir, taskBundle.evalPath),
     targets_path: toRelativeArtifactPath(outputDir, taskBundle.targetsPath),
     ...(taskBundle.filesPath
@@ -123,14 +123,14 @@ export function buildResultIndexArtifact(
   const artifactSubdir = (buildCoreResultIndexArtifact(result).result_dir ?? '').trim();
   const extraIndexFields = taskBundle
     ? {
-        task_dir: path.posix.join(artifactSubdir, 'task'),
-        eval_path: path.posix.join(artifactSubdir, 'task', 'EVAL.yaml'),
-        targets_path: path.posix.join(artifactSubdir, 'task', 'targets.yaml'),
+        test_dir: path.posix.join(artifactSubdir, 'test'),
+        eval_path: path.posix.join(artifactSubdir, 'test', 'EVAL.yaml'),
+        targets_path: path.posix.join(artifactSubdir, 'test', 'targets.yaml'),
         ...(taskBundle.filesPath
-          ? { files_path: path.posix.join(artifactSubdir, 'task', 'files') }
+          ? { files_path: path.posix.join(artifactSubdir, 'test', 'files') }
           : {}),
         ...(taskBundle.gradersPath
-          ? { graders_path: path.posix.join(artifactSubdir, 'task', 'graders') }
+          ? { graders_path: path.posix.join(artifactSubdir, 'test', 'graders') }
           : {}),
       }
     : undefined;
@@ -220,6 +220,7 @@ export async function writePerTestArtifacts(
     repoRoot?: string;
     sourceTests?: readonly EvalTest[];
     taskBundleTargets?: readonly TaskBundleTargetSelection[];
+    additionalArtifacts?: AdditionalResultArtifactsWriter;
     runtimeSource?: RunRuntimeSourceMetadata;
   },
 ): Promise<void> {
@@ -229,7 +230,7 @@ export async function writePerTestArtifacts(
     runId: options?.runId,
     duplicatePolicy: options?.duplicatePolicy,
     sourceTests: options?.sourceTests,
-    additionalArtifacts: createTaskBundleArtifactsWriter(options),
+    additionalArtifacts: options?.additionalArtifacts ?? createTaskBundleArtifactsWriter(options),
     runtimeSource: options?.runtimeSource,
   });
 }
@@ -249,6 +250,7 @@ export async function writeArtifactsFromResults(
     repoRoot?: string;
     sourceTests?: readonly EvalTest[];
     taskBundleTargets?: readonly TaskBundleTargetSelection[];
+    additionalArtifacts?: AdditionalResultArtifactsWriter;
     runtimeSource?: RunRuntimeSourceMetadata;
   },
 ): Promise<{
@@ -265,7 +267,7 @@ export async function writeArtifactsFromResults(
     duplicatePolicy: options?.duplicatePolicy,
     resultGroup: options?.resultGroup,
     sourceTests: options?.sourceTests,
-    additionalArtifacts: createTaskBundleArtifactsWriter(options),
+    additionalArtifacts: options?.additionalArtifacts ?? createTaskBundleArtifactsWriter(options),
     runtimeSource: options?.runtimeSource,
   });
 }

apps/cli/src/commands/eval/task-bundle.ts

@@ -15,7 +15,7 @@ import { stringify as stringifyYaml } from 'yaml';
 
 import { toSnakeCaseDeep } from '../../utils/case-conversion.js';
 
-const TASK_DIRNAME = 'task';
+const TEST_BUNDLE_DIRNAME = 'test';
 const TASK_EVAL_FILENAME = 'EVAL.yaml';
 const TASK_TARGETS_FILENAME = 'targets.yaml';
 const TASK_FILES_DIRNAME = 'files';
@@ -62,7 +62,7 @@ export interface MaterializeTaskBundleOptions {
 }
 
 export interface MaterializedTaskBundlePaths {
-  readonly taskDir: string;
+  readonly testDir: string;
   readonly evalPath: string;
   readonly targetsPath: string;
   readonly filesPath?: string;
@@ -950,14 +950,14 @@ export async function materializeTaskBundle(
     return undefined;
   }
 
-  const taskDir = path.join(options.outputDir, TASK_DIRNAME);
-  await mkdir(taskDir, { recursive: true });
+  const testDir = path.join(options.outputDir, TEST_BUNDLE_DIRNAME);
+  await mkdir(testDir, { recursive: true });
 
-  const copiedReferences = await copyReferences(options.test.source.references, taskDir, options);
+  const copiedReferences = await copyReferences(options.test.source.references, testDir, options);
   const rewrites = buildPathRewrites(copiedReferences);
   const evalCase = buildEvalCase(options.test, rewrites);
-  const evalPath = path.join(taskDir, TASK_EVAL_FILENAME);
-  const targetsPath = path.join(taskDir, TASK_TARGETS_FILENAME);
+  const evalPath = path.join(testDir, TASK_EVAL_FILENAME);
+  const targetsPath = path.join(testDir, TASK_TARGETS_FILENAME);
 
   await writeYamlFile(evalPath, {
     execution: { target: options.targetName },
@@ -966,14 +966,14 @@ export async function materializeTaskBundle(
   await writeYamlFile(targetsPath, { targets: targetDefinitions });
 
   return {
-    taskDir,
+    testDir,
     evalPath,
     targetsPath,
     ...(hasCopiedBucket(copiedReferences, 'files')
-      ? { filesPath: path.join(taskDir, TASK_FILES_DIRNAME) }
+      ? { filesPath: path.join(testDir, TASK_FILES_DIRNAME) }
       : {}),
     ...(hasCopiedBucket(copiedReferences, 'graders')
-      ? { gradersPath: path.join(taskDir, TASK_GRADERS_DIRNAME) }
+      ? { gradersPath: path.join(testDir, TASK_GRADERS_DIRNAME) }
       : {}),
   };
 }
@@ -982,7 +982,7 @@ export async function materializeTaskBundle(
  * Materialize a whole eval suite as a portable directory.
  *
  * This reuses the same source snapshots, dependency copying, path rewriting,
- * target slicing, and secret redaction used by per-result task bundles. The
+ * target slicing, and secret redaction used by per-result test bundles. The
  * output eval is intentionally explicit: inherited suite defaults are written
  * onto each bundled test case so the bundle can run without the source tree.
  */

apps/cli/src/commands/results/combine-run.ts

@@ -377,6 +377,7 @@ const MANIFEST_PATH_FIELDS = [
   'transcript_raw_path',
   'metrics_path',
   'raw_provider_log_path',
+  'test_dir',
   'task_dir',
   'eval_path',
   'targets_path',

apps/cli/src/commands/results/export.ts

@@ -20,12 +20,18 @@
  *   - To add new per-test workspace files, add them under each test directory.
  */
 
-import { readFileSync } from 'node:fs';
+import { cpSync, existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 
 import { command, flag, oneOf, option, optional, positional, string } from 'cmd-ts';
 
-import type { EvaluationResult, ExportDuplicatePolicy, IndexArtifactEntry } from '@agentv/core';
+import type {
+  AdditionalResultArtifactsWriter,
+  AdditionalResultIndexFields,
+  EvaluationResult,
+  ExportDuplicatePolicy,
+  IndexArtifactEntry,
+} from '@agentv/core';
 
 import { parseJsonlResults, writeArtifactsFromResults } from '../eval/artifact-writer.js';
 import {
@@ -51,6 +57,7 @@ export async function exportResults(
   options?: { duplicatePolicy?: ExportDuplicatePolicy },
 ): Promise<void> {
   const results = parseJsonlResults(content);
+  const sourceIndexRecords = parseIndexArtifactEntries(content);
 
   if (results.length === 0) {
     throw new Error(`No results found in ${sourceFile}`);
@@ -60,6 +67,11 @@ export async function exportResults(
     evalFile: sourceFile,
     runId: deriveExportRunId(sourceFile),
     duplicatePolicy: options?.duplicatePolicy ?? 'update',
+    additionalArtifacts: createExportBundleArtifactsWriter({
+      outputDir,
+      sourceBaseDir: path.dirname(sourceFile),
+      sourceRecordsByResult: buildSourceRecordMap(results, sourceIndexRecords),
+    }),
   });
 }
 
@@ -100,20 +112,109 @@ export function deriveExportRunId(sourceFile: string): string {
 export async function loadExportSource(
   source: string | undefined,
   cwd: string,
-): Promise<{ sourceFile: string; results: readonly EvaluationResult[] }> {
+): Promise<{
+  sourceFile: string;
+  results: readonly EvaluationResult[];
+  indexRecords?: readonly IndexArtifactEntry[];
+}> {
   const { sourceFile } = await resolveSourceFile(source, cwd);
   const { results } = await loadSharedResults(source, cwd);
-  return { sourceFile, results };
+  const indexRecords = isRunManifestPath(sourceFile)
+    ? readIndexArtifactEntries(sourceFile)
+    : undefined;
+  return { sourceFile, results, indexRecords };
 }
 
-function readIndexArtifactEntries(indexPath: string): IndexArtifactEntry[] {
-  return readFileSync(indexPath, 'utf8')
+function parseIndexArtifactEntries(content: string): IndexArtifactEntry[] {
+  return content
     .split(/\r?\n/)
     .map((line) => line.trim())
     .filter(Boolean)
     .map((line) => JSON.parse(line) as IndexArtifactEntry);
 }
 
+function readIndexArtifactEntries(indexPath: string): IndexArtifactEntry[] {
+  return parseIndexArtifactEntries(readFileSync(indexPath, 'utf8'));
+}
+
+function buildSourceRecordMap(
+  results: readonly EvaluationResult[],
+  sourceRecords: readonly IndexArtifactEntry[],
+): ReadonlyMap<EvaluationResult, IndexArtifactEntry> {
+  return new Map(
+    results.flatMap((result, index) => {
+      const sourceRecord = sourceRecords[index];
+      return sourceRecord ? [[result, sourceRecord] as const] : [];
+    }),
+  );
+}
+
+function isSafeRelativePath(relativePath: string | undefined): relativePath is string {
+  return (
+    typeof relativePath === 'string' &&
+    relativePath.trim().length > 0 &&
+    !path.isAbsolute(relativePath) &&
+    !relativePath.split(/[\\/]+/).includes('..')
+  );
+}
+
+function toRelativeArtifactPath(outputDir: string, filePath: string): string {
+  return path.relative(outputDir, filePath).split(path.sep).join('/');
+}
+
+function hasCopiedSubdir(testBundleDir: string, dirname: string): boolean {
+  return existsSync(path.join(testBundleDir, dirname));
+}
+
+function createExportBundleArtifactsWriter(options: {
+  readonly outputDir: string;
+  readonly sourceBaseDir: string;
+  readonly sourceRecordsByResult: ReadonlyMap<EvaluationResult, IndexArtifactEntry>;
+}): AdditionalResultArtifactsWriter | undefined {
+  if (options.sourceRecordsByResult.size === 0) {
+    return undefined;
+  }
+
+  return async ({ result, testDir }): Promise<AdditionalResultIndexFields | undefined> => {
+    const sourceRecord = options.sourceRecordsByResult.get(result);
+    const sourceBundleDir = sourceRecord?.test_dir ?? sourceRecord?.task_dir;
+    if (!isSafeRelativePath(sourceBundleDir)) {
+      return undefined;
+    }
+
+    const sourceBundlePath = path.join(options.sourceBaseDir, sourceBundleDir);
+    const testBundlePath = path.join(testDir, 'test');
+    if (existsSync(sourceBundlePath)) {
+      cpSync(sourceBundlePath, testBundlePath, { recursive: true, force: true });
+    }
+
+    return {
+      test_dir: toRelativeArtifactPath(options.outputDir, testBundlePath),
+      eval_path: toRelativeArtifactPath(options.outputDir, path.join(testBundlePath, 'EVAL.yaml')),
+      targets_path: toRelativeArtifactPath(
+        options.outputDir,
+        path.join(testBundlePath, 'targets.yaml'),
+      ),
+      ...(sourceRecord?.files_path || hasCopiedSubdir(testBundlePath, 'files')
+        ? {
+            files_path: toRelativeArtifactPath(
+              options.outputDir,
+              path.join(testBundlePath, 'files'),
+            ),
+          }
+        : {}),
+      ...(sourceRecord?.graders_path || hasCopiedSubdir(testBundlePath, 'graders')
+        ? {
+            graders_path: toRelativeArtifactPath(
+              options.outputDir,
+              path.join(testBundlePath, 'graders'),
+            ),
+          }
+        : {}),
+    };
+  };
+}
+
 export function buildProjectionBundleFromExportedIndex(options: {
   readonly sourceFile: string;
   readonly outputDir: string;
@@ -197,7 +298,7 @@ export const resultsExportCommand = command({
     const shouldIncludeRawContent = includeRawContent;
 
     try {
-      const { sourceFile, results } = await loadExportSource(source, cwd);
+      const { sourceFile, results, indexRecords } = await loadExportSource(source, cwd);
 
       const outputDir = out
         ? path.isAbsolute(out)
@@ -212,6 +313,7 @@ export const resultsExportCommand = command({
           cwd,
           duplicatePolicy: policy,
           includeRawContent: shouldIncludeRawContent,
+          indexRecords,
         });
 
       if (shouldDryRun) {
@@ -223,6 +325,11 @@ export const resultsExportCommand = command({
         evalFile: sourceFile,
         runId: deriveExportRunId(sourceFile),
         duplicatePolicy: policy,
+        additionalArtifacts: createExportBundleArtifactsWriter({
+          outputDir,
+          sourceBaseDir: path.dirname(sourceFile),
+          sourceRecordsByResult: buildSourceRecordMap(results, indexRecords ?? []),
+        }),
       });
 
       const bundlePath = shouldWriteProjectionBundle

apps/cli/src/commands/results/manifest.ts

@@ -66,6 +66,7 @@ export interface ResultManifestRecord {
   readonly external_trace?: ExternalTraceMetadataWire;
   readonly response_path?: string;
   readonly result_dir?: string;
+  readonly test_dir?: string;
   readonly task_dir?: string;
   readonly eval_path?: string;
   readonly targets_path?: string;

apps/cli/src/commands/results/projection-bundle.ts

@@ -95,6 +95,7 @@ export type ProjectionBundleArtifactRefs = Partial<
     | 'transcript_path'
     | 'transcript_raw_path'
     | 'metrics_path'
+    | 'test_dir'
     | 'task_dir'
     | 'eval_path'
     | 'targets_path'
@@ -173,6 +174,7 @@ function artifactRefs(
     transcript_path: indexEntry.transcript_path,
     transcript_raw_path: indexEntry.transcript_raw_path,
     metrics_path: indexEntry.metrics_path,
+    test_dir: indexEntry.test_dir,
     task_dir: indexEntry.task_dir,
     eval_path: indexEntry.eval_path,
     targets_path: indexEntry.targets_path,

apps/cli/src/commands/results/serve.ts

@@ -883,6 +883,7 @@ function resultArtifactTreeRootPaths(
 ): string[] {
   return [
     ...catalog.filter((entry) => entry.storage === 'local').map((entry) => entry.displayPath),
+    record.test_dir,
     record.task_dir,
     record.files_path,
     record.graders_path,
@@ -1212,6 +1213,7 @@ function attachRunDetailReadModelFields<T extends Record<string, unknown>>(
       ...(record.aggregation && { aggregation: record.aggregation }),
       ...(record.eval_path && { eval_path: record.eval_path }),
       ...(record.result_dir && { result_dir: record.result_dir }),
+      ...(record.test_dir && { test_dir: record.test_dir }),
       ...(record.summary_path && { summary_path: record.summary_path }),
       ...(record.grading_path && { grading_path: record.grading_path }),
       ...(record.timing_path && { timing_path: record.timing_path }),