From 5daaa6f010b298ff3ff9e585c4aa8d095e5c0884 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 13:04:08 +0200 Subject: [PATCH] feat(artifacts): snapshot environment recipe provenance --- apps/cli/src/commands/eval/artifact-writer.ts | 2 + apps/cli/src/commands/eval/task-bundle.ts | 3 + .../commands/eval/artifact-writer.test.ts | 119 ++++++++ .../docs/next/reference/result-artifacts.mdx | 2 + .../src/evaluation/environment/provenance.ts | 286 ++++++++++++++++++ .../evaluation/loaders/environment-recipe.ts | 58 +++- packages/core/src/evaluation/orchestrator.ts | 42 ++- .../core/src/evaluation/result-row-schema.ts | 1 + packages/core/src/evaluation/run-artifacts.ts | 169 ++++++++++- packages/core/src/evaluation/types.ts | 42 +++ packages/core/src/index.ts | 1 + .../evaluation/environment-provenance.test.ts | 76 +++++ .../loaders/environment-recipe.test.ts | 4 + 13 files changed, 793 insertions(+), 12 deletions(-) create mode 100644 packages/core/src/evaluation/environment/provenance.ts create mode 100644 packages/core/test/evaluation/environment-provenance.test.ts diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index 882d3126a..c4b13301b 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { type AdditionalResultArtifactsWriter, type AggregateGradingArtifact, + type EnvironmentSummaryWire, type EvalTest, type EvaluationResult, type ExperimentArtifactMetadata, @@ -64,6 +65,7 @@ export { export type { AggregateGradingArtifact, GradingArtifact, + EnvironmentSummaryWire, IndexArtifactEntry, ResultIndexArtifact, RunConfigArtifact, diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 2a1473db3..c0a6d50ff 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -812,7 +812,10 @@ function serializeEnvironment( rewrites: ReadonlyMap, ): Record { const { + authoredReference: _authoredReference, recipeFilePath: _recipeFilePath, + recipeFileSha256: _recipeFileSha256, + recipeSha256: _recipeSha256, sourceDir: _sourceDir, ...portableEnvironment } = environment; diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 17dc1c014..abd26d56d 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -1316,6 +1316,125 @@ describe('writeArtifactsFromResults', () => { expect(indexLine.runtime_source).toBeUndefined(); }); + it('writes host environment provenance as a sidecar with redacted setup inputs and logs', async () => { + const result = makeResult({ + testId: 'host-env', + environmentProvenance: { + schemaVersion: 'agentv.environment_provenance.v1', + authoredKind: 'file', + authoredReference: 'file://.agentv/environments/host.yaml', + recipeFilePath: '/repo/.agentv/environments/host.yaml', + recipeFileSha256: 'f'.repeat(64), + recipeSha256: 'a'.repeat(64), + type: 'host', + sourceDir: '/repo/.agentv/environments', + workdir: '/repo/workspaces/app', + setup: { + command: ['node', 'setup.mjs', '--api-key', ''], + args: { + repo: 'example/app', + commit: 'abc123', + api_key: '', + }, + }, + setupExecutions: [ + { + scope: 'environment', + name: 'setup', + status: 'success', + testId: '__environment_setup__', + workdir: '/repo/workspaces/app', + command: ['node', 'setup.mjs', '--api-key', ''], + cwd: '/repo/.agentv/environments', + output: + '{"repo_provenance":{"repo":"example/app","commit":"abc123"}}\\nused ', + exitCode: 0, + }, + ], + repoProvenance: { repo: 'example/app', commit: 'abc123' }, + }, + }); + + const paths = await writeArtifactsFromResults([result], testDir, { + evalFile: 'evals/host.eval.yaml', + }); + const [indexLine] = await readIndexLines(paths.indexPath); + const rowDir = expectRowDir(indexLine, 'host-env'); + const environmentPath = path.join(testDir, indexLine.environment_path ?? ''); + const environment = JSON.parse(await readFile(environmentPath, 'utf8')); + const resultJson = JSON.parse( + await readFile(path.join(testDir, rowDir, 'sample-1', 'result.json'), 'utf8'), + ); + const summary: RunSummaryArtifact = JSON.parse(await readFile(paths.summaryPath, 'utf8')); + + expect(indexLine.environment).toMatchObject({ + schema_version: 'agentv.environment_summary.v1', + type: 'host', + workdir: '/repo/workspaces/app', + recipe_sha256: 'a'.repeat(64), + authored_reference: 'file://.agentv/environments/host.yaml', + setup_status: 'success', + }); + expect(indexLine.environment_path).toBe(`${rowDir}/sample-1/environment.json`); + expect(indexLine.environment).not.toHaveProperty('setup_executions'); + expect(JSON.stringify(indexLine)).not.toContain('used '); + expect(environment.setup.command).toEqual(['node', 'setup.mjs', '--api-key', '']); + expect(environment.setup.args.api_key).toBe(''); + expect(environment.setup_executions[0].output).toContain('used '); + expect(environment.repo_provenance).toEqual({ repo: 'example/app', commit: 'abc123' }); + expect(resultJson.environment_path).toBe('./environment.json'); + expect(summary.metadata.environments?.[0]).toMatchObject({ + type: 'host', + recipe_sha256: 'a'.repeat(64), + }); + }); + + it('writes Docker environment provenance without setup logs in index rows', async () => { + const result = makeResult({ + testId: 'docker-env', + environmentProvenance: { + schemaVersion: 'agentv.environment_provenance.v1', + authoredKind: 'inline', + recipeSha256: 'b'.repeat(64), + type: 'docker', + sourceDir: '/repo/evals', + workdir: '/app', + docker: { + context: '/repo/environment', + dockerfile: '/repo/environment/Dockerfile', + image: 'ghcr.io/example/app@sha256:1234567890abcdef', + imageDigest: 'sha256:1234567890abcdef', + }, + }, + }); + + const paths = await writeArtifactsFromResults([result], testDir, { + evalFile: 'evals/docker.eval.yaml', + }); + const [indexLine] = await readIndexLines(paths.indexPath); + const environment = JSON.parse( + await readFile(path.join(testDir, indexLine.environment_path ?? ''), 'utf8'), + ); + + expect(indexLine.environment).toMatchObject({ + type: 'docker', + workdir: '/app', + docker: { + context: '/repo/environment', + dockerfile: '/repo/environment/Dockerfile', + image: 'ghcr.io/example/app@sha256:1234567890abcdef', + image_digest: 'sha256:1234567890abcdef', + }, + }); + expect(indexLine.environment).not.toHaveProperty('setup_executions'); + expect(environment).toMatchObject({ + type: 'docker', + docker: { + image_digest: 'sha256:1234567890abcdef', + }, + }); + }); + it('does not write experiment config metadata into public run artifacts', async () => { const experimentMetadata = { name: 'native-exp', diff --git a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx index 58d84f92a..0b7497d09 100644 --- a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx @@ -45,6 +45,7 @@ The default local layout is: graders/ sample-1/ result.json + environment.json # optional environment recipe provenance grading.json metrics.json target-execution.json # optional target runtime envelope @@ -97,6 +98,7 @@ reserved for rebuildable local state and are skipped by run discovery. | `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | | `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. | | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. | +| `environment.json` / `environment_path` | Redacted environment recipe provenance: authored inline/file reference, resolved recipe hash, host or Docker type, resolved workdir, setup command and typed args, setup log output/error, Docker context/image/digest fields when available, and repo provenance only when authored or emitted by setup. Index rows carry `environment_path` plus a compact `environment` summary; large setup logs stay in the sidecar. | Reproducing and reviewing the testbed without treating setup side effects as row metadata. Repository identity is opaque unless the environment recipe or setup output states it explicitly. | | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. | | `metrics.json` | Duration, token usage, cost, execution status, trajectory, and derived executor behavior such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, cost/latency reporting, metric-style graders, adapter projections, and lightweight analysis. | | `target-execution.json` | Provider-neutral target runtime envelope, including command, cwd, timeout, exit code or signal, error kind, timestamps, log truncation metadata, and artifact paths. | Distinguishing target task failures, target crashes, timeouts, cancellation, malformed provider output, and sandbox/runner failures from AgentV orchestrator failures. | diff --git a/packages/core/src/evaluation/environment/provenance.ts b/packages/core/src/evaluation/environment/provenance.ts new file mode 100644 index 000000000..3bb217027 --- /dev/null +++ b/packages/core/src/evaluation/environment/provenance.ts @@ -0,0 +1,286 @@ +import { createHash } from 'node:crypto'; + +import type { + DockerEnvironmentRecipe, + EnvironmentRecipe, + EnvironmentSetupConfig, +} from '../loaders/environment-recipe.js'; +import type { + EnvironmentRecipeProvenance, + EnvironmentSetupProvenance, + JsonObject, + JsonValue, +} from '../types.js'; +import type { EnvironmentSetupExecution } from '../workspace/setup.js'; + +const SECRET_KEY_PATTERN = + /(api[_-]?key|auth|credential|password|passwd|private[_-]?key|secret|token)/i; + +export function buildEnvironmentRecipeProvenance(params: { + readonly environment: EnvironmentRecipe | undefined; + readonly setupExecutions?: readonly EnvironmentSetupExecution[]; +}): EnvironmentRecipeProvenance | undefined { + const environment = params.environment; + if (!environment) { + return undefined; + } + const secretValues = collectSecretValues(environment); + const setupExecutions = params.setupExecutions + ?.filter((execution) => execution.workdir === environment.workdir) + .map((execution) => redactSetupExecution(execution, secretValues)); + const repoProvenance = setupExecutions ? extractRepoProvenance(setupExecutions) : undefined; + return { + schemaVersion: 'agentv.environment_provenance.v1', + authoredKind: environment.authoredReference ? 'file' : 'inline', + ...(environment.authoredReference ? { authoredReference: environment.authoredReference } : {}), + ...(environment.recipeFilePath ? { recipeFilePath: environment.recipeFilePath } : {}), + ...(environment.recipeFileSha256 ? { recipeFileSha256: environment.recipeFileSha256 } : {}), + recipeSha256: environment.recipeSha256 ?? fallbackRecipeSha256(environment), + type: environment.type, + sourceDir: environment.sourceDir, + workdir: environment.workdir, + ...(environment.setup ? { setup: redactSetupConfig(environment.setup, secretValues) } : {}), + ...(setupExecutions && setupExecutions.length > 0 ? { setupExecutions } : {}), + ...(environment.type === 'docker' ? { docker: dockerProvenance(environment) } : {}), + ...(repoProvenance !== undefined ? { repoProvenance } : {}), + }; +} + +function dockerProvenance( + environment: DockerEnvironmentRecipe, +): EnvironmentRecipeProvenance['docker'] { + const imageDigest = environment.image?.match(/@([^@\s]+)$/)?.[1]; + return { + ...(environment.context ? { context: environment.context } : {}), + ...(environment.dockerfile ? { dockerfile: environment.dockerfile } : {}), + ...(environment.image ? { image: environment.image } : {}), + ...(imageDigest ? { imageDigest } : {}), + }; +} + +function fallbackRecipeSha256(environment: EnvironmentRecipe): string { + return createHash('sha256') + .update( + stableJson({ + type: environment.type, + workdir: environment.workdir, + sourceDir: environment.sourceDir, + setup: environment.setup, + ...(environment.type === 'docker' + ? { + context: environment.context, + dockerfile: environment.dockerfile, + image: environment.image, + } + : {}), + }), + ) + .digest('hex'); +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableJson).join(',')}]`; + } + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record).sort(([a], [b]) => + a.localeCompare(b), + ); + return `{${entries + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function redactSetupConfig( + setup: EnvironmentSetupConfig, + secretValues: readonly string[], +): EnvironmentRecipeProvenance['setup'] { + return { + command: redactCommand(setup.command, secretValues), + ...(setup.args ? { args: redactJsonObject(setup.args, secretValues) } : {}), + ...(setup.env ? { env: redactStringRecord(setup.env, secretValues) } : {}), + ...(setup.timeout_seconds !== undefined ? { timeoutSeconds: setup.timeout_seconds } : {}), + }; +} + +function redactSetupExecution( + execution: EnvironmentSetupExecution, + secretValues: readonly string[], +): EnvironmentSetupProvenance { + return { + scope: execution.scope, + name: execution.name, + status: execution.status, + testId: execution.testId, + workdir: execution.workdir, + ...(execution.command !== undefined + ? { command: redactCommand(execution.command, secretValues) } + : {}), + ...(execution.cwd !== undefined ? { cwd: execution.cwd } : {}), + ...(execution.output !== undefined + ? { output: redactString(execution.output, secretValues) } + : {}), + ...(execution.error !== undefined + ? { error: redactString(execution.error, secretValues) } + : {}), + ...(execution.exitCode !== undefined ? { exitCode: execution.exitCode } : {}), + }; +} + +function redactCommand( + command: EnvironmentSetupConfig['command'], + secretValues: readonly string[], +): EnvironmentSetupConfig['command'] { + if (typeof command === 'string') { + return redactSecretAssignments(redactString(command, secretValues)); + } + return command.map((part, index) => { + const previous = index > 0 ? command[index - 1] : undefined; + if (previous && isSecretKey(previous.replace(/^-+/, ''))) { + return ''; + } + return redactSecretAssignments(redactString(part, secretValues)); + }); +} + +function redactJsonObject(value: JsonObject, secretValues: readonly string[]): JsonObject { + return redactJsonValue(value, secretValues) as JsonObject; +} + +function redactJsonValue( + value: JsonValue, + secretValues: readonly string[], + key?: string, +): JsonValue { + if (key && isSecretKey(key)) { + return ''; + } + if (typeof value === 'string') { + return redactString(value, secretValues); + } + if (Array.isArray(value)) { + return value.map((entry) => redactJsonValue(entry, secretValues)); + } + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([entryKey, entryValue]) => [ + entryKey, + redactJsonValue(entryValue, secretValues, entryKey), + ]), + ); + } + return value; +} + +function redactStringRecord( + value: Readonly>, + secretValues: readonly string[], +): Readonly> { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + isSecretKey(key) ? '' : redactString(entry, secretValues), + ]), + ); +} + +function collectSecretValues(environment: EnvironmentRecipe): string[] { + const values: string[] = []; + collectSecretValuesFromRecord(environment.env, values); + collectSecretValuesFromRecord(environment.setup?.env, values); + collectSecretValuesFromJson(environment.setup?.args, values); + if (environment.type === 'docker') { + collectSecretValuesFromRecord(environment.secrets, values); + } + return [...new Set(values.filter((value) => value.length >= 4))]; +} + +function collectSecretValuesFromRecord( + record: Readonly> | undefined, + values: string[], +): void { + for (const [key, value] of Object.entries(record ?? {})) { + if (isSecretKey(key)) { + values.push(value); + } + } +} + +function collectSecretValuesFromJson( + value: JsonValue | undefined, + values: string[], + key?: string, +): void { + if (value === undefined) { + return; + } + if (key && isSecretKey(key) && typeof value === 'string') { + values.push(value); + return; + } + if (Array.isArray(value)) { + for (const entry of value) { + collectSecretValuesFromJson(entry, values); + } + return; + } + if (value && typeof value === 'object') { + for (const [entryKey, entryValue] of Object.entries(value)) { + collectSecretValuesFromJson(entryValue, values, entryKey); + } + } +} + +function isSecretKey(key: string): boolean { + return SECRET_KEY_PATTERN.test(key); +} + +function redactString(value: string, secretValues: readonly string[]): string { + return secretValues.reduce((text, secret) => text.split(secret).join(''), value); +} + +function redactSecretAssignments(value: string): string { + return value.replace( + /((?:api[_-]?key|auth|credential|password|passwd|private[_-]?key|secret|token)[A-Z0-9_-]*=)([^\s]+)/gi, + '$1', + ); +} + +function extractRepoProvenance( + setupExecutions: readonly EnvironmentSetupProvenance[], +): JsonValue | undefined { + for (const execution of setupExecutions) { + const candidates = [execution.output, execution.error].filter( + (entry): entry is string => !!entry, + ); + for (const candidate of candidates) { + const parsed = parseJsonCandidate(candidate); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + const record = parsed as Record; + if (record.repo_provenance !== undefined) { + return record.repo_provenance; + } + if (record.repoProvenance !== undefined) { + return record.repoProvenance; + } + } + } + } + return undefined; +} + +function parseJsonCandidate(value: string): JsonValue | undefined { + const trimmed = value.trim(); + const candidates = [trimmed, ...trimmed.split(/\r?\n/).map((line) => line.trim())].filter( + (line) => line.startsWith('{') && line.endsWith('}'), + ); + for (const candidate of candidates) { + try { + return JSON.parse(candidate) as JsonValue; + } catch {} + } + return undefined; +} diff --git a/packages/core/src/evaluation/loaders/environment-recipe.ts b/packages/core/src/evaluation/loaders/environment-recipe.ts index 751e035bc..2f7fd8a23 100644 --- a/packages/core/src/evaluation/loaders/environment-recipe.ts +++ b/packages/core/src/evaluation/loaders/environment-recipe.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { readFile } from 'node:fs/promises'; import path from 'node:path'; @@ -16,7 +17,10 @@ export type EnvironmentSetupConfig = { }; type EnvironmentRecipeSource = { + readonly authoredReference?: string; readonly recipeFilePath?: string; + readonly recipeFileSha256?: string; + readonly recipeSha256?: string; readonly sourceDir: string; }; @@ -73,8 +77,10 @@ export async function resolveEnvironmentRecipe( } const recipePath = resolveReferencePath(raw, evalFileDir); let parsed: unknown; + let recipeText: string; try { - parsed = interpolateEnv(parseYamlValue(await readFile(recipePath, 'utf8')), process.env); + recipeText = await readFile(recipePath, 'utf8'); + parsed = interpolateEnv(parseYamlValue(recipeText), process.env); } catch (error) { throw new Error( `${location} recipe file not found or unreadable: ${raw} (${(error as Error).message})`, @@ -85,10 +91,17 @@ export async function resolveEnvironmentRecipe( `${location} recipe file ${recipePath} must contain the environment recipe directly, not an object wrapped in 'environment'.`, ); } - return parseEnvironmentRecipe(parsed, path.dirname(recipePath), location, recipePath); + return parseEnvironmentRecipe(parsed, path.dirname(recipePath), location, { + authoredReference: raw, + recipeFilePath: recipePath, + recipeFileSha256: sha256(recipeText), + recipeSha256: sha256(stableJson(parsed)), + }); } - return parseEnvironmentRecipe(raw, evalFileDir, location); + return parseEnvironmentRecipe(raw, evalFileDir, location, { + recipeSha256: sha256(stableJson(raw)), + }); } function resolveReferencePath(reference: string, evalFileDir: string): string { @@ -100,7 +113,12 @@ function parseEnvironmentRecipe( raw: unknown, baseDir: string, location: string, - recipeFilePath?: string, + source: { + readonly authoredReference?: string; + readonly recipeFilePath?: string; + readonly recipeFileSha256?: string; + readonly recipeSha256: string; + }, ): EnvironmentRecipe { if (!isJsonObject(raw)) { throw new Error(`${location} must be an object with type: host|docker and workdir.`); @@ -120,9 +138,14 @@ function parseEnvironmentRecipe( type, workdir: resolveHostPath(workdir, baseDir), sourceDir: baseDir, + recipeSha256: source.recipeSha256, ...(setup !== undefined && { setup }), ...(env !== undefined && { env }), - ...(recipeFilePath !== undefined && { recipeFilePath }), + ...(source.authoredReference !== undefined && { + authoredReference: source.authoredReference, + }), + ...(source.recipeFilePath !== undefined && { recipeFilePath: source.recipeFilePath }), + ...(source.recipeFileSha256 !== undefined && { recipeFileSha256: source.recipeFileSha256 }), }; } @@ -149,6 +172,7 @@ function parseEnvironmentRecipe( type, workdir, sourceDir: baseDir, + recipeSha256: source.recipeSha256, ...(context !== undefined && { context: resolveHostPath(context, baseDir) }), ...(dockerfile !== undefined && { dockerfile: resolveHostPath(dockerfile, baseDir) }), ...(image !== undefined && { image }), @@ -157,10 +181,32 @@ function parseEnvironmentRecipe( ...(parseMounts(raw.mounts, `${location}.mounts`, baseDir) ?? {}), ...(secrets !== undefined && { secrets }), ...(setup !== undefined && { setup }), - ...(recipeFilePath !== undefined && { recipeFilePath }), + ...(source.authoredReference !== undefined && { authoredReference: source.authoredReference }), + ...(source.recipeFilePath !== undefined && { recipeFilePath: source.recipeFilePath }), + ...(source.recipeFileSha256 !== undefined && { recipeFileSha256: source.recipeFileSha256 }), }; } +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableJson).join(',')}]`; + } + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record).sort(([a], [b]) => + a.localeCompare(b), + ); + return `{${entries + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + function resolveHostPath(value: string, baseDir: string): string { return path.isAbsolute(value) ? value : path.resolve(baseDir, value); } diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 7f9a9be27..5e367f16a 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import micromatch from 'micromatch'; import pLimit from 'p-limit'; +import { buildEnvironmentRecipeProvenance } from './environment/provenance.js'; import { runExtensionsForHook } from './extensions/runner.js'; import { readJsonFile } from './file-utils.js'; import { @@ -92,6 +93,7 @@ import { executeWorkspaceScript, } from './workspace/script-executor.js'; import { + type EnvironmentSetupExecution, type EvalCaseWorkspaceSetup, WorkspaceSetupError, captureWorkspaceFileChanges, @@ -449,6 +451,8 @@ export interface RunEvalCaseOptions { readonly sharedBaselineCommit?: string; /** Provider/runtime context produced by shared beforeAll extensions. */ readonly sharedExtensionState?: import('./extensions/runner.js').ExtensionRuntimeState; + /** Environment setup executions inherited from shared suite setup. */ + readonly sharedEnvironmentSetupExecutions?: readonly EnvironmentSetupExecution[]; /** Suite-level .code-workspace file (resolved from workspace.template) */ readonly suiteWorkspaceFile?: string; /** Real-time observability callbacks passed to the provider */ @@ -774,7 +778,7 @@ export async function gradePreparedEvalCase( }; } catch (error) { const evalRun = { durationMs: Date.now() - caseStartMs }; - const errorResult = buildErrorResult( + const errorResultBase = buildErrorResult( evalCase, target.name, nowFn(), @@ -786,7 +790,7 @@ export async function gradePreparedEvalCase( verbose, ); return { - ...errorResult, + ...errorResultBase, evalRun, fileChanges, workspacePath, @@ -998,6 +1002,7 @@ export async function runEvaluation( beforeAllOutput, repoManager, useStaticWorkspace, + environmentSetupExecutions: sharedEnvironmentSetupExecutions, extensionState: sharedExtensionState, } = sharedSetup; const targetHooks = options.targetHooks; @@ -1252,6 +1257,9 @@ export async function runEvaluation( const testWorkspacePath = usesSharedWorkspace ? sharedWorkspacePath : undefined; const testBaselineCommit = usesSharedWorkspace ? sharedBaselineCommit : undefined; const testExtensionState = usesSharedWorkspace ? sharedExtensionState : undefined; + const testEnvironmentSetupExecutions = usesSharedWorkspace + ? sharedEnvironmentSetupExecutions + : undefined; try { const graderProvider = await resolveGraderProvider(target); @@ -1285,6 +1293,7 @@ export async function runEvaluation( threshold: scoreThreshold, targetHooks: options.targetHooks, sharedExtensionState: testExtensionState, + sharedEnvironmentSetupExecutions: testEnvironmentSetupExecutions, replayRecording, evalFilePath, repoRoot: repoRootPath, @@ -1898,7 +1907,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise {}); const evalRun = { durationMs: Date.now() - caseStartMs }; - const errorResult = buildErrorResult( + const errorResultBase = buildErrorResult( evalCase, target.name, nowFn(), @@ -2415,6 +2444,9 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise; @@ -121,6 +123,26 @@ export interface RunRuntimeSourceMetadata { readonly wrapper_eval_file?: string; } +export interface EnvironmentSummaryWire { + readonly schema_version: 'agentv.environment_summary.v1'; + readonly type: EnvironmentRecipeProvenance['type']; + readonly workdir: string; + readonly recipe_sha256: string; + readonly authored_kind: EnvironmentRecipeProvenance['authoredKind']; + readonly authored_reference?: string; + readonly recipe_file_path?: string; + readonly recipe_file_sha256?: string; + readonly setup_status?: string; + readonly environment_path?: string; + readonly docker?: { + readonly context?: string; + readonly dockerfile?: string; + readonly image?: string; + readonly image_digest?: string; + readonly build_id?: string; + }; +} + export function buildTestTargetKey(testId?: string, target?: string, variant?: string): string { return `${testId ?? 'unknown'}::${target ?? 'unknown'}::${variant ?? ''}`; } @@ -219,8 +241,15 @@ export async function aggregateRunDir( runtimeSource, tags, ); + const summaryToWrite = + previousMetadata.environments && !summary.metadata.environments + ? { + ...summary, + metadata: { ...summary.metadata, environments: previousMetadata.environments }, + } + : summary; const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME); - await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); + await writeFile(summaryPath, `${JSON.stringify(summaryToWrite, null, 2)}\n`, 'utf8'); const targetSet = new Set(results.map((r) => r.target ?? 'unknown')); return { summaryPath, testCount: results.length, targetCount: targetSet.size }; @@ -280,6 +309,7 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ plannedTestCount?: number; runtimeSource?: RunRuntimeSourceMetadata; tags?: Record; + environments?: readonly EnvironmentSummaryWire[]; }> { try { const raw = await readFile(summaryPath, 'utf8'); @@ -288,6 +318,7 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ planned_test_count?: number; runtime_source?: RunRuntimeSourceMetadata; tags?: unknown; + environments?: unknown; }; }; const value = parsed.metadata?.planned_test_count; @@ -297,16 +328,39 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ ? parsed.metadata.runtime_source : undefined; const tags = normalizeStringRecord(parsed.metadata?.tags); + const environments = normalizeEnvironmentSummaries(parsed.metadata?.environments); return { ...(plannedTestCount !== undefined && { plannedTestCount }), ...(runtimeSource !== undefined && { runtimeSource }), ...(tags !== undefined && { tags }), + ...(environments !== undefined && { environments }), }; } catch { return {}; } } +function normalizeEnvironmentSummaries( + value: unknown, +): readonly EnvironmentSummaryWire[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const summaries = value.filter((entry): entry is EnvironmentSummaryWire => { + if (!isRecord(entry)) { + return false; + } + return ( + entry.schema_version === 'agentv.environment_summary.v1' && + (entry.type === 'host' || entry.type === 'docker') && + typeof entry.workdir === 'string' && + typeof entry.recipe_sha256 === 'string' && + (entry.authored_kind === 'inline' || entry.authored_kind === 'file') + ); + }); + return summaries.length > 0 ? summaries : undefined; +} + /** * Coerce an unknown value into a `Record`, dropping non-string * entries. Returns undefined when the value is not a plain object or has no @@ -494,6 +548,7 @@ export interface RunSummaryArtifact { * experiment namespace. Absent when no map-form tags were resolved. */ readonly tags?: Record; + readonly environments?: readonly EnvironmentSummaryWire[]; }; readonly run_summary: Record< string, @@ -566,6 +621,8 @@ export interface IndexArtifactEntry { readonly transcript_summary?: TranscriptSummaryWire; readonly metrics_path?: string; readonly file_changes_path?: string; + readonly environment_path?: string; + readonly environment?: EnvironmentSummaryWire; readonly artifact_pointers?: ResultArtifactPointersWire; readonly sample_index?: number; readonly retry_index?: number; @@ -624,6 +681,8 @@ export interface AgentVRunResultArtifact { readonly model: string; readonly grading_path: string; readonly metrics_path: string; + readonly environment_path?: string; + readonly environment?: EnvironmentSummaryWire; readonly file_changes_path?: string; readonly transcript_path?: string; readonly transcript_raw_path?: string; @@ -1187,6 +1246,80 @@ function buildResultTranscriptSummary(result: EvaluationResult): TranscriptSumma }); } +function environmentSummary( + provenance: EnvironmentRecipeProvenance | undefined, + environmentPath?: string, +): EnvironmentSummaryWire | undefined { + if (!provenance) { + return undefined; + } + const lastSetup = provenance.setupExecutions?.at(-1); + return dropUndefined({ + schema_version: 'agentv.environment_summary.v1', + type: provenance.type, + workdir: provenance.workdir, + recipe_sha256: provenance.recipeSha256, + authored_kind: provenance.authoredKind, + authored_reference: provenance.authoredReference, + recipe_file_path: provenance.recipeFilePath, + recipe_file_sha256: provenance.recipeFileSha256, + setup_status: lastSetup?.status, + environment_path: environmentPath, + docker: provenance.docker + ? dropUndefined({ + context: provenance.docker.context, + dockerfile: provenance.docker.dockerfile, + image: provenance.docker.image, + image_digest: provenance.docker.imageDigest, + build_id: provenance.docker.buildId, + }) + : undefined, + }) as unknown as EnvironmentSummaryWire; +} + +function runEnvironmentSummaries( + results: readonly EvaluationResult[], +): readonly EnvironmentSummaryWire[] | undefined { + const summaries = new Map(); + for (const result of results) { + for (const trial of materializedRunTrials(result)) { + const summary = environmentSummary(trial.result?.environmentProvenance); + if (summary) { + summaries.set( + JSON.stringify({ + type: summary.type, + workdir: summary.workdir, + recipe_sha256: summary.recipe_sha256, + authored_reference: summary.authored_reference, + }), + summary, + ); + } + } + } + return summaries.size > 0 ? [...summaries.values()] : undefined; +} + +async function writeEnvironmentArtifact(params: { + readonly result: EvaluationResult; + readonly sampleDir: string; +}): Promise<{ environmentPath?: string; environment?: EnvironmentSummaryWire }> { + const provenance = params.result.environmentProvenance; + if (!provenance) { + return {}; + } + const environmentPath = path.join(params.sampleDir, ENVIRONMENT_ARTIFACT_PATH); + await writeFile( + environmentPath, + `${JSON.stringify(toSnakeCaseDeep(provenance), null, 2)}\n`, + 'utf8', + ); + return { + environmentPath, + environment: environmentSummary(provenance, `./${ENVIRONMENT_ARTIFACT_PATH}`), + }; +} + function buildAgentVRunResultArtifact(params: { readonly trial: TrialResult; readonly result: EvaluationResult; @@ -1202,6 +1335,8 @@ function buildAgentVRunResultArtifact(params: { readonly targetExecutionPath?: string; readonly stdoutPath?: string; readonly stderrPath?: string; + readonly environmentPath?: string; + readonly environment?: EnvironmentSummaryWire; }): AgentVRunResultArtifact { const metrics = params.metricsArtifact.metrics; const fileChangesPath = params.hasFileChanges @@ -1223,6 +1358,8 @@ function buildAgentVRunResultArtifact(params: { model: params.result.target ?? 'unknown', grading_path: './grading.json', metrics_path: `./${CANONICAL_METRICS_ARTIFACT_PATH}`, + environment_path: params.environmentPath ? `./${params.environmentPath}` : undefined, + environment: params.environment, file_changes_path: fileChangesPath, transcript_path: params.hasTranscript ? `./${CANONICAL_TRANSCRIPT_ARTIFACT_PATH}` : undefined, transcript_raw_path: params.hasTranscript ? './transcript-raw.jsonl' : undefined, @@ -1414,6 +1551,10 @@ async function writeTrialRunArtifacts(params: { fileChangesArtifactPath: fileChangesPath ? CANONICAL_FILE_CHANGES_ARTIFACT_PATH : undefined, timing, }); + const environmentArtifact = await writeEnvironmentArtifact({ + result, + sampleDir: runDir, + }); await writeFile( path.join(runDir, 'result.json'), @@ -1431,6 +1572,10 @@ async function writeTrialRunArtifacts(params: { : undefined, stdoutPath: targetExecutionArtifacts.stdoutPath ? TARGET_STDOUT_ARTIFACT_PATH : undefined, stderrPath: targetExecutionArtifacts.stderrPath ? TARGET_STDERR_ARTIFACT_PATH : undefined, + environmentPath: environmentArtifact.environmentPath + ? ENVIRONMENT_ARTIFACT_PATH + : undefined, + environment: environmentArtifact.environment, }), null, 2, @@ -1943,6 +2088,7 @@ export function buildRunSummaryArtifact( runtime_source: runtimeSource, planned_test_count: plannedTestCount, tags: tags && Object.keys(tags).length > 0 ? tags : undefined, + environments: runEnvironmentSummaries(results), }, run_summary: runSummary, per_grader_summary: perEvaluatorSummary, @@ -2252,6 +2398,7 @@ export function buildIndexArtifactEntry( targetExecutionPath?: string; stdoutPath?: string; stderrPath?: string; + environmentPath?: string; artifactPointers?: ResultArtifactPointersWire; rawProviderLogPath?: string; extraIndexFields?: AdditionalResultIndexFields; @@ -2356,6 +2503,15 @@ export function buildIndexArtifactEntry( file_changes_path: options.fileChangesPath ? toRelativeArtifactPath(options.outputDir, options.fileChangesPath) : undefined, + environment_path: options.environmentPath + ? toRelativeArtifactPath(options.outputDir, options.environmentPath) + : undefined, + environment: environmentSummary( + result.environmentProvenance, + options.environmentPath + ? toRelativeArtifactPath(options.outputDir, options.environmentPath) + : undefined, + ), raw_provider_log_path: options.rawProviderLogPath ? toRelativeArtifactPath(options.outputDir, options.rawProviderLogPath) : undefined, @@ -3180,6 +3336,10 @@ export async function writePerTestArtifacts( isSingleRun && result.targetExecution ? path.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH) : undefined; + const singleEnvironmentPath = + isSingleRun && result.environmentProvenance + ? path.join(singleRunDir, ENVIRONMENT_ARTIFACT_PATH) + : undefined; const extraIndexFields = await collectAdditionalIndexFields( result, @@ -3204,6 +3364,7 @@ export async function writePerTestArtifacts( targetExecutionPath: singleTargetExecutionPath, stdoutPath: singleStdoutPath, stderrPath: singleStderrPath, + environmentPath: singleEnvironmentPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity, @@ -3309,6 +3470,10 @@ export async function writeArtifactsFromResults( isSingleRun && result.targetExecution ? path.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH) : undefined; + const singleEnvironmentPath = + isSingleRun && result.environmentProvenance + ? path.join(singleRunDir, ENVIRONMENT_ARTIFACT_PATH) + : undefined; return { result, testDir, @@ -3324,6 +3489,7 @@ export async function writeArtifactsFromResults( singleTargetExecutionPath, singleStdoutPath, singleStderrPath, + singleEnvironmentPath, identityId, }; }); @@ -3401,6 +3567,7 @@ export async function writeArtifactsFromResults( targetExecutionPath: plan.singleTargetExecutionPath, stdoutPath: plan.singleStdoutPath, stderrPath: plan.singleStderrPath, + environmentPath: plan.singleEnvironmentPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity: plan.projectionIdentity, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 65593ad0c..f2beded79 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1224,6 +1224,46 @@ export interface EvalRunOverride { readonly budgetUsd?: number; } +export interface EnvironmentSetupProvenance { + readonly scope: 'environment'; + readonly name: 'setup'; + readonly status: 'skipped' | 'success' | 'failed'; + readonly testId: string; + readonly workdir: string; + readonly command?: string | readonly string[]; + readonly cwd?: string; + readonly output?: string; + readonly error?: string; + readonly exitCode?: number; +} + +export interface EnvironmentRecipeProvenance { + readonly schemaVersion: 'agentv.environment_provenance.v1'; + readonly authoredKind: 'inline' | 'file'; + readonly authoredReference?: string; + readonly recipeFilePath?: string; + readonly recipeFileSha256?: string; + readonly recipeSha256: string; + readonly type: EnvironmentRecipe['type']; + readonly sourceDir: string; + readonly workdir: string; + readonly setup?: { + readonly command: string | readonly string[]; + readonly args?: JsonObject; + readonly env?: Readonly>; + readonly timeoutSeconds?: number; + }; + readonly setupExecutions?: readonly EnvironmentSetupProvenance[]; + readonly docker?: { + readonly context?: string; + readonly dockerfile?: string; + readonly image?: string; + readonly imageDigest?: string; + readonly buildId?: string; + }; + readonly repoProvenance?: JsonValue; +} + /** * Primary classification of evaluation outcome. * - 'ok': evaluation completed, score reflects model quality (score >= 0.8) @@ -1322,6 +1362,8 @@ export interface EvaluationResult { readonly afterEachOutput?: string; /** Unified diff of workspace file changes */ readonly fileChanges?: string; + /** Redacted environment recipe and setup provenance for run-bundle artifacts. */ + readonly environmentProvenance?: EnvironmentRecipeProvenance; /** Individual attempt results (only present when evaluate_options.repeat.count > 1) */ readonly trials?: readonly TrialResult[]; /** Aggregation metadata describing how the final score was computed from attempts */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2f4035ecf..cdaa11bc9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -87,6 +87,7 @@ export { type AggregateGradingArtifact, type GradingArtifact, type IndexArtifactEntry, + type EnvironmentSummaryWire, type ResultIndexArtifact, type RunRuntimeConfigSource, type RunRuntimeSourceMetadata, diff --git a/packages/core/test/evaluation/environment-provenance.test.ts b/packages/core/test/evaluation/environment-provenance.test.ts new file mode 100644 index 000000000..8d2e390a1 --- /dev/null +++ b/packages/core/test/evaluation/environment-provenance.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'bun:test'; + +import { buildEnvironmentRecipeProvenance } from '../../src/evaluation/environment/provenance.js'; + +describe('environment recipe provenance', () => { + it('redacts setup command, args, env, and logs while preserving emitted repo provenance', () => { + const provenance = buildEnvironmentRecipeProvenance({ + environment: { + type: 'host', + workdir: '/workspaces/app', + sourceDir: '/repo/.agentv/environments', + setup: { + command: ['node', 'setup.mjs', '--api-key', 'sk-live-secret'], + args: { + repo: 'example/app', + api_key: 'sk-live-secret', + nested: { token: 'nested-secret' }, + }, + env: { + SETUP_MODE: 'test', + GITHUB_TOKEN: 'github-secret', + }, + }, + }, + setupExecutions: [ + { + scope: 'environment', + name: 'setup', + status: 'success', + testId: 'case-1', + workdir: '/workspaces/app', + command: ['node', 'setup.mjs', '--api-key', 'sk-live-secret'], + output: + '{"repo_provenance":{"repo":"example/app","commit":"abc123"}}\nused sk-live-secret and github-secret', + exitCode: 0, + }, + ], + }); + + expect(provenance?.setup?.command).toEqual(['node', 'setup.mjs', '--api-key', '']); + expect(provenance?.setup?.args).toEqual({ + repo: 'example/app', + api_key: '', + nested: { token: '' }, + }); + expect(provenance?.setup?.env).toEqual({ + SETUP_MODE: 'test', + GITHUB_TOKEN: '', + }); + expect(provenance?.setupExecutions?.[0]?.output).toContain('used '); + expect(provenance?.setupExecutions?.[0]?.output).not.toContain('sk-live-secret'); + expect(provenance?.setupExecutions?.[0]?.output).not.toContain('github-secret'); + expect(provenance?.repoProvenance).toEqual({ repo: 'example/app', commit: 'abc123' }); + }); + + it('captures Docker authored context and image digest when runtime build details are absent', () => { + const provenance = buildEnvironmentRecipeProvenance({ + environment: { + type: 'docker', + workdir: '/app', + sourceDir: '/repo/evals', + context: '/repo/environment', + dockerfile: '/repo/environment/Dockerfile', + image: 'ghcr.io/example/app@sha256:1234567890abcdef', + }, + }); + + expect(provenance?.docker).toEqual({ + context: '/repo/environment', + dockerfile: '/repo/environment/Dockerfile', + image: 'ghcr.io/example/app@sha256:1234567890abcdef', + imageDigest: 'sha256:1234567890abcdef', + }); + expect(provenance?.recipeSha256).toMatch(/^[a-f0-9]{64}$/); + }); +}); diff --git a/packages/core/test/evaluation/loaders/environment-recipe.test.ts b/packages/core/test/evaluation/loaders/environment-recipe.test.ts index d8f238016..bd3731091 100644 --- a/packages/core/test/evaluation/loaders/environment-recipe.test.ts +++ b/packages/core/test/evaluation/loaders/environment-recipe.test.ts @@ -60,6 +60,7 @@ describe('environment recipe loading', () => { type: 'host', workdir: path.join(dir, 'workspaces/app'), sourceDir: dir, + recipeSha256: expect.any(String), setup: { command: './scripts/setup.sh', args: { @@ -92,6 +93,8 @@ describe('environment recipe loading', () => { type: 'host', workdir: path.join(recipeDir, 'checkout'), recipeFilePath: path.join(recipeDir, 'host.yaml'), + recipeFileSha256: expect.any(String), + recipeSha256: expect.any(String), }); }); }); @@ -129,6 +132,7 @@ describe('environment recipe loading', () => { dockerfile: path.join(dir, 'Dockerfile'), workdir: '/app', sourceDir: dir, + recipeSha256: expect.any(String), env: { NODE_ENV: 'test' }, resources: { cpus: 2, memory: '4g' }, mounts: [{ source: path.join(dir, 'fixtures'), target: '/fixtures', access: 'ro' }],