diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 959a59a83..439f43ded 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -571,20 +571,14 @@ export function resolveExperimentNamespace(params: { resultGroupName: string; }): { experiment: string; - source: RunRuntimeSourceMetadata['experiment_namespace_source']; } { if (params.cliExperiment) { - return { experiment: params.cliExperiment, source: 'cli' }; + return { experiment: params.cliExperiment }; } if (params.tagsExperiment) { - return { experiment: params.tagsExperiment, source: 'tags' }; + return { experiment: params.tagsExperiment }; } - const source: RunRuntimeSourceMetadata['experiment_namespace_source'] = params.isMultiEval - ? 'multi_eval' - : params.suiteName - ? 'eval_metadata' - : 'eval_filename'; - return { experiment: params.resultGroupName, source }; + return { experiment: params.resultGroupName }; } /** @@ -902,11 +896,9 @@ function buildRuntimeSourceMetadata(params: { readonly activeTestFiles: readonly string[]; readonly sourceTests: readonly EvalTest[]; readonly fileMetadata: ReadonlyMap; - readonly experimentNamespace: string; - readonly experimentNamespaceSource: RunRuntimeSourceMetadata['experiment_namespace_source']; readonly hasCliRuntimeConfig: boolean; }): RunRuntimeSourceMetadata { - const evalFiles = uniqueRuntimeSourcePaths( + const activeEvalFiles = uniqueRuntimeSourcePaths( params.activeTestFiles.map((filePath) => toRuntimeSourcePath(params.cwd, filePath)), ); const activeResolvedFiles = new Set( @@ -920,30 +912,22 @@ function buildRuntimeSourceMetadata(params: { const sourceFile = testSourceEvalPathForComparison(test); return sourceFile ? !activeResolvedFiles.has(sourceFile) : false; }); - const kind = - params.activeTestFiles.length > 1 - ? 'multi_eval' - : hasImportedSuite || hasNonActiveSourceFile - ? 'wrapper_eval' - : 'direct_suite'; - const wrapperEvalFile = - kind === 'wrapper_eval' - ? toRuntimeSourcePath(params.cwd, params.activeTestFiles[0]) - : undefined; + const isWrapperEval = + params.activeTestFiles.length === 1 && (hasImportedSuite || hasNonActiveSourceFile); + const wrapperEvalFile = isWrapperEval + ? toRuntimeSourcePath(params.cwd, params.activeTestFiles[0]) + : undefined; + const evalFiles = sourceEvalFiles.length > 0 ? sourceEvalFiles : activeEvalFiles; return { schema_version: 'agentv.runtime_source.v1', - kind, config_source: buildRuntimeConfigSource({ activeTestFiles: params.activeTestFiles, fileMetadata: params.fileMetadata, hasCliRuntimeConfig: params.hasCliRuntimeConfig, }), - experiment_namespace: params.experimentNamespace, - experiment_namespace_source: params.experimentNamespaceSource, eval_files: evalFiles, ...(wrapperEvalFile && { wrapper_eval_file: wrapperEvalFile }), - ...(sourceEvalFiles.length > 0 && { source_eval_files: sourceEvalFiles }), }; } @@ -1848,14 +1832,13 @@ export async function runEvalCommand( configTags: yamlConfig?.tags, cliTags: options.tagMap, }); - const { experiment: resolvedExperimentNamespace, source: experimentNamespaceSource } = - resolveExperimentNamespace({ - cliExperiment: resolvedExperiment.name, - tagsExperiment: normalizeString(resolvedTags?.experiment), - isMultiEval: resolvedTestFiles.length > 1, - suiteName: primarySuite?.metadata?.name, - resultGroupName, - }); + const { experiment: resolvedExperimentNamespace } = resolveExperimentNamespace({ + cliExperiment: resolvedExperiment.name, + tagsExperiment: normalizeString(resolvedTags?.experiment), + isMultiEval: resolvedTestFiles.length > 1, + suiteName: primarySuite?.metadata?.name, + resultGroupName, + }); // Normalize once so the row `experiment` field, AGENTV_EXPERIMENT, and the // emitted tags map all agree (and any invalid-name error surfaces in one place). const normalizedExperiment = normalizeExperimentName(resolvedExperimentNamespace); @@ -2266,8 +2249,6 @@ export async function runEvalCommand( activeTestFiles, sourceTests: activeSourceTests, fileMetadata, - experimentNamespace: normalizeExperimentName(options.experiment), - experimentNamespaceSource, hasCliRuntimeConfig, }); const hasPerFileRuntimeThresholds = diff --git a/apps/cli/src/commands/results/report.ts b/apps/cli/src/commands/results/report.ts index 22397aeea..d70e757f9 100644 --- a/apps/cli/src/commands/results/report.ts +++ b/apps/cli/src/commands/results/report.ts @@ -86,10 +86,7 @@ function serializeReportResult( const runtimeSource = manifestRecord?.runtime_source ?? summaryMetadata?.runtimeSource; const resultExperiment = (result as EvaluationResult & { experiment?: string }).experiment; const experimentNamespace = - runtimeSource?.experiment_namespace ?? - manifestRecord?.experiment ?? - summaryMetadata?.experiment ?? - resultExperiment; + manifestRecord?.experiment ?? summaryMetadata?.experiment ?? resultExperiment; const fallbackEvalFile = normalizeEvalFileLabel(manifestRecord?.eval_file) ?? summaryMetadata?.evalFile ?? @@ -113,7 +110,6 @@ function serializeReportResult( output: result.output, assertions: result.assertions, experiment: experimentNamespace, - experiment_namespace: experimentNamespace, runtime_source: runtimeSource, runtime_source_label: formatRuntimeSourceLabel(runtimeSource), runtime_config_source_label: formatRuntimeConfigSourceLabel(runtimeSource?.config_source), @@ -121,19 +117,6 @@ function serializeReportResult( }; } -function formatRuntimeKindLabel(kind: RunRuntimeSourceMetadata['kind'] | undefined): string { - switch (kind) { - case 'direct_suite': - return 'Direct suite'; - case 'wrapper_eval': - return 'Wrapper eval'; - case 'multi_eval': - return 'Multi-eval'; - default: - return 'Unknown source'; - } -} - function formatRuntimeConfigSourceLabel( source: RunRuntimeSourceMetadata['config_source'] | undefined, ): string { @@ -151,36 +134,11 @@ function formatRuntimeConfigSourceLabel( } } -function formatNamespaceSourceLabel( - source: RunRuntimeSourceMetadata['experiment_namespace_source'] | undefined, -): string { - switch (source) { - case 'cli': - return 'CLI namespace'; - case 'tags': - return 'Tags namespace'; - case 'eval_metadata': - return 'Eval metadata namespace'; - case 'eval_filename': - return 'Eval filename namespace'; - case 'multi_eval': - return 'Multi-eval namespace'; - default: - return ''; - } -} - function formatRuntimeSourceLabel(runtimeSource: RunRuntimeSourceMetadata | undefined): string { if (!runtimeSource) { return ''; } - return [ - formatRuntimeKindLabel(runtimeSource.kind), - formatNamespaceSourceLabel(runtimeSource.experiment_namespace_source), - formatRuntimeConfigSourceLabel(runtimeSource.config_source), - ] - .filter(Boolean) - .join(' · '); + return formatRuntimeConfigSourceLabel(runtimeSource.config_source); } function uniqueStrings(values: readonly (string | undefined)[]): string[] { @@ -201,13 +159,7 @@ function escapeHtml(value: string): string { function formatReportHeaderContext(rows: readonly Record[]): string { const experiments = uniqueStrings( - rows.map((row) => - typeof row.experiment_namespace === 'string' - ? row.experiment_namespace - : typeof row.experiment === 'string' - ? row.experiment - : undefined, - ), + rows.map((row) => (typeof row.experiment === 'string' ? row.experiment : undefined)), ); const runtimeSources = uniqueStrings( rows.map((row) => diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index b2d16a335..abccda5fe 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -1447,7 +1447,6 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext runtimeSource = deriveDashboardRuntimeSource({ summaryMetadata, records, - inferredExperiment: experiment, }); } else { // Run is in-progress with 0 results written yet — fall back to the @@ -1456,7 +1455,6 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext runtimeSource = deriveDashboardRuntimeSource({ summaryMetadata, records: [], - inferredExperiment: experiment, }); } } catch { @@ -1524,7 +1522,6 @@ async function handleRunDetail(c: C, { searchDir, projectId }: DataContext) { const runtimeSource = deriveDashboardRuntimeSource({ summaryMetadata, records, - inferredExperiment: records[0]?.experiment, }); // Surface run_dir + suite_filter for local runs so the UI can launch a // Dashboard-side resume against this exact run. Remote runs live in the @@ -1646,7 +1643,6 @@ function deriveDashboardRuntimeSource(params: { runtimeSource?: RunRuntimeSourceMetadata; runtime_source?: RunRuntimeSourceMetadata; }[]; - readonly inferredExperiment?: string; }): RunRuntimeSourceMetadata | undefined { const recordWithRuntimeSource = params.records.find( (record) => record.runtimeSource ?? record.runtime_source, @@ -1659,25 +1655,17 @@ function deriveDashboardRuntimeSource(params: { return explicit; } - const experimentNamespace = - params.summaryMetadata.experiment ?? - params.inferredExperiment ?? - params.records.find((record) => record.experiment)?.experiment ?? - 'default'; const evalFiles = uniqueRuntimeSourceValues([ params.summaryMetadata.evalFile, ...params.records.map((record) => record.evalPath ?? record.eval_path), ]); - if (evalFiles.length === 0 && !experimentNamespace) { + if (evalFiles.length === 0) { return undefined; } return { schema_version: 'agentv.runtime_source.v1', - kind: evalFiles.length > 1 ? 'multi_eval' : 'direct_suite', config_source: 'defaults', - experiment_namespace: experimentNamespace, - experiment_namespace_source: 'unknown', eval_files: evalFiles, }; } @@ -3173,7 +3161,6 @@ export function createApp( runtimeSource = deriveDashboardRuntimeSource({ summaryMetadata, records, - inferredExperiment: experiment, }); } } catch { diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 6b6dba4ee..97250cf00 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -26,7 +26,6 @@ import { type GradingArtifact, type IndexArtifactEntry, RESULT_INDEX_FILENAME, - RUN_CONFIG_FILENAME, type RunSummaryArtifact, type TimingArtifact, buildAggregateGradingArtifact, @@ -1177,10 +1176,7 @@ describe('writeArtifactsFromResults', () => { it('writes optional runtime source metadata to summary only', async () => { const runtimeSource = { schema_version: 'agentv.runtime_source.v1' as const, - kind: 'direct_suite' as const, config_source: 'cli_flags' as const, - experiment_namespace: 'cli-smoke', - experiment_namespace_source: 'cli' as const, eval_files: ['evals/smoke.eval.yaml'], }; const paths = await writeArtifactsFromResults([makeResult({ testId: 'alpha' })], testDir, { @@ -1199,7 +1195,7 @@ describe('writeArtifactsFromResults', () => { expect(indexLine.runtime_source).toBeUndefined(); }); - it('moves experiment config metadata to an internal run config sidecar', async () => { + it('does not write experiment config metadata into public run artifacts', async () => { const experimentMetadata = { name: 'native-exp', target: 'codex-target', @@ -1214,21 +1210,16 @@ describe('writeArtifactsFromResults', () => { const summary: RunSummaryArtifact = JSON.parse(await readFile(paths.summaryPath, 'utf8')); expect(summary.metadata).not.toHaveProperty('experiment_config'); - expect(summary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`); - - const runConfig = JSON.parse( - await readFile(path.join(paths.testArtifactDir, '.internal', RUN_CONFIG_FILENAME), 'utf8'), - ); - expect(runConfig).toEqual({ - schema_version: 'agentv.run_config.v1', - experiment_config: experimentMetadata, - }); + expect(summary.metadata).not.toHaveProperty('run_config_path'); + await expect( + readFile(path.join(paths.testArtifactDir, '.internal', 'run-config.json'), 'utf8'), + ).rejects.toThrow(); await aggregateRunDir(paths.testArtifactDir); const rewrittenSummary: RunSummaryArtifact = JSON.parse( await readFile(paths.summaryPath, 'utf8'), ); - expect(rewrittenSummary.metadata.run_config_path).toBe(`.internal/${RUN_CONFIG_FILENAME}`); + expect(rewrittenSummary.metadata).not.toHaveProperty('run_config_path'); }); it('omits duplicated root instances from run summary', () => { diff --git a/apps/cli/test/commands/eval/tag-filtering.test.ts b/apps/cli/test/commands/eval/tag-filtering.test.ts index 698beaf5e..5023d0495 100644 --- a/apps/cli/test/commands/eval/tag-filtering.test.ts +++ b/apps/cli/test/commands/eval/tag-filtering.test.ts @@ -144,20 +144,18 @@ describe('resolveExperimentNamespace', () => { it('prefers an explicit --experiment over tags.experiment and the default', () => { expect( resolveExperimentNamespace({ ...base, cliExperiment: 'cli-exp', tagsExperiment: 'tag-exp' }), - ).toEqual({ experiment: 'cli-exp', source: 'cli' }); + ).toEqual({ experiment: 'cli-exp' }); }); it('uses tags.experiment when no --experiment is given', () => { expect(resolveExperimentNamespace({ ...base, tagsExperiment: 'tag-exp' })).toEqual({ experiment: 'tag-exp', - source: 'tags', }); }); it('falls back to the eval-metadata default when neither is set (e.g. --tag experiment=)', () => { expect(resolveExperimentNamespace({ ...base })).toEqual({ experiment: 'my-suite', - source: 'eval_metadata', }); }); @@ -167,13 +165,13 @@ describe('resolveExperimentNamespace', () => { isMultiEval: false, resultGroupName: 'dataset', }), - ).toEqual({ experiment: 'dataset', source: 'eval_filename' }); + ).toEqual({ experiment: 'dataset' }); }); it('labels multi-eval runs when no CLI/tags experiment is set', () => { expect( resolveExperimentNamespace({ isMultiEval: true, resultGroupName: 'multi-eval' }), - ).toEqual({ experiment: 'multi-eval', source: 'multi_eval' }); + ).toEqual({ experiment: 'multi-eval' }); }); it('lets tags.experiment win over the multi-eval default', () => { @@ -183,7 +181,7 @@ describe('resolveExperimentNamespace', () => { tagsExperiment: 'tag-exp', resultGroupName: 'multi-eval', }), - ).toEqual({ experiment: 'tag-exp', source: 'tags' }); + ).toEqual({ experiment: 'tag-exp' }); }); }); diff --git a/apps/cli/test/commands/results/report.test.ts b/apps/cli/test/commands/results/report.test.ts index b4ff0c16c..580595b94 100644 --- a/apps/cli/test/commands/results/report.test.ts +++ b/apps/cli/test/commands/results/report.test.ts @@ -160,13 +160,9 @@ describe('results report', () => { experiment: 'named-smoke', runtimeSource: { schema_version: 'agentv.runtime_source.v1', - kind: 'wrapper_eval', config_source: 'inline_experiment', - experiment_namespace: 'named-smoke', - experiment_namespace_source: 'eval_metadata', - eval_files: ['evals/wrapper.eval.yaml'], + eval_files: ['evals/child.eval.yaml'], wrapper_eval_file: 'evals/wrapper.eval.yaml', - source_eval_files: ['evals/child.eval.yaml'], }, }); @@ -174,9 +170,7 @@ describe('results report', () => { const html = readFileSync(outputPath, 'utf8'); expect(html).toContain('Experiment namespace: named-smoke'); - expect(html).toContain( - 'Runtime source: Wrapper eval · Eval metadata namespace · Inline experiment config', - ); + expect(html).toContain('Runtime source: Inline experiment config'); }); it('embeds result text containing replacement tokens without corrupting the inline script', async () => { diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 2c46a1d46..b4e1139f2 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -893,13 +893,9 @@ describe('serve app', () => { const runDir = localRunDir(tempDir, experiment, filename); const runtimeSource = { schema_version: 'agentv.runtime_source.v1' as const, - kind: 'wrapper_eval' as const, config_source: 'inline_experiment' as const, - experiment_namespace: experiment, - experiment_namespace_source: 'eval_metadata' as const, - eval_files: ['evals/wrapper.eval.yaml'], + eval_files: ['evals/source.test.yaml'], wrapper_eval_file: 'evals/wrapper.eval.yaml', - source_eval_files: ['evals/source.test.yaml'], }; mkdirSync(runDir, { recursive: true }); writeFileSync( @@ -4236,10 +4232,7 @@ describe('serve app', () => { const runDir = path.join(runsDir, filename); const runtimeSource = { schema_version: 'agentv.runtime_source.v1' as const, - kind: 'direct_suite' as const, config_source: 'defaults' as const, - experiment_namespace: 'cli-smoke', - experiment_namespace_source: 'cli' as const, eval_files: ['examples/demo.eval.yaml'], }; mkdirSync(runDir, { recursive: true }); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 997cc125a..6d4d32be8 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -698,36 +698,18 @@ describe('agentv eval CLI', () => { ) as { metadata?: Record }; expect(benchmark.metadata?.experiment).toBe('native-exp'); expect(benchmark.metadata).not.toHaveProperty('experiment_config'); - expect(benchmark.metadata?.run_config_path).toBe('.internal/run-config.json'); - const runConfig = JSON.parse( - await readFile( + expect(benchmark.metadata).not.toHaveProperty('run_config_path'); + await expect( + readFile( path.join(runDirFromIndexPath(outputPath), '.internal', 'run-config.json'), 'utf8', ), - ) as { experiment_config?: Record }; - expect(runConfig.experiment_config).toMatchObject({ - target: 'codex-target', - repeat: { - count: 2, - strategy: 'pass_any', - early_exit: true, - }, - threshold: 0.8, - budget_usd: 3, - timeout_seconds: 12, - }); - expect((runConfig.experiment_config as Record).fingerprint).toMatch( - /^[a-f0-9]{64}$/, - ); + ).rejects.toThrow(); expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', - kind: 'wrapper_eval', config_source: 'mixed', - experiment_namespace: 'native-exp', - experiment_namespace_source: 'eval_metadata', - eval_files: ['native-exp.eval.yaml'], + eval_files: ['sample.test.yaml'], wrapper_eval_file: 'native-exp.eval.yaml', - source_eval_files: ['sample.test.yaml'], }); } finally { await rm(fixture.baseDir, { recursive: true, force: true }); @@ -906,10 +888,7 @@ describe('agentv eval CLI', () => { ) as { metadata?: Record }; expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', - kind: 'multi_eval', config_source: 'mixed', - experiment_namespace: 'multi-eval', - experiment_namespace_source: 'multi_eval', eval_files: ['first.eval.yaml', 'second.eval.yaml'], }); } finally { @@ -1202,10 +1181,7 @@ tests: ) as { metadata?: Record }; expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', - kind: 'direct_suite', config_source: 'inline_experiment', - experiment_namespace: 'cli-smoke', - experiment_namespace_source: 'cli', eval_files: ['sample.test.yaml'], }); } finally { diff --git a/apps/dashboard/src/components/RunList.mobile.spec.tsx b/apps/dashboard/src/components/RunList.mobile.spec.tsx index 86a791ce5..5e9efacd7 100644 --- a/apps/dashboard/src/components/RunList.mobile.spec.tsx +++ b/apps/dashboard/src/components/RunList.mobile.spec.tsx @@ -71,10 +71,7 @@ describe('buildRunListItemView', () => { experiment: 'smoke-suite', runtime_source: { schema_version: 'agentv.runtime_source.v1', - kind: 'multi_eval', config_source: 'mixed', - experiment_namespace: 'smoke-suite', - experiment_namespace_source: 'cli', eval_files: ['evals/a.eval.yaml', 'evals/b.eval.yaml'], }, }), @@ -82,7 +79,7 @@ describe('buildRunListItemView', () => { ); expect(view.experimentNamespace).toBe('smoke-suite'); - expect(view.runtimeSourceLabel).toBe('Multi-eval · CLI namespace · Mixed runtime config'); + expect(view.runtimeSourceLabel).toBe('Mixed runtime config'); expect(view.runtimeSourceTitle).toContain('evals/a.eval.yaml'); }); }); diff --git a/apps/dashboard/src/lib/run-detail-context.test.ts b/apps/dashboard/src/lib/run-detail-context.test.ts index b0c7b7a6e..8ca4fe358 100644 --- a/apps/dashboard/src/lib/run-detail-context.test.ts +++ b/apps/dashboard/src/lib/run-detail-context.test.ts @@ -78,24 +78,20 @@ describe('buildRunDetailHeader', () => { results: localRunResults, runtimeSource: { schema_version: 'agentv.runtime_source.v1', - kind: 'wrapper_eval', config_source: 'inline_experiment', - experiment_namespace: 'native-exp', - experiment_namespace_source: 'eval_metadata', - eval_files: ['evals/native-exp.eval.yaml'], + eval_files: ['evals/sample.eval.yaml'], wrapper_eval_file: 'evals/native-exp.eval.yaml', - source_eval_files: ['evals/sample.eval.yaml'], }, formatTimestamp: (timestamp) => timestamp, }); expect(header.sourceContext).toContainEqual({ label: 'Experiment namespace', - value: 'native-exp', + value: 'default', }); expect(header.sourceContext).toContainEqual({ label: 'Runtime source', - value: 'Wrapper eval · Eval metadata namespace · Inline experiment config', + value: 'Inline experiment config', }); }); }); diff --git a/apps/dashboard/src/lib/runtime-source.ts b/apps/dashboard/src/lib/runtime-source.ts index 6881a945c..d22ed3f98 100644 --- a/apps/dashboard/src/lib/runtime-source.ts +++ b/apps/dashboard/src/lib/runtime-source.ts @@ -1,18 +1,5 @@ import type { RunRuntimeSource } from './types'; -export function runtimeKindLabel(kind: RunRuntimeSource['kind'] | undefined): string { - switch (kind) { - case 'direct_suite': - return 'Direct suite'; - case 'wrapper_eval': - return 'Wrapper eval'; - case 'multi_eval': - return 'Multi-eval'; - default: - return 'Unknown source'; - } -} - export function runtimeConfigSourceLabel( source: RunRuntimeSource['config_source'] | undefined, ): string { @@ -26,24 +13,7 @@ export function runtimeConfigSourceLabel( case 'defaults': return 'Default runtime config'; default: - return 'Unknown runtime config'; - } -} - -export function experimentNamespaceSourceLabel( - source: RunRuntimeSource['experiment_namespace_source'] | undefined, -): string { - switch (source) { - case 'cli': - return 'CLI namespace'; - case 'eval_metadata': - return 'Eval metadata namespace'; - case 'eval_filename': - return 'Eval filename namespace'; - case 'multi_eval': - return 'Multi-eval namespace'; - default: - return 'Namespace source unknown'; + return 'Runtime config unknown'; } } @@ -51,20 +21,14 @@ export function experimentNamespaceLabel(input: { experiment?: string; runtime_source?: RunRuntimeSource; }): string { - return ( - input.runtime_source?.experiment_namespace?.trim() || input.experiment?.trim() || 'default' - ); + return input.experiment?.trim() || 'default'; } export function runtimeSourceSummary(runtimeSource: RunRuntimeSource | undefined): string { if (!runtimeSource) { return 'Runtime source unknown'; } - return [ - runtimeKindLabel(runtimeSource.kind), - experimentNamespaceSourceLabel(runtimeSource.experiment_namespace_source), - runtimeConfigSourceLabel(runtimeSource.config_source), - ].join(' · '); + return runtimeConfigSourceLabel(runtimeSource.config_source); } export function runtimeSourceTitle(runtimeSource: RunRuntimeSource | undefined): string { @@ -78,8 +42,5 @@ export function runtimeSourceTitle(runtimeSource: RunRuntimeSource | undefined): if (runtimeSource.wrapper_eval_file) { lines.push(`Wrapper eval: ${runtimeSource.wrapper_eval_file}`); } - if (runtimeSource.source_eval_files && runtimeSource.source_eval_files.length > 0) { - lines.push(`Source eval files: ${runtimeSource.source_eval_files.join(', ')}`); - } return lines.join('\n'); } diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index 206c2ec50..91fb91b91 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -160,7 +160,7 @@ export interface EvalCaseTrial { export type EvalTrialAggregation = | { - strategy: 'pass_at_k'; + strategy: 'pass_any'; passed_attempts?: number; total_attempts?: number; } @@ -320,19 +320,9 @@ export interface RunDetailResponse { export interface RunRuntimeSource { schema_version?: 'agentv.runtime_source.v1'; - kind?: 'direct_suite' | 'wrapper_eval' | 'multi_eval' | string; config_source?: 'defaults' | 'inline_experiment' | 'cli_flags' | 'mixed' | string; - experiment_namespace?: string; - experiment_namespace_source?: - | 'cli' - | 'eval_metadata' - | 'eval_filename' - | 'multi_eval' - | 'unknown' - | string; eval_files?: string[]; wrapper_eval_file?: string; - source_eval_files?: string[]; } export interface SuiteSummary { diff --git a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx index 24c05306e..58d84f92a 100644 --- a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx @@ -95,7 +95,6 @@ reserved for rebuildable local state and are skipped by run discovery. | File or field | Owns | Use it for | | --- | --- | --- | | `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. | -| `.internal/run-config.json` | Internal reproducibility metadata that is useful to rerun or audit the invocation but too noisy for the public aggregate summary, such as resolved experiment config and fingerprints. | Debugging run setup, reproducing wrapper-eval choices, and reviewing config provenance without repeating it in every row. | | `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. | | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. | | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. | @@ -108,12 +107,11 @@ reserved for rebuildable local state and are skipped by run discovery. | `test/` | Generated test bundle for the exact eval slice and target settings that produced a row. | Audit, external review, and rerun workflows that should not depend on a mutable source checkout. | | `artifact_pointers` | Offload indirection for large detached payload bytes. | Finding payloads published outside the primary metadata/control-plane branch, such as transcript bytes on `agentv/artifacts/v1`. | -`summary.json`, `.internal/run-config.json`, and `.internal/index.jsonl` are -complementary, not redundant. A run list should not scan every row just to show -pass rate or total duration, and a row reader should not parse aggregate summary -structures to find one case's grading or transcript. Keep aggregate questions on -`summary.json`; keep run setup details in `.internal/run-config.json`; keep row -and artifact discovery on `.internal/index.jsonl`. +`summary.json` and `.internal/index.jsonl` are complementary, not redundant. A +run list should not scan every row just to show pass rate or total duration, and +a row reader should not parse aggregate summary structures to find one case's +grading or transcript. Keep aggregate questions on `summary.json`; keep row and +artifact discovery on `.internal/index.jsonl`. ## Grading Contract @@ -178,7 +176,8 @@ adds providers and projections, but stable rows follow these rules: - Identity and filter fields live on the row, not only in directory names. - Sidecar references are explicit path fields, relative to the run directory. - Run-level provenance such as `runtime_source` belongs in `summary.json`, not - repeated on every row. + repeated on every row. It records runtime config provenance and eval file + paths; the experiment label remains on `experiment` and `tags.experiment`. - Large detached payloads may also have `artifact_pointers`, but ordinary sidecars should still be discoverable through path fields. - Unknown fields should be preserved by adapters when they rewrite or project @@ -244,6 +243,8 @@ When a run resolves a tags metadata map (from suite `tags`, project config `tags`, or `--tag key=value`), the resolved map is emitted as `tags` on each row and as `summary.json.metadata.tags`. Its reserved `experiment` key matches the row `experiment` field, so trend/compare views can group by `tags.experiment`. +Machine-readable durations use milliseconds in row and metrics fields such as +`duration_ms` and `total_ms`; renderers can derive human seconds for display. Use `repeat` for authoring configuration and `samples` for produced executions. The `sample-1/`, `sample-2/`, and later folders under a result diff --git a/docs/adr/0013-experiment-is-metadata-expressed-as-tags-experiment.md b/docs/adr/0013-experiment-is-metadata-expressed-as-tags-experiment.md index e9b052e0a..d882dd41c 100644 --- a/docs/adr/0013-experiment-is-metadata-expressed-as-tags-experiment.md +++ b/docs/adr/0013-experiment-is-metadata-expressed-as-tags-experiment.md @@ -15,11 +15,10 @@ source loop by giving evals a promptfoo-compatible way to author it. ## Context AgentV already treats `experiment` as run-grouping metadata: it is written to -`summary.json.metadata.experiment` and each `index.jsonl` row, with provenance -in `runtime_source.experiment_namespace_source`. Until now an eval could only -influence the namespace through the CLI `--experiment` flag or by falling back -to the suite `metadata.name` / eval filename. There was no first-class, -in-eval way to label the experiment. +`summary.json.metadata.experiment` and each `index.jsonl` row. Until now an eval +could only influence the namespace through the CLI `--experiment` flag or by +falling back to the suite `metadata.name` / eval filename. There was no +first-class, in-eval way to label the experiment. promptfoo — a widely used lowest-common-denominator eval contract — expresses run labels through `tags: Record` with no first-class @@ -63,11 +62,6 @@ The experiment namespace is resolved with precedence: --experiment (CLI) > tags.experiment > default (multi-eval / suite name / filename) ``` -`experiment_namespace_source` gains a new `tags` value for provenance, joining -the existing `cli`, `eval_metadata`, `eval_filename`, `multi_eval`, and -`unknown` values. The `experiment_namespace` / `experiment_namespace_source` -contract is otherwise unchanged. - The resolved tags map is emitted to `summary.json.metadata.tags` and to each `index.jsonl` row (mirroring promptfoo's `evals_to_tags`) so Dashboard trend/compare can group by `tags.experiment`. Experiment stays metadata: there diff --git a/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md b/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md index 2d7389229..ca5a3f04f 100644 --- a/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md +++ b/docs/adr/0017-output-artifact-and-workspace-resolver-contract.md @@ -49,12 +49,13 @@ bundle, and how a workspace is acquired. "manifest"/`bundle.json` for the frozen config. 6. **Merge `timing.json` into `metrics.json`** (sections: duration/tokens/cost always; execution/trajectory when a trace exists); drop `timing_path`, keep one `metrics_path`. -7. **Analytics = one pure `Build()`** (margin-lab shape) producing the `Summary` with - pass@k; add promptfoo-shaped `named_scores`/`derived_metrics` on rows. +7. **Analytics = one pure `Build()`** (margin-lab shape) producing status, + count, usage, runtime, case, and failure summaries; add promptfoo-shaped + `named_scores`/`derived_metrics` on rows. ### Multi-suite runs — one run_id, categorize by suite AND tags/experiment Confirms ADR-0009 + ADR-0012 (not a new decision): -- **One `` (one timestamp) per CLI invocation**, across any number of suite YAMLs — all suites' cases live under the single `/` bundle. **Never a separate timestamp/folder per suite.** `runtime_source.kind = multi_eval` records the multi-suite invocation. +- **One `` (one timestamp) per CLI invocation**, across any number of suite YAMLs — all suites' cases live under the single `/` bundle. **Never a separate timestamp/folder per suite.** `runtime_source.eval_files` records the active eval files. - **Identity = `eval_path` + `test_id`** (uuid-suffixed dir), so overlapping `test_id`s across suites don't collide. `suite`/`name` are **display/grouping metadata, not routing** (ADR-0009). - **Categorize by BOTH, orthogonally** (each `index.jsonl` row carries both): **`suite`** (+`eval_path`) = structural origin; **`tags`** (map, incl **`experiment`**) = semantic/campaign grouping. `experiment` = the run/campaign bucket; `suite` = the intra-run structural group; the Dashboard groups by any tag key, and suite is another grouping dimension. Reports filter/group by either axis. diff --git a/docs/plans/dashboard-tags-tab-brainstorm.md b/docs/plans/dashboard-tags-tab-brainstorm.md index 234d657a4..df1248d62 100644 --- a/docs/plans/dashboard-tags-tab-brainstorm.md +++ b/docs/plans/dashboard-tags-tab-brainstorm.md @@ -32,7 +32,7 @@ Artifact source of truth (verified in core): - `summary.json` `metadata.tags` (`Record`) is written via `aggregateRunDir` → `buildRunSummaryArtifact` in `packages/core/src/evaluation/run-artifacts.ts` (`tags` option threaded at `run-artifacts.ts:168,180,190`; round-tripped through `readRunSummaryMetadata` at `run-artifacts.ts:245-258`). - Each `index.jsonl` row carries `experiment` (string) and `tags` (`Record`): the `ResultIndexArtifact` type declares both at `run-artifacts.ts:477,479`, and `writePerTestArtifacts` writes each row with `experiment: options?.experiment` and `...(resolvedTags ? { tags: resolvedTags } : {})` at `run-artifacts.ts:2351-2371` (row push at `2369-2370`). -- Lockstep: the run-level `experiment` namespace is derived from the `tags` map's reserved `experiment` key (`run-artifacts.ts:435-439` — "The reserved key `experiment` feeds the experiment namespace"; `experiment_namespace_source: 'tags'` at `run-artifacts.ts:82,310`). The equality is actively enforced at run time by `syncTagsExperiment` (`apps/cli/src/commands/eval/run-eval.ts:~404-424`) with precedence resolved by `resolveExperimentNamespace` (CLI `--experiment` > `tags.experiment` > eval defaults, `run-eval.ts:~426-461`), so the top-level `experiment` field and `tags.experiment` stay equal. The Dashboard-side resolution should therefore read `record.experiment ?? record.tags?.experiment`. +- Lockstep: the run-level `experiment` label is derived from the `tags` map's reserved `experiment` key when present. The equality is actively enforced at run time by `syncTagsExperiment` with precedence resolved by `resolveExperimentNamespace` (CLI `--experiment` > `tags.experiment` > eval defaults), so the top-level `experiment` field and `tags.experiment` stay equal. The Dashboard-side resolution should therefore read `record.experiment ?? record.tags?.experiment`. - Confirmed drop point: neither `ResultManifestRecord` (`manifest.ts:25-76`) nor `LightweightResultRecord` (`manifest.ts:310-325`) declares `tags`, and the core row normalizer (`packages/core/src/evaluation/result-row-schema.ts:~189-221`) has no `tags` alias — the map is present in the raw JSONL but dropped by the CLI parse layer before any handler sees it. This is exactly the plumbing §4 must fix. --- diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index d4d5b5a89..dc3c5d907 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -110,27 +110,13 @@ const TIMING_SOURCE_VALUES = [ type TimingSource = (typeof TIMING_SOURCE_VALUES)[number]; -export type RunRuntimeSourceKind = 'direct_suite' | 'wrapper_eval' | 'multi_eval'; - export type RunRuntimeConfigSource = 'defaults' | 'inline_experiment' | 'cli_flags' | 'mixed'; -export type ExperimentNamespaceSource = - | 'cli' - | 'tags' - | 'eval_metadata' - | 'eval_filename' - | 'multi_eval' - | 'unknown'; - export interface RunRuntimeSourceMetadata { readonly schema_version: 'agentv.runtime_source.v1'; - readonly kind: RunRuntimeSourceKind; readonly config_source: RunRuntimeConfigSource; - readonly experiment_namespace: string; - readonly experiment_namespace_source: ExperimentNamespaceSource; readonly eval_files: readonly string[]; readonly wrapper_eval_file?: string; - readonly source_eval_files?: readonly string[]; } export function buildTestTargetKey(testId?: string, target?: string, variant?: string): string { @@ -218,9 +204,6 @@ export async function aggregateRunDir( const previousMetadata = await readRunSummaryMetadata(path.join(runDir, RUN_SUMMARY_FILENAME)); const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount; - const runConfigPath = options?.experimentMetadata - ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` - : previousMetadata.runConfigPath; const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource; const tags = options?.tags ?? previousMetadata.tags; @@ -233,11 +216,9 @@ export async function aggregateRunDir( options?.experimentMetadata, runtimeSource, tags, - runConfigPath, ); const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME); await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); - await writeRunConfigArtifact(runDir, options?.experimentMetadata); const targetSet = new Set(results.map((r) => r.target ?? 'unknown')); return { summaryPath, testCount: results.length, targetCount: targetSet.size }; @@ -295,7 +276,6 @@ async function resolveExistingResultManifestPath(runDir: string): Promise; }> { @@ -304,7 +284,6 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ const parsed = JSON.parse(raw) as { metadata?: { planned_test_count?: number; - run_config_path?: unknown; runtime_source?: RunRuntimeSourceMetadata; tags?: unknown; }; @@ -315,15 +294,9 @@ async function readRunSummaryMetadata(summaryPath: string): Promise<{ const runtimeSource = isRunRuntimeSourceMetadata(parsed.metadata?.runtime_source) ? parsed.metadata.runtime_source : undefined; - const runConfigPath = - typeof parsed.metadata?.run_config_path === 'string' && - parsed.metadata.run_config_path.trim().length > 0 - ? parsed.metadata.run_config_path - : undefined; const tags = normalizeStringRecord(parsed.metadata?.tags); return { ...(plannedTestCount !== undefined && { plannedTestCount }), - ...(runConfigPath !== undefined && { runConfigPath }), ...(runtimeSource !== undefined && { runtimeSource }), ...(tags !== undefined && { tags }), }; @@ -357,20 +330,10 @@ function isRunRuntimeSourceMetadata(value: unknown): value is RunRuntimeSourceMe const candidate = value as Partial; return ( candidate.schema_version === 'agentv.runtime_source.v1' && - (candidate.kind === 'direct_suite' || - candidate.kind === 'wrapper_eval' || - candidate.kind === 'multi_eval') && (candidate.config_source === 'defaults' || candidate.config_source === 'inline_experiment' || candidate.config_source === 'cli_flags' || candidate.config_source === 'mixed') && - typeof candidate.experiment_namespace === 'string' && - (candidate.experiment_namespace_source === 'cli' || - candidate.experiment_namespace_source === 'tags' || - candidate.experiment_namespace_source === 'eval_metadata' || - candidate.experiment_namespace_source === 'eval_filename' || - candidate.experiment_namespace_source === 'multi_eval' || - candidate.experiment_namespace_source === 'unknown') && Array.isArray(candidate.eval_files) && candidate.eval_files.every((entry) => typeof entry === 'string') ); @@ -520,12 +483,6 @@ export interface RunSummaryArtifact { readonly failed_cases: number; readonly errored_instances: number; }; - readonly pass_at_k: { - readonly k: number; - readonly passed_cases: number; - readonly total_cases: number; - readonly rate: number; - }; readonly usage: { readonly total_tokens: number; readonly input_tokens: number; @@ -546,7 +503,6 @@ export interface RunSummaryArtifact { readonly variants?: readonly string[]; readonly tests_run: readonly string[]; readonly experiment?: string; - readonly run_config_path?: string; readonly runtime_source?: RunRuntimeSourceMetadata; readonly planned_test_count?: number; /** @@ -574,7 +530,6 @@ export interface RunSummaryArtifact { export interface RunConfigArtifact { readonly schema_version: 'agentv.run_config.v1'; - readonly experiment_config?: ExperimentArtifactMetadata; } export interface AggregateGradingArtifact { @@ -1663,10 +1618,10 @@ export function buildRunSummaryArtifact( experiment?: string, runId?: string, plannedTestCount?: number, - experimentMetadata?: ExperimentArtifactMetadata, + _experimentMetadata?: ExperimentArtifactMetadata, runtimeSource?: RunRuntimeSourceMetadata, tags?: Record, - runConfigPath?: string, + _runConfigPath?: string, ): RunSummaryArtifact { const targetSet = new Set(); const variantSet = new Set(); @@ -1868,12 +1823,6 @@ export function buildRunSummaryArtifact( failed_cases: failedCases, errored_instances: erroredInstances, }, - pass_at_k: { - k: 1, - passed_cases: passedCases, - total_cases: caseSummaries.length, - rate: percentage(passedCases, caseSummaries.length), - }, usage: { total_tokens: runMetrics.tokens.total, input_tokens: runMetrics.tokens.input, @@ -1896,10 +1845,6 @@ export function buildRunSummaryArtifact( variants: variants.length > 0 ? variants : undefined, tests_run: testIds, experiment, - run_config_path: - experimentMetadata !== undefined - ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` - : runConfigPath, runtime_source: runtimeSource, planned_test_count: plannedTestCount, tags: tags && Object.keys(tags).length > 0 ? tags : undefined, @@ -1938,23 +1883,6 @@ export async function writeInitialRunSummaryArtifact( ); const summaryPath = path.join(runDir, RUN_SUMMARY_FILENAME); await writeFile(summaryPath, `${JSON.stringify(stub, null, 2)}\n`, 'utf8'); - await writeRunConfigArtifact(runDir, options.experimentMetadata); -} - -async function writeRunConfigArtifact( - runDir: string, - experimentMetadata: ExperimentArtifactMetadata | undefined, -): Promise { - if (!experimentMetadata) { - return; - } - const config: RunConfigArtifact = { - schema_version: 'agentv.run_config.v1', - experiment_config: experimentMetadata, - }; - const configPath = runInternalPath(runDir, RUN_CONFIG_FILENAME); - await mkdir(path.dirname(configPath), { recursive: true }); - await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); } export async function readRunConfigArtifact( @@ -3404,9 +3332,6 @@ export async function writeArtifactsFromResults( const previousMetadata = await readRunSummaryMetadata(summaryPath); const plannedTestCount = options?.plannedTestCount ?? previousMetadata.plannedTestCount; - const runConfigPath = options?.experimentMetadata - ? `${RUN_INTERNAL_DIRNAME}/${RUN_CONFIG_FILENAME}` - : previousMetadata.runConfigPath; const runtimeSource = options?.runtimeSource ?? previousMetadata.runtimeSource; const summaryTags = resolvedTags ?? previousMetadata.tags; const summary = buildRunSummaryArtifact( @@ -3418,10 +3343,8 @@ export async function writeArtifactsFromResults( options?.experimentMetadata, runtimeSource, summaryTags, - runConfigPath, ); await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8'); - await writeRunConfigArtifact(outputDir, options?.experimentMetadata); await mkdir(path.dirname(indexPath), { recursive: true }); await writeJsonlFile(indexPath, indexRecords); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b3829b187..e4f044424 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -82,9 +82,7 @@ export { type GradingArtifact, type IndexArtifactEntry, type ResultIndexArtifact, - type ExperimentNamespaceSource, type RunRuntimeConfigSource, - type RunRuntimeSourceKind, type RunRuntimeSourceMetadata, type RunConfigArtifact, type RunSummaryArtifact,