diff --git a/apps/cli/src/commands/results/combine-run.ts b/apps/cli/src/commands/results/combine-run.ts index fe48ecebe..a3b01f522 100644 --- a/apps/cli/src/commands/results/combine-run.ts +++ b/apps/cli/src/commands/results/combine-run.ts @@ -56,7 +56,6 @@ export interface CombineRunSource { readonly displayName: string; readonly manifestPath: string; readonly experiment: string; - readonly tags?: readonly string[]; } export interface DuplicateConflict { @@ -111,7 +110,6 @@ export interface CombineRunResult { readonly duplicateConflicts: readonly DuplicateConflict[]; readonly testCount: number; readonly targetCount: number; - readonly tags: readonly string[]; } function parseJsonlLine(line: string): ResultManifestRecord { @@ -538,7 +536,6 @@ export function buildCombineRunSources( options?: { ids?: readonly string[]; displayNames?: readonly string[]; - tags?: readonly string[][]; }, ): CombineRunSource[] { return sourcePaths.map((sourcePath, index) => { @@ -561,7 +558,6 @@ export function buildCombineRunSources( displayName: options?.displayNames?.[index] ?? path.basename(runDir), manifestPath, experiment: normalizeExperimentName(experiment), - tags: options?.tags?.[index], }; }); } @@ -632,7 +628,6 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult const summaryPath = path.join(runDir, 'summary.json'); writeJson(summaryPath, summaryWithMetadata); - const tags = [...new Set(loadedSources.flatMap((source) => source.tags ?? []))].sort(); return { runDir, runId: toRunId(options.cwd, runDir), @@ -644,6 +639,5 @@ export function combineRunSources(options: CombineRunOptions): CombineRunResult duplicateConflicts: conflicts, testCount: rows.length, targetCount: new Set(results.map((result) => result.target ?? 'unknown')).size, - tags, }; } diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index 93d776912..9a4617eb5 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -35,7 +35,6 @@ import { normalizeExperimentName, } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; -import { normalizeTags, writeRunTags } from './run-tags.js'; // ── In-memory run tracker ──────────────────────────────────────────────── @@ -146,7 +145,6 @@ interface RunEvalRequest { test_ids?: string[]; target?: string; experiment?: string; - tags?: string[]; threshold?: number; workers?: number; /** Resume an interrupted run: skip already-completed tests and append results to `output`. */ @@ -176,24 +174,12 @@ function validateResumeOptions(req: RunEvalRequest): string | undefined { return undefined; } -function parseInitialTags(value: unknown): string[] { - if (value === undefined) return []; - if (!Array.isArray(value)) { - throw new Error('tags must be an array of strings'); - } - return normalizeTags(value); -} - -function normalizeRunMetadata(req: RunEvalRequest): { experiment: string; tags: string[] } { +function normalizeRunMetadata(req: RunEvalRequest): { experiment: string } { const experiment = normalizeExperimentName(req.experiment); - const tags = parseInitialTags(req.tags); if ((req.resume || req.rerun_failed) && req.experiment?.trim()) { throw new Error('experiment cannot be changed when resuming an existing run'); } - if ((req.resume || req.rerun_failed) && tags.length > 0) { - throw new Error('initial tags can only be set when creating a new run'); - } - return { experiment, tags }; + return { experiment }; } function buildCliArgs(req: RunEvalRequest, experiment?: string): string[] { @@ -331,12 +317,6 @@ function openConsoleLogStream(outputDir: string): WriteStream | undefined { } } -function writeInitialRunTags(outputDir: string, tags: readonly string[]): void { - if (tags.length === 0) return; - mkdirSync(outputDir, { recursive: true }); - writeRunTags(path.join(outputDir, RESULT_INDEX_FILENAME), tags); -} - // ── Route registration ─────────────────────────────────────────────────── // biome-ignore lint/suspicious/noExplicitAny: Hono Context generic varies by route @@ -400,7 +380,7 @@ export function registerEvalRoutes( return c.json({ error: resumeError }, 400); } - let metadata: { experiment: string; tags: string[] }; + let metadata: { experiment: string }; try { metadata = normalizeRunMetadata(body); } catch (err) { @@ -440,7 +420,6 @@ export function registerEvalRoutes( activeRuns.set(runId, run); try { - writeInitialRunTags(outputDir, metadata.tags); const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'], @@ -576,7 +555,7 @@ export function registerEvalRoutes( return c.json({ error: 'Invalid JSON body' }, 400); } - let metadata: { experiment: string; tags: string[] }; + let metadata: { experiment: string }; try { metadata = normalizeRunMetadata(body); } catch (err) { @@ -636,7 +615,7 @@ export function registerEvalRoutes( return c.json({ error: resumeError }, 400); } - let metadata: { experiment: string; tags: string[] }; + let metadata: { experiment: string }; try { metadata = normalizeRunMetadata(body); } catch (err) { @@ -671,7 +650,6 @@ export function registerEvalRoutes( activeRuns.set(runId, run); try { - writeInitialRunTags(outputDir, metadata.tags); const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'], @@ -775,7 +753,7 @@ export function registerEvalRoutes( } catch { return c.json({ error: 'Invalid JSON body' }, 400); } - let metadata: { experiment: string; tags: string[] }; + let metadata: { experiment: string }; try { metadata = normalizeRunMetadata(body); } catch (err) { diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index ceb4c9754..412757a59 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -28,6 +28,8 @@ export interface ResultManifestRecord { readonly suite?: string; readonly category?: string; readonly experiment?: string; + /** promptfoo-shaped tag map (`Record`), e.g. `{experiment, team, env}`. */ + readonly tags?: Record; readonly target?: string; readonly variant?: string; readonly score: number; @@ -315,6 +317,8 @@ export interface LightweightResultRecord { readonly target?: string; readonly variant?: string; readonly experiment?: string; + /** promptfoo-shaped tag map from the JSONL row's `tags` field. */ + readonly tags?: Record; readonly score: number; readonly scores?: readonly Record[]; readonly executionStatus?: string; @@ -324,6 +328,24 @@ export interface LightweightResultRecord { readonly runtimeSource?: RunRuntimeSourceMetadata; } +/** + * Coerce a raw JSONL `tags` value into a `Record`, dropping + * non-string values. Returns undefined when the map is absent or empty so the + * lightweight record stays sparse for old runs that never wrote a tags map. + */ +export function normalizeTagMap(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return undefined; + } + const entries: [string, string][] = []; + for (const [key, raw] of Object.entries(value as Record)) { + if (typeof raw === 'string') { + entries.push([key, raw]); + } + } + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + export function loadLightweightResults(sourceFile: string): LightweightResultRecord[] { const resolvedSourceFile = resolveRunManifestPath(sourceFile); const content = readFileSync(resolvedSourceFile, 'utf8'); @@ -335,6 +357,7 @@ export function loadLightweightResults(sourceFile: string): LightweightResultRec target: record.target, variant: record.variant, experiment: record.experiment, + tags: normalizeTagMap(record.tags), score: record.score, scores: record.scores, executionStatus: record.execution_status, diff --git a/apps/cli/src/commands/results/remote-metadata.ts b/apps/cli/src/commands/results/remote-metadata.ts deleted file mode 100644 index ac2ba4a7f..000000000 --- a/apps/cli/src/commands/results/remote-metadata.ts +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Mutable metadata overlays for remote result runs. - * - * Remote run artifacts under `runs/**` on the results branch are treated as - * immutable fetched payloads. Editable fields, starting with tags, live in a - * small sidecar tree under `metadata/runs/**` inside the configured results repo - * checkout/branch. This is a remote-results implementation detail, not part of - * the local `.agentv/results//` layout. It keeps local - * edits pushable by normal Git sync without rewriting the fetched run bundle. - * - * To add another mutable field: create a sibling helper that maps the remote - * run manifest to the same metadata run directory, keep the on-disk keys - * snake_case, and compare the working tree file against the upstream Git ref - * so Dashboard can show pending local edits. - */ - -import { execFileSync } from 'node:child_process'; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; - -import { assertExpectedTagRevision, createTagRevision, normalizeTagRevision } from './run-state.js'; -import { RUN_TAGS_FILENAME, normalizeTags } from './run-tags.js'; - -const RESULTS_RUNS_DIR = 'runs'; -const REMOTE_METADATA_RUNS_DIR = path.join('metadata', 'runs'); - -interface TagsFile { - readonly tags: string[]; - readonly updatedAt?: string; - readonly tagRevision: string; -} - -interface RemoteRunMetadataPaths { - readonly runRelativePath: string; - readonly artifactTagsPath: string; - readonly artifactTagsGitPath: string; - readonly overlayTagsPath: string; - readonly overlayTagsGitPath: string; -} - -interface RemoteRunTagsContext { - readonly paths: RemoteRunMetadataPaths; - readonly artifactTags: TagsFile | undefined; - readonly baseOverlayTags: TagsFile | undefined; - readonly localOverlayTags: TagsFile | undefined; -} - -export interface RemoteRunTagState { - readonly tags: string[]; - readonly remoteTags: string[]; - readonly pendingTags?: string[]; - readonly dirty: boolean; - readonly updatedAt?: string; - readonly tagRevision: string; - readonly metadataPath: string; -} - -function cleanGitEnv(): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined && !(key.startsWith('GIT_') && key !== 'GIT_SSH_COMMAND')) { - env[key] = value; - } - } - return env; -} - -function runGit(repoDir: string, args: readonly string[]): string { - return execFileSync('git', [...args], { - cwd: repoDir, - encoding: 'utf8', - env: cleanGitEnv(), - stdio: ['ignore', 'pipe', 'pipe'], - }).trim(); -} - -function tryRunGit(repoDir: string, args: readonly string[]): string | undefined { - try { - return runGit(repoDir, args); - } catch { - return undefined; - } -} - -function toGitPath(filePath: string): string { - return filePath.split(path.sep).join('/'); -} - -function readTagsFile(filePath: string): TagsFile | undefined { - if (!existsSync(filePath)) return undefined; - try { - return parseTagsFile(readFileSync(filePath, 'utf8')); - } catch { - return undefined; - } -} - -function readTagsFromGit( - repoDir: string, - ref: string | undefined, - gitPath: string, -): TagsFile | undefined { - if (!ref) return undefined; - const content = tryRunGit(repoDir, ['show', `${ref}:${gitPath}`]); - if (content === undefined) return undefined; - try { - return parseTagsFile(content); - } catch { - return undefined; - } -} - -function parseTagsFile(content: string): TagsFile | undefined { - const parsed = JSON.parse(content) as unknown; - if (!parsed || typeof parsed !== 'object') return undefined; - const record = parsed as Record; - if (!Array.isArray(record.tags)) return undefined; - const tags = record.tags.filter((tag): tag is string => typeof tag === 'string'); - const updatedAt = typeof record.updated_at === 'string' ? record.updated_at : undefined; - return { - tags, - updatedAt, - tagRevision: normalizeTagRevision(record.tag_revision, tags, updatedAt), - }; -} - -function equalTags(a: readonly string[], b: readonly string[]): boolean { - if (a.length !== b.length) return false; - return a.every((tag, index) => tag === b[index]); -} - -function equalTagFiles(a: TagsFile | undefined, b: TagsFile | undefined): boolean { - if (a === undefined || b === undefined) { - return a === b; - } - return ( - equalTags(a.tags, b.tags) && a.updatedAt === b.updatedAt && a.tagRevision === b.tagRevision - ); -} - -function resolveComparisonRef(repoDir: string): string | undefined { - const upstream = tryRunGit(repoDir, [ - 'rev-parse', - '--abbrev-ref', - '--symbolic-full-name', - '@{upstream}', - ]); - if (upstream) return upstream; - return tryRunGit(repoDir, ['rev-parse', '--verify', 'HEAD']) ? 'HEAD' : undefined; -} - -function resolveRemoteRunMetadataPaths( - repoDir: string, - manifestPath: string, -): RemoteRunMetadataPaths { - const runsRoot = path.resolve(repoDir, RESULTS_RUNS_DIR); - const manifestDir = path.resolve(path.dirname(manifestPath)); - const runRelativePath = path.relative(runsRoot, manifestDir); - if ( - runRelativePath.length === 0 || - runRelativePath.startsWith('..') || - path.isAbsolute(runRelativePath) - ) { - throw new Error( - `Remote run manifest is outside the results repo runs directory: ${manifestPath}`, - ); - } - - const overlayTagsPath = path.join( - repoDir, - REMOTE_METADATA_RUNS_DIR, - runRelativePath, - RUN_TAGS_FILENAME, - ); - const artifactTagsPath = path.join(runsRoot, runRelativePath, RUN_TAGS_FILENAME); - - return { - runRelativePath, - artifactTagsPath, - artifactTagsGitPath: toGitPath(path.relative(repoDir, artifactTagsPath)), - overlayTagsPath, - overlayTagsGitPath: toGitPath(path.relative(repoDir, overlayTagsPath)), - }; -} - -function readRemoteRunTagsContext( - repoDir: string, - manifestPath: string, - comparisonRef = resolveComparisonRef(repoDir), -): RemoteRunTagsContext { - const paths = resolveRemoteRunMetadataPaths(repoDir, manifestPath); - const artifactTags = - readTagsFile(paths.artifactTagsPath) ?? - readTagsFromGit(repoDir, comparisonRef, paths.artifactTagsGitPath); - const baseOverlayTags = readTagsFromGit(repoDir, comparisonRef, paths.overlayTagsGitPath); - const localOverlayTags = readTagsFile(paths.overlayTagsPath); - - return { - paths, - artifactTags, - baseOverlayTags, - localOverlayTags, - }; -} - -function toRemoteRunTagState(context: RemoteRunTagsContext): RemoteRunTagState { - const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? []; - const effectiveTags = context.localOverlayTags?.tags ?? remoteTags; - const dirty = context.localOverlayTags - ? !equalTagFiles(context.localOverlayTags, context.baseOverlayTags) - : !equalTags(effectiveTags, remoteTags); - const updatedAt = - context.localOverlayTags?.updatedAt ?? - context.baseOverlayTags?.updatedAt ?? - context.artifactTags?.updatedAt; - const tagRevision = - context.localOverlayTags?.tagRevision ?? - context.baseOverlayTags?.tagRevision ?? - context.artifactTags?.tagRevision ?? - createTagRevision(effectiveTags, updatedAt); - - return { - tags: effectiveTags, - remoteTags, - ...(dirty && { pendingTags: effectiveTags }), - dirty, - updatedAt, - tagRevision, - metadataPath: context.paths.overlayTagsPath, - }; -} - -export function assertWritableResultsRepo(repoDir: string): void { - if (!existsSync(repoDir)) { - throw new Error('Writable results repo is not configured for remote metadata'); - } - const insideWorkTree = tryRunGit(repoDir, ['rev-parse', '--is-inside-work-tree']); - if (insideWorkTree !== 'true') { - throw new Error(`Configured results repo is not a writable git checkout: ${repoDir}`); - } -} - -export function isResultsRepoWorktreeDirty(repoDir: string): boolean { - if (!existsSync(repoDir)) return false; - const status = tryRunGit(repoDir, ['status', '--porcelain']); - return status !== undefined && status.trim().length > 0; -} - -export function readRemoteRunTags( - repoDir: string, - manifestPath: string, - comparisonRef?: string, -): RemoteRunTagState { - const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef); - return toRemoteRunTagState(context); -} - -export function writeRemoteRunTags( - repoDir: string, - manifestPath: string, - tags: readonly string[], - comparisonRef?: string, - expectedTagRevision?: string, -): RemoteRunTagState { - assertWritableResultsRepo(repoDir); - - const cleaned = normalizeTags(tags); - const context = readRemoteRunTagsContext(repoDir, manifestPath, comparisonRef); - const currentState = toRemoteRunTagState(context); - assertExpectedTagRevision(expectedTagRevision, currentState.tagRevision); - const remoteTags = context.baseOverlayTags?.tags ?? context.artifactTags?.tags ?? []; - - if ( - cleaned.length > 0 && - equalTags(cleaned, remoteTags) && - context.baseOverlayTags === undefined - ) { - rmSync(context.paths.overlayTagsPath, { force: true }); - return readRemoteRunTags(repoDir, manifestPath, comparisonRef); - } - - const updatedAt = new Date().toISOString(); - const entry = { - tags: cleaned, - updated_at: updatedAt, - tag_revision: createTagRevision(cleaned, updatedAt), - }; - mkdirSync(path.dirname(context.paths.overlayTagsPath), { recursive: true }); - writeFileSync(context.paths.overlayTagsPath, `${JSON.stringify(entry, null, 2)}\n`, 'utf8'); - return readRemoteRunTags(repoDir, manifestPath, comparisonRef); -} - -export function deleteRemoteRunTags( - repoDir: string, - manifestPath: string, - comparisonRef?: string, - expectedTagRevision?: string, -): RemoteRunTagState { - return writeRemoteRunTags(repoDir, manifestPath, [], comparisonRef, expectedTagRevision); -} diff --git a/apps/cli/src/commands/results/remote.ts b/apps/cli/src/commands/results/remote.ts index 6e759b863..80f03510f 100644 --- a/apps/cli/src/commands/results/remote.ts +++ b/apps/cli/src/commands/results/remote.ts @@ -24,20 +24,13 @@ import { syncResultsRepoForProject, } from '@agentv/core'; -import { RESULT_INDEX_FILENAME, relativeRunPathFromCwd } from '../eval/result-layout.js'; +import { relativeRunPathFromCwd } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; import { type ResultFileMeta, listResultFiles, listResultFilesFromRunsDir, } from '../inspect/utils.js'; -import { - type RemoteRunTagState, - assertWritableResultsRepo, - deleteRemoteRunTags, - readRemoteRunTags, - writeRemoteRunTags, -} from './remote-metadata.js'; // ── In-memory TTL cache for listGitRuns ──────────────────────────── // Avoids repeated expensive git ls-tree + git cat-file --batch operations @@ -130,29 +123,6 @@ export interface RemoteResultsStatus extends ResultsRepoStatus { readonly run_count: number; } -function relativeLocalRunPath(cwd: string, manifestPath: string): string | undefined { - const manifestDir = path.resolve(path.dirname(manifestPath)); - return relativeRunPathFromCwd(cwd, manifestDir); -} - -function remoteMetadataManifestPath( - cwd: string, - config: NormalizedResultsConfig, - meta: Pick, -): string | undefined { - if (meta.source === 'remote') { - return meta.path; - } - if (!meta.on_remote) { - return undefined; - } - const relativeRunPath = relativeLocalRunPath(cwd, meta.path); - if (!relativeRunPath) { - return undefined; - } - return path.join(config.path, 'runs', ...relativeRunPath.split('/'), RESULT_INDEX_FILENAME); -} - export interface ResultsPublishOverrides { readonly repo?: string; readonly repo_path?: string; @@ -533,78 +503,6 @@ export async function ensureRemoteRunAvailable( await materializeGitRun(config.path, relativeRunPath, getResultsStorageRef(config)); } -export async function readRemoteRunTagState( - cwd: string, - meta: Pick, - projectId?: string, -): Promise { - if (meta.source !== 'remote' && !meta.on_remote) return undefined; - const config = await loadNormalizedResultsConfig(cwd, projectId); - if (!config) return undefined; - const manifestPath = remoteMetadataManifestPath(cwd, config, meta); - if (!manifestPath) return undefined; - - try { - return readRemoteRunTags(config.path, manifestPath, getResultsStorageRef(config)); - } catch { - return undefined; - } -} - -export async function setRemoteRunTags( - cwd: string, - meta: Pick, - tags: readonly string[], - projectId?: string, - expectedTagRevision?: string, -): Promise { - if (meta.source !== 'remote' && !meta.on_remote) { - throw new Error('Remote metadata can only be set on remote runs'); - } - const config = await loadNormalizedResultsConfig(cwd, projectId); - if (!config) { - throw new Error('Writable results repo is not configured for remote metadata'); - } - const manifestPath = remoteMetadataManifestPath(cwd, config, meta); - if (!manifestPath) { - throw new Error('Remote metadata can only be set on remote runs'); - } - assertWritableResultsRepo(config.path); - return writeRemoteRunTags( - config.path, - manifestPath, - tags, - getResultsStorageRef(config), - expectedTagRevision, - ); -} - -export async function clearRemoteRunTags( - cwd: string, - meta: Pick, - projectId?: string, - expectedTagRevision?: string, -): Promise { - if (meta.source !== 'remote' && !meta.on_remote) { - throw new Error('Remote metadata can only be removed from remote runs'); - } - const config = await loadNormalizedResultsConfig(cwd, projectId); - if (!config) { - throw new Error('Writable results repo is not configured for remote metadata'); - } - const manifestPath = remoteMetadataManifestPath(cwd, config, meta); - if (!manifestPath) { - throw new Error('Remote metadata can only be removed from remote runs'); - } - assertWritableResultsRepo(config.path); - return deleteRemoteRunTags( - config.path, - manifestPath, - getResultsStorageRef(config), - expectedTagRevision, - ); -} - export async function maybeAutoExportRunArtifacts( payload: RemoteExportPayload, ): Promise { diff --git a/apps/cli/src/commands/results/run-state.ts b/apps/cli/src/commands/results/run-state.ts deleted file mode 100644 index cfc9c6588..000000000 --- a/apps/cli/src/commands/results/run-state.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createHash } from 'node:crypto'; - -export type RunFinalStateLifecycle = 'active' | 'hidden' | 'deleted'; - -export interface RunFinalState { - readonly lifecycle: RunFinalStateLifecycle; - readonly tags: string[]; -} - -export interface RunReadStateFields { - readonly final_state: RunFinalState; - readonly tag_revision: string; -} - -export class TagRevisionConflictError extends Error { - readonly expectedRevision: string; - readonly currentRevision: string; - - constructor(expectedRevision: string, currentRevision: string) { - super('Run tags changed. Refresh the run and try again.'); - this.name = 'TagRevisionConflictError'; - this.expectedRevision = expectedRevision; - this.currentRevision = currentRevision; - } -} - -export function createTagRevision(tags: readonly string[], updatedAt?: string): string { - const hash = createHash('sha256'); - hash.update('agentv.run_tags.v1\0'); - hash.update(JSON.stringify({ tags: [...tags], updated_at: updatedAt ?? '' })); - return `sha256:${hash.digest('hex')}`; -} - -export function normalizeTagRevision( - input: unknown, - tags: readonly string[], - updatedAt?: string, -): string { - return typeof input === 'string' && input.trim().length > 0 - ? input - : createTagRevision(tags, updatedAt); -} - -export function assertExpectedTagRevision( - expectedRevision: string | undefined, - currentRevision: string, -): void { - if (expectedRevision !== undefined && expectedRevision !== currentRevision) { - throw new TagRevisionConflictError(expectedRevision, currentRevision); - } -} - -export function materializeRunState(input?: { - readonly lifecycle?: RunFinalStateLifecycle; - readonly tags?: readonly string[]; - readonly tagRevision?: string; - readonly updatedAt?: string; -}): RunReadStateFields { - const tags = [...(input?.tags ?? [])]; - - return { - final_state: { - lifecycle: input?.lifecycle ?? 'active', - tags, - }, - tag_revision: input?.tagRevision ?? createTagRevision(tags, input?.updatedAt), - }; -} diff --git a/apps/cli/src/commands/results/run-tags.ts b/apps/cli/src/commands/results/run-tags.ts deleted file mode 100644 index 449827d65..000000000 --- a/apps/cli/src/commands/results/run-tags.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Per-run tag sidecar file helpers. - * - * Tags are stored as a `tags.json` sidecar next to the run manifest - * manifest. The sidecar is optional, mutable, and non-breaking — absence - * means the run has no user-assigned tags. - * - * Wire format (stored on disk): - * ```json - * { - * "tags": ["baseline", "v2-prompt"], - * "updated_at": "2026-04-10T00:00:00.000Z", - * "tag_revision": "sha256:..." - * } - * ``` - * - * Used by the Dashboard compare API so users can retroactively tag runs - * without changing the eval YAML or the run manifest itself. Tags are a - * mutable multi-valued list of free-form labels that lives alongside the - * immutable run_id. - * - * Validation rules: - * - Each tag is 1–60 characters after trimming - * - No control characters (\n, \t, DEL, etc.) - * - Tags are deduplicated case-sensitively - * - A run can have at most 20 tags - * - Writing an empty array records an intentional clear state - * - * To extend (e.g. add colored labels or descriptions): add optional fields - * to `RunTagsFile` and keep the schema additive so older files still parse. - */ - -import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; - -import { createTagRevision, normalizeTagRevision } from './run-state.js'; - -export const RUN_TAGS_FILENAME = 'tags.json'; - -/** Maximum number of tags per run. */ -export const MAX_TAGS_PER_RUN = 20; - -/** Maximum length of a single tag after trimming. */ -export const MAX_TAG_LENGTH = 60; - -export interface RunTagsFile { - /** Ordered, deduplicated list of user-assigned tags. */ - tags: string[]; - /** ISO-8601 timestamp of last update. */ - updated_at: string; - /** Optimistic-concurrency token for this materialized tag list. */ - tag_revision: string; -} - -/** Resolve the tags sidecar path given a run manifest path. */ -export function runTagsPath(manifestPath: string): string { - return path.join(path.dirname(manifestPath), RUN_TAGS_FILENAME); -} - -/** Read the tags for a run. Returns `undefined` if missing or unreadable. */ -export function readRunTags(manifestPath: string): RunTagsFile | undefined { - const fp = runTagsPath(manifestPath); - if (!existsSync(fp)) return undefined; - try { - const parsed = JSON.parse(readFileSync(fp, 'utf8')) as unknown; - if (!parsed || typeof parsed !== 'object') return undefined; - const record = parsed as Record; - if (!Array.isArray(record.tags)) return undefined; - const tags = record.tags.filter( - (t): t is string => typeof t === 'string' && t.trim().length > 0, - ); - const updatedAt = typeof record.updated_at === 'string' ? record.updated_at : ''; - return { - tags, - updated_at: updatedAt, - tag_revision: normalizeTagRevision(record.tag_revision, tags, updatedAt || undefined), - }; - } catch { - return undefined; - } -} - -/** - * Write tags for a run. Replaces any existing tags. Pass an empty array to - * record that tags were intentionally cleared while preserving the watermark. - */ -export function writeRunTags(manifestPath: string, tags: readonly string[]): RunTagsFile { - const cleaned = normalizeTags(tags); - const updatedAt = new Date().toISOString(); - const entry: RunTagsFile = { - tags: cleaned, - updated_at: updatedAt, - tag_revision: createTagRevision(cleaned, updatedAt), - }; - writeFileSync(runTagsPath(manifestPath), `${JSON.stringify(entry, null, 2)}\n`, 'utf8'); - return entry; -} - -/** Remove a run's tags sidecar. No-op if the file does not exist. */ -export function deleteRunTags(manifestPath: string): void { - const fp = runTagsPath(manifestPath); - if (existsSync(fp)) { - unlinkSync(fp); - } -} - -/** - * Trim, validate, and deduplicate an incoming tag array. Throws on any - * invalid entry so the caller can surface a user-friendly error. - */ -export function normalizeTags(tags: readonly string[]): string[] { - const seen = new Set(); - const out: string[] = []; - for (const raw of tags) { - if (typeof raw !== 'string') { - throw new Error('Tags must be strings'); - } - const trimmed = raw.trim(); - if (trimmed === '') continue; - if (trimmed.length > MAX_TAG_LENGTH) { - throw new Error(`Tag "${trimmed.slice(0, 20)}…" exceeds ${MAX_TAG_LENGTH} characters`); - } - // Reject control characters (newlines, tabs, DEL, etc.) — they break - // column headers in compare views and confuse test assertions. - for (let i = 0; i < trimmed.length; i++) { - const code = trimmed.charCodeAt(i); - if (code < 0x20 || code === 0x7f) { - throw new Error('Tag must not contain control characters'); - } - } - if (seen.has(trimmed)) continue; - seen.add(trimmed); - out.push(trimmed); - } - if (out.length > MAX_TAGS_PER_RUN) { - throw new Error(`Too many tags (max ${MAX_TAGS_PER_RUN})`); - } - return out; -} diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 3e618cfaa..f68e10bff 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -97,29 +97,19 @@ import { type ResultManifestRecord, loadLightweightResults, loadManifestResults, + normalizeTagMap, parseResultManifest, } from './manifest.js'; import { type SourcedResultFileMeta, - clearRemoteRunTags, confirmRemoteResultsMerge, ensureRemoteRunAvailable, findRunById, getRemoteResultsStatus, listMergedResultFiles, loadNormalizedResultsConfig, - readRemoteRunTagState, - setRemoteRunTags, syncRemoteResults, } from './remote.js'; -import { - type RunFinalState, - type RunReadStateFields, - TagRevisionConflictError, - assertExpectedTagRevision, - materializeRunState, -} from './run-state.js'; -import { readRunTags, writeRunTags } from './run-tags.js'; import { type StudioConfig, loadStudioConfig, saveStudioConfig } from './studio-config.js'; // ── Source resolution ──────────────────────────────────────────────────── @@ -1239,15 +1229,6 @@ interface DataContext { projectId?: string; } -interface RunTagFields { - readonly tags?: string[]; - readonly remote_tags?: string[]; - readonly pending_tags?: string[]; - readonly metadata_dirty?: boolean; - readonly final_state: RunFinalState; - readonly tag_revision: string; -} - // biome-ignore lint/suspicious/noExplicitAny: Hono Context generic varies by route type C = Context; @@ -1263,143 +1244,6 @@ function inferExperimentFromRunId(runId: string): string | undefined { return experiment; } -async function readRunTagFields( - searchDir: string, - meta: SourcedResultFileMeta, - projectId?: string, -): Promise { - if (meta.on_remote) { - const state = await readRemoteRunTagState(searchDir, meta, projectId); - if (state) { - return { - tags: state.tags, - remote_tags: state.remoteTags, - metadata_dirty: state.dirty, - ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), - ...materializeRunState({ - tags: state.tags, - tagRevision: state.tagRevision, - updatedAt: state.updatedAt, - }), - }; - } - } - - if (meta.source === 'local') { - const tagsEntry = readRunTags(meta.path); - const runState = materializeRunState({ - tags: tagsEntry?.tags ?? [], - tagRevision: tagsEntry?.tag_revision, - updatedAt: tagsEntry?.updated_at || undefined, - }); - return { - ...(tagsEntry ? { tags: tagsEntry.tags } : {}), - ...runState, - }; - } - - const state = await readRemoteRunTagState(searchDir, meta, projectId); - if (!state) { - return { - tags: [], - remote_tags: [], - metadata_dirty: false, - ...materializeRunState({ tags: [] }), - }; - } - - return { - tags: state.tags, - remote_tags: state.remoteTags, - metadata_dirty: state.dirty, - ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), - ...materializeRunState({ - tags: state.tags, - tagRevision: state.tagRevision, - updatedAt: state.updatedAt, - }), - }; -} - -function remoteTagMutationResponse(state: { - readonly tags: string[]; - readonly remoteTags: string[]; - readonly pendingTags?: string[]; - readonly dirty: boolean; - readonly updatedAt?: string; - readonly tagRevision: string; -}) { - return { - tags: state.tags, - remote_tags: state.remoteTags, - metadata_dirty: state.dirty, - ...(state.dirty && { pending_tags: state.pendingTags ?? state.tags }), - ...materializeRunState({ - tags: state.tags, - tagRevision: state.tagRevision, - updatedAt: state.updatedAt, - }), - updated_at: state.updatedAt ?? new Date().toISOString(), - }; -} - -function localTagMutationResponse(input: { - readonly tags: readonly string[]; - readonly updatedAt?: string; - readonly tagRevision?: string; -}): RunReadStateFields { - return materializeRunState({ - tags: input.tags, - tagRevision: input.tagRevision, - updatedAt: input.updatedAt, - }); -} - -function remoteMetadataErrorStatus(error: unknown): 400 | 409 { - if (error instanceof TagRevisionConflictError) { - return 409; - } - const message = error instanceof Error ? error.message : String(error); - if ( - message.includes('not configured') || - message.includes('not a writable git checkout') || - message.includes('outside the results repo runs directory') - ) { - return 409; - } - return 400; -} - -function tagMutationErrorBody(error: unknown) { - const message = error instanceof Error ? error.message : String(error); - if (error instanceof TagRevisionConflictError) { - return { - error: message, - expected_tag_revision: error.expectedRevision, - current_tag_revision: error.currentRevision, - }; - } - return { error: message }; -} - -function expectedTagRevisionFromRecord(record: Record): string | undefined { - const raw = record.expected_tag_revision ?? record.tag_revision ?? record.etag; - if (raw === undefined) return undefined; - if (typeof raw !== 'string') { - throw new Error('expected_tag_revision must be a string'); - } - return raw; -} - -function currentLocalTagRevision(manifestPath: string): string { - const tagsEntry = readRunTags(manifestPath); - return materializeRunState({ - tags: tagsEntry?.tags ?? [], - tagRevision: tagsEntry?.tag_revision, - updatedAt: tagsEntry?.updated_at || undefined, - }).tag_revision; -} - async function ensureRunReadable( searchDir: string, meta: SourcedResultFileMeta, @@ -1573,6 +1417,7 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext let target = m.target; let experiment = m.experiment ?? inferExperimentFromRunId(m.raw_filename); const summaryMetadata = readRunSummaryMetadataForDashboard(m.path); + let runTags: Record | undefined = summaryMetadata.tags; let runtimeSource: RunRuntimeSourceMetadata | undefined = summaryMetadata.runtimeSource; let timestamp = m.timestamp; let testCount = m.testCount; @@ -1587,6 +1432,7 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext const qualitySummary = summarizeQualityResults(records, passThreshold); target = records[0].target; experiment = records[0].experiment ?? experiment; + runTags = records[0].tags ?? runTags; timestamp = hasUsableTimestamp(timestamp) || !records[0].timestamp ? timestamp @@ -1617,7 +1463,6 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext // or running so the RunList can render a spinner instead of the // pass/fail dot derived from a 0% pass rate. const liveStatus = getActiveRunStatus(m.path); - const tagFields = await readRunTagFields(searchDir, m, projectId); return { filename: m.filename, display_name: m.displayName, @@ -1632,8 +1477,8 @@ async function handleRuns(c: C, { searchDir, agentvDir, projectId }: DataContext on_remote: m.on_remote, ...(target && { target }), ...(experiment && { experiment }), + ...(runTags && { run_tags: runTags }), ...(runtimeSource && { runtime_source: runtimeSource }), - ...tagFields, ...(liveStatus && { status: liveStatus }), }; }), @@ -1684,7 +1529,6 @@ async function handleRunDetail(c: C, { searchDir, projectId }: DataContext) { const resumeMeta = meta.source === 'local' ? deriveResumeMeta(searchDir, meta.path, summaryMetadata) : {}; const liveStatus = meta.source === 'local' ? getActiveRunStatus(meta.path) : undefined; - const tagFields = await readRunTagFields(searchDir, meta, projectId); const baseDir = path.dirname(meta.path); return c.json({ results: attachExternalTraceFields( @@ -1694,7 +1538,6 @@ async function handleRunDetail(c: C, { searchDir, projectId }: DataContext) { source: meta.source, source_label: meta.displayName, ...(runtimeSource && { runtime_source: runtimeSource }), - ...tagFields, ...(liveStatus && { status: liveStatus }), ...resumeMeta, }); @@ -1746,6 +1589,7 @@ function attachExternalTraceFields>( interface RunSummaryMetadataForDashboard { readonly evalFile?: string; readonly experiment?: string; + readonly tags?: Record; readonly plannedTestCount?: number; readonly runtimeSource?: RunRuntimeSourceMetadata; } @@ -1760,16 +1604,19 @@ function readRunSummaryMetadataForDashboard(manifestPath: string): RunSummaryMet metadata?: { eval_file?: string; experiment?: string; + tags?: Record; planned_test_count?: number; runtime_source?: RunRuntimeSourceMetadata; }; }; const planned = parsed.metadata?.planned_test_count; + const tags = normalizeTagMap(parsed.metadata?.tags); return { ...(typeof parsed.metadata?.eval_file === 'string' && parsed.metadata.eval_file.trim() && { evalFile: parsed.metadata.eval_file.trim() }), ...(typeof parsed.metadata?.experiment === 'string' && parsed.metadata.experiment.trim() && { experiment: parsed.metadata.experiment.trim() }), + ...(tags && { tags }), ...(typeof planned === 'number' && Number.isFinite(planned) && planned > 0 && { plannedTestCount: planned }), @@ -2367,10 +2214,122 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) } } -async function handleExperiments(c: C, { searchDir, agentvDir, projectId }: DataContext) { +/** + * The reserved tag key that also feeds the run-level experiment namespace. Runs + * written before the promptfoo `tags` map shipped carry only a top-level + * `experiment`, so grouping on this key falls back to + * `record.experiment ?? record.tags?.experiment ?? 'default'`. + */ +const EXPERIMENT_TAG_KEY = 'experiment'; + +/** Bucket label for runs whose rows do not carry the selected tag key. */ +function noKeyBucketLabel(key: string): string { + return `(no ${key})`; +} + +/** + * Resolve the grouping value a record contributes for a given tag key. For the + * reserved `experiment` key we honour the lockstep fallback so old runs (no tags + * map) still resolve; for any other key we read `record.tags?.[key]` and place + * records missing the key in a `(no )` bucket. + */ +function resolveTagGroupValue( + record: { readonly experiment?: string; readonly tags?: Record }, + key: string, +): string { + if (key === EXPERIMENT_TAG_KEY) { + return record.experiment ?? record.tags?.experiment ?? 'default'; + } + const value = record.tags?.[key]; + return value === undefined || value === '' ? noKeyBucketLabel(key) : value; +} + +/** + * Resolve the single run-level `{experiment, tags}` pair a run contributes, + * mirroring exactly the resolution `handleRuns` uses to build each `RunMeta` + * (`experiment` and `run_tags`). Because promptfoo tags are run-level (every row + * of a run carries the same map), grouping and the run-list detail filter must + * both key off this one source so a run lands in exactly one group per key and + * the group card's `run_count` matches the detail view's filtered runs. + */ +function resolveRunLevelTagFields( + meta: SourcedResultFileMeta, + records: readonly { readonly experiment?: string; readonly tags?: Record }[], + summaryMetadata: RunSummaryMetadataForDashboard, +): { experiment?: string; tags?: Record } { + const inferredExperiment = meta.experiment ?? inferExperimentFromRunId(meta.raw_filename); + const experiment = records[0]?.experiment ?? inferredExperiment; + const tags = records[0]?.tags ?? summaryMetadata.tags; + return { + ...(experiment !== undefined && { experiment }), + ...(tags !== undefined && { tags }), + }; +} + +/** + * Generalized `GET /api/tags`. Without `?key=` it enumerates the available tag + * keys (union of every row's `tags` map keys plus the synthetic `experiment` + * key, sorted). With `?key=` it groups runs by that key's values and returns + * per-value summaries (the shape the old `/api/experiments` returned, with + * `name` = the tag value). + */ +async function handleTags(c: C, { searchDir, agentvDir, projectId }: DataContext) { const { runs: metas } = await listMergedResultFiles(searchDir, undefined, projectId); const { threshold: pass_threshold } = loadStudioConfig(agentvDir); - const experimentMap = new Map< + const rawKey = c.req.query('key'); + const key = rawKey?.trim() ? rawKey.trim() : undefined; + + // Key enumeration mode: union each run's tag keys, always include + // `experiment`. Read the run-level `metadata.tags` from each run's + // `summary.json` (cheap) rather than parsing every run's full JSONL — promptfoo + // tags are run-level, so the summary map carries the same keys. Returned sorted + // for a stable dropdown. + if (!key) { + const keys = new Set([EXPERIMENT_TAG_KEY]); + for (const m of metas) { + try { + await ensureRunReadable(searchDir, m, projectId); + const summaryMetadata = readRunSummaryMetadataForDashboard(m.path); + for (const tagKey of Object.keys(summaryMetadata.tags ?? {})) { + keys.add(tagKey); + } + } catch { + // skip runs that fail to load + } + } + return c.json({ keys: [...keys].sort() }); + } + + const groups = await aggregateTagGroups(metas, key, { searchDir, projectId, pass_threshold }); + return c.json({ key, groups }); +} + +/** + * Group runs by a tag key's values and return per-value summaries. Grouping is + * per-run: each run resolves exactly ONE group value from its run-level + * `{experiment, tags}` (via {@link resolveRunLevelTagFields}), matching the + * `RunMeta` the run-list detail filter keys off, so `run_count` per group equals + * the detail view's filtered run count. Per-row metrics (targets, evals, pass + * rate, last run) are then accumulated into that one group. + */ +async function aggregateTagGroups( + metas: readonly SourcedResultFileMeta[], + key: string, + opts: { searchDir: string; projectId?: string; pass_threshold: number }, +): Promise< + Array<{ + name: string; + run_count: number; + target_count: number; + eval_count: number; + quality_count: number; + passed_count: number; + execution_error_count: number; + pass_rate: number; + last_run: string | null; + }> +> { + const groupMap = new Map< string, { targets: Set; @@ -2385,38 +2344,42 @@ async function handleExperiments(c: C, { searchDir, agentvDir, projectId }: Data for (const m of metas) { try { - const records = await loadLightweightResultsForMeta(searchDir, m, projectId); + const records = await loadLightweightResultsForMeta(opts.searchDir, m, opts.projectId); + const summaryMetadata = readRunSummaryMetadataForDashboard(m.path); + const value = resolveTagGroupValue( + resolveRunLevelTagFields(m, records, summaryMetadata), + key, + ); + const entry = groupMap.get(value) ?? { + targets: new Set(), + runFilenames: new Set(), + evalCount: 0, + qualityCount: 0, + passedCount: 0, + executionErrorCount: 0, + lastTimestamp: '', + }; + entry.runFilenames.add(m.filename); for (const r of records) { - const experiment = r.experiment ?? 'default'; - const entry = experimentMap.get(experiment) ?? { - targets: new Set(), - runFilenames: new Set(), - evalCount: 0, - qualityCount: 0, - passedCount: 0, - executionErrorCount: 0, - lastTimestamp: '', - }; - entry.runFilenames.add(m.filename); if (r.target) entry.targets.add(r.target); entry.evalCount++; if (isExecutionErrorResult(r)) { entry.executionErrorCount++; } else { entry.qualityCount++; - if (r.score >= pass_threshold) entry.passedCount++; + if (r.score >= opts.pass_threshold) entry.passedCount++; } if (r.timestamp && r.timestamp > entry.lastTimestamp) { entry.lastTimestamp = r.timestamp; } - experimentMap.set(experiment, entry); } + groupMap.set(value, entry); } catch { // skip runs that fail to load } } - const experiments = [...experimentMap.entries()].map(([name, entry]) => ({ + return [...groupMap.entries()].map(([name, entry]) => ({ name, run_count: entry.runFilenames.size, target_count: entry.targets.size, @@ -2427,7 +2390,21 @@ async function handleExperiments(c: C, { searchDir, agentvDir, projectId }: Data pass_rate: entry.qualityCount > 0 ? entry.passedCount / entry.qualityCount : 0, last_run: entry.lastTimestamp || null, })); +} +/** + * Backward-compatible `GET /api/experiments` alias. Serves the legacy + * `{ experiments: ExperimentSummary[] }` shape by grouping on the `experiment` + * key, so existing clients keep working during the Tags-tab migration. + */ +async function handleExperiments(c: C, ctx: DataContext) { + const { runs: metas } = await listMergedResultFiles(ctx.searchDir, undefined, ctx.projectId); + const { threshold: pass_threshold } = loadStudioConfig(ctx.agentvDir); + const experiments = await aggregateTagGroups(metas, EXPERIMENT_TAG_KEY, { + searchDir: ctx.searchDir, + projectId: ctx.projectId, + pass_threshold, + }); return c.json({ experiments }); } @@ -2435,19 +2412,6 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont const { runs: metas } = await listMergedResultFiles(searchDir, undefined, projectId); const { threshold: pass_threshold } = loadStudioConfig(agentvDir); - // Optional tag filter: `?tags=baseline,v2-prompt` keeps only runs that - // carry at least one of the given tags (OR semantics). Empty / missing - // param is a no-op. Filtering is applied before aggregation so it - // propagates through `cells[]`, `runs[]`, `experiments[]`, and - // `targets[]` uniformly. - const tagsParam = c.req.query('tags') ?? ''; - const filterTags = new Set( - tagsParam - .split(',') - .map((t) => t.trim()) - .filter(Boolean), - ); - type CompareTestEntry = { test_id: string; category?: string; @@ -2478,12 +2442,6 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont started_at: string; experiment: string; target: string; - tags?: string[]; - remote_tags?: string[]; - pending_tags?: string[]; - metadata_dirty?: boolean; - final_state: RunFinalState; - tag_revision: string; source: 'local' | 'remote'; eval_count: number; quality_count: number; @@ -2500,14 +2458,6 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont for (const m of metas) { try { - // Read tags before any heavy work so the `?tags=` filter can skip - // non-matching runs without loading their JSONL records. - const tagFields = await readRunTagFields(searchDir, m, projectId); - if (filterTags.size > 0) { - const runTags = tagFields.tags ?? []; - if (!runTags.some((t) => filterTags.has(t))) continue; - } - const records = await loadLightweightResultsForMeta(searchDir, m, projectId); const runTestMap = new Map(); let runEvalCount = 0; @@ -2520,7 +2470,10 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont let runStartedAt = m.timestamp; for (const r of records) { - const experiment = r.experiment ?? 'default'; + // Resolve experiment with the same lockstep tags.experiment fallback the + // Tags tab uses (`resolveTagGroupValue` on the reserved key), so a run + // never appears under a different experiment name across tabs. + const experiment = resolveTagGroupValue(r, EXPERIMENT_TAG_KEY); const target = r.target ?? 'default'; experimentsSet.add(experiment); targetsSet.add(target); @@ -2584,7 +2537,6 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont started_at: runStartedAt, experiment: runExperiment, target: runTarget, - ...tagFields, source: m.source, eval_count: runEvalCount, quality_count: runQualityCount, @@ -2907,105 +2859,6 @@ function directoryBrowseResultToWire(result: DirectoryBrowseResult) { }; } -async function handleRunTagsPut(c: C, { searchDir, projectId }: DataContext) { - const filename = c.req.param('filename') ?? ''; - const meta = await findRunById(searchDir, filename, projectId); - if (!meta) return c.json({ error: 'Run not found' }, 404); - let body: unknown; - try { - body = await c.req.json(); - } catch { - return c.json({ error: 'Invalid JSON' }, 400); - } - if (!body || typeof body !== 'object') { - return c.json({ error: 'Invalid payload' }, 400); - } - const tags = (body as Record).tags; - if (!Array.isArray(tags)) { - return c.json({ error: 'Missing tags array' }, 400); - } - let expectedTagRevision: string | undefined; - try { - expectedTagRevision = expectedTagRevisionFromRecord(body as Record); - } catch (err) { - return c.json({ error: (err as Error).message }, 400); - } - try { - if (meta.on_remote) { - const state = await setRemoteRunTags( - searchDir, - meta, - tags as string[], - projectId, - expectedTagRevision, - ); - return c.json(remoteTagMutationResponse(state)); - } - - assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path)); - const entry = writeRunTags(meta.path, tags as string[]); - const responseState = localTagMutationResponse({ - tags: entry?.tags ?? [], - updatedAt: entry?.updated_at, - tagRevision: entry?.tag_revision, - }); - return c.json({ - tags: entry?.tags ?? [], - ...responseState, - updated_at: entry?.updated_at ?? new Date().toISOString(), - }); - } catch (err) { - return c.json(tagMutationErrorBody(err), remoteMetadataErrorStatus(err)); - } -} - -async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext) { - const filename = c.req.param('filename') ?? ''; - const meta = await findRunById(searchDir, filename, projectId); - if (!meta) return c.json({ error: 'Run not found' }, 404); - let expectedTagRevision: string | undefined; - try { - const text = await c.req.text(); - if (text.trim().length > 0) { - const body = JSON.parse(text) as unknown; - if (!body || typeof body !== 'object') { - return c.json({ error: 'Invalid payload' }, 400); - } - expectedTagRevision = expectedTagRevisionFromRecord(body as Record); - } - } catch (err) { - return c.json( - { error: err instanceof SyntaxError ? 'Invalid JSON' : (err as Error).message }, - 400, - ); - } - try { - if (meta.on_remote) { - const state = await clearRemoteRunTags(searchDir, meta, projectId, expectedTagRevision); - return c.json({ - ok: true, - ...remoteTagMutationResponse(state), - }); - } - - assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path)); - const entry = writeRunTags(meta.path, []); - const responseState = localTagMutationResponse({ - tags: entry.tags, - updatedAt: entry.updated_at, - tagRevision: entry.tag_revision, - }); - return c.json({ - ok: true, - tags: entry.tags, - ...responseState, - updated_at: entry.updated_at, - }); - } catch (err) { - return c.json(tagMutationErrorBody(err), remoteMetadataErrorStatus(err)); - } -} - async function handleRunDelete(c: C, { searchDir, projectId }: DataContext) { const filename = c.req.param('filename') ?? ''; const meta = await findRunById(searchDir, filename, projectId); @@ -3106,7 +2959,6 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { { ids: runIds, displayNames: metas.map((meta) => meta.displayName), - tags: metas.map((meta) => readRunTags(meta.path)?.tags ?? []), }, ); const combined = combineRunSources({ @@ -3116,8 +2968,6 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { displayName, duplicatePolicy: duplicatePolicy as Exclude, }); - const tagEntry = - combined.tags.length > 0 ? writeRunTags(combined.manifestPath, combined.tags) : undefined; return c.json( { ok: true, @@ -3126,7 +2976,6 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { experiment: combined.experiment, combined_from_run_ids: combined.combinedFromRunIds, duplicate_conflicts: combined.duplicateConflicts, - ...(tagEntry && { tags: tagEntry.tags }), }, 201, ); @@ -3344,12 +3193,6 @@ export function createApp( target?: string; experiment?: string; runtime_source?: RunRuntimeSourceMetadata; - tags?: string[]; - remote_tags?: string[]; - pending_tags?: string[]; - metadata_dirty?: boolean; - final_state: RunFinalState; - tag_revision: string; source: 'local' | 'remote'; project_id: string; }> = []; @@ -3388,7 +3231,6 @@ export function createApp( } catch { // ignore enrichment errors } - const tagFields = await readRunTagFields(p.path, m, p.id); allRuns.push({ filename: m.filename, display_name: m.displayName, @@ -3403,7 +3245,6 @@ export function createApp( ...(target && { target }), ...(experiment && { experiment }), ...(runtimeSource && { runtime_source: runtimeSource }), - ...tagFields, project_id: p.id, }); } @@ -3450,18 +3291,6 @@ export function createApp( } return handleRunsCombine(c, defaultCtx); }); - app.put('/api/runs/:filename/tags', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return handleRunTagsPut(c, defaultCtx); - }); - app.delete('/api/runs/:filename/tags', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return handleRunTagsDelete(c, defaultCtx); - }); app.delete('/api/runs/:filename', (c) => { if (readOnly) { return c.json({ error: 'Dashboard is running in read-only mode' }, 403); @@ -3484,6 +3313,7 @@ export function createApp( ); app.get('/api/runs/:filename/evals/:evalId/files', (c) => handleEvalFiles(c, defaultCtx)); app.get('/api/runs/:filename/evals/:evalId/files/*', (c) => handleEvalFileContent(c, defaultCtx)); + app.get('/api/tags', (c) => handleTags(c, defaultCtx)); app.get('/api/experiments', (c) => handleExperiments(c, defaultCtx)); app.get('/api/compare', (c) => handleCompare(c, defaultCtx)); app.get('/api/targets', (c) => handleTargets(c, defaultCtx)); @@ -3577,18 +3407,6 @@ export function createApp( } return withProject(c, handleRunsCombine); }); - app.put('/api/projects/:projectId/runs/:filename/tags', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return withProject(c, handleRunTagsPut); - }); - app.delete('/api/projects/:projectId/runs/:filename/tags', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return withProject(c, handleRunTagsDelete); - }); app.delete('/api/projects/:projectId/runs/:filename', (c) => { if (readOnly) { return c.json({ error: 'Dashboard is running in read-only mode' }, 403); @@ -3619,6 +3437,7 @@ export function createApp( app.get('/api/projects/:projectId/runs/:filename/evals/:evalId/files/*', (c) => withProject(c, handleEvalFileContent), ); + app.get('/api/projects/:projectId/tags', (c) => withProject(c, handleTags)); app.get('/api/projects/:projectId/experiments', (c) => withProject(c, handleExperiments)); app.get('/api/projects/:projectId/compare', (c) => withProject(c, handleCompare)); app.get('/api/projects/:projectId/targets', (c) => withProject(c, handleTargets)); diff --git a/apps/cli/test/commands/results/remote-metadata.test.ts b/apps/cli/test/commands/results/remote-metadata.test.ts deleted file mode 100644 index 94280e7fe..000000000 --- a/apps/cli/test/commands/results/remote-metadata.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { execSync } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; - -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; - -import { - deleteRemoteRunTags, - isResultsRepoWorktreeDirty, - readRemoteRunTags, - writeRemoteRunTags, -} from '../../../src/commands/results/remote-metadata.js'; - -const RUN_TIMESTAMP = '2026-06-06T10-00-00-000Z'; - -function cleanGitEnv(): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined && !(key.startsWith('GIT_') && key !== 'GIT_SSH_COMMAND')) { - env[key] = value; - } - } - return env; -} - -function git(cmd: string, cwd: string): string { - return execSync(cmd, { - cwd, - encoding: 'utf8', - env: cleanGitEnv(), - stdio: ['ignore', 'pipe', 'pipe'], - }).trim(); -} - -function seedRepo( - repoDir: string, - options?: { readonly artifactTags?: readonly string[] }, -): string { - git('git init --quiet', repoDir); - git('git config user.email "test@example.com"', repoDir); - git('git config user.name "Test User"', repoDir); - - const runDir = path.join(repoDir, 'runs', 'default', RUN_TIMESTAMP); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha","score":1}\n'); - const artifactTags = options?.artifactTags ?? ['remote-baseline']; - if (artifactTags.length > 0) { - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify( - { tags: artifactTags, updated_at: '2026-06-06T09:00:00.000Z' }, - null, - 2, - )}\n`, - ); - } - git('git add runs', repoDir); - git('git commit --quiet -m "seed remote run"', repoDir); - return path.join(runDir, 'index.jsonl'); -} - -describe('remote metadata tags', () => { - let repoDir: string; - - beforeEach(() => { - repoDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-remote-metadata-test-')); - }); - - afterEach(() => { - rmSync(repoDir, { recursive: true, force: true }); - }); - - it('writes tag edits as a metadata overlay without mutating the run artifact', () => { - const manifestPath = seedRepo(repoDir); - const artifactTagsPath = path.join(path.dirname(manifestPath), 'tags.json'); - const originalArtifactTags = readFileSync(artifactTagsPath, 'utf8'); - - const state = writeRemoteRunTags(repoDir, manifestPath, ['pending', 'remote-baseline']); - - expect(state.tags).toEqual(['pending', 'remote-baseline']); - expect(state.remoteTags).toEqual(['remote-baseline']); - expect(state.pendingTags).toEqual(['pending', 'remote-baseline']); - expect(state.dirty).toBe(true); - expect(state.tagRevision).toStartWith('sha256:'); - expect(state.metadataPath).toContain( - path.join('metadata', 'runs', 'default', RUN_TIMESTAMP, 'tags.json'), - ); - expect(readFileSync(artifactTagsPath, 'utf8')).toBe(originalArtifactTags); - expect(existsSync(state.metadataPath)).toBe(true); - expect(isResultsRepoWorktreeDirty(repoDir)).toBe(true); - - const reloaded = readRemoteRunTags(repoDir, manifestPath); - expect(reloaded.tags).toEqual(['pending', 'remote-baseline']); - expect(reloaded.pendingTags).toEqual(['pending', 'remote-baseline']); - expect(reloaded.dirty).toBe(true); - expect(reloaded.tagRevision).toBe(state.tagRevision); - }); - - it('uses committed metadata overlays as the clean remote baseline', () => { - const manifestPath = seedRepo(repoDir); - const state = writeRemoteRunTags(repoDir, manifestPath, ['accepted']); - git('git add metadata', repoDir); - git('git commit --quiet -m "update tags"', repoDir); - - const reloaded = readRemoteRunTags(repoDir, manifestPath); - - expect(state.dirty).toBe(true); - expect(reloaded.tags).toEqual(['accepted']); - expect(reloaded.remoteTags).toEqual(['accepted']); - expect(reloaded.pendingTags).toBeUndefined(); - expect(reloaded.dirty).toBe(false); - expect(reloaded.tagRevision).toBe(state.tagRevision); - }); - - it('preserves the storage ref when unchanged remote artifact tags clear a local overlay', () => { - git('git init --quiet', repoDir); - git('git config user.email "test@example.com"', repoDir); - git('git config user.name "Test User"', repoDir); - writeFileSync(path.join(repoDir, 'README.md'), '# source branch\n'); - git('git add README.md', repoDir); - git('git commit --quiet -m "seed source branch"', repoDir); - git('git branch -M main', repoDir); - - git('git checkout --quiet -b agentv/results/v1', repoDir); - const runDir = path.join(repoDir, 'runs', 'default', RUN_TIMESTAMP); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), '{"test_id":"alpha","score":1}\n'); - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify({ tags: ['remote'], updated_at: '2026-06-06T09:00:00.000Z' }, null, 2)}\n`, - ); - git('git add runs', repoDir); - git('git commit --quiet -m "seed result artifact"', repoDir); - git('git checkout --quiet main', repoDir); - - const manifestPath = path.join(runDir, 'index.jsonl'); - const state = writeRemoteRunTags(repoDir, manifestPath, ['remote'], 'agentv/results/v1'); - - expect(state.tags).toEqual(['remote']); - expect(state.remoteTags).toEqual(['remote']); - expect(state.pendingTags).toBeUndefined(); - expect(state.dirty).toBe(false); - expect(existsSync(state.metadataPath)).toBe(false); - }); - - it('persists clearing remote tags as an empty pending overlay', () => { - const manifestPath = seedRepo(repoDir); - - const state = deleteRemoteRunTags(repoDir, manifestPath); - - expect(state.tags).toEqual([]); - expect(state.remoteTags).toEqual(['remote-baseline']); - expect(state.pendingTags).toEqual([]); - expect(state.dirty).toBe(true); - expect(readFileSync(state.metadataPath, 'utf8')).toContain('"tags": []'); - }); - - it('records an explicit clear revision when the remote baseline is already empty', () => { - const manifestPath = seedRepo(repoDir, { artifactTags: [] }); - - const state = writeRemoteRunTags(repoDir, manifestPath, []); - const metadata = JSON.parse(readFileSync(state.metadataPath, 'utf8')) as { - tags: string[]; - tag_revision: string; - }; - - expect(state.tags).toEqual([]); - expect(state.remoteTags).toEqual([]); - expect(state.pendingTags).toEqual([]); - expect(state.dirty).toBe(true); - expect(state.tagRevision).toStartWith('sha256:'); - expect(metadata.tags).toEqual([]); - expect(metadata.tag_revision).toBe(state.tagRevision); - }); - - it('rejects stale tag revisions before writing an overlay', () => { - const manifestPath = seedRepo(repoDir); - const before = readRemoteRunTags(repoDir, manifestPath); - - expect(() => - writeRemoteRunTags(repoDir, manifestPath, ['stale'], undefined, 'sha256:stale'), - ).toThrow('Run tags changed. Refresh the run and try again.'); - - const after = readRemoteRunTags(repoDir, manifestPath); - expect(after.tagRevision).toBe(before.tagRevision); - expect(after.tags).toEqual(before.tags); - }); - - it('rejects writes when the configured results path is not a git checkout', () => { - const runDir = path.join(repoDir, 'runs', 'default', RUN_TIMESTAMP); - mkdirSync(runDir, { recursive: true }); - const manifestPath = path.join(runDir, 'index.jsonl'); - writeFileSync(manifestPath, '{"test_id":"alpha","score":1}\n'); - - expect(() => writeRemoteRunTags(repoDir, manifestPath, ['blocked'])).toThrow( - 'not a writable git checkout', - ); - }); -}); diff --git a/apps/cli/test/commands/results/run-tags.test.ts b/apps/cli/test/commands/results/run-tags.test.ts deleted file mode 100644 index 7540ee586..000000000 --- a/apps/cli/test/commands/results/run-tags.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; - -import { - deleteRunTags, - readRunTags, - runTagsPath, - writeRunTags, -} from '../../../src/commands/results/run-tags.js'; - -describe('run tags sidecar', () => { - let tempDir: string; - let manifestPath: string; - - beforeEach(() => { - tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-run-tags-')); - const runDir = path.join(tempDir, '.agentv', 'results', 'default', '2026-clear-tags'); - mkdirSync(runDir, { recursive: true }); - manifestPath = path.join(runDir, 'index.jsonl'); - writeFileSync(manifestPath, '{"test_id":"alpha","score":1}\n', 'utf8'); - }); - - afterEach(() => { - rmSync(tempDir, { recursive: true, force: true }); - }); - - it('records empty tags as an explicit clear state with a tag revision', () => { - writeRunTags(manifestPath, ['baseline']); - - const cleared = writeRunTags(manifestPath, []); - const reloaded = readRunTags(manifestPath); - - expect(existsSync(runTagsPath(manifestPath))).toBe(true); - expect(cleared.tags).toEqual([]); - expect(cleared.tag_revision).toStartWith('sha256:'); - expect(reloaded).toEqual(cleared); - expect(readFileSync(runTagsPath(manifestPath), 'utf8')).toContain('"tags": []'); - expect(readFileSync(runTagsPath(manifestPath), 'utf8')).toContain('"tag_revision": "sha256:'); - }); - - it('changes tag_revision after replacement writes', () => { - const first = writeRunTags(manifestPath, ['baseline']); - const second = writeRunTags(manifestPath, ['candidate']); - - expect(first.tag_revision).toStartWith('sha256:'); - expect(second.tag_revision).toStartWith('sha256:'); - expect(second.tag_revision).not.toBe(first.tag_revision); - }); - - it('keeps physical sidecar deletion explicit', () => { - writeRunTags(manifestPath, []); - - deleteRunTags(manifestPath); - - expect(existsSync(runTagsPath(manifestPath))).toBe(false); - expect(readRunTags(manifestPath)).toBeUndefined(); - }); -}); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 87dbf36f7..98a08f30b 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -16,7 +16,6 @@ import { fileURLToPath } from 'node:url'; import { AGENTV_RESULTS_ARTIFACTS_REF, addProject, saveProjectRegistry } from '@agentv/core'; -import { createTagRevision } from '../../../src/commands/results/run-state.js'; import { createApp, loadResults, @@ -325,21 +324,6 @@ function writeDirtyRemoteRunArtifact( return timestamp; } -function writeRemoteTagMetadataOverlay( - repoDir: string, - experiment: string, - timestamp: string, - tags: readonly string[], -): string { - const metadataPath = path.join(repoDir, 'metadata', 'runs', timestamp, 'tags.json'); - mkdirSync(path.dirname(metadataPath), { recursive: true }); - writeFileSync( - metadataPath, - `${JSON.stringify({ tags, updated_at: '2026-06-06T12:00:00.000Z' }, null, 2)}\n`, - ); - return metadataPath; -} - function writeLocalRunArtifact( projectDir: string, experiment: string, @@ -1042,7 +1026,7 @@ describe('serve app', () => { }); }); - it('tags local runs with source metadata', async () => { + it('annotates local runs with source metadata', async () => { const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); const filename = '2026-03-25T10-00-00-000Z'; @@ -1059,8 +1043,6 @@ describe('serve app', () => { filename: string; source: string; on_remote: boolean; - final_state: { lifecycle: string; tags: string[] }; - tag_revision: string; }>; }; expect(data.runs).toHaveLength(1); @@ -1069,12 +1051,7 @@ describe('serve app', () => { filename, source: 'local', on_remote: false, - final_state: { - lifecycle: 'active', - tags: [], - }, }); - expect(data.runs[0].tag_revision).toStartWith('sha256:'); }); it('exposes experiment namespace and runtime source metadata for run list cards', async () => { @@ -1194,185 +1171,6 @@ describe('serve app', () => { expect(serialized).not.toContain('token='); }); - it('exposes materialized final state and tag revision for local run tags', async () => { - const runsDir = localResultsExperimentDir(tempDir); - mkdirSync(runsDir, { recursive: true }); - const filename = '2026-03-25T10-00-00-000Z'; - const runDir = path.join(runsDir, filename); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); - const updatedAt = '2026-06-21T10:15:00.000Z'; - const tagRevision = createTagRevision(['accepted'], updatedAt); - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify( - { - tags: ['accepted'], - updated_at: updatedAt, - tag_revision: tagRevision, - }, - null, - 2, - )}\n`, - ); - - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const listRes = await app.request('/api/runs'); - expect(listRes.status).toBe(200); - const listData = (await listRes.json()) as { - runs: Array<{ - tags: string[]; - final_state: { lifecycle: string; tags: string[] }; - tag_revision: string; - }>; - }; - expect(listData.runs[0]).toMatchObject({ - tags: ['accepted'], - final_state: { - lifecycle: 'active', - tags: ['accepted'], - }, - tag_revision: tagRevision, - }); - - const detailRes = await app.request(`/api/runs/${encodeURIComponent(filename)}`); - expect(detailRes.status).toBe(200); - const detailData = (await detailRes.json()) as { - tags: string[]; - final_state: { lifecycle: string; tags: string[] }; - tag_revision: string; - }; - expect(detailData).toMatchObject({ - tags: ['accepted'], - final_state: { - lifecycle: 'active', - tags: ['accepted'], - }, - tag_revision: tagRevision, - }); - }); - - it('preserves a local tag clear state after DELETE /tags', async () => { - const runsDir = localResultsExperimentDir(tempDir); - mkdirSync(runsDir, { recursive: true }); - const filename = '2026-03-25T10-30-00-000Z'; - const runDir = path.join(runsDir, filename); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); - const updatedAt = '2026-06-21T10:15:00.000Z'; - const tagRevision = createTagRevision(['accepted'], updatedAt); - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify( - { - tags: ['accepted'], - updated_at: updatedAt, - tag_revision: tagRevision, - }, - null, - 2, - )}\n`, - ); - - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const deleteRes = await app.request(`/api/runs/${encodeURIComponent(filename)}/tags`, { - method: 'DELETE', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ expected_tag_revision: tagRevision }), - }); - expect(deleteRes.status).toBe(200); - const deleteData = (await deleteRes.json()) as { - ok: boolean; - tags: string[]; - final_state: { lifecycle: string; tags: string[] }; - tag_revision: string; - updated_at: string; - }; - expect(deleteData.ok).toBe(true); - expect(deleteData.tags).toEqual([]); - expect(deleteData.final_state).toEqual({ - lifecycle: 'active', - tags: [], - }); - expect(deleteData.tag_revision).toStartWith('sha256:'); - expect(deleteData.tag_revision).not.toBe(tagRevision); - - const tagFile = JSON.parse(readFileSync(path.join(runDir, 'tags.json'), 'utf8')) as { - tags: string[]; - tag_revision: string; - }; - expect(tagFile.tags).toEqual([]); - expect(tagFile.tag_revision).toBe(deleteData.tag_revision); - - const reloadedApp = createApp([], tempDir, tempDir, undefined, { studioDir }); - const detailRes = await reloadedApp.request(`/api/runs/${encodeURIComponent(filename)}`); - expect(detailRes.status).toBe(200); - const detailData = (await detailRes.json()) as { - tags: string[]; - final_state: { lifecycle: string; tags: string[] }; - tag_revision: string; - }; - expect(detailData).toMatchObject({ - tags: [], - final_state: { - lifecycle: 'active', - tags: [], - }, - tag_revision: deleteData.tag_revision, - }); - }); - - it('rejects stale local tag writes with refresh-required details', async () => { - const runsDir = localResultsExperimentDir(tempDir); - mkdirSync(runsDir, { recursive: true }); - const filename = '2026-03-25T10-45-00-000Z'; - const runDir = path.join(runsDir, filename); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); - const updatedAt = '2026-06-21T10:15:00.000Z'; - const tagRevision = createTagRevision(['accepted'], updatedAt); - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify( - { - tags: ['accepted'], - updated_at: updatedAt, - tag_revision: tagRevision, - }, - null, - 2, - )}\n`, - ); - - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const staleRes = await app.request(`/api/runs/${encodeURIComponent(filename)}/tags`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ tags: ['stale'], expected_tag_revision: 'sha256:stale' }), - }); - expect(staleRes.status).toBe(409); - const staleData = (await staleRes.json()) as { - error: string; - expected_tag_revision: string; - current_tag_revision: string; - }; - expect(staleData).toEqual({ - error: 'Run tags changed. Refresh the run and try again.', - expected_tag_revision: 'sha256:stale', - current_tag_revision: tagRevision, - }); - - const tagFile = JSON.parse(readFileSync(path.join(runDir, 'tags.json'), 'utf8')) as { - tags: string[]; - tag_revision: string; - }; - expect(tagFile.tags).toEqual(['accepted']); - expect(tagFile.tag_revision).toBe(tagRevision); - }); - it('computes pass_rate using the configured dashboard threshold', async () => { const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); @@ -1797,63 +1595,6 @@ describe('serve app', () => { }); }, 15000); - it('edits synced local run tags through the remote metadata overlay', async () => { - const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); - const experiment = 'green-uat'; - const timestamp = '2026-03-26T10-45-00-000Z'; - const runId = writeRemoteRunArtifact(cloneDir, experiment, timestamp, RESULT_A); - writeLocalRunArtifact(tempDir, experiment, timestamp, RESULT_A); - - writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); - - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - const putRes = await app.request(`/api/runs/${encodeURIComponent(runId)}/tags`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ tags: ['needs-review'] }), - }); - - expect(putRes.status).toBe(200); - const putData = (await putRes.json()) as { - tags: string[]; - remote_tags: string[]; - pending_tags: string[]; - metadata_dirty: boolean; - }; - expect(putData).toMatchObject({ - tags: ['needs-review'], - remote_tags: [], - pending_tags: ['needs-review'], - metadata_dirty: true, - }); - - const localTagsPath = path.join(tempDir, '.agentv', 'results', timestamp, 'tags.json'); - const overlayTagsPath = path.join(cloneDir, 'metadata', 'runs', timestamp, 'tags.json'); - expect(existsSync(localTagsPath)).toBe(false); - expect(existsSync(overlayTagsPath)).toBe(true); - - const listRes = await app.request('/api/runs'); - expect(listRes.status).toBe(200); - const listData = (await listRes.json()) as { - runs: Array<{ - filename: string; - source: string; - on_remote: boolean; - tags: string[]; - pending_tags: string[]; - metadata_dirty: boolean; - }>; - }; - expect(listData.runs[0]).toMatchObject({ - filename: runId, - source: 'local', - on_remote: true, - tags: ['needs-review'], - pending_tags: ['needs-review'], - metadata_dirty: true, - }); - }, 15000); - it('computes git-native remote run list totals from materialized index rows', async () => { const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); const secondPass = { @@ -1946,121 +1687,6 @@ describe('serve app', () => { expect(existsSync(runManifestPath)).toBe(true); }, 15000); - it('edits remote run tags through metadata overlay and reloads effective tags', async () => { - const { remoteDir, cloneDir } = initializeRemoteRepo(tempDir); - const runId = writeRemoteRunArtifact( - cloneDir, - 'green-uat', - '2026-03-26T12-00-00-000Z', - RESULT_A, - ); - - writeResultsConfig(tempDir, { remote: `file://${remoteDir}`, path: cloneDir }); - - const filename = `remote::${runId}`; - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - const putRes = await app.request(`/api/runs/${encodeURIComponent(filename)}/tags`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ tags: ['pending-review', 'shared'] }), - }); - - expect(putRes.status).toBe(200); - const putData = (await putRes.json()) as { - tags: string[]; - remote_tags: string[]; - pending_tags: string[]; - metadata_dirty: boolean; - }; - expect(putData).toMatchObject({ - tags: ['pending-review', 'shared'], - remote_tags: [], - pending_tags: ['pending-review', 'shared'], - metadata_dirty: true, - }); - - const artifactTagsPath = path.join(cloneDir, 'runs', '2026-03-26T12-00-00-000Z', 'tags.json'); - const overlayTagsPath = path.join( - cloneDir, - 'metadata', - 'runs', - '2026-03-26T12-00-00-000Z', - 'tags.json', - ); - expect(existsSync(artifactTagsPath)).toBe(false); - expect(existsSync(overlayTagsPath)).toBe(true); - - const listRes = await app.request('/api/runs'); - expect(listRes.status).toBe(200); - const listData = (await listRes.json()) as { - runs: Array<{ - filename: string; - tags: string[]; - pending_tags: string[]; - metadata_dirty: boolean; - }>; - }; - expect(listData.runs[0]).toMatchObject({ - filename, - tags: ['pending-review', 'shared'], - pending_tags: ['pending-review', 'shared'], - metadata_dirty: true, - }); - - const detailRes = await app.request(`/api/runs/${encodeURIComponent(filename)}`); - expect(detailRes.status).toBe(200); - const detailData = (await detailRes.json()) as { - tags: string[]; - pending_tags: string[]; - metadata_dirty: boolean; - }; - expect(detailData).toMatchObject({ - tags: ['pending-review', 'shared'], - pending_tags: ['pending-review', 'shared'], - metadata_dirty: true, - }); - - const reloadedApp = createApp([], tempDir, tempDir, undefined, { studioDir }); - const reloadedRes = await reloadedApp.request('/api/runs'); - expect(reloadedRes.status).toBe(200); - const reloadedData = (await reloadedRes.json()) as { - runs: Array<{ tags: string[]; pending_tags: string[]; metadata_dirty: boolean }>; - }; - expect(reloadedData.runs[0]).toMatchObject({ - tags: ['pending-review', 'shared'], - pending_tags: ['pending-review', 'shared'], - metadata_dirty: true, - }); - }, 15000); - - it('rejects remote tag edits when the configured results path is not writable', async () => { - const plainResultsDir = path.join(tempDir, 'plain-results'); - const timestamp = '2026-03-26T13-00-00-000Z'; - const runDir = localRunDir(plainResultsDir, 'default', timestamp); - mkdirSync(runDir, { recursive: true }); - writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(RESULT_A)); - - writeResultsConfig(tempDir, { - remote: `file://${path.join(tempDir, 'missing.git')}`, - path: plainResultsDir, - }); - - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - const res = await app.request( - `/api/runs/${encodeURIComponent(`remote::${timestamp}`)}/tags`, - { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ tags: ['blocked'] }), - }, - ); - - expect(res.status).toBe(409); - const data = (await res.json()) as { error: string }; - expect(data.error).toContain('not a writable git checkout'); - expect(existsSync(path.join(runDir, 'tags.json'))).toBe(false); - }); - it('loads a local run detail without cloning or fetching the configured results repo', async () => { const remoteDir = path.join(tempDir, 'results-remote.git'); git(`git init --bare --initial-branch=main --quiet "${remoteDir}"`, tempDir); @@ -2169,6 +1795,118 @@ describe('serve app', () => { }); }); + describe('GET /api/tags', () => { + // Seed a local run whose JSONL rows carry a promptfoo-shaped `tags` map so + // the Tags-tab grouping endpoint has arbitrary keys to enumerate/group on. + // Real runs write the same run-level tags map into both `index.jsonl` rows + // and `summary.json` `metadata.tags` in lockstep (see run-artifacts.ts), and + // key enumeration reads the cheap summary source — so mirror the row tags + // into summary.json here to match production. + function createLocalTaggedRun( + baseDir: string, + timestamp: string, + record: Record, + ) { + const runDir = localRunDirFromRunId(baseDir, timestamp); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(record)); + const tags = record.tags as Record | undefined; + writeFileSync( + path.join(runDir, 'summary.json'), + JSON.stringify( + { + metadata: { + run_id: timestamp, + ...(typeof record.experiment === 'string' && { experiment: record.experiment }), + ...(tags && { tags }), + }, + }, + null, + 2, + ), + ); + } + + it('enumerates available tag keys (union of row keys plus synthetic experiment)', async () => { + createLocalTaggedRun(tempDir, '2026-05-01T10-00-00-000Z', { + ...RESULT_A, + experiment: 'v2', + tags: { experiment: 'v2', team: 'core', env: 'ci' }, + }); + createLocalTaggedRun(tempDir, '2026-05-01T11-00-00-000Z', { + ...RESULT_A, + experiment: 'v1', + tags: { experiment: 'v1', region: 'us' }, + }); + // Legacy run with no tags map — contributes only the experiment key. + createLocalTaggedRun(tempDir, '2026-05-01T12-00-00-000Z', { + ...RESULT_A, + experiment: 'legacy', + }); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request('/api/tags'); + expect(res.status).toBe(200); + const data = (await res.json()) as { keys: string[] }; + expect(data.keys).toEqual(['env', 'experiment', 'region', 'team']); + }); + + it('groups runs by a selected arbitrary key with a (no ) bucket', async () => { + createLocalTaggedRun(tempDir, '2026-05-02T10-00-00-000Z', { + ...RESULT_A, + experiment: 'v2', + tags: { experiment: 'v2', team: 'core' }, + }); + createLocalTaggedRun(tempDir, '2026-05-02T11-00-00-000Z', { + ...RESULT_A, + experiment: 'v1', + tags: { experiment: 'v1', team: 'core' }, + }); + // Run missing the `team` key lands in the `(no team)` bucket. + createLocalTaggedRun(tempDir, '2026-05-02T12-00-00-000Z', { + ...RESULT_A, + experiment: 'v1', + tags: { experiment: 'v1' }, + }); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request('/api/tags?key=team'); + expect(res.status).toBe(200); + const data = (await res.json()) as { + key: string; + groups: Array<{ name: string; run_count: number }>; + }; + expect(data.key).toBe('team'); + const byName = Object.fromEntries(data.groups.map((g) => [g.name, g.run_count])); + expect(byName.core).toBe(2); + expect(byName['(no team)']).toBe(1); + }); + + it('resolves the experiment key via the top-level fallback for old runs', async () => { + // Old run: top-level `experiment`, no tags map. + createLocalTaggedRun(tempDir, '2026-05-03T10-00-00-000Z', { + ...RESULT_A, + experiment: 'baseline', + }); + // Newer run: experiment lives in the tags map. + createLocalTaggedRun(tempDir, '2026-05-03T11-00-00-000Z', { + ...RESULT_A, + experiment: 'candidate', + tags: { experiment: 'candidate', team: 'core' }, + }); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request('/api/tags?key=experiment'); + expect(res.status).toBe(200); + const data = (await res.json()) as { + key: string; + groups: Array<{ name: string }>; + }; + expect(data.key).toBe('experiment'); + expect(data.groups.map((g) => g.name).sort()).toEqual(['baseline', 'candidate']); + }); + }); + describe('GET /api/projects/all-runs', () => { it('does not infer experiment names for live benchmark runs before records persist them', async () => { const homedirSpy = spyOn(os, 'homedir').mockReturnValue(path.join(tempDir, 'home')); @@ -2357,7 +2095,7 @@ describe('serve app', () => { } }, 15000); - it('commits and pushes dirty remote tag metadata through project sync', async () => { + it('commits and pushes dirty remote run artifacts through project sync', async () => { const previousHome = process.env.AGENTV_HOME; const homeDir = path.join(tempDir, 'agentv-home-project-sync-push'); process.env.AGENTV_HOME = homeDir; @@ -2383,11 +2121,7 @@ describe('serve app', () => { ], }); const runTimestamp = '2026-03-26T12-00-00-000Z'; - writeRemoteRunArtifact(cloneDir, 'project-sync-push', runTimestamp, RESULT_A); - writeRemoteTagMetadataOverlay(cloneDir, 'project-sync-push', runTimestamp, [ - 'pending-review', - 'shared', - ]); + writeDirtyRemoteRunArtifact(cloneDir, 'project-sync-push', runTimestamp, RESULT_A); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const res = await app.request('/api/projects/project-sync-push/remote/sync', { @@ -2412,7 +2146,7 @@ describe('serve app', () => { run_count: 1, }); expect(git(`git --git-dir "${remoteDir}" ls-tree -r --name-only main`, tempDir)).toContain( - `metadata/runs/${runTimestamp}/tags.json`, + `runs/${runTimestamp}/index.jsonl`, ); } finally { if (previousHome === undefined) { @@ -2508,16 +2242,19 @@ describe('serve app', () => { }); const runTimestamp = '2026-03-26T13-00-00-000Z'; - const relativeMetadataPath = path.posix.join('metadata', 'runs', runTimestamp, 'tags.json'); - writeRemoteTagMetadataOverlay(seedDir, 'project-sync-conflict', runTimestamp, ['base']); - git('git add metadata && git commit --quiet -m "seed tag metadata"', seedDir); + const relativeRunPath = path.posix.join('runs', runTimestamp, 'index.jsonl'); + const seedRunPath = path.join(seedDir, relativeRunPath); + const cloneRunPath = path.join(cloneDir, relativeRunPath); + mkdirSync(path.dirname(seedRunPath), { recursive: true }); + writeFileSync(seedRunPath, `${JSON.stringify({ ...RESULT_A, score: 0.5 })}\n`); + git('git add runs && git commit --quiet -m "seed run artifact"', seedDir); git('git push --quiet origin main', seedDir); git('git pull --ff-only --quiet', cloneDir); - writeRemoteTagMetadataOverlay(cloneDir, 'project-sync-conflict', runTimestamp, ['local']); - git('git add metadata && git commit --quiet -m "local tag metadata"', cloneDir); - writeRemoteTagMetadataOverlay(seedDir, 'project-sync-conflict', runTimestamp, ['remote']); - git('git add metadata && git commit --quiet -m "remote tag metadata"', seedDir); + writeFileSync(cloneRunPath, `${JSON.stringify({ ...RESULT_A, score: 0.75 })}\n`); + git('git add runs && git commit --quiet -m "local run edit"', cloneDir); + writeFileSync(seedRunPath, `${JSON.stringify({ ...RESULT_A, score: 0.25 })}\n`); + git('git add runs && git commit --quiet -m "remote run edit"', seedDir); git('git push --quiet origin main', seedDir); git('git fetch --quiet origin --prune', cloneDir); git('git merge origin/main || true', cloneDir); @@ -2546,9 +2283,9 @@ describe('serve app', () => { commit_created: false, }); expect(data.block_reason).toContain('unresolved git conflicts'); - expect(data.conflicted_paths).toContain(relativeMetadataPath); + expect(data.conflicted_paths).toContain(relativeRunPath); expect(data.git_status).toContain('UU'); - expect(readFileSync(path.join(cloneDir, relativeMetadataPath), 'utf8')).toContain( + expect(readFileSync(path.join(cloneDir, relativeRunPath), 'utf8')).toContain( '<<<<<<< HEAD', ); } finally { @@ -2745,7 +2482,7 @@ describe('serve app', () => { function seedRun( name: string, records: object[] = [RESULT_A], - opts?: { experiment?: string; tags?: string[]; baseDir?: string }, + opts?: { experiment?: string; baseDir?: string }, ): { runId: string; runDir: string; manifestPath: string } { const runDir = localRunDir(opts?.baseDir ?? tempDir, opts?.experiment ?? 'default', name); mkdirSync(runDir, { recursive: true }); @@ -2756,12 +2493,6 @@ describe('serve app', () => { ...records.map((record) => ({ ...record, experiment: opts?.experiment ?? 'default' })), ), ); - if (opts?.tags) { - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify({ tags: opts.tags, updated_at: '2026-04-10T00:00:00.000Z' }, null, 2)}\n`, - ); - } return { runId: name, runDir, @@ -2769,13 +2500,9 @@ describe('serve app', () => { }; } - it('combines two local finished runs into a new run workspace with unioned tags', async () => { - const first = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A], { - tags: ['baseline', 'shared'], - }); - const second = seedRun('2026-06-01T11-00-00-000Z', [RESULT_B], { - tags: ['shared', 'candidate'], - }); + it('combines two local finished runs into a new run workspace', async () => { + const first = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A]); + const second = seedRun('2026-06-01T11-00-00-000Z', [RESULT_B]); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); const res = await app.request('/api/runs/combine', { @@ -2805,10 +2532,6 @@ describe('serve app', () => { expect(detail.results.map((r) => r.testId).sort()).toEqual(['test-greeting', 'test-math']); const combinedDir = localRunDirFromRunId(tempDir, data.run_id); - const tags = JSON.parse(readFileSync(path.join(combinedDir, 'tags.json'), 'utf8')) as { - tags: string[]; - }; - expect(tags.tags.sort()).toEqual(['baseline', 'candidate', 'shared']); const benchmark = JSON.parse( readFileSync(path.join(combinedDir, 'summary.json'), 'utf8'), ) as { @@ -3069,7 +2792,6 @@ describe('serve app', () => { const runDir = localRunDir(opts?.baseDir ?? tempDir, opts?.experiment ?? 'default', name); mkdirSync(runDir, { recursive: true }); writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); - writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n'); return { runId: name, runDir, @@ -4054,13 +3776,12 @@ describe('serve app', () => { }); }); - // ── GET /api/compare (tag filter) ─────────────────────────────────── + // ── GET /api/compare ───────────────────────────────────────────────── describe('GET /api/compare', () => { function seedCompareFixture() { - // Four runs, each in its own run workspace, with the tags documented - // below. This setup exercises the OR filter semantics used by - // `/api/compare?tags=`. + // Four runs, each in its own run workspace, spanning two experiments and + // two targets. This exercises the aggregated matrix and per-run views. const runsDir = localResultsExperimentDir(tempDir); mkdirSync(runsDir, { recursive: true }); @@ -4070,7 +3791,6 @@ describe('serve app', () => { target: string; category: string; score: number; - tags?: string[]; }> = [ { name: '2026-04-01T10-00-00-000Z', @@ -4078,7 +3798,6 @@ describe('serve app', () => { target: 'gpt-4o', category: 'baseline', score: 1.0, - tags: ['baseline'], }, { name: '2026-04-02T10-00-00-000Z', @@ -4086,7 +3805,6 @@ describe('serve app', () => { target: 'claude', category: 'baseline', score: 0.9, - tags: ['baseline'], }, { name: '2026-04-03T10-00-00-000Z', @@ -4094,10 +3812,8 @@ describe('serve app', () => { target: 'gpt-4o', category: 'prompting', score: 0.85, - tags: ['v2-prompt'], }, { - // Intentionally untagged — should never match any tag filter. name: '2026-04-04T10-00-00-000Z', experiment: 'exp-b', target: 'claude', @@ -4120,12 +3836,6 @@ describe('serve app', () => { score: run.score, }), ); - if (run.tags && run.tags.length > 0) { - writeFileSync( - path.join(runDir, 'tags.json'), - `${JSON.stringify({ tags: run.tags, updated_at: '2026-04-10T00:00:00.000Z' }, null, 2)}\n`, - ); - } } } @@ -4142,7 +3852,6 @@ describe('serve app', () => { run_id: string; experiment: string; target: string; - tags?: string[]; tests?: Array<{ test_id: string; category?: string }>; }>; }; @@ -4174,67 +3883,6 @@ describe('serve app', () => { expect(cell?.tests?.[0]?.category).toBe('prompting'); expect(run?.tests?.[0]?.category).toBe('prompting'); }); - - it('filters to a single tag', async () => { - seedCompareFixture(); - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const res = await app.request('/api/compare?tags=baseline'); - expect(res.status).toBe(200); - const data = (await res.json()) as CompareJson; - - expect(data.runs).toHaveLength(2); - for (const run of data.runs ?? []) { - expect(run.tags ?? []).toContain('baseline'); - } - // Only exp-a is represented; targets narrow to the two used by exp-a runs. - expect(data.experiments).toEqual(['exp-a']); - expect(data.targets.sort()).toEqual(['claude', 'gpt-4o']); - expect(data.cells).toHaveLength(2); - }); - - it('applies OR semantics across multiple tags', async () => { - seedCompareFixture(); - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const res = await app.request('/api/compare?tags=baseline,v2-prompt'); - expect(res.status).toBe(200); - const data = (await res.json()) as CompareJson; - - // Three tagged runs; the untagged run is excluded. - expect(data.runs).toHaveLength(3); - expect(data.experiments.sort()).toEqual(['exp-a', 'exp-b']); - // (exp-a, gpt-4o), (exp-a, claude), (exp-b, gpt-4o) — the (exp-b, claude) - // cell is missing because the only contributing run was untagged. - expect(data.cells).toHaveLength(3); - const cellKeys = (data.cells ?? []).map((c) => `${c.experiment}::${c.target}`).sort(); - expect(cellKeys).toEqual(['exp-a::claude', 'exp-a::gpt-4o', 'exp-b::gpt-4o']); - }); - - it('returns empty payload when no runs match the filter', async () => { - seedCompareFixture(); - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - const res = await app.request('/api/compare?tags=nonexistent'); - expect(res.status).toBe(200); - const data = (await res.json()) as CompareJson; - - expect(data.runs).toEqual([]); - expect(data.cells).toEqual([]); - expect(data.experiments).toEqual([]); - expect(data.targets).toEqual([]); - }); - - it('ignores whitespace and empty segments in the tags query', async () => { - seedCompareFixture(); - const app = createApp([], tempDir, tempDir, undefined, { studioDir }); - - // ` , baseline , ` should parse to just ['baseline']. - const res = await app.request('/api/compare?tags=%20,%20baseline%20,%20'); - expect(res.status).toBe(200); - const data = (await res.json()) as CompareJson; - expect(data.runs).toHaveLength(2); - }); }); // ── SPA fallback ────────────────────────────────────────────────────── @@ -4312,7 +3960,7 @@ describe('serve app', () => { expect(data.command).toContain('--output .agentv/results/r1'); }); - it('builds a selected experiment output path and writes initial tags beside the new run', async () => { + it('builds a selected experiment output path for the new run', async () => { const app = makeAppForRun(); const res = await app.request('/api/eval/run', { method: 'POST', @@ -4320,7 +3968,6 @@ describe('serve app', () => { body: JSON.stringify({ suite_filter: 'examples/demo.eval.yaml', experiment: 'smoke', - tags: [' baseline ', 'baseline', 'prompt-v2'], }), }); @@ -4330,13 +3977,6 @@ describe('serve app', () => { expect(data.command).toContain(path.join('.agentv', 'results')); const outputDir = data.command.match(/--output ([^\s]+)/)?.[1]; expect(outputDir).toBeString(); - - const tagFile = JSON.parse( - readFileSync(path.join(outputDir as string, 'tags.json'), 'utf8'), - ) as { - tags: string[]; - }; - expect(tagFile.tags).toEqual(['baseline', 'prompt-v2']); }); it('builds --retry-errors from the request', async () => { @@ -4386,23 +4026,6 @@ describe('serve app', () => { expect(res.status).toBe(400); }); - it('rejects initial tags when resuming an existing run', async () => { - const app = makeAppForRun(); - const res = await app.request('/api/eval/run', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - suite_filter: 'examples/demo.eval.yaml', - output: '.agentv/results/r1', - resume: true, - tags: ['baseline'], - }), - }); - expect(res.status).toBe(400); - const data = (await res.json()) as { error: string }; - expect(data.error).toContain('creating a new run'); - }); - it('returns 403 in read-only mode for unscoped /api/eval/run', async () => { const app = makeAppForRun({ readOnly: true }); const res = await app.request('/api/eval/run', { @@ -4566,7 +4189,7 @@ describe('serve app', () => { expect(data.command).not.toContain('--dry-run'); }); - it('rejects invalid experiment and tag values', async () => { + it('rejects invalid experiment values', async () => { const app = createApp([], tempDir, undefined, undefined, { studioDir }); const badExperiment = await app.request('/api/eval/preview', { method: 'POST', @@ -4577,16 +4200,6 @@ describe('serve app', () => { }), }); expect(badExperiment.status).toBe(400); - - const badTag = await app.request('/api/eval/preview', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - suite_filter: 'examples/demo.eval.yaml', - tags: ['good', 'bad\nvalue'], - }), - }); - expect(badTag.status).toBe(400); }); }); diff --git a/apps/dashboard/src/components/AnalyticsCharts.tsx b/apps/dashboard/src/components/AnalyticsCharts.tsx index dd34dd306..f72dcdd1c 100644 --- a/apps/dashboard/src/components/AnalyticsCharts.tsx +++ b/apps/dashboard/src/components/AnalyticsCharts.tsx @@ -7,10 +7,9 @@ * fields and renders the following charts: * * 1. Normalized gain bar chart (horizontal bars, g per task × target) - * 2. Domain/tag heatmap (pass rate by tag × target) - * 3. Negative delta table (tasks where non-baseline scored worse) - * 4. Filterable score distribution histogram (experiment/category/time) - * 5. Trend-over-time line chart (mean score per target over time) + * 2. Negative delta table (tasks where non-baseline scored worse) + * 3. Filterable score distribution histogram (experiment/category/time) + * 4. Trend-over-time line chart (mean score per target over time) * * All charts use recharts styled with Tailwind-matching colors to * respect the Dashboard dark theme (gray-950 canvas, cyan accents, @@ -144,20 +143,15 @@ export function AnalyticsCharts({ data, projectId }: AnalyticsChartsProps) { )} - {/* 2. Tag heatmap */} - {data.runs && data.runs.length > 0 && targets.length > 1 && ( - - )} - - {/* 3. Negative delta table */} + {/* 2. Negative delta table */} {baseline && baselineData && ( )} - {/* 4. Score distribution histogram */} + {/* 3. Score distribution histogram */} - {/* 5. Trend over time */} + {/* 4. Trend over time */} {trendData.length > 1 && targets.length > 0 && ( )} @@ -301,98 +295,7 @@ function NormalizedGainChart({ data, baseline }: { data: CompareResponse; baseli ); } -// ── 2. Tag heatmap ───────────────────────────────────────────────────── - -function TagHeatmap({ runs, targets }: { runs: CompareRunEntry[]; targets: string[] }) { - const { tags, grid } = useMemo(() => { - // Collect all tags and compute pass rate per (tag, target) - const tagTargetMap = new Map>(); - for (const run of runs) { - for (const tag of run.tags ?? []) { - let targetMap = tagTargetMap.get(tag); - if (!targetMap) { - targetMap = new Map(); - tagTargetMap.set(tag, targetMap); - } - let entry = targetMap.get(run.target); - if (!entry) { - entry = { passed: 0, total: 0 }; - targetMap.set(run.target, entry); - } - entry.passed += run.passed_count; - entry.total += run.eval_count; - } - } - const allTags = [...tagTargetMap.keys()].sort(); - const gridData = allTags.map((tag) => { - const row: Record = { tag }; - const targetMap = tagTargetMap.get(tag); - if (!targetMap) return row; - for (const target of targets) { - const entry = targetMap.get(target); - row[target] = entry && entry.total > 0 ? entry.passed / entry.total : -1; - } - return row; - }); - return { tags: allTags, grid: gridData }; - }, [runs, targets]); - - if (tags.length === 0) return null; - - return ( - -
- - - - - {targets.map((t) => ( - - ))} - - - - {grid.map((row) => ( - - - {targets.map((target) => { - const val = row[target] as number; - if (val < 0) { - return ( - - ); - } - const pct = Math.round(val * 100); - const colorClass = - val >= 0.8 - ? 'bg-emerald-400/20 text-emerald-400' - : val >= 0.5 - ? 'bg-yellow-400/20 text-yellow-400' - : 'bg-red-400/20 text-red-400'; - return ( - - ); - })} - - ))} - -
Tag - {t} -
{row.tag as string} - — - - - {pct}% - -
-
- - ); -} - -// ── 3. Negative delta table ──────────────────────────────────────────── +// ── 2. Negative delta table ──────────────────────────────────────────── function NegativeDeltaTable({ data, baseline }: { data: CompareResponse; baseline: string }) { const negatives = useMemo(() => { @@ -444,7 +347,7 @@ function NegativeDeltaTable({ data, baseline }: { data: CompareResponse; baselin ); } -// ── 4. Score distribution histogram ──────────────────────────────────── +// ── 3. Score distribution histogram ──────────────────────────────────── const DEFAULT_DISTRIBUTION_FILTERS: ScoreDistributionFilters = { experiment: ALL_DISTRIBUTION_FILTER_VALUE, @@ -651,7 +554,7 @@ function scoreDistributionEmptyMessage( return 'No scores match the selected distribution filters.'; } -// ── 5. Trend over time ───────────────────────────────────────────────── +// ── 4. Trend over time ───────────────────────────────────────────────── interface TrendPoint { date: string; diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index cd95956a4..31eceb49d 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -4,8 +4,7 @@ * Two modes: * 1. Aggregated (default) — `(experiment, target)` matrix, one cell per pair. * 2. Per run — individual runs are first-class; users select - * 2+ runs to render a side-by-side comparison, - * and may attach retroactive tags to any run. + * 2+ runs to render a side-by-side comparison. * * Styling matches the rest of AgentV Dashboard: dark gray surfaces * (`bg-gray-900` / `border-gray-800`), cyan accents for interactive elements, @@ -14,8 +13,6 @@ * * Backend contract: * - `GET /api/compare` → { cells, runs? } - * - `PUT /api/runs/:runId/tags` → replaces sidecar tags.json - * - `DELETE /api/runs/:runId/tags` → records an empty tag state * * To extend with a new mode: add a value to `ViewMode`, a button in the mode * toggle, and a new body component in the content switch. Hooks in any new @@ -23,10 +20,8 @@ * hook order does not change across renders. */ -import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useMemo, useState } from 'react'; -import { deleteRunTagsApi, saveRunTagsApi } from '~/lib/api'; import { aggregateQualityCount, executionErrorCount } from '~/lib/result-summary'; import type { CompareCell, CompareResponse, CompareRunEntry, CompareTestResult } from '~/lib/types'; @@ -40,7 +35,7 @@ interface AnalyticsTabProps { error?: Error | null; /** Project scope. Undefined for the unscoped (root) compare view. */ projectId?: string; - /** Read-only mode disables tag editing. */ + /** Read-only mode. Reserved for surfaces that disable mutating actions. */ readOnly?: boolean; } @@ -48,117 +43,11 @@ type ViewMode = 'aggregated' | 'per-run'; // ── Top-level container ───────────────────────────────────────────────── -export function AnalyticsTab({ - data, - isLoading, - isError, - error, - projectId, - readOnly, -}: AnalyticsTabProps) { +export function AnalyticsTab({ data, isLoading, isError, error, projectId }: AnalyticsTabProps) { const [mode, setMode] = useState('aggregated'); - const [filterTags, setFilterTags] = useState([]); - // Chip list is derived from the UNFILTERED response so chips stay visible - // even when the active filter would otherwise hide the runs that supplied - // them. Sorted alphabetically for stable UI. - const { allTags, tagCounts } = useMemo(() => { - const counts = new Map(); - for (const run of data?.runs ?? []) { - for (const tag of run.tags ?? []) { - counts.set(tag, (counts.get(tag) ?? 0) + 1); - } - } - return { allTags: [...counts.keys()].sort(), tagCounts: counts }; - }, [data?.runs]); - - // When a filter is active, re-aggregate cells/runs client-side from the - // filtered subset of runs. This avoids a network round-trip on every chip - // click and keeps the backend responsible only for the initial fetch. - // Safe because the server already exposes per-run totals; we sum them per - // (experiment, target) bucket, weighting averages by quality_count so - // execution errors do not depress quality scores. - const filteredData = useMemo(() => { - if (!data) return data; - if (filterTags.length === 0) return data; - const filterSet = new Set(filterTags); - const filteredRuns = (data.runs ?? []).filter((r) => - (r.tags ?? []).some((t) => filterSet.has(t)), - ); - - type CellAccum = { - experiment: string; - target: string; - eval_count: number; - quality_count: number; - passed_count: number; - execution_error_count: number; - score_sum: number; - tests: CompareTestResult[]; - }; - const cellMap = new Map(); - const experimentsSet = new Set(); - const targetsSet = new Set(); - - for (const run of filteredRuns) { - experimentsSet.add(run.experiment); - targetsSet.add(run.target); - const key = `${run.experiment}::${run.target}`; - const entry = cellMap.get(key) ?? { - experiment: run.experiment, - target: run.target, - eval_count: 0, - quality_count: 0, - passed_count: 0, - execution_error_count: 0, - score_sum: 0, - tests: [], - }; - const runQualityCount = aggregateQualityCount(run); - entry.eval_count += run.eval_count; - entry.quality_count += runQualityCount; - entry.passed_count += run.passed_count; - entry.execution_error_count += executionErrorCount(run); - entry.score_sum += run.avg_score * runQualityCount; - for (const t of run.tests) entry.tests.push(t); - cellMap.set(key, entry); - } - - const cells: CompareCell[] = [...cellMap.values()].map((e) => { - // Dedupe tests by test_id, last-wins (same pattern as the server). - const dedup = new Map(); - for (const t of e.tests) dedup.set(t.test_id, t); - return { - experiment: e.experiment, - target: e.target, - eval_count: e.eval_count, - quality_count: e.quality_count, - passed_count: e.passed_count, - execution_error_count: e.execution_error_count, - pass_rate: e.quality_count > 0 ? e.passed_count / e.quality_count : 0, - avg_score: e.quality_count > 0 ? e.score_sum / e.quality_count : 0, - tests: [...dedup.values()].slice(-100), - }; - }); - - return { - ...data, - experiments: [...experimentsSet].sort(), - targets: [...targetsSet].sort(), - cells, - runs: filteredRuns, - }; - }, [data, filterTags]); - - const toggleFilterTag = (tag: string) => { - setFilterTags((prev) => (prev.includes(tag) ? prev.filter((x) => x !== tag) : [...prev, tag])); - }; - const clearFilterTags = () => setFilterTags([]); - - const runsCount = filteredData?.runs?.length ?? 0; + const runsCount = data?.runs?.length ?? 0; const underlyingHasData = data && data.cells.length > 0; - const filterYieldsNoRuns = - filterTags.length > 0 && filteredData && (filteredData.runs?.length ?? 0) === 0; return (
@@ -169,111 +58,16 @@ export function AnalyticsTab({ )} {!isLoading && !isError && !underlyingHasData && } - {!isLoading && !isError && underlyingHasData && ( + {!isLoading && !isError && underlyingHasData && data && ( <> - {allTags.length > 0 && ( - - )} - {filterYieldsNoRuns ? ( - `\`${t}\``).join(' + ')}`} - body="Clear the filter or pick a different tag combination." - action={{ label: 'Clear filter', onClick: clearFilterTags }} - /> - ) : ( - filteredData && ( - <> - {mode === 'aggregated' && ( - - )} - {mode === 'per-run' && ( - - )} - - ) - )} + {mode === 'aggregated' && } + {mode === 'per-run' && } )}
); } -// ── Tag filter bar ────────────────────────────────────────────────────── - -function TagFilterBar({ - allTags, - tagCounts, - selected, - onToggle, - onClear, -}: { - allTags: string[]; - tagCounts: Map; - selected: string[]; - onToggle: (tag: string) => void; - onClear: () => void; -}) { - const selectedSet = new Set(selected); - const anySelected = selected.length > 0; - return ( -
-
- - Filter by tag - - {allTags.map((tag) => { - const isActive = selectedSet.has(tag); - const count = tagCounts.get(tag) ?? 0; - return ( - - ); - })} - {anySelected && ( - - )} -
-

- Showing runs with any selected tag. -

-
- ); -} - // ── Header ────────────────────────────────────────────────────────────── function Header({ @@ -525,19 +319,10 @@ function TestBreakdown({ tests }: { tests: CompareTestResult[] }) { // ── Per-run view ──────────────────────────────────────────────────────── -function PerRunView({ - data, - projectId, - readOnly, -}: { - data: CompareResponse; - projectId?: string; - readOnly: boolean; -}) { +function PerRunView({ data }: { data: CompareResponse }) { const runs = data.runs ?? []; const [selected, setSelected] = useState>(new Set()); const [showingCompare, setShowingCompare] = useState(false); - const [editingRunId, setEditingRunId] = useState(null); const toggleSelect = (runId: string) => { setSelected((prev) => { @@ -575,7 +360,6 @@ function PerRunView({ Timestamp - Tags Experiment Target Tests @@ -590,11 +374,6 @@ function PerRunView({ run={run} checked={selected.has(run.run_id)} onToggle={() => toggleSelect(run.run_id)} - editing={editingRunId === run.run_id} - onStartEdit={() => setEditingRunId(run.run_id)} - onEndEdit={() => setEditingRunId(null)} - projectId={projectId} - readOnly={readOnly} /> ))} @@ -639,331 +418,48 @@ function PerRunRow({ run, checked, onToggle, - editing, - onStartEdit, - onEndEdit, - projectId, - readOnly, }: { run: CompareRunEntry; checked: boolean; onToggle: () => void; - editing: boolean; - onStartEdit: () => void; - onEndEdit: () => void; - projectId?: string; - readOnly: boolean; }) { const avgPct = Math.round(run.avg_score * 100); const qualityCount = aggregateQualityCount(run); const errors = executionErrorCount(run); - const canEdit = !readOnly; - const tagsBtnRef = useRef(null); - const tags = run.tags ?? []; - const metadataDirty = run.metadata_dirty === true; - const runLabel = tags[0] ?? run.run_id; const subLabel = runSubLabel(run.run_id); - const tagsButtonClass = - tags.length > 0 - ? 'inline-flex flex-wrap items-center gap-1 rounded-md px-1 py-0.5 transition-colors hover:bg-gray-800/60' - : metadataDirty - ? 'rounded-md border border-yellow-900/60 bg-yellow-950/20 px-2 py-0.5 text-xs text-yellow-300 transition-colors hover:border-yellow-700' - : 'rounded-md border border-dashed border-gray-700 px-2 py-0.5 text-xs text-gray-500 transition-colors hover:border-cyan-800 hover:text-cyan-400'; - - // Restore focus to the tags trigger button once the inline editor closes, - // so keyboard users don't lose their place in the table. - const wasEditing = useRef(editing); - useEffect(() => { - if (wasEditing.current && !editing) { - tagsBtnRef.current?.focus(); - } - wasEditing.current = editing; - }, [editing]); - - return ( - <> - - - - - -
- {formatTimestamp(run.started_at)} -
- {subLabel &&
{subLabel}
} - - - {canEdit ? ( - - ) : tags.length > 0 ? ( - - ) : ( - - {metadataDirty ? 'Pending clear' : '—'} - - )} - - {run.experiment} - {run.target} - -
{qualityCount}
- {errors > 0 &&
{errors} errors
} - - - - - {avgPct}% - - {editing && ( - - - - - - )} - - ); -} - -function TagChips({ tags, dirty }: { tags: string[]; dirty: boolean }) { - return ( - - {tags.map((t) => ( - - {t} - - ))} - {dirty ? ( - - Pending sync - - ) : null} - - ); -} - -/** - * Inline chip-based tag editor. - * - * Local state: a `string[]` staged edit of the run's tags. Chips show the - * current staged tags; an input at the end accepts new tags (commit with - * Enter or comma, delete the last chip with Backspace on an empty input). - * Save persists the whole array; Cancel / Escape discards. - * - * The backend's `writeRunTags` handles deduplication, length limits, and - * control-character rejection, so we only lightly normalize in the UI - * (trim + skip duplicates already in the staged array). - */ -function TagsEditor({ - runId, - currentTags, - tagRevision, - source, - projectId, - onClose, -}: { - runId: string; - currentTags: string[]; - tagRevision?: string; - source: 'local' | 'remote'; - projectId?: string; - onClose: () => void; -}) { - const [tags, setTags] = useState(currentTags); - const [input, setInput] = useState(''); - const [err, setErr] = useState(null); - const qc = useQueryClient(); - const inputRef = useRef(null); - - useEffect(() => { - inputRef.current?.focus(); - }, []); - - const saveMut = useMutation({ - mutationFn: () => saveRunTagsApi(runId, tags, projectId, tagRevision), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ['compare'] }); - qc.invalidateQueries({ queryKey: ['runs'] }); - qc.invalidateQueries({ queryKey: ['remote-status', projectId ?? ''] }); - if (projectId) { - qc.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }); - qc.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }); - } - onClose(); - }, - onError: (e: Error) => setErr(e.message), - }); - - const clearMut = useMutation({ - mutationFn: () => deleteRunTagsApi(runId, projectId, tagRevision), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ['compare'] }); - qc.invalidateQueries({ queryKey: ['runs'] }); - qc.invalidateQueries({ queryKey: ['remote-status', projectId ?? ''] }); - if (projectId) { - qc.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }); - qc.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }); - } - onClose(); - }, - onError: (e: Error) => setErr(e.message), - }); - - const busy = saveMut.isPending || clearMut.isPending; - const hasChanges = - tags.length !== currentTags.length || tags.some((t, i) => t !== currentTags[i]); - - const commitInput = () => { - const trimmed = input.trim(); - if (trimmed === '') return; - if (tags.includes(trimmed)) { - setInput(''); - return; - } - setTags([...tags, trimmed]); - setInput(''); - setErr(null); - }; - - const removeTag = (tag: string) => { - setTags(tags.filter((t) => t !== tag)); - }; return ( -
-
- Tag run - - Multi-valued. Enter or comma adds; Backspace removes the last chip. - -
- {source === 'remote' ? ( -
- Remote tag edits are saved as local metadata. Use Sync Metadata to push them to the - results repo. -
- ) : null} -
- {tags.map((t) => ( - - {t} - - - ))} + + { - setErr(null); - setInput(e.target.value); - }} - maxLength={60} - disabled={busy} - onKeyDown={(e) => { - if (e.key === 'Enter' || e.key === ',') { - e.preventDefault(); - commitInput(); - } else if (e.key === 'Backspace' && input === '' && tags.length > 0) { - e.preventDefault(); - setTags(tags.slice(0, -1)); - } else if (e.key === 'Escape') { - onClose(); - } - }} - onBlur={commitInput} + type="checkbox" + className="h-4 w-4 cursor-pointer rounded border-gray-700 bg-gray-900 text-cyan-500 accent-cyan-500 focus:ring-cyan-500" + checked={checked} + onChange={onToggle} + aria-label={`Select run ${run.run_id}`} /> -
-
- - {currentTags.length > 0 && ( - - )} - -
- {err && ( -
- {err} + + +
+ {formatTimestamp(run.started_at)}
- )} -
+ {subLabel &&
{subLabel}
} + + {run.experiment} + {run.target} + +
{qualityCount}
+ {errors > 0 &&
{errors} errors
} + + + + + {avgPct}% + ); } @@ -1090,24 +586,11 @@ function PerRunCompareView({ } function RunColumnHeader({ run }: { run: CompareRunEntry }) { - const tags = run.tags ?? []; return (
{formatTimestamp(run.started_at)}
- {tags.length > 0 && ( -
- {tags.map((t) => ( - - {t} - - ))} -
- )}
{run.experiment} · {run.target}
@@ -1154,28 +637,11 @@ function EmptyState() { ); } -function Notice({ - headline, - body, - action, -}: { - headline: string; - body: string; - action?: { label: string; onClick: () => void }; -}) { +function Notice({ headline, body }: { headline: string; body: string }) { return (

{headline}

{body}

- {action && ( - - )}
); } diff --git a/apps/dashboard/src/components/Breadcrumbs.tsx b/apps/dashboard/src/components/Breadcrumbs.tsx index f5421a3e6..5d7e84058 100644 --- a/apps/dashboard/src/components/Breadcrumbs.tsx +++ b/apps/dashboard/src/components/Breadcrumbs.tsx @@ -10,11 +10,11 @@ import { Link, useMatches } from '@tanstack/react-router'; import { categoryPath, evalPath, - experimentPath, jobPath, projectHomePath, runPath, suitePath, + tagValuePath, } from '~/lib/navigation'; import { useSidebarContext } from '~/lib/sidebar-context'; @@ -102,10 +102,10 @@ function deriveSegments(matches: ReturnType): BreadcrumbSegme label: params.evalId ?? 'Eval', to: evalPath(params.runId, params.evalId ?? 'Eval', params.projectId), }); - } else if (routeId.includes('/projects/$projectId_/experiments/$experimentName')) { + } else if (routeId.includes('/projects/$projectId_/tags/$key/$value')) { segments.push({ - label: params.experimentName ?? 'Experiment', - to: experimentPath(params.experimentName ?? 'Experiment', params.projectId), + label: params.value ?? 'Tag', + to: tagValuePath(params.key ?? 'experiment', params.value ?? 'Tag', params.projectId), }); } else if (routeId.includes('/runs/$runId/category/$category')) { if (!segments.some((s) => s.label === formatBreadcrumbRunLabel(params.runId))) { @@ -151,10 +151,10 @@ function deriveSegments(matches: ReturnType): BreadcrumbSegme label: params.evalId ?? 'Eval', to: evalPath(params.runId, params.evalId ?? 'Eval'), }); - } else if (routeId.includes('/experiments/$experimentName')) { + } else if (routeId.includes('/tags/$key/$value')) { segments.push({ - label: params.experimentName ?? 'Experiment', - to: experimentPath(params.experimentName ?? 'Experiment'), + label: params.value ?? 'Tag', + to: tagValuePath(params.key ?? 'experiment', params.value ?? 'Tag'), }); } else if (routeId === '/settings') { segments.push({ label: 'Settings', to: '/settings' }); diff --git a/apps/dashboard/src/components/ExperimentsTab.tsx b/apps/dashboard/src/components/ExperimentsTab.tsx deleted file mode 100644 index 75605eaa1..000000000 --- a/apps/dashboard/src/components/ExperimentsTab.tsx +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Experiments table showing experiments grouped across all runs. - * - * Displays experiment name, number of runs, targets, pass rate, and - * last run timestamp. Each row links to the experiment detail page. - * - * The table keeps the desktop column layout on mobile by using the same - * overflow container + fixed minimum width pattern as other Dashboard summary - * tables, so right-side metrics remain reachable instead of being clipped. - */ - -import { useQuery } from '@tanstack/react-query'; -import { Link } from '@tanstack/react-router'; - -import { projectExperimentsOptions, useExperiments } from '~/lib/api'; -import { aggregateQualityCount, executionErrorCount } from '~/lib/result-summary'; -import type { ExperimentSummary } from '~/lib/types'; - -import { PassRatePill } from './PassRatePill'; - -interface ExperimentsTabProps { - projectId?: string; -} - -export function ExperimentsTab({ projectId }: ExperimentsTabProps) { - const { data, isLoading } = projectId - ? useQuery(projectExperimentsOptions(projectId)) - : useExperiments(); - - if (isLoading) { - return ; - } - - const experiments = data?.experiments ?? []; - - if (experiments.length === 0) { - return ( -
-

No experiments found

-

- Experiments will appear here once evaluations are run with experiment labels. -

-
- ); - } - - return ( -
- - - - - - - - - - - - - - {experiments.map((exp: ExperimentSummary) => { - const qualityCount = aggregateQualityCount(exp); - const errors = executionErrorCount(exp); - return ( - - - - - - - - - - ); - })} - -
ExperimentRunsTargetsEvalsExecution ErrorsPass RateLast Run
- {projectId ? ( - - {exp.name} - - ) : ( - - {exp.name} - - )} - {exp.run_count} - {exp.target_count} - - {exp.passed_count} - / - {qualityCount} - - {errors > 0 ? errors : 0} - - - - {formatTimestamp(exp.last_run).date} -
-
- ); -} - -function formatTimestamp(ts: string | undefined | null): { date: string; full: string } { - if (!ts) return { date: 'N/A', full: 'N/A' }; - try { - const d = new Date(ts); - if (Number.isNaN(d.getTime())) return { date: 'N/A', full: 'N/A' }; - const full = d.toLocaleString(); - const diffMs = Date.now() - d.getTime(); - const diffMin = Math.floor(diffMs / 60_000); - const diffHour = Math.floor(diffMs / 3_600_000); - let date: string; - if (diffMin < 1) date = 'just now'; - else if (diffMin < 60) date = `${diffMin} min ago`; - else if (diffHour < 24) date = `${diffHour} hour${diffHour === 1 ? '' : 's'} ago`; - else date = d.toLocaleDateString(); - return { date, full }; - } catch { - return { date: 'N/A', full: 'N/A' }; - } -} - -function LoadingSkeleton() { - return ( -
-
-
-
-
- {['sk-1', 'sk-2', 'sk-3', 'sk-4', 'sk-5'].map((id) => ( -
-
-
-
-
-
-
- ))} -
-
- ); -} diff --git a/apps/dashboard/src/components/RunEvalModal.tsx b/apps/dashboard/src/components/RunEvalModal.tsx index fa5911c83..6d126aeeb 100644 --- a/apps/dashboard/src/components/RunEvalModal.tsx +++ b/apps/dashboard/src/components/RunEvalModal.tsx @@ -63,8 +63,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal const [testIds, setTestIds] = useState(prefill?.testIds ?? []); const [target, setTarget] = useState(prefill?.target ?? ''); const [experiment, setExperiment] = useState(DEFAULT_EXPERIMENT); - const [tagInput, setTagInput] = useState(''); - const [tags, setTags] = useState([]); const [threshold, setThreshold] = useState(''); const [thresholdEdited, setThresholdEdited] = useState(false); const [workers, setWorkers] = useState(''); @@ -107,8 +105,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setTestIds(prefill?.testIds ?? []); setTarget(prefill?.target ?? ''); setExperiment(DEFAULT_EXPERIMENT); - setTagInput(''); - setTags([]); setTestIdInput(''); setThreshold(''); setThresholdEdited(false); @@ -131,28 +127,16 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal // Build request body from form state const buildRequest = useCallback((): RunEvalRequest => { - const effectiveTags = mergeUniqueTags(tags, parseTagsInput(tagInput)); return buildRunEvalRequest({ suiteFilter, testIds, target, experiment, - tags: effectiveTags, thresholdInput: threshold, studioThreshold: studioConfig?.threshold, workers, }); - }, [ - experiment, - studioConfig?.threshold, - suiteFilter, - tagInput, - tags, - target, - testIds, - threshold, - workers, - ]); + }, [experiment, studioConfig?.threshold, suiteFilter, target, testIds, threshold, workers]); // Update CLI preview when form changes useEffect(() => { @@ -179,18 +163,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setTestIds(testIds.filter((t) => t !== id)); } - function addTagsFromInput() { - const nextTags = parseTagsInput(tagInput); - if (nextTags.length > 0) { - setTags(mergeUniqueTags(tags, nextTags)); - } - setTagInput(''); - } - - function removeTag(tag: string) { - setTags(tags.filter((t) => t !== tag)); - } - // Launch async function handleLaunch() { setError(null); @@ -344,77 +316,22 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal {/* Run metadata */}
-
-
- - setExperiment(e.target.value)} - placeholder={DEFAULT_EXPERIMENT} - className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" - /> -
-
- -
- setTagInput(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter' || e.key === ',') { - e.preventDefault(); - addTagsFromInput(); - } - }} - onBlur={addTagsFromInput} - placeholder="baseline, prompt-v2" - className="min-w-0 flex-1 rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" - /> - -
-
+
+ + setExperiment(e.target.value)} + placeholder={DEFAULT_EXPERIMENT} + className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" + />
- {tags.length > 0 && ( -
- {tags.map((tag) => ( - - {tag} - - - ))} -
- )} -

- Saved with the new run only; existing runs stay unchanged. -

{/* Advanced options */} @@ -506,24 +423,6 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal // ── Sub-components ─────────────────────────────────────────────────────── -function parseTagsInput(value: string): string[] { - return value - .split(',') - .map((tag) => tag.trim()) - .filter((tag) => tag.length > 0); -} - -function mergeUniqueTags(existing: string[], incoming: string[]): string[] { - const seen = new Set(existing); - const merged = [...existing]; - for (const tag of incoming) { - if (seen.has(tag)) continue; - seen.add(tag); - merged.push(tag); - } - return merged; -} - function ModalShell({ children, onClose, diff --git a/apps/dashboard/src/components/RunList.tsx b/apps/dashboard/src/components/RunList.tsx index 1e377133e..7bdb81808 100644 --- a/apps/dashboard/src/components/RunList.tsx +++ b/apps/dashboard/src/components/RunList.tsx @@ -77,7 +77,6 @@ interface RunListItemView { passing: boolean; passedCount: number; failedCount: number; - metadataDirty: boolean; experimentNamespace: string; runtimeSourceLabel?: string; runtimeSourceTitle?: string; @@ -113,7 +112,6 @@ export function buildRunListItemView(run: RunMeta, passThreshold: number): RunLi const passing = qualityCount > 0 ? run.pass_rate >= passThreshold : errors === 0; const passedCount = Math.round(run.pass_rate * qualityCount); const failedCount = qualityCount - passedCount; - const metadataDirty = run.metadata_dirty === true; const experimentNamespace = experimentNamespaceLabel(run); const runtimeSourceLabel = run.runtime_source ? runtimeSourceSummary(run.runtime_source) @@ -133,7 +131,6 @@ export function buildRunListItemView(run: RunMeta, passThreshold: number): RunLi passing, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle: runtimeSourceTooltip, @@ -219,7 +216,7 @@ export function RunList({ if (projectId) { await Promise.all([ queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'runs'] }), - queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'experiments'] }), + queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'tags'] }), queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'compare'] }), queryClient.invalidateQueries({ queryKey: ['projects', projectId, 'targets'] }), ]); @@ -227,7 +224,7 @@ export function RunList({ } await Promise.all([ queryClient.invalidateQueries({ queryKey: ['runs'] }), - queryClient.invalidateQueries({ queryKey: ['experiments'] }), + queryClient.invalidateQueries({ queryKey: ['tags'] }), queryClient.invalidateQueries({ queryKey: ['compare'] }), queryClient.invalidateQueries({ queryKey: ['targets'] }), ]); @@ -437,7 +434,6 @@ export function RunList({ qualityCount, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle, @@ -487,7 +483,6 @@ export function RunList({ />
- {metadataDirty ? : null} {ts.date} @@ -556,7 +551,6 @@ export function RunList({ qualityCount, passedCount, failedCount, - metadataDirty, experimentNamespace, runtimeSourceLabel, runtimeSourceTitle, @@ -620,7 +614,6 @@ export function RunList({ title={display.title} className="block min-w-0 truncate font-medium text-cyan-400 hover:text-cyan-300 hover:underline" /> - {metadataDirty ? : null}
@@ -771,17 +764,6 @@ function CloudOutlineIcon() { ); } -function PendingSyncBadge() { - return ( - - Pending sync - - ); -} - function RunSourceBadges({ experimentNamespace, runtimeSourceLabel, diff --git a/apps/dashboard/src/components/Sidebar.tsx b/apps/dashboard/src/components/Sidebar.tsx index 543e442dc..120fd0834 100644 --- a/apps/dashboard/src/components/Sidebar.tsx +++ b/apps/dashboard/src/components/Sidebar.tsx @@ -6,7 +6,7 @@ * - At run detail: shows nearby runs as local review context * - At eval detail: shows evals in the current run with pass/fail indicators * - At suite/category detail: shows the filtered review context - * - At experiment detail: shows nearby experiments + * - At tag-value detail: shows sibling values for the selected tag key * * Responsive behavior is handled by SidebarShell: * - md+ (≥768px): always-visible fixed left panel @@ -22,10 +22,10 @@ import { DEFAULT_APP_NAME, isPassing, projectCategorySuitesOptions, - projectExperimentsOptions, + projectTagGroupsOptions, + tagGroupsOptions, useCategorySuites, useEvalRuns, - useExperiments, useProjectList, useProjectRunDetail, useProjectRunList, @@ -41,6 +41,7 @@ import { import { shouldShowEvalSourceLabels } from '~/lib/run-detail-context'; import { formatRunDisplay } from '~/lib/run-label'; import { useSidebarContext } from '~/lib/sidebar-context'; +import { tagKeyLabel } from '~/lib/tag-key-label'; import type { EvalResult } from '~/lib/types'; import { BrandName } from './BrandName'; @@ -141,11 +142,11 @@ function useCurrentEvalIdentitySearch() { }; } -type ProjectTabId = 'runs' | 'experiments' | 'analytics' | 'targets'; +type ProjectTabId = 'runs' | 'tags' | 'analytics' | 'targets'; const projectNavItems: { id: ProjectTabId; label: string; description: string }[] = [ { id: 'runs', label: 'Recent Runs', description: 'Run review' }, - { id: 'experiments', label: 'Experiments', description: 'Grouped runs' }, + { id: 'tags', label: 'Tags', description: 'Grouped runs' }, { id: 'analytics', label: 'Analytics', description: 'Compare scores' }, { id: 'targets', label: 'Targets', description: 'Target results' }, ]; @@ -224,8 +225,8 @@ export function Sidebar() { to: '/projects/$projectId/runs/$runId', fuzzy: true, }); - const projectExperimentMatch = matchRoute({ - to: '/projects/$projectId/experiments/$experimentName', + const projectTagValueMatch = matchRoute({ + to: '/projects/$projectId/tags/$key/$value', fuzzy: true, }); const projectCategoryMatch = matchRoute({ @@ -293,15 +294,16 @@ export function Sidebar() { } if ( - projectExperimentMatch && - typeof projectExperimentMatch === 'object' && - 'projectId' in projectExperimentMatch + projectTagValueMatch && + typeof projectTagValueMatch === 'object' && + 'projectId' in projectTagValueMatch ) { - const { projectId, experimentName } = projectExperimentMatch as { + const { projectId, key, value } = projectTagValueMatch as { projectId: string; - experimentName: string; + key: string; + value: string; }; - return ; + return ; } // Project home (runs/experiments/targets) @@ -321,8 +323,8 @@ export function Sidebar() { to: '/runs/$runId/suite/$suite', fuzzy: true, }); - const experimentMatch = matchRoute({ - to: '/experiments/$experimentName', + const tagValueMatch = matchRoute({ + to: '/tags/$key/$value', fuzzy: true, }); @@ -341,13 +343,9 @@ export function Sidebar() { return ; } - if ( - experimentMatch && - typeof experimentMatch === 'object' && - 'experimentName' in experimentMatch - ) { - const { experimentName } = experimentMatch as { experimentName: string }; - return ; + if (tagValueMatch && typeof tagValueMatch === 'object' && 'key' in tagValueMatch) { + const { key, value } = tagValueMatch as { key: string; value: string }; + return ; } if (runMatch && typeof runMatch === 'object' && 'runId' in runMatch) { @@ -801,15 +799,17 @@ function ProjectCategorySidebar({ ); } -function ProjectExperimentSidebar({ +function ProjectTagValueSidebar({ projectId, - currentExperiment, + tagKey, + currentValue, }: { projectId: string; - currentExperiment: string; + tagKey: string; + currentValue: string; }) { - const { data } = useQuery(projectExperimentsOptions(projectId)); - const experiments = data?.experiments ?? []; + const { data } = useQuery(projectTagGroupsOptions(projectId, tagKey)); + const groups = data?.groups ?? []; return ( @@ -819,33 +819,33 @@ function ProjectExperimentSidebar({ } + search={{ tab: 'tags', key: tagKey } as Record} className="text-xs text-gray-400 hover:text-cyan-400" > - ← All experiments + ← All {tagKeyLabel(tagKey)} values