From 0c1b8c1c211c23fd47c0788ab5cf229a5bef909b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 23 Jun 2026 04:01:48 +0200 Subject: [PATCH 1/3] feat(dashboard): set run experiment and tags --- apps/cli/src/commands/results/eval-runner.ts | 80 +++++++++-- apps/cli/test/commands/results/serve.test.ts | 82 +++++++++++ .../dashboard/src/components/RunEvalModal.tsx | 131 +++++++++++++++++- .../src/components/run-eval-threshold.test.ts | 25 ++++ .../src/components/run-eval-threshold.ts | 6 + apps/dashboard/src/lib/types.ts | 2 + .../src/content/docs/docs/tools/dashboard.mdx | 2 + 7 files changed, 317 insertions(+), 11 deletions(-) diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index ac8f23fc7..eaa8a1535 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -27,10 +27,11 @@ import { listTargetNames, readTargetDefinitions } from '@agentv/core'; import type { Context } from 'hono'; import type { Hono } from 'hono'; -import { TARGET_FILE_CANDIDATES, discoverTargetsFile } from '../../utils/targets.js'; +import { TARGET_FILE_CANDIDATES } from '../../utils/targets.js'; import { discoverEvalFiles } from '../eval/discover.js'; -import { buildDefaultRunDir } from '../eval/result-layout.js'; +import { buildDefaultRunDir, normalizeExperimentName } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; +import { normalizeTags, writeRunTags } from './run-tags.js'; // ── In-memory run tracker ──────────────────────────────────────────────── @@ -140,6 +141,8 @@ interface RunEvalRequest { suite_filter?: string; test_ids?: string[]; target?: string; + experiment?: string; + tags?: string[]; threshold?: number; workers?: number; dry_run?: boolean; @@ -170,7 +173,27 @@ function validateResumeOptions(req: RunEvalRequest): string | undefined { return undefined; } -function buildCliArgs(req: RunEvalRequest): string[] { +function parseInitialTags(value: unknown): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) { + throw new Error('tags must be an array of strings'); + } + return normalizeTags(value); +} + +function normalizeRunMetadata(req: RunEvalRequest): { experiment: string; tags: string[] } { + const experiment = normalizeExperimentName(req.experiment); + const tags = parseInitialTags(req.tags); + if ((req.resume || req.rerun_failed) && req.experiment?.trim()) { + throw new Error('experiment cannot be changed when resuming an existing run'); + } + if ((req.resume || req.rerun_failed) && tags.length > 0) { + throw new Error('initial tags can only be set when creating a new run'); + } + return { experiment, tags }; +} + +function buildCliArgs(req: RunEvalRequest, experiment?: string): string[] { const args: string[] = ['eval']; // Suite filter (eval paths/globs) @@ -196,6 +219,10 @@ function buildCliArgs(req: RunEvalRequest): string[] { args.push('--target', req.target.trim()); } + if (experiment && req.experiment?.trim()) { + args.push('--experiment', experiment); + } + // Threshold if (req.threshold !== undefined && req.threshold !== null) { args.push('--threshold', String(req.threshold)); @@ -306,6 +333,12 @@ function openConsoleLogStream(outputDir: string): WriteStream | undefined { } } +function writeInitialRunTags(outputDir: string, tags: readonly string[]): void { + if (tags.length === 0) return; + mkdirSync(outputDir, { recursive: true }); + writeRunTags(path.join(outputDir, 'index.jsonl'), tags); +} + // ── Route registration ─────────────────────────────────────────────────── // biome-ignore lint/suspicious/noExplicitAny: Hono Context generic varies by route @@ -369,12 +402,19 @@ export function registerEvalRoutes( return c.json({ error: resumeError }, 400); } + let metadata: { experiment: string; tags: string[] }; + try { + metadata = normalizeRunMetadata(body); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } + const cliPaths = resolveCliPath(cwd); if (!cliPaths) { return c.json({ error: 'Cannot locate agentv CLI entry point' }, 500); } - const args = buildCliArgs(body); + const args = buildCliArgs(body, metadata.experiment); // Determine the output directory for this run. When the caller provides // an explicit --output (resume/rerun), use that path. Otherwise generate // the default path now so we can pass it via --output and later correlate @@ -382,7 +422,7 @@ export function registerEvalRoutes( // target in the sidebar before any results have been written). const outputDir = body.output?.trim() ? path.resolve(cwd, body.output.trim()) - : buildDefaultRunDir(cwd); + : buildDefaultRunDir(cwd, metadata.experiment); if (!body.output?.trim()) { args.push('--output', outputDir); } @@ -402,6 +442,7 @@ export function registerEvalRoutes( activeRuns.set(runId, run); try { + writeInitialRunTags(outputDir, metadata.tags); const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'], @@ -537,7 +578,14 @@ export function registerEvalRoutes( return c.json({ error: 'Invalid JSON body' }, 400); } - const args = buildCliArgs(body); + let metadata: { experiment: string; tags: string[] }; + try { + metadata = normalizeRunMetadata(body); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } + + const args = buildCliArgs(body, metadata.experiment); return c.json({ command: buildCliPreview(args) }); }); @@ -590,15 +638,22 @@ export function registerEvalRoutes( return c.json({ error: resumeError }, 400); } + let metadata: { experiment: string; tags: string[] }; + try { + metadata = normalizeRunMetadata(body); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } + const cliPaths = resolveCliPath(cwd); if (!cliPaths) { return c.json({ error: 'Cannot locate agentv CLI entry point' }, 500); } - const args = buildCliArgs(body); + const args = buildCliArgs(body, metadata.experiment); const outputDir = body.output?.trim() ? path.resolve(cwd, body.output.trim()) - : buildDefaultRunDir(cwd); + : buildDefaultRunDir(cwd, metadata.experiment); if (!body.output?.trim()) { args.push('--output', outputDir); } @@ -618,6 +673,7 @@ export function registerEvalRoutes( activeRuns.set(runId, run); try { + writeInitialRunTags(outputDir, metadata.tags); const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'], @@ -721,7 +777,13 @@ export function registerEvalRoutes( } catch { return c.json({ error: 'Invalid JSON body' }, 400); } - const args = buildCliArgs(body); + let metadata: { experiment: string; tags: string[] }; + try { + metadata = normalizeRunMetadata(body); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } + const args = buildCliArgs(body, metadata.experiment); return c.json({ command: buildCliPreview(args) }); }); } diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index f49dd814a..fe053553b 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -4026,6 +4026,33 @@ describe('serve app', () => { expect(data.command).toContain('--output .agentv/results/default/r1'); }); + it('builds a selected experiment output path and writes initial tags beside the new run', async () => { + const app = makeAppForRun(); + const res = await app.request('/api/eval/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + experiment: 'smoke', + tags: [' baseline ', 'baseline', 'prompt-v2'], + }), + }); + + expect(res.status).toBe(202); + const data = (await res.json()) as { command: string }; + expect(data.command).toContain('--experiment smoke'); + expect(data.command).toContain(path.join('.agentv', 'results', 'runs', 'smoke')); + const outputDir = data.command.match(/--output ([^\s]+)/)?.[1]; + expect(outputDir).toBeString(); + + const tagFile = JSON.parse( + readFileSync(path.join(outputDir as string, 'tags.json'), 'utf8'), + ) as { + tags: string[]; + }; + expect(tagFile.tags).toEqual(['baseline', 'prompt-v2']); + }); + it('builds --retry-errors from the request', async () => { const app = makeAppForRun(); const res = await app.request('/api/eval/run', { @@ -4073,6 +4100,23 @@ describe('serve app', () => { expect(res.status).toBe(400); }); + it('rejects initial tags when resuming an existing run', async () => { + const app = makeAppForRun(); + const res = await app.request('/api/eval/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + output: 'runs/r1', + resume: true, + tags: ['baseline'], + }), + }); + expect(res.status).toBe(400); + const data = (await res.json()) as { error: string }; + expect(data.error).toContain('creating a new run'); + }); + it('returns 403 in read-only mode for unscoped /api/eval/run', async () => { const app = makeAppForRun({ readOnly: true }); const res = await app.request('/api/eval/run', { @@ -4205,6 +4249,44 @@ describe('serve app', () => { const data = (await res.json()) as { command: string }; expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl'); }); + + it('emits --experiment for selected experiment requests', async () => { + const app = createApp([], tempDir, undefined, undefined, { studioDir }); + const res = await app.request('/api/eval/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + experiment: 'smoke', + }), + }); + expect(res.status).toBe(200); + const data = (await res.json()) as { command: string }; + expect(data.command).toContain('--experiment smoke'); + }); + + it('rejects invalid experiment and tag values', async () => { + const app = createApp([], tempDir, undefined, undefined, { studioDir }); + const badExperiment = await app.request('/api/eval/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + experiment: 'bad/name', + }), + }); + expect(badExperiment.status).toBe(400); + + const badTag = await app.request('/api/eval/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + suite_filter: 'examples/demo.eval.yaml', + tags: ['good', 'bad\nvalue'], + }), + }); + expect(badTag.status).toBe(400); + }); }); // ── GET /api/runs/:filename — run_dir + suite_filter for resume UI ───── diff --git a/apps/dashboard/src/components/RunEvalModal.tsx b/apps/dashboard/src/components/RunEvalModal.tsx index 486178304..7b4b2bcfd 100644 --- a/apps/dashboard/src/components/RunEvalModal.tsx +++ b/apps/dashboard/src/components/RunEvalModal.tsx @@ -36,6 +36,8 @@ import { getThresholdFieldValue, } from './run-eval-threshold'; +const DEFAULT_EXPERIMENT = 'default'; + // ── Props ──────────────────────────────────────────────────────────────── export interface RunEvalModalProps { @@ -60,6 +62,9 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal const [testIdInput, setTestIdInput] = useState(''); const [testIds, setTestIds] = useState(prefill?.testIds ?? []); const [target, setTarget] = useState(prefill?.target ?? ''); + const [experiment, setExperiment] = useState(DEFAULT_EXPERIMENT); + const [tagInput, setTagInput] = useState(''); + const [tags, setTags] = useState([]); const [threshold, setThreshold] = useState(''); const [thresholdEdited, setThresholdEdited] = useState(false); const [workers, setWorkers] = useState(''); @@ -102,6 +107,9 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setSuiteFilter(prefill?.suiteFilter ?? ''); setTestIds(prefill?.testIds ?? []); setTarget(prefill?.target ?? ''); + setExperiment(DEFAULT_EXPERIMENT); + setTagInput(''); + setTags([]); setTestIdInput(''); setThreshold(''); setThresholdEdited(false); @@ -125,16 +133,30 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal // Build request body from form state const buildRequest = useCallback((): RunEvalRequest => { + const effectiveTags = mergeUniqueTags(tags, parseTagsInput(tagInput)); return buildRunEvalRequest({ suiteFilter, testIds, target, + experiment, + tags: effectiveTags, thresholdInput: threshold, studioThreshold: studioConfig?.threshold, workers, dryRun, }); - }, [dryRun, studioConfig?.threshold, suiteFilter, target, testIds, threshold, workers]); + }, [ + dryRun, + experiment, + studioConfig?.threshold, + suiteFilter, + tagInput, + tags, + target, + testIds, + threshold, + workers, + ]); // Update CLI preview when form changes useEffect(() => { @@ -161,6 +183,18 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal setTestIds(testIds.filter((t) => t !== id)); } + function addTagsFromInput() { + const nextTags = parseTagsInput(tagInput); + if (nextTags.length > 0) { + setTags(mergeUniqueTags(tags, nextTags)); + } + setTagInput(''); + } + + function removeTag(tag: string) { + setTags(tags.filter((t) => t !== tag)); + } + // Launch async function handleLaunch() { setError(null); @@ -312,6 +346,81 @@ export function RunEvalModal({ open, onClose, projectId, prefill }: RunEvalModal + {/* Run metadata */} +
+
+
+ + setExperiment(e.target.value)} + placeholder={DEFAULT_EXPERIMENT} + className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" + /> +
+
+ +
+ setTagInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ',') { + e.preventDefault(); + addTagsFromInput(); + } + }} + onBlur={addTagsFromInput} + placeholder="baseline, prompt-v2" + className="min-w-0 flex-1 rounded-md border border-gray-700 bg-gray-800 px-3 py-1.5 text-sm text-white placeholder-gray-500 focus:border-cyan-600 focus:outline-none" + /> + +
+
+
+ {tags.length > 0 && ( +
+ {tags.map((tag) => ( + + {tag} + + + ))} +
+ )} +

+ Saved with the new run only; existing runs stay unchanged. +

+
+ {/* Advanced options */}