Skip to content

feat(dashboard): set run experiment and tags #1490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
christso merged 3 commits into from
Jun 23, 2026
Merged

feat(dashboard): set run experiment and tags #1490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0c1b8c1
feat(dashboard): set run experiment and tags
christso Jun 23, 2026
4af2cda
fix(dashboard): use canonical resume test path
christso Jun 23, 2026
8d1181b
fix(dashboard): expect experiment result root
christso Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 71 additions & 9 deletions apps/cli/src/commands/results/eval-runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ import { listTargetNames, readTargetDefinitions } from '@agentv/core';
import type { Context } from 'hono';
import type { Hono } from 'hono';

import { TARGET_FILE_CANDIDATES, discoverTargetsFile } from '../../utils/targets.js';
import { TARGET_FILE_CANDIDATES } from '../../utils/targets.js';
import { discoverEvalFiles } from '../eval/discover.js';
import { buildDefaultRunDir } from '../eval/result-layout.js';
import { buildDefaultRunDir, normalizeExperimentName } from '../eval/result-layout.js';
import { findRepoRoot } from '../eval/shared.js';
import { normalizeTags, writeRunTags } from './run-tags.js';

// ── In-memory run tracker ────────────────────────────────────────────────

Expand Down Expand Up @@ -140,6 +141,8 @@ interface RunEvalRequest {
suite_filter?: string;
test_ids?: string[];
target?: string;
experiment?: string;
tags?: string[];
threshold?: number;
workers?: number;
dry_run?: boolean;
Expand Down Expand Up @@ -170,7 +173,27 @@ function validateResumeOptions(req: RunEvalRequest): string | undefined {
return undefined;
}

function buildCliArgs(req: RunEvalRequest): string[] {
function parseInitialTags(value: unknown): string[] {
if (value === undefined) return [];
if (!Array.isArray(value)) {
throw new Error('tags must be an array of strings');
}
return normalizeTags(value);
}

function normalizeRunMetadata(req: RunEvalRequest): { experiment: string; tags: string[] } {
const experiment = normalizeExperimentName(req.experiment);
const tags = parseInitialTags(req.tags);
if ((req.resume || req.rerun_failed) && req.experiment?.trim()) {
throw new Error('experiment cannot be changed when resuming an existing run');
}
if ((req.resume || req.rerun_failed) && tags.length > 0) {
throw new Error('initial tags can only be set when creating a new run');
}
return { experiment, tags };
}

function buildCliArgs(req: RunEvalRequest, experiment?: string): string[] {
const args: string[] = ['eval'];

// Suite filter (eval paths/globs)
Expand All @@ -196,6 +219,10 @@ function buildCliArgs(req: RunEvalRequest): string[] {
args.push('--target', req.target.trim());
}

if (experiment && req.experiment?.trim()) {
args.push('--experiment', experiment);
}

// Threshold
if (req.threshold !== undefined && req.threshold !== null) {
args.push('--threshold', String(req.threshold));
Expand Down Expand Up @@ -306,6 +333,12 @@ function openConsoleLogStream(outputDir: string): WriteStream | undefined {
}
}

function writeInitialRunTags(outputDir: string, tags: readonly string[]): void {
if (tags.length === 0) return;
mkdirSync(outputDir, { recursive: true });
writeRunTags(path.join(outputDir, 'index.jsonl'), tags);
}

// ── Route registration ───────────────────────────────────────────────────

// biome-ignore lint/suspicious/noExplicitAny: Hono Context generic varies by route
Expand Down Expand Up @@ -369,20 +402,27 @@ export function registerEvalRoutes(
return c.json({ error: resumeError }, 400);
}

let metadata: { experiment: string; tags: string[] };
try {
metadata = normalizeRunMetadata(body);
} catch (err) {
return c.json({ error: (err as Error).message }, 400);
}

const cliPaths = resolveCliPath(cwd);
if (!cliPaths) {
return c.json({ error: 'Cannot locate agentv CLI entry point' }, 500);
}

const args = buildCliArgs(body);
const args = buildCliArgs(body, metadata.experiment);
// Determine the output directory for this run. When the caller provides
// an explicit --output (resume/rerun), use that path. Otherwise generate
// the default path now so we can pass it via --output and later correlate
// the filesystem run with this in-memory DashboardRun (needed to show the
// target in the sidebar before any results have been written).
const outputDir = body.output?.trim()
? path.resolve(cwd, body.output.trim())
: buildDefaultRunDir(cwd);
: buildDefaultRunDir(cwd, metadata.experiment);
if (!body.output?.trim()) {
args.push('--output', outputDir);
}
Expand All @@ -402,6 +442,7 @@ export function registerEvalRoutes(
activeRuns.set(runId, run);

try {
writeInitialRunTags(outputDir, metadata.tags);
const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], {
cwd,
stdio: ['ignore', 'pipe', 'pipe'],
Expand Down Expand Up @@ -537,7 +578,14 @@ export function registerEvalRoutes(
return c.json({ error: 'Invalid JSON body' }, 400);
}

const args = buildCliArgs(body);
let metadata: { experiment: string; tags: string[] };
try {
metadata = normalizeRunMetadata(body);
} catch (err) {
return c.json({ error: (err as Error).message }, 400);
}

const args = buildCliArgs(body, metadata.experiment);
return c.json({ command: buildCliPreview(args) });
});

Expand Down Expand Up @@ -590,15 +638,22 @@ export function registerEvalRoutes(
return c.json({ error: resumeError }, 400);
}

let metadata: { experiment: string; tags: string[] };
try {
metadata = normalizeRunMetadata(body);
} catch (err) {
return c.json({ error: (err as Error).message }, 400);
}

const cliPaths = resolveCliPath(cwd);
if (!cliPaths) {
return c.json({ error: 'Cannot locate agentv CLI entry point' }, 500);
}

const args = buildCliArgs(body);
const args = buildCliArgs(body, metadata.experiment);
const outputDir = body.output?.trim()
? path.resolve(cwd, body.output.trim())
: buildDefaultRunDir(cwd);
: buildDefaultRunDir(cwd, metadata.experiment);
if (!body.output?.trim()) {
args.push('--output', outputDir);
}
Expand All @@ -618,6 +673,7 @@ export function registerEvalRoutes(
activeRuns.set(runId, run);

try {
writeInitialRunTags(outputDir, metadata.tags);
const child = spawn(cliPaths.binPath, [...cliPaths.args, ...args], {
cwd,
stdio: ['ignore', 'pipe', 'pipe'],
Expand Down Expand Up @@ -721,7 +777,13 @@ export function registerEvalRoutes(
} catch {
return c.json({ error: 'Invalid JSON body' }, 400);
}
const args = buildCliArgs(body);
let metadata: { experiment: string; tags: string[] };
try {
metadata = normalizeRunMetadata(body);
} catch (err) {
return c.json({ error: (err as Error).message }, 400);
}
const args = buildCliArgs(body, metadata.experiment);
return c.json({ command: buildCliPreview(args) });
});
}
82 changes: 82 additions & 0 deletions apps/cli/test/commands/results/serve.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4026,6 +4026,33 @@ describe('serve app', () => {
expect(data.command).toContain('--output .agentv/results/default/r1');
});

it('builds a selected experiment output path and writes initial tags beside the new run', async () => {
const app = makeAppForRun();
const res = await app.request('/api/eval/run', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
suite_filter: 'examples/demo.eval.yaml',
experiment: 'smoke',
tags: [' baseline ', 'baseline', 'prompt-v2'],
}),
});

expect(res.status).toBe(202);
const data = (await res.json()) as { command: string };
expect(data.command).toContain('--experiment smoke');
expect(data.command).toContain(path.join('.agentv', 'results', 'smoke'));
const outputDir = data.command.match(/--output ([^\s]+)/)?.[1];
expect(outputDir).toBeString();

const tagFile = JSON.parse(
readFileSync(path.join(outputDir as string, 'tags.json'), 'utf8'),
) as {
tags: string[];
};
expect(tagFile.tags).toEqual(['baseline', 'prompt-v2']);
});

it('builds --retry-errors <path> from the request', async () => {
const app = makeAppForRun();
const res = await app.request('/api/eval/run', {
Expand Down Expand Up @@ -4073,6 +4100,23 @@ describe('serve app', () => {
expect(res.status).toBe(400);
});

it('rejects initial tags when resuming an existing run', async () => {
const app = makeAppForRun();
const res = await app.request('/api/eval/run', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
suite_filter: 'examples/demo.eval.yaml',
output: '.agentv/results/default/r1',
resume: true,
tags: ['baseline'],
}),
});
expect(res.status).toBe(400);
const data = (await res.json()) as { error: string };
expect(data.error).toContain('creating a new run');
});

it('returns 403 in read-only mode for unscoped /api/eval/run', async () => {
const app = makeAppForRun({ readOnly: true });
const res = await app.request('/api/eval/run', {
Expand Down Expand Up @@ -4205,6 +4249,44 @@ describe('serve app', () => {
const data = (await res.json()) as { command: string };
expect(data.command).toContain('--retry-errors .agentv/results/default/r0/index.jsonl');
});

it('emits --experiment for selected experiment requests', async () => {
const app = createApp([], tempDir, undefined, undefined, { studioDir });
const res = await app.request('/api/eval/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
suite_filter: 'examples/demo.eval.yaml',
experiment: 'smoke',
}),
});
expect(res.status).toBe(200);
const data = (await res.json()) as { command: string };
expect(data.command).toContain('--experiment smoke');
});

it('rejects invalid experiment and tag values', async () => {
const app = createApp([], tempDir, undefined, undefined, { studioDir });
const badExperiment = await app.request('/api/eval/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
suite_filter: 'examples/demo.eval.yaml',
experiment: 'bad/name',
}),
});
expect(badExperiment.status).toBe(400);

const badTag = await app.request('/api/eval/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
suite_filter: 'examples/demo.eval.yaml',
tags: ['good', 'bad\nvalue'],
}),
});
expect(badTag.status).toBe(400);
});
});

// ── GET /api/runs/:filename — run_dir + suite_filter for resume UI ─────
Expand Down
Loading
Loading