diff --git a/.gitignore b/.gitignore index d26e4e25a..5b5b0c353 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ examples/**/*.results.jsonl agent-orchestrator.yaml # Agent configuration and activity logs +AGENTS.md.local .agents/ .claude/ .codex/ diff --git a/AGENTS.md b/AGENTS.md index 8356f41c3..8e4348442 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,10 @@ This is a TypeScript monorepo for AgentV - an AI agent evaluation framework. +## Local Overrides + +If `AGENTS.md.local` exists in the repository root, read it after this file and follow it for machine-local workflow details. `AGENTS.md.local` is intentionally ignored by git; it is for local paths, private asset repositories, and environment-specific verification requirements. + ## High-Level Goals AgentV aims to provide a robust, declarative framework for evaluating AI agents. @@ -422,7 +426,9 @@ Before marking any branch as ready for review, complete this checklist: 6. **Dashboard UX verification**: For changes affecting config, scoring display, or dashboard API, use `agent-browser` to verify the Dashboard UI still renders and functions correctly (settings page loads, pass/fail indicators are correct, config saves work). -7. **Mark PR as ready** only after steps 1-6 have been completed AND red/green UAT evidence is included in the PR. +7. **Save visual evidence when required by local overrides:** If `AGENTS.md.local` specifies a private evidence repository or asset location, save Dashboard/docs/browser E2E screenshots there and include the resulting paths/commit in the handoff. + +8. **Mark PR as ready** only after steps 1-7 have been completed AND red/green UAT evidence is included in the PR. ## Documentation Updates @@ -463,7 +469,9 @@ Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore` ### Issue Workflow -Use Beads for live ownership and GitHub for external collaboration. Do not duplicate claim state in a separate live tracker. Push focused commits to the assigned branch and open/update the PR requested by the bead/user. Close the bead only after the scoped work is complete, pushed, and documented with verification evidence. +Use Beads for live ownership and GitHub for external collaboration. Do not duplicate claim state in a separate live tracker. Push focused commits to the assigned branch and open/update the PR requested by the bead/user. A branch, pushed commit, or draft PR is not done for ordinary scoped work. Close the bead only after the scoped work is complete, verified, merged to `main` through a PR, and documented with verification evidence. + +Exception: if the bead is part of an epic/worktree continuation and the work intentionally remains on an ongoing branch, open a draft PR and record the branch name, PR URL, worktree path, current head commit, and remaining scope in the epic bead or parent bead. In that case, keep the child/task bead open or in progress rather than closing it as completed until the PR is merged or the parent explicitly supersedes it. If a commit is a self-contained unit of completed, verified work, push it directly to its assigned remote branch instead of leaving it local for handoff. This applies to feature branches, artifact/documentation branches, and private asset repos. It does not override the rule against pushing directly to `main` in this repository. diff --git a/apps/cli/src/commands/results/delete-run.ts b/apps/cli/src/commands/results/delete-run.ts new file mode 100644 index 000000000..0ee04fa87 --- /dev/null +++ b/apps/cli/src/commands/results/delete-run.ts @@ -0,0 +1,78 @@ +/** + * Shared local run deletion primitive for `agentv results delete` and the + * Dashboard API. + * + * Deletes exactly one local run workspace directory under + * `.agentv/results/runs/`. Callers may pass a run ID, run workspace directory, + * or `index.jsonl` path. Remote runs and paths outside the local results tree + * are rejected before anything is removed. + */ + +import { existsSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +import { RESULT_INDEX_FILENAME, resolveRunManifestPath } from '../eval/result-layout.js'; +import { listResultFiles } from '../inspect/utils.js'; +import { resolveResultSourcePath } from './manifest.js'; +import { isRemoteRunId } from './remote.js'; + +export interface DeleteRunTarget { + readonly runId: string; + readonly runDir: string; + readonly manifestPath: string; +} + +export interface DeleteRunResult extends DeleteRunTarget { + readonly deleted: true; +} + +function localRunsRoot(cwd: string): string { + return path.resolve(cwd, '.agentv', 'results', 'runs'); +} + +function assertLocalRunManifest(cwd: string, manifestPath: string, runId: string): DeleteRunTarget { + const resolvedManifestPath = path.resolve(manifestPath); + if (path.basename(resolvedManifestPath) !== RESULT_INDEX_FILENAME) { + throw new Error('Expected a run workspace directory or index.jsonl manifest'); + } + + const runDir = path.dirname(resolvedManifestPath); + const runsRoot = localRunsRoot(cwd); + const relativeRunDir = path.relative(runsRoot, runDir); + if (relativeRunDir === '' || relativeRunDir.startsWith('..') || path.isAbsolute(relativeRunDir)) { + throw new Error('Run workspace is outside the local results directory'); + } + if (!existsSync(resolvedManifestPath)) { + throw new Error(`Run not found: ${runId}`); + } + + return { runId, runDir, manifestPath: resolvedManifestPath }; +} + +export function resolveDeleteRunTarget(cwd: string, runIdOrPath: string): DeleteRunTarget { + const requested = runIdOrPath.trim(); + if (!requested) { + throw new Error('Run ID is required'); + } + if (isRemoteRunId(requested)) { + throw new Error('Run deletion is only available for local runs'); + } + + const localMeta = listResultFiles(cwd).find((run) => run.filename === requested); + if (localMeta) { + return assertLocalRunManifest(cwd, localMeta.path, requested); + } + + const resolvedSource = resolveResultSourcePath(requested, cwd); + if (!existsSync(resolvedSource)) { + throw new Error(`Run not found: ${requested}`); + } + const manifestPath = resolveRunManifestPath(resolvedSource); + return assertLocalRunManifest(cwd, manifestPath, requested); +} + +export function deleteLocalRun(cwd: string, runIdOrPath: string): DeleteRunResult { + const target = resolveDeleteRunTarget(cwd, runIdOrPath); + rmSync(target.runDir, { recursive: true, force: false }); + return { ...target, deleted: true }; +} diff --git a/apps/cli/src/commands/results/delete.ts b/apps/cli/src/commands/results/delete.ts new file mode 100644 index 000000000..a16b5b5c0 --- /dev/null +++ b/apps/cli/src/commands/results/delete.ts @@ -0,0 +1,66 @@ +/** + * `agentv results delete` — remove one or more local run workspaces. + * + * The command requires confirmation unless `--yes` is passed. It accepts local + * run IDs, run workspace directories, or `index.jsonl` manifests and refuses + * remote runs. + */ + +import * as readline from 'node:readline/promises'; +import { command, flag, restPositionals, string } from 'cmd-ts'; + +import { deleteLocalRun, resolveDeleteRunTarget } from './delete-run.js'; + +async function confirm(message: string): Promise { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer = (await rl.question(`${message} [y/N] `)).trim().toLowerCase(); + return answer === 'y' || answer === 'yes'; + } finally { + rl.close(); + } +} + +export const resultsDeleteCommand = command({ + name: 'delete', + description: 'Delete one or more local run workspaces', + args: { + runs: restPositionals({ + type: string, + displayName: 'run', + description: 'Local run ID, run workspace directory, or index.jsonl manifest', + }), + yes: flag({ + long: 'yes', + short: 'y', + description: 'Skip confirmation prompt', + }), + }, + handler: async ({ runs, yes }) => { + if (runs.length === 0) { + console.error('Error: provide at least one local run ID or run workspace path'); + process.exit(1); + } + + const cwd = process.cwd(); + try { + const targets = runs.map((run) => resolveDeleteRunTarget(cwd, run)); + if (!yes) { + const confirmed = await confirm(`Delete ${targets.length} local run workspace(s)?`); + if (!confirmed) { + console.log('Cancelled.'); + return; + } + } + + for (const run of runs) { + const deleted = deleteLocalRun(cwd, run); + console.log(`Deleted ${deleted.runId}: ${deleted.runDir}`); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`Error: ${message}`); + process.exit(1); + } + }, +}); diff --git a/apps/cli/src/commands/results/index.ts b/apps/cli/src/commands/results/index.ts index 239682d62..e51d77d64 100644 --- a/apps/cli/src/commands/results/index.ts +++ b/apps/cli/src/commands/results/index.ts @@ -1,6 +1,7 @@ import { subcommands } from 'cmd-ts'; import { resultsCombineCommand } from './combine.js'; +import { resultsDeleteCommand } from './delete.js'; import { resultsExportCommand } from './export.js'; import { resultsFailuresCommand } from './failures.js'; import { resultsReportCommand } from './report.js'; @@ -13,6 +14,7 @@ export const resultsCommand = subcommands({ description: 'Inspect, export, and manage evaluation results', cmds: { combine: resultsCombineCommand, + delete: resultsDeleteCommand, export: resultsExportCommand, report: resultsReportCommand, summary: resultsSummaryCommand, diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 8aab15267..2c949323c 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -63,6 +63,7 @@ import { buildCombineRunSources, combineRunSources, } from './combine-run.js'; +import { deleteLocalRun } from './delete-run.js'; import { getActiveRunStatus, getActiveRunTarget, registerEvalRoutes } from './eval-runner.js'; import { loadLightweightResults, @@ -1078,6 +1079,25 @@ async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext) } } +async function handleRunDelete(c: C, { searchDir, projectId }: DataContext) { + const filename = c.req.param('filename') ?? ''; + const meta = await findRunById(searchDir, filename, projectId); + if (!meta) return c.json({ error: 'Run not found' }, 404); + if (meta.source === 'remote') { + return c.json({ error: 'Run deletion is only available for local runs' }, 400); + } + if (getActiveRunStatus(meta.path) === 'starting' || getActiveRunStatus(meta.path) === 'running') { + return c.json({ error: 'Run is still active' }, 409); + } + + try { + const deleted = deleteLocalRun(searchDir, filename); + return c.json({ ok: true, run_id: deleted.runId }); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } +} + function getLocalRunsRoot(searchDir: string): string { return path.join(searchDir, '.agentv', 'results', 'runs'); } @@ -1436,6 +1456,12 @@ export function createApp( } return handleRunTagsDelete(c, defaultCtx); }); + app.delete('/api/runs/:filename', (c) => { + if (readOnly) { + return c.json({ error: 'Dashboard is running in read-only mode' }, 403); + } + return handleRunDelete(c, defaultCtx); + }); app.get('/api/runs/:filename', (c) => handleRunDetail(c, defaultCtx)); app.get('/api/runs/:filename/log', (c) => handleRunLog(c, defaultCtx)); app.get('/api/runs/:filename/suites', (c) => handleRunSuites(c, defaultCtx)); @@ -1571,6 +1597,12 @@ export function createApp( } return withProject(c, handleRunTagsDelete); }); + app.delete('/api/projects/:projectId/runs/:filename', (c) => { + if (readOnly) { + return c.json({ error: 'Dashboard is running in read-only mode' }, 403); + } + return withProject(c, handleRunDelete); + }); app.get('/api/projects/:projectId/runs/:filename', (c) => withProject(c, handleRunDetail)); app.get('/api/projects/:projectId/runs/:filename/log', (c) => withProject(c, handleRunLog)); app.get('/api/projects/:projectId/runs/:filename/suites', (c) => withProject(c, handleRunSuites)); diff --git a/apps/cli/test/commands/results/delete.test.ts b/apps/cli/test/commands/results/delete.test.ts new file mode 100644 index 000000000..49357524a --- /dev/null +++ b/apps/cli/test/commands/results/delete.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { + deleteLocalRun, + resolveDeleteRunTarget, +} from '../../../src/commands/results/delete-run.js'; + +function toJsonl(record: object): string { + return `${JSON.stringify(record)}\n`; +} + +describe('results delete', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-results-delete-')); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + function seedRun(runId: string): string { + const runDir = path.join(tempDir, '.agentv', 'results', 'runs', ...runId.split('::')); + mkdirSync(runDir, { recursive: true }); + writeFileSync( + path.join(runDir, 'index.jsonl'), + toJsonl({ + timestamp: '2026-06-01T10:00:00.000Z', + test_id: 'test-a', + score: 1, + target: 'mock', + }), + 'utf8', + ); + writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n', 'utf8'); + return runDir; + } + + it('deletes a local run workspace by run ID', () => { + const runDir = seedRun('demo::2026-06-01T10-00-00-000Z'); + + const deleted = deleteLocalRun(tempDir, 'demo::2026-06-01T10-00-00-000Z'); + + expect(deleted.runId).toBe('demo::2026-06-01T10-00-00-000Z'); + expect(existsSync(runDir)).toBe(false); + }); + + it('resolves and deletes by workspace path', () => { + const runDir = seedRun('2026-06-01T10-00-00-000Z'); + + const target = resolveDeleteRunTarget(tempDir, runDir); + expect(target.runDir).toBe(runDir); + + deleteLocalRun(tempDir, runDir); + expect(existsSync(runDir)).toBe(false); + }); + + it('rejects remote IDs and paths outside the local runs directory', () => { + seedRun('2026-06-01T10-00-00-000Z'); + const outsideDir = path.join(tempDir, 'outside-run'); + mkdirSync(outsideDir, { recursive: true }); + writeFileSync(path.join(outsideDir, 'index.jsonl'), toJsonl({ score: 1 }), 'utf8'); + + expect(() => deleteLocalRun(tempDir, 'remote::2026-06-01T10-00-00-000Z')).toThrow('local runs'); + expect(() => deleteLocalRun(tempDir, outsideDir)).toThrow('outside the local results'); + }); + + it('reports missing run IDs as not found', () => { + expect(() => resolveDeleteRunTarget(tempDir, 'missing-run')).toThrow('Run not found'); + }); +}); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 2466f3703..43af6f991 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -1213,6 +1213,113 @@ describe('serve app', () => { }); }); + describe('run delete API', () => { + function seedRun( + name: string, + records: object[] = [RESULT_A], + opts?: { experiment?: string; baseDir?: string }, + ): { runId: string; runDir: string } { + const runsDir = path.join(opts?.baseDir ?? tempDir, '.agentv', 'results', 'runs'); + const runDir = opts?.experiment + ? path.join(runsDir, opts.experiment, name) + : path.join(runsDir, name); + mkdirSync(runDir, { recursive: true }); + writeFileSync(path.join(runDir, 'index.jsonl'), toJsonl(...records)); + writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n'); + return { + runId: opts?.experiment ? `${opts.experiment}::${name}` : name, + runDir, + }; + } + + it('deletes a local run workspace and rejects missing runs', async () => { + const run = seedRun('2026-06-01T10-00-00-000Z'); + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const deleted = await app.request(`/api/runs/${encodeURIComponent(run.runId)}`, { + method: 'DELETE', + }); + expect(deleted.status).toBe(200); + expect(existsSync(run.runDir)).toBe(false); + + const missing = await app.request(`/api/runs/${encodeURIComponent(run.runId)}`, { + method: 'DELETE', + }); + expect(missing.status).toBe(404); + }); + + it('rejects deleting remote runs', async () => { + const previousHome = process.env.AGENTV_HOME; + process.env.AGENTV_HOME = path.join(tempDir, 'agentv-home'); + try { + mkdirSync(path.join(tempDir, '.agentv'), { recursive: true }); + writeFileSync( + path.join(tempDir, '.agentv', 'config.yaml'), + `results: + mode: github + repo: EntityProcess/agentv-evals +`, + ); + const remoteRunDir = path.join( + process.env.AGENTV_HOME, + 'results', + 'EntityProcess-agentv-evals', + '.agentv', + 'results', + 'runs', + 'default', + '2026-06-01T11-00-00-000Z', + ); + mkdirSync(remoteRunDir, { recursive: true }); + writeFileSync(path.join(remoteRunDir, 'index.jsonl'), toJsonl(RESULT_B)); + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const res = await app.request( + `/api/runs/${encodeURIComponent('remote::2026-06-01T11-00-00-000Z')}`, + { method: 'DELETE' }, + ); + + expect(res.status).toBe(400); + expect(existsSync(remoteRunDir)).toBe(true); + } finally { + if (previousHome === undefined) { + process.env.AGENTV_HOME = undefined; + } else { + process.env.AGENTV_HOME = previousHome; + } + } + }); + + it('supports project-scoped run deletion within the selected project', async () => { + const homedirSpy = spyOn(os, 'homedir').mockReturnValue(path.join(tempDir, 'home')); + try { + const projectDir = path.join(tempDir, 'project-one'); + const otherProjectDir = path.join(tempDir, 'project-two'); + mkdirSync(path.join(projectDir, '.agentv'), { recursive: true }); + mkdirSync(path.join(otherProjectDir, '.agentv'), { recursive: true }); + const project = addProject(projectDir); + addProject(otherProjectDir); + + const run = seedRun('2026-06-01T10-00-00-000Z', [RESULT_A], { baseDir: projectDir }); + const otherRun = seedRun('2026-06-01T10-00-00-000Z', [RESULT_B], { + baseDir: otherProjectDir, + }); + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const deleted = await app.request( + `/api/projects/${project.id}/runs/${encodeURIComponent(run.runId)}`, + { method: 'DELETE' }, + ); + + expect(deleted.status).toBe(200); + expect(existsSync(run.runDir)).toBe(false); + expect(existsSync(otherRun.runDir)).toBe(true); + } finally { + homedirSpy.mockRestore(); + } + }); + }); + describe('GET /api/runs/:filename/evals/:evalId/files/*', () => { it('loads file content for experiment-scoped run ids', async () => { const runsDir = path.join(tempDir, '.agentv', 'results', 'runs', 'with-skills'); diff --git a/apps/dashboard/src/components/RunList.tsx b/apps/dashboard/src/components/RunList.tsx index 2afcd9e55..cafa67b7b 100644 --- a/apps/dashboard/src/components/RunList.tsx +++ b/apps/dashboard/src/components/RunList.tsx @@ -22,6 +22,7 @@ import { CombineRunsApiError, DEFAULT_PASS_THRESHOLD, combineRunsApi, + deleteRunApi, useStudioConfig, } from '~/lib/api'; import { formatRunLabel } from '~/lib/run-label'; @@ -76,6 +77,7 @@ export function RunList({ const [selectedRunIds, setSelectedRunIds] = useState([]); const [combineError, setCombineError] = useState(null); const [combineInFlight, setCombineInFlight] = useState(false); + const [deleteInFlight, setDeleteInFlight] = useState(false); const selectableRunIds = useMemo( () => runs @@ -173,6 +175,29 @@ export function RunList({ } } + async function handleDelete() { + if (selectedRunIds.length === 0 || deleteInFlight) return; + const count = selectedRunIds.length; + const confirmed = window.confirm( + `Delete ${count} local run${count === 1 ? '' : 's'}? This removes the run workspace and artifacts from disk.`, + ); + if (!confirmed) return; + + setCombineError(null); + setDeleteInFlight(true); + try { + for (const runId of selectedRunIds) { + await deleteRunApi(runId, projectId); + } + setSelectedRunIds([]); + await invalidateRunQueries(); + } catch (err) { + setCombineError((err as Error).message); + } finally { + setDeleteInFlight(false); + } + } + function toggleRun(runId: string) { setSelectedRunIds((current) => current.includes(runId) ? current.filter((id) => id !== runId) : [...current, runId], @@ -207,14 +232,24 @@ export function RunList({

{combineError &&

{combineError}

} - +
+ + +
)}
diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 5cc4f8d19..44c2fb5f7 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -574,6 +574,17 @@ export async function combineRunsApi( return res.json() as Promise; } +export async function deleteRunApi(runId: string, projectId?: string): Promise { + const url = projectId + ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(runId)}` + : `/api/runs/${encodeURIComponent(runId)}`; + const res = await fetch(url, { method: 'DELETE' }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.statusText })); + throw new Error((err as { error?: string }).error ?? `Failed to delete run: ${res.status}`); + } +} + // ── Run tag mutations ──────────────────────────────────────────────────── /** diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 472c1f69b..b5c6d6b2c 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -90,6 +90,16 @@ Click any run to see a breakdown by suite, per-test scores, target, duration, an AgentV Dashboard run detail showing 100% pass rate across 5 tests with scores and duration +## Run management + +In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. + +The same deletion primitive is available from the CLI: + +```bash +agentv results delete --yes +``` + ## Experiments The Experiments tab groups runs by experiment name so you can compare the impact of changes — for example, `with_skills` vs `without_skills`.