Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ examples/**/*.results.jsonl
agent-orchestrator.yaml

# Agent configuration and activity logs
AGENTS.md.local
.agents/
.claude/
.codex/
Expand Down
12 changes: 10 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

This is a TypeScript monorepo for AgentV - an AI agent evaluation framework.

## Local Overrides

If `AGENTS.md.local` exists in the repository root, read it after this file and follow it for machine-local workflow details. `AGENTS.md.local` is intentionally ignored by git; it is for local paths, private asset repositories, and environment-specific verification requirements.

## High-Level Goals

AgentV aims to provide a robust, declarative framework for evaluating AI agents.
Expand Down Expand Up @@ -422,7 +426,9 @@ Before marking any branch as ready for review, complete this checklist:

6. **Dashboard UX verification**: For changes affecting config, scoring display, or dashboard API, use `agent-browser` to verify the Dashboard UI still renders and functions correctly (settings page loads, pass/fail indicators are correct, config saves work).

7. **Mark PR as ready** only after steps 1-6 have been completed AND red/green UAT evidence is included in the PR.
7. **Save visual evidence when required by local overrides:** If `AGENTS.md.local` specifies a private evidence repository or asset location, save Dashboard/docs/browser E2E screenshots there and include the resulting paths/commit in the handoff.

8. **Mark PR as ready** only after steps 1-7 have been completed AND red/green UAT evidence is included in the PR.

## Documentation Updates

Expand Down Expand Up @@ -463,7 +469,9 @@ Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore`

### Issue Workflow

Use Beads for live ownership and GitHub for external collaboration. Do not duplicate claim state in a separate live tracker. Push focused commits to the assigned branch and open/update the PR requested by the bead/user. Close the bead only after the scoped work is complete, pushed, and documented with verification evidence.
Use Beads for live ownership and GitHub for external collaboration. Do not duplicate claim state in a separate live tracker. Push focused commits to the assigned branch and open/update the PR requested by the bead/user. A branch, pushed commit, or draft PR is not done for ordinary scoped work. Close the bead only after the scoped work is complete, verified, merged to `main` through a PR, and documented with verification evidence.

Exception: if the bead is part of an epic/worktree continuation and the work intentionally remains on an ongoing branch, open a draft PR and record the branch name, PR URL, worktree path, current head commit, and remaining scope in the epic bead or parent bead. In that case, keep the child/task bead open or in progress rather than closing it as completed until the PR is merged or the parent explicitly supersedes it.

If a commit is a self-contained unit of completed, verified work, push it directly to its assigned remote branch instead of leaving it local for handoff. This applies to feature branches, artifact/documentation branches, and private asset repos. It does not override the rule against pushing directly to `main` in this repository.

Expand Down
78 changes: 78 additions & 0 deletions apps/cli/src/commands/results/delete-run.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/**
* Shared local run deletion primitive for `agentv results delete` and the
* Dashboard API.
*
* Deletes exactly one local run workspace directory under
* `.agentv/results/runs/`. Callers may pass a run ID, run workspace directory,
* or `index.jsonl` path. Remote runs and paths outside the local results tree
* are rejected before anything is removed.
*/

import { existsSync, rmSync } from 'node:fs';
import path from 'node:path';

import { RESULT_INDEX_FILENAME, resolveRunManifestPath } from '../eval/result-layout.js';
import { listResultFiles } from '../inspect/utils.js';
import { resolveResultSourcePath } from './manifest.js';
import { isRemoteRunId } from './remote.js';

export interface DeleteRunTarget {
readonly runId: string;
readonly runDir: string;
readonly manifestPath: string;
}

export interface DeleteRunResult extends DeleteRunTarget {
readonly deleted: true;
}

function localRunsRoot(cwd: string): string {
return path.resolve(cwd, '.agentv', 'results', 'runs');
}

function assertLocalRunManifest(cwd: string, manifestPath: string, runId: string): DeleteRunTarget {
const resolvedManifestPath = path.resolve(manifestPath);
if (path.basename(resolvedManifestPath) !== RESULT_INDEX_FILENAME) {
throw new Error('Expected a run workspace directory or index.jsonl manifest');
}

const runDir = path.dirname(resolvedManifestPath);
const runsRoot = localRunsRoot(cwd);
const relativeRunDir = path.relative(runsRoot, runDir);
if (relativeRunDir === '' || relativeRunDir.startsWith('..') || path.isAbsolute(relativeRunDir)) {
throw new Error('Run workspace is outside the local results directory');
}
if (!existsSync(resolvedManifestPath)) {
throw new Error(`Run not found: ${runId}`);
}

return { runId, runDir, manifestPath: resolvedManifestPath };
}

export function resolveDeleteRunTarget(cwd: string, runIdOrPath: string): DeleteRunTarget {
const requested = runIdOrPath.trim();
if (!requested) {
throw new Error('Run ID is required');
}
if (isRemoteRunId(requested)) {
throw new Error('Run deletion is only available for local runs');
}

const localMeta = listResultFiles(cwd).find((run) => run.filename === requested);
if (localMeta) {
return assertLocalRunManifest(cwd, localMeta.path, requested);
}

const resolvedSource = resolveResultSourcePath(requested, cwd);
if (!existsSync(resolvedSource)) {
throw new Error(`Run not found: ${requested}`);
}
const manifestPath = resolveRunManifestPath(resolvedSource);
return assertLocalRunManifest(cwd, manifestPath, requested);
}

export function deleteLocalRun(cwd: string, runIdOrPath: string): DeleteRunResult {
const target = resolveDeleteRunTarget(cwd, runIdOrPath);
rmSync(target.runDir, { recursive: true, force: false });
return { ...target, deleted: true };
}
66 changes: 66 additions & 0 deletions apps/cli/src/commands/results/delete.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/**
* `agentv results delete` — remove one or more local run workspaces.
*
* The command requires confirmation unless `--yes` is passed. It accepts local
* run IDs, run workspace directories, or `index.jsonl` manifests and refuses
* remote runs.
*/

import * as readline from 'node:readline/promises';
import { command, flag, restPositionals, string } from 'cmd-ts';

import { deleteLocalRun, resolveDeleteRunTarget } from './delete-run.js';

async function confirm(message: string): Promise<boolean> {
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
try {
const answer = (await rl.question(`${message} [y/N] `)).trim().toLowerCase();
return answer === 'y' || answer === 'yes';
} finally {
rl.close();
}
}

export const resultsDeleteCommand = command({
name: 'delete',
description: 'Delete one or more local run workspaces',
args: {
runs: restPositionals({
type: string,
displayName: 'run',
description: 'Local run ID, run workspace directory, or index.jsonl manifest',
}),
yes: flag({
long: 'yes',
short: 'y',
description: 'Skip confirmation prompt',
}),
},
handler: async ({ runs, yes }) => {
if (runs.length === 0) {
console.error('Error: provide at least one local run ID or run workspace path');
process.exit(1);
}

const cwd = process.cwd();
try {
const targets = runs.map((run) => resolveDeleteRunTarget(cwd, run));
if (!yes) {
const confirmed = await confirm(`Delete ${targets.length} local run workspace(s)?`);
if (!confirmed) {
console.log('Cancelled.');
return;
}
}

for (const run of runs) {
const deleted = deleteLocalRun(cwd, run);
console.log(`Deleted ${deleted.runId}: ${deleted.runDir}`);
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error(`Error: ${message}`);
process.exit(1);
}
},
});
2 changes: 2 additions & 0 deletions apps/cli/src/commands/results/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { subcommands } from 'cmd-ts';

import { resultsCombineCommand } from './combine.js';
import { resultsDeleteCommand } from './delete.js';
import { resultsExportCommand } from './export.js';
import { resultsFailuresCommand } from './failures.js';
import { resultsReportCommand } from './report.js';
Expand All @@ -13,6 +14,7 @@ export const resultsCommand = subcommands({
description: 'Inspect, export, and manage evaluation results',
cmds: {
combine: resultsCombineCommand,
delete: resultsDeleteCommand,
export: resultsExportCommand,
report: resultsReportCommand,
summary: resultsSummaryCommand,
Expand Down
32 changes: 32 additions & 0 deletions apps/cli/src/commands/results/serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ import {
buildCombineRunSources,
combineRunSources,
} from './combine-run.js';
import { deleteLocalRun } from './delete-run.js';
import { getActiveRunStatus, getActiveRunTarget, registerEvalRoutes } from './eval-runner.js';
import {
loadLightweightResults,
Expand Down Expand Up @@ -1078,6 +1079,25 @@ async function handleRunTagsDelete(c: C, { searchDir, projectId }: DataContext)
}
}

async function handleRunDelete(c: C, { searchDir, projectId }: DataContext) {
const filename = c.req.param('filename') ?? '';
const meta = await findRunById(searchDir, filename, projectId);
if (!meta) return c.json({ error: 'Run not found' }, 404);
if (meta.source === 'remote') {
return c.json({ error: 'Run deletion is only available for local runs' }, 400);
}
if (getActiveRunStatus(meta.path) === 'starting' || getActiveRunStatus(meta.path) === 'running') {
return c.json({ error: 'Run is still active' }, 409);
}

try {
const deleted = deleteLocalRun(searchDir, filename);
return c.json({ ok: true, run_id: deleted.runId });
} catch (err) {
return c.json({ error: (err as Error).message }, 400);
}
}

function getLocalRunsRoot(searchDir: string): string {
return path.join(searchDir, '.agentv', 'results', 'runs');
}
Expand Down Expand Up @@ -1436,6 +1456,12 @@ export function createApp(
}
return handleRunTagsDelete(c, defaultCtx);
});
app.delete('/api/runs/:filename', (c) => {
if (readOnly) {
return c.json({ error: 'Dashboard is running in read-only mode' }, 403);
}
return handleRunDelete(c, defaultCtx);
});
app.get('/api/runs/:filename', (c) => handleRunDetail(c, defaultCtx));
app.get('/api/runs/:filename/log', (c) => handleRunLog(c, defaultCtx));
app.get('/api/runs/:filename/suites', (c) => handleRunSuites(c, defaultCtx));
Expand Down Expand Up @@ -1571,6 +1597,12 @@ export function createApp(
}
return withProject(c, handleRunTagsDelete);
});
app.delete('/api/projects/:projectId/runs/:filename', (c) => {
if (readOnly) {
return c.json({ error: 'Dashboard is running in read-only mode' }, 403);
}
return withProject(c, handleRunDelete);
});
app.get('/api/projects/:projectId/runs/:filename', (c) => withProject(c, handleRunDetail));
app.get('/api/projects/:projectId/runs/:filename/log', (c) => withProject(c, handleRunLog));
app.get('/api/projects/:projectId/runs/:filename/suites', (c) => withProject(c, handleRunSuites));
Expand Down
75 changes: 75 additions & 0 deletions apps/cli/test/commands/results/delete.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';

import {
deleteLocalRun,
resolveDeleteRunTarget,
} from '../../../src/commands/results/delete-run.js';

function toJsonl(record: object): string {
return `${JSON.stringify(record)}\n`;
}

describe('results delete', () => {
let tempDir: string;

beforeEach(() => {
tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-results-delete-'));
});

afterEach(() => {
rmSync(tempDir, { recursive: true, force: true });
});

function seedRun(runId: string): string {
const runDir = path.join(tempDir, '.agentv', 'results', 'runs', ...runId.split('::'));
mkdirSync(runDir, { recursive: true });
writeFileSync(
path.join(runDir, 'index.jsonl'),
toJsonl({
timestamp: '2026-06-01T10:00:00.000Z',
test_id: 'test-a',
score: 1,
target: 'mock',
}),
'utf8',
);
writeFileSync(path.join(runDir, 'tags.json'), '{"tags":["stale"]}\n', 'utf8');
return runDir;
}

it('deletes a local run workspace by run ID', () => {
const runDir = seedRun('demo::2026-06-01T10-00-00-000Z');

const deleted = deleteLocalRun(tempDir, 'demo::2026-06-01T10-00-00-000Z');

expect(deleted.runId).toBe('demo::2026-06-01T10-00-00-000Z');
expect(existsSync(runDir)).toBe(false);
});

it('resolves and deletes by workspace path', () => {
const runDir = seedRun('2026-06-01T10-00-00-000Z');

const target = resolveDeleteRunTarget(tempDir, runDir);
expect(target.runDir).toBe(runDir);

deleteLocalRun(tempDir, runDir);
expect(existsSync(runDir)).toBe(false);
});

it('rejects remote IDs and paths outside the local runs directory', () => {
seedRun('2026-06-01T10-00-00-000Z');
const outsideDir = path.join(tempDir, 'outside-run');
mkdirSync(outsideDir, { recursive: true });
writeFileSync(path.join(outsideDir, 'index.jsonl'), toJsonl({ score: 1 }), 'utf8');

expect(() => deleteLocalRun(tempDir, 'remote::2026-06-01T10-00-00-000Z')).toThrow('local runs');
expect(() => deleteLocalRun(tempDir, outsideDir)).toThrow('outside the local results');
});

it('reports missing run IDs as not found', () => {
expect(() => resolveDeleteRunTarget(tempDir, 'missing-run')).toThrow('Run not found');
});
});
Loading
Loading