diff --git a/.agents/conventions.md b/.agents/conventions.md index c0d1e3a11..f104b4fff 100644 --- a/.agents/conventions.md +++ b/.agents/conventions.md @@ -136,16 +136,16 @@ Before adding a new pointer family, verify that the artifact is large enough or Grader types use kebab-case everywhere. -- YAML config: `type: llm-grader`, `type: is-json`, `type: execution-metrics` +- YAML config: `type: llm-rubric`, `type: llm-rubric`, `type: script`, `type: is-json` - Internal TypeScript: `EvaluatorKind = 'llm-grader' | 'is-json' | ...` - Output `scores[].type`: `"llm-grader"`, `"is-json"` - Registry keys: `registry.register('llm-grader', ...)` -Source of truth: `EVALUATOR_KIND_VALUES` in `packages/core/src/evaluation/types.ts`. +Source of truth: `GRADER_KIND_VALUES` in `packages/core/src/evaluation/types.ts`. Backward compatibility: -- Snake_case is accepted in YAML by `normalizeGraderType()` in `grader-parser.ts`, for example `llm_judge` -> `llm-grader`. +- Snake_case is accepted in YAML by `normalizeGraderType()` in `grader-parser.ts`, for example `llm_rubric` -> `llm-rubric`. - Single-word types such as `contains`, `equals`, `regex`, `latency`, and `cost` are unchanged. Two type definitions exist and must stay in sync: diff --git a/.agents/product-boundary.md b/.agents/product-boundary.md index 5973b0c78..9a5cfff23 100644 --- a/.agents/product-boundary.md +++ b/.agents/product-boundary.md @@ -40,7 +40,7 @@ AgentV's core should remain minimal. Complex or domain-specific logic belongs in Prefer these extension points before adding a built-in: - `script` graders for custom executable evaluation logic -- plain assertion strings or `g-eval` for structured rubric criteria +- plain assertion strings or `llm-rubric` for structured rubric criteria - `llm-rubric` for promptfoo-compatible free-form rubric checks - `llm-grader` only when a custom prompt, custom grader target, or preprocessing is needed - CLI wrappers that consume AgentV JSON or JSONL output for post-processing such as aggregation, comparison, or reporting diff --git a/README.md b/README.md index 684c5067c..8534ac68c 100644 --- a/README.md +++ b/README.md @@ -18,33 +18,10 @@ Test AI targets on real repo tasks and measure what actually works. - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval. - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts. - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by label from `targets.yaml` or with an eval-local target object. -- **Experiment** is the run/result grouping label being measured over that corpus, such as `with-skills` or `without-skills`. Keep suite/category and target/model names out of this label. +- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag. - **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`. - **Default test** configures inherited per-test defaults such as score `threshold`. -- **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. - -```mermaid -flowchart LR - corpus["Eval suite / imports / tests
task corpus"] - category["Category
path-derived grouping"] - context["Workspace / fixtures / graders
task-owned context"] - experiment["Experiment
named run condition"] - target["Target
system under test"] - controls["Run controls
execution + gates"] - run["Run
concrete execution"] - artifacts["Run artifacts
summary.json + index.jsonl + sidecars"] - readers["Dashboard / compare / trend
derived readers"] - - corpus --> category - corpus --> run - context --> run - category --> run - experiment --> run - target --> run - controls --> run - run --> artifacts - artifacts --> readers -``` +- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. ## Quick start @@ -80,7 +57,7 @@ default_test: threshold: 0.8 workspace: - isolation: per_case + scope: attempt repos: - path: ./fixture repo: EntityProcess/agentv-contract-fixture @@ -89,14 +66,14 @@ workspace: tests: - id: fizzbuzz input: Write FizzBuzz in Python - assertions: + assert: - type: contains value: "fizz" - Implements correct FizzBuzz logic for multiples of 3, 5, and 15 - type: script command: ["python3", "./validators/check_syntax.py"] - - type: g-eval - criteria: + - type: llm-rubric + value: - outcome: Solution is simple and idiomatic Python weight: 0.5 - outcome: Handles the 3, 5, and 15 branches correctly @@ -104,10 +81,10 @@ tests: ``` Plain assertion strings are short-form rubric criteria: AgentV groups them into -`g-eval` and writes each criterion to `grading.json.assertion_results` for the -Dashboard. Use explicit `type: g-eval` when you need weights, required flags, or -`score_ranges`; use `type: llm-rubric` for promptfoo-compatible free-form rubric -assertions; use `type: llm-grader` only when you need a custom grader prompt, +`llm-rubric` and writes each criterion to `grading.json.assertion_results` for the +Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or +`score_ranges`; use string `value` for promptfoo-compatible free-form rubric +checks; use `type: llm-grader` only when you need a custom grader prompt, grader target, or preprocessing. Executable graders use `type: script`. The target can be an eval-local object when this eval needs target settings of its own: @@ -134,7 +111,20 @@ tests: `target: copilot-sdk` resolves the target label from `.agentv/targets.yaml` or `targets.yaml` and uses its default provider, model, hooks, and provider settings. The object form above starts from `copilot-sdk`, then applies the eval-local fields for this eval. If `extends` is omitted, the object defines the full target inline and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata. -Use `default_test.threshold` for the inherited per-test pass cutoff. Existing eval files with a top-level `threshold` still load during migration, and `--threshold` on the CLI still overrides YAML thresholds for a run. +Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file, matching promptfoo's external defaults pattern: + +```yaml +default_test: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml +``` + +AgentV makes `AGENTV_REPO_ROOT` available during eval/config interpolation. Projects that prefer a short name can define their own reference in `.agentv/config.yaml`; `global-default` below is just an example key: + +```yaml +refs: + global-default: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml +``` + +Then eval files in that project can use `default_test: ref://global-default`. **4. Run it:** ```bash @@ -195,11 +185,11 @@ const { results, summary } = await evaluate({ { id: 'fizzbuzz', input: 'Write FizzBuzz in Python', - assertions: [ + assert: [ { type: 'contains', value: 'fizz' }, 'Implements correct FizzBuzz logic for multiples of 3, 5, and 15', { type: 'script', command: ['python3', './validators/check_syntax.py'] }, - { type: 'g-eval', criteria: ['Solution is simple and idiomatic Python'] }, + { type: 'llm-rubric', value: ['Solution is simple and idiomatic Python'] }, ], }, ], @@ -227,7 +217,7 @@ export default defineEval({ }, threshold: 0.8, workspace: { - isolation: 'per_case', + scope: 'attempt', repos: [ { path: './fixture', @@ -240,11 +230,11 @@ export default defineEval({ { id: 'fizzbuzz', input: 'Write FizzBuzz in Python', - assertions: [ + assert: [ { type: 'contains', value: 'fizz' }, 'Implements correct FizzBuzz logic for multiples of 3, 5, and 15', { type: 'script', command: ['python3', './validators/check_syntax.py'] }, - { type: 'g-eval', criteria: ['Solution is simple and idiomatic Python'] }, + { type: 'llm-rubric', value: ['Solution is simple and idiomatic Python'] }, ], }, ], diff --git a/STRATEGY.md b/STRATEGY.md index 5f52388d9..b4056eabd 100644 --- a/STRATEGY.md +++ b/STRATEGY.md @@ -28,7 +28,7 @@ AgentV stays repo-native and workspace-native: it runs or imports evaluations ar ### Workspace-native evaluation -Make real repository workflows first-class: repo acquisition, hooks, pooled workspaces, replay/import paths, and reuse of existing harnesses. +Make real repository workflows first-class: repo acquisition, hooks, suite/attempt workspaces, replay/import paths, and reuse of existing harnesses. _Why it serves the approach:_ This keeps AgentV attached to the actual work the agent is being judged on instead of collapsing it into a synthetic runner. diff --git a/apps/cli/src/commands/convert/index.ts b/apps/cli/src/commands/convert/index.ts index 658aa3da9..83bc70f64 100644 --- a/apps/cli/src/commands/convert/index.ts +++ b/apps/cli/src/commands/convert/index.ts @@ -72,9 +72,9 @@ export function convertEvalsJsonToYaml(inputPath: string): string { lines.push('# AgentV features you can add:'); lines.push('# - type: is-json, contains, regex for deterministic graders'); lines.push('# - type: script for custom scoring scripts'); - lines.push('# - type: g-eval criteria with weights and score ranges for rubrics'); + lines.push('# - type: llm-rubric value arrays with weights and score ranges for rubrics'); lines.push('# - Multi-turn conversations via input message arrays'); - lines.push('# - Multiple assertions with weighted scoring'); + lines.push('# - Multiple assert entries with weighted scoring'); lines.push('# - Workspace isolation with repos and hooks'); lines.push(''); @@ -125,10 +125,10 @@ export function convertEvalsJsonToYaml(inputPath: string): string { ' # Promoted from evals.json expected_output, assertions[], and expectations[]', ); lines.push(' # Replace with type: is-json, contains, or regex for deterministic checks'); - lines.push(' assertions:'); - lines.push(' - name: agent-skills-criteria'); - lines.push(' type: g-eval'); - lines.push(' criteria:'); + lines.push(' assert:'); + lines.push(' - metric: agent-skills-criteria'); + lines.push(' type: llm-rubric'); + lines.push(' value:'); for (const criterion of test.criteria) { lines.push(` - id: ${quoteYamlString(criterion.id)}`); lines.push(` outcome: ${quoteYamlString(criterion.outcome)}`); diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 6579b77bd..743515edd 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -34,36 +34,35 @@ export default defineAssertion(({ output }) => { const EVAL_TEMPLATES: Record string> = { default: (name: string) => `description: ${name} evaluation suite -execution: - target: default +target: default tests: - id: sample-test criteria: Agent responds correctly input: "Hello, how are you?" expected_output: "I'm doing well" - assertions: + assert: - type: contains value: "well" `, rubric: (name: string) => `description: ${name} evaluation suite -execution: - target: default +target: default tests: - id: sample-test criteria: Agent responds correctly and completely input: "Hello, how are you?" expected_output: "I'm doing well, thank you for asking!" - assertions: - - type: llm-grader - rubric: - accuracy: + assert: + - metric: response-quality + type: llm-rubric + value: + - id: accuracy + outcome: Response is factually correct weight: 0.6 - criteria: Response is factually correct - completeness: + - id: completeness + outcome: Response addresses all parts of the question weight: 0.4 - criteria: Response addresses all parts of the question `, }; @@ -128,7 +127,7 @@ export const createAssertionCommand = command({ await mkdir(dir, { recursive: true }); await writeFile(filePath, content); console.log(`Created ${path.relative(process.cwd(), filePath)} (template: ${templateName})`); - console.log(`\nUse in EVAL.yaml:\n assertions:\n - type: ${name}`); + console.log(`\nUse in EVAL.yaml:\n assert:\n - type: ${name}`); }, }); diff --git a/apps/cli/src/commands/eval/commands/assert.ts b/apps/cli/src/commands/eval/commands/assert.ts index 4e7220edc..9a1a096ab 100644 --- a/apps/cli/src/commands/eval/commands/assert.ts +++ b/apps/cli/src/commands/eval/commands/assert.ts @@ -7,7 +7,7 @@ import { buildTraceFromMessages, executeScript } from '@agentv/core'; export const evalAssertCommand = command({ name: 'assert', - description: 'Run a single code-grader assertion from .agentv/graders/ and print the score', + description: 'Run a single script grader assertion from .agentv/graders/ and print the score', args: { graderName: positional({ type: string, @@ -62,8 +62,7 @@ export const evalAssertCommand = command({ process.exit(1); } - // Build payload matching CodeGrader's expected format (snake_case). - // Include all fields that defineCodeGrader validates as required. + // Build payload matching the script grader protocol (snake_case). const messages = [{ role: 'assistant' as const, content: resolvedOutput }]; const inputMessages = [{ role: 'user' as const, content: resolvedInput }]; const trace = buildTraceFromMessages({ diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 09b0f80ef..c52e39876 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -119,15 +119,10 @@ export const evalRunCommand = command({ long: 'verbose', description: 'Enable verbose logging', }), - workspaceMode: option({ - type: optional(string), - long: 'workspace-mode', - description: "Workspace mode: 'temp' (default), 'pooled', or 'static'", - }), workspacePath: option({ type: optional(string), long: 'workspace-path', - description: 'Static workspace directory path (used when workspace mode is static)', + description: 'Static workspace directory path to reuse for this run', }), keepWorkspaces: flag({ long: 'keep-workspaces', @@ -271,7 +266,6 @@ export const evalRunCommand = command({ cachePath: args.cachePath, noCache: args.noCache, verbose: args.verbose, - workspaceMode: args.workspaceMode, workspacePath: args.workspacePath, keepWorkspaces: args.keepWorkspaces, trace: false, diff --git a/apps/cli/src/commands/eval/commands/vitest.ts b/apps/cli/src/commands/eval/commands/vitest.ts index 524dfd8eb..878b019c1 100644 --- a/apps/cli/src/commands/eval/commands/vitest.ts +++ b/apps/cli/src/commands/eval/commands/vitest.ts @@ -9,7 +9,7 @@ function parseCommand(value: string | undefined): readonly string[] | undefined export const evalVitestCommand = command({ name: 'vitest', - description: 'Run Vitest workspace verifier files as an AgentV code-grader protocol adapter', + description: 'Run Vitest workspace verifier files as an AgentV script grader adapter', args: { testFiles: restPositionals({ type: string, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 80bdfc4a7..c8e5b193f 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -294,7 +294,6 @@ interface NormalizedOptions { readonly resume: boolean; readonly rerunFailed: boolean; readonly rerunFailedSource?: string; - readonly workspaceMode?: 'pooled' | 'temp' | 'static'; readonly workspacePath?: string; readonly keepWorkspaces: boolean; /** Removed: use --output instead */ @@ -380,10 +379,6 @@ function normalizeOptionalNumber(value: unknown): number | undefined { return undefined; } -function normalizeWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined { - return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined; -} - function normalizeStringArray(value: unknown): readonly string[] { if (Array.isArray(value)) { return value.filter((v): v is string => typeof v === 'string' && v.trim().length > 0); @@ -696,21 +691,7 @@ function normalizeOptions( const configOutputDir = normalizeString(config?.output?.dir); const cliWorkspacePath = normalizeString(rawOptions.workspacePath); const configWorkspacePath = normalizeString(yamlExecution?.workspace_path); - const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode); - const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode); - if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== 'static') { - throw new Error('--workspace-path requires --workspace-mode=static (or omit --workspace-mode)'); - } - const configWorkspaceMode = normalizeWorkspaceMode(yamlExecution?.workspace_mode); - if (configWorkspacePath && configWorkspaceMode && configWorkspaceMode !== 'static') { - throw new Error( - 'execution.workspace_path requires execution.workspace_mode: static when both are provided', - ); - } - const useConfigWorkspacePath = cliWorkspaceMode === undefined || cliWorkspaceMode === 'static'; - const workspacePath = - cliWorkspacePath ?? (useConfigWorkspacePath ? configWorkspacePath : undefined); - const workspaceMode = workspacePath ? 'static' : (cliWorkspaceMode ?? configWorkspaceMode); + const workspacePath = cliWorkspacePath ?? configWorkspacePath; const resultsRepo = normalizeString(rawOptions.resultsRepo); const resultsPush = normalizeBoolean(rawOptions.resultsPush); const resultsNoPush = normalizeBoolean(rawOptions.noResultsPush); @@ -772,7 +753,6 @@ function normalizeOptions( normalizeBoolean(rawOptions.resume) || normalizeString(rawOptions.rerunFailed) !== undefined, rerunFailed: normalizeString(rawOptions.rerunFailed) !== undefined, rerunFailedSource: normalizeString(rawOptions.rerunFailed), - workspaceMode, workspacePath, // Precedence: CLI > YAML config > TS config keepWorkspaces: @@ -865,7 +845,6 @@ const CLI_RUNTIME_SOURCE_OPTION_KEYS = [ 'recordReplay', 'recordReplayVariant', 'workspacePath', - 'workspaceMode', ] as const; function hasCliRuntimeSource(rawOptions: Record): boolean { @@ -1031,7 +1010,6 @@ function applyExperimentOptions( ...options, target: options.target, agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds, - workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, threshold: options.threshold ?? experiment.threshold, @@ -1760,7 +1738,6 @@ async function runSingleEvalFile(params: { evalCases: testCases, verbose: options.verbose, maxConcurrency: resolvedWorkers, - workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, keepWorkspaces: options.keepWorkspaces, trials: trialsConfig, @@ -2830,9 +2807,7 @@ export async function runEvalCommand( } // Hint about --keep-workspaces when workspaces were used but some cleaned up - const usedWorkspaces = - resultsWithWorkspaces.length > 0 || - (options.workspaceMode && options.workspaceMode !== 'static'); + const usedWorkspaces = resultsWithWorkspaces.length > 0; if (!options.keepWorkspaces && usedWorkspaces) { console.log('Use --keep-workspaces to preserve all workspaces for inspection.'); } diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index df6a2be20..28f4187b3 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -495,10 +495,22 @@ function parseSourceTestCase(test: EvalTest): Record { function withoutLegacyAssertionKeys(testCase: Record): Record { return Object.fromEntries( - Object.entries(testCase).filter(([key]) => key !== 'assert' && key !== 'evaluators'), + Object.entries(testCase).filter(([key]) => key !== 'assert' && key !== 'assertions'), ); } +function serializeGraderDefinition( + definition: Record, + rewrites: ReadonlyMap, +): unknown { + const serialized = rewritePathsDeep(toSnakeCaseDeep(definition), rewrites); + if (!isRecord(serialized)) { + return serialized; + } + const { name, metric, ...rest } = serialized; + return typeof name === 'string' && metric === undefined ? { metric: name, ...rest } : serialized; +} + function buildEvalCase( test: EvalTest, rewrites: ReadonlyMap, @@ -508,8 +520,8 @@ function buildEvalCase( if (graderDefinitions.length > 0) { return { ...withoutLegacyAssertionKeys(testCase), - assertions: graderDefinitions.map((grader) => - rewritePathsDeep(toSnakeCaseDeep(grader.definition), rewrites), + assert: graderDefinitions.map((grader) => + serializeGraderDefinition(grader.definition, rewrites), ), }; } diff --git a/apps/cli/src/commands/inspect/score.ts b/apps/cli/src/commands/inspect/score.ts index 75244e827..4141ed850 100644 --- a/apps/cli/src/commands/inspect/score.ts +++ b/apps/cli/src/commands/inspect/score.ts @@ -67,8 +67,7 @@ function parseKeyValues(s: string): Record { */ export function parseAssertSpec(spec: string): GraderConfig { const colonIdx = spec.indexOf(':'); - // Normalize snake_case to kebab-case for backward compat - const type = (colonIdx === -1 ? spec : spec.slice(0, colonIdx)).replace(/_/g, '-'); + const type = colonIdx === -1 ? spec : spec.slice(0, colonIdx); const params = colonIdx === -1 ? '' : spec.slice(colonIdx + 1); switch (type) { diff --git a/apps/cli/src/commands/pipeline/bench.ts b/apps/cli/src/commands/pipeline/bench.ts index daa95683c..539e374fc 100644 --- a/apps/cli/src/commands/pipeline/bench.ts +++ b/apps/cli/src/commands/pipeline/bench.ts @@ -1,5 +1,5 @@ /** - * `agentv pipeline bench` — Merge code-grader and LLM grader scores into final + * `agentv pipeline bench` — Merge script and LLM grader scores into final * benchmark artifacts. * * Reads code_grader_results and llm_grader_results from disk per test. @@ -64,7 +64,7 @@ export const evalBenchCommand = command({ const evaluators: EvaluatorScore[] = []; const allAssertions: { text: string; passed: boolean; evidence: string }[] = []; - // Collect code grader results + // Collect script grader results const codeResultsDir = join(testDir, 'code_grader_results'); try { const resultFiles = (await readdir(codeResultsDir)).filter((f) => f.endsWith('.json')); @@ -72,7 +72,7 @@ export const evalBenchCommand = command({ const result = JSON.parse(await readFile(join(codeResultsDir, file), 'utf8')); evaluators.push({ name: result.name, - type: result.type ?? 'code-grader', + type: result.type ?? 'script', score: result.score, weight: result.weight ?? 1.0, assertions: result.assertions ?? [], @@ -82,7 +82,7 @@ export const evalBenchCommand = command({ } } } catch { - // No code grader results + // No script grader results. } // Collect LLM grader scores from per-test disk results diff --git a/apps/cli/src/commands/pipeline/grade.ts b/apps/cli/src/commands/pipeline/grade.ts index ef4eb3773..3b2dc4bb0 100644 --- a/apps/cli/src/commands/pipeline/grade.ts +++ b/apps/cli/src/commands/pipeline/grade.ts @@ -4,7 +4,7 @@ * * All grader configs live in code_graders/.json. Each config has a `type` * field that determines how it's evaluated: - * - `code-grader` (or configs with a `command` field): executed as external scripts + * - `script` (or configs with a `command` field): executed as external scripts * - Built-in types (contains, regex, equals, etc.): evaluated in-process * * Results are written to code_grader_results/.json for pipeline bench. @@ -62,7 +62,7 @@ export interface GraderTask { /** * Run grader tasks with concurrency and progress feedback. - * Dispatches each task based on its config: code-graders are executed as + * Dispatches each task based on its config: script graders are executed as * external scripts, built-in types (contains, regex, etc.) are evaluated in-process. * Shared by `pipeline grade` and `pipeline run`. */ @@ -103,7 +103,7 @@ export async function runCodeGraders( writeProgress(); }; - /** Run an external code-grader script. */ + /** Run an external script grader. */ const executeCodeGrader = async (graderConfig: Record, task: GraderTask) => { const { testId, resultsDir, responseText, inputData } = task; const graderName = graderConfig.name as string; diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index efd07824c..61d121204 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -16,7 +16,7 @@ * ├── criteria.md * ├── expected_output.json (if present) * ├── llm_graders/.json - * └── code_graders/.json + * └── code_graders/.json # script/deterministic grader configs */ import { readFile } from 'node:fs/promises'; import { mkdir, writeFile } from 'node:fs/promises'; @@ -142,7 +142,7 @@ export const evalInputCommand = command({ await mkdir(testDir, { recursive: true }); testIds.push(test.id); - // input.json — aligned with eval YAML schema and code grader SDK field names + // input.json - aligned with eval YAML schema and script grader payload fields. const inputMessages = test.input.map((m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content, @@ -215,7 +215,7 @@ export const evalInputCommand = command({ console.log(' 1. Dispatch executor subagents — one per test case (all in parallel):'); console.log(' - Each reads //input.json'); console.log(' - Executes the task, writes //response.md'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log( ' 3. Dispatch grader subagents — one per (test × LLM grader) pair (all in parallel):', ); @@ -252,7 +252,7 @@ async function writeGraderConfigs( let hasLlmGraders = false; for (const assertion of assertions) { - if (assertion.type === 'script' || assertion.type === 'code-grader') { + if (assertion.type === 'script') { if (!hasCodeGraders) { await mkdir(codeGradersDir, { recursive: true }); hasCodeGraders = true; diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 8437f732d..4f842e9f8 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -64,7 +64,7 @@ function loadEnvFile(dir: string): Record { export const evalRunCommand = command({ name: 'run', description: - 'Extract inputs, invoke CLI targets, and run code graders (for agent targets, use pipeline input + subagents)', + 'Extract inputs, invoke CLI targets, and run script graders (for agent targets, use pipeline input + subagents)', args: { evalPath: positional({ type: string, @@ -90,7 +90,7 @@ export const evalRunCommand = command({ type: optional(oneOf(['code', 'none'])), long: 'grader-type', description: - 'Which grading phase to run: "code" runs code-graders inline, omit to skip grading (use pipeline grade separately)', + 'Which grading phase to run: "code" runs script graders inline, omit to skip grading (use pipeline grade separately)', }), target: option({ type: optional(string), @@ -347,7 +347,7 @@ export const evalRunCommand = command({ console.log(' 1. Dispatch executor subagents — one per test case (all in parallel):'); console.log(' - Each reads //input.json'); console.log(' - Executes the task, writes //response.md'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log( ' 3. Dispatch grader subagents — one per (test x LLM grader) pair (all in parallel):', ); @@ -363,18 +363,18 @@ export const evalRunCommand = command({ console.log(''); } - // ── Step 3: Run code graders (only when explicitly requested) ───── + // Step 3: Run script graders (only when explicitly requested). if (graderType !== 'code') { console.log(`\nDone. Results in ${outDir}`); console.log(''); if (targetKind === 'agent') { console.log(' The agent must now:'); console.log(' 1. Dispatch executor subagents to generate response.md files'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log(' 3. Dispatch grader subagents for llm_graders/ configs'); console.log(' 4. Merge scores: agentv pipeline bench '); } else { - console.log(' To run code graders: agentv pipeline grade '); + console.log(' To run script graders: agentv pipeline grade '); console.log(' Or re-run with --grader-type code to grade inline.'); } return; @@ -408,7 +408,7 @@ export const evalRunCommand = command({ const graderConcurrency = workers ?? 10; const { totalGraders, totalPassed } = await runCodeGraders(graderTasks, graderConcurrency); - console.log(`Graded ${totalGraders} code-grader(s): ${totalPassed} passed`); + console.log(`Graded ${totalGraders} script grader(s): ${totalPassed} passed`); console.log(''); console.log(`Results in ${outDir}`); console.log(''); @@ -439,7 +439,7 @@ async function writeGraderConfigs( let hasLlmGraders = false; for (const assertion of assertions) { - if (assertion.type === 'script' || assertion.type === 'code-grader') { + if (assertion.type === 'script') { if (!hasCodeGraders) { await mkdir(codeGradersDir, { recursive: true }); hasCodeGraders = true; diff --git a/apps/cli/src/commands/prepare/index.ts b/apps/cli/src/commands/prepare/index.ts index 5a6ad9aeb..84a8eb426 100644 --- a/apps/cli/src/commands/prepare/index.ts +++ b/apps/cli/src/commands/prepare/index.ts @@ -190,7 +190,7 @@ async function placePreparedWorkspace( await mkdir(path.dirname(destinationPath), { recursive: true }); await rm(destinationPath, { recursive: true, force: true }); - if (prepared.cleanupPolicy.mode !== 'static' && prepared.pool === undefined) { + if (prepared.cleanupPolicy.mode !== 'static') { await moveDirectory(sourcePath, destinationPath); return destinationPath; } @@ -333,7 +333,6 @@ async function prepareAttempt(options: { evalCases: suite.tests, testId: options.testId, verbose: false, - workspaceMode: 'temp', retainOnSuccess: 'keep', retainOnFailure: 'keep', }); diff --git a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts index 476c60897..85de7b3c9 100644 --- a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts +++ b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts @@ -164,11 +164,11 @@ export function agentSkillsToAgentVYamlObject(suite: ConvertedAgentSkillsSuite): input: test.prompt, ...(test.criteria.length > 0 ? { - assertions: [ + assert: [ { - name: 'agent-skills-criteria', - type: 'g-eval', - criteria: test.criteria.map((criterion) => ({ ...criterion })), + metric: 'agent-skills-criteria', + type: 'llm-rubric', + value: test.criteria.map((criterion) => ({ ...criterion })), }, ], } diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index b98f71a0e..12e944e35 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -14,8 +14,6 @@ * or as raw/downloadable text with ?raw=1 / ?download=1 * - GET /api/runs/:filename/evals/:evalId/trace-session — read an AgentV * trace sidecar through the Dashboard trace/session read model - * - GET /api/feedback — read feedback reviews - * - POST /api/feedback — write feedback reviews * - GET /api/projects — list registered projects * - POST /api/projects — register a project by path * - DELETE /api/projects/:projectId — unregister a project @@ -209,39 +207,6 @@ function bootstrapCurrentProject( return { currentProjectId: entry.id }; } -// ── Feedback persistence ───────────────────────────────────────────────── - -interface FeedbackReview { - test_id: string; - comment: string; - updated_at: string; -} - -interface FeedbackData { - reviews: FeedbackReview[]; -} - -function feedbackPath(resultDir: string): string { - return path.join(resultDir, 'feedback.json'); -} - -function readFeedback(cwd: string): FeedbackData { - const fp = feedbackPath(cwd); - if (!existsSync(fp)) { - return { reviews: [] }; - } - try { - return JSON.parse(readFileSync(fp, 'utf8')) as FeedbackData; - } catch (err) { - console.error(`Warning: could not parse ${fp}, starting fresh: ${(err as Error).message}`); - return { reviews: [] }; - } -} - -function writeFeedback(cwd: string, data: FeedbackData): void { - writeFileSync(feedbackPath(cwd), `${JSON.stringify(data, null, 2)}\n`, 'utf8'); -} - // ── Shared utilities (used by handler functions) ───────────────────────── export interface FileNode { @@ -2698,61 +2663,6 @@ function handleConfig( }); } -function handleFeedbackRead(c: C, { searchDir }: DataContext) { - return c.json(readFeedback(feedbackStoreDir(searchDir))); -} - -function feedbackStoreDir(searchDir: string): string { - const resultsDir = path.join(searchDir, '.agentv', 'results'); - return existsSync(resultsDir) ? resultsDir : searchDir; -} - -async function handleFeedbackWrite(c: C, resultDir: string) { - let body: unknown; - try { - body = await c.req.json(); - } catch { - return c.json({ error: 'Invalid JSON' }, 400); - } - - if (!body || typeof body !== 'object') { - return c.json({ error: 'Invalid payload' }, 400); - } - - const payload = body as Record; - if (!Array.isArray(payload.reviews)) { - return c.json({ error: 'Missing reviews array' }, 400); - } - - const incoming = payload.reviews as Record[]; - for (const review of incoming) { - if (typeof review.test_id !== 'string' || typeof review.comment !== 'string') { - return c.json({ error: 'Each review must have test_id and comment strings' }, 400); - } - } - - const existing = readFeedback(resultDir); - const now = new Date().toISOString(); - - for (const review of incoming) { - const newReview: FeedbackReview = { - test_id: review.test_id as string, - comment: review.comment as string, - updated_at: now, - }; - - const idx = existing.reviews.findIndex((r) => r.test_id === newReview.test_id); - if (idx >= 0) { - existing.reviews[idx] = newReview; - } else { - existing.reviews.push(newReview); - } - } - - writeFeedback(resultDir, existing); - return c.json(existing); -} - function expandHomePath(inputPath: string): string { if (inputPath === '~') return homedir(); if (inputPath.startsWith('~/') || inputPath.startsWith('~\\')) { @@ -3007,7 +2917,7 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { // ── Hono app factory ───────────────────────────────────────────────────── /** - * Create a Hono app with dashboard, result picker, and feedback API routes. + * Create a Hono app with dashboard and result picker API routes. * Accepts an empty results array for the empty-state dashboard. */ export function createApp( @@ -3326,19 +3236,6 @@ export function createApp( app.get('/api/compare', (c) => handleCompare(c, defaultCtx)); app.get('/api/targets', (c) => handleTargets(c, defaultCtx)); - // Feedback (unscoped — read uses defaultCtx.searchDir as resultDir) - app.get('/api/feedback', (c) => { - const data = readFeedback(resultDir); - return c.json(data); - }); - - app.post('/api/feedback', async (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return handleFeedbackWrite(c, resultDir); - }); - // Aggregated index (unscoped only) app.get('/api/index', async (c) => { const { runs: metas } = await listMergedResultFiles(searchDir, undefined, defaultCtx.projectId); @@ -3449,16 +3346,6 @@ export function createApp( app.get('/api/projects/:projectId/experiments', (c) => withProject(c, handleExperiments)); app.get('/api/projects/:projectId/compare', (c) => withProject(c, handleCompare)); app.get('/api/projects/:projectId/targets', (c) => withProject(c, handleTargets)); - app.get('/api/projects/:projectId/feedback', (c) => withProject(c, handleFeedbackRead)); - app.post('/api/projects/:projectId/feedback', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return withProject(c, (projectContext, ctx) => - handleFeedbackWrite(projectContext, feedbackStoreDir(ctx.searchDir)), - ); - }); - // ── Eval runner routes (discovery, launch, status) ──────────────────── registerEvalRoutes( @@ -3682,7 +3569,6 @@ export const resultsServeCommand = command({ } } - // Use the run directory for feedback storage (matches #764 behavior) const resultDir = sourceFile ? path.dirname(path.resolve(sourceFile)) : cwd; const app = createApp(results, resultDir, cwd, sourceFile, { readOnly, diff --git a/apps/cli/src/commands/workspace/clean.ts b/apps/cli/src/commands/workspace/clean.ts deleted file mode 100644 index b77ba7eb0..000000000 --- a/apps/cli/src/commands/workspace/clean.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { existsSync } from 'node:fs'; -import { readFile, readdir, rm } from 'node:fs/promises'; -import path from 'node:path'; -import { command, flag, option, optional, string } from 'cmd-ts'; - -import { getWorkspacePoolRoot } from '@agentv/core'; - -interface PoolMetadata { - fingerprint: string; - templatePath: string | null; - repos: readonly { - path?: string; - repo?: string; - source?: { type: string; url?: string; path?: string }; - }[]; - createdAt: string; -} - -async function confirm(message: string): Promise { - const readline = await import('node:readline'); - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - const answer = await new Promise((resolve) => { - rl.question(`${message} [y/N] `, resolve); - }); - rl.close(); - return answer.toLowerCase() === 'y'; -} - -export const cleanCommand = command({ - name: 'clean', - description: 'Remove workspace pool entries', - args: { - repo: option({ - type: optional(string), - long: 'repo', - description: 'Only remove pools containing this repo URL', - }), - force: flag({ - long: 'force', - short: 'f', - description: 'Skip confirmation prompt', - }), - }, - handler: async ({ repo, force }) => { - const poolRoot = getWorkspacePoolRoot(); - - if (!existsSync(poolRoot)) { - console.log('No workspace pool entries found.'); - return; - } - - if (repo) { - // Remove only pool entries matching the repo URL - const entries = await readdir(poolRoot, { withFileTypes: true }); - const poolDirs = entries.filter((e) => e.isDirectory()); - const matchingDirs: string[] = []; - - for (const dir of poolDirs) { - const poolDir = path.join(poolRoot, dir.name); - const metadataPath = path.join(poolDir, 'metadata.json'); - - try { - const raw = await readFile(metadataPath, 'utf-8'); - const metadata = JSON.parse(raw) as PoolMetadata; - - const hasRepo = metadata.repos?.some((r) => { - const value = r.repo ?? (r.source?.type === 'git' ? r.source.url : r.source?.path); - return value?.toLowerCase().includes(repo.toLowerCase()) ?? false; - }); - - if (hasRepo) { - matchingDirs.push(poolDir); - } - } catch { - // Skip entries without valid metadata - } - } - - if (matchingDirs.length === 0) { - console.log(`No workspace pool entries found matching repo "${repo}".`); - return; - } - - if (!force) { - const confirmed = await confirm( - `Remove ${matchingDirs.length} pool entry(s) matching repo "${repo}"?`, - ); - if (!confirmed) { - console.log('Cancelled.'); - return; - } - } - - for (const dir of matchingDirs) { - await rm(dir, { recursive: true, force: true }); - console.log(`Removed: ${path.basename(dir).slice(0, 12)}...`); - } - console.log('Done.'); - } else { - // Remove entire pool root - if (!force) { - const confirmed = await confirm(`Remove all workspace pool entries from ${poolRoot}?`); - if (!confirmed) { - console.log('Cancelled.'); - return; - } - } - - await rm(poolRoot, { recursive: true, force: true }); - console.log('Workspace pool cleaned.'); - } - }, -}); diff --git a/apps/cli/src/commands/workspace/index.ts b/apps/cli/src/commands/workspace/index.ts index 50dd3c00f..5eb9ebc64 100644 --- a/apps/cli/src/commands/workspace/index.ts +++ b/apps/cli/src/commands/workspace/index.ts @@ -1,15 +1,11 @@ import { subcommands } from 'cmd-ts'; -import { cleanCommand } from './clean.js'; import { depsCommand } from './deps.js'; -import { listCommand } from './list.js'; export const workspaceCommand = subcommands({ name: 'workspace', - description: 'Manage workspace pool', + description: 'Inspect workspace dependencies', cmds: { - list: listCommand, - clean: cleanCommand, deps: depsCommand, }, }); diff --git a/apps/cli/src/commands/workspace/list.ts b/apps/cli/src/commands/workspace/list.ts deleted file mode 100644 index 8187447de..000000000 --- a/apps/cli/src/commands/workspace/list.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { existsSync } from 'node:fs'; -import { readFile, readdir, stat } from 'node:fs/promises'; -import path from 'node:path'; -import { command } from 'cmd-ts'; - -import { getWorkspacePoolRoot } from '@agentv/core'; - -interface PoolMetadata { - fingerprint: string; - templatePath: string | null; - repos: readonly { - path?: string; - repo?: string; - source?: { type: string; url?: string; path?: string }; - }[]; - createdAt: string; -} - -async function getDirectorySize(dirPath: string): Promise { - let totalSize = 0; - try { - const entries = await readdir(dirPath, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = path.join(dirPath, entry.name); - if (entry.isDirectory()) { - totalSize += await getDirectorySize(fullPath); - } else { - const stats = await stat(fullPath); - totalSize += stats.size; - } - } - } catch { - // Directory might not be readable - } - return totalSize; -} - -function formatSize(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; -} - -export const listCommand = command({ - name: 'list', - description: 'List workspace pool entries', - args: {}, - handler: async () => { - const poolRoot = getWorkspacePoolRoot(); - - if (!existsSync(poolRoot)) { - console.log('No workspace pool entries found.'); - return; - } - - const entries = await readdir(poolRoot, { withFileTypes: true }); - const poolDirs = entries.filter((e) => e.isDirectory()); - - if (poolDirs.length === 0) { - console.log('No workspace pool entries found.'); - return; - } - - for (const dir of poolDirs) { - const poolDir = path.join(poolRoot, dir.name); - const fingerprint = dir.name; - - // Count slots - const poolEntries = await readdir(poolDir, { withFileTypes: true }); - const slots = poolEntries.filter((e) => e.isDirectory() && e.name.startsWith('slot-')); - - // Read metadata - const metadataPath = path.join(poolDir, 'metadata.json'); - let metadata: PoolMetadata | null = null; - try { - const raw = await readFile(metadataPath, 'utf-8'); - metadata = JSON.parse(raw) as PoolMetadata; - } catch { - // metadata.json might not exist - } - - // Compute disk size - const size = await getDirectorySize(poolDir); - - console.log(` ${fingerprint.slice(0, 12)}...`); - console.log(` Slots: ${slots.length}`); - console.log(` Size: ${formatSize(size)}`); - if (metadata) { - if (metadata.templatePath) { - console.log(` Template: ${metadata.templatePath}`); - } - if (metadata.repos && metadata.repos.length > 0) { - const repoSources = metadata.repos.map( - (r) => r.repo ?? (r.source?.type === 'git' ? r.source.url : r.source?.path), - ); - console.log(` Repos: ${repoSources.join(', ')}`); - } - console.log(` Created: ${metadata.createdAt}`); - } - console.log(); - } - }, -}); diff --git a/apps/cli/test/commands/convert/convert-evals-json.test.ts b/apps/cli/test/commands/convert/convert-evals-json.test.ts index 6c0db216c..f19084377 100644 --- a/apps/cli/test/commands/convert/convert-evals-json.test.ts +++ b/apps/cli/test/commands/convert/convert-evals-json.test.ts @@ -44,7 +44,10 @@ describe('convertEvalsJsonToYaml', () => { expect(yaml).toContain('criteria: |-'); expect(yaml).toContain('Something done'); expect(yaml).toContain('agent-skills-criteria'); - expect(yaml).toContain('type: g-eval'); + expect(yaml).toContain('assert:'); + expect(yaml).toContain('metric: agent-skills-criteria'); + expect(yaml).toContain('type: llm-rubric'); + expect(yaml).toContain('value:'); expect(yaml).toContain('expected-outcome'); expect(yaml).toContain('assertion-1'); expect(yaml).toContain('expectation-1'); @@ -63,7 +66,7 @@ describe('convertEvalsJsonToYaml', () => { const yaml = convertEvalsJsonToYaml(filePath); expect(yaml).toContain('id: "1"'); expect(yaml).toContain('Just a prompt'); - expect(yaml).not.toContain('assertions:'); + expect(yaml).not.toContain('assert:'); expect(yaml).not.toContain('expected_output:'); }); @@ -87,13 +90,18 @@ describe('convertEvalsJsonToYaml', () => { it('maps skill_name to tags.skill in the read adapter', () => { const filePath = writeTempJson({ skill_name: 'test-skill', - evals: [{ id: 1, prompt: 'Just a prompt' }], + evals: [{ id: 1, prompt: 'Just a prompt', assertions: ['Check A'] }], }); const yamlObject = agentSkillsToAgentVYamlObject(readAgentSkillsEvalsFile(filePath)); expect(yamlObject.tags).toEqual({ skill: 'test-skill' }); expect(yamlObject.metadata).toEqual({ source_adapter: 'agent-skills-evals-json' }); + expect(yamlObject.tests?.[0]?.assert?.[0]).toMatchObject({ + metric: 'agent-skills-criteria', + type: 'llm-rubric', + value: [{ id: 'assertion-1', outcome: 'Check A' }], + }); }); it('throws on invalid format', () => { diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 80952c0a9..5043cdb68 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -278,7 +278,7 @@ describe('buildGradingArtifact', () => { it('includes evaluators list with AgentV extensions', () => { const result = makeResult({ scores: [ - makeEvaluatorResult({ name: 'format-check', type: 'code-grader', score: 1.0 }), + makeEvaluatorResult({ name: 'format-check', type: 'script', score: 1.0 }), makeEvaluatorResult({ name: 'quality', type: 'llm-grader', score: 0.7 }), ], }); @@ -287,7 +287,7 @@ describe('buildGradingArtifact', () => { expect(grading.graders).toHaveLength(2); expect(grading.graders?.[0].name).toBe('format-check'); - expect(grading.graders?.[0].type).toBe('code-grader'); + expect(grading.graders?.[0].type).toBe('script'); expect(grading.graders?.[1].score).toBe(0.7); }); @@ -2404,7 +2404,7 @@ describe('writeArtifactsFromResults', () => { const parsedEval = parseYamlValue(taskEval) as Record; const [testCase] = parsedEval.tests as Record[]; - const [assertion] = testCase.assertions as Record[]; + const [assertion] = testCase.assert as Record[]; expect(parsedEval.target).toBe('gpt-4o'); expect(testCase.input).toBe('file://files/src/input.txt'); expect(assertion.prompt).toBe('file://graders/src/grader.md'); diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 624ed736c..6687eb02a 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -75,7 +75,7 @@ describe('agentv eval bundle', () => { value: ../data/input.txt - type: text value: Answer using the fixture. - assertions: + assert: - type: contains value: Mock `, @@ -184,7 +184,7 @@ tests: ../data/cases.yaml tests: - id: inline-case input: hello - assertions: + assert: - type: contains value: inline `, @@ -226,7 +226,7 @@ tests: tests: - id: missing-template input: hello - assertions: + assert: - type: contains value: Mock `, diff --git a/apps/cli/test/commands/eval/pipeline/bench.test.ts b/apps/cli/test/commands/eval/pipeline/bench.test.ts index c9b9ab0a6..0bd1bc193 100644 --- a/apps/cli/test/commands/eval/pipeline/bench.test.ts +++ b/apps/cli/test/commands/eval/pipeline/bench.test.ts @@ -30,7 +30,7 @@ describe('pipeline bench', () => { join(codeResultsDir, 'contains.json'), JSON.stringify({ name: 'contains', - type: 'code-grader', + type: 'script', score: 1.0, weight: 1.0, assertions: [{ text: 'Found keyword', passed: true }], diff --git a/apps/cli/test/commands/eval/pipeline/fixtures/builtin-test.eval.yaml b/apps/cli/test/commands/eval/pipeline/fixtures/builtin-test.eval.yaml index 7a6984d73..c4a0220cc 100644 --- a/apps/cli/test/commands/eval/pipeline/fixtures/builtin-test.eval.yaml +++ b/apps/cli/test/commands/eval/pipeline/fixtures/builtin-test.eval.yaml @@ -3,12 +3,12 @@ tests: - id: test-01 input: hello world criteria: Response echoes the input - assertions: - - name: has_hello + assert: + - metric: has_hello type: contains value: hello - - name: matches_pattern + - metric: matches_pattern type: regex value: "h[aeiou]llo" - - name: is_valid_json + - metric: is_valid_json type: is-json diff --git a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml index ca18544f3..d1ccf032c 100644 --- a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml +++ b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml @@ -3,12 +3,12 @@ tests: - id: test-01 input: hello world criteria: Response echoes the input - assertions: - - name: contains_hello - type: code-grader + assert: + - metric: contains_hello + type: script command: node grader-score-1.js weight: 1.0 - - name: relevance + - metric: relevance type: llm-grader prompt: Did the response echo the input? weight: 2.0 diff --git a/apps/cli/test/commands/eval/pipeline/fixtures/no-name.eval.yaml b/apps/cli/test/commands/eval/pipeline/fixtures/no-name.eval.yaml index a793e9c28..4ab4aacd6 100644 --- a/apps/cli/test/commands/eval/pipeline/fixtures/no-name.eval.yaml +++ b/apps/cli/test/commands/eval/pipeline/fixtures/no-name.eval.yaml @@ -2,7 +2,7 @@ tests: - id: test-01 input: hello world criteria: Response echoes the input - assertions: - - name: contains_hello + assert: + - metric: contains_hello type: contains value: hello diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index 9d7aeb4be..963b1e857 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -121,7 +121,7 @@ describe('materializeTaskBundle', () => { const taskTargets = await readFile(paths?.targetsPath ?? '', 'utf8'); const parsedEval = parseYamlValue(taskEval) as Record; const [testCase] = parsedEval.tests as Record[]; - const [assertion] = testCase.assertions as Record[]; + const [assertion] = testCase.assert as Record[]; expect(parsedEval.target).toBe('selected'); expect(parsedEval.execution).toBeUndefined(); diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index df5cd6c91..11584a69b 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -94,7 +94,7 @@ targets: ` workspace: template: ../template -assertions: +assert: ${assertionYaml .trim() .split('\n') @@ -126,8 +126,8 @@ describe('agentv grade prepared attempts', () => { const { evalPath, targetMarker, graderPayloadPath } = await writeFixtureProject( tempDir, ` -- name: workspace-check - type: code-grader +- metric: workspace-check + type: script command: ["bun", "../scripts/workspace-grader.ts"] `, ); @@ -247,7 +247,7 @@ describe('agentv grade prepared attempts', () => { const { evalPath, targetMarker } = await writeFixtureProject( tempDir, ` -- name: expected-tool-sequence +- metric: expected-tool-sequence type: tool-trajectory mode: exact expected: @@ -292,7 +292,7 @@ describe('agentv grade prepared attempts', () => { const { evalPath, targetMarker } = await writeFixtureProject( tempDir, ` -- name: expected-tool-sequence +- metric: expected-tool-sequence type: tool-trajectory mode: in_order expected: @@ -400,7 +400,7 @@ describe('agentv grade prepared attempts', () => { const { evalPath } = await writeFixtureProject( tempDir, ` -- name: expected-tool-sequence +- metric: expected-tool-sequence type: tool-trajectory mode: in_order expected: @@ -443,7 +443,7 @@ describe('agentv grade prepared attempts', () => { const { evalPath } = await writeFixtureProject( tempDir, ` -- name: expected-tool-sequence +- metric: expected-tool-sequence type: tool-trajectory mode: in_order expected: diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index ed850db91..17bc4206e 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -85,7 +85,7 @@ target: command: ["bun", "../scripts/hook.ts", "target_before_each"] assertions: - name: secret-grader - type: code-grader + type: script command: ["bun", "../scripts/grader.ts"] tests: - id: case-1 diff --git a/apps/cli/test/commands/results/export-e2e-providers.test.ts b/apps/cli/test/commands/results/export-e2e-providers.test.ts index c5207c1fb..574e84c00 100644 --- a/apps/cli/test/commands/results/export-e2e-providers.test.ts +++ b/apps/cli/test/commands/results/export-e2e-providers.test.ts @@ -73,7 +73,7 @@ const CODEX_RESULT = { scores: [ { name: 'edit_quality', - type: 'code-grader', + type: 'script', score: 0.9, assertions: [{ text: 'File edited correctly', passed: true }], }, diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 2d27c4c4b..0676c557a 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -723,165 +723,6 @@ describe('serve app', () => { }); }); - // ── GET /api/feedback ────────────────────────────────────────────────── - - describe('GET /api/feedback', () => { - it('returns empty reviews when no feedback file', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback'); - expect(res.status).toBe(200); - const data = await res.json(); - expect(data).toEqual({ reviews: [] }); - }); - }); - - // ── POST /api/feedback ───────────────────────────────────────────────── - - describe('POST /api/feedback', () => { - it('persists reviews to feedback.json', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Looks good!' }], - }), - }); - expect(res.status).toBe(200); - const data = (await res.json()) as { - reviews: { test_id: string; comment: string; updated_at: string }[]; - }; - expect(data.reviews).toHaveLength(1); - expect(data.reviews[0].test_id).toBe('test-greeting'); - expect(data.reviews[0].comment).toBe('Looks good!'); - expect(data.reviews[0].updated_at).toBeDefined(); - - // Verify file exists on disk - const fp = path.join(tempDir, 'feedback.json'); - expect(existsSync(fp)).toBe(true); - const onDisk = JSON.parse(readFileSync(fp, 'utf8')); - expect(onDisk.reviews).toHaveLength(1); - }); - - it('merges with existing reviews', async () => { - const app = makeApp(); - - // First review - await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'First review' }], - }), - }); - - // Second review (different test_id) - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-math', comment: 'Math looks off' }], - }), - }); - - const data = (await res.json()) as { reviews: { test_id: string }[] }; - expect(data.reviews).toHaveLength(2); - expect(data.reviews.map((r: { test_id: string }) => r.test_id).sort()).toEqual([ - 'test-greeting', - 'test-math', - ]); - }); - - it('overwrites duplicate test_id', async () => { - const app = makeApp(); - - // First review - await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Initial' }], - }), - }); - - // Overwrite same test_id - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Updated' }], - }), - }); - - const data = (await res.json()) as { reviews: { test_id: string; comment: string }[] }; - expect(data.reviews).toHaveLength(1); - expect(data.reviews[0].comment).toBe('Updated'); - }); - - it('accepts empty comment string', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: '' }], - }), - }); - expect(res.status).toBe(200); - const data = (await res.json()) as { - reviews: { comment: string }[]; - }; - expect(data.reviews[0].comment).toBe(''); - }); - - it('rejects invalid payload (400)', async () => { - const app = makeApp(); - - // Missing reviews array - const res1 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ foo: 'bar' }), - }); - expect(res1.status).toBe(400); - - // Invalid review entry - const res2 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ reviews: [{ test_id: 123 }] }), - }); - expect(res2.status).toBe(400); - - // Not an object - const res3 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: '"just a string"', - }); - expect(res3.status).toBe(400); - }); - - it('returns 403 in read-only mode', async () => { - const content = toJsonl(RESULT_A, RESULT_B); - const results = loadResults(content); - const app = createApp(results, tempDir, undefined, undefined, { - studioDir, - readOnly: true, - }); - - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'blocked' }], - }), - }); - - expect(res.status).toBe(403); - }); - }); - describe('GET /api/config', () => { it('includes read_only mode and dashboard mode in the config payload', async () => { const content = toJsonl(RESULT_A, RESULT_B); @@ -913,14 +754,6 @@ describe('serve app', () => { const html = await res.text(); expect(html).toContain('agentv'); }); - - it('serves feedback API with empty results', async () => { - const app = createApp([], tempDir, undefined, undefined, { studioDir }); - const res = await app.request('/api/feedback'); - expect(res.status).toBe(200); - const data = await res.json(); - expect(data).toEqual({ reviews: [] }); - }); }); // ── GET /api/runs ─────────────────────────────────────────────────── diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index c069b44c8..1b00d6dc0 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -601,7 +601,7 @@ describe('parseAssertSpec', () => { }); it('should parse token-usage spec with params', () => { - const config = parseAssertSpec('token_usage:max_total=2000,max_input=1500'); + const config = parseAssertSpec('token-usage:max_total=2000,max_input=1500'); expect(config.type).toBe('token-usage'); expect((config as { max_total: number }).max_total).toBe(2000); expect((config as { max_input: number }).max_input).toBe(1500); @@ -613,7 +613,7 @@ describe('parseAssertSpec', () => { }); it('should parse execution-metrics spec', () => { - const config = parseAssertSpec('execution_metrics:max_tool_calls=10,max_tokens=3000'); + const config = parseAssertSpec('execution-metrics:max_tool_calls=10,max_tokens=3000'); expect(config.type).toBe('execution-metrics'); expect((config as { max_tool_calls: number }).max_tool_calls).toBe(10); expect((config as { max_tokens: number }).max_tokens).toBe(3000); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 31dbce6d7..a89fabd8b 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -524,7 +524,6 @@ describe('agentv eval CLI', () => { expect(result.exitCode).toBe(0); const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ - workspaceMode: 'static', workspacePath, resultCount: 2, }); @@ -549,7 +548,6 @@ describe('agentv eval CLI', () => { expect(result.exitCode).toBe(0); const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ - workspaceMode: 'static', workspacePath, resultCount: 2, }); @@ -639,8 +637,8 @@ describe('agentv eval CLI', () => { ' model: gpt-5-codex', 'timeout_seconds: 12', 'threshold: 0.8', - 'budget_usd: 3', 'evaluate_options:', + ' budget_usd: 3', ' repeat:', ' count: 2', ' strategy: pass_any', @@ -825,7 +823,8 @@ describe('agentv eval CLI', () => { 'name: first', 'target: cli-target', 'timeout_seconds: 11', - 'budget_usd: 0.11', + 'evaluate_options:', + ' budget_usd: 0.11', 'tests:', ' - id: first-case', ' input: first', @@ -840,7 +839,8 @@ describe('agentv eval CLI', () => { 'name: second', 'target: file-target', 'timeout_seconds: 22', - 'budget_usd: 0.22', + 'evaluate_options:', + ' budget_usd: 0.22', 'tests:', ' - id: second-case', ' input: second', diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index c97edf54b..0e96f3d01 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -20,7 +20,6 @@ interface RunEvaluationOptionsLike { readonly evalCases?: ReadonlyArray; readonly verbose?: boolean; readonly maxConcurrency?: number; - readonly workspaceMode?: string; readonly workspacePath?: string; readonly trials?: { readonly count: number; @@ -191,7 +190,6 @@ async function maybeWriteDiagnostics( envLocalOnly: process.env.CLI_ENV_LOCAL_ONLY ?? null, budgetUsd: options.budgetUsd ?? null, maxConcurrency: options.maxConcurrency ?? null, - workspaceMode: options.workspaceMode ?? null, workspacePath: options.workspacePath ?? null, trials: options.trials ?? null, threshold: options.threshold ?? null, diff --git a/apps/dashboard/src/components/EvalDetail.tsx b/apps/dashboard/src/components/EvalDetail.tsx index fcd367667..8ce4d45da 100644 --- a/apps/dashboard/src/components/EvalDetail.tsx +++ b/apps/dashboard/src/components/EvalDetail.tsx @@ -1,5 +1,5 @@ /** - * Eval detail view with checks, source traceability, artifact files, and feedback. + * Eval detail view with checks, source traceability, artifact files, and artifacts. * * Layout: compact header → tabs → full-height content area. * Scores and assertions are only visible in the Checks tab. @@ -31,7 +31,6 @@ import type { SourceTraceability, } from '~/lib/types'; -import { FeedbackPanel } from './FeedbackPanel'; import type { FileNode } from './FileTree'; import { FileTree } from './FileTree'; import { MonacoViewer } from './MonacoViewer'; @@ -49,7 +48,7 @@ interface EvalDetailProps { onSelectTrial?: (trial: EvalCaseTrial, initialTab?: Tab) => void; } -type Tab = 'checks' | 'transcript' | 'source' | 'files' | 'feedback'; +type Tab = 'checks' | 'transcript' | 'source' | 'files'; /** Recursively find the first file node in the tree. */ function findFirstFile(nodes: FileNode[]): string | null { @@ -138,8 +137,6 @@ export function EvalDetail({ }: EvalDetailProps) { const [activeTab, setActiveTab] = useState(initialTab); const [selectedFilePath, setSelectedFilePath] = useState(initialSelectedFilePath); - const { data: config } = useStudioConfig(projectId); - const isReadOnly = config?.read_only === true; const detailResult = selectedTrial ? selectedTrialResult(result, selectedTrial) : result; const showAggregateRepeat = repeatGroup != null && selectedTrial == null; @@ -153,7 +150,6 @@ export function EvalDetail({ { id: 'transcript', label: 'Transcript' }, { id: 'source', label: 'Source' }, { id: 'files', label: 'Files' }, - ...(isReadOnly ? [] : [{ id: 'feedback' as const, label: 'Feedback' }]), ]; const openFile = (filePath: string) => { @@ -250,11 +246,6 @@ export function EvalDetail({ )} - {!isReadOnly && activeTab === 'feedback' && ( -
- -
- )} ); diff --git a/apps/dashboard/src/components/FeedbackPanel.tsx b/apps/dashboard/src/components/FeedbackPanel.tsx deleted file mode 100644 index 3901b0ca6..000000000 --- a/apps/dashboard/src/components/FeedbackPanel.tsx +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Feedback panel for leaving review comments on individual eval results. - * - * Reads existing feedback via the /api/feedback endpoint and persists - * new comments via POST /api/feedback. - */ - -import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { useCallback, useEffect, useState } from 'react'; - -import { useFeedback } from '~/lib/api'; - -interface FeedbackPanelProps { - testId: string; - projectId?: string; -} - -function feedbackUrl(projectId?: string): string { - return projectId ? `/api/projects/${encodeURIComponent(projectId)}/feedback` : '/api/feedback'; -} - -async function saveFeedback(testId: string, comment: string, projectId?: string) { - const res = await fetch(feedbackUrl(projectId), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ reviews: [{ test_id: testId, comment }] }), - }); - if (!res.ok) { - throw new Error(`Failed to save feedback: ${res.status}`); - } - return res.json(); -} - -export function FeedbackPanel({ testId, projectId }: FeedbackPanelProps) { - const { data } = useFeedback(projectId); - const queryClient = useQueryClient(); - - const existing = data?.reviews?.find((r) => r.test_id === testId); - const [comment, setComment] = useState(existing?.comment ?? ''); - const [saved, setSaved] = useState(false); - - // Sync when feedback data loads (existing?.comment captures testId changes - // since `existing` is derived from testId via the find() above) - useEffect(() => { - setComment(existing?.comment ?? ''); - setSaved(false); - }, [existing?.comment]); - - const mutation = useMutation({ - mutationFn: () => saveFeedback(testId, comment, projectId), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['feedback', projectId ?? ''] }); - setSaved(true); - setTimeout(() => setSaved(false), 2000); - }, - }); - - const handleSave = useCallback(() => { - mutation.mutate(); - }, [mutation]); - - return ( -
-

Feedback

-