From 7c3cc912aae32aa72a92b21444126d8e0ccd70dc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 01:34:39 +0200 Subject: [PATCH 1/8] feat(cli): adapt Agent Skills evals at CLI boundary --- apps/cli/src/commands/eval/run-eval.ts | 4 ++++ .../cli/src/commands/read-adapters/agent-skills-evals.ts | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 80bdfc4a7..8c7e96fca 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -85,6 +85,10 @@ import { } from './retry-errors.js'; import { resolveCachedRunDir, saveRunCache } from './run-cache.js'; import { findRepoRoot, resolveEvalPaths } from './shared.js'; +import { + agentSkillsToAgentVYamlObject, + readAgentSkillsEvalsFile, +} from '../read-adapters/agent-skills-evals.js'; import { calculateEvaluationSummary, formatEvaluationSummary, diff --git a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts index 476c60897..beb7826b8 100644 --- a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts +++ b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts @@ -53,7 +53,10 @@ function normalizedStringArray(value: unknown): readonly string[] { return isStringArray(value) ? value.map((item) => item.trim()).filter(Boolean) : []; } -export function parseAgentSkillsEvals(parsed: unknown, source: string): ConvertedAgentSkillsSuite { +export function parseAgentSkillsEvals( + parsed: unknown, + source: string, +): ConvertedAgentSkillsSuite { if (!isAgentSkillsEvalsFormat(parsed)) { throw new Error( "Not a valid Agent Skills evals.json: expected top-level 'skill_name' string and 'evals' array", @@ -148,7 +151,9 @@ export function isAgentSkillsEvalsJsonFile(filePath: string): boolean { } } -export function agentSkillsToAgentVYamlObject(suite: ConvertedAgentSkillsSuite): AgentVYamlObject { +export function agentSkillsToAgentVYamlObject( + suite: ConvertedAgentSkillsSuite, +): AgentVYamlObject { return { description: `Evals for ${suite.skillName} skill`, tags: { From d1166a02890779e9f968b4998d4fab6b4f651170 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 01:37:39 +0200 Subject: [PATCH 2/8] style: format Agent Skills adapter changes --- apps/cli/src/commands/eval/run-eval.ts | 4 ---- .../cli/src/commands/read-adapters/agent-skills-evals.ts | 9 ++------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 8c7e96fca..80bdfc4a7 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -85,10 +85,6 @@ import { } from './retry-errors.js'; import { resolveCachedRunDir, saveRunCache } from './run-cache.js'; import { findRepoRoot, resolveEvalPaths } from './shared.js'; -import { - agentSkillsToAgentVYamlObject, - readAgentSkillsEvalsFile, -} from '../read-adapters/agent-skills-evals.js'; import { calculateEvaluationSummary, formatEvaluationSummary, diff --git a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts index beb7826b8..476c60897 100644 --- a/apps/cli/src/commands/read-adapters/agent-skills-evals.ts +++ b/apps/cli/src/commands/read-adapters/agent-skills-evals.ts @@ -53,10 +53,7 @@ function normalizedStringArray(value: unknown): readonly string[] { return isStringArray(value) ? value.map((item) => item.trim()).filter(Boolean) : []; } -export function parseAgentSkillsEvals( - parsed: unknown, - source: string, -): ConvertedAgentSkillsSuite { +export function parseAgentSkillsEvals(parsed: unknown, source: string): ConvertedAgentSkillsSuite { if (!isAgentSkillsEvalsFormat(parsed)) { throw new Error( "Not a valid Agent Skills evals.json: expected top-level 'skill_name' string and 'evals' array", @@ -151,9 +148,7 @@ export function isAgentSkillsEvalsJsonFile(filePath: string): boolean { } } -export function agentSkillsToAgentVYamlObject( - suite: ConvertedAgentSkillsSuite, -): AgentVYamlObject { +export function agentSkillsToAgentVYamlObject(suite: ConvertedAgentSkillsSuite): AgentVYamlObject { return { description: `Evals for ${suite.skillName} skill`, tags: { From 4272ac4ca42bf89eafad92807689acd87a4eb9b1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 04:23:36 +0200 Subject: [PATCH 3/8] feat(eval): finalize promptfoo-aligned restructure --- .agents/conventions.md | 6 +- README.md | 6 +- STRATEGY.md | 2 +- apps/cli/src/commands/eval/commands/assert.ts | 5 +- apps/cli/src/commands/eval/commands/run.ts | 8 +- apps/cli/src/commands/eval/commands/vitest.ts | 2 +- apps/cli/src/commands/eval/run-eval.ts | 29 +- apps/cli/src/commands/eval/task-bundle.ts | 4 +- apps/cli/src/commands/inspect/score.ts | 3 +- apps/cli/src/commands/pipeline/bench.ts | 8 +- apps/cli/src/commands/pipeline/grade.ts | 6 +- apps/cli/src/commands/pipeline/input.ts | 8 +- apps/cli/src/commands/pipeline/run.ts | 16 +- apps/cli/src/commands/prepare/index.ts | 3 +- apps/cli/src/commands/results/serve.ts | 116 +- apps/cli/src/commands/workspace/clean.ts | 113 - apps/cli/src/commands/workspace/index.ts | 6 +- apps/cli/src/commands/workspace/list.ts | 104 - .../commands/eval/artifact-writer.test.ts | 4 +- .../test/commands/eval/pipeline/bench.test.ts | 2 +- .../pipeline/fixtures/input-test.eval.yaml | 2 +- .../commands/grade/grade-prepared.test.ts | 2 +- .../cli/test/commands/prepare/prepare.test.ts | 2 +- .../results/export-e2e-providers.test.ts | 2 +- apps/cli/test/commands/results/serve.test.ts | 167 - apps/cli/test/commands/trace/trace.test.ts | 4 +- apps/cli/test/eval.integration.test.ts | 10 +- apps/cli/test/fixtures/mock-run-evaluation.ts | 2 - apps/dashboard/src/components/EvalDetail.tsx | 13 +- .../src/components/FeedbackPanel.tsx | 89 - apps/dashboard/src/components/ResultTable.tsx | 17 +- apps/dashboard/src/lib/api.ts | 13 - apps/dashboard/src/lib/result-table.test.ts | 6 +- apps/dashboard/src/lib/result-table.ts | 28 +- apps/dashboard/src/lib/types.ts | 10 - .../docs/docs/evaluation/batch-cli.mdx | 4 +- .../docs/docs/evaluation/eval-cases.mdx | 4 +- .../docs/docs/evaluation/eval-files.mdx | 8 +- .../docs/docs/evaluation/experiments.mdx | 2 +- .../content/docs/docs/evaluation/rubrics.mdx | 24 +- .../docs/docs/evaluation/running-evals.mdx | 26 +- .../docs/docs/getting-started/quickstart.mdx | 2 +- .../content/docs/docs/graders/composite.mdx | 4 +- .../docs/docs/graders/execution-metrics.mdx | 10 +- .../content/docs/docs/graders/llm-graders.mdx | 14 +- .../docs/docs/graders/structured-data.mdx | 8 +- .../docs/docs/guides/agent-eval-layers.mdx | 10 +- .../docs/docs/guides/benchmark-provenance.mdx | 6 +- .../content/docs/docs/guides/human-review.mdx | 198 - .../docs/guides/workspace-architecture.mdx | 37 +- .../docs/docs/guides/workspace-pool.mdx | 220 - .../docs/docs/targets/configuration.mdx | 28 +- .../src/content/docs/docs/tools/dashboard.mdx | 5 +- .../src/content/docs/docs/tools/results.mdx | 4 +- .../src/content/docs/docs/tools/validate.mdx | 2 +- ...chmark-execution-behind-runner-boundary.md | 2 +- ...arate-experiments-from-eval-definitions.md | 52 +- ...ct-free-results-sync-without-force-push.md | 4 +- ...mptfoo-superset-eval-authoring-contract.md | 14 +- ...rtifact-and-workspace-resolver-contract.md | 8 +- ...gentv-eval-authoring-extensibility-plan.md | 2 +- ...s-agentv-schema-benchmark-research-plan.md | 2 +- ...at-promptfoo-compatible-extensions-plan.md | 10 +- ...026-07-02-cli-command-surface-alignment.md | 2 +- .../promptfoo-aligned-eval-restructure.md | 28 +- evals/agentv-self/agentv-self.eval.yaml | 6 +- evals/agentv-self/pr-workflow-guard.eval.yaml | 4 +- .../evals/code-grader-contract.eval.yaml | 2 +- .../contract/evals/release-gate.eval.yaml | 2 +- .../evals/repo-materialization.eval.yaml | 2 +- examples/features/README.md | 4 +- .../assert-extended/evals/dataset.eval.yaml | 16 +- .../basic/evals/dataset.eval.baseline.jsonl | 2 +- .../features/basic/evals/dataset.eval.yaml | 2 +- .../evals/dataset.eval.baseline.jsonl | 8 +- .../batch-cli/evals/dataset.eval.yaml | 8 +- .../evals/dataset.eval.baseline.jsonl | 2 +- .../code-grader-sdk/evals/dataset.eval.yaml | 2 +- .../code-grader-with-llm-calls/README.md | 4 +- .../contextual-precision.eval.baseline.jsonl | 6 +- .../evals/contextual-precision.eval.yaml | 2 +- .../contextual-recall.eval.baseline.jsonl | 6 +- .../evals/contextual-recall.eval.yaml | 2 +- .../composite/evals/dataset.eval.yaml | 4 +- .../evals/skill-trigger.EVAL.yaml | 2 +- .../graders/transcript-quality.ts | 2 +- .../features/deterministic-graders/README.md | 8 +- .../evals/dataset.eval.yaml | 4 +- .../evals/docker-example.EVAL.yaml | 2 +- .../features/document-extraction/README.md | 2 +- .../confusion-metrics.eval.baseline.jsonl | 10 +- .../evals/confusion-metrics.eval.yaml | 2 +- .../evals/field-accuracy.eval.baseline.jsonl | 4 +- .../evals/field-accuracy.eval.yaml | 4 +- .../graders/fuzzy_match.ts | 2 +- .../graders/header_confusion_metrics.ts | 2 +- .../graders/line_item_matching.ts | 2 +- .../graders/multi_field_fuzzy.ts | 2 +- .../eval-assert-demo/evals/dataset.eval.yaml | 4 +- .../evals/dataset.eval.baseline.jsonl | 2 +- .../execution-metrics/evals/dataset.eval.yaml | 2 +- .../scripts/check-metrics-present.ts | 2 +- .../file-changes-with-repos/evals/eval.yaml | 2 +- .../evals/dataset.eval.baseline.jsonl | 4 +- .../file-changes/evals/dataset.eval.yaml | 4 +- .../evals/dataset.eval.baseline.jsonl | 2 +- .../evals/dataset.eval.yaml | 2 +- .../evals/transcript-check.EVAL.yaml | 2 +- .../evals/dataset.eval.baseline.jsonl | 10 +- .../nlp-metrics/evals/dataset.eval.yaml | 14 +- .../repo-lifecycle/evals/pool-e2e.eval.yaml | 27 - .../rubric/evals/dataset.eval.baseline.jsonl | 2 +- .../features/rubric/evals/dataset.eval.yaml | 2 +- .../tool-evaluation-plugins/README.md | 4 +- .../evals/dataset.eval.baseline.jsonl | 6 +- .../evals/dataset.eval.yaml | 6 +- .../graders/tool-args-f1.ts | 2 +- .../graders/tool-call-f1.ts | 2 +- .../evals/dataset.eval.yaml | 6 +- .../tool-trajectory-simple/mock-agent.ts | 4 +- examples/features/trace-evaluation/README.md | 6 +- .../evals/dataset.eval.baseline.jsonl | 10 +- .../trace-evaluation/evals/dataset.eval.yaml | 14 +- .../trial-output-consistency/README.md | 4 +- .../evals/dataset.eval.yaml | 8 +- .../vitest-workspace-grader/README.md | 2 +- .../evals/dataset.eval.yaml | 2 +- .../workspace-artifact/evals/eval.yaml | 2 +- .../features/workspace-setup-script/README.md | 2 +- examples/showcase/README.md | 2 +- .../cross-repo-sync/evals/dataset.eval.yaml | 6 +- .../evals/dataset.eval.baseline.jsonl | 16 +- .../evals/dataset.eval.yaml | 2 +- .../evals/dataset.eval.baseline.jsonl | 44 +- .../export-screening/evals/dataset.eval.yaml | 2 +- .../showcase/grader-conformance/EVAL.yaml | 4 +- .../evals/encouragement.eval.baseline.jsonl | 16 +- .../evals/encouragement.eval.yaml | 2 +- .../evals/listening.eval.baseline.jsonl | 12 +- .../psychotherapy/evals/listening.eval.yaml | 2 +- .../evals/routing.eval.baseline.jsonl | 8 +- .../psychotherapy/evals/routing.eval.yaml | 2 +- .../tool-evaluation-plugins/README.md | 6 +- .../scripts/efficiency-scorer.ts | 2 +- .../scripts/pairwise-tool-compare.ts | 2 +- .../scripts/tool-selection-grader.ts | 2 +- .../tool-eval-demo.baseline.jsonl | 8 +- .../tool-eval-demo.eval.yaml | 10 +- .../evals/coding-agent-replay.eval.yaml | 6 +- .../evals/transcript-import.eval.yaml | 2 +- packages/core/src/evaluation/content.ts | 2 +- packages/core/src/evaluation/evaluate.ts | 15 +- packages/core/src/evaluation/experiment.ts | 2 +- .../src/evaluation/graders/code-grader.ts | 4 +- .../core/src/evaluation/graders/composite.ts | 7 +- packages/core/src/evaluation/graders/types.ts | 8 +- .../src/evaluation/loaders/config-loader.ts | 95 +- .../loaders/eval-yaml-transpiler.ts | 36 +- .../src/evaluation/loaders/grader-parser.ts | 126 +- .../src/evaluation/loaders/jsonl-parser.ts | 6 +- packages/core/src/evaluation/orchestrator.ts | 213 +- .../core/src/evaluation/prepared-workspace.ts | 56 +- .../src/evaluation/providers/copilot-log.ts | 2 +- .../core/src/evaluation/providers/types.ts | 2 +- .../registry/assertion-discovery.ts | 2 +- .../evaluation/registry/builtin-graders.ts | 1 - .../evaluation/registry/grader-discovery.ts | 2 +- .../evaluation/registry/grader-registry.ts | 4 +- packages/core/src/evaluation/results-repo.ts | 6 +- .../core/src/evaluation/trace-envelope.ts | 4 +- packages/core/src/evaluation/trace.ts | 2 +- packages/core/src/evaluation/types.ts | 41 +- .../evaluation/validation/eval-file.schema.ts | 21 +- .../evaluation/validation/eval-validator.ts | 37 +- .../src/evaluation/workspace/file-changes.ts | 2 +- .../core/src/evaluation/workspace/index.ts | 6 - .../src/evaluation/workspace/pool-manager.ts | 359 - .../core/src/evaluation/workspace/setup.ts | 754 +- packages/core/src/evaluation/yaml-parser.ts | 37 +- packages/core/src/index.ts | 1 - packages/core/src/paths.ts | 8 +- packages/core/src/runtime/exec.ts | 2 +- packages/core/src/runtime/target-proxy.ts | 2 +- .../evaluation/eval-inline-experiment.test.ts | 19 +- .../core/test/evaluation/extensions.test.ts | 8 +- .../evaluation/loaders/config-loader.test.ts | 98 +- .../loaders/eval-yaml-transpiler.test.ts | 12 +- .../loaders/fixtures/sdk-define-eval.eval.ts | 4 +- .../evaluation/loaders/grader-parser.test.ts | 245 +- .../evaluation/loaders/jsonl-parser.test.ts | 8 +- .../core/test/evaluation/orchestrator.test.ts | 301 +- .../evaluation/repo-schema-validation.test.ts | 4 +- .../evaluation/source-traceability.test.ts | 2 +- .../validation/eval-validator.test.ts | 4 +- .../workspace-config-parsing.test.ts | 12 +- .../evaluation/workspace/pool-manager.test.ts | 850 - .../test/evaluation/workspace/setup.test.ts | 41 +- packages/core/test/paths.test.ts | 3 - packages/sdk/README.md | 4 +- packages/sdk/src/assertion.ts | 14 - packages/sdk/src/eval.ts | 3 +- skills-data/agentv-bench/SKILL.md | 4 +- .../references/environment-adaptation.md | 2 +- skills-data/agentv-eval-migrations/SKILL.md | 6 +- .../references/breaking-changes.md | 66 +- skills-data/agentv-eval-writer/SKILL.md | 44 +- .../references/config.schema.json | 5 - .../references/eval.schema.json | 21370 +++++----------- 208 files changed, 7770 insertions(+), 19231 deletions(-) delete mode 100644 apps/cli/src/commands/workspace/clean.ts delete mode 100644 apps/cli/src/commands/workspace/list.ts delete mode 100644 apps/dashboard/src/components/FeedbackPanel.tsx delete mode 100644 apps/web/src/content/docs/docs/guides/human-review.mdx delete mode 100644 apps/web/src/content/docs/docs/guides/workspace-pool.mdx delete mode 100644 examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml delete mode 100644 packages/core/src/evaluation/workspace/pool-manager.ts delete mode 100644 packages/core/test/evaluation/workspace/pool-manager.test.ts diff --git a/.agents/conventions.md b/.agents/conventions.md index c0d1e3a11..d78e68376 100644 --- a/.agents/conventions.md +++ b/.agents/conventions.md @@ -136,16 +136,16 @@ Before adding a new pointer family, verify that the artifact is large enough or Grader types use kebab-case everywhere. -- YAML config: `type: llm-grader`, `type: is-json`, `type: execution-metrics` +- YAML config: `type: g-eval`, `type: llm-rubric`, `type: script`, `type: is-json` - Internal TypeScript: `EvaluatorKind = 'llm-grader' | 'is-json' | ...` - Output `scores[].type`: `"llm-grader"`, `"is-json"` - Registry keys: `registry.register('llm-grader', ...)` -Source of truth: `EVALUATOR_KIND_VALUES` in `packages/core/src/evaluation/types.ts`. +Source of truth: `GRADER_KIND_VALUES` in `packages/core/src/evaluation/types.ts`. Backward compatibility: -- Snake_case is accepted in YAML by `normalizeGraderType()` in `grader-parser.ts`, for example `llm_judge` -> `llm-grader`. +- Snake_case is accepted in YAML by `normalizeGraderType()` in `grader-parser.ts`, for example `llm_rubric` -> `llm-rubric`. - Single-word types such as `contains`, `equals`, `regex`, `latency`, and `cost` are unchanged. Two type definitions exist and must stay in sync: diff --git a/README.md b/README.md index 684c5067c..16b513e79 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ default_test: threshold: 0.8 workspace: - isolation: per_case + scope: attempt repos: - path: ./fixture repo: EntityProcess/agentv-contract-fixture @@ -89,7 +89,7 @@ workspace: tests: - id: fizzbuzz input: Write FizzBuzz in Python - assertions: + assert: - type: contains value: "fizz" - Implements correct FizzBuzz logic for multiples of 3, 5, and 15 @@ -227,7 +227,7 @@ export default defineEval({ }, threshold: 0.8, workspace: { - isolation: 'per_case', + scope: 'attempt', repos: [ { path: './fixture', diff --git a/STRATEGY.md b/STRATEGY.md index 5f52388d9..b4056eabd 100644 --- a/STRATEGY.md +++ b/STRATEGY.md @@ -28,7 +28,7 @@ AgentV stays repo-native and workspace-native: it runs or imports evaluations ar ### Workspace-native evaluation -Make real repository workflows first-class: repo acquisition, hooks, pooled workspaces, replay/import paths, and reuse of existing harnesses. +Make real repository workflows first-class: repo acquisition, hooks, suite/attempt workspaces, replay/import paths, and reuse of existing harnesses. _Why it serves the approach:_ This keeps AgentV attached to the actual work the agent is being judged on instead of collapsing it into a synthetic runner. diff --git a/apps/cli/src/commands/eval/commands/assert.ts b/apps/cli/src/commands/eval/commands/assert.ts index 4e7220edc..9a1a096ab 100644 --- a/apps/cli/src/commands/eval/commands/assert.ts +++ b/apps/cli/src/commands/eval/commands/assert.ts @@ -7,7 +7,7 @@ import { buildTraceFromMessages, executeScript } from '@agentv/core'; export const evalAssertCommand = command({ name: 'assert', - description: 'Run a single code-grader assertion from .agentv/graders/ and print the score', + description: 'Run a single script grader assertion from .agentv/graders/ and print the score', args: { graderName: positional({ type: string, @@ -62,8 +62,7 @@ export const evalAssertCommand = command({ process.exit(1); } - // Build payload matching CodeGrader's expected format (snake_case). - // Include all fields that defineCodeGrader validates as required. + // Build payload matching the script grader protocol (snake_case). const messages = [{ role: 'assistant' as const, content: resolvedOutput }]; const inputMessages = [{ role: 'user' as const, content: resolvedInput }]; const trace = buildTraceFromMessages({ diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 09b0f80ef..c52e39876 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -119,15 +119,10 @@ export const evalRunCommand = command({ long: 'verbose', description: 'Enable verbose logging', }), - workspaceMode: option({ - type: optional(string), - long: 'workspace-mode', - description: "Workspace mode: 'temp' (default), 'pooled', or 'static'", - }), workspacePath: option({ type: optional(string), long: 'workspace-path', - description: 'Static workspace directory path (used when workspace mode is static)', + description: 'Static workspace directory path to reuse for this run', }), keepWorkspaces: flag({ long: 'keep-workspaces', @@ -271,7 +266,6 @@ export const evalRunCommand = command({ cachePath: args.cachePath, noCache: args.noCache, verbose: args.verbose, - workspaceMode: args.workspaceMode, workspacePath: args.workspacePath, keepWorkspaces: args.keepWorkspaces, trace: false, diff --git a/apps/cli/src/commands/eval/commands/vitest.ts b/apps/cli/src/commands/eval/commands/vitest.ts index 524dfd8eb..878b019c1 100644 --- a/apps/cli/src/commands/eval/commands/vitest.ts +++ b/apps/cli/src/commands/eval/commands/vitest.ts @@ -9,7 +9,7 @@ function parseCommand(value: string | undefined): readonly string[] | undefined export const evalVitestCommand = command({ name: 'vitest', - description: 'Run Vitest workspace verifier files as an AgentV code-grader protocol adapter', + description: 'Run Vitest workspace verifier files as an AgentV script grader adapter', args: { testFiles: restPositionals({ type: string, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 80bdfc4a7..c8e5b193f 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -294,7 +294,6 @@ interface NormalizedOptions { readonly resume: boolean; readonly rerunFailed: boolean; readonly rerunFailedSource?: string; - readonly workspaceMode?: 'pooled' | 'temp' | 'static'; readonly workspacePath?: string; readonly keepWorkspaces: boolean; /** Removed: use --output instead */ @@ -380,10 +379,6 @@ function normalizeOptionalNumber(value: unknown): number | undefined { return undefined; } -function normalizeWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined { - return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined; -} - function normalizeStringArray(value: unknown): readonly string[] { if (Array.isArray(value)) { return value.filter((v): v is string => typeof v === 'string' && v.trim().length > 0); @@ -696,21 +691,7 @@ function normalizeOptions( const configOutputDir = normalizeString(config?.output?.dir); const cliWorkspacePath = normalizeString(rawOptions.workspacePath); const configWorkspacePath = normalizeString(yamlExecution?.workspace_path); - const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode); - const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode); - if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== 'static') { - throw new Error('--workspace-path requires --workspace-mode=static (or omit --workspace-mode)'); - } - const configWorkspaceMode = normalizeWorkspaceMode(yamlExecution?.workspace_mode); - if (configWorkspacePath && configWorkspaceMode && configWorkspaceMode !== 'static') { - throw new Error( - 'execution.workspace_path requires execution.workspace_mode: static when both are provided', - ); - } - const useConfigWorkspacePath = cliWorkspaceMode === undefined || cliWorkspaceMode === 'static'; - const workspacePath = - cliWorkspacePath ?? (useConfigWorkspacePath ? configWorkspacePath : undefined); - const workspaceMode = workspacePath ? 'static' : (cliWorkspaceMode ?? configWorkspaceMode); + const workspacePath = cliWorkspacePath ?? configWorkspacePath; const resultsRepo = normalizeString(rawOptions.resultsRepo); const resultsPush = normalizeBoolean(rawOptions.resultsPush); const resultsNoPush = normalizeBoolean(rawOptions.noResultsPush); @@ -772,7 +753,6 @@ function normalizeOptions( normalizeBoolean(rawOptions.resume) || normalizeString(rawOptions.rerunFailed) !== undefined, rerunFailed: normalizeString(rawOptions.rerunFailed) !== undefined, rerunFailedSource: normalizeString(rawOptions.rerunFailed), - workspaceMode, workspacePath, // Precedence: CLI > YAML config > TS config keepWorkspaces: @@ -865,7 +845,6 @@ const CLI_RUNTIME_SOURCE_OPTION_KEYS = [ 'recordReplay', 'recordReplayVariant', 'workspacePath', - 'workspaceMode', ] as const; function hasCliRuntimeSource(rawOptions: Record): boolean { @@ -1031,7 +1010,6 @@ function applyExperimentOptions( ...options, target: options.target, agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds, - workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, threshold: options.threshold ?? experiment.threshold, @@ -1760,7 +1738,6 @@ async function runSingleEvalFile(params: { evalCases: testCases, verbose: options.verbose, maxConcurrency: resolvedWorkers, - workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, keepWorkspaces: options.keepWorkspaces, trials: trialsConfig, @@ -2830,9 +2807,7 @@ export async function runEvalCommand( } // Hint about --keep-workspaces when workspaces were used but some cleaned up - const usedWorkspaces = - resultsWithWorkspaces.length > 0 || - (options.workspaceMode && options.workspaceMode !== 'static'); + const usedWorkspaces = resultsWithWorkspaces.length > 0; if (!options.keepWorkspaces && usedWorkspaces) { console.log('Use --keep-workspaces to preserve all workspaces for inspection.'); } diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index df6a2be20..f639f90a5 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -494,9 +494,7 @@ function parseSourceTestCase(test: EvalTest): Record { } function withoutLegacyAssertionKeys(testCase: Record): Record { - return Object.fromEntries( - Object.entries(testCase).filter(([key]) => key !== 'assert' && key !== 'evaluators'), - ); + return Object.fromEntries(Object.entries(testCase).filter(([key]) => key !== 'assert')); } function buildEvalCase( diff --git a/apps/cli/src/commands/inspect/score.ts b/apps/cli/src/commands/inspect/score.ts index 75244e827..4141ed850 100644 --- a/apps/cli/src/commands/inspect/score.ts +++ b/apps/cli/src/commands/inspect/score.ts @@ -67,8 +67,7 @@ function parseKeyValues(s: string): Record { */ export function parseAssertSpec(spec: string): GraderConfig { const colonIdx = spec.indexOf(':'); - // Normalize snake_case to kebab-case for backward compat - const type = (colonIdx === -1 ? spec : spec.slice(0, colonIdx)).replace(/_/g, '-'); + const type = colonIdx === -1 ? spec : spec.slice(0, colonIdx); const params = colonIdx === -1 ? '' : spec.slice(colonIdx + 1); switch (type) { diff --git a/apps/cli/src/commands/pipeline/bench.ts b/apps/cli/src/commands/pipeline/bench.ts index daa95683c..539e374fc 100644 --- a/apps/cli/src/commands/pipeline/bench.ts +++ b/apps/cli/src/commands/pipeline/bench.ts @@ -1,5 +1,5 @@ /** - * `agentv pipeline bench` — Merge code-grader and LLM grader scores into final + * `agentv pipeline bench` — Merge script and LLM grader scores into final * benchmark artifacts. * * Reads code_grader_results and llm_grader_results from disk per test. @@ -64,7 +64,7 @@ export const evalBenchCommand = command({ const evaluators: EvaluatorScore[] = []; const allAssertions: { text: string; passed: boolean; evidence: string }[] = []; - // Collect code grader results + // Collect script grader results const codeResultsDir = join(testDir, 'code_grader_results'); try { const resultFiles = (await readdir(codeResultsDir)).filter((f) => f.endsWith('.json')); @@ -72,7 +72,7 @@ export const evalBenchCommand = command({ const result = JSON.parse(await readFile(join(codeResultsDir, file), 'utf8')); evaluators.push({ name: result.name, - type: result.type ?? 'code-grader', + type: result.type ?? 'script', score: result.score, weight: result.weight ?? 1.0, assertions: result.assertions ?? [], @@ -82,7 +82,7 @@ export const evalBenchCommand = command({ } } } catch { - // No code grader results + // No script grader results. } // Collect LLM grader scores from per-test disk results diff --git a/apps/cli/src/commands/pipeline/grade.ts b/apps/cli/src/commands/pipeline/grade.ts index ef4eb3773..3b2dc4bb0 100644 --- a/apps/cli/src/commands/pipeline/grade.ts +++ b/apps/cli/src/commands/pipeline/grade.ts @@ -4,7 +4,7 @@ * * All grader configs live in code_graders/.json. Each config has a `type` * field that determines how it's evaluated: - * - `code-grader` (or configs with a `command` field): executed as external scripts + * - `script` (or configs with a `command` field): executed as external scripts * - Built-in types (contains, regex, equals, etc.): evaluated in-process * * Results are written to code_grader_results/.json for pipeline bench. @@ -62,7 +62,7 @@ export interface GraderTask { /** * Run grader tasks with concurrency and progress feedback. - * Dispatches each task based on its config: code-graders are executed as + * Dispatches each task based on its config: script graders are executed as * external scripts, built-in types (contains, regex, etc.) are evaluated in-process. * Shared by `pipeline grade` and `pipeline run`. */ @@ -103,7 +103,7 @@ export async function runCodeGraders( writeProgress(); }; - /** Run an external code-grader script. */ + /** Run an external script grader. */ const executeCodeGrader = async (graderConfig: Record, task: GraderTask) => { const { testId, resultsDir, responseText, inputData } = task; const graderName = graderConfig.name as string; diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index efd07824c..61d121204 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -16,7 +16,7 @@ * ├── criteria.md * ├── expected_output.json (if present) * ├── llm_graders/.json - * └── code_graders/.json + * └── code_graders/.json # script/deterministic grader configs */ import { readFile } from 'node:fs/promises'; import { mkdir, writeFile } from 'node:fs/promises'; @@ -142,7 +142,7 @@ export const evalInputCommand = command({ await mkdir(testDir, { recursive: true }); testIds.push(test.id); - // input.json — aligned with eval YAML schema and code grader SDK field names + // input.json - aligned with eval YAML schema and script grader payload fields. const inputMessages = test.input.map((m) => ({ role: m.role, content: typeof m.content === 'string' ? m.content : m.content, @@ -215,7 +215,7 @@ export const evalInputCommand = command({ console.log(' 1. Dispatch executor subagents — one per test case (all in parallel):'); console.log(' - Each reads //input.json'); console.log(' - Executes the task, writes //response.md'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log( ' 3. Dispatch grader subagents — one per (test × LLM grader) pair (all in parallel):', ); @@ -252,7 +252,7 @@ async function writeGraderConfigs( let hasLlmGraders = false; for (const assertion of assertions) { - if (assertion.type === 'script' || assertion.type === 'code-grader') { + if (assertion.type === 'script') { if (!hasCodeGraders) { await mkdir(codeGradersDir, { recursive: true }); hasCodeGraders = true; diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 8437f732d..4f842e9f8 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -64,7 +64,7 @@ function loadEnvFile(dir: string): Record { export const evalRunCommand = command({ name: 'run', description: - 'Extract inputs, invoke CLI targets, and run code graders (for agent targets, use pipeline input + subagents)', + 'Extract inputs, invoke CLI targets, and run script graders (for agent targets, use pipeline input + subagents)', args: { evalPath: positional({ type: string, @@ -90,7 +90,7 @@ export const evalRunCommand = command({ type: optional(oneOf(['code', 'none'])), long: 'grader-type', description: - 'Which grading phase to run: "code" runs code-graders inline, omit to skip grading (use pipeline grade separately)', + 'Which grading phase to run: "code" runs script graders inline, omit to skip grading (use pipeline grade separately)', }), target: option({ type: optional(string), @@ -347,7 +347,7 @@ export const evalRunCommand = command({ console.log(' 1. Dispatch executor subagents — one per test case (all in parallel):'); console.log(' - Each reads //input.json'); console.log(' - Executes the task, writes //response.md'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log( ' 3. Dispatch grader subagents — one per (test x LLM grader) pair (all in parallel):', ); @@ -363,18 +363,18 @@ export const evalRunCommand = command({ console.log(''); } - // ── Step 3: Run code graders (only when explicitly requested) ───── + // Step 3: Run script graders (only when explicitly requested). if (graderType !== 'code') { console.log(`\nDone. Results in ${outDir}`); console.log(''); if (targetKind === 'agent') { console.log(' The agent must now:'); console.log(' 1. Dispatch executor subagents to generate response.md files'); - console.log(' 2. Run code graders: agentv pipeline grade '); + console.log(' 2. Run script graders: agentv pipeline grade '); console.log(' 3. Dispatch grader subagents for llm_graders/ configs'); console.log(' 4. Merge scores: agentv pipeline bench '); } else { - console.log(' To run code graders: agentv pipeline grade '); + console.log(' To run script graders: agentv pipeline grade '); console.log(' Or re-run with --grader-type code to grade inline.'); } return; @@ -408,7 +408,7 @@ export const evalRunCommand = command({ const graderConcurrency = workers ?? 10; const { totalGraders, totalPassed } = await runCodeGraders(graderTasks, graderConcurrency); - console.log(`Graded ${totalGraders} code-grader(s): ${totalPassed} passed`); + console.log(`Graded ${totalGraders} script grader(s): ${totalPassed} passed`); console.log(''); console.log(`Results in ${outDir}`); console.log(''); @@ -439,7 +439,7 @@ async function writeGraderConfigs( let hasLlmGraders = false; for (const assertion of assertions) { - if (assertion.type === 'script' || assertion.type === 'code-grader') { + if (assertion.type === 'script') { if (!hasCodeGraders) { await mkdir(codeGradersDir, { recursive: true }); hasCodeGraders = true; diff --git a/apps/cli/src/commands/prepare/index.ts b/apps/cli/src/commands/prepare/index.ts index 5a6ad9aeb..84a8eb426 100644 --- a/apps/cli/src/commands/prepare/index.ts +++ b/apps/cli/src/commands/prepare/index.ts @@ -190,7 +190,7 @@ async function placePreparedWorkspace( await mkdir(path.dirname(destinationPath), { recursive: true }); await rm(destinationPath, { recursive: true, force: true }); - if (prepared.cleanupPolicy.mode !== 'static' && prepared.pool === undefined) { + if (prepared.cleanupPolicy.mode !== 'static') { await moveDirectory(sourcePath, destinationPath); return destinationPath; } @@ -333,7 +333,6 @@ async function prepareAttempt(options: { evalCases: suite.tests, testId: options.testId, verbose: false, - workspaceMode: 'temp', retainOnSuccess: 'keep', retainOnFailure: 'keep', }); diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index b98f71a0e..12e944e35 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -14,8 +14,6 @@ * or as raw/downloadable text with ?raw=1 / ?download=1 * - GET /api/runs/:filename/evals/:evalId/trace-session — read an AgentV * trace sidecar through the Dashboard trace/session read model - * - GET /api/feedback — read feedback reviews - * - POST /api/feedback — write feedback reviews * - GET /api/projects — list registered projects * - POST /api/projects — register a project by path * - DELETE /api/projects/:projectId — unregister a project @@ -209,39 +207,6 @@ function bootstrapCurrentProject( return { currentProjectId: entry.id }; } -// ── Feedback persistence ───────────────────────────────────────────────── - -interface FeedbackReview { - test_id: string; - comment: string; - updated_at: string; -} - -interface FeedbackData { - reviews: FeedbackReview[]; -} - -function feedbackPath(resultDir: string): string { - return path.join(resultDir, 'feedback.json'); -} - -function readFeedback(cwd: string): FeedbackData { - const fp = feedbackPath(cwd); - if (!existsSync(fp)) { - return { reviews: [] }; - } - try { - return JSON.parse(readFileSync(fp, 'utf8')) as FeedbackData; - } catch (err) { - console.error(`Warning: could not parse ${fp}, starting fresh: ${(err as Error).message}`); - return { reviews: [] }; - } -} - -function writeFeedback(cwd: string, data: FeedbackData): void { - writeFileSync(feedbackPath(cwd), `${JSON.stringify(data, null, 2)}\n`, 'utf8'); -} - // ── Shared utilities (used by handler functions) ───────────────────────── export interface FileNode { @@ -2698,61 +2663,6 @@ function handleConfig( }); } -function handleFeedbackRead(c: C, { searchDir }: DataContext) { - return c.json(readFeedback(feedbackStoreDir(searchDir))); -} - -function feedbackStoreDir(searchDir: string): string { - const resultsDir = path.join(searchDir, '.agentv', 'results'); - return existsSync(resultsDir) ? resultsDir : searchDir; -} - -async function handleFeedbackWrite(c: C, resultDir: string) { - let body: unknown; - try { - body = await c.req.json(); - } catch { - return c.json({ error: 'Invalid JSON' }, 400); - } - - if (!body || typeof body !== 'object') { - return c.json({ error: 'Invalid payload' }, 400); - } - - const payload = body as Record; - if (!Array.isArray(payload.reviews)) { - return c.json({ error: 'Missing reviews array' }, 400); - } - - const incoming = payload.reviews as Record[]; - for (const review of incoming) { - if (typeof review.test_id !== 'string' || typeof review.comment !== 'string') { - return c.json({ error: 'Each review must have test_id and comment strings' }, 400); - } - } - - const existing = readFeedback(resultDir); - const now = new Date().toISOString(); - - for (const review of incoming) { - const newReview: FeedbackReview = { - test_id: review.test_id as string, - comment: review.comment as string, - updated_at: now, - }; - - const idx = existing.reviews.findIndex((r) => r.test_id === newReview.test_id); - if (idx >= 0) { - existing.reviews[idx] = newReview; - } else { - existing.reviews.push(newReview); - } - } - - writeFeedback(resultDir, existing); - return c.json(existing); -} - function expandHomePath(inputPath: string): string { if (inputPath === '~') return homedir(); if (inputPath.startsWith('~/') || inputPath.startsWith('~\\')) { @@ -3007,7 +2917,7 @@ async function handleRunsCombine(c: C, { searchDir, projectId }: DataContext) { // ── Hono app factory ───────────────────────────────────────────────────── /** - * Create a Hono app with dashboard, result picker, and feedback API routes. + * Create a Hono app with dashboard and result picker API routes. * Accepts an empty results array for the empty-state dashboard. */ export function createApp( @@ -3326,19 +3236,6 @@ export function createApp( app.get('/api/compare', (c) => handleCompare(c, defaultCtx)); app.get('/api/targets', (c) => handleTargets(c, defaultCtx)); - // Feedback (unscoped — read uses defaultCtx.searchDir as resultDir) - app.get('/api/feedback', (c) => { - const data = readFeedback(resultDir); - return c.json(data); - }); - - app.post('/api/feedback', async (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return handleFeedbackWrite(c, resultDir); - }); - // Aggregated index (unscoped only) app.get('/api/index', async (c) => { const { runs: metas } = await listMergedResultFiles(searchDir, undefined, defaultCtx.projectId); @@ -3449,16 +3346,6 @@ export function createApp( app.get('/api/projects/:projectId/experiments', (c) => withProject(c, handleExperiments)); app.get('/api/projects/:projectId/compare', (c) => withProject(c, handleCompare)); app.get('/api/projects/:projectId/targets', (c) => withProject(c, handleTargets)); - app.get('/api/projects/:projectId/feedback', (c) => withProject(c, handleFeedbackRead)); - app.post('/api/projects/:projectId/feedback', (c) => { - if (readOnly) { - return c.json({ error: 'Dashboard is running in read-only mode' }, 403); - } - return withProject(c, (projectContext, ctx) => - handleFeedbackWrite(projectContext, feedbackStoreDir(ctx.searchDir)), - ); - }); - // ── Eval runner routes (discovery, launch, status) ──────────────────── registerEvalRoutes( @@ -3682,7 +3569,6 @@ export const resultsServeCommand = command({ } } - // Use the run directory for feedback storage (matches #764 behavior) const resultDir = sourceFile ? path.dirname(path.resolve(sourceFile)) : cwd; const app = createApp(results, resultDir, cwd, sourceFile, { readOnly, diff --git a/apps/cli/src/commands/workspace/clean.ts b/apps/cli/src/commands/workspace/clean.ts deleted file mode 100644 index b77ba7eb0..000000000 --- a/apps/cli/src/commands/workspace/clean.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { existsSync } from 'node:fs'; -import { readFile, readdir, rm } from 'node:fs/promises'; -import path from 'node:path'; -import { command, flag, option, optional, string } from 'cmd-ts'; - -import { getWorkspacePoolRoot } from '@agentv/core'; - -interface PoolMetadata { - fingerprint: string; - templatePath: string | null; - repos: readonly { - path?: string; - repo?: string; - source?: { type: string; url?: string; path?: string }; - }[]; - createdAt: string; -} - -async function confirm(message: string): Promise { - const readline = await import('node:readline'); - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - const answer = await new Promise((resolve) => { - rl.question(`${message} [y/N] `, resolve); - }); - rl.close(); - return answer.toLowerCase() === 'y'; -} - -export const cleanCommand = command({ - name: 'clean', - description: 'Remove workspace pool entries', - args: { - repo: option({ - type: optional(string), - long: 'repo', - description: 'Only remove pools containing this repo URL', - }), - force: flag({ - long: 'force', - short: 'f', - description: 'Skip confirmation prompt', - }), - }, - handler: async ({ repo, force }) => { - const poolRoot = getWorkspacePoolRoot(); - - if (!existsSync(poolRoot)) { - console.log('No workspace pool entries found.'); - return; - } - - if (repo) { - // Remove only pool entries matching the repo URL - const entries = await readdir(poolRoot, { withFileTypes: true }); - const poolDirs = entries.filter((e) => e.isDirectory()); - const matchingDirs: string[] = []; - - for (const dir of poolDirs) { - const poolDir = path.join(poolRoot, dir.name); - const metadataPath = path.join(poolDir, 'metadata.json'); - - try { - const raw = await readFile(metadataPath, 'utf-8'); - const metadata = JSON.parse(raw) as PoolMetadata; - - const hasRepo = metadata.repos?.some((r) => { - const value = r.repo ?? (r.source?.type === 'git' ? r.source.url : r.source?.path); - return value?.toLowerCase().includes(repo.toLowerCase()) ?? false; - }); - - if (hasRepo) { - matchingDirs.push(poolDir); - } - } catch { - // Skip entries without valid metadata - } - } - - if (matchingDirs.length === 0) { - console.log(`No workspace pool entries found matching repo "${repo}".`); - return; - } - - if (!force) { - const confirmed = await confirm( - `Remove ${matchingDirs.length} pool entry(s) matching repo "${repo}"?`, - ); - if (!confirmed) { - console.log('Cancelled.'); - return; - } - } - - for (const dir of matchingDirs) { - await rm(dir, { recursive: true, force: true }); - console.log(`Removed: ${path.basename(dir).slice(0, 12)}...`); - } - console.log('Done.'); - } else { - // Remove entire pool root - if (!force) { - const confirmed = await confirm(`Remove all workspace pool entries from ${poolRoot}?`); - if (!confirmed) { - console.log('Cancelled.'); - return; - } - } - - await rm(poolRoot, { recursive: true, force: true }); - console.log('Workspace pool cleaned.'); - } - }, -}); diff --git a/apps/cli/src/commands/workspace/index.ts b/apps/cli/src/commands/workspace/index.ts index 50dd3c00f..5eb9ebc64 100644 --- a/apps/cli/src/commands/workspace/index.ts +++ b/apps/cli/src/commands/workspace/index.ts @@ -1,15 +1,11 @@ import { subcommands } from 'cmd-ts'; -import { cleanCommand } from './clean.js'; import { depsCommand } from './deps.js'; -import { listCommand } from './list.js'; export const workspaceCommand = subcommands({ name: 'workspace', - description: 'Manage workspace pool', + description: 'Inspect workspace dependencies', cmds: { - list: listCommand, - clean: cleanCommand, deps: depsCommand, }, }); diff --git a/apps/cli/src/commands/workspace/list.ts b/apps/cli/src/commands/workspace/list.ts deleted file mode 100644 index 8187447de..000000000 --- a/apps/cli/src/commands/workspace/list.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { existsSync } from 'node:fs'; -import { readFile, readdir, stat } from 'node:fs/promises'; -import path from 'node:path'; -import { command } from 'cmd-ts'; - -import { getWorkspacePoolRoot } from '@agentv/core'; - -interface PoolMetadata { - fingerprint: string; - templatePath: string | null; - repos: readonly { - path?: string; - repo?: string; - source?: { type: string; url?: string; path?: string }; - }[]; - createdAt: string; -} - -async function getDirectorySize(dirPath: string): Promise { - let totalSize = 0; - try { - const entries = await readdir(dirPath, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = path.join(dirPath, entry.name); - if (entry.isDirectory()) { - totalSize += await getDirectorySize(fullPath); - } else { - const stats = await stat(fullPath); - totalSize += stats.size; - } - } - } catch { - // Directory might not be readable - } - return totalSize; -} - -function formatSize(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; -} - -export const listCommand = command({ - name: 'list', - description: 'List workspace pool entries', - args: {}, - handler: async () => { - const poolRoot = getWorkspacePoolRoot(); - - if (!existsSync(poolRoot)) { - console.log('No workspace pool entries found.'); - return; - } - - const entries = await readdir(poolRoot, { withFileTypes: true }); - const poolDirs = entries.filter((e) => e.isDirectory()); - - if (poolDirs.length === 0) { - console.log('No workspace pool entries found.'); - return; - } - - for (const dir of poolDirs) { - const poolDir = path.join(poolRoot, dir.name); - const fingerprint = dir.name; - - // Count slots - const poolEntries = await readdir(poolDir, { withFileTypes: true }); - const slots = poolEntries.filter((e) => e.isDirectory() && e.name.startsWith('slot-')); - - // Read metadata - const metadataPath = path.join(poolDir, 'metadata.json'); - let metadata: PoolMetadata | null = null; - try { - const raw = await readFile(metadataPath, 'utf-8'); - metadata = JSON.parse(raw) as PoolMetadata; - } catch { - // metadata.json might not exist - } - - // Compute disk size - const size = await getDirectorySize(poolDir); - - console.log(` ${fingerprint.slice(0, 12)}...`); - console.log(` Slots: ${slots.length}`); - console.log(` Size: ${formatSize(size)}`); - if (metadata) { - if (metadata.templatePath) { - console.log(` Template: ${metadata.templatePath}`); - } - if (metadata.repos && metadata.repos.length > 0) { - const repoSources = metadata.repos.map( - (r) => r.repo ?? (r.source?.type === 'git' ? r.source.url : r.source?.path), - ); - console.log(` Repos: ${repoSources.join(', ')}`); - } - console.log(` Created: ${metadata.createdAt}`); - } - console.log(); - } - }, -}); diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 80952c0a9..31b34bf33 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -278,7 +278,7 @@ describe('buildGradingArtifact', () => { it('includes evaluators list with AgentV extensions', () => { const result = makeResult({ scores: [ - makeEvaluatorResult({ name: 'format-check', type: 'code-grader', score: 1.0 }), + makeEvaluatorResult({ name: 'format-check', type: 'script', score: 1.0 }), makeEvaluatorResult({ name: 'quality', type: 'llm-grader', score: 0.7 }), ], }); @@ -287,7 +287,7 @@ describe('buildGradingArtifact', () => { expect(grading.graders).toHaveLength(2); expect(grading.graders?.[0].name).toBe('format-check'); - expect(grading.graders?.[0].type).toBe('code-grader'); + expect(grading.graders?.[0].type).toBe('script'); expect(grading.graders?.[1].score).toBe(0.7); }); diff --git a/apps/cli/test/commands/eval/pipeline/bench.test.ts b/apps/cli/test/commands/eval/pipeline/bench.test.ts index c9b9ab0a6..0bd1bc193 100644 --- a/apps/cli/test/commands/eval/pipeline/bench.test.ts +++ b/apps/cli/test/commands/eval/pipeline/bench.test.ts @@ -30,7 +30,7 @@ describe('pipeline bench', () => { join(codeResultsDir, 'contains.json'), JSON.stringify({ name: 'contains', - type: 'code-grader', + type: 'script', score: 1.0, weight: 1.0, assertions: [{ text: 'Found keyword', passed: true }], diff --git a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml index ca18544f3..6b71c722f 100644 --- a/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml +++ b/apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml @@ -5,7 +5,7 @@ tests: criteria: Response echoes the input assertions: - name: contains_hello - type: code-grader + type: script command: node grader-score-1.js weight: 1.0 - name: relevance diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index df5cd6c91..c1dbca458 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -127,7 +127,7 @@ describe('agentv grade prepared attempts', () => { tempDir, ` - name: workspace-check - type: code-grader + type: script command: ["bun", "../scripts/workspace-grader.ts"] `, ); diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index ed850db91..17bc4206e 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -85,7 +85,7 @@ target: command: ["bun", "../scripts/hook.ts", "target_before_each"] assertions: - name: secret-grader - type: code-grader + type: script command: ["bun", "../scripts/grader.ts"] tests: - id: case-1 diff --git a/apps/cli/test/commands/results/export-e2e-providers.test.ts b/apps/cli/test/commands/results/export-e2e-providers.test.ts index c5207c1fb..574e84c00 100644 --- a/apps/cli/test/commands/results/export-e2e-providers.test.ts +++ b/apps/cli/test/commands/results/export-e2e-providers.test.ts @@ -73,7 +73,7 @@ const CODEX_RESULT = { scores: [ { name: 'edit_quality', - type: 'code-grader', + type: 'script', score: 0.9, assertions: [{ text: 'File edited correctly', passed: true }], }, diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 2d27c4c4b..0676c557a 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -723,165 +723,6 @@ describe('serve app', () => { }); }); - // ── GET /api/feedback ────────────────────────────────────────────────── - - describe('GET /api/feedback', () => { - it('returns empty reviews when no feedback file', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback'); - expect(res.status).toBe(200); - const data = await res.json(); - expect(data).toEqual({ reviews: [] }); - }); - }); - - // ── POST /api/feedback ───────────────────────────────────────────────── - - describe('POST /api/feedback', () => { - it('persists reviews to feedback.json', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Looks good!' }], - }), - }); - expect(res.status).toBe(200); - const data = (await res.json()) as { - reviews: { test_id: string; comment: string; updated_at: string }[]; - }; - expect(data.reviews).toHaveLength(1); - expect(data.reviews[0].test_id).toBe('test-greeting'); - expect(data.reviews[0].comment).toBe('Looks good!'); - expect(data.reviews[0].updated_at).toBeDefined(); - - // Verify file exists on disk - const fp = path.join(tempDir, 'feedback.json'); - expect(existsSync(fp)).toBe(true); - const onDisk = JSON.parse(readFileSync(fp, 'utf8')); - expect(onDisk.reviews).toHaveLength(1); - }); - - it('merges with existing reviews', async () => { - const app = makeApp(); - - // First review - await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'First review' }], - }), - }); - - // Second review (different test_id) - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-math', comment: 'Math looks off' }], - }), - }); - - const data = (await res.json()) as { reviews: { test_id: string }[] }; - expect(data.reviews).toHaveLength(2); - expect(data.reviews.map((r: { test_id: string }) => r.test_id).sort()).toEqual([ - 'test-greeting', - 'test-math', - ]); - }); - - it('overwrites duplicate test_id', async () => { - const app = makeApp(); - - // First review - await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Initial' }], - }), - }); - - // Overwrite same test_id - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'Updated' }], - }), - }); - - const data = (await res.json()) as { reviews: { test_id: string; comment: string }[] }; - expect(data.reviews).toHaveLength(1); - expect(data.reviews[0].comment).toBe('Updated'); - }); - - it('accepts empty comment string', async () => { - const app = makeApp(); - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: '' }], - }), - }); - expect(res.status).toBe(200); - const data = (await res.json()) as { - reviews: { comment: string }[]; - }; - expect(data.reviews[0].comment).toBe(''); - }); - - it('rejects invalid payload (400)', async () => { - const app = makeApp(); - - // Missing reviews array - const res1 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ foo: 'bar' }), - }); - expect(res1.status).toBe(400); - - // Invalid review entry - const res2 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ reviews: [{ test_id: 123 }] }), - }); - expect(res2.status).toBe(400); - - // Not an object - const res3 = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: '"just a string"', - }); - expect(res3.status).toBe(400); - }); - - it('returns 403 in read-only mode', async () => { - const content = toJsonl(RESULT_A, RESULT_B); - const results = loadResults(content); - const app = createApp(results, tempDir, undefined, undefined, { - studioDir, - readOnly: true, - }); - - const res = await app.request('/api/feedback', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - reviews: [{ test_id: 'test-greeting', comment: 'blocked' }], - }), - }); - - expect(res.status).toBe(403); - }); - }); - describe('GET /api/config', () => { it('includes read_only mode and dashboard mode in the config payload', async () => { const content = toJsonl(RESULT_A, RESULT_B); @@ -913,14 +754,6 @@ describe('serve app', () => { const html = await res.text(); expect(html).toContain('agentv'); }); - - it('serves feedback API with empty results', async () => { - const app = createApp([], tempDir, undefined, undefined, { studioDir }); - const res = await app.request('/api/feedback'); - expect(res.status).toBe(200); - const data = await res.json(); - expect(data).toEqual({ reviews: [] }); - }); }); // ── GET /api/runs ─────────────────────────────────────────────────── diff --git a/apps/cli/test/commands/trace/trace.test.ts b/apps/cli/test/commands/trace/trace.test.ts index c069b44c8..1b00d6dc0 100644 --- a/apps/cli/test/commands/trace/trace.test.ts +++ b/apps/cli/test/commands/trace/trace.test.ts @@ -601,7 +601,7 @@ describe('parseAssertSpec', () => { }); it('should parse token-usage spec with params', () => { - const config = parseAssertSpec('token_usage:max_total=2000,max_input=1500'); + const config = parseAssertSpec('token-usage:max_total=2000,max_input=1500'); expect(config.type).toBe('token-usage'); expect((config as { max_total: number }).max_total).toBe(2000); expect((config as { max_input: number }).max_input).toBe(1500); @@ -613,7 +613,7 @@ describe('parseAssertSpec', () => { }); it('should parse execution-metrics spec', () => { - const config = parseAssertSpec('execution_metrics:max_tool_calls=10,max_tokens=3000'); + const config = parseAssertSpec('execution-metrics:max_tool_calls=10,max_tokens=3000'); expect(config.type).toBe('execution-metrics'); expect((config as { max_tool_calls: number }).max_tool_calls).toBe(10); expect((config as { max_tokens: number }).max_tokens).toBe(3000); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 31dbce6d7..a89fabd8b 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -524,7 +524,6 @@ describe('agentv eval CLI', () => { expect(result.exitCode).toBe(0); const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ - workspaceMode: 'static', workspacePath, resultCount: 2, }); @@ -549,7 +548,6 @@ describe('agentv eval CLI', () => { expect(result.exitCode).toBe(0); const diagnostics = await readDiagnostics(fixture); expect(diagnostics).toMatchObject({ - workspaceMode: 'static', workspacePath, resultCount: 2, }); @@ -639,8 +637,8 @@ describe('agentv eval CLI', () => { ' model: gpt-5-codex', 'timeout_seconds: 12', 'threshold: 0.8', - 'budget_usd: 3', 'evaluate_options:', + ' budget_usd: 3', ' repeat:', ' count: 2', ' strategy: pass_any', @@ -825,7 +823,8 @@ describe('agentv eval CLI', () => { 'name: first', 'target: cli-target', 'timeout_seconds: 11', - 'budget_usd: 0.11', + 'evaluate_options:', + ' budget_usd: 0.11', 'tests:', ' - id: first-case', ' input: first', @@ -840,7 +839,8 @@ describe('agentv eval CLI', () => { 'name: second', 'target: file-target', 'timeout_seconds: 22', - 'budget_usd: 0.22', + 'evaluate_options:', + ' budget_usd: 0.22', 'tests:', ' - id: second-case', ' input: second', diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index c97edf54b..0e96f3d01 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -20,7 +20,6 @@ interface RunEvaluationOptionsLike { readonly evalCases?: ReadonlyArray; readonly verbose?: boolean; readonly maxConcurrency?: number; - readonly workspaceMode?: string; readonly workspacePath?: string; readonly trials?: { readonly count: number; @@ -191,7 +190,6 @@ async function maybeWriteDiagnostics( envLocalOnly: process.env.CLI_ENV_LOCAL_ONLY ?? null, budgetUsd: options.budgetUsd ?? null, maxConcurrency: options.maxConcurrency ?? null, - workspaceMode: options.workspaceMode ?? null, workspacePath: options.workspacePath ?? null, trials: options.trials ?? null, threshold: options.threshold ?? null, diff --git a/apps/dashboard/src/components/EvalDetail.tsx b/apps/dashboard/src/components/EvalDetail.tsx index fcd367667..8ce4d45da 100644 --- a/apps/dashboard/src/components/EvalDetail.tsx +++ b/apps/dashboard/src/components/EvalDetail.tsx @@ -1,5 +1,5 @@ /** - * Eval detail view with checks, source traceability, artifact files, and feedback. + * Eval detail view with checks, source traceability, artifact files, and artifacts. * * Layout: compact header → tabs → full-height content area. * Scores and assertions are only visible in the Checks tab. @@ -31,7 +31,6 @@ import type { SourceTraceability, } from '~/lib/types'; -import { FeedbackPanel } from './FeedbackPanel'; import type { FileNode } from './FileTree'; import { FileTree } from './FileTree'; import { MonacoViewer } from './MonacoViewer'; @@ -49,7 +48,7 @@ interface EvalDetailProps { onSelectTrial?: (trial: EvalCaseTrial, initialTab?: Tab) => void; } -type Tab = 'checks' | 'transcript' | 'source' | 'files' | 'feedback'; +type Tab = 'checks' | 'transcript' | 'source' | 'files'; /** Recursively find the first file node in the tree. */ function findFirstFile(nodes: FileNode[]): string | null { @@ -138,8 +137,6 @@ export function EvalDetail({ }: EvalDetailProps) { const [activeTab, setActiveTab] = useState(initialTab); const [selectedFilePath, setSelectedFilePath] = useState(initialSelectedFilePath); - const { data: config } = useStudioConfig(projectId); - const isReadOnly = config?.read_only === true; const detailResult = selectedTrial ? selectedTrialResult(result, selectedTrial) : result; const showAggregateRepeat = repeatGroup != null && selectedTrial == null; @@ -153,7 +150,6 @@ export function EvalDetail({ { id: 'transcript', label: 'Transcript' }, { id: 'source', label: 'Source' }, { id: 'files', label: 'Files' }, - ...(isReadOnly ? [] : [{ id: 'feedback' as const, label: 'Feedback' }]), ]; const openFile = (filePath: string) => { @@ -250,11 +246,6 @@ export function EvalDetail({ )} - {!isReadOnly && activeTab === 'feedback' && ( -
- -
- )} ); diff --git a/apps/dashboard/src/components/FeedbackPanel.tsx b/apps/dashboard/src/components/FeedbackPanel.tsx deleted file mode 100644 index 3901b0ca6..000000000 --- a/apps/dashboard/src/components/FeedbackPanel.tsx +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Feedback panel for leaving review comments on individual eval results. - * - * Reads existing feedback via the /api/feedback endpoint and persists - * new comments via POST /api/feedback. - */ - -import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { useCallback, useEffect, useState } from 'react'; - -import { useFeedback } from '~/lib/api'; - -interface FeedbackPanelProps { - testId: string; - projectId?: string; -} - -function feedbackUrl(projectId?: string): string { - return projectId ? `/api/projects/${encodeURIComponent(projectId)}/feedback` : '/api/feedback'; -} - -async function saveFeedback(testId: string, comment: string, projectId?: string) { - const res = await fetch(feedbackUrl(projectId), { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ reviews: [{ test_id: testId, comment }] }), - }); - if (!res.ok) { - throw new Error(`Failed to save feedback: ${res.status}`); - } - return res.json(); -} - -export function FeedbackPanel({ testId, projectId }: FeedbackPanelProps) { - const { data } = useFeedback(projectId); - const queryClient = useQueryClient(); - - const existing = data?.reviews?.find((r) => r.test_id === testId); - const [comment, setComment] = useState(existing?.comment ?? ''); - const [saved, setSaved] = useState(false); - - // Sync when feedback data loads (existing?.comment captures testId changes - // since `existing` is derived from testId via the find() above) - useEffect(() => { - setComment(existing?.comment ?? ''); - setSaved(false); - }, [existing?.comment]); - - const mutation = useMutation({ - mutationFn: () => saveFeedback(testId, comment, projectId), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['feedback', projectId ?? ''] }); - setSaved(true); - setTimeout(() => setSaved(false), 2000); - }, - }); - - const handleSave = useCallback(() => { - mutation.mutate(); - }, [mutation]); - - return ( -
-

Feedback

-