diff --git a/apps/cli/src/commands/eval/index.ts b/apps/cli/src/commands/eval/index.ts index 9b65fd6f5..69835f0f6 100644 --- a/apps/cli/src/commands/eval/index.ts +++ b/apps/cli/src/commands/eval/index.ts @@ -9,7 +9,7 @@ import { evalVitestCommand } from './commands/vitest.js'; export const evalCommand = subcommands({ name: 'eval', description: - 'Evaluation commands. Shorthand: `agentv eval ` aliases `agentv eval run `.', + 'Evaluation commands. Shorthand: eval files run with `eval run`; verifier-looking test files run with `eval vitest`.', cmds: { run: evalRunCommand, assert: evalAssertCommand, diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index e18ecfa2f..5aba9068d 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -62,6 +62,7 @@ export const app = subcommands({ * implicit `run` subcommand for backward-compatible `agentv eval `. */ const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate', 'bundle', 'vitest']); +const VITEST_VERIFIER_RE = /(?:^|[/\\])(?:EVAL|[^/\\]+[.-](?:test|spec))\.[cm]?[jt]sx?$/i; /** * Top-level CLI command names (excluding `eval` itself). @@ -100,6 +101,10 @@ export function shouldRunBeforeSessionHook(argv: string[]): boolean { return !(argv[2] === 'eval' && argv[3] === 'vitest'); } +export function inferEvalSubcommand(arg: string | undefined): 'run' | 'vitest' { + return arg && VITEST_VERIFIER_RE.test(arg) ? 'vitest' : 'run'; +} + /** * Preprocess argv for convenience aliases: * - `--eval-id` → `--test-id` @@ -122,11 +127,12 @@ export function preprocessArgv(argv: string[]): string[] { } } - // Implicit `run` subcommand: `agentv eval []` → `agentv eval run []` + // Implicit eval subcommand: `agentv eval []` injects the inferred command // when the first arg after `eval` is absent or is not a known eval subcommand. // Backward-compat: `eval` used to be a direct command; now it is a subcommands group. // Bare `agentv eval` falls through to the run handler so its TTY check can launch - // the interactive wizard. + // the interactive wizard. Vitest-looking verifier files use the protocol adapter + // directly so deterministic workspace graders can stay short in eval YAML. // Only applies when `eval` is the top-level subcommand. // Exception: `--help` / `-h` should show the eval group help, not run's help. const evalIdx = result.indexOf('eval'); @@ -139,7 +145,7 @@ export function preprocessArgv(argv: string[]): string[] { const isHelp = nextArg === '--help' || nextArg === '-h'; const isKnownSubcommand = nextArg !== undefined && EVAL_SUBCOMMANDS.has(nextArg); if (!isHelp && !isKnownSubcommand) { - result.splice(evalIdx + 1, 0, 'run'); + result.splice(evalIdx + 1, 0, inferEvalSubcommand(nextArg)); } } } diff --git a/apps/cli/test/commands/eval/vitest.test.ts b/apps/cli/test/commands/eval/vitest.test.ts index a71fc9915..f43368c8f 100644 --- a/apps/cli/test/commands/eval/vitest.test.ts +++ b/apps/cli/test/commands/eval/vitest.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -39,13 +39,19 @@ const report = { ], }; -async function runCli(args: readonly string[], cwd: string, input: string) { +async function runCli( + args: readonly string[], + cwd: string, + input: string, + env: Record = {}, +) { return execa('bun', ['--no-env-file', CLI_ENTRY, ...args], { cwd, input, env: { AGENTV_HOME: path.join(cwd, '.agentv-home'), AGENTV_NO_UPDATE_CHECK: '1', + ...env, }, }); } @@ -127,4 +133,62 @@ process.exit(1); const workspaceEntries = await readdir(workspacePath); expect(workspaceEntries.some((entry) => entry.startsWith('.agentv-vitest-'))).toBe(false); }); + + it('infers the Vitest adapter for verifier-looking eval paths', async () => { + const workspacePath = path.join(tempDir, 'workspace'); + const gradersPath = path.join(tempDir, 'graders'); + const binPath = path.join(tempDir, 'bin'); + const fakeBunx = path.join(binPath, 'bunx'); + await mkdir(workspacePath, { recursive: true }); + await mkdir(gradersPath, { recursive: true }); + await mkdir(binPath, { recursive: true }); + await writeFile( + path.join(gradersPath, 'welcome-banner.test.ts'), + 'import { expect, it } from "vitest";\n', + 'utf8', + ); + await writeFile( + fakeBunx, + `#!/usr/bin/env bun +import { writeFileSync } from 'node:fs'; + +const args = process.argv.slice(2); +writeFileSync('vitest-args.json', JSON.stringify(args)); +const outputArg = args.find((arg) => arg.startsWith('--outputFile=')); +if (!outputArg) throw new Error('missing outputFile arg'); +writeFileSync(outputArg.slice('--outputFile='.length), JSON.stringify(${JSON.stringify(report)})); +process.exit(1); +`, + 'utf8', + ); + await chmod(fakeBunx, 0o755); + + const payload = JSON.stringify({ + criteria: 'Verify the workspace', + expected_output: [], + input_files: [], + input: [{ role: 'user', content: 'Update the welcome banner' }], + workspace_path: workspacePath, + }); + + const result = await runCli(['eval', 'graders/welcome-banner.test.ts'], tempDir, payload, { + PATH: `${binPath}:${process.env.PATH ?? ''}`, + }); + + const output = JSON.parse(result.stdout); + expect(output.score).toBe(0.5); + expect(output.details).toMatchObject({ + vitest_success: false, + num_total_tests: 2, + num_passed_tests: 1, + num_failed_tests: 1, + }); + + const vitestArgs = JSON.parse( + await readFile(path.join(workspacePath, 'vitest-args.json'), 'utf8'), + ) as string[]; + expect(vitestArgs[0]).toBe('vitest'); + expect(vitestArgs[1]).toBe('run'); + expect(vitestArgs[2]).toMatch(/^\.agentv-vitest-.+\/0-welcome-banner\.test\.ts$/); + }); }); diff --git a/apps/cli/test/unit/preprocess-argv.test.ts b/apps/cli/test/unit/preprocess-argv.test.ts index e58f8c331..71ee3d55f 100644 --- a/apps/cli/test/unit/preprocess-argv.test.ts +++ b/apps/cli/test/unit/preprocess-argv.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'bun:test'; import { + inferEvalSubcommand, preprocessArgv, shouldRunBeforeSessionHook, usesDeprecatedStudioAlias, @@ -25,6 +26,22 @@ describe('preprocessArgv', () => { expect(result).toEqual(['node', 'agentv', 'eval', 'run', 'file.yaml', '--verbose']); }); + it('inserts `vitest` when eval is followed by a verifier test file', () => { + const result = preprocessArgv(['node', 'agentv', 'eval', 'graders/welcome-banner.test.ts']); + expect(result).toEqual([ + 'node', + 'agentv', + 'eval', + 'vitest', + 'graders/welcome-banner.test.ts', + ]); + }); + + it('inserts `vitest` for Vercel-style EVAL.ts verifier files', () => { + const result = preprocessArgv(['node', 'agentv', 'eval', 'evals/task/EVAL.ts']); + expect(result).toEqual(['node', 'agentv', 'eval', 'vitest', 'evals/task/EVAL.ts']); + }); + it('does not insert `run` when eval is followed by a known subcommand', () => { const argv = ['node', 'agentv', 'eval', 'assert', 'grader-name', '--output', 'test']; expect(preprocessArgv(argv)).toEqual(argv); @@ -114,4 +131,17 @@ describe('preprocessArgv', () => { ); }); }); + + describe('inferEvalSubcommand', () => { + it.each([ + ['graders/welcome-banner.test.ts', 'vitest'], + ['graders/welcome-banner.spec.tsx', 'vitest'], + ['vercel/evals/task/EVAL.ts', 'vitest'], + ['evals/greeting.eval.ts', 'run'], + ['evals/dataset.eval.yaml', 'run'], + ['graders/custom-grader.ts', 'run'], + ] as const)('infers %s as %s', (input, expected) => { + expect(inferEvalSubcommand(input)).toBe(expected); + }); + }); }); diff --git a/apps/web/src/content/docs/docs/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/evaluation/sdk.mdx index 2e97a3123..fdd7a34e5 100644 --- a/apps/web/src/content/docs/docs/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/evaluation/sdk.mdx @@ -46,7 +46,7 @@ Use the simplest surface that matches the job: - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput` and `assert`. - **`defineAssertion` / `defineCodeGrader`** when the grading logic itself must execute code. -- **`agentv eval vitest`** for deterministic workspace checks that fit normal Vitest `expect(...)` tests. +- **`agentv eval `** for deterministic workspace checks that fit normal Vitest `expect(...)` tests. There is no separate first-party Python authoring SDK today. Python-facing workflows should either emit canonical YAML/JSONL or implement executable graders that consume the standard `snake_case` wire format. @@ -297,7 +297,7 @@ it('links to the dashboard', () => { assertions: - name: vitest-welcome-banner type: code-grader - command: [agentv, eval, vitest, graders/welcome-banner.test.ts] + command: [agentv, eval, graders/welcome-banner.test.ts] ``` Use `defineWorkspaceGrader` only for tiny one-off file checks or custom score shaping: @@ -313,7 +313,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ ]); ``` -`defineCodeGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, vitest, ...]` without a custom wrapper. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. +`defineCodeGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. For detailed patterns, input/output contracts, and language-agnostic examples, see [Code Graders](/docs/graders/code-graders/). diff --git a/apps/web/src/content/docs/docs/graders/code-graders.mdx b/apps/web/src/content/docs/docs/graders/code-graders.mdx index 73b9a8ffd..c2422a192 100644 --- a/apps/web/src/content/docs/docs/graders/code-graders.mdx +++ b/apps/web/src/content/docs/docs/graders/code-graders.mdx @@ -244,10 +244,10 @@ Then use AgentV's built-in Vitest adapter as the `code-grader` command. The adap assertions: - name: vitest-welcome-banner type: code-grader - command: [agentv, eval, vitest, graders/welcome-banner.test.ts] + command: [agentv, eval, graders/welcome-banner.test.ts] ``` -Use `agentv eval vitest --in-workspace verifiers/welcome-banner.test.ts` when the verifier file is already materialized inside the prepared workspace. Use the SDK's `defineVitestWorkspaceGrader()` only when embedding the adapter in a custom script or custom command. See `examples/features/vitest-workspace-grader/` for a runnable example. +AgentV infers the Vitest adapter for verifier-looking files such as `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts`. Use `agentv eval vitest --in-workspace verifiers/welcome-banner.test.ts` when the verifier file is already materialized inside the prepared workspace or you need other adapter options. Use the SDK's `defineVitestWorkspaceGrader()` only when embedding the adapter in a custom script or custom command. See `examples/features/vitest-workspace-grader/` for a runnable example. ### Lower-Level Workspace Helpers diff --git a/examples/features/vitest-workspace-grader/README.md b/examples/features/vitest-workspace-grader/README.md index aaf5782a8..d3b3b74f5 100644 --- a/examples/features/vitest-workspace-grader/README.md +++ b/examples/features/vitest-workspace-grader/README.md @@ -5,7 +5,7 @@ Demonstrates the preferred deterministic workspace grader path: write normal Vit ## Files - `graders/welcome-banner.test.ts`: plain Vitest verifier that reads `app/page.tsx` -- `evals/dataset.eval.yaml`: eval case that runs the verifier through `agentv eval vitest` +- `evals/dataset.eval.yaml`: eval case that runs the verifier through `agentv eval ` - `.agentv/targets.yaml`: mock CLI target that updates the workspace ## Run @@ -38,9 +38,9 @@ The eval YAML calls AgentV's built-in adapter directly: assertions: - name: vitest-welcome-banner type: code-grader - command: [agentv, eval, vitest, graders/welcome-banner.test.ts] + command: [agentv, eval, graders/welcome-banner.test.ts] ``` -The local example uses a source-relative CLI path so it can run before the next AgentV package release. In a normal project, use the installed `agentv` binary form above. +AgentV infers the built-in Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` verifier files. The local example uses a source-relative CLI path so it can run before the next AgentV package release. In a normal project, use the installed `agentv` binary form above. Use lower-level `defineCodeGrader` scripts when the grader needs custom scoring, multi-stage setup, external commands beyond a test runner, or structured `details` that do not map cleanly to individual test outcomes. diff --git a/examples/features/vitest-workspace-grader/evals/dataset.eval.yaml b/examples/features/vitest-workspace-grader/evals/dataset.eval.yaml index f1076b018..bd1ad7eb8 100644 --- a/examples/features/vitest-workspace-grader/evals/dataset.eval.yaml +++ b/examples/features/vitest-workspace-grader/evals/dataset.eval.yaml @@ -24,6 +24,5 @@ tests: "bun", "../../../../apps/cli/src/cli.ts", "eval", - "vitest", "../graders/welcome-banner.test.ts", ] diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ea6c5cec8..dc4b67a03 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -77,12 +77,12 @@ Then reference the verifier directly from eval YAML through AgentV's built-in co assertions: - name: vitest-welcome-banner type: code-grader - command: [agentv, eval, vitest, graders/welcome-banner.test.ts] + command: [agentv, eval, graders/welcome-banner.test.ts] ``` The command reads the normal code-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV assertion, and computes score as `passed / total`. -Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script or when you need custom command options: +Use the explicit `agentv eval vitest` subcommand when you need adapter options such as `--cwd`, `--in-workspace`, or `--vitest-command`. Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script: ```typescript #!/usr/bin/env bun diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 823670ff6..50298c0b9 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -365,6 +365,13 @@ Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidenc Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config` SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`. When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace). +For deterministic workspace checks that fit normal Vitest `expect(...)` tests, prefer a plain verifier file and the built-in adapter: +```yaml +- name: welcome_banner + type: code-grader + command: [agentv, eval, graders/welcome-banner.test.ts] +``` +AgentV infers the Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` files. Use the explicit `agentv eval vitest` subcommand only when you need adapter flags such as `--cwd`, `--in-workspace`, or `--vitest-command`. See docs at https://agentv.dev/graders/code-graders/ ### llm-grader @@ -644,7 +651,7 @@ export default defineCodeGrader(({ output, trace }) => { }); ``` -`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: `. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. +`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: `. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`. ### Convention-Based Discovery