feat(cli): infer Vitest verifier eval commands

apps/cli/src/commands/eval/index.ts

@@ -9,7 +9,7 @@ import { evalVitestCommand } from './commands/vitest.js';
 export const evalCommand = subcommands({
   name: 'eval',
   description:
-    'Evaluation commands. Shorthand: `agentv eval <eval-paths...>` aliases `agentv eval run <eval-paths...>`.',
+    'Evaluation commands. Shorthand: eval files run with `eval run`; verifier-looking test files run with `eval vitest`.',
   cmds: {
     run: evalRunCommand,
     assert: evalAssertCommand,

apps/cli/src/index.ts

@@ -62,6 +62,7 @@ export const app = subcommands({
  * implicit `run` subcommand for backward-compatible `agentv eval <paths>`.
  */
 const EVAL_SUBCOMMANDS = new Set(['run', 'assert', 'aggregate', 'bundle', 'vitest']);
+const VITEST_VERIFIER_RE = /(?:^|[/\\])(?:EVAL|[^/\\]+[.-](?:test|spec))\.[cm]?[jt]sx?$/i;
 
 /**
  * Top-level CLI command names (excluding `eval` itself).
@@ -100,6 +101,10 @@ export function shouldRunBeforeSessionHook(argv: string[]): boolean {
   return !(argv[2] === 'eval' && argv[3] === 'vitest');
 }
 
+export function inferEvalSubcommand(arg: string | undefined): 'run' | 'vitest' {
+  return arg && VITEST_VERIFIER_RE.test(arg) ? 'vitest' : 'run';
+}
+
 /**
  * Preprocess argv for convenience aliases:
  * - `--eval-id` → `--test-id`
@@ -122,11 +127,12 @@ export function preprocessArgv(argv: string[]): string[] {
     }
   }
 
-  // Implicit `run` subcommand: `agentv eval [<arg>]` → `agentv eval run [<arg>]`
+  // Implicit eval subcommand: `agentv eval [<arg>]` injects the inferred command
   // when the first arg after `eval` is absent or is not a known eval subcommand.
   // Backward-compat: `eval` used to be a direct command; now it is a subcommands group.
   // Bare `agentv eval` falls through to the run handler so its TTY check can launch
-  // the interactive wizard.
+  // the interactive wizard. Vitest-looking verifier files use the protocol adapter
+  // directly so deterministic workspace graders can stay short in eval YAML.
   // Only applies when `eval` is the top-level subcommand.
   // Exception: `--help` / `-h` should show the eval group help, not run's help.
   const evalIdx = result.indexOf('eval');
@@ -139,7 +145,7 @@ export function preprocessArgv(argv: string[]): string[] {
       const isHelp = nextArg === '--help' || nextArg === '-h';
       const isKnownSubcommand = nextArg !== undefined && EVAL_SUBCOMMANDS.has(nextArg);
       if (!isHelp && !isKnownSubcommand) {
-        result.splice(evalIdx + 1, 0, 'run');
+        result.splice(evalIdx + 1, 0, inferEvalSubcommand(nextArg));
       }
     }
   }

apps/cli/test/commands/eval/vitest.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
-import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -39,13 +39,19 @@ const report = {
   ],
 };
 
-async function runCli(args: readonly string[], cwd: string, input: string) {
+async function runCli(
+  args: readonly string[],
+  cwd: string,
+  input: string,
+  env: Record<string, string> = {},
+) {
   return execa('bun', ['--no-env-file', CLI_ENTRY, ...args], {
     cwd,
     input,
     env: {
       AGENTV_HOME: path.join(cwd, '.agentv-home'),
       AGENTV_NO_UPDATE_CHECK: '1',
+      ...env,
     },
   });
 }
@@ -127,4 +133,62 @@ process.exit(1);
     const workspaceEntries = await readdir(workspacePath);
     expect(workspaceEntries.some((entry) => entry.startsWith('.agentv-vitest-'))).toBe(false);
   });
+
+  it('infers the Vitest adapter for verifier-looking eval paths', async () => {
+    const workspacePath = path.join(tempDir, 'workspace');
+    const gradersPath = path.join(tempDir, 'graders');
+    const binPath = path.join(tempDir, 'bin');
+    const fakeBunx = path.join(binPath, 'bunx');
+    await mkdir(workspacePath, { recursive: true });
+    await mkdir(gradersPath, { recursive: true });
+    await mkdir(binPath, { recursive: true });
+    await writeFile(
+      path.join(gradersPath, 'welcome-banner.test.ts'),
+      'import { expect, it } from "vitest";\n',
+      'utf8',
+    );
+    await writeFile(
+      fakeBunx,
+      `#!/usr/bin/env bun
+import { writeFileSync } from 'node:fs';
+
+const args = process.argv.slice(2);
+writeFileSync('vitest-args.json', JSON.stringify(args));
+const outputArg = args.find((arg) => arg.startsWith('--outputFile='));
+if (!outputArg) throw new Error('missing outputFile arg');
+writeFileSync(outputArg.slice('--outputFile='.length), JSON.stringify(${JSON.stringify(report)}));
+process.exit(1);
+`,
+      'utf8',
+    );
+    await chmod(fakeBunx, 0o755);
+
+    const payload = JSON.stringify({
+      criteria: 'Verify the workspace',
+      expected_output: [],
+      input_files: [],
+      input: [{ role: 'user', content: 'Update the welcome banner' }],
+      workspace_path: workspacePath,
+    });
+
+    const result = await runCli(['eval', 'graders/welcome-banner.test.ts'], tempDir, payload, {
+      PATH: `${binPath}:${process.env.PATH ?? ''}`,
+    });
+
+    const output = JSON.parse(result.stdout);
+    expect(output.score).toBe(0.5);
+    expect(output.details).toMatchObject({
+      vitest_success: false,
+      num_total_tests: 2,
+      num_passed_tests: 1,
+      num_failed_tests: 1,
+    });
+
+    const vitestArgs = JSON.parse(
+      await readFile(path.join(workspacePath, 'vitest-args.json'), 'utf8'),
+    ) as string[];
+    expect(vitestArgs[0]).toBe('vitest');
+    expect(vitestArgs[1]).toBe('run');
+    expect(vitestArgs[2]).toMatch(/^\.agentv-vitest-.+\/0-welcome-banner\.test\.ts$/);
+  });
 });

apps/cli/test/unit/preprocess-argv.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'bun:test';
 
 import {
+  inferEvalSubcommand,
   preprocessArgv,
   shouldRunBeforeSessionHook,
   usesDeprecatedStudioAlias,
@@ -25,6 +26,22 @@ describe('preprocessArgv', () => {
       expect(result).toEqual(['node', 'agentv', 'eval', 'run', 'file.yaml', '--verbose']);
     });
 
+    it('inserts `vitest` when eval is followed by a verifier test file', () => {
+      const result = preprocessArgv(['node', 'agentv', 'eval', 'graders/welcome-banner.test.ts']);
+      expect(result).toEqual([
+        'node',
+        'agentv',
+        'eval',
+        'vitest',
+        'graders/welcome-banner.test.ts',
+      ]);
+    });
+
+    it('inserts `vitest` for Vercel-style EVAL.ts verifier files', () => {
+      const result = preprocessArgv(['node', 'agentv', 'eval', 'evals/task/EVAL.ts']);
+      expect(result).toEqual(['node', 'agentv', 'eval', 'vitest', 'evals/task/EVAL.ts']);
+    });
+
     it('does not insert `run` when eval is followed by a known subcommand', () => {
       const argv = ['node', 'agentv', 'eval', 'assert', 'grader-name', '--output', 'test'];
       expect(preprocessArgv(argv)).toEqual(argv);
@@ -114,4 +131,17 @@ describe('preprocessArgv', () => {
       );
     });
   });
+
+  describe('inferEvalSubcommand', () => {
+    it.each([
+      ['graders/welcome-banner.test.ts', 'vitest'],
+      ['graders/welcome-banner.spec.tsx', 'vitest'],
+      ['vercel/evals/task/EVAL.ts', 'vitest'],
+      ['evals/greeting.eval.ts', 'run'],
+      ['evals/dataset.eval.yaml', 'run'],
+      ['graders/custom-grader.ts', 'run'],
+    ] as const)('infers %s as %s', (input, expected) => {
+      expect(inferEvalSubcommand(input)).toBe(expected);
+    });
+  });
 });

apps/web/src/content/docs/docs/evaluation/sdk.mdx

@@ -46,7 +46,7 @@ Use the simplest surface that matches the job:
 - **`evaluate({ specFile })`** when you want library control around an existing YAML suite.
 - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput` and `assert`.
 - **`defineAssertion` / `defineCodeGrader`** when the grading logic itself must execute code.
-- **`agentv eval vitest`** for deterministic workspace checks that fit normal Vitest `expect(...)` tests.
+- **`agentv eval <verifier.test.ts>`** for deterministic workspace checks that fit normal Vitest `expect(...)` tests.
 
 There is no separate first-party Python authoring SDK today. Python-facing workflows should either emit canonical YAML/JSONL or implement executable graders that consume the standard `snake_case` wire format.
 
@@ -297,7 +297,7 @@ it('links to the dashboard', () => {
 assertions:
   - name: vitest-welcome-banner
     type: code-grader
-    command: [agentv, eval, vitest, graders/welcome-banner.test.ts]
+    command: [agentv, eval, graders/welcome-banner.test.ts]
 ```
 
 Use `defineWorkspaceGrader` only for tiny one-off file checks or custom score shaping:
@@ -313,7 +313,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [
 ]);
 ```
 
-`defineCodeGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, vitest, ...]` without a custom wrapper. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name.
+`defineCodeGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name.
 
 For detailed patterns, input/output contracts, and language-agnostic examples, see [Code Graders](/docs/graders/code-graders/).
 

apps/web/src/content/docs/docs/graders/code-graders.mdx

@@ -244,10 +244,10 @@ Then use AgentV's built-in Vitest adapter as the `code-grader` command. The adap
 assertions:
   - name: vitest-welcome-banner
     type: code-grader
-    command: [agentv, eval, vitest, graders/welcome-banner.test.ts]
+    command: [agentv, eval, graders/welcome-banner.test.ts]
 ```
 
-Use `agentv eval vitest --in-workspace verifiers/welcome-banner.test.ts` when the verifier file is already materialized inside the prepared workspace. Use the SDK's `defineVitestWorkspaceGrader()` only when embedding the adapter in a custom script or custom command. See `examples/features/vitest-workspace-grader/` for a runnable example.
+AgentV infers the Vitest adapter for verifier-looking files such as `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts`. Use `agentv eval vitest --in-workspace verifiers/welcome-banner.test.ts` when the verifier file is already materialized inside the prepared workspace or you need other adapter options. Use the SDK's `defineVitestWorkspaceGrader()` only when embedding the adapter in a custom script or custom command. See `examples/features/vitest-workspace-grader/` for a runnable example.
 
 ### Lower-Level Workspace Helpers
 

examples/features/vitest-workspace-grader/README.md

@@ -5,7 +5,7 @@ Demonstrates the preferred deterministic workspace grader path: write normal Vit
 ## Files
 
 - `graders/welcome-banner.test.ts`: plain Vitest verifier that reads `app/page.tsx`
-- `evals/dataset.eval.yaml`: eval case that runs the verifier through `agentv eval vitest`
+- `evals/dataset.eval.yaml`: eval case that runs the verifier through `agentv eval <verifier.test.ts>`
 - `.agentv/targets.yaml`: mock CLI target that updates the workspace
 
 ## Run
@@ -38,9 +38,9 @@ The eval YAML calls AgentV's built-in adapter directly:
 assertions:
   - name: vitest-welcome-banner
     type: code-grader
-    command: [agentv, eval, vitest, graders/welcome-banner.test.ts]
+    command: [agentv, eval, graders/welcome-banner.test.ts]
 ```
 
-The local example uses a source-relative CLI path so it can run before the next AgentV package release. In a normal project, use the installed `agentv` binary form above.
+AgentV infers the built-in Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` verifier files. The local example uses a source-relative CLI path so it can run before the next AgentV package release. In a normal project, use the installed `agentv` binary form above.
 
 Use lower-level `defineCodeGrader` scripts when the grader needs custom scoring, multi-stage setup, external commands beyond a test runner, or structured `details` that do not map cleanly to individual test outcomes.

examples/features/vitest-workspace-grader/evals/dataset.eval.yaml

@@ -24,6 +24,5 @@ tests:
             "bun",
             "../../../../apps/cli/src/cli.ts",
             "eval",
-            "vitest",
             "../graders/welcome-banner.test.ts",
           ]

packages/sdk/README.md

@@ -77,12 +77,12 @@ Then reference the verifier directly from eval YAML through AgentV's built-in co
 assertions:
   - name: vitest-welcome-banner
     type: code-grader
-    command: [agentv, eval, vitest, graders/welcome-banner.test.ts]
+    command: [agentv, eval, graders/welcome-banner.test.ts]
 ```
 
 The command reads the normal code-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV assertion, and computes score as `passed / total`.
 
-Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script or when you need custom command options:
+Use the explicit `agentv eval vitest` subcommand when you need adapter options such as `--cwd`, `--in-workspace`, or `--vitest-command`. Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script:
 
 ```typescript
 #!/usr/bin/env bun

skills-data/agentv-eval-writer/SKILL.md

@@ -365,6 +365,13 @@ Contract: stdin JSON -> stdout JSON `{score, assertions: [{text, passed, evidenc
 Raw stdin uses snake_case and includes: `criteria`, `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config`
 SDK handlers receive the same payload in camelCase: `expectedOutput`, `traceSummary`, `tokenUsage`, `costUsd`, `durationMs`, `startTime`, `endTime`, `fileChanges`, `workspacePath`.
 When a workspace is configured, `workspace_path` is the absolute path to the workspace dir (also available as `AGENTV_WORKSPACE_PATH` env var). Use this for functional grading (e.g., running `npm test` in the workspace).
+For deterministic workspace checks that fit normal Vitest `expect(...)` tests, prefer a plain verifier file and the built-in adapter:
+```yaml
+- name: welcome_banner
+  type: code-grader
+  command: [agentv, eval, graders/welcome-banner.test.ts]
+```
+AgentV infers the Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` files. Use the explicit `agentv eval vitest` subcommand only when you need adapter flags such as `--cwd`, `--in-workspace`, or `--vitest-command`.
 See docs at https://agentv.dev/graders/code-graders/
 
 ### llm-grader
@@ -644,7 +651,7 @@ export default defineCodeGrader(({ output, trace }) => {
 });
 ```
 
-`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: <name>`. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`.
+`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: <name>`. `defineCodeGrader()` scripts are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`.
 
 ### Convention-Based Discovery