refactor(grading): rename script grader wording

AGENTS.md

@@ -62,7 +62,7 @@ These baseline rules apply to every repo change. They summarize the most common
 ## Repo Map
 
 - `packages/core/`: evaluation engine, providers, grading, project registry, and the programmatic API.
-- `packages/sdk/`: lightweight assertion SDK such as `defineAssertion` and `defineCodeGrader`.
+- `packages/sdk/`: lightweight assertion SDK such as `defineAssertion` and `defineScriptGrader`.
 - `apps/cli/`: published CLI surface for `agentv`.
 - `apps/web/src/content/docs/`: public product and CLI docs on agentv.dev.
 - `examples/`: examples that double as reference material and integration coverage.

apps/cli/src/commands/eval/commands/vitest.ts

@@ -1,6 +1,6 @@
 import { command, flag, number, option, optional, restPositionals, string } from 'cmd-ts';
 
-import { runCodeGrader, runVitestWorkspaceGrader } from '@agentv/sdk';
+import { runScriptGrader, runVitestWorkspaceGrader } from '@agentv/sdk';
 
 function parseCommand(value: string | undefined): readonly string[] | undefined {
   const trimmed = value?.trim();
@@ -42,7 +42,7 @@ export const evalVitestCommand = command({
     }),
   },
   handler: async ({ testFiles, cwd, vitestCommand, timeoutMs, inWorkspace, passWithNoTests }) => {
-    await runCodeGrader((input) => {
+    await runScriptGrader((input) => {
       if (testFiles.length === 0) {
         throw new Error('Provide at least one Vitest verifier file.');
       }

apps/cli/src/commands/pipeline/bench.ts

@@ -2,7 +2,7 @@
  * `agentv pipeline bench` — Merge script and LLM grader scores into final
  * benchmark artifacts.
  *
- * Reads code_grader_results and llm_grader_results from disk per test.
+ * Reads script_grader_results and llm_grader_results from disk per test.
  *
  * Writes:
  *   - <test-id>/grading.json  (per-test grading breakdown)
@@ -65,7 +65,7 @@ export const evalBenchCommand = command({
       const allAssertions: { text: string; passed: boolean; evidence: string }[] = [];
 
       // Collect script grader results
-      const codeResultsDir = join(testDir, 'code_grader_results');
+      const codeResultsDir = join(testDir, 'script_grader_results');
       try {
         const resultFiles = (await readdir(codeResultsDir)).filter((f) => f.endsWith('.json'));
         for (const file of resultFiles) {

apps/cli/src/commands/pipeline/grade.ts

@@ -2,15 +2,15 @@
  * `agentv pipeline grade` — Run grader assertions against response.md files
  * in an export directory produced by `pipeline input`.
  *
- * All grader configs live in code_graders/<name>.json. Each config has a `type`
+ * All grader configs live in script_graders/<name>.json. Each config has a `type`
  * field that determines how it's evaluated:
  * - `script` (or configs with a `command` field): executed as external scripts
  * - Built-in types (contains, regex, equals, etc.): evaluated in-process
  *
- * Results are written to code_grader_results/<name>.json for pipeline bench.
+ * Results are written to script_grader_results/<name>.json for pipeline bench.
  *
  * Export directory additions:
- *   <out-dir>/<suite>/<test-id>/code_grader_results/<name>.json
+ *   <out-dir>/<suite>/<test-id>/script_grader_results/<name>.json
  */
 import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
@@ -66,7 +66,7 @@ export interface GraderTask {
  * external scripts, built-in types (contains, regex, etc.) are evaluated in-process.
  * Shared by `pipeline grade` and `pipeline run`.
  */
-export async function runCodeGraders(
+export async function runScriptGraders(
   tasks: GraderTask[],
   concurrency: number,
 ): Promise<{ totalGraders: number; totalPassed: number }> {
@@ -86,13 +86,13 @@ export async function runCodeGraders(
   const executeGrader = async (task: GraderTask) => {
     const { testDir, resultsDir, graderFile, responseText } = task;
     const graderConfig = JSON.parse(
-      await readFile(join(testDir, 'code_graders', graderFile), 'utf8'),
+      await readFile(join(testDir, 'script_graders', graderFile), 'utf8'),
     );
 
     // Dispatch: configs with a `command` field are external scripts;
     // all others are built-in deterministic assertions evaluated in-process.
     if (graderConfig.command) {
-      await executeCodeGrader(graderConfig, task);
+      await executeScriptGrader(graderConfig, task);
     } else {
       await executeBuiltinGrader(graderConfig, responseText, resultsDir);
     }
@@ -104,7 +104,7 @@ export async function runCodeGraders(
   };
 
   /** Run an external script grader. */
-  const executeCodeGrader = async (graderConfig: Record<string, unknown>, task: GraderTask) => {
+  const executeScriptGrader = async (graderConfig: Record<string, unknown>, task: GraderTask) => {
     const { testId, resultsDir, responseText, inputData } = task;
     const graderName = graderConfig.name as string;
     const graderType = typeof graderConfig.type === 'string' ? graderConfig.type : 'script';
@@ -299,12 +299,12 @@ export const evalGradeCommand = command({
     for (const testId of testIds) {
       const subpath = safeSuiteName ? [safeSuiteName, testId] : [testId];
       const testDir = join(exportDir, ...subpath);
-      const codeGradersDir = join(testDir, 'code_graders');
-      const resultsDir = join(testDir, 'code_grader_results');
+      const scriptGradersDir = join(testDir, 'script_graders');
+      const resultsDir = join(testDir, 'script_grader_results');
 
       let graderFiles: string[];
       try {
-        graderFiles = (await readdir(codeGradersDir)).filter((f: string) => f.endsWith('.json'));
+        graderFiles = (await readdir(scriptGradersDir)).filter((f: string) => f.endsWith('.json'));
       } catch {
         continue; // No graders for this test
       }
@@ -320,7 +320,7 @@ export const evalGradeCommand = command({
       }
     }
 
-    const { totalGraders, totalPassed } = await runCodeGraders(tasks, maxWorkers);
+    const { totalGraders, totalPassed } = await runScriptGraders(tasks, maxWorkers);
     console.log(`Graded ${totalGraders} grader(s): ${totalPassed} passed`);
   },
 });

apps/cli/src/commands/pipeline/input.ts

@@ -16,7 +16,7 @@
  *           ├── criteria.md
  *           ├── expected_output.json    (if present)
  *           ├── llm_graders/<name>.json
- *           └── code_graders/<name>.json   # script/deterministic grader configs
+ *           └── script_graders/<name>.json   # script/deterministic grader configs
  */
 import { readFile } from 'node:fs/promises';
 import { mkdir, writeFile } from 'node:fs/promises';
@@ -234,7 +234,7 @@ export const evalInputCommand = command({
 });
 
 interface GraderCounts {
-  codeGraders: number;
+  scriptGraders: number;
   llmGraders: number;
   builtinAssertions: number;
 }
@@ -244,21 +244,21 @@ async function writeGraderConfigs(
   assertions: readonly GraderConfig[],
   evalDir: string,
 ): Promise<GraderCounts> {
-  const counts: GraderCounts = { codeGraders: 0, llmGraders: 0, builtinAssertions: 0 };
-  const codeGradersDir = join(testDir, 'code_graders');
+  const counts: GraderCounts = { scriptGraders: 0, llmGraders: 0, builtinAssertions: 0 };
+  const scriptGradersDir = join(testDir, 'script_graders');
   const llmGradersDir = join(testDir, 'llm_graders');
 
-  let hasCodeGraders = false;
+  let hasScriptGraders = false;
   let hasLlmGraders = false;
 
   for (const assertion of assertions) {
     if (assertion.type === 'script') {
-      if (!hasCodeGraders) {
-        await mkdir(codeGradersDir, { recursive: true });
-        hasCodeGraders = true;
+      if (!hasScriptGraders) {
+        await mkdir(scriptGradersDir, { recursive: true });
+        hasScriptGraders = true;
       }
       const config = assertion as ScriptGraderConfig;
-      await writeJson(join(codeGradersDir, `${config.name}.json`), {
+      await writeJson(join(scriptGradersDir, `${config.name}.json`), {
         name: config.name,
         type: 'script',
         command: config.command,
@@ -305,12 +305,12 @@ async function writeGraderConfigs(
         config: {},
       });
     } else if (BUILTIN_ASSERTION_TYPES.has(assertion.type)) {
-      if (!hasCodeGraders) {
-        await mkdir(codeGradersDir, { recursive: true });
-        hasCodeGraders = true;
+      if (!hasScriptGraders) {
+        await mkdir(scriptGradersDir, { recursive: true });
+        hasScriptGraders = true;
       }
       const config = assertion as GraderConfig & { value?: unknown; flags?: string };
-      await writeJson(join(codeGradersDir, `${config.name}.json`), {
+      await writeJson(join(scriptGradersDir, `${config.name}.json`), {
         name: config.name,
         type: config.type,
         value: config.value,

apps/cli/src/commands/pipeline/run.ts

@@ -25,7 +25,7 @@ import { buildDefaultRunDir } from '../eval/result-layout.js';
 import { findRepoRoot } from '../eval/shared.js';
 import { selectTarget } from '../eval/targets.js';
 import type { GraderTask } from './grade.js';
-import { runCodeGraders } from './grade.js';
+import { runScriptGraders } from './grade.js';
 
 /**
  * Convert a Message[] array to plain text.
@@ -386,12 +386,12 @@ export const evalRunCommand = command({
     for (const testId of testIds) {
       const subpath = safeSuiteName ? [safeSuiteName, testId] : [testId];
       const testDir = join(outDir, ...subpath);
-      const codeGradersDir = join(testDir, 'code_graders');
-      const resultsDir = join(testDir, 'code_grader_results');
+      const scriptGradersDir = join(testDir, 'script_graders');
+      const resultsDir = join(testDir, 'script_grader_results');
 
       let graderFiles: string[];
       try {
-        graderFiles = (await readdir(codeGradersDir)).filter((f) => f.endsWith('.json'));
+        graderFiles = (await readdir(scriptGradersDir)).filter((f) => f.endsWith('.json'));
       } catch {
         continue;
       }
@@ -407,7 +407,7 @@ export const evalRunCommand = command({
     }
 
     const graderConcurrency = workers ?? 10;
-    const { totalGraders, totalPassed } = await runCodeGraders(graderTasks, graderConcurrency);
+    const { totalGraders, totalPassed } = await runScriptGraders(graderTasks, graderConcurrency);
     console.log(`Graded ${totalGraders} script grader(s): ${totalPassed} passed`);
     console.log('');
     console.log(`Results in ${outDir}`);
@@ -432,20 +432,20 @@ async function writeGraderConfigs(
   assertions: readonly GraderConfig[],
   evalDir: string,
 ): Promise<void> {
-  const codeGradersDir = join(testDir, 'code_graders');
+  const scriptGradersDir = join(testDir, 'script_graders');
   const llmGradersDir = join(testDir, 'llm_graders');
 
-  let hasCodeGraders = false;
+  let hasScriptGraders = false;
   let hasLlmGraders = false;
 
   for (const assertion of assertions) {
     if (assertion.type === 'script') {
-      if (!hasCodeGraders) {
-        await mkdir(codeGradersDir, { recursive: true });
-        hasCodeGraders = true;
+      if (!hasScriptGraders) {
+        await mkdir(scriptGradersDir, { recursive: true });
+        hasScriptGraders = true;
       }
       const config = assertion as ScriptGraderConfig;
-      await writeJson(join(codeGradersDir, `${config.name}.json`), {
+      await writeJson(join(scriptGradersDir, `${config.name}.json`), {
         name: config.name,
         type: 'script',
         command: config.command,

apps/cli/test/commands/eval/assert.test.ts

@@ -156,7 +156,7 @@ describe('agentv eval assert', () => {
     }
   }, 30_000);
 
-  it('sends only canonical wire fields to code graders', async () => {
+  it('sends only canonical wire fields to script graders', async () => {
     const { baseDir } = await createGraderFixture();
     try {
       const result = await execa(

apps/cli/test/commands/eval/pipeline/bench.test.ts

@@ -8,14 +8,14 @@ const CLI_ENTRY = join(import.meta.dirname, '../../../../src/cli.ts');
 describe('pipeline bench', () => {
   beforeEach(async () => {
     const testDir = join(OUT_DIR, 'test-01');
-    const codeResultsDir = join(testDir, 'code_grader_results');
+    const codeResultsDir = join(testDir, 'script_grader_results');
     const llmGradersDir = join(testDir, 'llm_graders');
     const llmResultsDir = join(testDir, 'llm_grader_results');
-    const codeGradersDir = join(testDir, 'code_graders');
+    const scriptGradersDir = join(testDir, 'script_graders');
     await mkdir(codeResultsDir, { recursive: true });
     await mkdir(llmGradersDir, { recursive: true });
     await mkdir(llmResultsDir, { recursive: true });
-    await mkdir(codeGradersDir, { recursive: true });
+    await mkdir(scriptGradersDir, { recursive: true });
 
     await writeFile(
       join(OUT_DIR, 'manifest.json'),
@@ -46,7 +46,7 @@ describe('pipeline bench', () => {
       }),
     );
     await writeFile(
-      join(codeGradersDir, 'contains.json'),
+      join(scriptGradersDir, 'contains.json'),
       JSON.stringify({
         name: 'contains',
         command: ['echo'],

apps/cli/test/commands/eval/pipeline/grade.test.ts

@@ -8,8 +8,8 @@ const CLI_ENTRY = join(import.meta.dirname, '../../../../src/cli.ts');
 describe('pipeline grade', () => {
   beforeEach(async () => {
     const testDir = join(OUT_DIR, 'test-01');
-    const codeGradersDir = join(testDir, 'code_graders');
-    await mkdir(codeGradersDir, { recursive: true });
+    const scriptGradersDir = join(testDir, 'script_graders');
+    await mkdir(scriptGradersDir, { recursive: true });
 
     await writeFile(join(testDir, 'response.md'), 'hello world');
     await writeFile(
@@ -20,7 +20,7 @@ describe('pipeline grade', () => {
       }),
     );
     await writeFile(
-      join(codeGradersDir, 'always_pass.json'),
+      join(scriptGradersDir, 'always_pass.json'),
       JSON.stringify({
         name: 'always_pass',
         command: [
@@ -46,12 +46,12 @@ describe('pipeline grade', () => {
     await rm(OUT_DIR, { recursive: true, force: true });
   });
 
-  it('writes code_grader_results/<name>.json with score and assertions', async () => {
+  it('writes script_grader_results/<name>.json with score and assertions', async () => {
     const { execa } = await import('execa');
     await execa('bun', [CLI_ENTRY, 'pipeline', 'grade', OUT_DIR]);
 
     const result = JSON.parse(
-      await readFile(join(OUT_DIR, 'test-01', 'code_grader_results', 'always_pass.json'), 'utf8'),
+      await readFile(join(OUT_DIR, 'test-01', 'script_grader_results', 'always_pass.json'), 'utf8'),
     );
     expect(result.score).toBe(1);
     expect(result.name).toBe('always_pass');
@@ -65,7 +65,7 @@ describe('pipeline grade — builtin assertions', () => {
 
   beforeEach(async () => {
     const testDir = join(BUILTIN_OUT, 'test-01');
-    const builtinGradersDir = join(testDir, 'code_graders');
+    const builtinGradersDir = join(testDir, 'script_graders');
     await mkdir(builtinGradersDir, { recursive: true });
 
     await writeFile(join(testDir, 'response.md'), 'hello world');
@@ -127,15 +127,18 @@ describe('pipeline grade — builtin assertions', () => {
     await execa('bun', [CLI_ENTRY, 'pipeline', 'grade', BUILTIN_OUT]);
 
     const containsResult = JSON.parse(
-      await readFile(join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_hello.json'), 'utf8'),
+      await readFile(
+        join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_hello.json'),
+        'utf8',
+      ),
     );
     expect(containsResult.score).toBe(1);
     expect(containsResult.type).toBe('contains');
     expect(containsResult.assertions[0].passed).toBe(true);
 
     const regexResult = JSON.parse(
       await readFile(
-        join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'matches_pattern.json'),
+        join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'matches_pattern.json'),
         'utf8',
       ),
     );
@@ -144,7 +147,7 @@ describe('pipeline grade — builtin assertions', () => {
 
     const failingContainsResult = JSON.parse(
       await readFile(
-        join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_goodbye.json'),
+        join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_goodbye.json'),
         'utf8',
       ),
     );
@@ -154,7 +157,7 @@ describe('pipeline grade — builtin assertions', () => {
 
   it('applies negate to invert score', async () => {
     await writeFile(
-      join(BUILTIN_OUT, 'test-01', 'code_graders', 'has_goodbye.json'),
+      join(BUILTIN_OUT, 'test-01', 'script_graders', 'has_goodbye.json'),
       JSON.stringify({
         name: 'has_goodbye',
         type: 'contains',
@@ -169,7 +172,7 @@ describe('pipeline grade — builtin assertions', () => {
 
     const result = JSON.parse(
       await readFile(
-        join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_goodbye.json'),
+        join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_goodbye.json'),
         'utf8',
       ),
     );