From 59200f82cf0faa39c008ee25a804f9a182828ffc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 14:02:55 +0200 Subject: [PATCH] refactor(grading): rename script grader wording # Conflicts: # apps/web/src/content/docs/docs/next/index.mdx # examples/README.md # examples/contract/evals/script-grader-contract.eval.yaml # examples/features/README.md # examples/features/batch-cli/README.md # examples/features/code-grader-sdk/evals/dataset.eval.baseline.jsonl # examples/features/code-grader-sdk/evals/dataset.eval.yaml # examples/features/deterministic-graders/README.md # examples/features/document-extraction/evals/field-accuracy.eval.yaml # examples/features/prompt-template-sdk/evals/suite.yaml # examples/features/script-grader-sdk/README.md # examples/features/script-grader-sdk/evals/dataset.eval.baseline.jsonl # examples/features/script-grader-sdk/evals/dataset.eval.yaml # examples/features/script-grader-sdk/evals/suite.baseline.jsonl # examples/features/script-grader-sdk/evals/suite.yaml # skills-data/agentv-bench/SKILL.md # skills-data/agentv-bench/agents/analyzer.md # skills-data/agentv-bench/agents/comparator.md # skills-data/agentv-bench/references/eval-yaml-spec.md --- AGENTS.md | 2 +- apps/cli/src/commands/eval/commands/vitest.ts | 4 +- apps/cli/src/commands/pipeline/bench.ts | 4 +- apps/cli/src/commands/pipeline/grade.ts | 22 +- apps/cli/src/commands/pipeline/input.ts | 26 +- apps/cli/src/commands/pipeline/run.ts | 22 +- apps/cli/test/commands/eval/assert.test.ts | 2 +- .../test/commands/eval/pipeline/bench.test.ts | 8 +- .../test/commands/eval/pipeline/grade.test.ts | 25 +- .../test/commands/eval/pipeline/input.test.ts | 14 +- .../eval/pipeline/pipeline-e2e.test.ts | 2 +- apps/cli/test/commands/eval/vitest.test.ts | 2 +- .../content/docs/docs/next/evaluation/sdk.mdx | 18 +- .../docs/next/graders/custom-assertions.mdx | 4 +- .../docs/docs/next/graders/python-helpers.mdx | 4 +- .../{code-graders.mdx => script-graders.mdx} | 16 +- apps/web/src/content/docs/docs/next/index.mdx | 3 +- .../agent-plugin-review.eval.yaml | 18 +- .../deploy-auto/deploy-execute.eval.yaml | 4 +- evals/agentv-dev/skills/README.md | 2 +- .../skills/output-correctness.eval.yaml | 14 +- .../skills/skill-invocation.eval.yaml | 16 +- .../skills/skill-selection.eval.yaml | 16 +- .../agentv-self/graders/pr-workflow-safety.ts | 4 +- .../graders/required-file-reads.ts | 6 +- examples/README.md | 16 +- ....yaml => script-grader-contract.eval.yaml} | 10 +- .../fixtures/repo-materialization-note.txt | 2 +- .../scripts/check-repo-materialization.ts | 2 +- ...load.ts => check-script-grader-payload.ts} | 4 +- examples/contract/scripts/check-workspace.ts | 2 +- examples/features/README.md | 14 +- examples/features/batch-cli/README.md | 4 +- .../graders/check-batch-cli-output.ts | 6 +- .../code-grader-sdk/evals/example.txt | 1 - examples/features/composite/evals/suite.yaml | 6 +- examples/features/copilot-log-eval/README.md | 8 +- .../evals/skill-trigger.EVAL.yaml | 4 +- .../graders/transcript-quality.ts | 4 +- .../features/default-graders/evals/suite.yaml | 2 +- .../features/deterministic-graders/README.md | 4 +- .../graders/assertions.ts | 6 +- examples/features/docker-workspace/README.md | 2 +- .../evals/docker-example.EVAL.yaml | 2 +- .../features/document-extraction/README.md | 6 +- .../evals/field-accuracy.eval.yaml | 6 +- .../graders/fuzzy_match.ts | 4 +- .../graders/header_confusion_metrics.ts | 2 +- .../graders/line_item_matching.ts | 2 +- .../graders/multi_field_fuzzy.ts | 2 +- .../scripts/aggregate_metrics.ts | 2 +- examples/features/eval-assert-demo/README.md | 12 +- .../eval-assert-demo/evals/suite.yaml | 6 +- examples/features/execution-metrics/README.md | 2 +- .../execution-metrics/evals/suite.yaml | 4 +- .../scripts/check-efficiency.ts | 6 +- .../scripts/check-metrics-present.ts | 6 +- .../scripts/check-file-changes.ts | 2 +- .../functional-grading/evals/suite.yaml | 2 +- .../scripts/functional-check.ts | 2 +- examples/features/import-claude/README.md | 8 +- .../graders/transcript-quality.ts | 4 +- examples/features/nlp-metrics/README.md | 6 +- .../features/nlp-metrics/evals/suite.yaml | 4 +- examples/features/nlp-metrics/graders/bleu.ts | 6 +- .../nlp-metrics/graders/levenshtein.ts | 6 +- .../features/nlp-metrics/graders/rouge.ts | 6 +- .../nlp-metrics/graders/similarity.ts | 6 +- .../features/prompt-template-sdk/README.md | 2 +- .../prompt-template-sdk/evals/suite.yaml | 4 +- examples/features/rubric/evals/suite.yaml | 2 +- .../.agentv/targets.yaml | 0 .../CHANGELOG.md | 2 +- .../README.md | 22 +- .../bun.lock | 2 +- .../script-grader-sdk/evals/example.txt | 1 + .../evals/python.instructions.md | 0 .../evals/suite.baseline.jsonl | 0 .../evals/suite.yaml | 8 +- .../package.json | 2 +- .../scripts/verify-attachments.ts | 8 +- .../README.md | 16 +- .../bun.lock | 2 +- .../contextual-precision.eval.baseline.jsonl | 0 .../evals/contextual-precision.eval.yaml | 0 .../contextual-recall.eval.baseline.jsonl | 0 .../evals/contextual-recall.eval.yaml | 0 .../package.json | 2 +- .../scripts/contextual-precision.ts | 4 +- .../scripts/contextual-recall.ts | 4 +- .../scripts/utils.ts | 0 examples/features/sdk-python/README.md | 6 +- .../scripts/check_expected_output.py | 8 +- .../sdk-python/src/agentv_py/__init__.py | 27 +- .../sdk-python/src/agentv_py/grader.py | 31 +- .../features/sdk-python/tests/test_grader.py | 12 +- .../tool-evaluation-plugins/README.md | 2 +- .../tool-evaluation-plugins/evals/suite.yaml | 2 +- .../graders/tool-args-f1.ts | 8 +- .../graders/tool-call-f1.ts | 8 +- examples/features/trace-evaluation/README.md | 4 +- .../trace-evaluation/evals/suite.yaml | 4 +- .../trace-evaluation/graders/error-spans.ts | 4 +- .../trace-evaluation/graders/span-count.ts | 4 +- .../trace-evaluation/graders/span-duration.ts | 4 +- .../graders/trial-consistency.ts | 4 +- .../vitest-workspace-grader/README.md | 2 +- .../workspace-artifact/evals/suite.yaml | 2 +- .../scripts/check-csv-artifact.ts | 2 +- .../archetypes/coding-agent/README.md | 2 +- examples/showcase/README.md | 2 +- examples/showcase/bug-fix-benchmark/README.md | 2 +- examples/showcase/cross-repo-sync/README.md | 2 +- .../showcase/cross-repo-sync/evals/suite.yaml | 2 +- .../cross-repo-sync/scripts/validate-sync.ts | 8 +- .../evals/validate_output.py | 2 +- examples/showcase/export-screening/README.md | 4 +- .../evals/validate_risk_output.ts | 4 +- .../grader-conformance/conformance-check.ts | 8 +- .../graders/keyword-grader.ts | 4 +- .../showcase/multi-model-benchmark/README.md | 2 +- .../psychotherapy/evals/validate_output.py | 2 +- .../tool-evaluation-plugins/README.md | 6 +- .../scripts/efficiency-scorer.ts | 6 +- .../scripts/pairwise-tool-compare.ts | 6 +- .../scripts/tool-selection-grader.ts | 6 +- .../tool-eval-demo.eval.yaml | 2 +- examples/showcase/trace-evaluation/README.md | 2 +- .../graders/recovery-check.ts | 4 +- .../trace-evaluation/graders/replay-proof.ts | 120 ++--- .../trace-evaluation/scripts/prove-replay.ts | 2 +- .../src/evaluation/graders/code-grader.ts | 442 +----------------- .../core/src/evaluation/graders/composite.ts | 2 +- packages/core/src/evaluation/graders/index.ts | 4 +- .../evaluation/graders/prompt-resolution.ts | 2 +- .../src/evaluation/graders/script-grader.ts | 431 +++++++++++++++++ .../src/evaluation/loaders/grader-parser.ts | 2 +- .../registry/assertion-discovery.ts | 4 +- .../evaluation/registry/builtin-graders.ts | 11 +- .../evaluation/registry/grader-discovery.ts | 4 +- .../evaluation/validation/eval-file.schema.ts | 4 +- .../evaluation/validation/eval-validator.ts | 2 +- packages/core/src/evaluation/yaml-parser.ts | 2 +- .../test/evaluation/execution-metrics.test.ts | 10 +- packages/core/test/evaluation/graders.test.ts | 26 +- ...st.ts => script-grader-plain-text.test.ts} | 8 +- .../evaluation/loaders/grader-parser.test.ts | 22 +- .../core/test/evaluation/orchestrator.test.ts | 2 +- ...t.ts => script-grader-file-backed.test.ts} | 14 +- ...st.ts => script-grader-multimodal.test.ts} | 14 +- .../core/test/fixtures/test-define-grader.ts | 2 +- packages/sdk/README.md | 22 +- packages/sdk/src/assertion.ts | 24 +- packages/sdk/src/deprecation.ts | 12 +- packages/sdk/src/graders.ts | 19 +- packages/sdk/src/index.ts | 45 +- packages/sdk/src/prompt-template.ts | 8 +- packages/sdk/src/runtime.ts | 38 +- packages/sdk/src/schemas.ts | 44 +- packages/sdk/src/target-client.ts | 8 +- packages/sdk/src/vitest.ts | 31 +- packages/sdk/src/workspace.ts | 34 +- ...r.test.ts => define-script-grader.test.ts} | 48 +- packages/sdk/test/deprecation.test.ts | 6 +- packages/sdk/test/file-backed-output.test.ts | 16 +- packages/sdk/test/grader-helpers.test.ts | 3 +- .../sdk/test/vitest-workspace-grader.test.ts | 8 +- packages/sdk/test/workspace-grader.test.ts | 6 +- skills-data/agentv-bench/SKILL.md | 14 +- skills-data/agentv-bench/agents/analyzer.md | 2 +- skills-data/agentv-bench/agents/comparator.md | 7 +- skills-data/agentv-bench/agents/grader.md | 4 +- .../references/description-optimization.md | 4 +- .../references/environment-adaptation.md | 18 +- .../agentv-bench/references/eval-yaml-spec.md | 24 +- .../migrating-from-skill-creator.md | 8 +- .../references/subagent-pipeline.md | 12 +- skills-data/agentv-eval-review/SKILL.md | 4 +- .../agentv-eval-review/scripts/lint_eval.py | 2 +- skills-data/agentv-eval-writer/SKILL.md | 8 +- .../references/custom-evaluators.md | 14 +- .../references/python-helpers.md | 10 +- 182 files changed, 1239 insertions(+), 1154 deletions(-) rename apps/web/src/content/docs/docs/next/graders/{code-graders.mdx => script-graders.mdx} (96%) rename examples/contract/evals/{code-grader-contract.eval.yaml => script-grader-contract.eval.yaml} (58%) rename examples/contract/scripts/{check-code-grader-payload.ts => check-script-grader-payload.ts} (96%) delete mode 100644 examples/features/code-grader-sdk/evals/example.txt rename examples/features/{code-grader-sdk => script-grader-sdk}/.agentv/targets.yaml (100%) rename examples/features/{code-grader-sdk => script-grader-sdk}/CHANGELOG.md (92%) rename examples/features/{code-grader-sdk => script-grader-sdk}/README.md (59%) rename examples/features/{code-grader-sdk => script-grader-sdk}/bun.lock (89%) create mode 100644 examples/features/script-grader-sdk/evals/example.txt rename examples/features/{code-grader-sdk => script-grader-sdk}/evals/python.instructions.md (100%) rename examples/features/{code-grader-sdk => script-grader-sdk}/evals/suite.baseline.jsonl (100%) rename examples/features/{code-grader-sdk => script-grader-sdk}/evals/suite.yaml (77%) rename examples/features/{code-grader-sdk => script-grader-sdk}/package.json (71%) rename examples/features/{code-grader-sdk => script-grader-sdk}/scripts/verify-attachments.ts (85%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/README.md (91%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/bun.lock (89%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/evals/contextual-precision.eval.baseline.jsonl (100%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/evals/contextual-precision.eval.yaml (100%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/evals/contextual-recall.eval.baseline.jsonl (100%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/evals/contextual-recall.eval.yaml (100%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/package.json (69%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/scripts/contextual-precision.ts (97%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/scripts/contextual-recall.ts (98%) rename examples/features/{code-grader-with-llm-calls => script-grader-with-llm-calls}/scripts/utils.ts (100%) create mode 100644 packages/core/src/evaluation/graders/script-grader.ts rename packages/core/test/evaluation/graders/{code-grader-plain-text.test.ts => script-grader-plain-text.test.ts} (93%) rename packages/core/test/evaluation/{code-grader-file-backed.test.ts => script-grader-file-backed.test.ts} (92%) rename packages/core/test/evaluation/{code-grader-multimodal.test.ts => script-grader-multimodal.test.ts} (97%) rename packages/sdk/test/{define-code-grader.test.ts => define-script-grader.test.ts} (88%) diff --git a/AGENTS.md b/AGENTS.md index 6cb2ce7b6..517ca114c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ These baseline rules apply to every repo change. They summarize the most common ## Repo Map - `packages/core/`: evaluation engine, providers, grading, project registry, and the programmatic API. -- `packages/sdk/`: lightweight assertion SDK such as `defineAssertion` and `defineCodeGrader`. +- `packages/sdk/`: lightweight assertion SDK such as `defineAssertion` and `defineScriptGrader`. - `apps/cli/`: published CLI surface for `agentv`. - `apps/web/src/content/docs/`: public product and CLI docs on agentv.dev. - `examples/`: examples that double as reference material and integration coverage. diff --git a/apps/cli/src/commands/eval/commands/vitest.ts b/apps/cli/src/commands/eval/commands/vitest.ts index 878b019c1..d8b6a20fc 100644 --- a/apps/cli/src/commands/eval/commands/vitest.ts +++ b/apps/cli/src/commands/eval/commands/vitest.ts @@ -1,6 +1,6 @@ import { command, flag, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { runCodeGrader, runVitestWorkspaceGrader } from '@agentv/sdk'; +import { runScriptGrader, runVitestWorkspaceGrader } from '@agentv/sdk'; function parseCommand(value: string | undefined): readonly string[] | undefined { const trimmed = value?.trim(); @@ -42,7 +42,7 @@ export const evalVitestCommand = command({ }), }, handler: async ({ testFiles, cwd, vitestCommand, timeoutMs, inWorkspace, passWithNoTests }) => { - await runCodeGrader((input) => { + await runScriptGrader((input) => { if (testFiles.length === 0) { throw new Error('Provide at least one Vitest verifier file.'); } diff --git a/apps/cli/src/commands/pipeline/bench.ts b/apps/cli/src/commands/pipeline/bench.ts index 539e374fc..e12db1a69 100644 --- a/apps/cli/src/commands/pipeline/bench.ts +++ b/apps/cli/src/commands/pipeline/bench.ts @@ -2,7 +2,7 @@ * `agentv pipeline bench` — Merge script and LLM grader scores into final * benchmark artifacts. * - * Reads code_grader_results and llm_grader_results from disk per test. + * Reads script_grader_results and llm_grader_results from disk per test. * * Writes: * - /grading.json (per-test grading breakdown) @@ -65,7 +65,7 @@ export const evalBenchCommand = command({ const allAssertions: { text: string; passed: boolean; evidence: string }[] = []; // Collect script grader results - const codeResultsDir = join(testDir, 'code_grader_results'); + const codeResultsDir = join(testDir, 'script_grader_results'); try { const resultFiles = (await readdir(codeResultsDir)).filter((f) => f.endsWith('.json')); for (const file of resultFiles) { diff --git a/apps/cli/src/commands/pipeline/grade.ts b/apps/cli/src/commands/pipeline/grade.ts index 3b2dc4bb0..4e7961c75 100644 --- a/apps/cli/src/commands/pipeline/grade.ts +++ b/apps/cli/src/commands/pipeline/grade.ts @@ -2,15 +2,15 @@ * `agentv pipeline grade` — Run grader assertions against response.md files * in an export directory produced by `pipeline input`. * - * All grader configs live in code_graders/.json. Each config has a `type` + * All grader configs live in script_graders/.json. Each config has a `type` * field that determines how it's evaluated: * - `script` (or configs with a `command` field): executed as external scripts * - Built-in types (contains, regex, equals, etc.): evaluated in-process * - * Results are written to code_grader_results/.json for pipeline bench. + * Results are written to script_grader_results/.json for pipeline bench. * * Export directory additions: - * ///code_grader_results/.json + * ///script_grader_results/.json */ import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; @@ -66,7 +66,7 @@ export interface GraderTask { * external scripts, built-in types (contains, regex, etc.) are evaluated in-process. * Shared by `pipeline grade` and `pipeline run`. */ -export async function runCodeGraders( +export async function runScriptGraders( tasks: GraderTask[], concurrency: number, ): Promise<{ totalGraders: number; totalPassed: number }> { @@ -86,13 +86,13 @@ export async function runCodeGraders( const executeGrader = async (task: GraderTask) => { const { testDir, resultsDir, graderFile, responseText } = task; const graderConfig = JSON.parse( - await readFile(join(testDir, 'code_graders', graderFile), 'utf8'), + await readFile(join(testDir, 'script_graders', graderFile), 'utf8'), ); // Dispatch: configs with a `command` field are external scripts; // all others are built-in deterministic assertions evaluated in-process. if (graderConfig.command) { - await executeCodeGrader(graderConfig, task); + await executeScriptGrader(graderConfig, task); } else { await executeBuiltinGrader(graderConfig, responseText, resultsDir); } @@ -104,7 +104,7 @@ export async function runCodeGraders( }; /** Run an external script grader. */ - const executeCodeGrader = async (graderConfig: Record, task: GraderTask) => { + const executeScriptGrader = async (graderConfig: Record, task: GraderTask) => { const { testId, resultsDir, responseText, inputData } = task; const graderName = graderConfig.name as string; const graderType = typeof graderConfig.type === 'string' ? graderConfig.type : 'script'; @@ -299,12 +299,12 @@ export const evalGradeCommand = command({ for (const testId of testIds) { const subpath = safeSuiteName ? [safeSuiteName, testId] : [testId]; const testDir = join(exportDir, ...subpath); - const codeGradersDir = join(testDir, 'code_graders'); - const resultsDir = join(testDir, 'code_grader_results'); + const scriptGradersDir = join(testDir, 'script_graders'); + const resultsDir = join(testDir, 'script_grader_results'); let graderFiles: string[]; try { - graderFiles = (await readdir(codeGradersDir)).filter((f: string) => f.endsWith('.json')); + graderFiles = (await readdir(scriptGradersDir)).filter((f: string) => f.endsWith('.json')); } catch { continue; // No graders for this test } @@ -320,7 +320,7 @@ export const evalGradeCommand = command({ } } - const { totalGraders, totalPassed } = await runCodeGraders(tasks, maxWorkers); + const { totalGraders, totalPassed } = await runScriptGraders(tasks, maxWorkers); console.log(`Graded ${totalGraders} grader(s): ${totalPassed} passed`); }, }); diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index 61d121204..8652c4985 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -16,7 +16,7 @@ * ├── criteria.md * ├── expected_output.json (if present) * ├── llm_graders/.json - * └── code_graders/.json # script/deterministic grader configs + * └── script_graders/.json # script/deterministic grader configs */ import { readFile } from 'node:fs/promises'; import { mkdir, writeFile } from 'node:fs/promises'; @@ -234,7 +234,7 @@ export const evalInputCommand = command({ }); interface GraderCounts { - codeGraders: number; + scriptGraders: number; llmGraders: number; builtinAssertions: number; } @@ -244,21 +244,21 @@ async function writeGraderConfigs( assertions: readonly GraderConfig[], evalDir: string, ): Promise { - const counts: GraderCounts = { codeGraders: 0, llmGraders: 0, builtinAssertions: 0 }; - const codeGradersDir = join(testDir, 'code_graders'); + const counts: GraderCounts = { scriptGraders: 0, llmGraders: 0, builtinAssertions: 0 }; + const scriptGradersDir = join(testDir, 'script_graders'); const llmGradersDir = join(testDir, 'llm_graders'); - let hasCodeGraders = false; + let hasScriptGraders = false; let hasLlmGraders = false; for (const assertion of assertions) { if (assertion.type === 'script') { - if (!hasCodeGraders) { - await mkdir(codeGradersDir, { recursive: true }); - hasCodeGraders = true; + if (!hasScriptGraders) { + await mkdir(scriptGradersDir, { recursive: true }); + hasScriptGraders = true; } const config = assertion as ScriptGraderConfig; - await writeJson(join(codeGradersDir, `${config.name}.json`), { + await writeJson(join(scriptGradersDir, `${config.name}.json`), { name: config.name, type: 'script', command: config.command, @@ -305,12 +305,12 @@ async function writeGraderConfigs( config: {}, }); } else if (BUILTIN_ASSERTION_TYPES.has(assertion.type)) { - if (!hasCodeGraders) { - await mkdir(codeGradersDir, { recursive: true }); - hasCodeGraders = true; + if (!hasScriptGraders) { + await mkdir(scriptGradersDir, { recursive: true }); + hasScriptGraders = true; } const config = assertion as GraderConfig & { value?: unknown; flags?: string }; - await writeJson(join(codeGradersDir, `${config.name}.json`), { + await writeJson(join(scriptGradersDir, `${config.name}.json`), { name: config.name, type: config.type, value: config.value, diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 4f842e9f8..70678d7c0 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -25,7 +25,7 @@ import { buildDefaultRunDir } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; import { selectTarget } from '../eval/targets.js'; import type { GraderTask } from './grade.js'; -import { runCodeGraders } from './grade.js'; +import { runScriptGraders } from './grade.js'; /** * Convert a Message[] array to plain text. @@ -386,12 +386,12 @@ export const evalRunCommand = command({ for (const testId of testIds) { const subpath = safeSuiteName ? [safeSuiteName, testId] : [testId]; const testDir = join(outDir, ...subpath); - const codeGradersDir = join(testDir, 'code_graders'); - const resultsDir = join(testDir, 'code_grader_results'); + const scriptGradersDir = join(testDir, 'script_graders'); + const resultsDir = join(testDir, 'script_grader_results'); let graderFiles: string[]; try { - graderFiles = (await readdir(codeGradersDir)).filter((f) => f.endsWith('.json')); + graderFiles = (await readdir(scriptGradersDir)).filter((f) => f.endsWith('.json')); } catch { continue; } @@ -407,7 +407,7 @@ export const evalRunCommand = command({ } const graderConcurrency = workers ?? 10; - const { totalGraders, totalPassed } = await runCodeGraders(graderTasks, graderConcurrency); + const { totalGraders, totalPassed } = await runScriptGraders(graderTasks, graderConcurrency); console.log(`Graded ${totalGraders} script grader(s): ${totalPassed} passed`); console.log(''); console.log(`Results in ${outDir}`); @@ -432,20 +432,20 @@ async function writeGraderConfigs( assertions: readonly GraderConfig[], evalDir: string, ): Promise { - const codeGradersDir = join(testDir, 'code_graders'); + const scriptGradersDir = join(testDir, 'script_graders'); const llmGradersDir = join(testDir, 'llm_graders'); - let hasCodeGraders = false; + let hasScriptGraders = false; let hasLlmGraders = false; for (const assertion of assertions) { if (assertion.type === 'script') { - if (!hasCodeGraders) { - await mkdir(codeGradersDir, { recursive: true }); - hasCodeGraders = true; + if (!hasScriptGraders) { + await mkdir(scriptGradersDir, { recursive: true }); + hasScriptGraders = true; } const config = assertion as ScriptGraderConfig; - await writeJson(join(codeGradersDir, `${config.name}.json`), { + await writeJson(join(scriptGradersDir, `${config.name}.json`), { name: config.name, type: 'script', command: config.command, diff --git a/apps/cli/test/commands/eval/assert.test.ts b/apps/cli/test/commands/eval/assert.test.ts index 0302a958a..21b119163 100644 --- a/apps/cli/test/commands/eval/assert.test.ts +++ b/apps/cli/test/commands/eval/assert.test.ts @@ -156,7 +156,7 @@ describe('agentv eval assert', () => { } }, 30_000); - it('sends only canonical wire fields to code graders', async () => { + it('sends only canonical wire fields to script graders', async () => { const { baseDir } = await createGraderFixture(); try { const result = await execa( diff --git a/apps/cli/test/commands/eval/pipeline/bench.test.ts b/apps/cli/test/commands/eval/pipeline/bench.test.ts index 0bd1bc193..766ad20ed 100644 --- a/apps/cli/test/commands/eval/pipeline/bench.test.ts +++ b/apps/cli/test/commands/eval/pipeline/bench.test.ts @@ -8,14 +8,14 @@ const CLI_ENTRY = join(import.meta.dirname, '../../../../src/cli.ts'); describe('pipeline bench', () => { beforeEach(async () => { const testDir = join(OUT_DIR, 'test-01'); - const codeResultsDir = join(testDir, 'code_grader_results'); + const codeResultsDir = join(testDir, 'script_grader_results'); const llmGradersDir = join(testDir, 'llm_graders'); const llmResultsDir = join(testDir, 'llm_grader_results'); - const codeGradersDir = join(testDir, 'code_graders'); + const scriptGradersDir = join(testDir, 'script_graders'); await mkdir(codeResultsDir, { recursive: true }); await mkdir(llmGradersDir, { recursive: true }); await mkdir(llmResultsDir, { recursive: true }); - await mkdir(codeGradersDir, { recursive: true }); + await mkdir(scriptGradersDir, { recursive: true }); await writeFile( join(OUT_DIR, 'manifest.json'), @@ -46,7 +46,7 @@ describe('pipeline bench', () => { }), ); await writeFile( - join(codeGradersDir, 'contains.json'), + join(scriptGradersDir, 'contains.json'), JSON.stringify({ name: 'contains', command: ['echo'], diff --git a/apps/cli/test/commands/eval/pipeline/grade.test.ts b/apps/cli/test/commands/eval/pipeline/grade.test.ts index cf9abac39..8cab7a3b4 100644 --- a/apps/cli/test/commands/eval/pipeline/grade.test.ts +++ b/apps/cli/test/commands/eval/pipeline/grade.test.ts @@ -8,8 +8,8 @@ const CLI_ENTRY = join(import.meta.dirname, '../../../../src/cli.ts'); describe('pipeline grade', () => { beforeEach(async () => { const testDir = join(OUT_DIR, 'test-01'); - const codeGradersDir = join(testDir, 'code_graders'); - await mkdir(codeGradersDir, { recursive: true }); + const scriptGradersDir = join(testDir, 'script_graders'); + await mkdir(scriptGradersDir, { recursive: true }); await writeFile(join(testDir, 'response.md'), 'hello world'); await writeFile( @@ -20,7 +20,7 @@ describe('pipeline grade', () => { }), ); await writeFile( - join(codeGradersDir, 'always_pass.json'), + join(scriptGradersDir, 'always_pass.json'), JSON.stringify({ name: 'always_pass', command: [ @@ -46,12 +46,12 @@ describe('pipeline grade', () => { await rm(OUT_DIR, { recursive: true, force: true }); }); - it('writes code_grader_results/.json with score and assertions', async () => { + it('writes script_grader_results/.json with score and assertions', async () => { const { execa } = await import('execa'); await execa('bun', [CLI_ENTRY, 'pipeline', 'grade', OUT_DIR]); const result = JSON.parse( - await readFile(join(OUT_DIR, 'test-01', 'code_grader_results', 'always_pass.json'), 'utf8'), + await readFile(join(OUT_DIR, 'test-01', 'script_grader_results', 'always_pass.json'), 'utf8'), ); expect(result.score).toBe(1); expect(result.name).toBe('always_pass'); @@ -65,7 +65,7 @@ describe('pipeline grade — builtin assertions', () => { beforeEach(async () => { const testDir = join(BUILTIN_OUT, 'test-01'); - const builtinGradersDir = join(testDir, 'code_graders'); + const builtinGradersDir = join(testDir, 'script_graders'); await mkdir(builtinGradersDir, { recursive: true }); await writeFile(join(testDir, 'response.md'), 'hello world'); @@ -127,7 +127,10 @@ describe('pipeline grade — builtin assertions', () => { await execa('bun', [CLI_ENTRY, 'pipeline', 'grade', BUILTIN_OUT]); const containsResult = JSON.parse( - await readFile(join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_hello.json'), 'utf8'), + await readFile( + join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_hello.json'), + 'utf8', + ), ); expect(containsResult.score).toBe(1); expect(containsResult.type).toBe('contains'); @@ -135,7 +138,7 @@ describe('pipeline grade — builtin assertions', () => { const regexResult = JSON.parse( await readFile( - join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'matches_pattern.json'), + join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'matches_pattern.json'), 'utf8', ), ); @@ -144,7 +147,7 @@ describe('pipeline grade — builtin assertions', () => { const failingContainsResult = JSON.parse( await readFile( - join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_goodbye.json'), + join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_goodbye.json'), 'utf8', ), ); @@ -154,7 +157,7 @@ describe('pipeline grade — builtin assertions', () => { it('applies negate to invert score', async () => { await writeFile( - join(BUILTIN_OUT, 'test-01', 'code_graders', 'has_goodbye.json'), + join(BUILTIN_OUT, 'test-01', 'script_graders', 'has_goodbye.json'), JSON.stringify({ name: 'has_goodbye', type: 'contains', @@ -169,7 +172,7 @@ describe('pipeline grade — builtin assertions', () => { const result = JSON.parse( await readFile( - join(BUILTIN_OUT, 'test-01', 'code_grader_results', 'has_goodbye.json'), + join(BUILTIN_OUT, 'test-01', 'script_grader_results', 'has_goodbye.json'), 'utf8', ), ); diff --git a/apps/cli/test/commands/eval/pipeline/input.test.ts b/apps/cli/test/commands/eval/pipeline/input.test.ts index f994207c6..fbea67255 100644 --- a/apps/cli/test/commands/eval/pipeline/input.test.ts +++ b/apps/cli/test/commands/eval/pipeline/input.test.ts @@ -27,14 +27,14 @@ describe('pipeline input', () => { expect(input.input).toHaveLength(1); expect(input.input[0].content).toBe('hello world'); - const codeGrader = JSON.parse( + const scriptGrader = JSON.parse( await readFile( - join(OUT_DIR, 'input-test', 'test-01', 'code_graders', 'contains_hello.json'), + join(OUT_DIR, 'input-test', 'test-01', 'script_graders', 'contains_hello.json'), 'utf8', ), ); - expect(codeGrader.command).toBeDefined(); - expect(codeGrader.name).toBe('contains_hello'); + expect(scriptGrader.command).toBeDefined(); + expect(scriptGrader.name).toBe('contains_hello'); const llmGrader = JSON.parse( await readFile( @@ -71,14 +71,14 @@ describe('pipeline input', () => { expect(manifest.experiment).toBe('without_skills'); }, 30_000); - it('writes code_graders/.json for deterministic assertions', async () => { + it('writes script_graders/.json for deterministic assertions', async () => { const { execa } = await import('execa'); const builtinEvalPath = join(FIXTURE_DIR, 'builtin-test.eval.yaml'); await execa('bun', [CLI_ENTRY, 'pipeline', 'input', builtinEvalPath, '--out', OUT_DIR]); const containsGrader = JSON.parse( await readFile( - join(OUT_DIR, 'builtin-test', 'test-01', 'code_graders', 'has_hello.json'), + join(OUT_DIR, 'builtin-test', 'test-01', 'script_graders', 'has_hello.json'), 'utf8', ), ); @@ -88,7 +88,7 @@ describe('pipeline input', () => { const regexGrader = JSON.parse( await readFile( - join(OUT_DIR, 'builtin-test', 'test-01', 'code_graders', 'matches_pattern.json'), + join(OUT_DIR, 'builtin-test', 'test-01', 'script_graders', 'matches_pattern.json'), 'utf8', ), ); diff --git a/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts b/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts index fef9a62cb..2d013b2b9 100644 --- a/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts +++ b/apps/cli/test/commands/eval/pipeline/pipeline-e2e.test.ts @@ -36,7 +36,7 @@ describe('eval pipeline e2e', () => { await execa('bun', [CLI_ENTRY, 'pipeline', 'grade', outDir]); const gradeResult = JSON.parse( await readFile( - join(outDir, 'input-test', 'test-01', 'code_grader_results', 'contains_hello.json'), + join(outDir, 'input-test', 'test-01', 'script_grader_results', 'contains_hello.json'), 'utf8', ), ); diff --git a/apps/cli/test/commands/eval/vitest.test.ts b/apps/cli/test/commands/eval/vitest.test.ts index f43368c8f..c5a7fa939 100644 --- a/apps/cli/test/commands/eval/vitest.test.ts +++ b/apps/cli/test/commands/eval/vitest.test.ts @@ -67,7 +67,7 @@ describe('agentv eval vitest', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('runs external verifier files through the code-grader protocol', async () => { + it('runs external verifier files through the script-grader protocol', async () => { const workspacePath = path.join(tempDir, 'workspace'); const gradersPath = path.join(tempDir, 'graders'); const fakeVitest = path.join(tempDir, 'fake-vitest.ts'); diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 56500e871..9b99ba303 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -15,7 +15,7 @@ AgentV currently provides two npm packages for programmatic use: ## Installation ```bash -# User-facing SDK (evaluate, defineEval, graders, defineAssertion, defineCodeGrader) +# User-facing SDK (evaluate, defineEval, graders, defineAssertion, defineScriptGrader) npm install @agentv/sdk # Core configuration helpers (defineConfig) @@ -32,7 +32,7 @@ npm install @agentv/sdk ``` ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; ``` The general policy is hard convergence for same-week or unreleased surface names: use the correct package, field, or wire name instead of carrying aliases. `@agentv/eval` was already published, then deprecated on npm, and has been removed from this repository. New docs, examples, scaffolds, and skills should use `@agentv/sdk` directly. @@ -45,7 +45,7 @@ Use the simplest surface that matches the job: - **`defineEval()` / `evalSuite()`** when you want a `.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput`. -- **`defineAssertion` / `defineCodeGrader`** when the grading logic itself must execute code. +- **`defineAssertion` / `defineScriptGrader`** when the grading logic itself must execute code. - **`agentv eval `** for deterministic workspace checks that fit normal Vitest `expect(...)` tests. There is no separate first-party Python authoring SDK today. Python-facing workflows should either emit canonical YAML/JSONL or implement executable graders that consume the standard `snake_case` wire format. @@ -146,7 +146,7 @@ export default defineEval({ prompt: 'Grade whether the answer is useful.', target: 'grader-target', }), - graders.codeGrader(['bun', 'run', 'graders/check.ts'], { name: 'scripted-check' }), + graders.scriptGrader(['bun', 'run', 'graders/check.ts'], { name: 'scripted-check' }), ], }, ], @@ -264,12 +264,12 @@ assert: ## Script Graders -Use `defineCodeGrader` from `@agentv/sdk` for full control over scoring with an explicit assertions array: +Use `defineScriptGrader` from `@agentv/sdk` for full control over scoring with an explicit assertions array: ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, traceSummary }) => ({ +export default defineScriptGrader(({ output, traceSummary }) => ({ score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, assert: [ { text: 'Answer is not empty', passed: (output ?? '').length > 0 }, @@ -311,9 +311,9 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ ]); ``` -`defineCodeGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. +`defineScriptGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. -For detailed patterns, input/output contracts, and language-agnostic examples, see [Script Graders](/docs/graders/code-graders/). +For detailed patterns, input/output contracts, and language-agnostic examples, see [Script Graders](/docs/graders/script-graders/). ## Wire Format vs SDK Format diff --git a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx index f9cc6748d..d6d753009 100644 --- a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx @@ -14,11 +14,11 @@ AgentV provides two SDK functions for custom evaluation logic: | Function | Best For | Discovery | |----------|----------|-----------| | `defineAssertion()` | Pass/fail checks, reusable assertion types | Convention-based (`.agentv/assertions/`) | -| `defineCodeGrader()` | Full scoring control with explicit assertions array | Referenced via `type: script` + `command:` | +| `defineScriptGrader()` | Full scoring control with explicit assertions array | Referenced via `type: script` + `command:` | **Use `defineAssertion()`** when you want a named assertion type that can be referenced across eval files without specifying a command path. It uses a simplified result contract focused on `pass` and optional `score`. -**Use `defineCodeGrader()`** when you need full control over scoring with explicit `assertions` arrays, or when the grader is a one-off grader tied to a specific eval. See [Script Graders](/docs/graders/code-graders/) for details. +**Use `defineScriptGrader()`** when you need full control over scoring with explicit `assertions` arrays, or when the grader is a one-off grader tied to a specific eval. See [Script Graders](/docs/graders/script-graders/) for details. Both functions handle stdin/stdout JSON parsing, snake_case-to-camelCase conversion, Zod validation, and error handling automatically. diff --git a/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx b/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx index 604a0106c..37f102f47 100644 --- a/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx +++ b/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx @@ -35,14 +35,14 @@ Use canonical fields instead: ## Example ```python -from agentv_py.grader import Assertion, CodeGraderResult, define_script +from agentv_py.grader import Assertion, ScriptGraderResult, define_script def evaluate(context): actual = context.output or "" expected = context.expected_output[0]["content"] passed = actual.strip() == expected.strip() - return CodeGraderResult( + return ScriptGraderResult( score=1.0 if passed else 0.0, assertions=[ Assertion( diff --git a/apps/web/src/content/docs/docs/next/graders/code-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx similarity index 96% rename from apps/web/src/content/docs/docs/next/graders/code-graders.mdx rename to apps/web/src/content/docs/docs/next/graders/script-graders.mdx index 64ecf46c0..81a78bc27 100644 --- a/apps/web/src/content/docs/docs/next/graders/code-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -128,13 +128,13 @@ print(json.dumps({ The repo-local helper in `examples/features/sdk-python/` wraps the same contract for that example checkout: ```python -from agentv_py.grader import Assertion, CodeGraderResult, define_script +from agentv_py.grader import Assertion, ScriptGraderResult, define_script def evaluate(context): candidate = context.output or "" passed = "42" in candidate - return CodeGraderResult( + return ScriptGraderResult( score=1.0 if passed else 0.0, assertions=[ Assertion( @@ -186,13 +186,13 @@ assert: ## TypeScript SDK -The `@agentv/sdk` package provides a declarative API with automatic stdin/stdout handling. Use `defineCodeGrader` to skip protocol boilerplate: +The `@agentv/sdk` package provides a declarative API with automatic stdin/stdout handling. Use `defineScriptGrader` to skip protocol boilerplate: ```typescript #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, criteria }) => { +export default defineScriptGrader(({ output, criteria }) => { const outputText = output ?? ''; const assertions: Array<{ text: string; passed: boolean }> = []; @@ -266,7 +266,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ Prefer Vitest verifiers when the checks naturally fit `expect(...)`. Use `defineWorkspaceGrader` when you need a very small custom script, custom weighting, or details that do not map cleanly to individual test outcomes. -**SDK exports:** `defineCodeGrader`, `defineVitestWorkspaceGrader`, `defineWorkspaceGrader`, `Message`, `ToolCall`, `Trace`, `TraceSummary`, `CodeGraderInput`, `CodeGraderResult`, `Workspace`, `WorkspaceAssertion` +**SDK exports:** `defineScriptGrader`, `defineVitestWorkspaceGrader`, `defineWorkspaceGrader`, `Message`, `ToolCall`, `Trace`, `TraceSummary`, `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` ## Target Access @@ -291,9 +291,9 @@ Use `createTargetClient` from the SDK: ```typescript #!/usr/bin/env bun -import { createTargetClient, defineCodeGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(async ({ input, output }) => { +export default defineScriptGrader(async ({ input, output }) => { const inputText = input .filter((message) => message.role === 'user') .map((message) => typeof message.content === 'string' ? message.content : '') diff --git a/apps/web/src/content/docs/docs/next/index.mdx b/apps/web/src/content/docs/docs/next/index.mdx index b7a7fb680..46a62ed46 100644 --- a/apps/web/src/content/docs/docs/next/index.mdx +++ b/apps/web/src/content/docs/docs/next/index.mdx @@ -49,8 +49,7 @@ Use this topic map when you are an AI agent trying to decide which primitive or | Goal | Start here | Why | | --- | --- | --- | | Create a first eval | [Quickstart](/docs/getting-started/quickstart/) → [Eval files](/docs/evaluation/eval-files/) | Defines the smallest runnable YAML shape before adding advanced fields. | -| Run or resume evals | [Running evals](/docs/evaluation/running-evals/) → [WIP checkpoints](/docs/tools/wip-checkpoints/) | Covers `agentv eval`, concurrency, `--resume`, `--rerun-failed`, and remote partial-run recovery. | -| Choose graders | [Rubrics](/docs/evaluation/rubrics/) → [Script graders](/docs/graders/code-graders/) → [LLM graders](/docs/graders/llm-rubrics/) | Keeps deterministic checks, rubric scoring, and LLM judgment separate. | +| Choose graders | [Rubrics](/docs/evaluation/rubrics/) → [Script graders](/docs/graders/script-graders/) → [LLM graders](/docs/graders/llm-rubrics/) | Keeps deterministic checks, rubric scoring, and LLM judgment separate. | | Evaluate tool use or agents | [Tool trajectory](/docs/graders/tool-trajectory/) → [Coding agents](/docs/targets/coding-agents/) → [CLI provider](/docs/targets/cli-provider/) | Shows how targets, transcripts, and tool-call assertions compose. | | Share and inspect results | [Result artifact contract](/docs/reference/result-artifacts/) → [Results](/docs/tools/results/) → [Dashboard](/docs/tools/dashboard/) | Explains canonical run bundles, local artifacts, reports, remote result repositories, and Dashboard review flows. | | Compare runs | [Compare](/docs/tools/compare/) → [Dashboard Analytics](/docs/tools/dashboard/#analytics) | Use CLI metrics for automation and Dashboard analytics for interactive inspection. | diff --git a/evals/agentic-engineering/agent-plugin-review.eval.yaml b/evals/agentic-engineering/agent-plugin-review.eval.yaml index 8df315947..7c535ff1d 100644 --- a/evals/agentic-engineering/agent-plugin-review.eval.yaml +++ b/evals/agentic-engineering/agent-plugin-review.eval.yaml @@ -16,7 +16,7 @@ tests: input: | Review the deploy-auto plugin in this repo for completeness. Check that every skill has a corresponding eval file. - assertions: + assert: - type: skill-trigger skill: agent-plugin-review - type: contains @@ -30,7 +30,7 @@ tests: criteria: Identifies eval files using bare .yaml instead of .eval.yaml input: | Review the eval files under evals/deploy-auto/ for naming convention issues. - assertions: + assert: - type: contains value: .eval.yaml - type: rubrics @@ -44,7 +44,7 @@ tests: input: | Review evals/deploy-auto/deploy-plan.yaml for eval quality issues. Check assertion coverage and expected_output format. - assertions: + assert: - type: rubrics criteria: - Flags that no assertions are defined in deploy-plan.yaml @@ -55,7 +55,7 @@ tests: criteria: Identifies eval file paths missing leading slash input: | Review evals/deploy-auto/deploy-plan.yaml for file path formatting issues. - assertions: + assert: - type: rubrics criteria: - Flags that file paths are missing a leading slash @@ -66,7 +66,7 @@ tests: input: | Review evals/deploy-auto/deploy-plan.yaml for structural improvements. Look at how inputs are organized across test cases. - assertions: + assert: - type: rubrics criteria: - Identifies the repeated SKILL.md file input across all 3 tests @@ -77,7 +77,7 @@ tests: input: | Review the deploy-auto plugin's workflow architecture. Check whether phases enforce prerequisites before proceeding. - assertions: + assert: - type: rubrics criteria: - Flags that deploy-execute does not check for deploy-plan.md before starting @@ -89,7 +89,7 @@ tests: input: | Review evals/deploy-auto/deploy-execute.eval.yaml for factual accuracy. Cross-check expected outputs against what the skills actually document. - assertions: + assert: - type: rubrics criteria: - Flags the contradiction between pytest (skill) and python -m unittest (eval) @@ -100,7 +100,7 @@ tests: input: | Review plugins/deploy-auto/skills/deploy-plan/SKILL.md for cross-reference issues. Check that referenced commands and skills actually exist. - assertions: + assert: - type: rubrics criteria: - Flags that /deploy-execute is referenced but does not exist as a slash command @@ -111,7 +111,7 @@ tests: criteria: Identifies hardcoded local paths in deploy-execute skill input: | Review plugins/deploy-auto/skills/deploy-execute/SKILL.md for portability issues. - assertions: + assert: - type: rubrics criteria: - Flags the hardcoded path C:\Users\admin\.kube\config diff --git a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml index 2e00f579e..5343c4606 100644 --- a/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml +++ b/evals/agentic-engineering/workspace-template/evals/deploy-auto/deploy-execute.eval.yaml @@ -10,7 +10,7 @@ tests: value: "/plugins/deploy-auto/skills/deploy-execute/SKILL.md" - type: text value: "Execute the deployment plan at ./output/deploy-plan.md" - assertions: + assert: - type: rubrics criteria: - Reads the deployment plan file @@ -26,7 +26,7 @@ tests: value: "/plugins/deploy-auto/skills/deploy-execute/SKILL.md" - type: text value: "The API service health check is failing after deployment. What should happen?" - assertions: + assert: - type: contains value: rollback - type: rubrics diff --git a/evals/agentv-dev/skills/README.md b/evals/agentv-dev/skills/README.md index 0abf2691e..dbbf80d00 100644 --- a/evals/agentv-dev/skills/README.md +++ b/evals/agentv-dev/skills/README.md @@ -76,7 +76,7 @@ Pattern for repo-driven file tests: - type: text value: | - assertions: + assert: - type: contains value: - type: rubrics diff --git a/evals/agentv-dev/skills/output-correctness.eval.yaml b/evals/agentv-dev/skills/output-correctness.eval.yaml index 054aa8646..f42c26118 100644 --- a/evals/agentv-dev/skills/output-correctness.eval.yaml +++ b/evals/agentv-dev/skills/output-correctness.eval.yaml @@ -23,7 +23,7 @@ tests: correctly summarises a paragraph. Output **only** the YAML — no surrounding prose or fences. - assertions: + assert: - type: contains value: "description:" - type: contains @@ -50,7 +50,7 @@ tests: single test entry under `tests:` that uses **three** assertions: one `contains`, one `regex`, and one `rubrics` (with at least two criteria). Output only the YAML for the test entry. - assertions: + assert: - type: regex value: "type:\\s+contains" - type: regex @@ -78,7 +78,7 @@ tests: 1. List all bundled skills as JSON. 2. Print the agentv-bench skill including its bundled support files. 3. Print the on-disk path of the agentv-bench skill. - assertions: + assert: - type: regex value: "agentv\\s+skills\\s+list\\s+--json" - type: regex @@ -99,7 +99,7 @@ tests: describe what this skill does and when it should be invoked. Quote at most one short phrase from the skill text; do not invent capabilities the skill does not mention. - assertions: + assert: - type: rubrics criteria: - Description accurately reflects what agentv-trace-analyst does (analysing eval traces and failure modes) @@ -119,7 +119,7 @@ tests: The file above is the live `agentv-dev` plugin entrypoint. Reproduce the list of available bundled skill names as a markdown bulleted list. - assertions: + assert: - type: contains value: agentv-bench - type: contains @@ -148,7 +148,7 @@ tests: single command should I run to load the `lint-rules` reference file without pulling in the whole skill? Reply with the command on its own line. - assertions: + assert: - type: regex value: "agentv\\s+skills\\s+get\\s+agentv-governance\\s+--ref\\s+lint-rules" @@ -166,7 +166,7 @@ tests: to grade an answer's correctness. Include whatever fields the skill says an `llm-grader` requires (for example target/model or rubric criteria). Output only the YAML for the single assertion. - assertions: + assert: - type: regex value: "type:\\s+llm-grader" - type: rubrics diff --git a/evals/agentv-dev/skills/skill-invocation.eval.yaml b/evals/agentv-dev/skills/skill-invocation.eval.yaml index 6fddee7b2..368afe842 100644 --- a/evals/agentv-dev/skills/skill-invocation.eval.yaml +++ b/evals/agentv-dev/skills/skill-invocation.eval.yaml @@ -27,7 +27,7 @@ tests: `agentv` CLI to print the skill's SKILL.md to stdout? Reply with the command on its own line. - assertions: + assert: - type: regex value: "agentv\\s+skills\\s+get\\s+agentv-bench" - type: rubrics @@ -42,7 +42,7 @@ tests: bundled supporting files too — so I can inspect its reference material. What single `agentv skills` command should I run? - assertions: + assert: - type: contains value: --full - type: regex @@ -53,7 +53,7 @@ tests: input: | I just installed `agentv` globally. How do I list every skill that ships bundled with the CLI? Give me the single command. - assertions: + assert: - type: regex value: "agentv\\s+skills\\s+list" - type: rubrics @@ -66,7 +66,7 @@ tests: input: | I ran `agentv skills get does-not-exist`. What happened, and what command should I run instead to discover the correct skill name? - assertions: + assert: - type: rubrics criteria: - Identifies that `does-not-exist` is not a valid skill name @@ -79,7 +79,7 @@ tests: I want to pipe the list of skill names into `jq` so I can build a shell-script loop over each name. Which `agentv skills` invocation gives me machine-readable output suitable for `jq`? - assertions: + assert: - type: contains value: --json - type: regex @@ -91,7 +91,7 @@ tests: Where on disk does the installed `agentv` CLI keep its bundled skill files? I want the absolute path so I can open one in my editor. Give me the single CLI command that prints that path. - assertions: + assert: - type: regex value: "agentv\\s+skills\\s+path" - type: rubrics @@ -105,7 +105,7 @@ tests: I want to dump the SKILL.md of every bundled agentv skill into a single stream so I can pass them all into another tool's context. What's the single `agentv skills get` command that does this? - assertions: + assert: - type: contains value: --all - type: regex @@ -116,7 +116,7 @@ tests: input: | I want to read the skill that helps with debugging eval failures and analysing traces. Give me the single CLI command to print its content. - assertions: + assert: - type: contains value: agentv-trace-analyst - type: regex diff --git a/evals/agentv-dev/skills/skill-selection.eval.yaml b/evals/agentv-dev/skills/skill-selection.eval.yaml index ec8754e72..39816527b 100644 --- a/evals/agentv-dev/skills/skill-selection.eval.yaml +++ b/evals/agentv-dev/skills/skill-selection.eval.yaml @@ -25,7 +25,7 @@ tests: input: | Task: I have a YAML eval file and I need to run it against an LLM target and see the pass rate. Which skill? - assertions: + assert: - type: contains value: agentv-bench - type: rubrics @@ -38,7 +38,7 @@ tests: input: | Task: I want to author a brand-new `.eval.yaml` file for my custom skill from scratch. Which skill should I load? - assertions: + assert: - type: contains value: agentv-eval-writer - type: rubrics @@ -52,7 +52,7 @@ tests: Task: I already wrote my `.eval.yaml` and want a second pair of eyes to audit it for missing assertions, bad rubric phrasing, and structural issues. Which skill? - assertions: + assert: - type: contains value: agentv-eval-review - type: rubrics @@ -65,7 +65,7 @@ tests: input: | Task: I'm building a red-team eval and need to add a `governance:` block with OWASP and MITRE coverage metadata. Which skill? - assertions: + assert: - type: contains value: agentv-governance - type: rubrics @@ -78,7 +78,7 @@ tests: input: | Task: I already wrote the `governance:` block in my `.eval.yaml` and want to lint it against the supported vocabulary before CI runs. Which skill? - assertions: + assert: - type: contains value: agentv-governance - type: rubrics @@ -92,7 +92,7 @@ tests: Task: My eval run produced an `index.jsonl` with several failing rows. I want to dig into the per-test traces and figure out what went wrong. Which skill? - assertions: + assert: - type: contains value: agentv-trace-analyst - type: rubrics @@ -105,7 +105,7 @@ tests: input: | Task: I'm starting a new eval project. I'll need to author the `.eval.yaml` from scratch *and* run it against a target. Which skill or skills? - assertions: + assert: - type: contains value: agentv-eval-writer - type: contains @@ -120,7 +120,7 @@ tests: input: | Task: I want to deploy my Node.js app to a Kubernetes cluster. Which agentv skill should I load? - assertions: + assert: - type: equals value: none - type: rubrics diff --git a/evals/agentv-self/graders/pr-workflow-safety.ts b/evals/agentv-self/graders/pr-workflow-safety.ts index 16027f4c7..0c12af86f 100644 --- a/evals/agentv-self/graders/pr-workflow-safety.ts +++ b/evals/agentv-self/graders/pr-workflow-safety.ts @@ -8,7 +8,7 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { type Message, type ToolCall, type TraceEvent, defineCodeGrader } from '@agentv/sdk'; +import { type Message, type ToolCall, type TraceEvent, defineScriptGrader } from '@agentv/sdk'; type Assertion = { text: string; passed: boolean; evidence?: string }; @@ -209,7 +209,7 @@ function fixtureManifestAssertion(workspacePath: string | null | undefined): Ass }; } -export default defineCodeGrader(({ output, messages, trace, workspacePath }) => { +export default defineScriptGrader(({ output, messages, trace, workspacePath }) => { const response = outputText(output, messages); const normalizedResponse = normalizeText(response); const lines = outputLines(response); diff --git a/evals/agentv-self/graders/required-file-reads.ts b/evals/agentv-self/graders/required-file-reads.ts index e1249d832..e0aaab51d 100644 --- a/evals/agentv-self/graders/required-file-reads.ts +++ b/evals/agentv-self/graders/required-file-reads.ts @@ -2,9 +2,9 @@ /** * Verifies that an agent inspected required repo guidance files from the * prepared workspace. This is intentionally suite-local: the general primitive - * is code-grader, while this file encodes AgentV self-eval expectations. + * is script-grader, while this file encodes AgentV self-eval expectations. */ -import { type Message, type ToolCall, type TraceEvent, defineCodeGrader } from '@agentv/sdk'; +import { type Message, type ToolCall, type TraceEvent, defineScriptGrader } from '@agentv/sdk'; type Assertion = { text: string; passed: boolean; evidence?: string }; @@ -141,7 +141,7 @@ function evidenceForRequiredFile(call: ToolCall, expectedFile: string): string | return undefined; } -export default defineCodeGrader(({ config, messages, trace }) => { +export default defineScriptGrader(({ config, messages, trace }) => { const requiredFiles = stringArray(config?.requiredFiles ?? config?.required_files); if (requiredFiles.length === 0) { diff --git a/examples/README.md b/examples/README.md index 3130aa8ab..938875479 100644 --- a/examples/README.md +++ b/examples/README.md @@ -41,7 +41,7 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [composite](features/composite/) - Composite grader patterns - [weighted-graders](features/weighted-graders/) - Weighted graders - [execution-metrics](features/execution-metrics/) - Metrics tracking (tokens, cost, latency) -- [code-grader-with-llm-calls](features/code-grader-with-llm-calls/) - Code graders with target proxy for LLM calls +- [script-grader-with-llm-calls](features/script-grader-with-llm-calls/) - script graders with target proxy for LLM calls - [batch-cli](features/batch-cli/) - Batch CLI evaluation - [document-extraction](features/document-extraction/) - Document data extraction - [local-cli](features/local-cli/) - Local CLI targets @@ -52,7 +52,7 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it ### SDK -- [code-grader-sdk](features/code-grader-sdk/) - TypeScript SDK for code graders using `defineCodeGrader()` +- [script-grader-sdk](features/script-grader-sdk/) - TypeScript SDK for script graders using `defineScriptGrader()` - [vitest-workspace-grader](features/vitest-workspace-grader/) - Built-in AgentV adapter for Vitest workspace verifier files - [sdk-custom-assertion](features/sdk-custom-assertion/) - Custom assertion types using `defineAssertion()` - [sdk-programmatic-api](features/sdk-programmatic-api/) - Programmatic evaluation using `evaluate()` @@ -81,7 +81,7 @@ Each example follows this structure: example-name/ ├── evals/ │ ├── suite.yaml # Primary eval file -│ ├── *.ts or *.py # Code graders (optional) +│ ├── *.ts or *.py # script graders (optional) │ └── *.md # LLM grader prompts (optional) ├── scripts/ # Helper scripts (optional) ├── .agentv/ @@ -92,7 +92,7 @@ example-name/ ### Using `@agentv/sdk` -For TypeScript code graders, add a `package.json`: +For TypeScript script graders, add a `package.json`: ```json { @@ -105,14 +105,14 @@ For TypeScript code graders, add a `package.json`: } ``` -Then write type-safe code graders: +Then write type-safe script graders: ```typescript #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output }) => ({ +export default defineScriptGrader(({ output }) => ({ score: (output ?? '').includes('expected') ? 1.0 : 0.0, - assertions: [{ text: 'Found expected content', passed: (output ?? '').includes('expected') }], + assert: [{ text: 'Found expected content', passed: (output ?? '').includes('expected') }], })); ``` diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/script-grader-contract.eval.yaml similarity index 58% rename from examples/contract/evals/code-grader-contract.eval.yaml rename to examples/contract/evals/script-grader-contract.eval.yaml index c576bf799..0c6a91599 100644 --- a/examples/contract/evals/code-grader-contract.eval.yaml +++ b/examples/contract/evals/script-grader-contract.eval.yaml @@ -1,17 +1,17 @@ -name: code-grader-contract -description: Release gate verifying the code-grader stdin payload contract. +name: script-grader-contract +description: Release gate verifying the script-grader stdin payload contract. target: github-models-contract tests: - - id: code-grader-stdin-payload + - id: script-grader-stdin-payload input: - role: system content: "Use the literal token RELEASE as the first word in your response." - role: user content: "Use the literal token CONTRACT as the second word in your response." assert: - - metric: code-grader-payload-contract + - metric: script-grader-payload-contract type: script - command: [ "bun", "../scripts/check-code-grader-payload.ts" ] + command: [ "bun", "../scripts/check-script-grader-payload.ts" ] - Response includes both literal tokens RELEASE and CONTRACT. diff --git a/examples/contract/fixtures/repo-materialization-note.txt b/examples/contract/fixtures/repo-materialization-note.txt index c98e26fd2..f1cf2da15 100644 --- a/examples/contract/fixtures/repo-materialization-note.txt +++ b/examples/contract/fixtures/repo-materialization-note.txt @@ -1,4 +1,4 @@ AGENTV_REPO_FILE_SUBSTITUTION_READY This file is injected through a type:file input so the release contract covers -prompt file substitution while the code-grader verifies workspace state. +prompt file substitution while the script-grader verifies workspace state. diff --git a/examples/contract/scripts/check-repo-materialization.ts b/examples/contract/scripts/check-repo-materialization.ts index 9841c639f..be2f69e52 100644 --- a/examples/contract/scripts/check-repo-materialization.ts +++ b/examples/contract/scripts/check-repo-materialization.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Deterministic code grader for the repo materialization release contract. + * Deterministic script grader for the repo materialization release contract. * * It verifies that AgentV cloned the owned public fixture repo, checked out a * specific historical commit, and resolved a type:file input into the prompt diff --git a/examples/contract/scripts/check-code-grader-payload.ts b/examples/contract/scripts/check-script-grader-payload.ts similarity index 96% rename from examples/contract/scripts/check-code-grader-payload.ts rename to examples/contract/scripts/check-script-grader-payload.ts index 9be6d6a6a..7860c064e 100644 --- a/examples/contract/scripts/check-code-grader-payload.ts +++ b/examples/contract/scripts/check-script-grader-payload.ts @@ -1,8 +1,8 @@ #!/usr/bin/env bun /** - * Deterministic code grader for the code-grader stdin payload contract. + * Deterministic script grader for the script-grader stdin payload contract. * - * It verifies that AgentV sends the canonical wire payload to code graders: + * It verifies that AgentV sends the canonical wire payload to script graders: * final answer text in `output`, full transcript in `messages`, test context in * `input`/`criteria`, and no deprecated `answer` alias. */ diff --git a/examples/contract/scripts/check-workspace.ts b/examples/contract/scripts/check-workspace.ts index 0e59765f5..ad113b6c2 100644 --- a/examples/contract/scripts/check-workspace.ts +++ b/examples/contract/scripts/check-workspace.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Deterministic code grader for the release contract eval. + * Deterministic script grader for the release contract eval. * * It verifies that AgentV created a workspace from the local template and * exposed the workspace path to graders. This keeps the release gate public diff --git a/examples/features/README.md b/examples/features/README.md index dac7e8b53..d01a7e2eb 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -17,7 +17,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the ### Grade response quality with an LLM judge | Example | Description | |---------|-------------| -| [rubric](rubric/) | Boolean rubric criteria — pass/fail each with a code grader or LLM check | +| [rubric](rubric/) | Boolean rubric criteria — pass/fail each with a script grader or LLM check | | [weighted-graders](weighted-graders/) | Multiple named `llm-rubric` assertions with per-grader weights | | [composite](composite/) | Safety gate and weighted aggregation patterns | | [threshold-grader](threshold-grader/) | Pass a test if a configurable percentage of sub-graders pass | @@ -32,16 +32,16 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [assert](assert/) | Core built-ins: `contains`, `regex`, `is-json`, `equals`, `starts-with`, `ends-with` | | [assert-extended](assert-extended/) | Extended variants: `contains-any`, `icontains`, `icontains-all`, regex flags | | [deterministic-graders](deterministic-graders/) | Full showcase of all deterministic assertion types | -| [nlp-metrics](nlp-metrics/) | ROUGE, BLEU, cosine/Jaccard similarity, Levenshtein as code graders | +| [nlp-metrics](nlp-metrics/) | ROUGE, BLEU, cosine/Jaccard similarity, Levenshtein as script graders | --- ### Write custom graders in code | Example | Description | |---------|-------------| -| [code-grader-sdk](code-grader-sdk/) | TypeScript code graders using `defineCodeGrader()` from `@agentv/sdk` | -| [code-grader-with-llm-calls](code-grader-with-llm-calls/) | Code graders that make LLM calls via a target proxy | -| [eval-assert-demo](eval-assert-demo/) | Code graders runnable both in a suite and individually via `agentv eval assert` | +| [script-grader-sdk](script-grader-sdk/) | TypeScript script graders using `defineScriptGrader()` from `@agentv/sdk` | +| [script-grader-with-llm-calls](script-grader-with-llm-calls/) | script graders that make LLM calls via a target proxy | +| [eval-assert-demo](eval-assert-demo/) | script graders runnable both in a suite and individually via `agentv eval assert` | | [functional-grading](functional-grading/) | Install dependencies, compile, and run tests against agent-generated code | --- @@ -140,8 +140,8 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [basic-jsonl](basic-jsonl/) | Getting started | | [batch-cli](batch-cli/) | Workspace & targets | | [benchmark-tooling](benchmark-tooling/) | Benchmarking | -| [code-grader-sdk](code-grader-sdk/) | Custom graders | -| [code-grader-with-llm-calls](code-grader-with-llm-calls/) | Custom graders | +| [script-grader-sdk](script-grader-sdk/) | Custom graders | +| [script-grader-with-llm-calls](script-grader-with-llm-calls/) | Custom graders | | [compare](compare/) | Benchmarking | | [composite](composite/) | LLM grading | | [copilot-log-eval](copilot-log-eval/) | Offline evaluation | diff --git a/examples/features/batch-cli/README.md b/examples/features/batch-cli/README.md index 7cf28c019..2773e5419 100644 --- a/examples/features/batch-cli/README.md +++ b/examples/features/batch-cli/README.md @@ -10,7 +10,7 @@ This example demonstrates an **external batch runner** pattern for a (synthetic) 3. **Batch processing**: `batch-cli-runner.ts` reads the CSV and applies synthetic AML screening rules, writing **actual responses** as JSONL to a temporary file. Each JSONL record includes `output` with `tool_calls` for trace extraction. -4. **Evaluation**: AgentV compares the actual JSONL output against the ground truth in `evals/suite.yaml` using graders like `code_grader` and `tool_trajectory`. +4. **Evaluation**: AgentV compares the actual JSONL output against the ground truth in `evals/suite.yaml` using graders like `script` and `tool-trajectory`. ## Batch error handling (missing JSONL id) @@ -64,4 +64,4 @@ cd examples/features/batch-cli # Run AgentV against the batch CLI target # NOTE: This requires the CLI provider to support batching + JSONL batch output. bun agentv eval ./evals/suite.yaml --target batch_cli -``` \ No newline at end of file +``` diff --git a/examples/features/batch-cli/graders/check-batch-cli-output.ts b/examples/features/batch-cli/graders/check-batch-cli-output.ts index 1c5e85d46..2370d58e3 100644 --- a/examples/features/batch-cli/graders/check-batch-cli-output.ts +++ b/examples/features/batch-cli/graders/check-batch-cli-output.ts @@ -1,11 +1,11 @@ #!/usr/bin/env bun /** - * Batch CLI Output Grader - Code Grader + * Batch CLI Output Grader - script grader * * Validates that the batch CLI runner produces the expected decision * by comparing candidate output against expected_output or input. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -66,7 +66,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ expectedOutput, input, output }) => { +export default defineScriptGrader(({ expectedOutput, input, output }) => { const outputText = getMessageText(output ?? []); const expectedDecision = findExpectedDecisionFromExpectedMessages(expectedOutput) ?? diff --git a/examples/features/code-grader-sdk/evals/example.txt b/examples/features/code-grader-sdk/evals/example.txt deleted file mode 100644 index 5bc8587c5..000000000 --- a/examples/features/code-grader-sdk/evals/example.txt +++ /dev/null @@ -1 +0,0 @@ -This is a placeholder attachment for the code_grader SDK demo. diff --git a/examples/features/composite/evals/suite.yaml b/examples/features/composite/evals/suite.yaml index 3af679ca0..b33d6e5f4 100644 --- a/examples/features/composite/evals/suite.yaml +++ b/examples/features/composite/evals/suite.yaml @@ -32,8 +32,8 @@ tests: safety: 0.3 quality: 0.7 - # Example 2: Code Grader Aggregator (Safety Gate Pattern) - - id: code-grader-safety-gate + # Example 2: script grader Aggregator (Safety Gate Pattern) + - id: script-grader-safety-gate input: - role: user content: "Explain quantum computing in simple terms." @@ -57,7 +57,7 @@ tests: type: script path: bun run ../scripts/safety-gate-aggregator.js - # Example 3: Strict OR with a local code-grader aggregator + # Example 3: Strict OR with a local script-grader aggregator - id: strict-or-local input: - role: user diff --git a/examples/features/copilot-log-eval/README.md b/examples/features/copilot-log-eval/README.md index d0ac4b635..f95e87355 100644 --- a/examples/features/copilot-log-eval/README.md +++ b/examples/features/copilot-log-eval/README.md @@ -5,7 +5,7 @@ from disk with deterministic graders. **No LLM API key needed.** Graders used: - `skill-trigger` — checks whether a specific skill was invoked -- `code-grader` — custom TypeScript grader inspecting the full `Message[]` with tool calls +- `script-grader` — custom TypeScript grader inspecting the full `Message[]` with tool calls ## Setup @@ -44,7 +44,7 @@ allagents workspace init (setup hook) ↓ copilot-log provider (reads from disk) Message[] with tool calls ├─ skill-trigger grader (deterministic) → pass/fail - └─ code-grader (graders/transcript-quality.ts) → pass/fail + └─ script-grader (graders/transcript-quality.ts) → pass/fail ``` ## Graders @@ -53,8 +53,8 @@ Message[] with tool calls Checks whether the `csv-analyzer` skill was (or was not) invoked. Inspects tool call names and skill invocation events in the transcript. -### transcript-quality (code-grader) -Custom grader using `defineCodeGrader` from `@agentv/sdk`. Validates: +### transcript-quality (script-grader) +Custom grader using `defineScriptGrader` from `@agentv/sdk`. Validates: 1. Transcript contains assistant messages 2. Tool calls were recorded (inspects `Message[].toolCalls`) 3. Response addresses the CSV analysis question diff --git a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml b/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml index 6ae36e0ef..51272fa38 100644 --- a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml +++ b/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml @@ -1,8 +1,8 @@ -# Copilot Log evaluation example — deterministic + code-grader assertions. +# Copilot Log evaluation example — deterministic + script-grader assertions. # # Reads a Copilot CLI session transcript from disk and evaluates it with: # 1. skill-trigger — deterministic check for skill invocation -# 2. code-grader — custom TypeScript grader inspecting full Message[] +# 2. script-grader — custom TypeScript grader inspecting full Message[] # # No LLM API key needed — all graders are deterministic. # diff --git a/examples/features/copilot-log-eval/graders/transcript-quality.ts b/examples/features/copilot-log-eval/graders/transcript-quality.ts index b1b2b3b4d..9d6a49d9d 100644 --- a/examples/features/copilot-log-eval/graders/transcript-quality.ts +++ b/examples/features/copilot-log-eval/graders/transcript-quality.ts @@ -15,7 +15,7 @@ * type: script * command: ["bun", "run", "../graders/transcript-quality.ts"] */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function getMessageText( messages: readonly { role: string; content?: unknown }[], @@ -36,7 +36,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output }) => { +export default defineScriptGrader(({ output }) => { const outputText = getMessageText(output ?? []); const assertions: Array<{ text: string; passed: boolean; evidence?: string }> = []; diff --git a/examples/features/default-graders/evals/suite.yaml b/examples/features/default-graders/evals/suite.yaml index 41017a229..9adf7afef 100644 --- a/examples/features/default-graders/evals/suite.yaml +++ b/examples/features/default-graders/evals/suite.yaml @@ -1,5 +1,5 @@ # Default Graders Example -# Demonstrates root-level assertionsions that apply to all tests +# Demonstrates root-level assertions that apply to all tests name: default-graders-example description: Root-level graders that automatically apply to every test diff --git a/examples/features/deterministic-graders/README.md b/examples/features/deterministic-graders/README.md index 5ea7a3442..fab733c8f 100644 --- a/examples/features/deterministic-graders/README.md +++ b/examples/features/deterministic-graders/README.md @@ -4,7 +4,7 @@ Demonstrates how a single parameterised `script` grader can replace a family of ## Why a Script Grader? -AgentV's design philosophy keeps the core minimal. Instead of adding `contains`, `regex`, `is-json` as built-in grader types, you write a small code grader and drive it with YAML `config`: +AgentV's design philosophy keeps the core minimal. Instead of adding `contains`, `regex`, `is-json` as built-in grader types, you write a small script grader and drive it with YAML `config`: ```yaml graders: @@ -31,7 +31,7 @@ Set `negated: true` in config to invert any assertion. ## Files -- `graders/assertions.ts` — Parameterised script grader using `defineCodeGrader` from `@agentv/sdk` +- `graders/assertions.ts` — Parameterised script grader using `defineScriptGrader` from `@agentv/sdk` - `evals/suite.yaml` — Example tests covering every assertion type ## Setup diff --git a/examples/features/deterministic-graders/graders/assertions.ts b/examples/features/deterministic-graders/graders/assertions.ts index d057bb6ee..827e7c8a2 100644 --- a/examples/features/deterministic-graders/graders/assertions.ts +++ b/examples/features/deterministic-graders/graders/assertions.ts @@ -2,7 +2,7 @@ /** * Parameterized assertion grader. * - * A single code grader that handles common deterministic checks + * A single script grader that handles common deterministic checks * (contains, regex, JSON validation, etc.) driven by YAML config. * * Config fields (passed via grader `config` in YAML): @@ -10,7 +10,7 @@ * value – expected substring, pattern, or prefix (not used for is-json) * negated – when true, inverts the assertion (default: false) */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; type AssertionType = 'contains' | 'icontains' | 'equals' | 'regex' | 'starts-with' | 'is-json'; @@ -55,7 +55,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, criteria, config }) => { +export default defineScriptGrader(({ output, criteria, config }) => { const outputText = getMessageText(output ?? []); const type = (config?.type as AssertionType) ?? 'contains'; const value = config?.value as string | undefined; diff --git a/examples/features/docker-workspace/README.md b/examples/features/docker-workspace/README.md index c2be56972..79bcdaf77 100644 --- a/examples/features/docker-workspace/README.md +++ b/examples/features/docker-workspace/README.md @@ -1,6 +1,6 @@ # Docker Workspace Example -This example demonstrates how to run code-grader evaluations inside Docker containers. +This example demonstrates how to run script-grader evaluations inside Docker containers. ## Use Case diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index 0a012199f..0a59bb125 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -1,5 +1,5 @@ # Docker Workspace Example -# Demonstrates running a code-grader inside a Docker container. +# Demonstrates running a script-grader inside a Docker container. # # This eval sends a coding prompt to an agent, then grades the agent's output # inside a Docker container that has the target repository and test environment. diff --git a/examples/features/document-extraction/README.md b/examples/features/document-extraction/README.md index 1f79a27e0..e2dd1429d 100644 --- a/examples/features/document-extraction/README.md +++ b/examples/features/document-extraction/README.md @@ -3,14 +3,14 @@ This folder demonstrates two evaluation patterns for document extraction: 1. **`field_accuracy`** (built-in) - Per-test-case scoring with pass/fail per field -2. **`code-grader`** (custom) - TP/TN/FP/FN metrics for cross-document aggregation +2. **`script-grader`** (custom) - TP/TN/FP/FN metrics for cross-document aggregation ## When to Use Each Pattern | Pattern | Use Case | Output | |---------|----------|--------| | `field_accuracy` | Simple pass/fail scoring per test case | Score (0-1) per test case | -| `code-grader` with `details.metrics` | Aggregate precision/recall across documents | TP/TN/FP/FN per field | +| `script-grader` with `details.metrics` | Aggregate precision/recall across documents | TP/TN/FP/FN per field | ## Quick Start @@ -53,7 +53,7 @@ graders: ## Pattern 2: Confusion Metrics (`confusion-metrics.eval.yaml`) -Uses a custom `code-grader` that emits `details.metrics` with TP/TN/FP/FN per field: +Uses a custom `script-grader` that emits `details.metrics` with TP/TN/FP/FN per field: ```yaml graders: diff --git a/examples/features/document-extraction/evals/field-accuracy.eval.yaml b/examples/features/document-extraction/evals/field-accuracy.eval.yaml index 84ca5dce1..d060e3a34 100644 --- a/examples/features/document-extraction/evals/field-accuracy.eval.yaml +++ b/examples/features/document-extraction/evals/field-accuracy.eval.yaml @@ -9,7 +9,7 @@ # - Exact matching for invoice numbers and currency codes # - Date matching with format normalization # - Numeric tolerance for currency amounts -# - Fuzzy matching via code-grader plugins +# - Fuzzy matching via script-grader plugins # - Line item array validation # # EXPECTED SCORES: @@ -47,7 +47,7 @@ assert: weight: 1.0 # Party information - # Note: For fuzzy matching (OCR variations), use code-grader with fuzzy_match.ts + # Note: For fuzzy matching (OCR variations), use script-grader with fuzzy_match.ts - path: supplier.name match: exact required: true @@ -213,7 +213,7 @@ tests: # ============================================ # Test Case 2: Supplier Name Spacing Variation - # Demonstrates fuzzy matching via code-grader with config pass-through + # Demonstrates fuzzy matching via script-grader with config pass-through # ============================================ - id: invoice-002 conversation_id: document-extraction diff --git a/examples/features/document-extraction/graders/fuzzy_match.ts b/examples/features/document-extraction/graders/fuzzy_match.ts index c85c84a58..7970c3e6f 100644 --- a/examples/features/document-extraction/graders/fuzzy_match.ts +++ b/examples/features/document-extraction/graders/fuzzy_match.ts @@ -1,8 +1,8 @@ #!/usr/bin/env bun /** - * Fuzzy String Matching code-grader Example + * Fuzzy String Matching script-grader Example * - * This script demonstrates how to implement fuzzy string matching as a code-grader + * This script demonstrates how to implement fuzzy string matching as a script-grader * grader. Use this approach for comparing extracted text that may have OCR errors, * formatting variations, or minor typos. * diff --git a/examples/features/document-extraction/graders/header_confusion_metrics.ts b/examples/features/document-extraction/graders/header_confusion_metrics.ts index 2cc2a32f2..0876b9619 100644 --- a/examples/features/document-extraction/graders/header_confusion_metrics.ts +++ b/examples/features/document-extraction/graders/header_confusion_metrics.ts @@ -2,7 +2,7 @@ /** * Header Field Confusion Metrics Grader * - * A code-grader that compares header fields and classifies them as TP/TN/FP/FN + * A script-grader that compares header fields and classifies them as TP/TN/FP/FN * based on empty vs non-empty expected/parsed values. * * Classification rules (per attribute): diff --git a/examples/features/document-extraction/graders/line_item_matching.ts b/examples/features/document-extraction/graders/line_item_matching.ts index adb8874c8..a24ff9e5c 100644 --- a/examples/features/document-extraction/graders/line_item_matching.ts +++ b/examples/features/document-extraction/graders/line_item_matching.ts @@ -2,7 +2,7 @@ /** * Line Item Matching Grader * - * A code-grader that matches expected line items to parsed line items using + * A script-grader that matches expected line items to parsed line items using * greedy matching before scoring. This handles reordered and duplicate items. * * Matching strategy (greedy): diff --git a/examples/features/document-extraction/graders/multi_field_fuzzy.ts b/examples/features/document-extraction/graders/multi_field_fuzzy.ts index 388397f59..d60922767 100644 --- a/examples/features/document-extraction/graders/multi_field_fuzzy.ts +++ b/examples/features/document-extraction/graders/multi_field_fuzzy.ts @@ -2,7 +2,7 @@ /** * Multi-Field Fuzzy Matcher * - * A configurable code-grader that compares multiple fields using Levenshtein similarity. + * A configurable script-grader that compares multiple fields using Levenshtein similarity. * Configuration is passed via YAML properties that become stdin config. * * Usage in suite.yaml: diff --git a/examples/features/document-extraction/scripts/aggregate_metrics.ts b/examples/features/document-extraction/scripts/aggregate_metrics.ts index 2eea67ed8..3aa05fb23 100644 --- a/examples/features/document-extraction/scripts/aggregate_metrics.ts +++ b/examples/features/document-extraction/scripts/aggregate_metrics.ts @@ -123,7 +123,7 @@ function printTable(aggregated: Record): void { if (fields.length === 0) { console.log('No metrics found in evaluation results.'); - console.log('Make sure your code graders emit a `details.metrics` object.'); + console.log('Make sure your script graders emit a `details.metrics` object.'); return; } diff --git a/examples/features/eval-assert-demo/README.md b/examples/features/eval-assert-demo/README.md index 141f8c61f..6ffdb7814 100644 --- a/examples/features/eval-assert-demo/README.md +++ b/examples/features/eval-assert-demo/README.md @@ -1,6 +1,6 @@ # Eval Assert Demo -Demonstrates code graders that can be run both as part of an eval suite and individually via `agentv eval assert`. +Demonstrates script graders that can be run both as part of an eval suite and individually via `agentv eval assert`. ## Graders @@ -9,7 +9,7 @@ Demonstrates code graders that can be run both as part of an eval suite and indi | `.agentv/graders/keyword-check.ts` | Checks answer contains expected keywords (Paris, France) | | `.agentv/graders/length-check.ts` | Validates answer word count is between 5 and 50 | -Both graders use `defineCodeGrader` from `@agentv/sdk`. +Both graders use `defineScriptGrader` from `@agentv/sdk`. ## Running the Full Eval @@ -56,13 +56,13 @@ Input: "What is the capital of France? Answer in one concise sentence." Expected: "The capital of France is Paris." Criteria: - Output contains 'Paris' - - [code-grader] keyword-check: Checks that the answer mentions Paris and France - - [code-grader] length-check: Ensures answer is between 5 and 50 words + - [script-grader] keyword-check: Checks that the answer mentions Paris and France + - [script-grader] length-check: Ensures answer is between 5 and 50 words ``` ## How It Works -When running the eval, the transpiler emits natural-language instructions for each code grader: +When running the eval, the transpiler emits natural-language instructions for each script grader: ``` Run `agentv eval assert keyword-check --agent-output --agent-input ` and check the result. @@ -71,4 +71,4 @@ The command returns JSON: {"score": 0-1, "assertions": [{"text": "...", "passed" A score >= 0.5 means pass (exit 0); below 0.5 means fail (exit 1). ``` -This allows external grading agents to execute code graders directly without understanding their internal implementation. +This allows external grading agents to execute script graders directly without understanding their internal implementation. diff --git a/examples/features/eval-assert-demo/evals/suite.yaml b/examples/features/eval-assert-demo/evals/suite.yaml index 8f7e845dd..ff107ce2d 100644 --- a/examples/features/eval-assert-demo/evals/suite.yaml +++ b/examples/features/eval-assert-demo/evals/suite.yaml @@ -1,9 +1,9 @@ # Eval Assert Demo -# Demonstrates code graders with `agentv eval assert` CLI integration. -# Each code-grader assertion can also be run standalone: +# Demonstrates script graders with `agentv eval assert` CLI integration. +# Each script-grader assertion can also be run standalone: # agentv eval assert keyword-check --agent-output "..." --agent-input "..." -description: Code graders with eval assert CLI integration +description: script graders with eval assert CLI integration target: llm diff --git a/examples/features/execution-metrics/README.md b/examples/features/execution-metrics/README.md index 813c77880..6ddddc966 100644 --- a/examples/features/execution-metrics/README.md +++ b/examples/features/execution-metrics/README.md @@ -7,7 +7,7 @@ Demonstrates execution metrics tracking (tokens, cost, latency) in evaluations. - Automatic token usage tracking (input, output, cached) - Cost tracking in USD - Execution duration in milliseconds -- Using metrics in code graders for performance evaluation +- Using metrics in script graders for performance evaluation - Metrics available in evaluation results ## Running diff --git a/examples/features/execution-metrics/evals/suite.yaml b/examples/features/execution-metrics/evals/suite.yaml index 5827df41e..6115fbef8 100644 --- a/examples/features/execution-metrics/evals/suite.yaml +++ b/examples/features/execution-metrics/evals/suite.yaml @@ -127,8 +127,8 @@ tests: Agent completes the task within the specified cost budget. # ========================================== - # Example 6: Combining with code_grader - # Use execution_metrics for thresholds, code_grader for custom logic + # Example 6: Combining with script_grader + # Use execution_metrics for thresholds, script_grader for custom logic # ========================================== - id: hybrid-evaluation diff --git a/examples/features/execution-metrics/scripts/check-efficiency.ts b/examples/features/execution-metrics/scripts/check-efficiency.ts index 84ec19c1e..d73093f3b 100644 --- a/examples/features/execution-metrics/scripts/check-efficiency.ts +++ b/examples/features/execution-metrics/scripts/check-efficiency.ts @@ -1,11 +1,11 @@ #!/usr/bin/env bun /** - * Efficiency Check - Code Grader for Execution Metrics + * Efficiency Check - script grader for Execution Metrics * * Demonstrates how to evaluate agent efficiency using execution metrics * available in the trace payload. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; // Configurable thresholds const THRESHOLDS = { @@ -15,7 +15,7 @@ const THRESHOLDS = { maxDurationMs: 10000, }; -export default defineCodeGrader(({ trace, tokenUsage, costUsd, durationMs }) => { +export default defineScriptGrader(({ trace, tokenUsage, costUsd, durationMs }) => { const assertions: Array<{ text: string; passed: boolean }> = []; if (!trace) { diff --git a/examples/features/execution-metrics/scripts/check-metrics-present.ts b/examples/features/execution-metrics/scripts/check-metrics-present.ts index 489bc056f..b472b3dc7 100644 --- a/examples/features/execution-metrics/scripts/check-metrics-present.ts +++ b/examples/features/execution-metrics/scripts/check-metrics-present.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Check Metrics Present - Code Grader Plugin + * Check Metrics Present - script grader Plugin * * Verifies that execution metrics are present in the trace payload. * This is a simple sanity check that metrics collection is working. @@ -11,9 +11,9 @@ * type: script * command: ["bun", "run", "../scripts/check-metrics-present.ts"] */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ trace, tokenUsage, costUsd, durationMs }) => { +export default defineScriptGrader(({ trace, tokenUsage, costUsd, durationMs }) => { const assertions: Array<{ text: string; passed: boolean }> = []; if (!trace) { diff --git a/examples/features/file-changes-with-repos/scripts/check-file-changes.ts b/examples/features/file-changes-with-repos/scripts/check-file-changes.ts index a150c3fdf..f11396bff 100644 --- a/examples/features/file-changes-with-repos/scripts/check-file-changes.ts +++ b/examples/features/file-changes-with-repos/scripts/check-file-changes.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Code grader: verifies file_changes captures BOTH workspace-root files + * script grader: verifies file_changes captures BOTH workspace-root files * and changes inside nested git repos. * * Expected diff should include: diff --git a/examples/features/functional-grading/evals/suite.yaml b/examples/features/functional-grading/evals/suite.yaml index 0be172d4d..af1410347 100644 --- a/examples/features/functional-grading/evals/suite.yaml +++ b/examples/features/functional-grading/evals/suite.yaml @@ -2,7 +2,7 @@ # Demonstrates using workspace_path to run commands in the agent's workspace. # # The workspace-template/ contains a TypeScript project with function stubs. -# The mock agent implements the functions, then the code grader: +# The mock agent implements the functions, then the script grader: # 1. Installs dependencies (npm install) # 2. Runs typecheck (tsc --noEmit) # 3. Compiles (tsc) diff --git a/examples/features/functional-grading/scripts/functional-check.ts b/examples/features/functional-grading/scripts/functional-check.ts index b4c933b1e..534242368 100644 --- a/examples/features/functional-grading/scripts/functional-check.ts +++ b/examples/features/functional-grading/scripts/functional-check.ts @@ -23,7 +23,7 @@ if (!workspacePath) { { text: 'workspace_path not provided — cannot run functional checks', passed: false, - evidence: 'Code grader requires workspace_path to execute commands in the workspace', + evidence: 'script grader requires workspace_path to execute commands in the workspace', }, ], }), diff --git a/examples/features/import-claude/README.md b/examples/features/import-claude/README.md index a5fab5470..aad1c4d4f 100644 --- a/examples/features/import-claude/README.md +++ b/examples/features/import-claude/README.md @@ -4,7 +4,7 @@ Demonstrates importing a Claude Code session transcript and grading it offline with deterministic graders. **No LLM API key needed.** Graders used: -- `code-grader` — custom TypeScript grader inspecting the full `Message[]` with tool calls +- `script-grader` — custom TypeScript grader inspecting the full `Message[]` with tool calls ## Setup @@ -38,7 +38,7 @@ agentv eval evals/transcript-check.EVAL.yaml ~/.claude/projects//.jsonl ↓ agentv import claude (reads from disk, converts to Message[]) .agentv/transcripts/claude-.jsonl - ↓ code-grader (deterministic) + ↓ script-grader (deterministic) pass/fail ``` @@ -51,9 +51,9 @@ The import pipeline: ## Graders -### transcript-quality (code-grader) +### transcript-quality (script-grader) -Custom grader using `defineCodeGrader` from `@agentv/sdk`. Validates: +Custom grader using `defineScriptGrader` from `@agentv/sdk`. Validates: 1. Transcript contains at least one assistant message 2. Tool calls were recorded with outputs 3. No empty assistant messages diff --git a/examples/features/import-claude/graders/transcript-quality.ts b/examples/features/import-claude/graders/transcript-quality.ts index 19ab36656..d02358e50 100644 --- a/examples/features/import-claude/graders/transcript-quality.ts +++ b/examples/features/import-claude/graders/transcript-quality.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output }) => { +export default defineScriptGrader(({ output }) => { const assertions: Array<{ text: string; passed: boolean }> = []; // Check 1: Has assistant messages diff --git a/examples/features/nlp-metrics/README.md b/examples/features/nlp-metrics/README.md index 04a3c111f..0707ad480 100644 --- a/examples/features/nlp-metrics/README.md +++ b/examples/features/nlp-metrics/README.md @@ -1,6 +1,6 @@ # NLP Metrics Examples -Demonstrates how to implement common NLP evaluation metrics as AgentV `code_grader` graders — no external dependencies required. +Demonstrates how to implement common NLP evaluation metrics as AgentV `script_grader` graders — no external dependencies required. ## Graders @@ -11,7 +11,7 @@ Demonstrates how to implement common NLP evaluation metrics as AgentV `code_grad | `graders/similarity.ts` | Cosine + Jaccard | Paraphrasing — token-overlap similarity | | `graders/levenshtein.ts` | Levenshtein distance | Extraction — character-level edit distance | -Each grader is a standalone TypeScript file that uses `defineCodeGrader` from `@agentv/sdk`. Scores are normalised to the 0–1 range expected by AgentV. +Each grader is a standalone TypeScript file that uses `defineScriptGrader` from `@agentv/sdk`. Scores are normalised to the 0–1 range expected by AgentV. ## Running @@ -38,7 +38,7 @@ bun agentv eval prompt eval --grading-brief \ ## How It Works -Each grader receives the candidate answer and reference text via the `defineCodeGrader` handler, computes the relevant metric from scratch, and returns a `CodeGraderResult` with: +Each grader receives the candidate answer and reference text via the `defineScriptGrader` handler, computes the relevant metric from scratch, and returns a `ScriptGraderResult` with: - **score** — normalised 0–1 value - **hits / misses** — threshold checks for quick pass/fail diff --git a/examples/features/nlp-metrics/evals/suite.yaml b/examples/features/nlp-metrics/evals/suite.yaml index 63dba4978..eabe70d21 100644 --- a/examples/features/nlp-metrics/evals/suite.yaml +++ b/examples/features/nlp-metrics/evals/suite.yaml @@ -1,9 +1,9 @@ # NLP Metrics Grader Examples # Demonstrates ROUGE, BLEU, cosine similarity, and Levenshtein distance -# as code_grader graders — no external dependencies required. +# as script_grader graders — no external dependencies required. name: nlp-metrics -description: NLP text-quality metrics using code_grader graders +description: NLP text-quality metrics using script_grader graders target: llm diff --git a/examples/features/nlp-metrics/graders/bleu.ts b/examples/features/nlp-metrics/graders/bleu.ts index 3b04a438d..8260bb4a0 100644 --- a/examples/features/nlp-metrics/graders/bleu.ts +++ b/examples/features/nlp-metrics/graders/bleu.ts @@ -1,12 +1,12 @@ #!/usr/bin/env bun /** - * BLEU Code Grader + * BLEU script grader * * Computes a BLEU-like score between candidate and reference text. * BLEU (Bilingual Evaluation Understudy) measures n-gram precision with a * brevity penalty, commonly used for translation evaluation. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function tokenize(text: string): string[] { return text @@ -84,7 +84,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const outputText = getMessageText(output ?? []); const reference = getMessageText(expectedOutput); diff --git a/examples/features/nlp-metrics/graders/levenshtein.ts b/examples/features/nlp-metrics/graders/levenshtein.ts index 8b11fcd72..fdf5095ac 100644 --- a/examples/features/nlp-metrics/graders/levenshtein.ts +++ b/examples/features/nlp-metrics/graders/levenshtein.ts @@ -1,12 +1,12 @@ #!/usr/bin/env bun /** - * Levenshtein Distance Code Grader + * Levenshtein Distance script grader * * Computes normalised edit distance between candidate and reference text. * The score is 1 - (distance / maxLength), so identical strings score 1.0 * and completely different strings score close to 0. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function levenshteinDistance(a: string, b: string): number { const m = a.length; @@ -50,7 +50,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const outputText = getMessageText(output ?? []); const reference = getMessageText(expectedOutput); diff --git a/examples/features/nlp-metrics/graders/rouge.ts b/examples/features/nlp-metrics/graders/rouge.ts index 8ed17e001..ae364648a 100644 --- a/examples/features/nlp-metrics/graders/rouge.ts +++ b/examples/features/nlp-metrics/graders/rouge.ts @@ -1,12 +1,12 @@ #!/usr/bin/env bun /** - * ROUGE-N Code Grader + * ROUGE-N script grader * * Computes ROUGE-1 and ROUGE-2 F1 scores between candidate and reference text. * ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures n-gram * overlap, commonly used for summarisation evaluation. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function tokenize(text: string): string[] { return text @@ -66,7 +66,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const outputText = getMessageText(output ?? []); const reference = getMessageText(expectedOutput); diff --git a/examples/features/nlp-metrics/graders/similarity.ts b/examples/features/nlp-metrics/graders/similarity.ts index df4e7c4da..a92c00434 100644 --- a/examples/features/nlp-metrics/graders/similarity.ts +++ b/examples/features/nlp-metrics/graders/similarity.ts @@ -1,12 +1,12 @@ #!/usr/bin/env bun /** - * Cosine Similarity Code Grader + * Cosine Similarity script grader * * Computes cosine similarity between candidate and reference text using * token-overlap (bag-of-words) vectors. This is a lightweight alternative to * embedding-based similarity that requires no external dependencies. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function tokenize(text: string): string[] { return text @@ -68,7 +68,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const outputText = getMessageText(output ?? []); const reference = getMessageText(expectedOutput); diff --git a/examples/features/prompt-template-sdk/README.md b/examples/features/prompt-template-sdk/README.md index b04f1f7ba..6cfd102f7 100644 --- a/examples/features/prompt-template-sdk/README.md +++ b/examples/features/prompt-template-sdk/README.md @@ -7,7 +7,7 @@ This example demonstrates using TypeScript files for custom LLM grader prompts u - **Type-safe prompt generation**: Full TypeScript support with autocomplete for context fields - **Conditional logic**: Use JavaScript/TypeScript conditionals for dynamic prompts - **Config pass-through**: Access custom config from YAML in your prompt template -- **Same pattern as code graders**: Follows the familiar subprocess pattern +- **Same pattern as script graders**: Follows the familiar subprocess pattern ## How It Works diff --git a/examples/features/prompt-template-sdk/evals/suite.yaml b/examples/features/prompt-template-sdk/evals/suite.yaml index 1d265c818..1af7da8ec 100644 --- a/examples/features/prompt-template-sdk/evals/suite.yaml +++ b/examples/features/prompt-template-sdk/evals/suite.yaml @@ -1,6 +1,6 @@ # Prompt Template SDK Demo # Demonstrates using TypeScript/JavaScript files for custom grader prompts. -# Uses the same explicit script pattern as code_grader for consistency. +# Uses the same explicit script pattern as script_grader for consistency. description: Demonstrates TypeScript prompt templates for custom LLM grader prompts @@ -24,7 +24,7 @@ tests: assert: - metric: custom-prompt-eval type: llm-rubric - # Executable prompt template using explicit script array (matches code_grader pattern) + # Executable prompt template using explicit script array. prompt: command: [ bun, run, ../prompts/custom-grader.ts ] - The CLI provides a clear answer about TypeScript benefits. diff --git a/examples/features/rubric/evals/suite.yaml b/examples/features/rubric/evals/suite.yaml index 522b5fb11..2f3b6fa20 100644 --- a/examples/features/rubric/evals/suite.yaml +++ b/examples/features/rubric/evals/suite.yaml @@ -135,7 +135,7 @@ tests: - Has docstring documentation - Handles edge cases (None, empty string) - # Additional code grader for syntax checking + # Additional script grader for syntax checking - metric: python_syntax type: script command: [ "uv", "run", "python", "check_syntax.py" ] diff --git a/examples/features/code-grader-sdk/.agentv/targets.yaml b/examples/features/script-grader-sdk/.agentv/targets.yaml similarity index 100% rename from examples/features/code-grader-sdk/.agentv/targets.yaml rename to examples/features/script-grader-sdk/.agentv/targets.yaml diff --git a/examples/features/code-grader-sdk/CHANGELOG.md b/examples/features/script-grader-sdk/CHANGELOG.md similarity index 92% rename from examples/features/code-grader-sdk/CHANGELOG.md rename to examples/features/script-grader-sdk/CHANGELOG.md index 243d56b15..9ac026119 100644 --- a/examples/features/code-grader-sdk/CHANGELOG.md +++ b/examples/features/script-grader-sdk/CHANGELOG.md @@ -1,4 +1,4 @@ -# @agentv-examples/code-grader-sdk +# @agentv-examples/script-grader-sdk ## 0.0.2 diff --git a/examples/features/code-grader-sdk/README.md b/examples/features/script-grader-sdk/README.md similarity index 59% rename from examples/features/code-grader-sdk/README.md rename to examples/features/script-grader-sdk/README.md index 9ef3eca9f..f6af588a7 100644 --- a/examples/features/code-grader-sdk/README.md +++ b/examples/features/script-grader-sdk/README.md @@ -1,11 +1,11 @@ -# Code Grader SDK Helper +# script grader SDK Helper -Demonstrates how a TypeScript `code-grader` can use `defineCodeGrader` from `@agentv/sdk` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. +Demonstrates how a TypeScript `script-grader` can use `defineScriptGrader` from `@agentv/sdk` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. ## Files -- `evals/suite.yaml`: Example test that uses a `code-grader`. -- `scripts/verify-attachments.ts`: Code grader script using `defineCodeGrader`. +- `evals/suite.yaml`: Example test that uses a script grader. +- `scripts/verify-attachments.ts`: Script grader script using `defineScriptGrader`. - `evals/example.txt`, `evals/python.instructions.md`: Attachment fixtures. ## Setup @@ -21,10 +21,10 @@ bun run build # Builds @agentv/core package ### Standalone Test -Test the SDK-based code grader directly with a mock payload: +Test the SDK-based script grader directly with a mock payload: ```bash -cd examples/features/code-grader-sdk +cd examples/features/script-grader-sdk cat << 'EOF' | bun run scripts/verify-attachments.ts { "criteria": "The CLI echoes the prompt and lists attachment names.", @@ -42,24 +42,24 @@ From the repository root: ```bash cd examples/features -bun agentv eval code-grader-sdk/evals/suite.yaml --target local_cli +bun agentv eval script-grader-sdk/evals/suite.yaml --target local_cli ``` This requires a CLI target named `local_cli` configured in `.agentv/targets.yaml`. ## API -The `defineCodeGrader` helper: +The `defineScriptGrader` helper: - Reads JSON from stdin automatically - Converts snake_case to camelCase - Validates input and output with Zod schemas - Handles errors gracefully ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, criteria }) => ({ +export default defineScriptGrader(({ output, criteria }) => ({ score: (output ?? '').includes(criteria) ? 1.0 : 0.0, - assertions: [{ text: 'Check passed', passed: (output ?? '').includes(criteria) }], + assert: [{ text: 'Check passed', passed: (output ?? '').includes(criteria) }], })); ``` diff --git a/examples/features/code-grader-sdk/bun.lock b/examples/features/script-grader-sdk/bun.lock similarity index 89% rename from examples/features/code-grader-sdk/bun.lock rename to examples/features/script-grader-sdk/bun.lock index 56c0dae9e..4b99380a9 100644 --- a/examples/features/code-grader-sdk/bun.lock +++ b/examples/features/script-grader-sdk/bun.lock @@ -3,7 +3,7 @@ "configVersion": 1, "workspaces": { "": { - "name": "agentv-example-code-grader-sdk", + "name": "agentv-example-script-grader-sdk", "dependencies": { "@agentv/sdk": "file:../../../packages/sdk", }, diff --git a/examples/features/script-grader-sdk/evals/example.txt b/examples/features/script-grader-sdk/evals/example.txt new file mode 100644 index 000000000..9c31f3649 --- /dev/null +++ b/examples/features/script-grader-sdk/evals/example.txt @@ -0,0 +1 @@ +This is a placeholder attachment for the script_grader SDK demo. diff --git a/examples/features/code-grader-sdk/evals/python.instructions.md b/examples/features/script-grader-sdk/evals/python.instructions.md similarity index 100% rename from examples/features/code-grader-sdk/evals/python.instructions.md rename to examples/features/script-grader-sdk/evals/python.instructions.md diff --git a/examples/features/code-grader-sdk/evals/suite.baseline.jsonl b/examples/features/script-grader-sdk/evals/suite.baseline.jsonl similarity index 100% rename from examples/features/code-grader-sdk/evals/suite.baseline.jsonl rename to examples/features/script-grader-sdk/evals/suite.baseline.jsonl diff --git a/examples/features/code-grader-sdk/evals/suite.yaml b/examples/features/script-grader-sdk/evals/suite.yaml similarity index 77% rename from examples/features/code-grader-sdk/evals/suite.yaml rename to examples/features/script-grader-sdk/evals/suite.yaml index 92b1eaaf7..5b3a170e5 100644 --- a/examples/features/code-grader-sdk/evals/suite.yaml +++ b/examples/features/script-grader-sdk/evals/suite.yaml @@ -1,7 +1,7 @@ -# Code Grader SDK Helper Demo -# Demonstrates using the optional TypeScript helper to parse code_grader stdin. +# script grader SDK Helper Demo +# Demonstrates using the optional TypeScript helper to parse script_grader stdin. -description: Demonstrates TypeScript helpers for code_grader payloads +description: Demonstrates TypeScript helpers for script_grader payloads # Uses the CLI target defined in .agentv/targets.yaml target: local_cli @@ -9,7 +9,7 @@ target: local_cli tags: [ agent ] tests: - - id: code-grader-sdk-attachments + - id: script-grader-sdk-attachments input: - role: system diff --git a/examples/features/code-grader-sdk/package.json b/examples/features/script-grader-sdk/package.json similarity index 71% rename from examples/features/code-grader-sdk/package.json rename to examples/features/script-grader-sdk/package.json index c2bddac45..7cc81ab6b 100644 --- a/examples/features/code-grader-sdk/package.json +++ b/examples/features/script-grader-sdk/package.json @@ -1,5 +1,5 @@ { - "name": "agentv-example-code-grader-sdk", + "name": "agentv-example-script-grader-sdk", "private": true, "type": "module", "dependencies": { diff --git a/examples/features/code-grader-sdk/scripts/verify-attachments.ts b/examples/features/script-grader-sdk/scripts/verify-attachments.ts similarity index 85% rename from examples/features/code-grader-sdk/scripts/verify-attachments.ts rename to examples/features/script-grader-sdk/scripts/verify-attachments.ts index 00dd1c634..608dead77 100755 --- a/examples/features/code-grader-sdk/scripts/verify-attachments.ts +++ b/examples/features/script-grader-sdk/scripts/verify-attachments.ts @@ -1,18 +1,18 @@ #!/usr/bin/env bun /** - * Code Grader SDK Demo + * script grader SDK Demo * - * Uses the declarative defineCodeGrader helper to verify attachments + * Uses the declarative defineScriptGrader helper to verify attachments * are referenced in the candidate output. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function fileName(path: string): string { const parts = path.split('/'); return parts[parts.length - 1] ?? path; } -export default defineCodeGrader(({ expectedOutput, output, inputFiles }) => { +export default defineScriptGrader(({ expectedOutput, output, inputFiles }) => { const outputText = output ?? ''; const assertions: Array<{ text: string; passed: boolean }> = []; diff --git a/examples/features/code-grader-with-llm-calls/README.md b/examples/features/script-grader-with-llm-calls/README.md similarity index 91% rename from examples/features/code-grader-with-llm-calls/README.md rename to examples/features/script-grader-with-llm-calls/README.md index 6b95e9865..e1e9b1dd6 100644 --- a/examples/features/code-grader-with-llm-calls/README.md +++ b/examples/features/script-grader-with-llm-calls/README.md @@ -1,6 +1,6 @@ -# Code Grader with LLM Calls +# script grader with LLM Calls -This example demonstrates how code grader graders can make LLM calls through a secure local proxy without needing direct API credentials. +This example demonstrates how script grader graders can make LLM calls through a secure local proxy without needing direct API credentials. This example implements two RAG metrics: - **Contextual Precision**: Evaluates whether relevant documents are ranked higher @@ -117,7 +117,7 @@ This produces: `["Node A", "Node B", "Node C"]` **Potential Solutions:** -All solutions below can be implemented entirely in the code grader - no core AgentV changes required. The code grader receives the full `expectedOutput` structure: +All solutions below can be implemented entirely in the script grader - no core AgentV changes required. The script grader receives the full `expectedOutput` structure: ```typescript // Available in input.expectedOutput @@ -149,7 +149,7 @@ The target proxy is designed with security in mind: ## Configuration -Enable target access by adding a `target` block to your `code_grader` grader: +Enable target access by adding a `target` block to your `script_grader` grader: ```yaml graders: @@ -168,9 +168,9 @@ graders: ## Usage in Code ```typescript -import { createTargetClient, defineCodeGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(async ({ question, config }) => { +export default defineScriptGrader(async ({ question, config }) => { const target = createTargetClient(); const retrievalContext = config?.retrieval_context ?? []; @@ -231,10 +231,10 @@ The `createTargetClient()` function reads these automatically. # From the agentv monorepo root: # Run contextual precision evaluation -agentv eval examples/features/code-grader-with-llm-calls/evals/contextual-precision.eval.yaml --target gemini-llm +agentv eval examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml --target gemini-llm # Run contextual recall evaluation -agentv eval examples/features/code-grader-with-llm-calls/evals/contextual-recall.eval.yaml --target gemini-llm +agentv eval examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml --target gemini-llm ``` ### Expected Results diff --git a/examples/features/code-grader-with-llm-calls/bun.lock b/examples/features/script-grader-with-llm-calls/bun.lock similarity index 89% rename from examples/features/code-grader-with-llm-calls/bun.lock rename to examples/features/script-grader-with-llm-calls/bun.lock index ca3b9d00d..80b50f2db 100644 --- a/examples/features/code-grader-with-llm-calls/bun.lock +++ b/examples/features/script-grader-with-llm-calls/bun.lock @@ -3,7 +3,7 @@ "configVersion": 1, "workspaces": { "": { - "name": "code-grader-with-llm-calls-example", + "name": "script-grader-with-llm-calls-example", "dependencies": { "@agentv/sdk": "file:../../../packages/sdk", }, diff --git a/examples/features/code-grader-with-llm-calls/evals/contextual-precision.eval.baseline.jsonl b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.baseline.jsonl similarity index 100% rename from examples/features/code-grader-with-llm-calls/evals/contextual-precision.eval.baseline.jsonl rename to examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.baseline.jsonl diff --git a/examples/features/code-grader-with-llm-calls/evals/contextual-precision.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml similarity index 100% rename from examples/features/code-grader-with-llm-calls/evals/contextual-precision.eval.yaml rename to examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml diff --git a/examples/features/code-grader-with-llm-calls/evals/contextual-recall.eval.baseline.jsonl b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.baseline.jsonl similarity index 100% rename from examples/features/code-grader-with-llm-calls/evals/contextual-recall.eval.baseline.jsonl rename to examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.baseline.jsonl diff --git a/examples/features/code-grader-with-llm-calls/evals/contextual-recall.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml similarity index 100% rename from examples/features/code-grader-with-llm-calls/evals/contextual-recall.eval.yaml rename to examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml diff --git a/examples/features/code-grader-with-llm-calls/package.json b/examples/features/script-grader-with-llm-calls/package.json similarity index 69% rename from examples/features/code-grader-with-llm-calls/package.json rename to examples/features/script-grader-with-llm-calls/package.json index 4ee69c6d2..d758a38e7 100644 --- a/examples/features/code-grader-with-llm-calls/package.json +++ b/examples/features/script-grader-with-llm-calls/package.json @@ -1,5 +1,5 @@ { - "name": "code-grader-with-llm-calls-example", + "name": "script-grader-with-llm-calls-example", "type": "module", "private": true, "dependencies": { diff --git a/examples/features/code-grader-with-llm-calls/scripts/contextual-precision.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts similarity index 97% rename from examples/features/code-grader-with-llm-calls/scripts/contextual-precision.ts rename to examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts index ec9098541..b69f68e2c 100644 --- a/examples/features/code-grader-with-llm-calls/scripts/contextual-precision.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts @@ -15,7 +15,7 @@ * Requires `target: { max_calls: N }` in the grader YAML config, * where N >= number of retrieval context nodes to evaluate. */ -import { createTargetClient, defineCodeGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; import { extractRetrievalContext } from './utils.js'; interface RelevanceResult { @@ -42,7 +42,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(async (input) => { +export default defineScriptGrader(async (input) => { const { input: inputMessages, criteria, expectedOutput } = input; const inputText = getMessageText(inputMessages, 'user'); diff --git a/examples/features/code-grader-with-llm-calls/scripts/contextual-recall.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts similarity index 98% rename from examples/features/code-grader-with-llm-calls/scripts/contextual-recall.ts rename to examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts index d01a4c011..013a317d4 100644 --- a/examples/features/code-grader-with-llm-calls/scripts/contextual-recall.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts @@ -19,7 +19,7 @@ * Requires `target: { max_calls: N }` in the grader YAML config, * where N >= 2 (one for statement extraction + one for attribution check). */ -import { createTargetClient, defineCodeGrader } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; import { extractRetrievalContext } from './utils.js'; interface StatementExtractionResult { @@ -51,7 +51,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(async (input) => { +export default defineScriptGrader(async (input) => { const { input: inputMessages, criteria, expectedOutput } = input; const inputText = getMessageText(inputMessages, 'user'); diff --git a/examples/features/code-grader-with-llm-calls/scripts/utils.ts b/examples/features/script-grader-with-llm-calls/scripts/utils.ts similarity index 100% rename from examples/features/code-grader-with-llm-calls/scripts/utils.ts rename to examples/features/script-grader-with-llm-calls/scripts/utils.ts diff --git a/examples/features/sdk-python/README.md b/examples/features/sdk-python/README.md index a482c44cd..56e4b9a31 100644 --- a/examples/features/sdk-python/README.md +++ b/examples/features/sdk-python/README.md @@ -4,16 +4,16 @@ This example is the smallest repo-local Python helper surface for AgentV. It is intentionally scoped to two jobs: -- build Python `code-grader` scripts over the existing stdin/stdout contract +- build Python `script-grader` scripts over the existing stdin/stdout contract - author AgentV-shaped eval definitions and emit canonical YAML/JSONL It does **not** add a native Python runner. Evaluations still run through the AgentV CLI. ## Layout -- `src/agentv_py/grader.py` - canonical code-grader helper and target proxy client +- `src/agentv_py/grader.py` - canonical script-grader helper and target proxy client - `src/agentv_py/evals.py` - YAML/JSONL authoring helpers plus optional CLI invocation -- `scripts/check_expected_output.py` - example Python code-grader +- `scripts/check_expected_output.py` - example Python script-grader - `scripts/build_eval.py` - example eval definition builder - `evals/` - generated canonical AgentV YAML/JSONL - `tests/` - focused pytest coverage for wire fields and emitted shapes diff --git a/examples/features/sdk-python/scripts/check_expected_output.py b/examples/features/sdk-python/scripts/check_expected_output.py index 75134b56e..9794c826f 100644 --- a/examples/features/sdk-python/scripts/check_expected_output.py +++ b/examples/features/sdk-python/scripts/check_expected_output.py @@ -2,14 +2,14 @@ from __future__ import annotations -from agentv_py.grader import Assertion, CodeGraderContext, CodeGraderResult, define_code_grader +from agentv_py.grader import Assertion, ScriptGraderContext, ScriptGraderResult, define_script_grader -def evaluate(context: CodeGraderContext) -> CodeGraderResult: +def evaluate(context: ScriptGraderContext) -> ScriptGraderResult: expected = context.expected_output[0]["content"] if context.expected_output else "" actual = context.output or "" passed = actual.strip() == expected.strip() - return CodeGraderResult( + return ScriptGraderResult( score=1.0 if passed else 0.0, assertions=[ Assertion( @@ -21,4 +21,4 @@ def evaluate(context: CodeGraderContext) -> CodeGraderResult: if __name__ == "__main__": - define_code_grader(evaluate) + define_script_grader(evaluate) diff --git a/examples/features/sdk-python/src/agentv_py/__init__.py b/examples/features/sdk-python/src/agentv_py/__init__.py index c24b0e0d5..5aeaec85a 100644 --- a/examples/features/sdk-python/src/agentv_py/__init__.py +++ b/examples/features/sdk-python/src/agentv_py/__init__.py @@ -1,17 +1,34 @@ -"""Minimal Python helpers for AgentV code-graders and eval authoring.""" +"""Minimal Python helpers for AgentV script-graders and eval authoring.""" from .evals import EvalDefinition, EvalTest, JsonlCase, render_eval_yaml, render_jsonl, run_agentv_eval, write_eval_yaml, write_jsonl -from .grader import Assertion, CodeGraderContext, CodeGraderResult, TargetClient, define_code_grader, emit_grader_result, load_grader_input, run_code_grader +from .grader import ( + Assertion, + CodeGraderContext, + ScriptGraderResult, + ScriptGraderContext, + ScriptGraderResult, + TargetClient, + define_script_grader, + define_script_grader, + emit_grader_result, + load_grader_input, + run_script_grader, + run_script_grader, +) __all__ = [ "Assertion", + "ScriptGraderContext", + "ScriptGraderResult", "CodeGraderContext", - "CodeGraderResult", + "ScriptGraderResult", "TargetClient", - "define_code_grader", + "define_script_grader", + "define_script_grader", "emit_grader_result", "load_grader_input", - "run_code_grader", + "run_script_grader", + "run_script_grader", "EvalDefinition", "EvalTest", "JsonlCase", diff --git a/examples/features/sdk-python/src/agentv_py/grader.py b/examples/features/sdk-python/src/agentv_py/grader.py index 6ff9a72af..42bda63da 100644 --- a/examples/features/sdk-python/src/agentv_py/grader.py +++ b/examples/features/sdk-python/src/agentv_py/grader.py @@ -1,4 +1,4 @@ -"""Helpers for Python code-graders over AgentV's canonical stdin/stdout contract.""" +"""Helpers for Python script-graders over AgentV's canonical stdin/stdout contract.""" from __future__ import annotations @@ -50,7 +50,7 @@ def to_wire(self) -> dict[str, Any]: @dataclass(frozen=True) -class CodeGraderResult: +class ScriptGraderResult: score: float assertions: list[Assertion] = field(default_factory=list) details: Mapping[str, Any] | None = None @@ -67,7 +67,7 @@ def to_wire(self) -> dict[str, Any]: @dataclass -class CodeGraderContext: +class ScriptGraderContext: criteria: str expected_output: list[Any] output: str | None @@ -88,7 +88,7 @@ class CodeGraderContext: config: Mapping[str, Any] | None @classmethod - def from_wire(cls, payload: Mapping[str, Any]) -> "CodeGraderContext": + def from_wire(cls, payload: Mapping[str, Any]) -> "ScriptGraderContext": forbidden = sorted(_FORBIDDEN_WIRE_FIELDS.intersection(payload.keys())) if forbidden: names = ", ".join(forbidden) @@ -163,13 +163,13 @@ def optional_string(name: str) -> str | None: ) -def load_grader_input(stdin_text: str | None = None) -> CodeGraderContext: +def load_grader_input(stdin_text: str | None = None) -> ScriptGraderContext: raw_text = stdin_text if stdin_text is not None else sys.stdin.read() payload = json.loads(raw_text) - return CodeGraderContext.from_wire(_require_mapping(payload, "stdin payload")) + return ScriptGraderContext.from_wire(_require_mapping(payload, "stdin payload")) -def emit_grader_result(result: CodeGraderResult) -> None: +def emit_grader_result(result: ScriptGraderResult) -> None: sys.stdout.write(f"{json.dumps(result.to_wire(), indent=2)}\n") @@ -240,17 +240,17 @@ def get_info(self) -> Any: return self._request("GET", "/info") -CodeGraderHandler = Callable[[CodeGraderContext], CodeGraderResult] +ScriptGraderHandler = Callable[[ScriptGraderContext], ScriptGraderResult] -def run_code_grader(handler: CodeGraderHandler, stdin_text: str | None = None) -> int: +def run_script_grader(handler: ScriptGraderHandler, stdin_text: str | None = None) -> int: try: context = load_grader_input(stdin_text=stdin_text) emit_grader_result(handler(context)) return 0 except Exception as error: emit_grader_result( - CodeGraderResult( + ScriptGraderResult( score=0.0, assertions=[Assertion(text=f"Evaluation failed: {error}", passed=False)], ) @@ -258,5 +258,12 @@ def run_code_grader(handler: CodeGraderHandler, stdin_text: str | None = None) - return 1 -def define_code_grader(handler: CodeGraderHandler) -> None: - raise SystemExit(run_code_grader(handler)) +def define_script_grader(handler: ScriptGraderHandler) -> None: + raise SystemExit(run_script_grader(handler)) + + +CodeGraderContext = ScriptGraderContext +ScriptGraderResult = ScriptGraderResult +CodeGraderHandler = ScriptGraderHandler +run_script_grader = run_script_grader +define_script_grader = define_script_grader diff --git a/examples/features/sdk-python/tests/test_grader.py b/examples/features/sdk-python/tests/test_grader.py index adf2ea64b..f8f074f20 100644 --- a/examples/features/sdk-python/tests/test_grader.py +++ b/examples/features/sdk-python/tests/test_grader.py @@ -5,7 +5,7 @@ import pytest -from agentv_py.grader import Assertion, CodeGraderContext, CodeGraderResult, load_grader_input, run_code_grader +from agentv_py.grader import Assertion, ScriptGraderContext, ScriptGraderResult, load_grader_input, run_script_grader def canonical_payload() -> dict: @@ -44,7 +44,7 @@ def test_load_grader_input_rejects_deprecated_wire_aliases() -> None: payload["output_text"] = payload["output"] with pytest.raises(ValueError, match="Deprecated wire fields"): - CodeGraderContext.from_wire(payload) + ScriptGraderContext.from_wire(payload) def test_load_grader_input_reads_output_path(tmp_path: Path) -> None: @@ -59,15 +59,15 @@ def test_load_grader_input_reads_output_path(tmp_path: Path) -> None: assert context.output == "loaded from file" -def test_run_code_grader_emits_canonical_result(capsys: pytest.CaptureFixture[str]) -> None: - def handler(_: CodeGraderContext) -> CodeGraderResult: - return CodeGraderResult( +def test_run_script_grader_emits_canonical_result(capsys: pytest.CaptureFixture[str]) -> None: + def handler(_: ScriptGraderContext) -> ScriptGraderResult: + return ScriptGraderResult( score=1.0, assertions=[Assertion(text="Exact match", passed=True)], details={"source": "pytest"}, ) - exit_code = run_code_grader(handler, stdin_text=json.dumps(canonical_payload())) + exit_code = run_script_grader(handler, stdin_text=json.dumps(canonical_payload())) assert exit_code == 0 emitted = json.loads(capsys.readouterr().out) diff --git a/examples/features/tool-evaluation-plugins/README.md b/examples/features/tool-evaluation-plugins/README.md index c9ab15309..f4c040d06 100644 --- a/examples/features/tool-evaluation-plugins/README.md +++ b/examples/features/tool-evaluation-plugins/README.md @@ -1,6 +1,6 @@ # Tool-Call F1 Scoring -Code grader plugins that compute **F1 scores** over tool calls, comparing expected tools against actual agent behavior. +script grader plugins that compute **F1 scores** over tool calls, comparing expected tools against actual agent behavior. ## Graders diff --git a/examples/features/tool-evaluation-plugins/evals/suite.yaml b/examples/features/tool-evaluation-plugins/evals/suite.yaml index 1e7b7cb75..6e1ad5408 100644 --- a/examples/features/tool-evaluation-plugins/evals/suite.yaml +++ b/examples/features/tool-evaluation-plugins/evals/suite.yaml @@ -1,6 +1,6 @@ # Tool-Call F1 Scoring Example # -# Demonstrates using code_grader plugins to compute F1 scores over tool calls. +# Demonstrates using script_grader plugins to compute F1 scores over tool calls. # The graders compare expected tools (from grader config) against actual # tool calls in the agent's output messages. # diff --git a/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts b/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts index c81af75c2..eb3967356 100644 --- a/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts +++ b/examples/features/tool-evaluation-plugins/graders/tool-args-f1.ts @@ -22,7 +22,7 @@ * args: { query: "weather" } * - tool: fetch */ -import { type CodeGraderInput, defineCodeGrader } from '@agentv/sdk'; +import { type ScriptGraderInput, defineScriptGrader } from '@agentv/sdk'; interface ExpectedTool { tool: string; @@ -34,7 +34,7 @@ interface ActualCall { input: Record; } -function extractActualCalls(input: CodeGraderInput): ActualCall[] { +function extractActualCalls(input: ScriptGraderInput): ActualCall[] { const calls: ActualCall[] = []; for (const msg of input.output ?? []) { if (msg.role === 'assistant' && msg.toolCalls) { @@ -62,7 +62,7 @@ function argsMatch(expected: Record, actual: Record { +export default defineScriptGrader(({ output, config, ...rest }) => { const rawExpected = config?.expectedTools ?? config?.expected_tools; if (!rawExpected || !Array.isArray(rawExpected) || rawExpected.length === 0) { return { @@ -80,7 +80,7 @@ export default defineCodeGrader(({ output, config, ...rest }) => { typeof e === 'string' ? { tool: e } : (e as ExpectedTool), ); - const input: CodeGraderInput = { output, config, ...rest }; + const input: ScriptGraderInput = { output, config, ...rest }; const actualCalls = extractActualCalls(input); // Greedy matching: for each expected tool, find first unmatched actual call diff --git a/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts b/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts index 5cc267632..da7e258ce 100644 --- a/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts +++ b/examples/features/tool-evaluation-plugins/graders/tool-call-f1.ts @@ -20,9 +20,9 @@ * command: ["bun", "run", "../graders/tool-call-f1.ts"] * expected_tools: ["search", "fetch"] */ -import { type CodeGraderInput, defineCodeGrader } from '@agentv/sdk'; +import { type ScriptGraderInput, defineScriptGrader } from '@agentv/sdk'; -function extractActualTools(input: CodeGraderInput): string[] { +function extractActualTools(input: ScriptGraderInput): string[] { const tools: string[] = []; for (const msg of input.output ?? []) { if (msg.role === 'assistant' && msg.toolCalls) { @@ -34,7 +34,7 @@ function extractActualTools(input: CodeGraderInput): string[] { return tools; } -export default defineCodeGrader(({ output, config, ...rest }) => { +export default defineScriptGrader(({ output, config, ...rest }) => { const expectedTools: string[] = (config?.expectedTools as string[]) ?? (config?.expected_tools as string[]) ?? []; @@ -50,7 +50,7 @@ export default defineCodeGrader(({ output, config, ...rest }) => { }; } - const input: CodeGraderInput = { output, config, ...rest }; + const input: ScriptGraderInput = { output, config, ...rest }; const actualTools = extractActualTools(input); const actualSet = new Set(actualTools); diff --git a/examples/features/trace-evaluation/README.md b/examples/features/trace-evaluation/README.md index 01e2c94bc..f4ea4d499 100644 --- a/examples/features/trace-evaluation/README.md +++ b/examples/features/trace-evaluation/README.md @@ -1,6 +1,6 @@ # Trace-Based Evaluation -Demonstrates how to evaluate agent internals — LLM call counts, tool executions, errors, and step durations — using code graders that inspect `context.trace`. +Demonstrates how to evaluate agent internals — LLM call counts, tool executions, errors, and step durations — using script graders that inspect `context.trace`. ## Graders @@ -12,7 +12,7 @@ Demonstrates how to evaluate agent internals — LLM call counts, tool execution ## Available Trace Data -Code graders receive `trace` with these fields: +script graders receive `trace` with these fields: ```typescript interface TraceSummary { diff --git a/examples/features/trace-evaluation/evals/suite.yaml b/examples/features/trace-evaluation/evals/suite.yaml index 086faa71b..072266de6 100644 --- a/examples/features/trace-evaluation/evals/suite.yaml +++ b/examples/features/trace-evaluation/evals/suite.yaml @@ -1,12 +1,12 @@ # Trace-Based Evaluation Examples # # Demonstrates how to evaluate agent internals (LLM calls, tool executions, -# errors, and durations) using code graders that inspect trace. +# errors, and durations) using script graders that inspect trace. # # Validate with: # bun agentv validate examples/features/trace-evaluation/evals/suite.yaml -description: Trace-based evaluation of agent internals using code graders +description: Trace-based evaluation of agent internals using script graders target: llm diff --git a/examples/features/trace-evaluation/graders/error-spans.ts b/examples/features/trace-evaluation/graders/error-spans.ts index eaac72d68..25a89724e 100644 --- a/examples/features/trace-evaluation/graders/error-spans.ts +++ b/examples/features/trace-evaluation/graders/error-spans.ts @@ -5,9 +5,9 @@ * Detects errors in agent traces by inspecting trace.errorCount * and optionally checking for specific tool failures. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ trace, config }) => { +export default defineScriptGrader(({ trace, config }) => { if (!trace) { return { score: 0, diff --git a/examples/features/trace-evaluation/graders/span-count.ts b/examples/features/trace-evaluation/graders/span-count.ts index 3458aff96..83f939314 100644 --- a/examples/features/trace-evaluation/graders/span-count.ts +++ b/examples/features/trace-evaluation/graders/span-count.ts @@ -5,14 +5,14 @@ * Validates that the number of LLM calls and tool executions stays * within configurable thresholds using trace data. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; const DEFAULTS = { maxLlmCalls: 10, maxToolCalls: 15, }; -export default defineCodeGrader(({ trace, config }) => { +export default defineScriptGrader(({ trace, config }) => { if (!trace) { return { score: 0, diff --git a/examples/features/trace-evaluation/graders/span-duration.ts b/examples/features/trace-evaluation/graders/span-duration.ts index c5304a8a1..471d04aec 100644 --- a/examples/features/trace-evaluation/graders/span-duration.ts +++ b/examples/features/trace-evaluation/graders/span-duration.ts @@ -5,11 +5,11 @@ * Validates that no individual tool execution exceeds a time threshold * using trace.toolDurations data. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; const DEFAULT_MAX_SPAN_MS = 5000; -export default defineCodeGrader(({ trace, config, durationMs }) => { +export default defineScriptGrader(({ trace, config, durationMs }) => { if (!trace) { return { score: 0, diff --git a/examples/features/trial-output-consistency/graders/trial-consistency.ts b/examples/features/trial-output-consistency/graders/trial-consistency.ts index 0922f0a0b..26b86c0a6 100644 --- a/examples/features/trial-output-consistency/graders/trial-consistency.ts +++ b/examples/features/trial-output-consistency/graders/trial-consistency.ts @@ -15,7 +15,7 @@ * 1 trial → score 1.0 (perfect consistency by definition) * 2+ trials → average pairwise cosine similarity */ -import { createTargetClient, defineCodeGrader, z } from '@agentv/sdk'; +import { createTargetClient, defineScriptGrader, z } from '@agentv/sdk'; const ConfigSchema = z.object({ trialOutputs: z.array(z.string()), @@ -110,7 +110,7 @@ function averagePairwiseSimilarity(vectors: number[][]): number { // ── Grader ─────────────────────────────────────────────────────────────── -export default defineCodeGrader(async (input) => { +export default defineScriptGrader(async (input) => { const parsed = ConfigSchema.safeParse(input.config ?? {}); if (!parsed.success) { return { diff --git a/examples/features/vitest-workspace-grader/README.md b/examples/features/vitest-workspace-grader/README.md index fc78df72d..3bd600b9d 100644 --- a/examples/features/vitest-workspace-grader/README.md +++ b/examples/features/vitest-workspace-grader/README.md @@ -43,4 +43,4 @@ assert: AgentV infers the built-in Vitest adapter for `*.test.ts`, `*.spec.ts`, and Vercel-style `EVAL.ts` verifier files. The local example uses a source-relative CLI path so it can run before the next AgentV package release. In a normal project, use the installed `agentv` binary form above. -Use lower-level `defineCodeGrader` scripts when the grader needs custom scoring, multi-stage setup, external commands beyond a test runner, or structured `details` that do not map cleanly to individual test outcomes. +Use lower-level `defineScriptGrader` scripts when the grader needs custom scoring, multi-stage setup, external commands beyond a test runner, or structured `details` that do not map cleanly to individual test outcomes. diff --git a/examples/features/workspace-artifact/evals/suite.yaml b/examples/features/workspace-artifact/evals/suite.yaml index 98c0b8194..1dac56e0d 100644 --- a/examples/features/workspace-artifact/evals/suite.yaml +++ b/examples/features/workspace-artifact/evals/suite.yaml @@ -8,7 +8,7 @@ # directly into workspace_path (the temp workspace created from the # template). AgentV takes a baseline snapshot before the agent runs and # diffs it afterwards, populating file_changes with the new CSV content. -# A code grader then checks the CSV is present via {{file_changes}}. +# A script grader then checks the CSV is present via {{file_changes}}. # # RED (before fix): Without workspace configured, agents like Copilot that # save artifacts to their session-state path can't be evaluated because diff --git a/examples/features/workspace-artifact/scripts/check-csv-artifact.ts b/examples/features/workspace-artifact/scripts/check-csv-artifact.ts index 7836489ee..c06f39b41 100644 --- a/examples/features/workspace-artifact/scripts/check-csv-artifact.ts +++ b/examples/features/workspace-artifact/scripts/check-csv-artifact.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Code grader: checks that file_changes contains outputs/report.csv + * script grader: checks that file_changes contains outputs/report.csv * with a header row and at least one data row. * * This grader is intentionally self-contained — no LLM required. diff --git a/examples/red-team/archetypes/coding-agent/README.md b/examples/red-team/archetypes/coding-agent/README.md index dbf7cdf37..68f142887 100644 --- a/examples/red-team/archetypes/coding-agent/README.md +++ b/examples/red-team/archetypes/coding-agent/README.md @@ -48,7 +48,7 @@ runtime. Tool names referenced (you can rename to match your own agent): - **`fixtures/injected-readme.md`** — a benign-looking README with an HTML comment carrying an injection payload, used by `readme-issue-url-injection`. - **`fixtures/validators/check_imports_are_real.py`** — a tiny stdlib-only - Python script used by a `code-grader` assertion in + Python script used by a `script-grader` assertion in `supply-chain-slopsquatting` to flag suggested imports that are not real PyPI packages. diff --git a/examples/showcase/README.md b/examples/showcase/README.md index c5f7f5d20..827c6e0d2 100644 --- a/examples/showcase/README.md +++ b/examples/showcase/README.md @@ -27,7 +27,7 @@ End-to-end real-world evaluation scenarios. Each example is runnable and demonst | Example | Description | |---------|-------------| -| [tool-evaluation-plugins](tool-evaluation-plugins/) | Tool selection correctness, efficiency scoring, and pairwise comparison as code-grader plugins — includes a decision table for when to use plugins vs the built-in `tool-trajectory` grader | +| [tool-evaluation-plugins](tool-evaluation-plugins/) | Tool selection correctness, efficiency scoring, and pairwise comparison as script-grader plugins — includes a decision table for when to use plugins vs the built-in `tool-trajectory` grader | | [trace-evaluation](trace-evaluation/) | Replay-first coding-agent trace fixtures with target substitution, fresh deterministic graders, and an imported Codex transcript fixture | --- diff --git a/examples/showcase/bug-fix-benchmark/README.md b/examples/showcase/bug-fix-benchmark/README.md index 99f45d9f3..34e540e88 100644 --- a/examples/showcase/bug-fix-benchmark/README.md +++ b/examples/showcase/bug-fix-benchmark/README.md @@ -91,7 +91,7 @@ tests: - id: my-bug-fix input: | Fix the bug: - assertions: + assert: - type: contains value: "" - "The fix correctly addresses the root cause" diff --git a/examples/showcase/cross-repo-sync/README.md b/examples/showcase/cross-repo-sync/README.md index ad40ede22..6d48bfc5b 100644 --- a/examples/showcase/cross-repo-sync/README.md +++ b/examples/showcase/cross-repo-sync/README.md @@ -42,7 +42,7 @@ bun agentv eval ./evals/suite.yaml ├── scripts/ │ ├── setup.ts # before_each: clone repo │ ├── reset.ts # after_each: git reset -│ └── validate-sync.ts # Code grader +│ └── validate-sync.ts # script grader ├── .agentv/ │ └── targets.yaml # Mock CLI agent └── package.json diff --git a/examples/showcase/cross-repo-sync/evals/suite.yaml b/examples/showcase/cross-repo-sync/evals/suite.yaml index de1cb7f98..28e2cc53d 100644 --- a/examples/showcase/cross-repo-sync/evals/suite.yaml +++ b/examples/showcase/cross-repo-sync/evals/suite.yaml @@ -27,7 +27,7 @@ tests: content: | agentv just merged eval spec v2 (PR #262). Update the agentevals spec docs to reflect: 4 new deterministic assertion types, required - gates, assertions field at test/suite level, tests-as-string-path. + gates, assert field at test/suite level, tests-as-string-path. assert: - metric: sync-check type: script diff --git a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts index be752dfa5..cc9f82c85 100644 --- a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts +++ b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts @@ -1,19 +1,19 @@ /** - * Code grader for cross-repo sync validation. + * script grader for cross-repo sync validation. * * Compares the agent's fileChanges against the ground truth diff: * - File-level overlap: which expected files were modified * - Keyword matching: key terms that should appear in modifications * - * Pass-through config (from assertions block in YAML): + * Pass-through config (from assert block in YAML): * - expected_files_modified: string[] — paths that should appear in fileChanges * - expected_keywords: string[] — terms that should appear in the diff * - ground_truth: string — path to the ground truth diff file (from metadata) */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -defineCodeGrader(({ fileChanges, config }) => { +defineScriptGrader(({ fileChanges, config }) => { const assertions: Array<{ text: string; passed: boolean }> = []; // Config keys are camelCased by the SDK runtime (expected_files_modified → expectedFilesModified) diff --git a/examples/showcase/cw-incident-triage/evals/validate_output.py b/examples/showcase/cw-incident-triage/evals/validate_output.py index bb9fbcb3e..e6bdd27a4 100644 --- a/examples/showcase/cw-incident-triage/evals/validate_output.py +++ b/examples/showcase/cw-incident-triage/evals/validate_output.py @@ -68,7 +68,7 @@ def validate_json_format(candidate_answer: str, required_keys: list[str]) -> dic def main(): - """Main entry point for AgentV code grader.""" + """Main entry point for AgentV script grader.""" # AgentV passes eval data via stdin as JSON try: eval_data = json.load(sys.stdin) diff --git a/examples/showcase/export-screening/README.md b/examples/showcase/export-screening/README.md index 7e5c31cfc..0087b4296 100644 --- a/examples/showcase/export-screening/README.md +++ b/examples/showcase/export-screening/README.md @@ -14,7 +14,7 @@ Trade compliance teams screen shipments to identify potential dual-use goods req 1. **Multi-class classification** (Low/Medium/High) 2. **Structured JSON output** with reasoning -3. **Code grader** for format validation and accuracy checking +3. **script grader** for format validation and accuracy checking 4. **Wrapper-based metrics** (confusion matrix + precision/recall/F1 + policy-weighted overall) 5. **Multi-sample CI gating** — run eval N times, aggregate results, and gate on aggregated metrics @@ -83,7 +83,7 @@ Each case contains: - **Expected output**: Expert risk assessment (`riskLevel: High|Medium|Low`) - **Outcome description**: Explanation for human reviewers -### 2. Code Grader (`validate_risk_output.ts`) +### 2. script grader (`validate_risk_output.ts`) The grader: 1. Validates JSON format and required fields diff --git a/examples/showcase/export-screening/evals/validate_risk_output.ts b/examples/showcase/export-screening/evals/validate_risk_output.ts index 75433f3b4..f77b005b5 100644 --- a/examples/showcase/export-screening/evals/validate_risk_output.ts +++ b/examples/showcase/export-screening/evals/validate_risk_output.ts @@ -7,7 +7,7 @@ * * Returns structured output that enables post-processing for metrics. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; const VALID_RISK_LEVELS = new Set(['High', 'Medium', 'Low']); const REQUIRED_KEYS = ['riskLevel', 'reasoning']; @@ -78,7 +78,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const outputText = getMessageText(output ?? []); const assertions: Array<{ text: string; passed: boolean; evidence?: string }> = []; diff --git a/examples/showcase/grader-conformance/conformance-check.ts b/examples/showcase/grader-conformance/conformance-check.ts index b81b06c48..e5f301b0d 100644 --- a/examples/showcase/grader-conformance/conformance-check.ts +++ b/examples/showcase/grader-conformance/conformance-check.ts @@ -3,7 +3,7 @@ * Grader Conformance Harness * * Runs an grader N times per fixture and validates: - * - Compatibility: output matches CodeGraderResult schema (score, assertions) + * - Compatibility: output matches ScriptGraderResult schema (score, assertions) * - Consistency: flip-rate, agreement, and variance meet thresholds * * Usage: @@ -98,8 +98,8 @@ const maxFlipRate = Number.parseFloat(values['max-flip-rate'] ?? '0'); // ── Grader invocation ──────────────────────────────────────────────── -function buildCodeGraderInput(fixture: Fixture): string { - // Build a minimal CodeGraderInput in the snake_case wire format +function buildScriptGraderInput(fixture: Fixture): string { + // Build a minimal ScriptGraderInput in the snake_case wire format return JSON.stringify({ criteria: fixture.criteria, output: fixture.answer, @@ -234,7 +234,7 @@ async function main(): Promise { let allConsistent = true; for (const fixture of fixtures) { - const input = buildCodeGraderInput(fixture); + const input = buildScriptGraderInput(fixture); const scores: number[] = []; const errors: string[] = []; let compatible = true; diff --git a/examples/showcase/grader-conformance/graders/keyword-grader.ts b/examples/showcase/grader-conformance/graders/keyword-grader.ts index 7d5d76d5c..a2a840113 100644 --- a/examples/showcase/grader-conformance/graders/keyword-grader.ts +++ b/examples/showcase/grader-conformance/graders/keyword-grader.ts @@ -6,7 +6,7 @@ * appear in the candidate output. Produces stable scores for unambiguous * cases and variable scores for partial matches. */ -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; function getMessageText( messages: readonly { role: string; content?: unknown }[], @@ -27,7 +27,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ output, expectedOutput, criteria }) => { +export default defineScriptGrader(({ output, expectedOutput, criteria }) => { const outputText = getMessageText(output ?? []); const expectedOutputText = getMessageText(expectedOutput); const candidate = (outputText ?? '').toLowerCase().trim(); diff --git a/examples/showcase/multi-model-benchmark/README.md b/examples/showcase/multi-model-benchmark/README.md index a12f54ed5..2466c619e 100644 --- a/examples/showcase/multi-model-benchmark/README.md +++ b/examples/showcase/multi-model-benchmark/README.md @@ -117,7 +117,7 @@ agentv eval evals/benchmark.eval.yaml --target gemini-llm Three LLM graders score each response. Weights control their contribution to the aggregate score: ```yaml -assertions: +assert: - name: accuracy weight: 3.0 # Most important — factual correctness - name: completeness diff --git a/examples/showcase/psychotherapy/evals/validate_output.py b/examples/showcase/psychotherapy/evals/validate_output.py index 861107258..42ce828fb 100644 --- a/examples/showcase/psychotherapy/evals/validate_output.py +++ b/examples/showcase/psychotherapy/evals/validate_output.py @@ -216,7 +216,7 @@ def validate_routing_schema(parsed: dict[str, Any]) -> list[str]: def main(): - """Main entry point for AgentV code grader.""" + """Main entry point for AgentV script grader.""" # AgentV passes eval data via stdin as JSON try: eval_data = json.load(sys.stdin) diff --git a/examples/showcase/tool-evaluation-plugins/README.md b/examples/showcase/tool-evaluation-plugins/README.md index aed000d0f..36752b1cc 100644 --- a/examples/showcase/tool-evaluation-plugins/README.md +++ b/examples/showcase/tool-evaluation-plugins/README.md @@ -1,6 +1,6 @@ # Tool Evaluation Plugin Patterns -This showcase demonstrates **plugin-based tool evaluation patterns** that complement AgentV's built-in `tool_trajectory` grader. These patterns are intentionally implemented as plugins (code graders) rather than built-ins because they involve domain-specific logic or semantic evaluation. +This showcase demonstrates **plugin-based tool evaluation patterns** that complement AgentV's built-in `tool_trajectory` grader. These patterns are intentionally implemented as plugins (script graders) rather than built-ins because they involve domain-specific logic or semantic evaluation. ## When to Use Plugins vs Built-ins @@ -61,7 +61,7 @@ npx agentv eval examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.ya ## Input Contract -All code graders receive a JSON object on stdin with: +All script graders receive a JSON object on stdin with: ```json { @@ -99,7 +99,7 @@ All code graders receive a JSON object on stdin with: ## Output Contract -Code graders must output JSON with: +script graders must output JSON with: ```json { diff --git a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts index 2acc32056..6b65d39d9 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/efficiency-scorer.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Tool Efficiency Scorer - Code Grader Plugin + * Tool Efficiency Scorer - script grader Plugin * * Evaluates agent efficiency based on execution metrics: * - Token usage relative to task complexity @@ -19,7 +19,7 @@ * type: script * command: ["bun", "run", "scripts/efficiency-scorer.ts"] */ -import { type TraceSummary, defineCodeGrader } from '@agentv/sdk'; +import { type TraceSummary, defineScriptGrader } from '@agentv/sdk'; // Configurable thresholds (customize for your domain) const THRESHOLDS = { @@ -77,7 +77,7 @@ function calculateExplorationRatio(trace: TraceSummary): number { return explorationCount / total; } -export default defineCodeGrader(({ trace, criteria, tokenUsage, costUsd }) => { +export default defineScriptGrader(({ trace, criteria, tokenUsage, costUsd }) => { const assertions: Array<{ text: string; passed: boolean }> = []; const scores: number[] = []; diff --git a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts index 2b5e3cbe8..1635ce958 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/pairwise-tool-compare.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Pairwise Tool Comparison - Code Grader Plugin + * Pairwise Tool Comparison - script grader Plugin * * Compares tool usage quality between two agent responses with * position bias mitigation (runs comparison twice with swapped order). @@ -17,7 +17,7 @@ * type: script * command: ["bun", "run", "scripts/pairwise-tool-compare.ts"] */ -import { type Message, defineCodeGrader } from '@agentv/sdk'; +import { type Message, defineScriptGrader } from '@agentv/sdk'; interface ToolSummary { tools: string[]; @@ -119,7 +119,7 @@ function getMessageText( return ''; } -export default defineCodeGrader((input) => { +export default defineScriptGrader((input) => { const candidate = getMessageText(input.output ?? []); const reference = getMessageText(input.expectedOutput); diff --git a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts index 68ce5c83e..ef366a57a 100644 --- a/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts +++ b/examples/showcase/tool-evaluation-plugins/scripts/tool-selection-grader.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Tool Selection Grader - Code Grader Plugin + * Tool Selection Grader - script grader Plugin * * Evaluates whether the agent selected the RIGHT tools for the task. * This is a semantic evaluation that requires understanding task requirements @@ -17,7 +17,7 @@ * type: script * command: ["bun", "run", "scripts/tool-selection-grader.ts"] */ -import { type Message, defineCodeGrader } from '@agentv/sdk'; +import { type Message, defineScriptGrader } from '@agentv/sdk'; interface ExtractedToolCall { tool: string; @@ -68,7 +68,7 @@ function getMessageText( return ''; } -export default defineCodeGrader(({ input, criteria, output }) => { +export default defineScriptGrader(({ input, criteria, output }) => { const inputText = getMessageText(input, 'user'); const assertions: Array<{ text: string; passed: boolean }> = []; diff --git a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml index 0ef7009c2..6f30ce475 100644 --- a/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml +++ b/examples/showcase/tool-evaluation-plugins/tool-eval-demo.eval.yaml @@ -1,5 +1,5 @@ # Tool Evaluation Plugins Demo -# Demonstrates plugin-based (code grader) tool evaluation patterns +# Demonstrates plugin-based (script grader) tool evaluation patterns # # These patterns complement the built-in tool_trajectory grader with # semantic evaluation capabilities that require domain-specific logic. diff --git a/examples/showcase/trace-evaluation/README.md b/examples/showcase/trace-evaluation/README.md index 1fae2bccb..dc436f2e6 100644 --- a/examples/showcase/trace-evaluation/README.md +++ b/examples/showcase/trace-evaluation/README.md @@ -67,7 +67,7 @@ bun examples/showcase/trace-evaluation/scripts/prove-replay.ts The proof script runs the replay eval with common LLM API keys blanked. It then verifies: - the result target is `replay_coding_agent`, -- the proof code grader ran once per test, +- the proof script grader ran once per test, - replayed target metrics are preserved, - deterministic graders produced fresh scores for `tool-trajectory`, `execution-metrics`, `recovery-check`, and `replay-proof`. diff --git a/examples/showcase/trace-evaluation/graders/recovery-check.ts b/examples/showcase/trace-evaluation/graders/recovery-check.ts index 158c8e19a..104281988 100644 --- a/examples/showcase/trace-evaluation/graders/recovery-check.ts +++ b/examples/showcase/trace-evaluation/graders/recovery-check.ts @@ -1,5 +1,5 @@ #!/usr/bin/env bun -import { type Message, type ToolCall, defineCodeGrader } from '@agentv/sdk'; +import { type Message, type ToolCall, defineScriptGrader } from '@agentv/sdk'; function allToolCalls(output: readonly Message[] | null | undefined): ToolCall[] { return (output ?? []).flatMap((message) => [...(message.toolCalls ?? [])]); @@ -23,7 +23,7 @@ function assistantText(output: readonly Message[] | null | undefined): string { .toLowerCase(); } -export default defineCodeGrader(({ messages }) => { +export default defineScriptGrader(({ messages }) => { const toolCalls = allToolCalls(messages); const text = assistantText(messages); diff --git a/examples/showcase/trace-evaluation/graders/replay-proof.ts b/examples/showcase/trace-evaluation/graders/replay-proof.ts index c210b57c4..901236f09 100644 --- a/examples/showcase/trace-evaluation/graders/replay-proof.ts +++ b/examples/showcase/trace-evaluation/graders/replay-proof.ts @@ -1,65 +1,67 @@ #!/usr/bin/env bun import { appendFileSync } from 'node:fs'; -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ trace, tokenUsage, costUsd, durationMs, messages, config }) => { - const proofLog = process.env.AGENTV_TRACE_SHOWCASE_PROOF_LOG; - const testId = (process.env.AGENTV_EVAL_TEST_ID ?? process.env.AGENTV_TEST_ID) || 'unknown'; - const nestedConfig = config?.config as Record | undefined; - const requireMetrics = - (config?.requireMetrics as boolean | undefined) ?? - (config?.require_metrics as boolean | undefined) ?? - (nestedConfig?.requireMetrics as boolean | undefined) ?? - (nestedConfig?.require_metrics as boolean | undefined) ?? - true; +export default defineScriptGrader( + ({ trace, tokenUsage, costUsd, durationMs, messages, config }) => { + const proofLog = process.env.AGENTV_TRACE_SHOWCASE_PROOF_LOG; + const testId = (process.env.AGENTV_EVAL_TEST_ID ?? process.env.AGENTV_TEST_ID) || 'unknown'; + const nestedConfig = config?.config as Record | undefined; + const requireMetrics = + (config?.requireMetrics as boolean | undefined) ?? + (config?.require_metrics as boolean | undefined) ?? + (nestedConfig?.requireMetrics as boolean | undefined) ?? + (nestedConfig?.require_metrics as boolean | undefined) ?? + true; - if (proofLog) { - appendFileSync( - proofLog, - `${JSON.stringify({ - kind: 'grader_run', - grader: 'replay-proof', - test_id: testId, - event_count: trace?.eventCount ?? 0, - llm_call_count: trace?.llmCallCount ?? 0, - token_usage_present: tokenUsage !== null && tokenUsage !== undefined, - cost_usd_present: costUsd !== null && costUsd !== undefined, - duration_ms_present: durationMs !== null && durationMs !== undefined, - })}\n`, - 'utf8', - ); - } + if (proofLog) { + appendFileSync( + proofLog, + `${JSON.stringify({ + kind: 'grader_run', + grader: 'replay-proof', + test_id: testId, + event_count: trace?.eventCount ?? 0, + llm_call_count: trace?.llmCallCount ?? 0, + token_usage_present: tokenUsage !== null && tokenUsage !== undefined, + cost_usd_present: costUsd !== null && costUsd !== undefined, + duration_ms_present: durationMs !== null && durationMs !== undefined, + })}\n`, + 'utf8', + ); + } - const assistantMessages = messages?.filter((message) => message.role === 'assistant') ?? []; - const assertions = [ - { - text: `Trace has tool calls (${trace?.eventCount ?? 0})`, - passed: (trace?.eventCount ?? 0) > 0, - }, - { - text: `Trace has assistant model turns (${trace?.llmCallCount ?? 0})`, - passed: (trace?.llmCallCount ?? 0) > 0, - }, - { - text: 'Provider metrics are present on replayed output', - passed: - !requireMetrics || - (tokenUsage !== null && - tokenUsage !== undefined && - costUsd !== null && - costUsd !== undefined && - durationMs !== null && - durationMs !== undefined), - }, - { - text: `Assistant messages available (${assistantMessages.length})`, - passed: assistantMessages.length > 0, - }, - ]; + const assistantMessages = messages?.filter((message) => message.role === 'assistant') ?? []; + const assertions = [ + { + text: `Trace has tool calls (${trace?.eventCount ?? 0})`, + passed: (trace?.eventCount ?? 0) > 0, + }, + { + text: `Trace has assistant model turns (${trace?.llmCallCount ?? 0})`, + passed: (trace?.llmCallCount ?? 0) > 0, + }, + { + text: 'Provider metrics are present on replayed output', + passed: + !requireMetrics || + (tokenUsage !== null && + tokenUsage !== undefined && + costUsd !== null && + costUsd !== undefined && + durationMs !== null && + durationMs !== undefined), + }, + { + text: `Assistant messages available (${assistantMessages.length})`, + passed: assistantMessages.length > 0, + }, + ]; - const passed = assertions.filter((assertion) => assertion.passed).length; - return { - score: passed / assertions.length, - assertions, - }; -}); + const passed = assertions.filter((assertion) => assertion.passed).length; + return { + score: passed / assertions.length, + assertions, + }; + }, +); diff --git a/examples/showcase/trace-evaluation/scripts/prove-replay.ts b/examples/showcase/trace-evaluation/scripts/prove-replay.ts index 2589e39df..37f5e30b5 100644 --- a/examples/showcase/trace-evaluation/scripts/prove-replay.ts +++ b/examples/showcase/trace-evaluation/scripts/prove-replay.ts @@ -72,7 +72,7 @@ try { ), ); - const requiredScoreTypes = ['tool-trajectory', 'execution-metrics', 'code-grader']; + const requiredScoreTypes = ['tool-trajectory', 'execution-metrics', 'script-grader']; const requiredScoreNames = [ 'expected-tool-sequence', 'recovery-sequence', diff --git a/packages/core/src/evaluation/graders/code-grader.ts b/packages/core/src/evaluation/graders/code-grader.ts index f730765de..ea3c0848f 100644 --- a/packages/core/src/evaluation/graders/code-grader.ts +++ b/packages/core/src/evaluation/graders/code-grader.ts @@ -1,431 +1,11 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; - -import { execFileWithStdin, execShellWithStdin } from '../../runtime/exec.js'; -import { - DEFAULT_MAX_CALLS, - type TargetProxyUsageMetadata, - createTargetProxy, -} from '../../runtime/target-proxy.js'; -import { serializeSnakeCaseBoundaryPayload } from '../case-conversion.js'; -import { type ContentImage, isContentArray } from '../content.js'; -import type { AssertionEntry, JsonObject, TargetAccessConfig } from '../types.js'; -import { getRepoCheckoutTargets } from '../workspace/repo-checkout.js'; -import { clampScore, isNonEmptyString, parseJsonSafe, scoreToVerdict } from './scoring.js'; -import type { EvaluationContext, EvaluationScore, Grader } from './types.js'; - -/** Threshold in bytes above which output is written to a temp file instead of inlined. */ -const FILE_BACKED_OUTPUT_THRESHOLD = 50_000; - -/** Regex matching `data:;base64,` URIs. */ -const DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s; - -/** - * Convert ContentImage blocks in message arrays for script grader consumption. - * - * - Data URI images (`data:image/png;base64,...`) → decoded, written to temp file, replaced with file path. - * - Non-URI images (already a path or URL) → `source` carried through as `path`. - * - ContentText, ContentFile blocks → passed through unchanged. - * - Messages with plain string content → passed through unchanged. - * - * Returns the original array when no image blocks exist (zero-copy fast path). - */ -export async function materializeContentForGrader( - messages: readonly Record[] | null | undefined, - getWorkDir: () => Promise, -): Promise[] | null> { - if (!messages || messages.length === 0) return messages ?? null; - - // Fast path: skip if no image blocks exist - let hasAnyImage = false; - for (const msg of messages) { - if (isContentArray(msg.content)) { - for (const block of msg.content) { - if (block.type === 'image') { - hasAnyImage = true; - break; - } - } - } - if (hasAnyImage) break; - } - if (!hasAnyImage) return messages; - - let counter = 0; - const result: Record[] = []; - - for (const msg of messages) { - if (!isContentArray(msg.content)) { - result.push(msg); - continue; - } - - if (!msg.content.some((b) => b.type === 'image')) { - result.push(msg); - continue; - } - - const blocks: Record[] = []; - for (const block of msg.content) { - if (block.type !== 'image') { - blocks.push({ ...block }); - continue; - } - - const img = block as ContentImage; - const match = DATA_URI_RE.exec(img.source); - - if (match) { - const [, mediaType, base64Data] = match; - const ext = mediaType.split('/')[1] === 'jpeg' ? 'jpg' : (mediaType.split('/')[1] ?? 'bin'); - const dir = await getWorkDir(); - const filePath = join(dir, `img-${counter++}.${ext}`); - await writeFile(filePath, Buffer.from(base64Data, 'base64')); - blocks.push({ type: 'image', media_type: img.media_type, path: filePath }); - } else { - // Already a path or URL → carry through as path - blocks.push({ type: 'image', media_type: img.media_type, path: img.source }); - } - } - - result.push({ ...msg, content: blocks }); - } - - return result; -} - -export interface CodeGraderOptions { - readonly command: readonly string[]; - readonly cwd?: string; - readonly agentTimeoutMs?: number; - /** Pass-through configuration from YAML (any unrecognized properties) */ - readonly config?: Record; - /** Target access config - when present, enables target invocation */ - readonly target?: TargetAccessConfig; -} - -export class CodeGrader implements Grader { - readonly kind = 'script'; - - private readonly command: readonly string[]; - private readonly cwd?: string; - private readonly agentTimeoutMs?: number; - private readonly config?: Record; - private readonly target?: TargetAccessConfig; - - constructor(options: CodeGraderOptions) { - this.command = options.command; - this.cwd = options.cwd; - this.agentTimeoutMs = options.agentTimeoutMs; - this.config = options.config; - this.target = options.target; - } - - async evaluate(context: EvaluationContext): Promise { - // Lazy temp dir for materialized image files - let imageTmpDir: string | undefined; - const getImageDir = async () => { - if (!imageTmpDir) { - imageTmpDir = await mkdtemp(join(tmpdir(), 'agentv-img-')); - } - return imageTmpDir; - }; - - const transcriptMessages = context.trace?.messages ?? context.output ?? []; - - // Materialize transcript multimodal content (data URIs → temp files, source → path) - const materializedMessages = await materializeContentForGrader( - transcriptMessages as unknown as readonly Record[] | undefined, - getImageDir, - ); - - // Determine whether to use file-backed output for large final answers - let outputForPayload: string | null = context.candidate; - let outputPath: string | undefined; - - if (outputForPayload !== null) { - const serialized = JSON.stringify(outputForPayload); - if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) { - const tmpDir = await mkdtemp(join(tmpdir(), 'agentv-grader-')); - outputPath = join(tmpDir, 'output.json'); - await writeFile(outputPath, serialized); - outputForPayload = null; - } - } - - const traceForPayload = context.trace - ? { - ...context.trace, - messages: materializedMessages ?? context.trace.messages, - } - : null; - - // Build payload (camelCase internally, converted to snake_case for graders) - const payload = { - criteria: context.evalCase.criteria, - expectedOutput: await materializeContentForGrader( - context.evalCase.expected_output as readonly Record[], - getImageDir, - ), - output: outputForPayload, - messages: materializedMessages ?? [], - outputPath, - inputFiles: context.evalCase.file_paths, - input: await materializeContentForGrader( - context.evalCase.input as readonly Record[], - getImageDir, - ), - metadata: context.evalCase.metadata ?? null, - trace: traceForPayload, - traceSummary: context.trace - ? { - eventCount: context.trace.eventCount, - toolCalls: context.trace.toolCalls, - errorCount: context.trace.errorCount, - toolDurations: context.trace.toolDurations, - llmCallCount: context.trace.llmCallCount, - } - : null, - tokenUsage: context.tokenUsage ?? null, - costUsd: context.costUsd ?? null, - durationMs: context.durationMs ?? null, - startTime: context.startTime ?? null, - endTime: context.endTime ?? null, - fileChanges: context.fileChanges ?? null, - workspacePath: context.workspacePath ?? null, - config: this.config ?? null, - }; - - const inputPayload = JSON.stringify(serializeSnakeCaseBoundaryPayload(payload), null, 2); - - // Set up target proxy if configured and grader provider is available - let proxyEnv: Record | undefined; - let proxyShutdown: (() => Promise) | undefined; - let getProxyUsage: (() => TargetProxyUsageMetadata) | undefined; - - if (this.target !== undefined && context.graderProvider) { - const maxCalls = this.target.max_calls ?? DEFAULT_MAX_CALLS; - const proxy = await createTargetProxy({ - defaultProvider: context.graderProvider, - targetResolver: context.targetResolver, - availableTargets: context.availableTargets, - maxCalls, - }); - proxyEnv = { - AGENTV_TARGET_PROXY_URL: proxy.url, - AGENTV_TARGET_PROXY_TOKEN: proxy.token, - }; - proxyShutdown = proxy.shutdown; - getProxyUsage = proxy.getUsageMetadata; - } - - // Build workspace env if workspace path is available - const workspaceEnv = context.workspacePath - ? { AGENTV_WORKSPACE_PATH: context.workspacePath } - : undefined; - - // Merge proxy and workspace env vars - const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : undefined; - - try { - let stdout: string; - let exitCode = 0; - let execStderr = ''; - if (context.dockerConfig) { - // Docker execution mode: run grader inside a container - const { DockerWorkspaceProvider } = await import('../workspace/docker-workspace.js'); - const dockerProvider = new DockerWorkspaceProvider(context.dockerConfig); - const result = await dockerProvider.runGraderInContainer({ - command: [...this.command], - stdin: inputPayload, - repoCheckouts: getRepoCheckoutTargets(context.evalCase.workspace?.repos), - }); - exitCode = result.exitCode; - stdout = result.stdout.trim(); - execStderr = result.stderr; - } else { - const result = await runScriptRaw( - this.command, - inputPayload, - this.agentTimeoutMs, - this.cwd, - env, - ); - exitCode = result.exitCode; - stdout = result.stdout.trim(); - execStderr = result.stderr; - } - // Non-zero exit with JSON stdout, or with stderr output, is treated as an error - // (script signaled failure through the protocol or wrote an error message). - // Non-zero exit with plain stdout and no stderr uses the exit-code convention — - // score 0 (fail), stdout becomes the assertion text. - const looksLikeJson = stdout.startsWith('{') || stdout.startsWith('['); - const hasStderr = execStderr.trim().length > 0; - if (exitCode !== 0 && (looksLikeJson || hasStderr)) { - const trimmedErr = formatStderr(execStderr); - throw new Error( - trimmedErr.length > 0 - ? `Code evaluator exited with code ${exitCode}: ${trimmedErr}` - : `Code evaluator exited with code ${exitCode}`, - ); - } - const rawParsed = parseJsonSafe(stdout); - // Only treat stdout as the JSON protocol if it parsed as a plain object. - // Bare JSON scalars (numbers, booleans, strings) fall through to the plain-text path. - const parsed = - rawParsed != null && typeof rawParsed === 'object' && !Array.isArray(rawParsed) - ? rawParsed - : undefined; - // Plain-text fallback: exit code is pass/fail, stdout is the assertion text. - // For numeric scores or multi-aspect results, use the JSON protocol instead. - const passed = exitCode === 0; - const assertions: AssertionEntry[] = - parsed != null && Array.isArray(parsed?.assertions) - ? parsed.assertions - .filter( - (a: unknown): a is { text: string; passed: boolean; evidence?: string } => - typeof a === 'object' && - a !== null && - typeof (a as Record).text === 'string', - ) - .map((a) => ({ - text: String(a.text), - passed: Boolean(a.passed), - ...(typeof a.evidence === 'string' ? { evidence: a.evidence } : {}), - })) - : parsed == null - ? [{ text: stdout.trim() || (passed ? 'exit 0' : `exit ${exitCode}`), passed }] - : []; - // When the script omits `score` but returns `assertions`, derive score as passing/total. - const score = - parsed != null - ? clampScore( - typeof parsed.score === 'number' - ? parsed.score - : assertions.length > 0 - ? assertions.filter((a) => a.passed).length / assertions.length - : 0, - ) - : passed - ? 1 - : 0; - // Capture optional structured details from code judge output - const details = - parsed?.details && typeof parsed.details === 'object' && !Array.isArray(parsed.details) - ? (parsed.details as JsonObject) - : undefined; - - // Build evaluator raw request with proxy metadata if used - const proxyUsage = getProxyUsage?.(); - const graderRawRequest: JsonObject = { - command: this.command, - ...(this.cwd ? { cwd: this.cwd } : {}), - ...(proxyUsage - ? { - target_proxy: { - call_count: proxyUsage.callCount, - max_calls: proxyUsage.maxCalls, - }, - } - : {}), - }; - - return { - score, - verdict: scoreToVerdict(score), - assertions, - expectedAspectCount: assertions.length || 1, - graderRawRequest, - ...(details ? { details } : {}), - tokenUsage: proxyUsage?.tokenUsage, - }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const proxyUsage = getProxyUsage?.(); - return { - score: 0, - verdict: 'fail', - assertions: [{ text: `Code evaluator failed: ${message}`, passed: false }], - expectedAspectCount: 1, - graderRawRequest: { - command: this.command, - ...(this.cwd ? { cwd: this.cwd } : {}), - ...(proxyUsage - ? { - target_proxy: { - call_count: proxyUsage.callCount, - max_calls: proxyUsage.maxCalls, - }, - } - : {}), - error: message, - }, - tokenUsage: proxyUsage?.tokenUsage, - }; - } finally { - // Always shut down the proxy when done - if (proxyShutdown) { - await proxyShutdown(); - } - // Clean up temp file for file-backed output - if (outputPath) { - await rm(dirname(outputPath), { recursive: true, force: true }).catch(() => {}); - } - // Clean up temp dir for materialized images - if (imageTmpDir) { - await rm(imageTmpDir, { recursive: true, force: true }).catch(() => {}); - } - } - } -} - -/** Run a script and return raw stdout/stderr/exitCode without throwing. */ -async function runScriptRaw( - scriptPath: readonly string[] | string, - input: string, - agentTimeoutMs?: number, - cwd?: string, - env?: Record, -): Promise<{ stdout: string; stderr: string; exitCode: number }> { - return typeof scriptPath === 'string' - ? execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) - : execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }); -} - -export async function executeScript( - scriptPath: readonly string[] | string, - input: string, - agentTimeoutMs?: number, - cwd?: string, - env?: Record, -): Promise { - const { stdout, stderr, exitCode } = await runScriptRaw( - scriptPath, - input, - agentTimeoutMs, - cwd, - env, - ); - - if (exitCode !== 0) { - const trimmedErr = formatStderr(stderr); - throw new Error( - trimmedErr.length > 0 - ? `Code evaluator exited with code ${exitCode}: ${trimmedErr}` - : `Code evaluator exited with code ${exitCode}`, - ); - } - - return stdout.trim(); -} - -function formatStderr(stderr: string): string { - const trimmed = stderr.trim(); - const maxLength = 2000; - if (trimmed.length <= maxLength) { - return trimmed; - } - const tail = trimmed.slice(-maxLength); - return `...(truncated, last ${maxLength} chars)\n${tail}`; -} +// Compatibility re-export for older imports. New internal code uses script-grader.ts. +export { + ScriptGrader, + ScriptGrader as CodeGrader, + executeScript, + materializeContentForGrader, +} from './script-grader.js'; +export type { + ScriptGraderOptions, + ScriptGraderOptions as CodeGraderOptions, +} from './script-grader.js'; diff --git a/packages/core/src/evaluation/graders/composite.ts b/packages/core/src/evaluation/graders/composite.ts index 19b68edce..5acb8a3bf 100644 --- a/packages/core/src/evaluation/graders/composite.ts +++ b/packages/core/src/evaluation/graders/composite.ts @@ -5,9 +5,9 @@ import type { CompositeGraderConfig, JsonObject, } from '../types.js'; -import { executeScript } from './code-grader.js'; import { buildOutputSchema, freeformEvaluationSchema } from './llm-grader.js'; import { clampScore, parseJsonFromText, parseJsonSafe, scoreToVerdict } from './scoring.js'; +import { executeScript } from './script-grader.js'; import type { ChildGraderResult, EvaluationContext, diff --git a/packages/core/src/evaluation/graders/index.ts b/packages/core/src/evaluation/graders/index.ts index d7a118338..979c253a5 100644 --- a/packages/core/src/evaluation/graders/index.ts +++ b/packages/core/src/evaluation/graders/index.ts @@ -22,8 +22,8 @@ export { } from './scoring.js'; // Graders -export { CodeGrader, executeScript } from './code-grader.js'; -export type { CodeGraderOptions } from './code-grader.js'; +export { ScriptGrader, executeScript } from './script-grader.js'; +export type { ScriptGraderOptions } from './script-grader.js'; export { CompositeGrader } from './composite.js'; export type { CompositeGraderOptions } from './composite.js'; diff --git a/packages/core/src/evaluation/graders/prompt-resolution.ts b/packages/core/src/evaluation/graders/prompt-resolution.ts index d7dfd9173..fa3b28c2e 100644 --- a/packages/core/src/evaluation/graders/prompt-resolution.ts +++ b/packages/core/src/evaluation/graders/prompt-resolution.ts @@ -19,7 +19,7 @@ import type { Message } from '../providers/types.js'; import { VALID_TEMPLATE_VARIABLES } from '../template-variables.js'; import type { Trace } from '../trace.js'; import type { EvalTest, PromptScriptConfig } from '../types.js'; -import { executeScript } from './code-grader.js'; +import { executeScript } from './script-grader.js'; export interface ResolveCustomPromptContext { readonly evalCase: EvalTest; diff --git a/packages/core/src/evaluation/graders/script-grader.ts b/packages/core/src/evaluation/graders/script-grader.ts new file mode 100644 index 000000000..ca2989df9 --- /dev/null +++ b/packages/core/src/evaluation/graders/script-grader.ts @@ -0,0 +1,431 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { execFileWithStdin, execShellWithStdin } from '../../runtime/exec.js'; +import { + DEFAULT_MAX_CALLS, + type TargetProxyUsageMetadata, + createTargetProxy, +} from '../../runtime/target-proxy.js'; +import { serializeSnakeCaseBoundaryPayload } from '../case-conversion.js'; +import { type ContentImage, isContentArray } from '../content.js'; +import type { AssertionEntry, JsonObject, TargetAccessConfig } from '../types.js'; +import { getRepoCheckoutTargets } from '../workspace/repo-checkout.js'; +import { clampScore, isNonEmptyString, parseJsonSafe, scoreToVerdict } from './scoring.js'; +import type { EvaluationContext, EvaluationScore, Grader } from './types.js'; + +/** Threshold in bytes above which output is written to a temp file instead of inlined. */ +const FILE_BACKED_OUTPUT_THRESHOLD = 50_000; + +/** Regex matching `data:;base64,` URIs. */ +const DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s; + +/** + * Convert ContentImage blocks in message arrays for script grader consumption. + * + * - Data URI images (`data:image/png;base64,...`) → decoded, written to temp file, replaced with file path. + * - Non-URI images (already a path or URL) → `source` carried through as `path`. + * - ContentText, ContentFile blocks → passed through unchanged. + * - Messages with plain string content → passed through unchanged. + * + * Returns the original array when no image blocks exist (zero-copy fast path). + */ +export async function materializeContentForGrader( + messages: readonly Record[] | null | undefined, + getWorkDir: () => Promise, +): Promise[] | null> { + if (!messages || messages.length === 0) return messages ?? null; + + // Fast path: skip if no image blocks exist + let hasAnyImage = false; + for (const msg of messages) { + if (isContentArray(msg.content)) { + for (const block of msg.content) { + if (block.type === 'image') { + hasAnyImage = true; + break; + } + } + } + if (hasAnyImage) break; + } + if (!hasAnyImage) return messages; + + let counter = 0; + const result: Record[] = []; + + for (const msg of messages) { + if (!isContentArray(msg.content)) { + result.push(msg); + continue; + } + + if (!msg.content.some((b) => b.type === 'image')) { + result.push(msg); + continue; + } + + const blocks: Record[] = []; + for (const block of msg.content) { + if (block.type !== 'image') { + blocks.push({ ...block }); + continue; + } + + const img = block as ContentImage; + const match = DATA_URI_RE.exec(img.source); + + if (match) { + const [, mediaType, base64Data] = match; + const ext = mediaType.split('/')[1] === 'jpeg' ? 'jpg' : (mediaType.split('/')[1] ?? 'bin'); + const dir = await getWorkDir(); + const filePath = join(dir, `img-${counter++}.${ext}`); + await writeFile(filePath, Buffer.from(base64Data, 'base64')); + blocks.push({ type: 'image', media_type: img.media_type, path: filePath }); + } else { + // Already a path or URL → carry through as path + blocks.push({ type: 'image', media_type: img.media_type, path: img.source }); + } + } + + result.push({ ...msg, content: blocks }); + } + + return result; +} + +export interface ScriptGraderOptions { + readonly command: readonly string[]; + readonly cwd?: string; + readonly agentTimeoutMs?: number; + /** Pass-through configuration from YAML (any unrecognized properties) */ + readonly config?: Record; + /** Target access config - when present, enables target invocation */ + readonly target?: TargetAccessConfig; +} + +export class ScriptGrader implements Grader { + readonly kind = 'script'; + + private readonly command: readonly string[]; + private readonly cwd?: string; + private readonly agentTimeoutMs?: number; + private readonly config?: Record; + private readonly target?: TargetAccessConfig; + + constructor(options: ScriptGraderOptions) { + this.command = options.command; + this.cwd = options.cwd; + this.agentTimeoutMs = options.agentTimeoutMs; + this.config = options.config; + this.target = options.target; + } + + async evaluate(context: EvaluationContext): Promise { + // Lazy temp dir for materialized image files + let imageTmpDir: string | undefined; + const getImageDir = async () => { + if (!imageTmpDir) { + imageTmpDir = await mkdtemp(join(tmpdir(), 'agentv-img-')); + } + return imageTmpDir; + }; + + const transcriptMessages = context.trace?.messages ?? context.output ?? []; + + // Materialize transcript multimodal content (data URIs → temp files, source → path) + const materializedMessages = await materializeContentForGrader( + transcriptMessages as unknown as readonly Record[] | undefined, + getImageDir, + ); + + // Determine whether to use file-backed output for large final answers + let outputForPayload: string | null = context.candidate; + let outputPath: string | undefined; + + if (outputForPayload !== null) { + const serialized = JSON.stringify(outputForPayload); + if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) { + const tmpDir = await mkdtemp(join(tmpdir(), 'agentv-grader-')); + outputPath = join(tmpDir, 'output.json'); + await writeFile(outputPath, serialized); + outputForPayload = null; + } + } + + const traceForPayload = context.trace + ? { + ...context.trace, + messages: materializedMessages ?? context.trace.messages, + } + : null; + + // Build payload (camelCase internally, converted to snake_case for graders) + const payload = { + criteria: context.evalCase.criteria, + expectedOutput: await materializeContentForGrader( + context.evalCase.expected_output as readonly Record[], + getImageDir, + ), + output: outputForPayload, + messages: materializedMessages ?? [], + outputPath, + inputFiles: context.evalCase.file_paths, + input: await materializeContentForGrader( + context.evalCase.input as readonly Record[], + getImageDir, + ), + metadata: context.evalCase.metadata ?? null, + trace: traceForPayload, + traceSummary: context.trace + ? { + eventCount: context.trace.eventCount, + toolCalls: context.trace.toolCalls, + errorCount: context.trace.errorCount, + toolDurations: context.trace.toolDurations, + llmCallCount: context.trace.llmCallCount, + } + : null, + tokenUsage: context.tokenUsage ?? null, + costUsd: context.costUsd ?? null, + durationMs: context.durationMs ?? null, + startTime: context.startTime ?? null, + endTime: context.endTime ?? null, + fileChanges: context.fileChanges ?? null, + workspacePath: context.workspacePath ?? null, + config: this.config ?? null, + }; + + const inputPayload = JSON.stringify(serializeSnakeCaseBoundaryPayload(payload), null, 2); + + // Set up target proxy if configured and grader provider is available + let proxyEnv: Record | undefined; + let proxyShutdown: (() => Promise) | undefined; + let getProxyUsage: (() => TargetProxyUsageMetadata) | undefined; + + if (this.target !== undefined && context.graderProvider) { + const maxCalls = this.target.max_calls ?? DEFAULT_MAX_CALLS; + const proxy = await createTargetProxy({ + defaultProvider: context.graderProvider, + targetResolver: context.targetResolver, + availableTargets: context.availableTargets, + maxCalls, + }); + proxyEnv = { + AGENTV_TARGET_PROXY_URL: proxy.url, + AGENTV_TARGET_PROXY_TOKEN: proxy.token, + }; + proxyShutdown = proxy.shutdown; + getProxyUsage = proxy.getUsageMetadata; + } + + // Build workspace env if workspace path is available + const workspaceEnv = context.workspacePath + ? { AGENTV_WORKSPACE_PATH: context.workspacePath } + : undefined; + + // Merge proxy and workspace env vars + const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : undefined; + + try { + let stdout: string; + let exitCode = 0; + let execStderr = ''; + if (context.dockerConfig) { + // Docker execution mode: run grader inside a container + const { DockerWorkspaceProvider } = await import('../workspace/docker-workspace.js'); + const dockerProvider = new DockerWorkspaceProvider(context.dockerConfig); + const result = await dockerProvider.runGraderInContainer({ + command: [...this.command], + stdin: inputPayload, + repoCheckouts: getRepoCheckoutTargets(context.evalCase.workspace?.repos), + }); + exitCode = result.exitCode; + stdout = result.stdout.trim(); + execStderr = result.stderr; + } else { + const result = await runScriptRaw( + this.command, + inputPayload, + this.agentTimeoutMs, + this.cwd, + env, + ); + exitCode = result.exitCode; + stdout = result.stdout.trim(); + execStderr = result.stderr; + } + // Non-zero exit with JSON stdout, or with stderr output, is treated as an error + // (script signaled failure through the protocol or wrote an error message). + // Non-zero exit with plain stdout and no stderr uses the exit-code convention — + // score 0 (fail), stdout becomes the assertion text. + const looksLikeJson = stdout.startsWith('{') || stdout.startsWith('['); + const hasStderr = execStderr.trim().length > 0; + if (exitCode !== 0 && (looksLikeJson || hasStderr)) { + const trimmedErr = formatStderr(execStderr); + throw new Error( + trimmedErr.length > 0 + ? `Script evaluator exited with code ${exitCode}: ${trimmedErr}` + : `Script evaluator exited with code ${exitCode}`, + ); + } + const rawParsed = parseJsonSafe(stdout); + // Only treat stdout as the JSON protocol if it parsed as a plain object. + // Bare JSON scalars (numbers, booleans, strings) fall through to the plain-text path. + const parsed = + rawParsed != null && typeof rawParsed === 'object' && !Array.isArray(rawParsed) + ? rawParsed + : undefined; + // Plain-text fallback: exit code is pass/fail, stdout is the assertion text. + // For numeric scores or multi-aspect results, use the JSON protocol instead. + const passed = exitCode === 0; + const assertions: AssertionEntry[] = + parsed != null && Array.isArray(parsed?.assertions) + ? parsed.assertions + .filter( + (a: unknown): a is { text: string; passed: boolean; evidence?: string } => + typeof a === 'object' && + a !== null && + typeof (a as Record).text === 'string', + ) + .map((a) => ({ + text: String(a.text), + passed: Boolean(a.passed), + ...(typeof a.evidence === 'string' ? { evidence: a.evidence } : {}), + })) + : parsed == null + ? [{ text: stdout.trim() || (passed ? 'exit 0' : `exit ${exitCode}`), passed }] + : []; + // When the script omits `score` but returns `assertions`, derive score as passing/total. + const score = + parsed != null + ? clampScore( + typeof parsed.score === 'number' + ? parsed.score + : assertions.length > 0 + ? assertions.filter((a) => a.passed).length / assertions.length + : 0, + ) + : passed + ? 1 + : 0; + // Capture optional structured details from code judge output + const details = + parsed?.details && typeof parsed.details === 'object' && !Array.isArray(parsed.details) + ? (parsed.details as JsonObject) + : undefined; + + // Build evaluator raw request with proxy metadata if used + const proxyUsage = getProxyUsage?.(); + const graderRawRequest: JsonObject = { + command: this.command, + ...(this.cwd ? { cwd: this.cwd } : {}), + ...(proxyUsage + ? { + target_proxy: { + call_count: proxyUsage.callCount, + max_calls: proxyUsage.maxCalls, + }, + } + : {}), + }; + + return { + score, + verdict: scoreToVerdict(score), + assertions, + expectedAspectCount: assertions.length || 1, + graderRawRequest, + ...(details ? { details } : {}), + tokenUsage: proxyUsage?.tokenUsage, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const proxyUsage = getProxyUsage?.(); + return { + score: 0, + verdict: 'fail', + assertions: [{ text: `Script evaluator failed: ${message}`, passed: false }], + expectedAspectCount: 1, + graderRawRequest: { + command: this.command, + ...(this.cwd ? { cwd: this.cwd } : {}), + ...(proxyUsage + ? { + target_proxy: { + call_count: proxyUsage.callCount, + max_calls: proxyUsage.maxCalls, + }, + } + : {}), + error: message, + }, + tokenUsage: proxyUsage?.tokenUsage, + }; + } finally { + // Always shut down the proxy when done + if (proxyShutdown) { + await proxyShutdown(); + } + // Clean up temp file for file-backed output + if (outputPath) { + await rm(dirname(outputPath), { recursive: true, force: true }).catch(() => {}); + } + // Clean up temp dir for materialized images + if (imageTmpDir) { + await rm(imageTmpDir, { recursive: true, force: true }).catch(() => {}); + } + } + } +} + +/** Run a script and return raw stdout/stderr/exitCode without throwing. */ +async function runScriptRaw( + scriptPath: readonly string[] | string, + input: string, + agentTimeoutMs?: number, + cwd?: string, + env?: Record, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + return typeof scriptPath === 'string' + ? execShellWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }) + : execFileWithStdin(scriptPath, input, { cwd, timeoutMs: agentTimeoutMs, env }); +} + +export async function executeScript( + scriptPath: readonly string[] | string, + input: string, + agentTimeoutMs?: number, + cwd?: string, + env?: Record, +): Promise { + const { stdout, stderr, exitCode } = await runScriptRaw( + scriptPath, + input, + agentTimeoutMs, + cwd, + env, + ); + + if (exitCode !== 0) { + const trimmedErr = formatStderr(stderr); + throw new Error( + trimmedErr.length > 0 + ? `Script evaluator exited with code ${exitCode}: ${trimmedErr}` + : `Script evaluator exited with code ${exitCode}`, + ); + } + + return stdout.trim(); +} + +function formatStderr(stderr: string): string { + const trimmed = stderr.trim(); + const maxLength = 2000; + if (trimmed.length <= maxLength) { + return trimmed; + } + const tail = trimmed.slice(-maxLength); + return `...(truncated, last ${maxLength} chars)\n${tail}`; +} diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 8a5553fcc..54e66c244 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -39,7 +39,7 @@ export function normalizeGraderType(type: string): string { function removedGraderReplacement(type: string): string | undefined { const replacements: Record = { - 'code-grader': 'script', + 'script-grader': 'script', 'code-judge': 'script', 'g-eval': 'llm-rubric', rubrics: 'llm-rubric with value', diff --git a/packages/core/src/evaluation/registry/assertion-discovery.ts b/packages/core/src/evaluation/registry/assertion-discovery.ts index 7cf12fb79..ec06b5aef 100644 --- a/packages/core/src/evaluation/registry/assertion-discovery.ts +++ b/packages/core/src/evaluation/registry/assertion-discovery.ts @@ -11,7 +11,7 @@ import path from 'node:path'; import fg from 'fast-glob'; -import { CodeGrader } from '../graders/code-grader.js'; +import { ScriptGrader } from '../graders/script-grader.js'; import type { GraderFactoryFn } from './grader-registry.js'; import type { GraderRegistry } from './grader-registry.js'; @@ -64,7 +64,7 @@ export async function discoverAssertions( } const factory: GraderFactoryFn = (_config, context) => { - return new CodeGrader({ + return new ScriptGrader({ command: ['bun', 'run', filePath], agentTimeoutMs: context.agentTimeoutMs, }); diff --git a/packages/core/src/evaluation/registry/builtin-graders.ts b/packages/core/src/evaluation/registry/builtin-graders.ts index 304e770ff..76bc17ed1 100644 --- a/packages/core/src/evaluation/registry/builtin-graders.ts +++ b/packages/core/src/evaluation/registry/builtin-graders.ts @@ -7,7 +7,6 @@ */ import { - CodeGrader, CompositeGrader, CostGrader, ExecutionMetricsGrader, @@ -15,6 +14,7 @@ import { type Grader, LatencyGrader, LlmGrader, + ScriptGrader, SkillTriggerGrader, TokenUsageGrader, ToolTrajectoryGrader, @@ -43,7 +43,6 @@ import { isAgentProvider } from '../providers/types.js'; import type { Provider } from '../providers/types.js'; import type { ToolTrajectoryGraderConfig } from '../trace.js'; import type { - CodeGraderConfig, CompositeGraderConfig, ContainsAllGraderConfig, ContainsAnyGraderConfig, @@ -188,9 +187,9 @@ export const llmRubricFactory: GraderFactoryFn = (config, context) => llmGraderFactory(config as LlmRubricGraderConfig, context); /** Factory for subprocess-backed script evaluators. */ -export const codeFactory: GraderFactoryFn = (config, context) => { - const c = config as ScriptGraderConfig | CodeGraderConfig; - return new CodeGrader({ +export const scriptFactory: GraderFactoryFn = (config, context) => { + const c = config as ScriptGraderConfig; + return new ScriptGrader({ command: c.command, cwd: c.resolvedCwd ?? c.cwd, agentTimeoutMs: context.agentTimeoutMs, @@ -448,7 +447,7 @@ export function createBuiltinRegistry(): GraderRegistry { registry .register('llm-grader', llmGraderFactory) .register('llm-rubric', llmRubricFactory) - .register('script', codeFactory) + .register('script', scriptFactory) .register('composite', compositeFactory) .register('tool-trajectory', toolTrajectoryFactory) .register('field-accuracy', fieldAccuracyFactory) diff --git a/packages/core/src/evaluation/registry/grader-discovery.ts b/packages/core/src/evaluation/registry/grader-discovery.ts index 9735fc707..022828b54 100644 --- a/packages/core/src/evaluation/registry/grader-discovery.ts +++ b/packages/core/src/evaluation/registry/grader-discovery.ts @@ -11,7 +11,7 @@ import path from 'node:path'; import fg from 'fast-glob'; -import { CodeGrader } from '../graders/code-grader.js'; +import { ScriptGrader } from '../graders/script-grader.js'; import type { GraderFactoryFn } from './grader-registry.js'; import type { GraderRegistry } from './grader-registry.js'; @@ -65,7 +65,7 @@ export async function discoverGraders( } const factory: GraderFactoryFn = (_config, context) => { - return new CodeGrader({ + return new ScriptGrader({ command: ['bun', 'run', filePath], agentTimeoutMs: context.agentTimeoutMs, }); diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 862d52b01..3d1c8fe4a 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -104,7 +104,7 @@ const RubricItemSchema = z.object({ // --- Type-specific evaluator schemas --- -const CodeGraderSchema = EvaluatorCommonSchema.extend({ +const ScriptGraderSchema = EvaluatorCommonSchema.extend({ type: z.literal('script'), command: z.union([z.string(), z.array(z.string())]), cwd: z.string().optional(), @@ -283,7 +283,7 @@ const PromptfooAssertionSchema = EvaluatorCommonSchema.extend({ /** Union of all grader types */ const EvaluatorSchema = z.union([ - CodeGraderSchema, + ScriptGraderSchema, LlmGraderSchema, PromptfooAssertionSchema, IncludeSchema, diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 750f9b6e1..759d8e694 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -58,7 +58,7 @@ const REMOVED_ASSERTION_TYPE_REPLACEMENTS = new Map([ ['g-eval', 'llm-rubric'], ['rubrics', 'llm-rubric with value'], ['rubric', 'llm-rubric with value'], - ['code-grader', 'script'], + ['script-grader', 'script'], ['code-judge', 'script'], ['llm-judge', 'llm-grader'], ]); diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index ba4d9991c..8f63ca457 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -12,7 +12,7 @@ import { normalizeExperimentConfig, normalizeExperimentRunOverride, } from './experiment.js'; -import { executeScript } from './graders/code-grader.js'; +import { executeScript } from './graders/script-grader.js'; import { collectResolvedInputFilePaths } from './input-message-utils.js'; import { type NunjucksFilterMap, diff --git a/packages/core/test/evaluation/execution-metrics.test.ts b/packages/core/test/evaluation/execution-metrics.test.ts index 83a01e000..6dbc6900e 100644 --- a/packages/core/test/evaluation/execution-metrics.test.ts +++ b/packages/core/test/evaluation/execution-metrics.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'bun:test'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { CodeGrader } from '../../src/evaluation/graders.js'; +import { ScriptGrader } from '../../src/evaluation/graders.js'; import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; import { type TraceComputeResult, @@ -250,7 +250,7 @@ describe('Code Grader Metrics Integration', () => { reference_answer: '', file_paths: [], criteria: 'Test outcome', - evaluator: 'code-grader', + evaluator: 'script', }; const baseTarget: ResolvedTarget = { @@ -259,12 +259,12 @@ describe('Code Grader Metrics Integration', () => { config: { response: '{}' }, }; - it('passes trace to code-grader scripts', async () => { + it('passes trace to script grader scripts', async () => { // Use external script file for cross-platform compatibility const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-trace-summary.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const trace: TraceSummary = { eventCount: 3, @@ -303,7 +303,7 @@ describe('Code Grader Metrics Integration', () => { const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-no-trace-summary.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: baseTestCase, diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index 7112974bd..3f5688137 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -3,11 +3,11 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { - CodeGrader, CostGrader, FieldAccuracyGrader, LatencyGrader, LlmGrader, + ScriptGrader, TokenUsageGrader, } from '../../src/evaluation/graders.js'; import { assembleLlmGraderPrompt } from '../../src/evaluation/graders/llm-grader-prompt.js'; @@ -1097,8 +1097,8 @@ describe('LlmGrader (llm-grader)', () => { }); }); -describe('CodeGrader', () => { - it('passes required fields to code-grader scripts', async () => { +describe('ScriptGrader', () => { + it('passes required fields to script grader scripts', async () => { const graderProvider = new StubProvider(textResponse('{}')); const evalCaseWithExpectedMessages: EvalTest = { @@ -1112,7 +1112,7 @@ describe('CodeGrader', () => { const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: evalCaseWithExpectedMessages, @@ -1139,7 +1139,7 @@ describe('CodeGrader', () => { const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader-error.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: baseTestCase, @@ -1157,13 +1157,13 @@ describe('CodeGrader', () => { expect(failedAssertions[0].text).toContain('test-error'); }); - it('works with code grader stdin/stdout contract', async () => { + it('works with script grader stdin/stdout contract', async () => { const graderProvider = new StubProvider(textResponse('Logging improvements applied')); const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['bun', 'run', join(__dirname, '../fixtures/test-define-grader.ts')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: baseTestCase, @@ -1181,13 +1181,13 @@ describe('CodeGrader', () => { expect(result.assertions.filter((a) => a.passed).length).toBeGreaterThan(0); }); - it('captures optional details from code grader output', async () => { + it('captures optional details from script grader output', async () => { const graderProvider = new StubProvider(textResponse('{}')); const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader-with-details.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: { @@ -1212,13 +1212,13 @@ describe('CodeGrader', () => { expect(result.details?.f1).toBeCloseTo(0.769); }); - it('passes workspace_path to code grader via payload and env var', async () => { + it('passes workspace_path to script grader via payload and env var', async () => { const graderProvider = new StubProvider(textResponse('{}')); const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader-workspace.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: baseTestCase, @@ -1239,13 +1239,13 @@ describe('CodeGrader', () => { expect(passedTexts2).toContain('payload and env var match'); }); - it('omits details when not returned by code grader', async () => { + it('omits details when not returned by script grader', async () => { const graderProvider = new StubProvider(textResponse('{}')); const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader.cjs')]; - const evaluator = new CodeGrader({ command: script }); + const evaluator = new ScriptGrader({ command: script }); const result = await evaluator.evaluate({ evalCase: { diff --git a/packages/core/test/evaluation/graders/code-grader-plain-text.test.ts b/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts similarity index 93% rename from packages/core/test/evaluation/graders/code-grader-plain-text.test.ts rename to packages/core/test/evaluation/graders/script-grader-plain-text.test.ts index 86f9b8709..5f74e9784 100644 --- a/packages/core/test/evaluation/graders/code-grader-plain-text.test.ts +++ b/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts @@ -1,5 +1,5 @@ /** - * Tests for code-grader plain-text fallback. + * Tests for script-grader plain-text fallback. * * When a script emits non-JSON stdout, the grader uses the exit code as * pass/fail (0 = score 1, non-zero = score 0) and stdout as the assertion @@ -7,15 +7,15 @@ */ import { describe, expect, it } from 'vitest'; -import { CodeGrader } from '../../../src/evaluation/graders/code-grader.js'; +import { ScriptGrader } from '../../../src/evaluation/graders/script-grader.js'; import type { EvaluationContext } from '../../../src/evaluation/graders/types.js'; const ctx = { candidate: '', evalCase: { id: 'test', input: [] } } as unknown as EvaluationContext; const grader = (cmd: string) => - new CodeGrader({ command: ['bash', '-c', cmd], agentTimeoutMs: 10_000 }); + new ScriptGrader({ command: ['bash', '-c', cmd], agentTimeoutMs: 10_000 }); -describe('code-grader plain-text fallback', () => { +describe('script-grader plain-text fallback', () => { it('exit 0 with empty stdout → score 1, assertion text "exit 0"', async () => { const result = await grader('true').evaluate(ctx); expect(result.score).toBe(1); diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index 9446136a8..77e93297b 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -6,7 +6,6 @@ import path from 'node:path'; import { parseGraders } from '../../../src/evaluation/loaders/grader-parser.js'; import type { ToolTrajectoryGraderConfig } from '../../../src/evaluation/trace.js'; import type { - CodeGraderConfig, CompositeGraderConfig, ContainsGraderConfig, EqualsGraderConfig, @@ -15,6 +14,7 @@ import type { LlmGraderConfig, LlmRubricGraderConfig, RegexGraderConfig, + ScriptGraderConfig, } from '../../../src/evaluation/types.js'; describe('parseGraders - deterministic assertion types', () => { @@ -657,7 +657,7 @@ describe('parseGraders - script config pass-through', () => { const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); expect(evaluators).toHaveLength(1); - const config = evaluators?.[0] as CodeGraderConfig; + const config = evaluators?.[0] as ScriptGraderConfig; expect(config.type).toBe('script'); expect(config.name).toBe('fuzzy-matcher'); expect(config.config).toEqual({ @@ -684,7 +684,7 @@ describe('parseGraders - script config pass-through', () => { const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); expect(evaluators).toHaveLength(1); - const config = evaluators?.[0] as CodeGraderConfig; + const config = evaluators?.[0] as ScriptGraderConfig; expect(config.type).toBe('script'); expect(config.config).toBeUndefined(); }); @@ -713,7 +713,7 @@ describe('parseGraders - script config pass-through', () => { const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); expect(evaluators).toHaveLength(1); - const config = evaluators?.[0] as CodeGraderConfig; + const config = evaluators?.[0] as ScriptGraderConfig; expect(config.weight).toBe(2.0); expect(config.required).toBe(true); expect(config.min_score).toBe(0.75); @@ -735,7 +735,7 @@ describe('parseGraders - script config pass-through', () => { const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); expect(evaluators).toHaveLength(1); - const config = evaluators?.[0] as CodeGraderConfig; + const config = evaluators?.[0] as ScriptGraderConfig; if (process.platform === 'win32') { expect(config.command).toEqual(['cmd.exe', '/c', './test_script.ts']); } else { @@ -743,7 +743,7 @@ describe('parseGraders - script config pass-through', () => { } }); - it('rejects removed code-grader script alias', async () => { + it('rejects removed script-grader script alias', async () => { await expect( parseGraders( { @@ -785,19 +785,19 @@ describe('parseGraders - kebab-case type normalization', () => { expect((evaluators?.[0] as LlmGraderConfig).target).toBe('grader-low-cost-a'); }); - it('rejects removed code-grader type', async () => { + it('rejects removed script-grader type', async () => { const rawEvalCase = { assert: [ { metric: 'kebab-code', - type: 'code-grader', + type: 'script-grader', command: ['bun', 'run', './test_script.ts'], }, ], }; await expect(parseGraders(rawEvalCase, undefined, [tempDir], 'test-case')).rejects.toThrow( - /Unsupported grader 'code-grader'.*Use 'script'/, + /Unsupported grader 'script-grader'.*Use 'script'/, ); }); @@ -816,7 +816,7 @@ describe('parseGraders - kebab-case type normalization', () => { expect(evaluators).toHaveLength(1); expect(evaluators?.[0].type).toBe('script'); - expect((evaluators?.[0] as CodeGraderConfig).command).toEqual([ + expect((evaluators?.[0] as ScriptGraderConfig).command).toEqual([ 'bun', 'run', './test_script.ts', @@ -2121,7 +2121,7 @@ describe('parseGraders - required field', () => { 'test-1', ); expect(evaluators).toHaveLength(1); - const config = evaluators?.[0] as CodeGraderConfig; + const config = evaluators?.[0] as ScriptGraderConfig; expect(config.required).toBe(true); }); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 9fcb170e1..f57e10d43 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -1687,7 +1687,7 @@ Reference: \${ref}\`); { name: 'ts-prompt-eval', type: 'llm-grader', - // Use explicit script array (matches code-grader pattern) + // Use explicit script array (matches script grader pattern) resolvedPromptScript: ['bun', 'run', promptPath], }, ], diff --git a/packages/core/test/evaluation/code-grader-file-backed.test.ts b/packages/core/test/evaluation/script-grader-file-backed.test.ts similarity index 92% rename from packages/core/test/evaluation/code-grader-file-backed.test.ts rename to packages/core/test/evaluation/script-grader-file-backed.test.ts index 2106e9f05..a30bedd04 100644 --- a/packages/core/test/evaluation/code-grader-file-backed.test.ts +++ b/packages/core/test/evaluation/script-grader-file-backed.test.ts @@ -4,7 +4,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { CodeGrader } from '../../src/evaluation/graders/code-grader.js'; +import { ScriptGrader } from '../../src/evaluation/graders/script-grader.js'; import type { EvalTest } from '../../src/evaluation/types.js'; const baseTestCase: EvalTest = { @@ -16,7 +16,7 @@ const baseTestCase: EvalTest = { reference_answer: 'Expected answer', file_paths: [], criteria: 'Test criteria', - evaluator: 'code-grader', + evaluator: 'script', }; /** Create a grader script that echoes the received stdin payload. */ @@ -73,7 +73,7 @@ console.log(JSON.stringify({ return [process.execPath, script]; } -describe('CodeGrader file-backed output', () => { +describe('ScriptGrader file-backed output', () => { let tmpDir: string; beforeEach(async () => { @@ -88,7 +88,7 @@ describe('CodeGrader file-backed output', () => { const command = await createEchoGrader(tmpDir); const smallOutput = [{ role: 'assistant' as const, content: 'short response' }]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); const result = await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', @@ -105,7 +105,7 @@ describe('CodeGrader file-backed output', () => { const largeContent = 'x'.repeat(60_000); const largeOutput = [{ role: 'assistant' as const, content: largeContent }]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); const result = await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', @@ -126,7 +126,7 @@ describe('CodeGrader file-backed output', () => { const largeContent = 'x'.repeat(60_000); const largeOutput = [{ role: 'assistant' as const, content: largeContent }]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); const result = await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', @@ -141,7 +141,7 @@ describe('CodeGrader file-backed output', () => { it('preserves structured expected_output, input, and config in stdin', async () => { const command = await createPayloadShapeGrader(tmpDir); - const evaluator = new CodeGrader({ command, config: { mode: 'strict' } }); + const evaluator = new ScriptGrader({ command, config: { mode: 'strict' } }); const result = await evaluator.evaluate({ evalCase: { ...baseTestCase, diff --git a/packages/core/test/evaluation/code-grader-multimodal.test.ts b/packages/core/test/evaluation/script-grader-multimodal.test.ts similarity index 97% rename from packages/core/test/evaluation/code-grader-multimodal.test.ts rename to packages/core/test/evaluation/script-grader-multimodal.test.ts index 47a951563..18805a642 100644 --- a/packages/core/test/evaluation/code-grader-multimodal.test.ts +++ b/packages/core/test/evaluation/script-grader-multimodal.test.ts @@ -4,8 +4,8 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { materializeContentForGrader } from '../../src/evaluation/graders/code-grader.js'; -import { CodeGrader } from '../../src/evaluation/graders/code-grader.js'; +import { materializeContentForGrader } from '../../src/evaluation/graders/script-grader.js'; +import { ScriptGrader } from '../../src/evaluation/graders/script-grader.js'; import type { EvalTest } from '../../src/evaluation/types.js'; const baseTestCase: EvalTest = { @@ -17,7 +17,7 @@ const baseTestCase: EvalTest = { reference_answer: 'A chart', file_paths: [], criteria: 'Describes the image correctly', - evaluator: 'code-grader', + evaluator: 'script', }; /** Encode a string as base64 data URI. */ @@ -244,7 +244,7 @@ describe('materializeContentForGrader', () => { }); }); -describe('CodeGrader multimodal integration', () => { +describe('ScriptGrader multimodal integration', () => { let tmpDir: string; beforeEach(async () => { @@ -259,7 +259,7 @@ describe('CodeGrader multimodal integration', () => { const command = await createPayloadEchoGrader(tmpDir); const output = [{ role: 'assistant' as const, content: 'Hello world' }]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); const result = await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', @@ -290,7 +290,7 @@ describe('CodeGrader multimodal integration', () => { }, ]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); const result = await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', @@ -327,7 +327,7 @@ describe('CodeGrader multimodal integration', () => { }, ]; - const evaluator = new CodeGrader({ command }); + const evaluator = new ScriptGrader({ command }); await evaluator.evaluate({ evalCase: baseTestCase, candidate: 'answer', diff --git a/packages/core/test/fixtures/test-define-grader.ts b/packages/core/test/fixtures/test-define-grader.ts index 09b74f280..5483db597 100644 --- a/packages/core/test/fixtures/test-define-grader.ts +++ b/packages/core/test/fixtures/test-define-grader.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /** - * Test fixture for the code-grader stdin/stdout contract. + * Test fixture for the script-grader stdin/stdout contract. */ import { readFileSync } from 'node:fs'; diff --git a/packages/sdk/README.md b/packages/sdk/README.md index dcb8e1bcc..f78f6200b 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -18,7 +18,7 @@ npm install @agentv/sdk ``` ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; ``` `@agentv/eval` was a temporary deprecated compatibility package for this SDK. It is no longer published from this repository. Use `@agentv/sdk` directly. @@ -71,13 +71,13 @@ export default defineAssertion(({ output }) => ({ Checks support `pass: boolean` for simple checks and `score: number` (0-1) for granular scoring. -### defineCodeGrader (full control) +### defineScriptGrader (full control) ```typescript #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, traceSummary }) => ({ +export default defineScriptGrader(({ output, traceSummary }) => ({ score: (output ?? '').length > 0 ? 1.0 : 0.0, assert: [ { text: 'Output received', passed: (output ?? '').length > 0 }, @@ -103,7 +103,7 @@ it('links to the dashboard', () => { }); ``` -Then reference the verifier directly from eval YAML through AgentV's built-in code-grader adapter: +Then reference the verifier directly from eval YAML through AgentV's built-in script-grader adapter: ```yaml assert: @@ -112,7 +112,7 @@ assert: command: [agentv, eval, graders/welcome-banner.test.ts] ``` -The command reads the normal code-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV assertion, and computes score as `passed / total`. +The command reads the normal script-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV assertion, and computes score as `passed / total`. Use the explicit `agentv eval vitest` subcommand when you need adapter options such as `--cwd`, `--in-workspace`, or `--vitest-command`. Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script: @@ -189,7 +189,7 @@ export default defineEval({ prompt: 'Grade whether the answer is useful.', target: 'grader-target', }), - graders.codeGrader(['bun', 'run', 'graders/check.ts'], { metric: 'scripted-check' }), + graders.scriptGrader(['bun', 'run', 'graders/check.ts'], { metric: 'scripted-check' }), ], }, ], @@ -223,15 +223,15 @@ export default defineEval({ }); ``` -Python workflows should emit canonical YAML/JSONL or implement code graders over the stdin/stdout contract. The repo-local helper under `examples/features/sdk-python/` is an example, not a promised published Python package. +Python workflows should emit canonical YAML/JSONL or implement script graders over the stdin/stdout contract. The repo-local helper under `examples/features/sdk-python/` is an example, not a promised published Python package. ## Exports - `evaluate(config)` - Run evaluations programmatically from inline tests or an eval spec file - `defineAssertion(handler)` - Define a custom assertion (pass/fail + optional score) -- `defineCodeGrader(handler)` - Define a code grader (full score control) +- `defineScriptGrader(handler)` - Define a script grader (full score control) - `defineVitestWorkspaceGrader(options)` - Embed the Vitest workspace verifier adapter in a custom script -- `defineWorkspaceGrader(handler)` - Define a workspace-aware code grader with file assertion helpers +- `defineWorkspaceGrader(handler)` - Define a workspace-aware script grader with file assertion helpers - `definePromptTemplate(handler)` - Define a dynamic prompt template - `defineEval(definition)` / `evalSuite(definition)` - Define a YAML-aligned `.eval.ts` suite - `graders` - Catalog of built-in AgentV grader config helpers @@ -239,7 +239,7 @@ Python workflows should emit canonical YAML/JSONL or implement code graders over - `toEvalYamlObject(definition)` / `serializeEvalYaml(definition)` - Lower or serialize canonical eval YAML - `EvalConfig`, `EvalRunResult`, `EvalSummary`, `EvalTestInput`, `EvalAssertionInput` - Programmatic evaluation types - `AssertionContext`, `AssertionScore` - Assertion types -- `CodeGraderInput`, `CodeGraderResult`, `Workspace`, `WorkspaceAssertion` - Grader types +- `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` - Grader types - `TraceSummary`, `Message`, `ToolCall` - Trace data types - `createTargetClient()` - LLM target proxy for graders - `z` - Re-exported Zod for custom config schemas diff --git a/packages/sdk/src/assertion.ts b/packages/sdk/src/assertion.ts index 72b87be99..c546351ef 100644 --- a/packages/sdk/src/assertion.ts +++ b/packages/sdk/src/assertion.ts @@ -2,7 +2,7 @@ * Runtime for custom assertion evaluators. * Handles stdin parsing, validation, error handling, and output formatting. * - * Assertions receive the same input as code graders but use a simplified result + * Assertions receive the same input as script graders but use a simplified result * contract focused on pass/fail with optional score granularity. */ import { readFileSync } from 'node:fs'; @@ -10,16 +10,16 @@ import { toCamelCaseDeep } from '@agentv/core'; import { enrichInput } from './deprecation.js'; import { - type CodeGraderInput, - CodeGraderInputSchema, - type CodeGraderResult, - CodeGraderResultSchema, + type ScriptGraderInput, + ScriptGraderInputSchema, + type ScriptGraderResult, + ScriptGraderResultSchema, } from './schemas.js'; /** * Context provided to assertion handlers. */ -export type AssertionContext = CodeGraderInput; +export type AssertionContext = ScriptGraderInput; /** * Known built-in assertion types. Custom types are extensible via string. @@ -133,9 +133,9 @@ function formatError(error: unknown): string { } /** - * Normalize an AssertionScore to a CodeGraderResult for wire compatibility. + * Normalize an AssertionScore to a ScriptGraderResult for wire compatibility. */ -function normalizeScore(result: AssertionScore): CodeGraderResult { +function normalizeScore(result: AssertionScore): ScriptGraderResult { let score: number; if (result.score !== undefined) { score = clampScore(result.score); @@ -161,11 +161,11 @@ export async function runAssertion(handler: AssertionHandler): Promise { const stdin = readStdin(); const rawInput = JSON.parse(stdin) as Record; const camelInput = toCamelCaseDeep(rawInput); - const input = CodeGraderInputSchema.parse(camelInput); + const input = ScriptGraderInputSchema.parse(camelInput); // Lazy file-backed output loading if (input.outputPath && (input.output === null || input.output === undefined)) { - let cachedOutput: CodeGraderInput['output'] | undefined; + let cachedOutput: ScriptGraderInput['output'] | undefined; const filePath = input.outputPath; Object.defineProperty(input, 'output', { get() { @@ -185,11 +185,11 @@ export async function runAssertion(handler: AssertionHandler): Promise { // Run handler const rawResult = await handler(input); const normalized = normalizeScore(rawResult); - const result = CodeGraderResultSchema.parse(normalized); + const result = ScriptGraderResultSchema.parse(normalized); console.log(JSON.stringify(result, null, 2)); } catch (error) { const errorMessage = formatError(error); - const errorResult: CodeGraderResult = { + const errorResult: ScriptGraderResult = { score: 0, assertions: [{ text: `Assertion failed: ${errorMessage}`, passed: false }], }; diff --git a/packages/sdk/src/deprecation.ts b/packages/sdk/src/deprecation.ts index 35d80939f..8bcbea344 100644 --- a/packages/sdk/src/deprecation.ts +++ b/packages/sdk/src/deprecation.ts @@ -1,20 +1,20 @@ /** - * Input enrichment utilities for code grader and assertion runtimes. + * Input enrichment utilities for script grader and assertion runtimes. * * With the removal of text convenience accessors (`inputText`, `outputText`, - * `expectedOutputText`) from CodeGraderInput, this module is a no-op pass-through. + * `expectedOutputText`) from ScriptGraderInput, this module is a no-op pass-through. * Kept for backward compatibility — existing runtimes call `enrichInput()` and * the call is harmless. */ -import type { CodeGraderInput } from './schemas.js'; +import type { ScriptGraderInput } from './schemas.js'; /** - * Enrich a validated CodeGraderInput. + * Enrich a validated ScriptGraderInput. * * Previously populated text convenience accessors; now a no-op pass-through since - * those fields were removed. Code graders should extract text from `Message.content` + * those fields were removed. script graders should extract text from `Message.content` * using `getTextContent()` from `@agentv/core` instead. */ -export function enrichInput(input: CodeGraderInput): CodeGraderInput { +export function enrichInput(input: ScriptGraderInput): ScriptGraderInput { return input; } diff --git a/packages/sdk/src/graders.ts b/packages/sdk/src/graders.ts index 614eb281b..b61142282 100644 --- a/packages/sdk/src/graders.ts +++ b/packages/sdk/src/graders.ts @@ -99,13 +99,13 @@ export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig readonly preprocessors?: readonly EvalPreprocessor[]; } -export interface CodeGraderTargetOptions { +export interface ScriptGraderTargetOptions { readonly maxCalls?: number; } -export interface CodeGraderOptions extends GraderHelperOptions { +export interface ScriptGraderOptions extends GraderHelperOptions { readonly cwd?: string; - readonly target?: true | CodeGraderTargetOptions; + readonly target?: true | ScriptGraderTargetOptions; readonly config?: Readonly>; readonly preprocessors?: readonly EvalPreprocessor[]; } @@ -114,11 +114,17 @@ export interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonCon readonly type: 'script'; readonly command: GraderCommand; readonly cwd?: string; - readonly target?: true | CodeGraderTargetOptions; + readonly target?: true | ScriptGraderTargetOptions; readonly config?: Readonly>; readonly preprocessors?: readonly EvalPreprocessor[]; } +/** @deprecated Use ScriptGraderTargetOptions. */ +export type CodeGraderTargetOptions = ScriptGraderTargetOptions; + +/** @deprecated Use ScriptGraderOptions. */ +export type CodeGraderOptions = ScriptGraderOptions; + /** @deprecated Use ScriptGraderConfig with type: 'script'. */ export type CodeGraderConfig = ScriptGraderConfig; @@ -224,16 +230,17 @@ export function llmGrader(options: LlmGraderOptions = {}): LlmGraderConfig { ); } +/** @deprecated Use scriptGrader. */ export function codeGrader( command: GraderCommand, - options: CodeGraderOptions = {}, + options: ScriptGraderOptions = {}, ): ScriptGraderConfig { return scriptGrader(command, options); } export function scriptGrader( command: GraderCommand, - options: CodeGraderOptions = {}, + options: ScriptGraderOptions = {}, ): ScriptGraderConfig { return withCommon( { diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 38689cd67..101fa9422 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -17,12 +17,12 @@ * })); * ``` * - * @example Code grader (full control) + * @example script grader (full control) * ```typescript * #!/usr/bin/env bun - * import { defineCodeGrader } from '@agentv/sdk'; + * import { defineScriptGrader } from '@agentv/sdk'; * - * export default defineCodeGrader(({ output, traceSummary }) => { + * export default defineScriptGrader(({ output, traceSummary }) => { * return { * score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, * assertions: [ @@ -62,6 +62,8 @@ // Re-export schemas and types export { + ScriptGraderInputSchema, + ScriptGraderResultSchema, CodeGraderInputSchema, CodeGraderResultSchema, TRACE_REDACTION_LEVELS, @@ -90,6 +92,8 @@ export { ContentImageSchema, ContentFileSchema, ContentSchema, + type ScriptGraderInput, + type ScriptGraderResult, type CodeGraderInput, type CodeGraderResult, type TraceArtifact, @@ -167,6 +171,9 @@ export { llmRubricGrader, regexGrader, scriptGrader, + type ScriptGraderConfig, + type ScriptGraderOptions, + type ScriptGraderTargetOptions, type CodeGraderConfig, type CodeGraderOptions, type CodeGraderTargetOptions, @@ -188,7 +195,6 @@ export { type LlmRubricGraderConfig, type RegexGraderConfig, type RegexGraderOptions, - type ScriptGraderConfig, } from './graders.js'; // Re-export target client @@ -221,6 +227,7 @@ export { export { defineVitestWorkspaceGrader, runVitestWorkspaceGrader, + vitestReportToScriptGraderResult, vitestReportToCodeGraderResult, type VitestWorkspaceGraderOptions, } from './vitest.js'; @@ -238,14 +245,19 @@ export type { import { type AssertionHandler, runAssertion } from './assertion.js'; import { type PromptTemplateHandler, runPromptTemplate } from './prompt-template.js'; -import { type CodeGraderHandler, runCodeGrader } from './runtime.js'; +import { + type CodeGraderHandler, + type ScriptGraderHandler, + runCodeGrader, + runScriptGrader, +} from './runtime.js'; -export { runCodeGrader }; -export type { CodeGraderHandler }; +export { runCodeGrader, runScriptGrader }; +export type { CodeGraderHandler, ScriptGraderHandler }; export type { PromptTemplateHandler }; /** - * Define a code grader with automatic stdin/stdout handling. + * Define a script grader with automatic stdin/stdout handling. * * This function: * 1. Reads JSON from stdin (snake_case format) @@ -258,9 +270,9 @@ export type { PromptTemplateHandler }; * * @example * ```typescript - * import { defineCodeGrader } from '@agentv/sdk'; + * import { defineScriptGrader } from '@agentv/sdk'; * - * export default defineCodeGrader(({ trace }) => { + * export default defineScriptGrader(({ trace }) => { * if (!trace) { * return { score: 0.5, assertions: [{ text: 'No trace available', passed: false }] }; * } @@ -275,21 +287,26 @@ export type { PromptTemplateHandler }; * * @example With typed config * ```typescript - * import { defineCodeGrader, z } from '@agentv/sdk'; + * import { defineScriptGrader, z } from '@agentv/sdk'; * * const ConfigSchema = z.object({ * maxToolCalls: z.number().default(10), * }); * - * export default defineCodeGrader(({ trace, config }) => { + * export default defineScriptGrader(({ trace, config }) => { * const { maxToolCalls } = ConfigSchema.parse(config ?? {}); * // Use maxToolCalls... * }); * ``` */ -export function defineCodeGrader(handler: CodeGraderHandler): void { +export function defineScriptGrader(handler: ScriptGraderHandler): void { // Run immediately when module is loaded - runCodeGrader(handler); + runScriptGrader(handler); +} + +/** @deprecated Use defineScriptGrader. */ +export function defineCodeGrader(handler: ScriptGraderHandler): void { + defineScriptGrader(handler); } /** diff --git a/packages/sdk/src/prompt-template.ts b/packages/sdk/src/prompt-template.ts index 322edc325..45b8f6f35 100644 --- a/packages/sdk/src/prompt-template.ts +++ b/packages/sdk/src/prompt-template.ts @@ -6,13 +6,13 @@ import { readFileSync } from 'node:fs'; import { toCamelCaseDeep } from '@agentv/core'; import { enrichInput } from './deprecation.js'; -import { type CodeGraderInput, PromptTemplateInputSchema } from './schemas.js'; +import { PromptTemplateInputSchema, type ScriptGraderInput } from './schemas.js'; /** * Handler function type for prompt templates. * Returns the prompt string to use for evaluation. */ -export type PromptTemplateHandler = (input: CodeGraderInput) => string | Promise; +export type PromptTemplateHandler = (input: ScriptGraderInput) => string | Promise; /** * Read stdin synchronously (works in both Node.js and Bun). @@ -68,9 +68,9 @@ export async function runPromptTemplate(handler: PromptTemplateHandler): Promise * * @example * ```typescript - * import { definePromptTemplate, type CodeGraderInput } from '@agentv/sdk'; + * import { definePromptTemplate, type ScriptGraderInput } from '@agentv/sdk'; * - * export default definePromptTemplate((ctx: CodeGraderInput) => { + * export default definePromptTemplate((ctx: ScriptGraderInput) => { * const question = ctx.input * .filter((message) => message.role === 'user') * .map((message) => typeof message.content === 'string' ? message.content : '') diff --git a/packages/sdk/src/runtime.ts b/packages/sdk/src/runtime.ts index 9d9e011c5..b4d442d40 100644 --- a/packages/sdk/src/runtime.ts +++ b/packages/sdk/src/runtime.ts @@ -1,5 +1,5 @@ /** - * Runtime for code grader evaluators. + * Runtime for script grader evaluators. * Handles stdin parsing, validation, error handling, and output formatting. */ import { readFileSync } from 'node:fs'; @@ -7,18 +7,18 @@ import { toCamelCaseDeep } from '@agentv/core'; import { enrichInput } from './deprecation.js'; import { - type CodeGraderInput, - CodeGraderInputSchema, - type CodeGraderResult, - CodeGraderResultSchema, + type ScriptGraderInput, + ScriptGraderInputSchema, + type ScriptGraderResult, + ScriptGraderResultSchema, } from './schemas.js'; /** - * Handler function type for code graders. + * Handler function type for script graders. */ -export type CodeGraderHandler = ( - input: CodeGraderInput, -) => CodeGraderResult | Promise; +export type ScriptGraderHandler = ( + input: ScriptGraderInput, +) => ScriptGraderResult | Promise; /** * Read stdin synchronously (works in both Node.js and Bun). @@ -48,10 +48,10 @@ function formatError(error: unknown): string { } /** - * Run a code grader handler with full stdin/stdout handling. - * This is the internal implementation called by defineCodeGrader. + * Run a script grader handler with full stdin/stdout handling. + * This is the internal implementation called by defineScriptGrader. */ -export async function runCodeGrader(handler: CodeGraderHandler): Promise { +export async function runScriptGrader(handler: ScriptGraderHandler): Promise { try { // 1. Read stdin const stdin = readStdin(); @@ -63,11 +63,11 @@ export async function runCodeGrader(handler: CodeGraderHandler): Promise { const camelInput = toCamelCaseDeep(rawInput); // 4. Validate input with Zod - const input = CodeGraderInputSchema.parse(camelInput); + const input = ScriptGraderInputSchema.parse(camelInput); // 5. Set up lazy file-backed output loading if applicable if (input.outputPath && (input.output === null || input.output === undefined)) { - let cachedOutput: CodeGraderInput['output'] | undefined; + let cachedOutput: ScriptGraderInput['output'] | undefined; const filePath = input.outputPath; Object.defineProperty(input, 'output', { get() { @@ -88,7 +88,7 @@ export async function runCodeGrader(handler: CodeGraderHandler): Promise { const rawResult = await handler(input); // 8. Validate and normalize output - const result = CodeGraderResultSchema.parse({ + const result = ScriptGraderResultSchema.parse({ ...rawResult, score: clampScore(rawResult.score), }); @@ -98,7 +98,7 @@ export async function runCodeGrader(handler: CodeGraderHandler): Promise { } catch (error) { // Output failure result const errorMessage = formatError(error); - const errorResult: CodeGraderResult = { + const errorResult: ScriptGraderResult = { score: 0, assertions: [{ text: `Evaluation failed: ${errorMessage}`, passed: false }], }; @@ -106,3 +106,9 @@ export async function runCodeGrader(handler: CodeGraderHandler): Promise { process.exit(1); } } + +/** @deprecated Use ScriptGraderHandler. */ +export type CodeGraderHandler = ScriptGraderHandler; + +/** @deprecated Use runScriptGrader. */ +export const runCodeGrader = runScriptGrader; diff --git a/packages/sdk/src/schemas.ts b/packages/sdk/src/schemas.ts index b1f45c5d8..bd335e15d 100644 --- a/packages/sdk/src/schemas.ts +++ b/packages/sdk/src/schemas.ts @@ -1,5 +1,5 @@ /** - * Zod schemas for code grader input/output validation. + * Zod schemas for script grader input/output validation. * Provides both compile-time types and runtime validation. * * ## Content model @@ -286,12 +286,12 @@ export const TraceSchema = TraceSummarySchema.extend({ }); /** - * Code grader input schema (camelCase, converted from snake_case wire format). + * Script grader input schema (camelCase, converted from snake_case wire format). * * `output` is the final answer/scored result only. Transcript-aware graders * should inspect `messages`, `trace.messages`, or `trace.events`. */ -export const CodeGraderInputSchema = z.object({ +export const ScriptGraderInputSchema = z.object({ criteria: z.string(), expectedOutput: z.array(MessageSchema), output: z.string().nullable().optional(), @@ -314,9 +314,9 @@ export const CodeGraderInputSchema = z.object({ }); /** - * Code grader result schema (validated before output). + * Script grader result schema (validated before output). */ -export const CodeGraderResultSchema = z.object({ +export const ScriptGraderResultSchema = z.object({ score: z.number().min(0).max(1), assertions: z .array( @@ -335,8 +335,8 @@ export const CodeGraderResultSchema = z.object({ /** * Inferred types from schemas. */ -export type CodeGraderInput = z.infer; -export type CodeGraderResult = z.infer; +export type ScriptGraderInput = z.infer; +export type ScriptGraderResult = z.infer; export type TraceSummary = z.infer; export type Trace = z.infer; @@ -363,18 +363,26 @@ export type Content = z.infer; /** * Prompt template input schema (camelCase, converted from snake_case wire format). - * Uses the same schema as CodeGraderInput since the orchestrator sends identical payloads. + * Uses the same schema as ScriptGraderInput since the orchestrator sends identical payloads. */ -export const PromptTemplateInputSchema = CodeGraderInputSchema; +export const PromptTemplateInputSchema = ScriptGraderInputSchema; -export type PromptTemplateInput = CodeGraderInput; +export type PromptTemplateInput = ScriptGraderInput; // ── Backward-compat aliases (deprecated) ──────────────────────────────────────── -/** @deprecated Use CodeGraderInputSchema */ -export const CodeJudgeInputSchema = CodeGraderInputSchema; -/** @deprecated Use CodeGraderResultSchema */ -export const CodeJudgeResultSchema = CodeGraderResultSchema; -/** @deprecated Use CodeGraderInput */ -export type CodeJudgeInput = CodeGraderInput; -/** @deprecated Use CodeGraderResult */ -export type CodeJudgeResult = CodeGraderResult; +/** @deprecated Use ScriptGraderInputSchema */ +export const CodeGraderInputSchema = ScriptGraderInputSchema; +/** @deprecated Use ScriptGraderResultSchema */ +export const CodeGraderResultSchema = ScriptGraderResultSchema; +/** @deprecated Use ScriptGraderInput */ +export type CodeGraderInput = ScriptGraderInput; +/** @deprecated Use ScriptGraderResult */ +export type CodeGraderResult = ScriptGraderResult; +/** @deprecated Use ScriptGraderInputSchema */ +export const CodeJudgeInputSchema = ScriptGraderInputSchema; +/** @deprecated Use ScriptGraderResultSchema */ +export const CodeJudgeResultSchema = ScriptGraderResultSchema; +/** @deprecated Use ScriptGraderInput */ +export type CodeJudgeInput = ScriptGraderInput; +/** @deprecated Use ScriptGraderResult */ +export type CodeJudgeResult = ScriptGraderResult; diff --git a/packages/sdk/src/target-client.ts b/packages/sdk/src/target-client.ts index 433e46cc7..7c1937f4e 100644 --- a/packages/sdk/src/target-client.ts +++ b/packages/sdk/src/target-client.ts @@ -1,5 +1,5 @@ /** - * Client for invoking configured targets from code-grader scripts. + * Client for invoking configured targets from script-grader scripts. * * Environment variables (set automatically by AgentV when `target` config is present): * - AGENTV_TARGET_PROXY_URL: The URL of the local proxy server @@ -97,16 +97,16 @@ export class TargetInvocationError extends Error { * * This function reads the proxy URL and token from environment variables * that are automatically set by AgentV when a `target` config block is present - * on a `code_grader` evaluator. + * on a `script` evaluator. * * @returns A target client if environment variables are set, otherwise undefined * @throws TargetNotAvailableError if token is missing when URL is present * * @example * ```typescript - * import { createTargetClient, defineCodeGrader } from '@agentv/sdk'; + * import { createTargetClient, defineScriptGrader } from '@agentv/sdk'; * - * export default defineCodeGrader(async ({ input, criteria, output }) => { + * export default defineScriptGrader(async ({ input, criteria, output }) => { * const target = createTargetClient(); * const question = input * .filter((message) => message.role === 'user') diff --git a/packages/sdk/src/vitest.ts b/packages/sdk/src/vitest.ts index c1df6ed96..5030e5a9c 100644 --- a/packages/sdk/src/vitest.ts +++ b/packages/sdk/src/vitest.ts @@ -2,7 +2,7 @@ * Vitest workspace verifier adapter. * * This module keeps deterministic workspace verification in familiar Vitest - * tests while translating the JSON reporter output into AgentV's code-grader + * tests while translating the JSON reporter output into AgentV's script-grader * result contract. */ import { spawn } from 'node:child_process'; @@ -10,8 +10,12 @@ import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import nodePath from 'node:path'; -import { runCodeGrader } from './runtime.js'; -import { type CodeGraderInput, type CodeGraderResult, CodeGraderResultSchema } from './schemas.js'; +import { runScriptGrader } from './runtime.js'; +import { + type ScriptGraderInput, + type ScriptGraderResult, + ScriptGraderResultSchema, +} from './schemas.js'; export interface VitestWorkspaceGraderOptions { /** @@ -84,7 +88,7 @@ interface VitestJsonReport { readonly testResults?: readonly VitestFileResult[]; } -function workspacePathFrom(input: CodeGraderInput): string | undefined { +function workspacePathFrom(input: ScriptGraderInput): string | undefined { const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH; return workspacePath?.trim() ? workspacePath : undefined; } @@ -181,10 +185,10 @@ function assertionText(file: VitestFileResult, assertion: VitestAssertionResult) return title || file.name || 'Vitest assertion'; } -export function vitestReportToCodeGraderResult( +export function vitestReportToScriptGraderResult( report: VitestJsonReport, options: Pick = {}, -): CodeGraderResult { +): ScriptGraderResult { const assertions = (report.testResults ?? []).flatMap((file) => (file.assertionResults ?? []).map((item) => { const passed = item.status === 'passed'; @@ -202,7 +206,7 @@ export function vitestReportToCodeGraderResult( if (assertions.length === 0) { const passed = options.passWithNoTests === true; - return CodeGraderResultSchema.parse({ + return ScriptGraderResultSchema.parse({ score: passed ? 1 : 0, assertions: [{ text: 'Vitest reported no tests', passed }], details: { @@ -217,7 +221,7 @@ export function vitestReportToCodeGraderResult( } const passedCount = assertions.filter((item) => item.passed).length; - return CodeGraderResultSchema.parse({ + return ScriptGraderResultSchema.parse({ score: passedCount / assertions.length, assertions, details: { @@ -231,6 +235,9 @@ export function vitestReportToCodeGraderResult( }); } +/** @deprecated Use vitestReportToScriptGraderResult. */ +export const vitestReportToCodeGraderResult = vitestReportToScriptGraderResult; + function runCommand( command: readonly string[], options: { @@ -330,8 +337,8 @@ async function readVitestReport( export async function runVitestWorkspaceGrader( options: VitestWorkspaceGraderOptions, - input: CodeGraderInput, -): Promise { + input: ScriptGraderInput, +): Promise { const workspacePath = workspacePathFrom(input); if (!workspacePath) { return { @@ -384,7 +391,7 @@ export async function runVitestWorkspaceGrader( }); const report = await readVitestReport(result, outputFile); - return vitestReportToCodeGraderResult(report, options); + return vitestReportToScriptGraderResult(report, options); } catch (error) { return { score: 0, @@ -404,5 +411,5 @@ export async function runVitestWorkspaceGrader( } export function defineVitestWorkspaceGrader(options: VitestWorkspaceGraderOptions): void { - runCodeGrader((input) => runVitestWorkspaceGrader(options, input)); + runScriptGrader((input) => runVitestWorkspaceGrader(options, input)); } diff --git a/packages/sdk/src/workspace.ts b/packages/sdk/src/workspace.ts index b9c5a13a9..4ab1a32e2 100644 --- a/packages/sdk/src/workspace.ts +++ b/packages/sdk/src/workspace.ts @@ -1,7 +1,7 @@ /** * Workspace grader helpers for deterministic file assertions. * - * `defineWorkspaceGrader()` wraps the code-grader runtime with a small + * `defineWorkspaceGrader()` wraps the script-grader runtime with a small * workspace object so graders can read files and return assertion arrays * without hand-rolling stdin parsing, workspace path fallback, file reads, or * score aggregation. @@ -9,8 +9,12 @@ import { readFile, stat } from 'node:fs/promises'; import nodePath from 'node:path'; -import { runCodeGrader } from './runtime.js'; -import { type CodeGraderInput, type CodeGraderResult, CodeGraderResultSchema } from './schemas.js'; +import { runScriptGrader } from './runtime.js'; +import { + type ScriptGraderInput, + type ScriptGraderResult, + ScriptGraderResultSchema, +} from './schemas.js'; export interface WorkspaceAssertion { readonly text: string; @@ -21,7 +25,7 @@ export interface WorkspaceAssertion { type Awaitable = T | Promise; export type WorkspaceGraderReturn = - | CodeGraderResult + | ScriptGraderResult | WorkspaceAssertion | readonly Awaitable[]; @@ -49,7 +53,7 @@ export interface Workspace { readText(relativePath: string): Promise; } -export type WorkspaceGraderContext = CodeGraderInput & { +export type WorkspaceGraderContext = ScriptGraderInput & { readonly workspace: Workspace; }; @@ -63,7 +67,7 @@ interface ResolvedWorkspacePath { readonly error?: string; } -function workspacePathFrom(input: CodeGraderInput): string | undefined { +function workspacePathFrom(input: ScriptGraderInput): string | undefined { const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH; if (!workspacePath?.trim()) { return undefined; @@ -143,11 +147,11 @@ async function readFileForAssertion( } } -function isCodeGraderResult(value: WorkspaceGraderReturn): value is CodeGraderResult { +function isScriptGraderResult(value: WorkspaceGraderReturn): value is ScriptGraderResult { return !Array.isArray(value) && typeof value === 'object' && value !== null && 'score' in value; } -export function createWorkspace(input: CodeGraderInput): Workspace { +export function createWorkspace(input: ScriptGraderInput): Workspace { const workspacePath = workspacePathFrom(input); const textCache = new Map>(); @@ -272,15 +276,15 @@ export function createWorkspace(input: CodeGraderInput): Workspace { export async function normalizeWorkspaceGraderResult( result: WorkspaceGraderReturn, -): Promise { - if (isCodeGraderResult(result)) { - return CodeGraderResultSchema.parse(result); +): Promise { + if (isScriptGraderResult(result)) { + return ScriptGraderResultSchema.parse(result); } const assertions = Array.isArray(result) ? await Promise.all(result) : [result]; const passed = assertions.filter((item) => item.passed).length; - return CodeGraderResultSchema.parse({ + return ScriptGraderResultSchema.parse({ score: assertions.length > 0 ? passed / assertions.length : 0, assertions, }); @@ -288,8 +292,8 @@ export async function normalizeWorkspaceGraderResult( export async function runWorkspaceGrader( handler: WorkspaceGraderHandler, - input: CodeGraderInput, -): Promise { + input: ScriptGraderInput, +): Promise { return normalizeWorkspaceGraderResult( await handler({ ...input, @@ -299,5 +303,5 @@ export async function runWorkspaceGrader( } export function defineWorkspaceGrader(handler: WorkspaceGraderHandler): void { - runCodeGrader((input) => runWorkspaceGrader(handler, input)); + runScriptGrader((input) => runWorkspaceGrader(handler, input)); } diff --git a/packages/sdk/test/define-code-grader.test.ts b/packages/sdk/test/define-script-grader.test.ts similarity index 88% rename from packages/sdk/test/define-code-grader.test.ts rename to packages/sdk/test/define-script-grader.test.ts index 6f55257a2..bd2c1b1b1 100644 --- a/packages/sdk/test/define-code-grader.test.ts +++ b/packages/sdk/test/define-script-grader.test.ts @@ -1,9 +1,6 @@ import { describe, expect, it } from 'bun:test'; import { - CodeGraderInputSchema, - type CodeGraderResult, - CodeGraderResultSchema, // Backward-compat aliases CodeJudgeInputSchema, CodeJudgeResultSchema, @@ -12,6 +9,9 @@ import { ContentSchema, ContentTextSchema, MessageSchema, + ScriptGraderInputSchema, + type ScriptGraderResult, + ScriptGraderResultSchema, } from '../src/schemas.js'; const makeTrace = (overrides: Record = {}) => ({ @@ -173,7 +173,7 @@ describe('MessageSchema content variants', () => { }); }); -describe('CodeGraderInputSchema', () => { +describe('ScriptGraderInputSchema', () => { const validInput = { criteria: 'The answer should be 4', expectedOutput: [{ role: 'assistant', content: '4' }], @@ -182,7 +182,7 @@ describe('CodeGraderInputSchema', () => { }; it('parses valid input', () => { - const result = CodeGraderInputSchema.parse(validInput); + const result = ScriptGraderInputSchema.parse(validInput); expect(result.criteria).toBe('The answer should be 4'); }); @@ -191,7 +191,7 @@ describe('CodeGraderInputSchema', () => { ...validInput, trace: makeTrace(), }; - const result = CodeGraderInputSchema.parse(inputWithTrace); + const result = ScriptGraderInputSchema.parse(inputWithTrace); expect(result.trace?.eventCount).toBe(3); expect(result.trace?.toolCalls).toEqual({ read: 2, write: 1 }); }); @@ -201,7 +201,7 @@ describe('CodeGraderInputSchema', () => { ...validInput, trace: null, }; - const result = CodeGraderInputSchema.parse(inputWithNullTrace); + const result = ScriptGraderInputSchema.parse(inputWithNullTrace); expect(result.trace).toBeNull(); }); @@ -210,7 +210,7 @@ describe('CodeGraderInputSchema', () => { ...validInput, config: { maxToolCalls: 10, strictMode: true }, }; - const result = CodeGraderInputSchema.parse(inputWithConfig); + const result = ScriptGraderInputSchema.parse(inputWithConfig); expect(result.config).toEqual({ maxToolCalls: 10, strictMode: true }); }); @@ -226,7 +226,7 @@ describe('CodeGraderInputSchema', () => { }, ], }; - const result = CodeGraderInputSchema.parse(inputWithOutput); + const result = ScriptGraderInputSchema.parse(inputWithOutput); expect(result.output).toBe('Reading file...'); expect(result.messages?.[0].toolCalls?.[0].tool).toBe('read'); }); @@ -245,7 +245,7 @@ describe('CodeGraderInputSchema', () => { }, ], }; - const result = CodeGraderInputSchema.parse(inputWithImages); + const result = ScriptGraderInputSchema.parse(inputWithImages); const content = result.messages?.[0].content as { type: string; path?: string }[]; expect(content).toHaveLength(2); expect(content[1].type).toBe('image'); @@ -265,7 +265,7 @@ describe('CodeGraderInputSchema', () => { }, ], }; - const result = CodeGraderInputSchema.parse(inputWithContentArray); + const result = ScriptGraderInputSchema.parse(inputWithContentArray); const content = result.input[0].content as { type: string }[]; expect(content).toHaveLength(2); }); @@ -280,21 +280,21 @@ describe('CodeGraderInputSchema', () => { }, ], }; - const result = CodeGraderInputSchema.parse(inputWithStructuredExpectedOutput); + const result = ScriptGraderInputSchema.parse(inputWithStructuredExpectedOutput); expect(result.expectedOutput[0].content).toEqual({ riskLevel: 'High' }); }); }); -describe('CodeGraderResultSchema', () => { +describe('ScriptGraderResultSchema', () => { it('parses valid result with all fields', () => { - const result: CodeGraderResult = { + const result: ScriptGraderResult = { score: 0.8, assertions: [ { text: 'Correct answer', passed: true }, { text: 'Missing explanation', passed: false }, ], }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.score).toBe(0.8); expect(parsed.assertions).toEqual([ { text: 'Correct answer', passed: true }, @@ -304,29 +304,29 @@ describe('CodeGraderResultSchema', () => { it('defaults assertions to empty array', () => { const result = { score: 0.5 }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.assertions).toEqual([]); }); it('defaults assertions to empty array when omitted', () => { const result = { score: 1.0 }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.assertions).toEqual([]); }); it('rejects score below 0', () => { const result = { score: -0.5 }; - expect(() => CodeGraderResultSchema.parse(result)).toThrow(); + expect(() => ScriptGraderResultSchema.parse(result)).toThrow(); }); it('rejects score above 1', () => { const result = { score: 1.5 }; - expect(() => CodeGraderResultSchema.parse(result)).toThrow(); + expect(() => ScriptGraderResultSchema.parse(result)).toThrow(); }); it('accepts boundary scores 0 and 1', () => { - expect(CodeGraderResultSchema.parse({ score: 0 }).score).toBe(0); - expect(CodeGraderResultSchema.parse({ score: 1 }).score).toBe(1); + expect(ScriptGraderResultSchema.parse({ score: 0 }).score).toBe(0); + expect(ScriptGraderResultSchema.parse({ score: 1 }).score).toBe(1); }); it('accepts optional details object', () => { @@ -341,7 +341,7 @@ describe('CodeGraderResultSchema', () => { recall: 0.714, }, }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.details).toEqual({ tp: 5, tn: 2, @@ -354,7 +354,7 @@ describe('CodeGraderResultSchema', () => { it('allows details to be omitted', () => { const result = { score: 0.5 }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.details).toBeUndefined(); }); @@ -372,7 +372,7 @@ describe('CodeGraderResultSchema', () => { }, }, }; - const parsed = CodeGraderResultSchema.parse(result); + const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.details?.alignment).toHaveLength(2); expect(parsed.details?.metrics).toBeDefined(); }); diff --git a/packages/sdk/test/deprecation.test.ts b/packages/sdk/test/deprecation.test.ts index 6a63b03c7..87f1fc978 100644 --- a/packages/sdk/test/deprecation.test.ts +++ b/packages/sdk/test/deprecation.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it } from 'bun:test'; import { enrichInput } from '../src/deprecation.js'; -import { CodeGraderInputSchema } from '../src/schemas.js'; +import { ScriptGraderInputSchema } from '../src/schemas.js'; /** - * Build a minimal valid CodeGraderInput for testing. + * Build a minimal valid ScriptGraderInput for testing. */ function buildInput(overrides?: Record) { - return CodeGraderInputSchema.parse({ + return ScriptGraderInputSchema.parse({ criteria: 'The answer should be 4', expectedOutput: [{ role: 'assistant', content: '4' }], inputFiles: [], diff --git a/packages/sdk/test/file-backed-output.test.ts b/packages/sdk/test/file-backed-output.test.ts index 27de99630..827967e1f 100644 --- a/packages/sdk/test/file-backed-output.test.ts +++ b/packages/sdk/test/file-backed-output.test.ts @@ -3,9 +3,9 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { type CodeGraderInput, CodeGraderInputSchema } from '../src/schemas.js'; +import { type ScriptGraderInput, ScriptGraderInputSchema } from '../src/schemas.js'; -describe('CodeGraderInputSchema with outputPath', () => { +describe('ScriptGraderInputSchema with outputPath', () => { const validInput = { criteria: 'The answer should be 4', expectedOutput: [{ role: 'assistant', content: '4' }], @@ -19,18 +19,18 @@ describe('CodeGraderInputSchema with outputPath', () => { outputPath: '/tmp/test/output.json', output: null, }; - const result = CodeGraderInputSchema.parse(inputWithPath); + const result = ScriptGraderInputSchema.parse(inputWithPath); expect(result.outputPath).toBe('/tmp/test/output.json'); expect(result.output).toBeNull(); }); it('allows outputPath to be omitted (backward compat)', () => { - const result = CodeGraderInputSchema.parse(validInput); + const result = ScriptGraderInputSchema.parse(validInput); expect(result.outputPath).toBeUndefined(); }); it('allows both output and outputPath to be omitted', () => { - const result = CodeGraderInputSchema.parse(validInput); + const result = ScriptGraderInputSchema.parse(validInput); expect(result.output).toBeUndefined(); expect(result.outputPath).toBeUndefined(); }); @@ -52,7 +52,7 @@ describe('Lazy file-backed output loading', () => { const filePath = join(tmpDir, 'output.json'); writeFileSync(filePath, JSON.stringify(answer)); - const input: CodeGraderInput = CodeGraderInputSchema.parse({ + const input: ScriptGraderInput = ScriptGraderInputSchema.parse({ criteria: 'test', expectedOutput: [], output: null, @@ -62,7 +62,7 @@ describe('Lazy file-backed output loading', () => { }); // Set up lazy loading (simulates what runtime.ts does) - let cachedOutput: CodeGraderInput['output'] | undefined; + let cachedOutput: ScriptGraderInput['output'] | undefined; Object.defineProperty(input, 'output', { get() { if (cachedOutput === undefined) { @@ -84,7 +84,7 @@ describe('Lazy file-backed output loading', () => { }); it('uses inline output when outputPath is absent', () => { - const input: CodeGraderInput = CodeGraderInputSchema.parse({ + const input: ScriptGraderInput = ScriptGraderInputSchema.parse({ criteria: 'test', expectedOutput: [], output: 'inline', diff --git a/packages/sdk/test/grader-helpers.test.ts b/packages/sdk/test/grader-helpers.test.ts index 236bfc64b..c6b4b7588 100644 --- a/packages/sdk/test/grader-helpers.test.ts +++ b/packages/sdk/test/grader-helpers.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from 'bun:test'; import { - codeGrader, containsGrader, defineEval, equalsGrader, @@ -56,7 +55,7 @@ describe('grader helper config builders', () => { temperature: 0, }); expect( - codeGrader(['bun', 'run', 'graders/check.ts'], { + scriptGrader(['bun', 'run', 'graders/check.ts'], { metric: 'scripted-check', cwd: 'graders', target: { maxCalls: 2 }, diff --git a/packages/sdk/test/vitest-workspace-grader.test.ts b/packages/sdk/test/vitest-workspace-grader.test.ts index f4f78736e..ef45da4dd 100644 --- a/packages/sdk/test/vitest-workspace-grader.test.ts +++ b/packages/sdk/test/vitest-workspace-grader.test.ts @@ -3,8 +3,8 @@ import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { CodeGraderInputSchema } from '../src/schemas.js'; -import { runVitestWorkspaceGrader, vitestReportToCodeGraderResult } from '../src/vitest.js'; +import { ScriptGraderInputSchema } from '../src/schemas.js'; +import { runVitestWorkspaceGrader, vitestReportToScriptGraderResult } from '../src/vitest.js'; const mixedVitestReport = { success: false, @@ -39,7 +39,7 @@ const mixedVitestReport = { }; function buildInput(overrides?: Record) { - return CodeGraderInputSchema.parse({ + return ScriptGraderInputSchema.parse({ criteria: 'Verify the workspace with Vitest', expectedOutput: [], inputFiles: [], @@ -68,7 +68,7 @@ describe('Vitest workspace grader adapter', () => { }); it('maps individual Vitest test outcomes to AgentV assertions', () => { - const result = vitestReportToCodeGraderResult(mixedVitestReport); + const result = vitestReportToScriptGraderResult(mixedVitestReport); expect(result.score).toBe(0.5); expect(result.assertions).toEqual([ diff --git a/packages/sdk/test/workspace-grader.test.ts b/packages/sdk/test/workspace-grader.test.ts index c6feb2374..b7aab2420 100644 --- a/packages/sdk/test/workspace-grader.test.ts +++ b/packages/sdk/test/workspace-grader.test.ts @@ -3,7 +3,7 @@ import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { CodeGraderInputSchema } from '../src/schemas.js'; +import { ScriptGraderInputSchema } from '../src/schemas.js'; import { createWorkspace, normalizeWorkspaceGraderResult, @@ -11,7 +11,7 @@ import { } from '../src/workspace.js'; function buildInput(overrides?: Record) { - return CodeGraderInputSchema.parse({ + return ScriptGraderInputSchema.parse({ criteria: 'Verify the workspace', expectedOutput: [], inputFiles: [], @@ -122,7 +122,7 @@ describe('workspace grader helpers', () => { expect(result.assertions[0].evidence).toContain('inside the workspace'); }); - it('passes through explicit CodeGraderResult objects', async () => { + it('passes through explicit ScriptGraderResult objects', async () => { const result = await runWorkspaceGrader( () => ({ score: 0.75, diff --git a/skills-data/agentv-bench/SKILL.md b/skills-data/agentv-bench/SKILL.md index ad69accc0..c1e1c7f3e 100644 --- a/skills-data/agentv-bench/SKILL.md +++ b/skills-data/agentv-bench/SKILL.md @@ -65,7 +65,7 @@ Before running or optimizing, understand what you're working with. AgentV supports two evaluation formats: -**EVAL.yaml** (native, full features) — supports workspaces, code graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation. +**EVAL.yaml** (native, full features) — supports workspaces, script graders, multi-turn conversations, tool trajectory scoring, workspace file tracking, multi-provider targets. Use this for agent evaluation. ```yaml # example.eval.yaml @@ -73,7 +73,7 @@ tests: - id: basic-code-review input: "Review this TypeScript file for bugs and suggest improvements" criteria: "Identifies the null pointer bug on line 12 and suggests a fix" - assertions: + assert: - type: contains value: "null" - Review identifies the null pointer bug and suggests a concrete fix @@ -113,7 +113,7 @@ Start with 2-3 realistic test cases — the kind of thing a real user would actu Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment. -**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `code-grader`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type. +**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type. Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-rubric`. @@ -196,17 +196,17 @@ grading until all responses are present. **In subagent mode**, grading has three phases. **All three are required — do not stop after phase 1.** -**Phase 1: Code graders** (deterministic, zero-cost) +**Phase 1: script graders** (deterministic, zero-cost) ```bash agentv pipeline grade ``` This evaluates all deterministic assertions against `response.md` files. Two types are handled: -- **`code-grader` scripts** — external scripts executed against the response (arbitrary logic, any language) +- **`script-grader` scripts** — external scripts executed against the response (arbitrary logic, any language) - **Built-in assertion types** — evaluated in-process: `contains`, `contains-any`, `contains-all`, `icontains`, `regex`, `equals`, `starts-with`, `ends-with`, `is-json`, and variants -Both types are configured by `pipeline input` into `code_graders/.json` and graded by `pipeline grade`. Results are written to `/code_grader_results/.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline. +Both types are configured by `pipeline input` into `script_graders/.json` and graded by `pipeline grade`. Results are written to `/script_grader_results/.json`. Alternatively, pass `--grader-type code` to `pipeline run` to run these inline. **Do not dispatch LLM grader subagents for tests that only have `contains`, `regex`, or other built-in assertions** — `pipeline grade` handles them entirely, at zero cost. To detect which tests need Phase 2, check whether `/llm_graders/` contains any `.json` config files — `pipeline input` only writes there for `llm-rubric` assertions. Tests with an empty (or missing) `llm_graders/` directory are done after Phase 1. @@ -242,7 +242,7 @@ agentv pipeline bench agentv results validate ``` -`pipeline bench` reads LLM grader results from `llm_grader_results/.json` per test automatically, merges with code-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`. +`pipeline bench` reads LLM grader results from `llm_grader_results/.json` per test automatically, merges with script-grader scores, computes weighted pass_rate, and writes `grading.json` + `index.jsonl` + `summary.json`. > **Diagnosing `pass_rate=0`:** If `pipeline bench` reports `pass_rate=0` across the board, do **not** assume the tests genuinely failed. First verify the grading pipeline ran correctly: check that `/llm_grader_results/.json` exists and is non-empty for each test. If these files are absent or empty, the grader subagents failed to produce output (most common cause: `agents/grader.md` was not embedded in the subagent prompts — see Phase 2). Treat `pass_rate=0` as a real signal only after confirming grader results exist. diff --git a/skills-data/agentv-bench/agents/analyzer.md b/skills-data/agentv-bench/agents/analyzer.md index fd60c1477..53f6c69d5 100644 --- a/skills-data/agentv-bench/agents/analyzer.md +++ b/skills-data/agentv-bench/agents/analyzer.md @@ -123,7 +123,7 @@ If a section has no findings, include the header with "None found." underneath. - **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config. - **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on. - **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-rubric` → deterministic saves the most). -- **Handle all grader types:** Process `code-grader`, `tool-trajectory`, `llm-rubric`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades. +- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades. - **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific. - **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning. diff --git a/skills-data/agentv-bench/agents/comparator.md b/skills-data/agentv-bench/agents/comparator.md index 754a6f234..b3d6cacd4 100644 --- a/skills-data/agentv-bench/agents/comparator.md +++ b/skills-data/agentv-bench/agents/comparator.md @@ -24,8 +24,7 @@ You are the Blind Comparator for AgentV's evaluation workflow. Your job is to co You will receive: - `outputs`: Array of evaluation outputs to compare. Each contains: - `target_id`: The provider/configuration identifier (DO NOT read this during scoring) - - `answer`: The candidate response text - - `evaluator_results`: Array of grader scores and details (code-grader, tool-trajectory, llm-rubric, deterministic) + - `evaluator_results`: Array of grader scores and details (script, tool-trajectory, llm-rubric, deterministic) - `workspace_changes`: File changes made during workspace evaluation (if applicable) - `tool_calls`: Tool invocations and results from multi-turn conversations (if applicable) - `conversation`: Full multi-turn conversation history (if applicable) @@ -91,7 +90,7 @@ For each content criterion, define: **Grader-Specific Scoring** — when grader results are present: -- **code-grader**: Factor in pass/fail results, test coverage, assertion hit rates +- **script-grader**: Factor in pass/fail results, test coverage, assertion hit rates - **tool-trajectory**: Factor in tool call accuracy, sequence correctness, unnecessary tool calls - **llm-rubric**: Factor in existing LLM grader scores as a reference signal (not as ground truth) - **deterministic**: Factor in exact match / keyword hit rates @@ -236,7 +235,7 @@ Also produce a human-readable markdown summary: - **Be evidence-based**: Every score must cite specific evidence from the output. - **Evaluate substance over style**: Correct, complete answers with rough formatting score higher than polished but incorrect answers. - **Handle missing data gracefully**: If an output lacks workspace changes or tool calls but others have them, score what is present — do not penalize for data the target wasn't expected to produce. -- **Respect grader signals**: When code-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily. +- **Respect grader signals**: When script-grader or tool-trajectory results exist, they represent objective ground truth. Weight these heavily. ## Edge Cases diff --git a/skills-data/agentv-bench/agents/grader.md b/skills-data/agentv-bench/agents/grader.md index 376783339..121e578f0 100644 --- a/skills-data/agentv-bench/agents/grader.md +++ b/skills-data/agentv-bench/agents/grader.md @@ -3,7 +3,7 @@ name: grader description: >- Grade a candidate response for an AgentV evaluation test case. Evaluates all assertion types natively — deterministic checks via string operations, LLM grading - via Claude's own reasoning, code-grader via Bash script execution. Zero CLI dependency. + via Claude's own reasoning, script-grader via Bash script execution. Zero CLI dependency. Dispatch this agent after a candidate completes a test case. model: inherit color: yellow @@ -72,7 +72,7 @@ For LLM-graded types: be rigorous and fair. Score based on substance, not exact | Type | How to evaluate | |------|----------------| -| `code-grader` | Run: `bun ` or `python `. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` | +| `script-grader` | Run: `bun ` or `python `. Pass response via file. Parse stdout JSON: `{"score": N, "reason": "..."}` | **Composite assertions** — evaluate sub-assertions, then aggregate per the configured mode (weighted_average, min, max, all_pass). diff --git a/skills-data/agentv-bench/references/description-optimization.md b/skills-data/agentv-bench/references/description-optimization.md index 6e90abcc2..d021e26b0 100644 --- a/skills-data/agentv-bench/references/description-optimization.md +++ b/skills-data/agentv-bench/references/description-optimization.md @@ -35,12 +35,12 @@ it's a natural utterance): tests: - id: should-trigger-casual-optimize input: "ok so I have this agent that keeps failing on the code review tasks, can you help me figure out why and fix it" - assertions: + assert: - type: skill-trigger skill: agentv-bench - id: should-not-trigger-build-error input: "my TypeScript build is failing with type errors in src/auth.ts" - assertions: + assert: - type: skill-trigger skill: agentv-bench should_trigger: false diff --git a/skills-data/agentv-bench/references/environment-adaptation.md b/skills-data/agentv-bench/references/environment-adaptation.md index f5a81c844..ef0cab3ee 100644 --- a/skills-data/agentv-bench/references/environment-adaptation.md +++ b/skills-data/agentv-bench/references/environment-adaptation.md @@ -29,28 +29,28 @@ to any platform with skill-discovery mechanisms. All listed providers support sk ## Unsupported Providers: Use a Code-Grader The built-in `skill-trigger` grader covers Claude, Copilot, Pi, Codex and VS Code out -of the box. For providers with different tool-call formats, write a code-grader that inspects +of the box. For providers with different tool-call formats, write a script-grader that inspects the agent's transcript messages or tool call trace. -A code-grader receives the full evaluation context including the final `output` string, +A script-grader receives the full evaluation context including the final `output` string, transcript `messages`, and structured `trace`. Inspect `messages` or `trace.events` for tool calls; reserve `output` for final-answer text checks. ```yaml -# Example: code-grader for Codex skill-trigger detection +# Example: script-grader for Codex skill-trigger detection tests: - id: should-trigger-codex input: "Analyze this CSV file" - assertions: + assert: - type: script command: [bun, run, ./judges/codex-skill-trigger.ts] ``` ```typescript // judges/codex-skill-trigger.ts -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ messages }) => { +export default defineScriptGrader(({ messages }) => { const skillName = 'csv-analyzer'; const toolCalls = messages.flatMap((msg) => msg.toolCalls ?? []); const firstTool = toolCalls[0]; @@ -65,7 +65,7 @@ export default defineCodeGrader(({ messages }) => { if (cmd.includes(skillName)) { return { score: 1, - assertions: [ + assert: [ { text: `Skill "${skillName}" triggered via command`, passed: true, evidence: cmd }, ], }; @@ -78,14 +78,14 @@ export default defineCodeGrader(({ messages }) => { if (path.includes(skillName)) { return { score: 1, - assertions: [{ text: 'Skill file accessed', passed: true, evidence: path }], + assert: [{ text: 'Skill file accessed', passed: true, evidence: path }], }; } } return { score: 0, - assertions: [ + assert: [ { text: `First tool was not a skill invocation for "${skillName}"`, passed: false, diff --git a/skills-data/agentv-bench/references/eval-yaml-spec.md b/skills-data/agentv-bench/references/eval-yaml-spec.md index 6b6303837..8dedf8017 100644 --- a/skills-data/agentv-bench/references/eval-yaml-spec.md +++ b/skills-data/agentv-bench/references/eval-yaml-spec.md @@ -45,7 +45,7 @@ including `criteria` and `expected_output` when present. When `assertions` is present, the list is explicit: run only the declared assertions/graders. `expected_output` remains reference data for graders that consume it, -such as `llm-rubric`, `code-grader`, or `field-accuracy`; it does not trigger an additional +such as `llm-rubric`, `script`, or `field-accuracy`; it does not trigger an additional default `llm-rubric`. When the declared assertion strings fully express the semantic contract, do not also add a duplicate `criteria` block. @@ -194,15 +194,15 @@ Same as contains variants but explicitly case-insensitive. ### Script-based assertions -#### `code-grader` +#### `script-grader` - **Fields:** `path` (string, required — path to script), `command` (string[], optional — custom command) -- **Script SDK:** Use `defineCodeGrader` from `@agentv/sdk`: +- **Script SDK:** Use `defineScriptGrader` from `@agentv/sdk`: ```typescript - import { defineCodeGrader } from '@agentv/sdk'; - export default defineCodeGrader(({ output, trace }) => ({ + import { defineScriptGrader } from '@agentv/sdk'; + export default defineScriptGrader(({ output, trace }) => ({ score: (output ?? '').includes('expected') ? 1 : 0, - assertions: [{ text: 'Contains expected', passed: (output ?? '').includes('expected') }], + assert: [{ text: 'Contains expected', passed: (output ?? '').includes('expected') }], })); ``` - **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{"score": N, "assertions": [...]}` @@ -268,7 +268,7 @@ Each line in the results JSONL file is an `EvaluationResult` object. In JSONL, f - `assertions` (array of `{text, passed, evidence?}`) - `weight` (number, optional) - `verdict` (string: `pass` | `fail` | `skip`) -- `details` (object, optional — structured data from code graders) +- `details` (object, optional — structured data from script graders) - `reasoning` (string, optional) ## 6. Eval Set Support @@ -301,7 +301,7 @@ Extracts inputs, target commands, and grader configs from an eval YAML file. │ ├── invoke.json ← {kind, command?, cwd?, timeout_ms?} │ ├── criteria.md ← human-readable success criteria │ ├── expected_output.json ← (if present) -│ ├── code_graders/.json ← {name, command, weight, config?} +│ ├── script_graders/.json ← {name, command, weight, config?} │ └── llm_graders/.json ← {name, weight, threshold?, prompt_content} ``` @@ -321,15 +321,15 @@ Extracts inputs, target commands, and grader configs from an eval YAML file. ### `agentv pipeline grade ` -Runs code-grader assertions against `response.md` files in each test directory. +Runs script-grader assertions against `response.md` files in each test directory. **Prerequisites:** `pipeline input` has been run and `response.md` exists in each test dir. -**Output:** `/code_grader_results/.json` for each code grader, containing: +**Output:** `/script_grader_results/.json` for each script grader, containing: ```json { "name": "grader-name", - "type": "code-grader", + "type": "script-grader", "score": 1.0, "weight": 1.0, "assertions": [{"text": "...", "passed": true}] @@ -338,7 +338,7 @@ Runs code-grader assertions against `response.md` files in each test directory. ### `agentv pipeline bench ` -Merges code-grader results with LLM grader scores and produces final artifacts. +Merges script-grader results with LLM grader scores and produces final artifacts. LLM grader results are read from disk at `/llm_grader_results/.json` per test. diff --git a/skills-data/agentv-bench/references/migrating-from-skill-creator.md b/skills-data/agentv-bench/references/migrating-from-skill-creator.md index 71d2103db..3f6410822 100644 --- a/skills-data/agentv-bench/references/migrating-from-skill-creator.md +++ b/skills-data/agentv-bench/references/migrating-from-skill-creator.md @@ -20,7 +20,7 @@ AgentV automatically: - Converts `assertions` → LLM-grader graders - Resolves `files[]` paths relative to the evals.json directory -If you're using the `agentv-bench` skill, it orchestrates these same AgentV commands. Code graders, grading, and artifact generation remain in AgentV core; the skill just orchestrates and summarizes the existing outputs. +If you're using the `agentv-bench` skill, it orchestrates these same AgentV commands. script graders, grading, and artifact generation remain in AgentV core; the skill just orchestrates and summarizes the existing outputs. ## What You Gain @@ -29,7 +29,7 @@ Moving from skill-creator's eval loop to AgentV's lifecycle skill gives you: | Capability | skill-creator | AgentV lifecycle skill | |-----------|---------------|----------------------| | Workspace isolation | ❌ | ✅ Clone repos, run setup/teardown scripts | -| Code graders | ❌ | ✅ Python/TypeScript grader scripts via `defineCodeGrader()` | +| script graders | ❌ | ✅ Python/TypeScript grader scripts via `defineScriptGrader()` | | Tool trajectory scoring | ❌ | ✅ Evaluate tool call sequences | | Multi-provider comparison | with-skill vs without-skill | N-way: Claude, GPT, Copilot, Gemini, custom CLI | | Multi-turn evaluation | ❌ | ✅ Conversation tracking with `conversation_id` | @@ -64,14 +64,14 @@ When evals.json becomes limiting, convert to EVAL.yaml for the full feature set: # Convert evals.json to EVAL.yaml agentv convert evals.json -# Edit the generated YAML to add workspace config, code graders, etc. +# Edit the generated YAML to add workspace config, script graders, etc. # Then run with the full lifecycle agentv eval eval.yaml ``` EVAL.yaml unlocks: - **Workspace setup/teardown** — clone repos, install dependencies, clean up after tests -- **Code graders** — write graders in Python or TypeScript, not just LLM prompts +- **script graders** — write graders in Python or TypeScript, not just LLM prompts - **Rubric-based grading** — multi-dimensional scoring with weighted criteria - **Retry policies** — automatic retries for flaky tests with configurable backoff - **Test groups** — organize tests by category with shared config diff --git a/skills-data/agentv-bench/references/subagent-pipeline.md b/skills-data/agentv-bench/references/subagent-pipeline.md index 81f09e476..ee548ee7e 100644 --- a/skills-data/agentv-bench/references/subagent-pipeline.md +++ b/skills-data/agentv-bench/references/subagent-pipeline.md @@ -50,12 +50,12 @@ code grading in one step. When `--out` is omitted, the output directory defaults # Extract inputs and invoke all CLI targets in parallel: agentv pipeline run evals/repro.eval.yaml -# Also run code graders inline (instead of using pipeline grade separately): +# Also run script graders inline (instead of using pipeline grade separately): agentv pipeline run evals/repro.eval.yaml --grader-type code ``` By default, `pipeline run` extracts inputs and invokes targets only. Pass `--grader-type code` -to also run code-graders inline, or use `agentv pipeline grade ` as a separate step. +to also run script-graders inline, or use `agentv pipeline grade ` as a separate step. The run directory is printed to stdout. Then continue to the grading and merge phases described in SKILL.md Step 3. @@ -109,7 +109,7 @@ grading until all responses are present. ### Step 3 onward: Grade and merge -See SKILL.md Step 3 "Grading" section for the three-phase grading process (code graders → +See SKILL.md Step 3 "Grading" section for the three-phase grading process (script graders → LLM grading → merge and validate). ## Step-by-Step Fine-Grained Control (CLI targets) @@ -122,7 +122,7 @@ agentv pipeline input evals/repro.eval.yaml # Step 2: run_tests.py invokes CLI targets (or use pipeline run instead) -# Step 3: Run code graders +# Step 3: Run script graders agentv pipeline grade # Step 4: Subagent does LLM grading, writes results to llm_grader_results/.json per test @@ -174,9 +174,9 @@ the eval.yaml. The target is recorded in `manifest.json` — one run = one targe ├── criteria.md ← grading criteria ├── response.md ← target/agent output ├── timing.json ← execution timing - ├── code_graders/.json ← grader configs written by `pipeline input`: code-grader scripts AND built-in types (contains, regex, equals, etc.) + ├── script_graders/.json ← grader configs written by `pipeline input`: script-grader scripts AND built-in types (contains, regex, equals, etc.) ├── llm_graders/.json ← LLM grader configs - ├── code_grader_results/.json ← code grader results + ├── script_grader_results/.json ← script grader results ├── llm_grader_results/.json ← LLM grader results (written by grader subagents; one file per grader) └── grading.json ← merged grading (written by `pipeline bench` — do NOT write here directly) ``` diff --git a/skills-data/agentv-eval-review/SKILL.md b/skills-data/agentv-eval-review/SKILL.md index f3f5ddbb5..0520c8c22 100644 --- a/skills-data/agentv-eval-review/SKILL.md +++ b/skills-data/agentv-eval-review/SKILL.md @@ -21,9 +21,9 @@ Walk every target eval file and report violations grouped by severity (error > w - File extension is `.eval.yaml` (error if not). - `description` field is present at the top level (error if missing). -- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assertions` (error if missing). +- Each entry under `tests` has `id`, `input`, and at least one of `criteria` / `expected_output` / `assert` (error if missing). - File-typed inputs (`type: file`) use a leading `/` in their `path` (error if relative). -- Tests have an `assertions` block — flag tests that rely solely on `expected_output` (warning). +- Tests have an `assert` block — flag tests that rely solely on `expected_output` (warning). - Flag `criteria` that duplicates assertion strings when `assertions` already express the grading contract (warning — remove the duplicate `criteria`). - Prefer plain assertion strings over multiple named `type: llm-rubric` blocks when the default LLM rubric grader can evaluate the checks (info unless custom prompts or grader targets are present). - Detect `expected_output` prose patterns like "The agent should..." or "The output is..." (warning — `expected_output` should be a golden/reference answer; scoring rules belong in `assertions` or, for implicit-grader cases, `criteria`). diff --git a/skills-data/agentv-eval-review/scripts/lint_eval.py b/skills-data/agentv-eval-review/scripts/lint_eval.py index 1ba450885..9fb507496 100644 --- a/skills-data/agentv-eval-review/scripts/lint_eval.py +++ b/skills-data/agentv-eval-review/scripts/lint_eval.py @@ -8,7 +8,7 @@ - description field present - Each test has id, input, criteria - File paths in type:file use leading / - - assertions blocks present (not relying solely on expected_output) + - assert blocks present (not relying solely on expected_output) - expected_output does not contain evaluation criteria prose - Repeated file inputs across tests (should use top-level input) - Naming prefix consistency across eval files in same directory diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 9e7e1171a..1179740f4 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -681,12 +681,12 @@ export default defineAssertion(({ output, trace }) => { Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass` is given, score is 1 (pass) or 0 (fail). -### defineCodeGrader (full control) +### defineScriptGrader (full control) ```typescript #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, trace }) => { +export default defineScriptGrader(({ output, trace }) => { const finalOutput = output ?? ''; return { score: finalOutput.length > 0 && (trace?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, @@ -698,7 +698,7 @@ export default defineCodeGrader(({ output, trace }) => { }); ``` -`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: `. `defineCodeGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`. +`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: `. `defineScriptGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`. ### Convention-Based Discovery diff --git a/skills-data/agentv-eval-writer/references/custom-evaluators.md b/skills-data/agentv-eval-writer/references/custom-evaluators.md index b8ce0357a..df8b0f10c 100644 --- a/skills-data/agentv-eval-writer/references/custom-evaluators.md +++ b/skills-data/agentv-eval-writer/references/custom-evaluators.md @@ -53,14 +53,14 @@ ```typescript import { createTargetClient, - defineCodeGrader, + defineScriptGrader, defineEval, definePromptTemplate, graders, } from '@agentv/sdk'; ``` -- `defineCodeGrader(fn)` - Wraps evaluation function with stdin/stdout handling +- `defineScriptGrader(fn)` - Wraps evaluation function with stdin/stdout handling - `defineEval(definition)` - Defines a YAML-aligned `.eval.ts` suite - `graders` - Helper catalog that returns ordinary AgentV `assert` entries - `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured) @@ -119,7 +119,7 @@ assert: ```python #!/usr/bin/env python3 -from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader +from agentv_py.grader import Assertion, ScriptGraderResult, define_script_grader def evaluate(context): @@ -128,23 +128,23 @@ def evaluate(context): for kw in ["async", "await"]: assertions.append(Assertion(text=f"Keyword '{kw}'", passed=kw in candidate)) passed = sum(1 for item in assertions if item.passed) - return CodeGraderResult( + return ScriptGraderResult( score=passed / max(len(assertions), 1), assertions=assertions, ) if __name__ == "__main__": - define_code_grader(evaluate) + define_script_grader(evaluate) ``` ## TypeScript Example ```typescript #!/usr/bin/env bun -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, expectedOutput }) => { +export default defineScriptGrader(({ output, expectedOutput }) => { const candidate = output ?? ''; const expected = expectedOutput ?.map((message) => (typeof message.content === 'string' ? message.content : '')) diff --git a/skills-data/agentv-eval-writer/references/python-helpers.md b/skills-data/agentv-eval-writer/references/python-helpers.md index 88ea64a19..bffe46c5d 100644 --- a/skills-data/agentv-eval-writer/references/python-helpers.md +++ b/skills-data/agentv-eval-writer/references/python-helpers.md @@ -16,8 +16,8 @@ Use it when the user wants Python-based custom graders or wants to emit AgentV Y - `agentv_py.grader` - `load_grader_input()` - - `run_code_grader(handler)` - - `define_code_grader(handler)` + - `run_script_grader(handler)` + - `define_script_grader(handler)` - `TargetClient.from_env()` - `agentv_py.evals` - `EvalDefinition` @@ -30,21 +30,21 @@ Use it when the user wants Python-based custom graders or wants to emit AgentV Y ## Example ```python -from agentv_py.grader import Assertion, CodeGraderResult, define_code_grader +from agentv_py.grader import Assertion, ScriptGraderResult, define_script_grader def evaluate(context): actual = context.output or "" expected = context.expected_output[0]["content"] passed = actual.strip() == expected.strip() - return CodeGraderResult( + return ScriptGraderResult( score=1.0 if passed else 0.0, assertions=[Assertion(text="Exact match", passed=passed)], ) if __name__ == "__main__": - define_code_grader(evaluate) + define_script_grader(evaluate) ``` ## YAML helper example