From 402f46f43ca6b33ad78f7f890d846163189cd4e3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 07:27:34 +0200 Subject: [PATCH] docs(sdk): clarify custom assertion terminology --- README.md | 3 +- .../content/docs/docs/next/evaluation/sdk.mdx | 7 ++- .../docs/next/getting-started/quickstart.mdx | 2 +- .../docs/next/graders/custom-assertions.mdx | 14 +++-- .../docs/docs/next/graders/custom-graders.mdx | 3 +- .../docs/docs/next/graders/script-graders.mdx | 2 + .../docs/docs/v4.42.4/evaluation/sdk.mdx | 57 ++++++++++--------- .../v4.42.4/getting-started/quickstart.mdx | 6 +- .../docs/v4.42.4/graders/code-graders.mdx | 2 + .../v4.42.4/graders/custom-assertions.mdx | 22 ++++--- .../docs/v4.42.4/graders/custom-graders.mdx | 47 +++++++-------- examples/features/script-grader-sdk/README.md | 6 +- .../features/sdk-custom-assertion/README.md | 2 + packages/sdk/README.md | 10 ++-- packages/sdk/package.json | 2 +- packages/sdk/src/index.ts | 10 ++-- packages/sdk/src/schemas.ts | 2 +- skills-data/agentv-eval-writer/SKILL.md | 6 +- .../references/custom-evaluators.md | 6 +- 19 files changed, 121 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index e08710e75..d6e1f7795 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,8 @@ export default defineEval({ Full docs at [agentv.dev/docs](https://agentv.dev/docs/getting-started/introduction/). - [Eval files](https://agentv.dev/docs/evaluation/eval-files/) — format and structure -- [Custom graders](https://agentv.dev/docs/graders/custom-graders/) — script graders in any language +- [Custom assertions](https://agentv.dev/docs/graders/custom-assertions/) — reusable assertion types +- [Script graders](https://agentv.dev/docs/graders/script-graders/) — command-backed graders in any language - [Rubrics](https://agentv.dev/docs/evaluation/rubrics/) — structured criteria scoring - [Targets](https://agentv.dev/docs/targets/configuration/) — configure agents and providers - [Compare results](https://agentv.dev/docs/tools/compare/) — A/B testing and regression detection diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 2e55aca0f..e8ea95a53 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -46,7 +46,8 @@ Use the simplest surface that matches the job: - **`defineEval()` / `evalSuite()`** when you want a `.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput`. -- **`defineAssertion` / `defineScriptGrader`** when the grading logic itself must execute code. +- **`defineAssertion`** when you want a reusable assertion type discovered from `.agentv/assertions/`. +- **`defineScriptGrader`** when you need a command-backed grader with explicit score and assertion-result control. - **`agentv eval `** for deterministic workspace checks that fit normal Vitest `expect(...)` tests. There is no separate first-party Python authoring SDK today. Python-facing workflows should either emit canonical YAML/JSONL or implement executable graders that consume the standard `snake_case` wire format. @@ -209,6 +210,8 @@ assert: Use `defineAssertion` from `@agentv/sdk` to create reusable assertion types. Place them in `.agentv/assertions/` — they're auto-discovered by filename. +This is the custom assertion path, not the custom grader path. It matches Promptfoo's assertion terminology for normal eval checks, while extending Promptfoo's fixed custom logic types (`javascript`, `python`, `ruby`, `webhook`) with arbitrary discovered AgentV type names. + ### Pass/Fail Pattern ```typescript @@ -312,7 +315,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ ]); ``` -`defineScriptGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. +`defineScriptGrader`, `defineVitestWorkspaceGrader`, and `defineWorkspaceGrader` custom scripts are graders referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest verifier files can use `command: [agentv, eval, graders/check.test.ts]` without a custom wrapper; use `agentv eval vitest` when you need adapter flags. `defineAssertion` uses convention-based discovery instead — just place it in `.agentv/assertions/` and reference it by assertion type name. For detailed patterns, input/output contracts, and language-agnostic examples, see [Script Graders](/docs/graders/script-graders/). diff --git a/apps/web/src/content/docs/docs/next/getting-started/quickstart.mdx b/apps/web/src/content/docs/docs/next/getting-started/quickstart.mdx index 43971a8b3..f0abe7591 100644 --- a/apps/web/src/content/docs/docs/next/getting-started/quickstart.mdx +++ b/apps/web/src/content/docs/docs/next/getting-started/quickstart.mdx @@ -72,5 +72,5 @@ Results appear in `.agentv/results//.internal/index.jsonl` with scores, - Learn about [eval file formats](/docs/evaluation/eval-files/) - Configure [targets](/docs/targets/configuration/) for different providers -- Create [custom graders](/docs/graders/custom-graders/) +- Choose [custom assertions](/docs/graders/custom-assertions/) or [script graders](/docs/graders/script-graders/) - If setup drifts, rerun: `agentv init` diff --git a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx index fcd12376c..494ef7413 100644 --- a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx @@ -6,7 +6,7 @@ sidebar: slug: docs/graders/custom-assertions --- -Custom assertions let you add evaluation logic that goes beyond built-in types. Define a TypeScript function, drop it in `.agentv/assertions/`, and reference it by name in your YAML eval files. +Custom assertions let you add reusable assertion types that go beyond built-in types. Define a TypeScript function, drop it in `.agentv/assertions/`, and reference it by name in your YAML eval files. ## When to Use Each Approach @@ -14,15 +14,21 @@ AgentV provides two SDK functions for custom evaluation logic: | Function | Best For | Discovery | |----------|----------|-----------| -| `defineAssertion()` | Pass/fail checks, reusable assertion types | Convention-based (`.agentv/assertions/`) | -| `defineScriptGrader()` | Full scoring control with explicit assertions array | Referenced via `type: script` + `command:` | +| `defineAssertion()` | Reusable assertion types with pass/fail plus optional score | Convention-based (`.agentv/assertions/`) | +| `defineScriptGrader()` | Command-backed scorer with full score and assertion-result control | Referenced via `type: script` + `command:` | **Use `defineAssertion()`** when you want a named assertion type that can be referenced across eval files without specifying a command path. It uses a simplified result contract focused on `pass` and optional `score`. -**Use `defineScriptGrader()`** when you need full control over scoring with explicit `assertions` arrays, or when the grader is a one-off grader tied to a specific eval. See [Script Graders](/docs/graders/script-graders/) for details. +**Use `defineScriptGrader()`** when the scoring component is a command-backed grader: it needs explicit score calculation, custom assertion-result arrays, workspace commands, or LLM calls through a grader target. See [Script Graders](/docs/graders/script-graders/) for details. Both functions handle stdin/stdout JSON parsing, snake_case-to-camelCase conversion, Zod validation, and error handling automatically. +## Promptfoo Terminology + +Promptfoo calls normal eval checks assertions. Its custom code paths use fixed assertion types such as `javascript`, `python`, `ruby`, and `webhook`, and its Node API exposes assertion-oriented helpers such as `runAssertion()` and `runAssertions()`. + +AgentV follows that framing for `assert:` entries and `defineAssertion()`. The AgentV extension is convention discovery: any file in `.agentv/assertions/` becomes an assertion type name such as `word-count` or `has-citation`. Reserve custom grader or script grader wording for command-backed or LLM-backed scoring components, especially `type: script` entries built with `defineScriptGrader()`. + ## Installation ```bash diff --git a/apps/web/src/content/docs/docs/next/graders/custom-graders.mdx b/apps/web/src/content/docs/docs/next/graders/custom-graders.mdx index f88c90908..62525824b 100644 --- a/apps/web/src/content/docs/docs/next/graders/custom-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/custom-graders.mdx @@ -6,7 +6,7 @@ sidebar: slug: docs/graders/custom-graders --- -AgentV supports multiple grader types that can be combined for comprehensive evaluation. +AgentV supports multiple grader types that can be combined for comprehensive evaluation. Use this page for command-backed or LLM-backed scoring components. For reusable assertion types discovered from `.agentv/assertions/`, see [Custom Assertions](/docs/graders/custom-assertions/). ## Grader Types @@ -83,5 +83,6 @@ If any grader has `required: true` and scores below its required threshold, the - **Use plain assertion strings first for semantic checks** — AgentV treats them as rubric criteria - **Use script graders for deterministic checks** — exact value matching, format validation, schema compliance - **Use `llm-rubric` for semantic evaluation** — meaning, quality, helpfulness, or weighted itemized scoring +- **Use custom assertions for reusable pass/fail types** — define them with `defineAssertion()` and reference them by discovered type name - **Combine grader types** for comprehensive coverage - **Test script graders locally** before running full evaluations diff --git a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx index a95b60044..3d3acaaed 100644 --- a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -8,6 +8,8 @@ slug: docs/graders/script-graders Script graders are scripts that evaluate agent responses deterministically. Write them in any language — Python, TypeScript, Node, or any executable. +Use script graders when you need a command-backed scoring component with explicit score control. If you only need a reusable assertion type that can be referenced by name from `.agentv/assertions/`, use [Custom Assertions](/docs/graders/custom-assertions/) instead. + ## Contract Script graders receive eval context via stdin JSON and return a result via stdout. diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx index 83f81f091..1186478b3 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx @@ -12,13 +12,13 @@ YAML remains AgentV's canonical, portable eval format. The SDK surfaces below ar AgentV currently provides two npm packages for programmatic use: -- **`@agentv/sdk`** — YAML-aligned eval authoring, custom assertions, and code graders +- **`@agentv/sdk`** — YAML-aligned eval authoring, custom assertions, and script graders - **`@agentv/core`** — programmatic evaluation API and typed configuration ## Installation ```bash -# Lightweight SDK (defineEval, graders, defineAssertion, defineCodeGrader) +# Lightweight SDK (defineEval, graders, defineAssertion, defineScriptGrader) npm install @agentv/sdk # Programmatic API (evaluate, defineConfig) @@ -35,7 +35,7 @@ npm install @agentv/sdk ``` ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; ``` The general policy is hard convergence for same-week or unreleased surface names: use the correct package, field, or wire name instead of carrying aliases. The package rename is the exception because `@agentv/eval` was already published. It remains a temporary deprecated compatibility package that re-exports `@agentv/sdk` for existing consumers, but it should not appear in new docs, examples, scaffolds, or skills except as migration guidance. @@ -48,11 +48,12 @@ Use the simplest surface that matches the job: - **`defineEval()` / `evalSuite()`** when you want a `.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. - **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput` and `assert`. -- **`defineAssertion` / `defineCodeGrader`** when the grading logic itself must execute code. +- **`defineAssertion`** when you want a reusable assertion type discovered from `.agentv/assertions/`. +- **`defineScriptGrader`** when you need a command-backed grader with explicit score and assertion-result control. There is no separate first-party Python authoring SDK today. Python-facing workflows should either emit canonical YAML/JSONL or implement executable graders that consume the standard `snake_case` wire format. -For example, the repo-local helper in `examples/features/sdk-python/` can build YAML-shaped cases while keeping `assertions` as the durable contract: +For example, the repo-local helper in `examples/features/sdk-python/` can build YAML-shaped cases while keeping `assert` as the durable authored contract: ```python from agentv_py.evals import EvalDefinition, JsonlCase, write_eval_yaml, write_jsonl @@ -61,7 +62,7 @@ from agentv_py.evals import EvalDefinition, JsonlCase, write_eval_yaml, write_js def rag_faithfulness(): return { "name": "rag-faithfulness", - "type": "llm-grader", + "type": "llm-rubric", "target": "grader-target", "prompt": "Grade whether the answer is supported by the retrieved context.", } @@ -74,7 +75,7 @@ write_jsonl( id="grounded-answer", input=[{"role": "user", "content": "Answer using the retrieved context."}], expected_output=[{"role": "assistant", "content": "The answer cites the source material."}], - extra={"assertions": [rag_faithfulness()]}, + extra={"assert": [rag_faithfulness()]}, ) ], ) @@ -113,7 +114,7 @@ export default defineEval({ input: 'Say hello', inputFiles: ['../fixtures/per-test-note.md'], expectedOutput: 'Hello from the mock target', - assertions: [graders.contains('Hello')], + assert: [graders.contains('Hello')], }, ], }); @@ -124,11 +125,11 @@ Useful companion helpers: - `toEvalYamlObject()` returns the canonical snake_case object. - `serializeEvalYaml()` returns YAML text using the same canonical field names. -The durable field remains `assertions`. This helper does not introduce a second YAML vocabulary. +The durable authored field remains `assert`. This helper does not introduce a second YAML vocabulary. ## Built-In Grader Helpers -`@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assertions` entries and serialize to the same canonical YAML you could write by hand. +`@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand. ```typescript import { defineEval, graders } from '@agentv/sdk'; @@ -139,35 +140,35 @@ export default defineEval({ { id: 'json-greeting', input: 'Return a JSON greeting.', - assertions: [ + assert: [ graders.contains('Hello', { name: 'mentions-hello' }), graders.exact('{"message":"Hello"}', { name: 'exact-json', minScore: 1 }), graders.regex(/"message"\s*:/, { name: 'message-key' }), graders.json({ name: 'valid-json', required: true }), - graders.rubrics(['Greets the user'], { name: 'rubric-review' }), - graders.llmGrader({ + graders.llmRubric(['Greets the user'], { name: 'rubric-review' }), + graders.llmRubric(undefined, { name: 'llm-review', prompt: 'Grade whether the answer is useful.', target: 'grader-target', }), - graders.codeGrader(['bun', 'run', 'graders/check.ts'], { name: 'scripted-check' }), + graders.scriptGrader(['bun', 'run', 'graders/check.ts'], { name: 'scripted-check' }), ], }, ], }); ``` -The catalog covers `contains`, `equals`/`exact`, `regex`, `is-json`/`json`, `rubrics`, `llm-grader`, and `code-grader`. CamelCase SDK options such as `minScore`, `maxSteps`, and rubric `scoreRanges` lower to `min_score`, `max_steps`, and `score_ranges` when AgentV loads or serializes the suite. +The catalog covers `contains`, `equals`/`exact`, `regex`, `is-json`/`json`, `llm-rubric`, and `script`. CamelCase SDK options such as `minScore`, `maxSteps`, and rubric `scoreRanges` lower to `min_score`, `max_steps`, and `score_ranges` when AgentV loads or serializes the suite. ## AgentV-Native Helper Factories -If you are coming from Braintrust `scores` or DeepEval metrics, keep the reusable logic AgentV-native: write helper factories that return `graders.*` configs, then let `defineEval()` lower them to ordinary `assertions`. +If you are coming from Braintrust `scores` or DeepEval metrics, keep the reusable logic AgentV-native: write helper factories that return `graders.*` configs, then let `defineEval()` lower them to ordinary `assert` entries. ```typescript import { defineEval, graders } from '@agentv/sdk'; function ragFaithfulness() { - return graders.llmGrader({ + return graders.llmRubric(undefined, { name: 'rag-faithfulness', target: 'grader-target', prompt: [ @@ -184,7 +185,7 @@ export default defineEval({ id: 'grounded-answer', input: 'Answer the question using the retrieved context.', expectedOutput: 'The answer cites the source material.', - assertions: [ + assert: [ graders.contains('source', { name: 'mentions-source' }), ragFaithfulness(), ], @@ -196,12 +197,12 @@ export default defineEval({ The helper above serializes to the same shape you could write by hand: ```yaml -assertions: +assert: - name: mentions-source type: contains value: source - name: rag-faithfulness - type: llm-grader + type: llm-rubric target: grader-target prompt: |- Grade whether the answer is supported by the retrieved context. @@ -212,6 +213,8 @@ assertions: Use `defineAssertion` from `@agentv/sdk` to create reusable assertion types. Place them in `.agentv/assertions/` — they're auto-discovered by filename. +This is the custom assertion path, not the custom grader path. It matches Promptfoo's assertion terminology for normal eval checks, while extending Promptfoo's fixed custom logic types (`javascript`, `python`, `ruby`, `webhook`) with arbitrary discovered AgentV type names. + ### Pass/Fail Pattern ```typescript @@ -260,20 +263,20 @@ Convention-based discovery maps filename → assertion type: Reference directly in your eval file — no `command:` needed: ```yaml -assertions: +assert: - type: word-count - type: contains value: "Hello" ``` -## Code Graders +## Script Graders -Use `defineCodeGrader` from `@agentv/sdk` for full control over scoring with an explicit assertions array: +Use `defineScriptGrader` from `@agentv/sdk` for full control over scoring with an explicit assertions array: ```typescript -import { defineCodeGrader } from '@agentv/sdk'; +import { defineScriptGrader } from '@agentv/sdk'; -export default defineCodeGrader(({ output, traceSummary }) => ({ +export default defineScriptGrader(({ output, traceSummary }) => ({ score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, assertions: [ { text: 'Answer is not empty', passed: (output ?? '').length > 0 }, @@ -282,9 +285,9 @@ export default defineCodeGrader(({ output, traceSummary }) => ({ })); ``` -`defineCodeGrader` graders are referenced in YAML with `type: code-grader` and `command: [bun, run, grader.ts]`. `defineAssertion` uses convention-based discovery instead — just place in `.agentv/assertions/` and reference by name. +`defineScriptGrader` scripts are graders referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. `defineAssertion` uses convention-based discovery instead — just place it in `.agentv/assertions/` and reference it by assertion type name. -For detailed patterns, input/output contracts, and language-agnostic examples, see [Code Graders](/docs/v4.42.4/graders/code-graders/). +For detailed patterns, input/output contracts, and language-agnostic examples, see [Code Graders](/docs/v4.42.4/graders/code-graders/) in this versioned doc set. ## Wire Format vs SDK Format diff --git a/apps/web/src/content/docs/docs/v4.42.4/getting-started/quickstart.mdx b/apps/web/src/content/docs/docs/v4.42.4/getting-started/quickstart.mdx index bb3a23aa9..fcba3976f 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/getting-started/quickstart.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/getting-started/quickstart.mdx @@ -57,9 +57,9 @@ tests: expected_output: "42" - assertions: + assert: - name: math_check - type: code-grader + type: script command: [./validators/check_math.py] ``` @@ -75,5 +75,5 @@ Results appear in `.agentv/results/runs//index.jsonl` with scores, re - Learn about [eval file formats](/docs/v4.42.4/evaluation/eval-files/) - Configure [targets](/docs/v4.42.4/targets/configuration/) for different providers -- Create [custom graders](/docs/v4.42.4/graders/custom-graders/) +- Choose [custom assertions](/docs/v4.42.4/graders/custom-assertions/) or command-backed [custom graders](/docs/v4.42.4/graders/custom-graders/) - If setup drifts, rerun: `agentv init` diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/code-graders.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/code-graders.mdx index ff0671552..9f899a16d 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/code-graders.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/code-graders.mdx @@ -10,6 +10,8 @@ pagefind: false Code graders are scripts that evaluate agent responses deterministically. Write them in any language — Python, TypeScript, Node, or any executable. +Use command-backed graders when you need explicit score control. If you only need a reusable assertion type that can be referenced by name from `.agentv/assertions/`, use [Custom Assertions](/docs/v4.42.4/graders/custom-assertions/) instead. + ## Contract Code graders receive eval context via stdin JSON and return a result via stdout. diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/custom-assertions.mdx index 8ad3bc45a..bea77647e 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/custom-assertions.mdx @@ -8,7 +8,7 @@ editUrl: false pagefind: false --- -Custom assertions let you add evaluation logic that goes beyond built-in types. Define a TypeScript function, drop it in `.agentv/assertions/`, and reference it by name in your YAML eval files. +Custom assertions let you add reusable assertion types that go beyond built-in types. Define a TypeScript function, drop it in `.agentv/assertions/`, and reference it by name in your YAML eval files. ## When to Use Each Approach @@ -16,15 +16,21 @@ AgentV provides two SDK functions for custom evaluation logic: | Function | Best For | Discovery | |----------|----------|-----------| -| `defineAssertion()` | Pass/fail checks, reusable assertion types | Convention-based (`.agentv/assertions/`) | -| `defineCodeGrader()` | Full scoring control with explicit assertions array | Referenced via `type: code-grader` + `command:` | +| `defineAssertion()` | Reusable assertion types with pass/fail plus optional score | Convention-based (`.agentv/assertions/`) | +| `defineScriptGrader()` | Command-backed scorer with full score and assertion-result control | Referenced via `type: script` + `command:` | **Use `defineAssertion()`** when you want a named assertion type that can be referenced across eval files without specifying a command path. It uses a simplified result contract focused on `pass` and optional `score`. -**Use `defineCodeGrader()`** when you need full control over scoring with explicit `assertions` arrays, or when the grader is a one-off grader tied to a specific eval. See [Code Graders](/docs/v4.42.4/graders/code-graders/) for details. +**Use `defineScriptGrader()`** when the scoring component is a command-backed grader: it needs explicit score calculation, custom assertion-result arrays, workspace commands, or LLM calls through a grader target. See [Code Graders](/docs/v4.42.4/graders/code-graders/) for the command-backed grader contract in this versioned doc set. Both functions handle stdin/stdout JSON parsing, snake_case-to-camelCase conversion, Zod validation, and error handling automatically. +## Promptfoo Terminology + +Promptfoo calls normal eval checks assertions. Its custom code paths use fixed assertion types such as `javascript`, `python`, `ruby`, and `webhook`, and its Node API exposes assertion-oriented helpers such as `runAssertion()` and `runAssertions()`. + +AgentV follows that framing for `assert:` entries and `defineAssertion()`. The AgentV extension is convention discovery: any file in `.agentv/assertions/` becomes an assertion type name such as `word-count` or `has-citation`. Reserve custom grader or script grader wording for command-backed or LLM-backed scoring components, especially `type: script` entries built with `defineScriptGrader()`. + ## Installation ```bash @@ -52,7 +58,7 @@ Custom assertion types cannot override built-in types (`contains`, `equals`, `is Reference the assertion by type name directly -- no `command:` path needed: ```yaml -assertions: +assert: - type: word-count - type: contains value: "Hello" @@ -114,7 +120,7 @@ The handler must return an `AssertionScore` object: ## Context Available to Assertions -The handler receives an `AssertionContext` with the same fields as a code grader: +The handler receives an `AssertionContext` with the same fields as a script grader: | Field | Type | Description | |-------|------|-------------| @@ -231,7 +237,7 @@ tests: criteria: Agent gives a multi-word greeting input: "Say hello and introduce yourself" expected_output: "Hello! I'm an AI assistant here to help you." - assertions: + assert: - type: contains value: "Hello" - type: word-count @@ -240,7 +246,7 @@ tests: criteria: Agent gives a short but valid response input: "What is 2+2?" expected_output: "The answer is 4." - assertions: + assert: - type: contains value: "4" - type: word-count diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/custom-graders.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/custom-graders.mdx index 6690791b2..bb6b98b3f 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/custom-graders.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/custom-graders.mdx @@ -8,27 +8,26 @@ editUrl: false pagefind: false --- -AgentV supports multiple grader types that can be combined for comprehensive evaluation. +AgentV supports multiple grader types that can be combined for comprehensive evaluation. Use this page for command-backed or LLM-backed scoring components. For reusable assertion types discovered from `.agentv/assertions/`, see [Custom Assertions](/docs/v4.42.4/graders/custom-assertions/). ## Grader Types | Type | Description | Use Case | |------|-------------|----------| -| `code_grader` | Deterministic command (Python/TS/any) | Exact matching, format validation, programmatic checks | -| `llm_grader` | LLM-based evaluation with custom prompt | Semantic evaluation, nuance, subjective quality | -| `rubrics` | Structured rubric grader via `assertions` | Multi-criterion grading with weights | +| `script` | Deterministic command (Python/TS/any) | Exact matching, format validation, programmatic checks | +| `llm-rubric` | LLM-backed rubric grading, including custom prompts | Semantic evaluation, nuance, weighted criteria | ## Referencing Graders -Graders are configured using `assertions` — either top-level (applies to all tests) or per-test: +Graders are configured using `assert` — either top-level (applies to all tests) or per-test: ### Top-Level (Default for All Tests) ```yaml description: My evaluation -assertions: +assert: - name: correctness - type: llm-grader + type: llm-rubric prompt: ./graders/correctness.md tests: @@ -42,11 +41,11 @@ tests: ```yaml tests: - id: test-1 - criteria: Returns valid JSON input: Generate a JSON config - assertions: + assert: + - Returns valid JSON - name: json_check - type: code-grader + type: script command: [./validators/check_json.py] ``` @@ -57,37 +56,35 @@ Use multiple graders on the same case for comprehensive scoring: ```yaml tests: - id: code-generation - criteria: Generates correct Python code input: Write a sorting function - assertions: - - type: rubrics - criteria: - - Code is syntactically valid - - Handles edge cases (empty list, single element) - - Uses appropriate algorithm + assert: + - Code is syntactically valid + - Handles edge cases such as empty lists and single-element lists + - Uses an appropriate algorithm - name: syntax_check - type: code-grader + type: script command: [./validators/check_syntax.py] - name: quality_review - type: llm-grader + type: llm-rubric prompt: ./graders/code_quality.md ``` Each grader produces its own score. Results appear in `scores[]` in the output JSONL. -For multiple graders in `assertions`, the test score is the weighted mean: +For multiple graders in `assert`, the test score is the weighted mean: ``` final_score = sum(score_i * weight_i) / sum(weight_i) ``` If `weight` is omitted, it defaults to `1.0` (equal weighting). -If any grader has `required: true` (or `required: `) and scores below its required threshold, the overall test score is forced to `0`. +If any grader has `required: true` and scores below its required threshold, the overall test score is forced to `0`. Use `min_score` for a custom threshold. ## Best Practices -- **Use code graders for deterministic checks** — exact value matching, format validation, schema compliance -- **Use LLM graders for semantic evaluation** — meaning, quality, helpfulness -- **Use rubrics for structured multi-criteria grading** — when you need weighted, itemized scoring +- **Use plain assertion strings first for semantic checks** — AgentV treats them as rubric criteria +- **Use script graders for deterministic checks** — exact value matching, format validation, schema compliance +- **Use `llm-rubric` for semantic evaluation** — meaning, quality, helpfulness, or weighted itemized scoring +- **Use custom assertions for reusable pass/fail types** — define them with `defineAssertion()` and reference them by discovered type name - **Combine grader types** for comprehensive coverage -- **Test code graders locally** before running full evaluations +- **Test script graders locally** before running full evaluations diff --git a/examples/features/script-grader-sdk/README.md b/examples/features/script-grader-sdk/README.md index f6af588a7..5a52c58fb 100644 --- a/examples/features/script-grader-sdk/README.md +++ b/examples/features/script-grader-sdk/README.md @@ -1,6 +1,8 @@ # script grader SDK Helper -Demonstrates how a TypeScript `script-grader` can use `defineScriptGrader` from `@agentv/sdk` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. +Demonstrates how a TypeScript script grader can use `defineScriptGrader` from `@agentv/sdk` for a declarative, low-boilerplate approach while still consuming the canonical AgentV wire format. + +Use this pattern for command-backed graders referenced with `type: script`. For reusable assertion types discovered from `.agentv/assertions/`, use `defineAssertion()` instead. ## Files @@ -60,6 +62,6 @@ import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, criteria }) => ({ score: (output ?? '').includes(criteria) ? 1.0 : 0.0, - assert: [{ text: 'Check passed', passed: (output ?? '').includes(criteria) }], + assertions: [{ text: 'Check passed', passed: (output ?? '').includes(criteria) }], })); ``` diff --git a/examples/features/sdk-custom-assertion/README.md b/examples/features/sdk-custom-assertion/README.md index 35df71ee5..20f6bbb3c 100644 --- a/examples/features/sdk-custom-assertion/README.md +++ b/examples/features/sdk-custom-assertion/README.md @@ -24,3 +24,5 @@ agentv eval evals/suite.yaml - **`defineAssertion()`** — simplest way to add custom evaluation logic - **Convention discovery** — files in `.agentv/assertions/` are auto-discovered by type name - **Pass/fail with reasoning** — return `{ pass, reasoning }` for clear results + +Use this pattern for reusable assertion types. Use `defineScriptGrader()` with `type: script` when you need a command-backed grader with full score control. diff --git a/packages/sdk/README.md b/packages/sdk/README.md index f78f6200b..7428ac4e8 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -1,6 +1,6 @@ # @agentv/sdk -Public lightweight SDK for AgentV - run evaluations programmatically, build YAML-aligned eval suites, and write custom graders and prompt templates around the canonical AgentV eval model. +Public lightweight SDK for AgentV - run evaluations programmatically, build YAML-aligned eval suites, and write custom assertions, script graders, and prompt templates around the canonical AgentV eval model. ## Installation @@ -79,14 +79,14 @@ import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, traceSummary }) => ({ score: (output ?? '').length > 0 ? 1.0 : 0.0, - assert: [ + assertions: [ { text: 'Output received', passed: (output ?? '').length > 0 }, { text: 'Trace summary available', passed: traceSummary !== null }, ], })); ``` -Both functions handle stdin/stdout parsing, snake_case conversion, Zod validation, and error handling automatically. +Both functions handle stdin/stdout parsing, snake_case conversion, Zod validation, and error handling automatically. Use `defineAssertion()` for reusable assertion types discovered from `.agentv/assertions/`; use `defineScriptGrader()` for command-backed graders referenced with `type: script` and `command:`. ### Vitest workspace verifiers (preferred deterministic workspace checks) @@ -228,8 +228,8 @@ Python workflows should emit canonical YAML/JSONL or implement script graders ov ## Exports - `evaluate(config)` - Run evaluations programmatically from inline tests or an eval spec file -- `defineAssertion(handler)` - Define a custom assertion (pass/fail + optional score) -- `defineScriptGrader(handler)` - Define a script grader (full score control) +- `defineAssertion(handler)` - Define a custom assertion type (pass/fail + optional score) +- `defineScriptGrader(handler)` - Define a script grader (command-backed full score control) - `defineVitestWorkspaceGrader(options)` - Embed the Vitest workspace verifier adapter in a custom script - `defineWorkspaceGrader(handler)` - Define a workspace-aware script grader with file assertion helpers - `definePromptTemplate(handler)` - Define a dynamic prompt template diff --git a/packages/sdk/package.json b/packages/sdk/package.json index faae633ca..d5841519c 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,7 +1,7 @@ { "name": "@agentv/sdk", "version": "5.0.0-next.1", - "description": "Evaluation SDK for AgentV - build custom code judges", + "description": "Evaluation SDK for AgentV - build custom assertions and script graders", "type": "module", "repository": { "type": "git", diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 101fa9422..d492586b1 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -1,7 +1,7 @@ /** * AgentV Evaluation SDK * - * Build custom graders for AI agent outputs. + * Build custom assertions, script graders, and eval authoring helpers for AI agent outputs. * * @example Custom assertion (simplest way to add evaluation logic) * ```typescript @@ -341,11 +341,13 @@ export function definePromptTemplate(handler: PromptTemplateHandler): void { } /** - * Define a custom assertion grader with automatic stdin/stdout handling. + * Define a custom assertion with automatic stdin/stdout handling. * - * Assertions are the simplest way to add custom evaluation logic. They receive + * Assertions are the simplest way to add reusable custom checks. They receive * the full evaluation context and return a pass/fail result with optional - * granular scoring. + * granular scoring. Place these files in `.agentv/assertions/` and reference + * them by discovered assertion type name. Use defineScriptGrader for + * command-backed graders referenced with `type: script`. * * This function: * 1. Reads JSON from stdin (snake_case format) diff --git a/packages/sdk/src/schemas.ts b/packages/sdk/src/schemas.ts index bd335e15d..518bef134 100644 --- a/packages/sdk/src/schemas.ts +++ b/packages/sdk/src/schemas.ts @@ -268,7 +268,7 @@ export const MessageSchema = z.object({ }); /** - * Derived evaluation trace read model exposed to custom graders. + * Derived evaluation trace read model exposed to custom assertions and script graders. * * Top-level summary fields (`eventCount`, `toolCalls`, `errorCount`) remain * available for existing metric graders; full transcript/tool evidence is under diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 1179740f4..786eb69c2 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -665,7 +665,7 @@ export function ragFaithfulness() { Use the helper in `assert: [ragFaithfulness()]`; do not create new YAML terms like `scores`. -### defineAssertion (recommended for custom checks) +### defineAssertion (recommended for reusable custom assertions) ```typescript #!/usr/bin/env bun import { defineAssertion } from '@agentv/sdk'; @@ -681,6 +681,8 @@ export default defineAssertion(({ output, trace }) => { Assertions support both `pass: boolean` and `score: number` (0-1). If only `pass` is given, score is 1 (pass) or 0 (fail). +Use `defineAssertion()` when you want a reusable assertion type discovered from `.agentv/assertions/` and referenced by filename as `type: `. This follows Promptfoo's normal eval terminology: custom logic is an assertion, with Promptfoo using fixed assertion types such as `javascript`, `python`, `ruby`, and `webhook`. AgentV extends that model by allowing arbitrary discovered assertion type names. + ### defineScriptGrader (full control) ```typescript #!/usr/bin/env bun @@ -698,7 +700,7 @@ export default defineScriptGrader(({ output, trace }) => { }); ``` -`defineAssertion()` files go in `.agentv/assertions/` and are referenced by filename as `type: `. `defineScriptGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`. +Use `defineScriptGrader()` when the custom component is a command-backed grader with explicit score control, custom assertion-result arrays, workspace commands, or LLM calls through a grader target. `defineScriptGrader()` scripts are referenced in YAML with `type: script` and `command: [bun, run, grader.ts]`. Plain Vitest workspace verifier files can use `command: [agentv, eval, graders/check.test.ts]`. ### Convention-Based Discovery diff --git a/skills-data/agentv-eval-writer/references/custom-evaluators.md b/skills-data/agentv-eval-writer/references/custom-evaluators.md index df8b0f10c..a01a02707 100644 --- a/skills-data/agentv-eval-writer/references/custom-evaluators.md +++ b/skills-data/agentv-eval-writer/references/custom-evaluators.md @@ -1,4 +1,8 @@ -# Custom Graders +# Custom Graders and Assertions + +Use **custom assertions** for reusable assertion types discovered from `.agentv/assertions/` via `defineAssertion()`. Use **script graders** for command-backed or LLM-backed scoring components referenced with `type: script` and `command:`. + +Promptfoo normally calls eval custom logic assertions and uses fixed assertion types such as `javascript`, `python`, `ruby`, and `webhook`. AgentV follows that assertion terminology for `defineAssertion()` and extends it with arbitrary discovered assertion type names. ## Wire Format