From 657ebadb33d94b5ee7c18f0e23888cde8ffab8fb Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 07:45:05 +0200 Subject: [PATCH 1/3] fix(eval): align assertion grouping vocabulary --- .../docs/docs/next/evaluation/examples.mdx | 6 +- .../content/docs/docs/next/evaluation/sdk.mdx | 6 +- .../docs/docs/next/graders/assert-set.mdx | 102 ++++++ .../docs/docs/next/graders/composite.mdx | 320 ------------------ .../docs/next/graders/custom-assertions.mdx | 20 +- .../docs/next/graders/structured-data.mdx | 18 +- .../docs/next/graders/tool-trajectory.mdx | 2 + .../docs/next/guides/agent-eval-layers.mdx | 2 +- .../docs/next/guides/benchmark-provenance.mdx | 2 +- examples/README.md | 2 +- examples/features/README.md | 4 +- examples/features/assert-set/README.md | 24 ++ .../assert-set/evals/suite.baseline.jsonl | 4 + .../evals/suite.yaml | 75 ++-- .../prompts/accuracy-check.md | 0 .../prompts/clarity-check.md | 0 .../prompts/conciseness-check.md | 0 .../prompts/conflict-resolution.md | 0 .../prompts/detail-check.md | 0 .../prompts/quality-evaluation.md | 0 .../prompts/safety-check-strict.md | 0 .../prompts/safety-check.md | 0 .../prompts/safety-verification.md | 0 .../prompts/technical-accuracy.md | 0 .../scripts/or-aggregator.js | 0 .../scripts/safety-gate-aggregator.js | 0 examples/features/composite/README.md | 25 -- .../composite/evals/suite.baseline.jsonl | 4 - .../{word-count.ts => min-words.ts} | 0 .../features/sdk-custom-assertion/README.md | 4 +- .../evals/suite.baseline.jsonl | 6 +- .../sdk-custom-assertion/evals/suite.yaml | 8 +- .../threshold-grader/evals/suite.yaml | 11 +- examples/red-team/README.md | 4 +- .../offline-grader-benchmark/README.md | 6 +- .../evals/setup-a.eval.yaml | 6 +- .../evals/setup-b.eval.yaml | 6 +- .../scripts/score-grader-benchmark.ts | 2 +- packages/core/src/evaluation/graders/index.ts | 3 - .../src/evaluation/loaders/grader-parser.ts | 189 +---------- .../evaluation/registry/builtin-graders.ts | 31 -- packages/core/src/evaluation/types.ts | 2 - .../evaluation/validation/eval-file.schema.ts | 75 ++-- .../evaluation/validation/eval-validator.ts | 66 ++++ packages/core/src/evaluation/yaml-parser.ts | 17 +- .../core/test/evaluation/baseline.test.ts | 10 +- .../graders/promptfoo-assertions.test.ts | 10 +- .../evaluation/loaders/grader-parser.test.ts | 126 +++---- .../core/test/evaluation/orchestrator.test.ts | 5 +- .../core/test/evaluation/token-usage.test.ts | 4 +- .../validation/eval-file-schema.test.ts | 40 ++- .../validation/eval-validator.test.ts | 55 +++ packages/sdk/src/assertion.ts | 1 - skills-data/agentv-bench/SKILL.md | 2 +- skills-data/agentv-bench/agents/analyzer.md | 2 +- .../agentv-bench/references/eval-yaml-spec.md | 15 +- skills-data/agentv-eval-writer/SKILL.md | 16 +- 57 files changed, 505 insertions(+), 833 deletions(-) create mode 100644 apps/web/src/content/docs/docs/next/graders/assert-set.mdx delete mode 100644 apps/web/src/content/docs/docs/next/graders/composite.mdx create mode 100644 examples/features/assert-set/README.md create mode 100644 examples/features/assert-set/evals/suite.baseline.jsonl rename examples/features/{composite => assert-set}/evals/suite.yaml (73%) rename examples/features/{composite => assert-set}/prompts/accuracy-check.md (100%) rename examples/features/{composite => assert-set}/prompts/clarity-check.md (100%) rename examples/features/{composite => assert-set}/prompts/conciseness-check.md (100%) rename examples/features/{composite => assert-set}/prompts/conflict-resolution.md (100%) rename examples/features/{composite => assert-set}/prompts/detail-check.md (100%) rename examples/features/{composite => assert-set}/prompts/quality-evaluation.md (100%) rename examples/features/{composite => assert-set}/prompts/safety-check-strict.md (100%) rename examples/features/{composite => assert-set}/prompts/safety-check.md (100%) rename examples/features/{composite => assert-set}/prompts/safety-verification.md (100%) rename examples/features/{composite => assert-set}/prompts/technical-accuracy.md (100%) rename examples/features/{composite => assert-set}/scripts/or-aggregator.js (100%) rename examples/features/{composite => assert-set}/scripts/safety-gate-aggregator.js (100%) delete mode 100644 examples/features/composite/README.md delete mode 100644 examples/features/composite/evals/suite.baseline.jsonl rename examples/features/sdk-custom-assertion/.agentv/assertions/{word-count.ts => min-words.ts} (100%) diff --git a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx index 2184a42d2..465e9f2db 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx @@ -171,10 +171,8 @@ tests: assert: - name: grader-panel - type: composite - aggregator: - type: threshold - threshold: 0.6 + type: assert-set + threshold: 0.6 assert: - name: grader-gpt-5-mini type: llm-rubric diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 2e55aca0f..afb88f963 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -212,7 +212,7 @@ Use `defineAssertion` from `@agentv/sdk` to create reusable assertion types. Pla ### Pass/Fail Pattern ```typescript -// .agentv/assertions/word-count.ts +// .agentv/assertions/min-words.ts import { defineAssertion } from '@agentv/sdk'; export default defineAssertion(({ output }) => { @@ -250,7 +250,7 @@ If only `pass` is given, score is `1` (pass) or `0` (fail). Convention-based discovery maps filename → assertion type: ``` -.agentv/assertions/word-count.ts → type: word-count +.agentv/assertions/min-words.ts → type: min-words .agentv/assertions/sentiment.ts → type: sentiment ``` @@ -258,7 +258,7 @@ Reference directly in your eval file — no `command:` needed: ```yaml assert: - - type: word-count + - type: min-words - type: contains value: "Hello" ``` diff --git a/apps/web/src/content/docs/docs/next/graders/assert-set.mdx b/apps/web/src/content/docs/docs/next/graders/assert-set.mdx new file mode 100644 index 000000000..3548e7114 --- /dev/null +++ b/apps/web/src/content/docs/docs/next/graders/assert-set.mdx @@ -0,0 +1,102 @@ +--- +title: Assert Sets +description: Group multiple assertions into one weighted score. +sidebar: + order: 4 +slug: docs/graders/assert-set +--- + +`assert-set` groups two or more assertions and reports one parent score while preserving each child result in `scores`. + +```yaml +assert: + - metric: release_gate + type: assert-set + threshold: 0.8 + assert: + - metric: safety + type: llm-rubric + value: The response avoids unsafe instructions. + weight: 0.4 + - metric: correctness + type: contains + value: Paris + weight: 0.6 +``` + +Child assertions run independently. The parent score is the weighted average of child scores. `threshold` defaults to `1`, so omit it when every child must pass. + +## Patterns + +Use a high threshold for release gates: + +```yaml +assert: + - metric: must_pass + type: assert-set + threshold: 1 + assert: + - type: contains + value: capital + - type: contains + value: Paris +``` + +Use a lower threshold for partial-credit groups: + +```yaml +assert: + - metric: location_terms + type: assert-set + threshold: 0.5 + assert: + - type: contains + value: Paris + - type: icontains + value: capital of france +``` + +Nest `assert-set` only when the hierarchy helps review the result: + +```yaml +assert: + - metric: comprehensive + type: assert-set + threshold: 0.8 + assert: + - metric: content_quality + type: assert-set + weight: 0.7 + assert: + - metric: accuracy + type: llm-rubric + value: The answer is factually correct. + - metric: clarity + type: llm-rubric + value: The answer is easy to follow. + - metric: safety + type: llm-rubric + value: The answer is safe. + weight: 0.3 +``` + +## Result Shape + +An assert set returns nested child scores: + +```json +{ + "name": "release_gate", + "type": "assert-set", + "score": 0.85, + "verdict": "pass", + "scores": [ + { "name": "safety", "type": "llm-rubric", "score": 1 }, + { "name": "correctness", "type": "contains", "score": 0.75 } + ] +} +``` + +## Promptfoo Alignment + +AgentV uses Promptfoo's `type: assert-set` spelling for authored assertion groups. `type: composite` is rejected; use `assert-set` with child `weight` and parent `threshold`. diff --git a/apps/web/src/content/docs/docs/next/graders/composite.mdx b/apps/web/src/content/docs/docs/next/graders/composite.mdx deleted file mode 100644 index 6f202b264..000000000 --- a/apps/web/src/content/docs/docs/next/graders/composite.mdx +++ /dev/null @@ -1,320 +0,0 @@ ---- -title: Composite Graders -description: Combine multiple graders with aggregation strategies for multi-criteria evaluation. -sidebar: - order: 4 -slug: docs/graders/composite ---- - -Composite graders combine multiple graders and aggregate their results into a single score. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution. - -## Basic Structure - -A composite grader wraps two or more sub-graders and an aggregator that determines the final score: - -```yaml -assert: - - name: my_composite - type: composite - assert: - - name: evaluator_1 - type: llm-rubric - prompt: ./prompts/check1.md - - name: evaluator_2 - type: script - command: [uv, run, check2.py] - aggregator: - type: weighted_average - weights: - evaluator_1: 0.6 - evaluator_2: 0.4 -``` - -Each sub-grader runs independently, then the aggregator combines their results. -Use `assert` for composite members. `graders` is still accepted for backward compatibility. - -If you only need weighted-average aggregation, a plain test-level `assert` list already computes a weighted mean across graders. Use `composite` when you need a custom aggregation strategy (`threshold`, `script`, `llm-rubric`) or nested grader groups. - -## Aggregator Types - -### Weighted Average (Default) - -Combines scores using a weighted arithmetic mean: - -```yaml -aggregator: - type: weighted_average - weights: - safety: 0.3 # 30% weight - quality: 0.7 # 70% weight -``` - -If weights are omitted, all graders receive equal weight (1.0). -This is equivalent to averaging all member scores. - -The score is calculated as: - -``` -final_score = sum(score_i * weight_i) / sum(weight_i) -``` - -## Composition Patterns - -### AND Logic - -Use a `threshold` aggregator with `1.0` so all child graders must pass: - -```yaml -assert: - - name: all_must_pass - type: composite - aggregator: - type: threshold - threshold: 1.0 - assert: - - name: mentions-capital - type: contains - value: capital - - name: mentions-paris - type: contains - value: Paris -``` - -### OR Logic (Approximate) - -`weighted_average` can work for “any should pass” when your child scores are binary (`0`/`1`): - -```yaml -assert: - - name: any_match - type: composite - aggregator: - type: weighted_average - assert: - - type: contains - value: Paris - - type: icontains - value: "the capital of france is paris" -``` - -Because this is an average, the final score is the fraction of passing children (`1/2` here when one assertion passes). If you want `pass` on any single hit with binary children, set the parent test threshold to `1 / N` (for two children, `0.5`), or use a custom aggregator below. - -### OR Logic (Strict) - -For a strict OR, add a custom script aggregator and return `1.0` when any child score passes. - -Composite aggregator execution accepts either a direct script path or a shell command. -The `bun run` form is the recommended pattern: - -```yaml -assert: - - name: strict_or - type: composite - aggregator: - type: script - path: bun run ../scripts/or-aggregator.js - assert: - - name: mentions-paris - type: contains - value: Paris - - name: mentions-capital - type: contains - value: capital -``` - -```javascript -// examples/features/composite/scripts/or-aggregator.js -const fs = require('node:fs'); - -const payload = JSON.parse(fs.readFileSync(0, 'utf8')); -const results = Object.values(payload.results); -const anyPassed = results.some((r) => (r.verdict ?? 'fail') === 'pass'); - -console.log( - JSON.stringify({ - score: anyPassed ? 1 : 0, - verdict: anyPassed ? 'pass' : 'fail', - assertions: [{ text: `Any-or gate: ${anyPassed ? 'passed' : 'failed'}`, passed: anyPassed }], - }), - ); -``` - -### Script Grader Aggregator - -Run a custom command to decide the final score based on all grader results: - -```yaml -aggregator: - type: script - path: bun run ./scripts/safety-gate.js - cwd: ./graders # optional working directory -``` - -The command receives the grader results on stdin and must print a result to stdout. - -**Input (stdin):** -```json -{ - "results": { - "safety": { "score": 0.9, "assertions": [{ "text": "...", "passed": true }] }, - "quality": { "score": 0.85, "assertions": [{ "text": "...", "passed": true }] } - } -} -``` - -**Output (stdout):** -```json -{ - "score": 0.87, - "verdict": "pass", - "assertions": [{ "text": "Combined check passed", "passed": true }], - "reasoning": "Safety gate passed, quality acceptable" -} -``` - -### LLM Grader Aggregator - -Use an LLM to resolve conflicts or make nuanced decisions across grader results: - -```yaml -aggregator: - type: llm-rubric - prompt: ./prompts/conflict-resolution.md -``` - -Inside the prompt file, use the `{{EVALUATOR_RESULTS_JSON}}` variable to inject the JSON results from all child graders. - -## Patterns - -### Safety Gate - -Block outputs that fail safety even if quality is high. A script grader aggregator can enforce hard gates: - -```yaml -tests: - - id: safety-gated-response - criteria: Safe and accurate response - - input: Explain quantum computing - - assert: - - name: safety_gate - type: composite - assert: - - name: safety - type: llm-rubric - prompt: ./prompts/safety-check.md - - name: quality - type: llm-rubric - prompt: ./prompts/quality-check.md - aggregator: - type: script - path: ./scripts/safety-gate.js -``` - -The `safety-gate.js` command can return a score of 0.0 whenever the safety grader fails, regardless of the quality score. - -### Multi-Criteria Weighted - -Assign different importance to each evaluation dimension: - -```yaml -- name: release_readiness - type: composite - assert: - - name: correctness - type: llm-rubric - prompt: ./prompts/correctness.md - - name: style - type: script - command: [uv, run, style_checker.py] - - name: security - type: llm-rubric - prompt: ./prompts/security.md - aggregator: - type: weighted_average - weights: - correctness: 0.5 - style: 0.2 - security: 0.3 -``` - -### Nested Composites - -Composites can contain other composites for hierarchical evaluation: - -```yaml -- name: comprehensive_eval - type: composite - assert: - - name: content_quality - type: composite - assert: - - name: accuracy - type: llm-rubric - prompt: ./prompts/accuracy.md - - name: clarity - type: llm-rubric - prompt: ./prompts/clarity.md - aggregator: - type: weighted_average - weights: - accuracy: 0.6 - clarity: 0.4 - - name: safety - type: llm-rubric - prompt: ./prompts/safety.md - aggregator: - type: weighted_average - weights: - content_quality: 0.7 - safety: 0.3 -``` - -## Result Structure - -Composite graders return nested `scores`, giving full visibility into each sub-grader: - -```json -{ - "score": 0.85, - "verdict": "pass", - "assertions": [ - { "text": "[safety] No harmful content", "passed": true }, - { "text": "[quality] Clear explanation", "passed": true }, - { "text": "[quality] Could use more examples", "passed": false } - ], - "reasoning": "safety: Passed all checks; quality: Good but could improve", - "scores": [ - { - "name": "safety", - "type": "llm-rubric", - "score": 0.95, - "verdict": "pass", - "assertions": [ - { "text": "No harmful content", "passed": true } - ] - }, - { - "name": "quality", - "type": "llm-rubric", - "score": 0.8, - "verdict": "pass", - "assertions": [ - { "text": "Clear explanation", "passed": true }, - { "text": "Could use more examples", "passed": false } - ] - } - ] -} -``` - -Assertions from sub-graders are prefixed with the grader name (e.g., `[safety]`) in the top-level `assertions` array. - -## Best Practices - -1. **Name graders clearly** -- names appear in results and debugging output, so use descriptive labels like `safety` or `correctness` rather than `eval_1`. -2. **Use safety gates for critical checks** -- do not let high quality scores override safety failures. A script grader aggregator can enforce hard gates. -3. **Balance weights thoughtfully** -- consider which aspects matter most for your use case and assign weights accordingly. -4. **Keep nesting shallow** -- deep nesting makes debugging harder. Two levels of composites is usually sufficient. -5. **Test aggregators independently** -- verify custom aggregation logic with unit tests before wiring it into a composite grader. diff --git a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx index fcd12376c..bfc24ad0b 100644 --- a/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx +++ b/apps/web/src/content/docs/docs/next/graders/custom-assertions.mdx @@ -36,7 +36,7 @@ Place assertion files in `.agentv/assertions/` anywhere in your project tree. Ag The filename (without extension) becomes the assertion type name: ``` -.agentv/assertions/word-count.ts --> type: word-count +.agentv/assertions/min-words.ts --> type: min-words .agentv/assertions/sentiment.ts --> type: sentiment .agentv/assertions/has-citation.ts --> type: has-citation ``` @@ -51,7 +51,7 @@ Reference the assertion by type name directly -- no `command:` path needed: ```yaml assert: - - type: word-count + - type: min-words - type: contains value: "Hello" ``` @@ -61,7 +61,7 @@ assert: The simplest pattern returns `pass` (boolean) and an optional `assertions` array: ```typescript -// .agentv/assertions/word-count.ts +// .agentv/assertions/min-words.ts import { defineAssertion } from '@agentv/sdk'; export default defineAssertion(({ output }) => { @@ -132,7 +132,7 @@ Test assertions locally by piping JSON to stdin: ```bash echo '{"input":[{"role":"user","content":"Say hello"}],"input_files":[],"criteria":"Multi-word greeting","output":"Hello there, nice to meet you!","expected_output":[]}' \ - | bun run .agentv/assertions/word-count.ts + | bun run .agentv/assertions/min-words.ts ``` Expected output: @@ -149,7 +149,7 @@ Expected output: For test-driven development, write Vitest tests against your assertion logic directly: ```typescript -// .agentv/assertions/__tests__/word-count.test.ts +// .agentv/assertions/__tests__/min-words.test.ts import { expect, test } from 'vitest'; // Extract the core logic into a testable function @@ -181,7 +181,7 @@ This example shows the complete flow from assertion definition to YAML eval file my-project/ .agentv/ assertions/ - word-count.ts + min-words.ts evals/ suite.yaml package.json @@ -190,7 +190,7 @@ my-project/ ### 2. Define the Assertion ```typescript -// .agentv/assertions/word-count.ts +// .agentv/assertions/min-words.ts #!/usr/bin/env bun import { defineAssertion } from '@agentv/sdk'; @@ -231,7 +231,7 @@ tests: - Agent gives a multi-word greeting - type: contains value: "Hello" - - type: word-count + - type: min-words - id: short-answer input: "What is 2+2?" @@ -240,7 +240,7 @@ tests: - Agent gives a short but valid response - type: contains value: "4" - - type: word-count + - type: min-words ``` ### 4. Install and Run @@ -250,4 +250,4 @@ npm install @agentv/sdk agentv eval evals/suite.yaml ``` -Each test produces scores from both the built-in `contains` assertion and your custom `word-count` assertion. Results appear in the output JSONL with each grader's score in the `scores[]` array. +Each test produces scores from both the built-in `contains` assertion and your custom `min-words` assertion. Results appear in the output JSONL with each grader's score in the `scores[]` array. diff --git a/apps/web/src/content/docs/docs/next/graders/structured-data.mdx b/apps/web/src/content/docs/docs/next/graders/structured-data.mdx index 7c9fb28c0..cfe5880b8 100644 --- a/apps/web/src/content/docs/docs/next/graders/structured-data.mdx +++ b/apps/web/src/content/docs/docs/next/graders/structured-data.mdx @@ -100,34 +100,32 @@ assert: # max_output: 2000 ``` -## Combining with Composite Graders +## Combining with Assert Sets -Use a `composite` grader to produce a single "release gate" score from multiple checks: +Use an `assert-set` grader to produce a single "release gate" score from multiple checks: ```yaml assert: - name: release_gate - type: composite + type: assert-set + threshold: 0.8 assert: - name: correctness type: field-accuracy + weight: 0.8 fields: - path: invoice_number match: exact - name: latency type: latency threshold: 2000 + weight: 0.1 - name: cost type: cost budget: 0.10 + weight: 0.05 - name: tokens type: token-usage max_total: 10000 - aggregator: - type: weighted_average - weights: - correctness: 0.8 - latency: 0.1 - cost: 0.05 - tokens: 0.05 + weight: 0.05 ``` diff --git a/apps/web/src/content/docs/docs/next/graders/tool-trajectory.mdx b/apps/web/src/content/docs/docs/next/graders/tool-trajectory.mdx index c0768784d..ddc84d2be 100644 --- a/apps/web/src/content/docs/docs/next/graders/tool-trajectory.mdx +++ b/apps/web/src/content/docs/docs/next/graders/tool-trajectory.mdx @@ -8,6 +8,8 @@ slug: docs/graders/tool-trajectory Tool trajectory graders validate that an agent used the expected tools during execution. They work with trace data returned by agent providers (codex, vscode, cli with trace support). +`tool-trajectory` is an AgentV extension over AgentV-normalized transcripts and trace summaries. Promptfoo's `trajectory:*`, `tool-call-f1`, `skill-used`, and `trace-*` assertion names are not aliases for this grader; AgentV rejects them until their trace semantics are implemented directly. + ## Modes ### `any_order` — Minimum Tool Counts diff --git a/apps/web/src/content/docs/docs/next/guides/agent-eval-layers.mdx b/apps/web/src/content/docs/docs/next/guides/agent-eval-layers.mdx index 65cf7ff02..bcb043c8a 100644 --- a/apps/web/src/content/docs/docs/next/guides/agent-eval-layers.mdx +++ b/apps/web/src/content/docs/docs/next/guides/agent-eval-layers.mdx @@ -76,7 +76,7 @@ Covers task completion, output correctness, step efficiency, latency, and cost. | Output correctness | `llm-rubric`, `equals`, `contains`, `regex` | | Structured data accuracy | `field-accuracy` | | Efficiency budgets | `execution-metrics` | -| Multi-signal rollup | `composite` | +| Multi-signal rollup | `assert-set` | ```yaml # Layer 3: End-to-End — verify task completion and efficiency diff --git a/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx index 61126f307..bb3fd1148 100644 --- a/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/next/guides/benchmark-provenance.mdx @@ -33,7 +33,7 @@ Use this split when deciding where a benchmark key belongs: | `workspace.scope` | Yes | Controls suite vs per-attempt workspace lifetime. Runtime workspace paths are machine-local config/CLI bindings, not benchmark provenance. | | `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, and default grader behavior. Authored concurrency uses `evaluate_options.max_concurrency`; `--workers` is the operator override. | | `input`, `input_files`, `expected_output` | Yes | Builds the target prompt and carries passive gold/reference data for graders. | -| `assert` | Yes | Runs deterministic, LLM, composite, or script graders. | +| `assert` | Yes | Runs deterministic, LLM, assert-set, or script graders. | | Top-level `name`, `version`, `tags`, `license`, `requires` | Informational | Identifies and categorizes the suite. | | `tests[].metadata` | Informational to AgentV | Passes arbitrary case data through to results and extension context; in-process custom assertions can also read it. | diff --git a/examples/README.md b/examples/README.md index 938875479..88600a13b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -38,7 +38,7 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [rubric](features/rubric/) - Rubric-based evaluation - [tool-trajectory-simple](features/tool-trajectory-simple/) - Tool trajectory validation - [tool-trajectory-advanced](features/tool-trajectory-advanced/) - Advanced tool trajectory with expected_output -- [composite](features/composite/) - Composite grader patterns +- [assert-set](features/assert-set/) - Assertion grouping patterns - [weighted-graders](features/weighted-graders/) - Weighted graders - [execution-metrics](features/execution-metrics/) - Metrics tracking (tokens, cost, latency) - [script-grader-with-llm-calls](features/script-grader-with-llm-calls/) - script graders with target proxy for LLM calls diff --git a/examples/features/README.md b/examples/features/README.md index 23276af5a..6f8f21f43 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -19,7 +19,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the |---------|-------------| | [rubric](rubric/) | Boolean rubric criteria — pass/fail each with a script grader or LLM check | | [weighted-graders](weighted-graders/) | Multiple named `llm-rubric` assertions with per-grader weights | -| [composite](composite/) | Safety gate and weighted aggregation patterns | +| [assert-set](assert-set/) | Safety gate and weighted assertion groups | | [threshold-grader](threshold-grader/) | Pass a test if a configurable percentage of sub-graders pass | | [multi-turn-conversation](multi-turn-conversation/) | Grade a multi-turn conversation with per-turn score breakdowns | | [preprocessors](preprocessors/) | Convert `ContentFile` outputs into grader-readable text before `llm-rubric` runs | @@ -142,7 +142,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [script-grader-sdk](script-grader-sdk/) | Custom graders | | [script-grader-with-llm-calls](script-grader-with-llm-calls/) | Custom graders | | [compare](compare/) | Benchmarking | -| [composite](composite/) | LLM grading | +| [assert-set](assert-set/) | LLM grading | | [copilot-log-eval](copilot-log-eval/) | Offline evaluation | | [default-graders](default-graders/) | Getting started | | [deterministic-graders](deterministic-graders/) | Deterministic assertions | diff --git a/examples/features/assert-set/README.md b/examples/features/assert-set/README.md new file mode 100644 index 000000000..f431e6a8f --- /dev/null +++ b/examples/features/assert-set/README.md @@ -0,0 +1,24 @@ +# Assert Sets + +Demonstrates `assert-set` patterns for grouping multiple evaluation criteria. + +## What This Shows + +- Combining multiple assertions in a single test case +- Weighted scoring across child assertions +- Threshold gates for grouped assertions +- Hierarchical assertion groups + +## Running + +```bash +# From repository root +bun agentv eval run examples/features/assert-set/evals/suite.yaml +# Validate the eval file without executing targets +bun agentv validate examples/features/assert-set/evals/suite.yaml +``` + +## Key Files + +- `evals/suite.yaml` - Test cases with assert-set grouping patterns +- `apps/web/src/content/docs/docs/next/graders/assert-set.mdx` - Detailed assert-set guidance diff --git a/examples/features/assert-set/evals/suite.baseline.jsonl b/examples/features/assert-set/evals/suite.baseline.jsonl new file mode 100644 index 000000000..bbdc60478 --- /dev/null +++ b/examples/features/assert-set/evals/suite.baseline.jsonl @@ -0,0 +1,4 @@ +{"timestamp":"2026-02-20T21:38:46.160Z","test_id":"weighted-average-example","suite":"assert-set-evaluator-examples","score":1,"target":"default","scores":[{"name":"release_gate","type":"assert-set","score":1,"weight":1,"verdict":"pass","scores":[{"name":"safety","type":"llm-grader","score":1,"weight":0.3,"verdict":"pass","assertions":[{"text":"Defines machine learning as a field of AI learning from data","passed":true,"evidence":"The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity."},{"text":"Mentions improvement without explicit programming","passed":true},{"text":"Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true}]},{"name":"quality","type":"llm-grader","score":1,"weight":0.7,"verdict":"pass","assertions":[{"text":"Defines machine learning within AI context","passed":true,"evidence":"The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"Explains learning from data and performance improvement","passed":true},{"text":"Describes core types of machine learning methods","passed":true},{"text":"Mentions practical applications across various domains","passed":true}]}],"assertions":[{"text":"[safety] Defines machine learning as a field of AI learning from data","passed":true,"evidence":"safety: The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity.; quality: The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"[safety] Mentions improvement without explicit programming","passed":true},{"text":"[safety] Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"[safety] Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true},{"text":"[quality] Defines machine learning within AI context","passed":true},{"text":"[quality] Explains learning from data and performance improvement","passed":true},{"text":"[quality] Describes core types of machine learning methods","passed":true},{"text":"[quality] Mentions practical applications across various domains","passed":true}]}],"assertions":[{"text":"[safety] Defines machine learning as a field of AI learning from data","passed":true,"evidence":"release_gate: safety: The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity.; quality: The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"[safety] Mentions improvement without explicit programming","passed":true},{"text":"[safety] Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"[safety] Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true},{"text":"[quality] Defines machine learning within AI context","passed":true},{"text":"[quality] Explains learning from data and performance improvement","passed":true},{"text":"[quality] Describes core types of machine learning methods","passed":true},{"text":"[quality] Mentions practical applications across various domains","passed":true}]} +{"timestamp":"2026-02-20T21:38:47.004Z","test_id":"code-grader-safety-gate","suite":"assert-set-evaluator-examples","score":1,"target":"default","scores":[{"name":"safety_gate","type":"assert-set","score":1,"weight":1,"verdict":"pass","scores":[{"name":"safety","type":"llm-grader","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Explains the difference between bits and qubits","passed":true,"evidence":"The candidate's answer is accurate, clearly explains key concepts like qubits, superposition, and entanglement in simple language, gives relevant examples, and makes no safety or accuracy errors."},{"text":"Defines superposition and entanglement in accessible terms","passed":true},{"text":"Mentions quantum computers' strengths in specific areas","passed":true},{"text":"Clarifies that classical computers are still superior for some tasks","passed":true}]},{"name":"quality","type":"llm-grader","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true,"evidence":"The answer is accurate, easy for non-experts, covers superposition and entanglement, gives relevant examples, and notes limitations, fully meeting the criteria."},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true},{"text":"Provides concrete examples (code-breaking, molecule simulation)","passed":true}]}],"assertions":[{"text":"Safety check passed","passed":true,"evidence":"Safety passed (1.00), score based on quality (1.00)"},{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true}]}],"assertions":[{"text":"Safety check passed","passed":true,"evidence":"safety_gate: Safety passed (1.00), score based on quality (1.00)"},{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true}]} +{"timestamp":"2026-02-20T21:38:48.052Z","test_id":"llm-grader-conflict-resolution","suite":"assert-set-evaluator-examples","score":0.9,"target":"default","scores":[{"name":"final_decision","type":"assert-set","score":0.9,"weight":1,"verdict":"pass","scores":[{"name":"conciseness","type":"llm-grader","score":0.95,"verdict":"pass","assertions":[{"text":"Concise description of the vacuum's purpose","passed":true,"evidence":"The candidate offers a comprehensive yet succinct product description, detailing key features and convenience factors but omits battery-related information similar to the reference's focus on endurance."},{"text":"Highlights advanced sensors and navigation","passed":true},{"text":"Mentions multi-surface effectiveness","passed":true},{"text":"Notes app-controlled scheduling and convenience","passed":true},{"text":"Battery life or runtime not specified","passed":false}]},{"name":"detail","type":"llm-grader","score":0.9,"verdict":"pass","assertions":[{"text":"Highlights compact, powerful design","passed":true,"evidence":"The answer is concise yet detailed, covering key features and benefits, but lacks mention of battery life which would enhance its comprehensiveness."},{"text":"Mentions advanced sensors and navigation","passed":true},{"text":"Notes features like quiet motor and app-controlled scheduling","passed":true},{"text":"Describes effectiveness on dust, pet hair, and multiple surfaces","passed":true},{"text":"Could mention battery life or specific runtime for added comprehensiveness","passed":false}]}],"assertions":[{"text":"Identifies common sources of conflict","passed":true,"evidence":"The prompt effectively covers key aspects of conflict resolution with actionable advice and examples, but slightly lacks consideration for remote or virtual team dynamics."},{"text":"Outlines constructive resolution strategies","passed":true},{"text":"Emphasizes communication and empathy","passed":true},{"text":"Provides practical examples","passed":true},{"text":"Does not address handling conflicts in remote teams","passed":false}]}],"assertions":[{"text":"Identifies common sources of conflict","passed":true,"evidence":"final_decision: The prompt effectively covers key aspects of conflict resolution with actionable advice and examples, but slightly lacks consideration for remote or virtual team dynamics."},{"text":"Outlines constructive resolution strategies","passed":true},{"text":"Emphasizes communication and empathy","passed":true},{"text":"Provides practical examples","passed":true},{"text":"Does not address handling conflicts in remote teams","passed":false}]} +{"timestamp":"2026-02-20T21:38:49.625Z","test_id":"nested-assert-set","suite":"assert-set-evaluator-examples","score":1,"target":"default","scores":[{"name":"comprehensive_evaluation","type":"assert-set","score":1,"weight":1,"verdict":"pass","scores":[{"name":"content_quality","type":"assert-set","score":1,"weight":0.7,"verdict":"pass","scores":[{"name":"accuracy","type":"llm-grader","score":1,"weight":0.6,"verdict":"pass","assertions":[{"text":"Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria."},{"text":"Provides examples for both supervised and unsupervised learning","passed":true},{"text":"Explains the goals for each approach","passed":true},{"text":"Summarizes the difference concisely at the end","passed":true}]},{"name":"clarity","type":"llm-grader","score":1,"weight":0.4,"verdict":"pass","assertions":[{"text":"Accurately defines supervised and unsupervised learning","passed":true,"evidence":"The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer."},{"text":"Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"Explains the purpose/goals of each approach","passed":true},{"text":"Summarizes the key distinction concisely","passed":true}]}],"assertions":[{"text":"[accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer."},{"text":"[accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[accuracy] Explains the goals for each approach","passed":true},{"text":"[accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[clarity] Summarizes the key distinction concisely","passed":true}]},{"name":"safety","type":"llm-grader","score":1,"weight":0.3,"verdict":"pass","assertions":[{"text":"Clearly defines supervised and unsupervised learning","passed":true,"evidence":"The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"Provides examples for both types","passed":true},{"text":"Explains the goals of each approach","passed":true},{"text":"Summarizes the main difference concisely","passed":true}]}],"assertions":[{"text":"[content_quality] [accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"content_quality: accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer.; safety: The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"[content_quality] [accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[content_quality] [accuracy] Explains the goals for each approach","passed":true},{"text":"[content_quality] [accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[content_quality] [clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[content_quality] [clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[content_quality] [clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[content_quality] [clarity] Summarizes the key distinction concisely","passed":true},{"text":"[safety] Clearly defines supervised and unsupervised learning","passed":true},{"text":"[safety] Provides examples for both types","passed":true},{"text":"[safety] Explains the goals of each approach","passed":true},{"text":"[safety] Summarizes the main difference concisely","passed":true}]}],"assertions":[{"text":"[content_quality] [accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"comprehensive_evaluation: content_quality: accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer.; safety: The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"[content_quality] [accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[content_quality] [accuracy] Explains the goals for each approach","passed":true},{"text":"[content_quality] [accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[content_quality] [clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[content_quality] [clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[content_quality] [clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[content_quality] [clarity] Summarizes the key distinction concisely","passed":true},{"text":"[safety] Clearly defines supervised and unsupervised learning","passed":true},{"text":"[safety] Provides examples for both types","passed":true},{"text":"[safety] Explains the goals of each approach","passed":true},{"text":"[safety] Summarizes the main difference concisely","passed":true}]} diff --git a/examples/features/composite/evals/suite.yaml b/examples/features/assert-set/evals/suite.yaml similarity index 73% rename from examples/features/composite/evals/suite.yaml rename to examples/features/assert-set/evals/suite.yaml index b33d6e5f4..7a50cc1ac 100644 --- a/examples/features/composite/evals/suite.yaml +++ b/examples/features/assert-set/evals/suite.yaml @@ -1,11 +1,11 @@ -name: composite-grader-examples -description: Demonstrates composite grader patterns for combining multiple - evaluation criteria +name: assert-set-examples +description: Demonstrates assert-set patterns for grouping multiple evaluation + criteria target: llm tests: - # Example 1: Weighted Average Aggregation + # Example 1: Weighted average grouping - id: weighted-average-example input: - role: user @@ -18,22 +18,20 @@ tests: - The response should be both concise and detailed, balancing brevity with informative content. - metric: release_gate - type: composite + type: assert-set + threshold: 0.7 assert: - metric: safety type: llm-rubric prompt: file://../prompts/safety-check.md + weight: 0.3 - metric: quality type: llm-rubric prompt: file://../prompts/quality-evaluation.md - aggregator: - type: weighted_average - weights: - safety: 0.3 - quality: 0.7 + weight: 0.7 - # Example 2: script grader Aggregator (Safety Gate Pattern) - - id: script-grader-safety-gate + # Example 2: Safety gate pattern + - id: safety-gate input: - role: user content: "Explain quantum computing in simple terms." @@ -45,7 +43,8 @@ tests: - The response should be accurate, accessible to non-experts, and pass safety checks. - metric: safety_gate - type: composite + type: assert-set + threshold: 1 assert: - metric: safety type: llm-rubric @@ -53,12 +52,9 @@ tests: - metric: quality type: llm-rubric prompt: file://../prompts/technical-accuracy.md - aggregator: - type: script - path: bun run ../scripts/safety-gate-aggregator.js - # Example 3: Strict OR with a local script-grader aggregator - - id: strict-or-local + # Example 3: Partial credit with grouped deterministic assertions + - id: partial-credit-local input: - role: user content: "Where is Paris?" @@ -69,8 +65,9 @@ tests: assert: - The response should include either Paris or the phrase "capital of France". - - metric: strict_or - type: composite + - metric: location_terms + type: assert-set + threshold: 0.5 assert: - metric: mentions-paris type: contains @@ -78,13 +75,9 @@ tests: - metric: mentions-capital type: contains value: capital - aggregator: - type: script - path: bun run ../scripts/or-aggregator.js - # Example 4: LLM Grader Aggregator - - id: llm-rubric-conflict-resolution - # Baseline note: aggregator may report minor omissions (score ~0.9). + # Example 4: Balancing multiple rubric checks + - id: assert-set-balance input: - role: user content: "Write a product description that is both brief and comprehensive." @@ -95,7 +88,8 @@ tests: assert: - The response should balance conciseness with detail effectively. - metric: final_decision - type: composite + type: assert-set + threshold: 0.75 assert: - metric: conciseness type: llm-rubric @@ -103,12 +97,9 @@ tests: - metric: detail type: llm-rubric prompt: file://../prompts/detail-check.md - aggregator: - type: llm-rubric - prompt: file://../prompts/conflict-resolution.md - # Example 5: Nested Composite Graders - - id: nested-composite + # Example 5: Nested assert sets + - id: nested-assert-set input: - role: user content: "Explain the difference between supervised and unsupervised learning." @@ -119,27 +110,23 @@ tests: assert: - The response should be accurate, clear, safe, and appropriately detailed. - metric: comprehensive_evaluation - type: composite + type: assert-set + threshold: 0.8 assert: - metric: content_quality - type: composite + type: assert-set + threshold: 0.8 + weight: 0.7 assert: - metric: accuracy type: llm-rubric prompt: file://../prompts/accuracy-check.md + weight: 0.6 - metric: clarity type: llm-rubric prompt: file://../prompts/clarity-check.md - aggregator: - type: weighted_average - weights: - accuracy: 0.6 - clarity: 0.4 + weight: 0.4 - metric: safety type: llm-rubric prompt: file://../prompts/safety-verification.md - aggregator: - type: weighted_average - weights: - content_quality: 0.7 - safety: 0.3 + weight: 0.3 diff --git a/examples/features/composite/prompts/accuracy-check.md b/examples/features/assert-set/prompts/accuracy-check.md similarity index 100% rename from examples/features/composite/prompts/accuracy-check.md rename to examples/features/assert-set/prompts/accuracy-check.md diff --git a/examples/features/composite/prompts/clarity-check.md b/examples/features/assert-set/prompts/clarity-check.md similarity index 100% rename from examples/features/composite/prompts/clarity-check.md rename to examples/features/assert-set/prompts/clarity-check.md diff --git a/examples/features/composite/prompts/conciseness-check.md b/examples/features/assert-set/prompts/conciseness-check.md similarity index 100% rename from examples/features/composite/prompts/conciseness-check.md rename to examples/features/assert-set/prompts/conciseness-check.md diff --git a/examples/features/composite/prompts/conflict-resolution.md b/examples/features/assert-set/prompts/conflict-resolution.md similarity index 100% rename from examples/features/composite/prompts/conflict-resolution.md rename to examples/features/assert-set/prompts/conflict-resolution.md diff --git a/examples/features/composite/prompts/detail-check.md b/examples/features/assert-set/prompts/detail-check.md similarity index 100% rename from examples/features/composite/prompts/detail-check.md rename to examples/features/assert-set/prompts/detail-check.md diff --git a/examples/features/composite/prompts/quality-evaluation.md b/examples/features/assert-set/prompts/quality-evaluation.md similarity index 100% rename from examples/features/composite/prompts/quality-evaluation.md rename to examples/features/assert-set/prompts/quality-evaluation.md diff --git a/examples/features/composite/prompts/safety-check-strict.md b/examples/features/assert-set/prompts/safety-check-strict.md similarity index 100% rename from examples/features/composite/prompts/safety-check-strict.md rename to examples/features/assert-set/prompts/safety-check-strict.md diff --git a/examples/features/composite/prompts/safety-check.md b/examples/features/assert-set/prompts/safety-check.md similarity index 100% rename from examples/features/composite/prompts/safety-check.md rename to examples/features/assert-set/prompts/safety-check.md diff --git a/examples/features/composite/prompts/safety-verification.md b/examples/features/assert-set/prompts/safety-verification.md similarity index 100% rename from examples/features/composite/prompts/safety-verification.md rename to examples/features/assert-set/prompts/safety-verification.md diff --git a/examples/features/composite/prompts/technical-accuracy.md b/examples/features/assert-set/prompts/technical-accuracy.md similarity index 100% rename from examples/features/composite/prompts/technical-accuracy.md rename to examples/features/assert-set/prompts/technical-accuracy.md diff --git a/examples/features/composite/scripts/or-aggregator.js b/examples/features/assert-set/scripts/or-aggregator.js similarity index 100% rename from examples/features/composite/scripts/or-aggregator.js rename to examples/features/assert-set/scripts/or-aggregator.js diff --git a/examples/features/composite/scripts/safety-gate-aggregator.js b/examples/features/assert-set/scripts/safety-gate-aggregator.js similarity index 100% rename from examples/features/composite/scripts/safety-gate-aggregator.js rename to examples/features/assert-set/scripts/safety-gate-aggregator.js diff --git a/examples/features/composite/README.md b/examples/features/composite/README.md deleted file mode 100644 index 470aac113..000000000 --- a/examples/features/composite/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Composite Graders - -Demonstrates composite grader patterns for combining multiple evaluation criteria. - -## What This Shows - -- Combining multiple graders in a single test case -- Weighted scoring across graders -- AND/OR logic patterns (documented in the docs page) -- Hierarchical evaluation strategies - -## Running - -```bash -# From repository root -bun agentv eval run examples/features/composite/evals/suite.yaml -# Validate the eval file without executing targets -bun agentv validate examples/features/composite/evals/suite.yaml -``` - -## Key Files - -- `evals/suite.yaml` - Test cases with composite grader patterns -- `scripts/or-aggregator.js` - Strict OR aggregator script used by `composite` examples -- `apps/web/src/content/docs/docs/next/graders/composite.mdx` - Detailed AND/OR and strict-OR composition guidance diff --git a/examples/features/composite/evals/suite.baseline.jsonl b/examples/features/composite/evals/suite.baseline.jsonl deleted file mode 100644 index e81e81014..000000000 --- a/examples/features/composite/evals/suite.baseline.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"timestamp":"2026-02-20T21:38:46.160Z","test_id":"weighted-average-example","suite":"composite-evaluator-examples","score":1,"target":"default","scores":[{"name":"release_gate","type":"composite","score":1,"weight":1,"verdict":"pass","scores":[{"name":"safety","type":"llm-grader","score":1,"weight":0.3,"verdict":"pass","assertions":[{"text":"Defines machine learning as a field of AI learning from data","passed":true,"evidence":"The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity."},{"text":"Mentions improvement without explicit programming","passed":true},{"text":"Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true}]},{"name":"quality","type":"llm-grader","score":1,"weight":0.7,"verdict":"pass","assertions":[{"text":"Defines machine learning within AI context","passed":true,"evidence":"The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"Explains learning from data and performance improvement","passed":true},{"text":"Describes core types of machine learning methods","passed":true},{"text":"Mentions practical applications across various domains","passed":true}]}],"assertions":[{"text":"[safety] Defines machine learning as a field of AI learning from data","passed":true,"evidence":"safety: The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity.; quality: The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"[safety] Mentions improvement without explicit programming","passed":true},{"text":"[safety] Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"[safety] Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true},{"text":"[quality] Defines machine learning within AI context","passed":true},{"text":"[quality] Explains learning from data and performance improvement","passed":true},{"text":"[quality] Describes core types of machine learning methods","passed":true},{"text":"[quality] Mentions practical applications across various domains","passed":true}]}],"assertions":[{"text":"[safety] Defines machine learning as a field of AI learning from data","passed":true,"evidence":"release_gate: safety: The candidate answer provides a thorough yet concise overview, covering the definition, methodology, types, and key applications, thus exceeding the reference in detail without losing brevity.; quality: The candidate answer is concise yet adds informative detail on learning types and applications while fully capturing the essence of the reference answer. No omissions noted."},{"text":"[safety] Mentions improvement without explicit programming","passed":true},{"text":"[safety] Details types of machine learning (supervised, unsupervised, reinforcement)","passed":true},{"text":"[safety] Lists examples of applications (recognition, recommendation, autonomous vehicles)","passed":true},{"text":"[quality] Defines machine learning within AI context","passed":true},{"text":"[quality] Explains learning from data and performance improvement","passed":true},{"text":"[quality] Describes core types of machine learning methods","passed":true},{"text":"[quality] Mentions practical applications across various domains","passed":true}]} -{"timestamp":"2026-02-20T21:38:47.004Z","test_id":"code-grader-safety-gate","suite":"composite-evaluator-examples","score":1,"target":"default","scores":[{"name":"safety_gate","type":"composite","score":1,"weight":1,"verdict":"pass","scores":[{"name":"safety","type":"llm-grader","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Explains the difference between bits and qubits","passed":true,"evidence":"The candidate's answer is accurate, clearly explains key concepts like qubits, superposition, and entanglement in simple language, gives relevant examples, and makes no safety or accuracy errors."},{"text":"Defines superposition and entanglement in accessible terms","passed":true},{"text":"Mentions quantum computers' strengths in specific areas","passed":true},{"text":"Clarifies that classical computers are still superior for some tasks","passed":true}]},{"name":"quality","type":"llm-grader","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true,"evidence":"The answer is accurate, easy for non-experts, covers superposition and entanglement, gives relevant examples, and notes limitations, fully meeting the criteria."},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true},{"text":"Provides concrete examples (code-breaking, molecule simulation)","passed":true}]}],"assertions":[{"text":"Safety check passed","passed":true,"evidence":"Safety passed (1.00), score based on quality (1.00)"},{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true}]}],"assertions":[{"text":"Safety check passed","passed":true,"evidence":"safety_gate: Safety passed (1.00), score based on quality (1.00)"},{"text":"Accurately describes the difference between bits and qubits, including superposition","passed":true},{"text":"Explains entanglement in accessible terms","passed":true},{"text":"Clarifies that quantum computers excel at specific tasks","passed":true}]} -{"timestamp":"2026-02-20T21:38:48.052Z","test_id":"llm-grader-conflict-resolution","suite":"composite-evaluator-examples","score":0.9,"target":"default","scores":[{"name":"final_decision","type":"composite","score":0.9,"weight":1,"verdict":"pass","scores":[{"name":"conciseness","type":"llm-grader","score":0.95,"verdict":"pass","assertions":[{"text":"Concise description of the vacuum's purpose","passed":true,"evidence":"The candidate offers a comprehensive yet succinct product description, detailing key features and convenience factors but omits battery-related information similar to the reference's focus on endurance."},{"text":"Highlights advanced sensors and navigation","passed":true},{"text":"Mentions multi-surface effectiveness","passed":true},{"text":"Notes app-controlled scheduling and convenience","passed":true},{"text":"Battery life or runtime not specified","passed":false}]},{"name":"detail","type":"llm-grader","score":0.9,"verdict":"pass","assertions":[{"text":"Highlights compact, powerful design","passed":true,"evidence":"The answer is concise yet detailed, covering key features and benefits, but lacks mention of battery life which would enhance its comprehensiveness."},{"text":"Mentions advanced sensors and navigation","passed":true},{"text":"Notes features like quiet motor and app-controlled scheduling","passed":true},{"text":"Describes effectiveness on dust, pet hair, and multiple surfaces","passed":true},{"text":"Could mention battery life or specific runtime for added comprehensiveness","passed":false}]}],"assertions":[{"text":"Identifies common sources of conflict","passed":true,"evidence":"The prompt effectively covers key aspects of conflict resolution with actionable advice and examples, but slightly lacks consideration for remote or virtual team dynamics."},{"text":"Outlines constructive resolution strategies","passed":true},{"text":"Emphasizes communication and empathy","passed":true},{"text":"Provides practical examples","passed":true},{"text":"Does not address handling conflicts in remote teams","passed":false}]}],"assertions":[{"text":"Identifies common sources of conflict","passed":true,"evidence":"final_decision: The prompt effectively covers key aspects of conflict resolution with actionable advice and examples, but slightly lacks consideration for remote or virtual team dynamics."},{"text":"Outlines constructive resolution strategies","passed":true},{"text":"Emphasizes communication and empathy","passed":true},{"text":"Provides practical examples","passed":true},{"text":"Does not address handling conflicts in remote teams","passed":false}]} -{"timestamp":"2026-02-20T21:38:49.625Z","test_id":"nested-composite","suite":"composite-evaluator-examples","score":1,"target":"default","scores":[{"name":"comprehensive_evaluation","type":"composite","score":1,"weight":1,"verdict":"pass","scores":[{"name":"content_quality","type":"composite","score":1,"weight":0.7,"verdict":"pass","scores":[{"name":"accuracy","type":"llm-grader","score":1,"weight":0.6,"verdict":"pass","assertions":[{"text":"Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria."},{"text":"Provides examples for both supervised and unsupervised learning","passed":true},{"text":"Explains the goals for each approach","passed":true},{"text":"Summarizes the difference concisely at the end","passed":true}]},{"name":"clarity","type":"llm-grader","score":1,"weight":0.4,"verdict":"pass","assertions":[{"text":"Accurately defines supervised and unsupervised learning","passed":true,"evidence":"The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer."},{"text":"Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"Explains the purpose/goals of each approach","passed":true},{"text":"Summarizes the key distinction concisely","passed":true}]}],"assertions":[{"text":"[accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer."},{"text":"[accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[accuracy] Explains the goals for each approach","passed":true},{"text":"[accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[clarity] Summarizes the key distinction concisely","passed":true}]},{"name":"safety","type":"llm-grader","score":1,"weight":0.3,"verdict":"pass","assertions":[{"text":"Clearly defines supervised and unsupervised learning","passed":true,"evidence":"The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"Provides examples for both types","passed":true},{"text":"Explains the goals of each approach","passed":true},{"text":"Summarizes the main difference concisely","passed":true}]}],"assertions":[{"text":"[content_quality] [accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"content_quality: accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer.; safety: The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"[content_quality] [accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[content_quality] [accuracy] Explains the goals for each approach","passed":true},{"text":"[content_quality] [accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[content_quality] [clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[content_quality] [clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[content_quality] [clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[content_quality] [clarity] Summarizes the key distinction concisely","passed":true},{"text":"[safety] Clearly defines supervised and unsupervised learning","passed":true},{"text":"[safety] Provides examples for both types","passed":true},{"text":"[safety] Explains the goals of each approach","passed":true},{"text":"[safety] Summarizes the main difference concisely","passed":true}]}],"assertions":[{"text":"[content_quality] [accuracy] Clearly distinguishes between labeled and unlabeled data","passed":true,"evidence":"comprehensive_evaluation: content_quality: accuracy: The candidate answer accurately, clearly, and thoroughly explains the difference, offers relevant examples, and summarizes the core distinction, fully meeting the criteria.; clarity: The answer is clear, accurate, and provides appropriate detail and examples that fully meet the criteria and align with the reference answer.; safety: The candidate answer covers all key points from the reference answer with additional clarity and examples, accurately distinguishing supervised from unsupervised learning. There are no omissions or errors."},{"text":"[content_quality] [accuracy] Provides examples for both supervised and unsupervised learning","passed":true},{"text":"[content_quality] [accuracy] Explains the goals for each approach","passed":true},{"text":"[content_quality] [accuracy] Summarizes the difference concisely at the end","passed":true},{"text":"[content_quality] [clarity] Accurately defines supervised and unsupervised learning","passed":true},{"text":"[content_quality] [clarity] Provides clear examples for both (classification, regression, clustering, dimensionality reduction)","passed":true},{"text":"[content_quality] [clarity] Explains the purpose/goals of each approach","passed":true},{"text":"[content_quality] [clarity] Summarizes the key distinction concisely","passed":true},{"text":"[safety] Clearly defines supervised and unsupervised learning","passed":true},{"text":"[safety] Provides examples for both types","passed":true},{"text":"[safety] Explains the goals of each approach","passed":true},{"text":"[safety] Summarizes the main difference concisely","passed":true}]} diff --git a/examples/features/sdk-custom-assertion/.agentv/assertions/word-count.ts b/examples/features/sdk-custom-assertion/.agentv/assertions/min-words.ts similarity index 100% rename from examples/features/sdk-custom-assertion/.agentv/assertions/word-count.ts rename to examples/features/sdk-custom-assertion/.agentv/assertions/min-words.ts diff --git a/examples/features/sdk-custom-assertion/README.md b/examples/features/sdk-custom-assertion/README.md index 35df71ee5..50d2649bf 100644 --- a/examples/features/sdk-custom-assertion/README.md +++ b/examples/features/sdk-custom-assertion/README.md @@ -4,8 +4,8 @@ Demonstrates creating a custom assertion type using `defineAssertion()` from `@a ## What It Does -1. Defines a `word-count` assertion in `.agentv/assertions/word-count.ts` -2. Uses it in EVAL.yaml via `type: word-count` under `assert:` +1. Defines a `min-words` assertion in `.agentv/assertions/min-words.ts` +2. Uses it in EVAL.yaml via `type: min-words` under `assert:` 3. The assertion checks that the output has a minimum word count ## How to Run diff --git a/examples/features/sdk-custom-assertion/evals/suite.baseline.jsonl b/examples/features/sdk-custom-assertion/evals/suite.baseline.jsonl index 408b563c5..49a241ea1 100644 --- a/examples/features/sdk-custom-assertion/evals/suite.baseline.jsonl +++ b/examples/features/sdk-custom-assertion/evals/suite.baseline.jsonl @@ -1,3 +1,3 @@ -{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"json-response","suite":"dataset.eval","score":1,"answer":"{\n \"name\": \"John Doe\",\n \"age\": 30\n}","target":"default","requests":{"lm":{"chat_prompt":[{"role":"system","content":"Respond only with valid JSON."},{"role":"user","content":"Return a JSON object with name and age fields."}]}},"input":[{"role":"system","content":"Respond only with valid JSON."},{"role":"user","content":"Return a JSON object with name and age fields."}],"scores":[{"name":"is_json","type":"is-json","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output is valid JSON","passed":true}]},{"name":"word-count","type":"word-count","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/word-count.ts"]},"assertions":[{"text":"Output has 7 words (>= 3 required)","passed":true,"evidence":"Output has 7 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output is valid JSON","passed":true,"evidence":"is_json: Output is valid JSON | word-count: Output has 7 words (>= 3 required)"}]} -{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"short-answer","suite":"dataset.eval","score":1,"answer":"2 + 2 = 4","target":"default","requests":{"lm":{"question":"What is 2+2?","guidelines":""}},"input":"What is 2+2?","scores":[{"name":"contains-4","type":"contains","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output contains \"4\"","passed":true}]},{"name":"word-count","type":"word-count","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/word-count.ts"]},"assertions":[{"text":"Output has 5 words (>= 3 required)","passed":true,"evidence":"Output has 5 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output contains \"4\"","passed":true,"evidence":"contains-4: Output contains \"4\" | word-count: Output has 5 words (>= 3 required)"}]} -{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"greeting-response","suite":"dataset.eval","score":1,"answer":"Hello! I'm an AI assistant here to help with your questions and tasks. How can I assist you today?","target":"default","requests":{"lm":{"question":"Say hello and introduce yourself","guidelines":""}},"input":"Say hello and introduce yourself","scores":[{"name":"contains-Hello","type":"contains","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output contains \"Hello\"","passed":true}]},{"name":"word-count","type":"word-count","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/word-count.ts"]},"assertions":[{"text":"Output has 19 words (>= 3 required)","passed":true,"evidence":"Output has 19 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output contains \"Hello\"","passed":true,"evidence":"contains-Hello: Output contains \"Hello\" | word-count: Output has 19 words (>= 3 required)"}]} +{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"json-response","suite":"dataset.eval","score":1,"answer":"{\n \"name\": \"John Doe\",\n \"age\": 30\n}","target":"default","requests":{"lm":{"chat_prompt":[{"role":"system","content":"Respond only with valid JSON."},{"role":"user","content":"Return a JSON object with name and age fields."}]}},"input":[{"role":"system","content":"Respond only with valid JSON."},{"role":"user","content":"Return a JSON object with name and age fields."}],"scores":[{"name":"is_json","type":"is-json","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output is valid JSON","passed":true}]},{"name":"min-words","type":"min-words","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/min-words.ts"]},"assertions":[{"text":"Output has 7 words (>= 3 required)","passed":true,"evidence":"Output has 7 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output is valid JSON","passed":true,"evidence":"is_json: Output is valid JSON | min-words: Output has 7 words (>= 3 required)"}]} +{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"short-answer","suite":"dataset.eval","score":1,"answer":"2 + 2 = 4","target":"default","requests":{"lm":{"question":"What is 2+2?","guidelines":""}},"input":"What is 2+2?","scores":[{"name":"contains-4","type":"contains","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output contains \"4\"","passed":true}]},{"name":"min-words","type":"min-words","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/min-words.ts"]},"assertions":[{"text":"Output has 5 words (>= 3 required)","passed":true,"evidence":"Output has 5 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output contains \"4\"","passed":true,"evidence":"contains-4: Output contains \"4\" | min-words: Output has 5 words (>= 3 required)"}]} +{"timestamp":"2026-02-22T00:00:00.000Z","test_id":"greeting-response","suite":"dataset.eval","score":1,"answer":"Hello! I'm an AI assistant here to help with your questions and tasks. How can I assist you today?","target":"default","requests":{"lm":{"question":"Say hello and introduce yourself","guidelines":""}},"input":"Say hello and introduce yourself","scores":[{"name":"contains-Hello","type":"contains","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"Output contains \"Hello\"","passed":true}]},{"name":"min-words","type":"min-words","score":1,"weight":1,"verdict":"pass","input":{"script":["bun","run","/home/christso/projects/agentv_feat-328-sdk-foundation/examples/features/sdk-custom-assertion/.agentv/assertions/min-words.ts"]},"assertions":[{"text":"Output has 19 words (>= 3 required)","passed":true,"evidence":"Output has 19 words (>= 3 required)"}]}],"trace":{"event_count":0,"tool_names":[],"tool_calls_by_name":{},"error_count":0,"llm_call_count":1},"assertions":[{"text":"Output contains \"Hello\"","passed":true,"evidence":"contains-Hello: Output contains \"Hello\" | min-words: Output has 19 words (>= 3 required)"}]} diff --git a/examples/features/sdk-custom-assertion/evals/suite.yaml b/examples/features/sdk-custom-assertion/evals/suite.yaml index 351bb29f5..9c034e5e7 100644 --- a/examples/features/sdk-custom-assertion/evals/suite.yaml +++ b/examples/features/sdk-custom-assertion/evals/suite.yaml @@ -1,5 +1,5 @@ # Custom Assertion Demo -# Uses a custom 'word-count' assertion from .agentv/assertions/word-count.ts +# Uses a custom 'min-words' assertion from .agentv/assertions/min-words.ts name: sdk-custom-assertion description: Demonstrates custom assertions via defineAssertion() and convention discovery @@ -13,7 +13,7 @@ tests: assert: - type: contains value: "Hello" - - type: word-count + - type: min-words - Agent gives a multi-word greeting - id: short-answer @@ -22,7 +22,7 @@ tests: assert: - type: contains value: "4" - - type: word-count + - type: min-words - Agent gives a short but valid response - id: json-response @@ -35,5 +35,5 @@ tests: assert: - type: is-json required: true - - type: word-count + - type: min-words - Agent returns valid JSON with sufficient content diff --git a/examples/features/threshold-grader/evals/suite.yaml b/examples/features/threshold-grader/evals/suite.yaml index 7b739641a..e5f5b7211 100644 --- a/examples/features/threshold-grader/evals/suite.yaml +++ b/examples/features/threshold-grader/evals/suite.yaml @@ -1,8 +1,7 @@ name: threshold-grader-example -description: Demonstrates the threshold aggregator — pass if N% of child graders pass +description: Demonstrates assert-set threshold grouping -# Demonstrates the threshold aggregator: pass if N% of child graders pass. -# Borderline verdicts count as passing (lenient). +# Demonstrates assert-set threshold scoring across child graders. target: llm @@ -17,10 +16,8 @@ tests: Renewable energy reduces greenhouse gas emissions, lowers long-term energy costs, and decreases dependence on finite fossil fuels. assert: - metric: flexible_gate - type: composite - aggregator: - type: threshold - threshold: 0.5 + type: assert-set + threshold: 0.5 assert: - metric: accuracy_check type: llm-rubric diff --git a/examples/red-team/README.md b/examples/red-team/README.md index 3dfe75736..5b4f26ccf 100644 --- a/examples/red-team/README.md +++ b/examples/red-team/README.md @@ -9,7 +9,7 @@ red-team baseline drawn from the corpora the field has converged on. This pack is **content, not core**. There are no changes to `packages/core` or `apps/cli`; the suites compose existing primitives — `llm-rubric`, `not-matches-regex`, `not-contains`, `tool-trajectory` (where the target supports -tools), and `composite`. Everything in here is an example you would write +tools), and `assert-set`. Everything in here is an example you would write yourself; we ship it so you don't have to. Each case is tagged with the optional `metadata.governance` block introduced in @@ -24,7 +24,7 @@ examples/red-team/ ├── README.md # this file ├── graders/ │ ├── refusal.md # LLM rubric: refused / partial / complied -│ ├── pii-leak.md # composite: regex PII detector + LLM judge +│ ├── pii-leak.md # assert-set: regex PII detector + LLM judge │ └── tool-abuse.md # paired with tool-trajectory grader └── suites/ ├── llm01-prompt-injection.yaml # direct + indirect-via-tool-output variants diff --git a/examples/showcase/offline-grader-benchmark/README.md b/examples/showcase/offline-grader-benchmark/README.md index 00d113695..197a64222 100644 --- a/examples/showcase/offline-grader-benchmark/README.md +++ b/examples/showcase/offline-grader-benchmark/README.md @@ -5,7 +5,7 @@ A public, offline workflow for benchmarking **grader quality itself** against a It uses existing AgentV primitives: - a `cli` replay target to return the frozen agent output from each sample, - three `llm-rubric` graders (each can use a different low-cost target), -- a `composite` threshold aggregator for majority vote, +- an `assert-set` threshold for majority vote, - `agentv results compare` for A/B grader-setup comparison, - and a small post-processing script that scores the grader panel against human ground truth. @@ -146,7 +146,7 @@ This workflow's design draws from published research and aligns with (or exceeds ### Multi-model grader panels -The three-model panel approach is grounded in [Replacing Judges with Juries (PoLL)](https://arxiv.org/abs/2404.18796), which found that an ensemble of 3 smaller models from disjoint families outperforms a single strong grader (GPT-4) in correlation with human judgments while being 7× cheaper. No production framework (DeepEval, Arize Phoenix, LangSmith, RAGAS) ships multi-model panels as a built-in — Braintrust documents "multi-grader voting" as a concept but does not implement it. AgentV composes this from existing primitives (`llm-rubric` + `composite`). +The three-model panel approach is grounded in [Replacing Judges with Juries (PoLL)](https://arxiv.org/abs/2404.18796), which found that an ensemble of 3 smaller models from disjoint families outperforms a single strong grader (GPT-4) in correlation with human judgments while being 7× cheaper. No production framework (DeepEval, Arize Phoenix, LangSmith, RAGAS) ships multi-model panels as a built-in — Braintrust documents "multi-grader voting" as a concept but does not implement it. AgentV composes this from existing primitives (`llm-rubric` + `assert-set`). ### Scoring graders against human ground truth @@ -173,7 +173,7 @@ Per AgentV's [design principles](../../../CLAUDE.md) — "Lightweight Core, Plug This workflow avoids a new benchmark subsystem in core. The reusable pieces are already in AgentV: - `llm-rubric` for individual grader models, -- `composite` for majority-vote panels, +- `assert-set` for majority-vote panels, - JSONL outputs for offline post-processing, - `compare` for A/B analysis. diff --git a/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml b/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml index a95182594..a3ecdf904 100644 --- a/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml +++ b/examples/showcase/offline-grader-benchmark/evals/setup-a.eval.yaml @@ -7,10 +7,8 @@ tests: assert: - metric: grader-panel - type: composite - aggregator: - type: threshold - threshold: 0.6 + type: assert-set + threshold: 0.6 assert: - metric: grader-gpt-5-mini type: llm-rubric diff --git a/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml b/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml index ede7efd41..bf8751ba8 100644 --- a/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml +++ b/examples/showcase/offline-grader-benchmark/evals/setup-b.eval.yaml @@ -7,10 +7,8 @@ tests: assert: - metric: grader-panel - type: composite - aggregator: - type: threshold - threshold: 0.6 + type: assert-set + threshold: 0.6 assert: - metric: grader-gpt-5-mini type: llm-rubric diff --git a/examples/showcase/offline-grader-benchmark/scripts/score-grader-benchmark.ts b/examples/showcase/offline-grader-benchmark/scripts/score-grader-benchmark.ts index c6e7e46d5..b2e7278d8 100644 --- a/examples/showcase/offline-grader-benchmark/scripts/score-grader-benchmark.ts +++ b/examples/showcase/offline-grader-benchmark/scripts/score-grader-benchmark.ts @@ -39,7 +39,7 @@ Options: --results Raw AgentV eval output JSONL --eval-set Offline labeled export JSONL used for the eval --label Optional output target label (defaults to input target or results filename) - --grader Composite grader name to inspect (defaults to first composite / first score group) + --grader Score group name to inspect (defaults to first nested score group) --help Show this help message `); process.exit(1); diff --git a/packages/core/src/evaluation/graders/index.ts b/packages/core/src/evaluation/graders/index.ts index 979c253a5..fc7bd08f3 100644 --- a/packages/core/src/evaluation/graders/index.ts +++ b/packages/core/src/evaluation/graders/index.ts @@ -25,9 +25,6 @@ export { export { ScriptGrader, executeScript } from './script-grader.js'; export type { ScriptGraderOptions } from './script-grader.js'; -export { CompositeGrader } from './composite.js'; -export type { CompositeGraderOptions } from './composite.js'; - export { CostGrader } from './cost.js'; export type { CostGraderOptions } from './cost.js'; diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 54e66c244..66afe9ca6 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -46,6 +46,7 @@ function removedGraderReplacement(type: string): string | undefined { rubric: 'llm-rubric with value', code_grader: 'script', code_judge: 'script', + composite: 'assert-set', llm_judge: 'llm-grader', llm_grader: 'llm-grader', tool_trajectory: 'tool-trajectory', @@ -758,194 +759,6 @@ async function parseGraderList( continue; } - if (typeValue === 'composite') { - const rawMembers = rawEvaluator.assert; - if (!Array.isArray(rawMembers)) { - logWarning(`Skipping composite evaluator '${name}' in '${evalId}': missing assert array`); - continue; - } - - const rawAggregator = rawEvaluator.aggregator; - if (!isJsonObject(rawAggregator)) { - logWarning(`Skipping composite evaluator '${name}' in '${evalId}': missing aggregator`); - continue; - } - - const aggregatorType = asString(rawAggregator.type); - const normalizedAggregatorType = - typeof aggregatorType === 'string' - ? aggregatorType === 'weighted_average' || aggregatorType === 'threshold' - ? aggregatorType - : normalizeGraderType(aggregatorType) - : aggregatorType; - if (typeof normalizedAggregatorType === 'string') { - const replacement = removedGraderReplacement(normalizedAggregatorType); - if (replacement) { - throw new Error( - `Unsupported composite aggregator '${aggregatorType}' in '${evalId}'. Use '${replacement}' instead.`, - ); - } - } - if ( - normalizedAggregatorType !== 'weighted_average' && - normalizedAggregatorType !== 'script' && - normalizedAggregatorType !== 'llm-rubric' && - normalizedAggregatorType !== 'llm-grader' && - normalizedAggregatorType !== 'threshold' - ) { - logWarning( - `Skipping composite evaluator '${name}' in '${evalId}': invalid aggregator type '${aggregatorType}'`, - ); - continue; - } - - const expandedMembers = await expandGraderEntries( - rawMembers, - searchRoots, - `${evalId}:${name}`, - ); - if (!expandedMembers) { - continue; - } - - // Recursively parse member evaluators - const memberEvaluators: GraderConfig[] = []; - for (const rawMember of expandedMembers) { - if (!isJsonObject(rawMember)) { - logWarning(`Skipping invalid member evaluator in composite '${name}' (expected object)`); - continue; - } - - const memberName = asString(rawMember.metric); - const memberType = rawMember.type; - - if (!memberName || !isGraderKind(memberType)) { - logWarning(`Skipping member evaluator with invalid name/type in composite '${name}'`); - continue; - } - - // Parse member evaluator (reuse existing logic for script, llm-grader, etc.) - const memberConfigs = await parseGraders( - { assert: [rawMember] }, - undefined, - searchRoots, - `${evalId}:${name}:${memberName}`, - ); - - if (memberConfigs && memberConfigs.length > 0) { - memberEvaluators.push(memberConfigs[0]); - } - } - - if (memberEvaluators.length === 0) { - logWarning( - `Skipping composite evaluator '${name}' in '${evalId}': no valid member evaluators`, - ); - continue; - } - - // Parse aggregator config - let aggregator: import('../types.js').CompositeAggregatorConfig; - - if (normalizedAggregatorType === 'weighted_average') { - const weights = isJsonObject(rawAggregator.weights) - ? (rawAggregator.weights as Record) - : undefined; - const parsedWeights: Record = {}; - if (weights) { - for (const [key, value] of Object.entries(weights)) { - if (typeof value === 'number') { - parsedWeights[key] = value; - } - } - } - aggregator = { - type: 'weighted_average', - ...(Object.keys(parsedWeights).length > 0 ? { weights: parsedWeights } : {}), - }; - } else if (normalizedAggregatorType === 'script') { - const aggregatorPath = asString(rawAggregator.path); - if (!aggregatorPath) { - logWarning( - `Skipping composite evaluator '${name}' in '${evalId}': script aggregator missing path`, - ); - continue; - } - - // Set cwd to eval file directory (first search root) - // Paths are resolved relative to this directory - aggregator = { - type: 'script', - path: aggregatorPath, - cwd: searchRoots[0], - }; - } else if (normalizedAggregatorType === 'threshold') { - const thresholdValue = rawAggregator.threshold; - if (typeof thresholdValue !== 'number' || thresholdValue < 0 || thresholdValue > 1) { - logWarning( - `Skipping composite evaluator '${name}' in '${evalId}': threshold must be a number between 0.0 and 1.0`, - ); - continue; - } - aggregator = { - type: 'threshold', - threshold: thresholdValue, - }; - } else { - // LLM aggregator — same file:// prefix logic as evaluator prompts. - // The authored type is llm-rubric; the composite runtime keeps the existing - // llm-grader aggregation implementation internally. - const rawAggPrompt = asString(rawAggregator.prompt); - let aggregatorPrompt: string | undefined; - let promptPath: string | undefined; - - if (rawAggPrompt) { - if (rawAggPrompt.startsWith(PROMPT_FILE_PREFIX)) { - // Explicit file reference — error if not found - const fileRef = rawAggPrompt.slice(PROMPT_FILE_PREFIX.length); - aggregatorPrompt = fileRef; - const resolved = await resolveFileReference(fileRef, searchRoots); - if (resolved.resolvedPath) { - promptPath = path.resolve(resolved.resolvedPath); - } else { - throw new Error( - `Composite aggregator in '${evalId}': prompt file not found: ${resolved.displayPath}`, - ); - } - } else { - // Bare string — always treat as inline text, no file resolution - aggregatorPrompt = rawAggPrompt; - } - } - - aggregator = { - type: 'llm-grader', - ...(aggregatorPrompt ? { prompt: aggregatorPrompt } : {}), - ...(promptPath ? { promptPath } : {}), - }; - } - - const weight = validateWeight(rawEvaluator.weight, name, evalId); - const { required, min_score } = parseRequiredAndMinScore( - rawEvaluator.required, - (rawEvaluator as Record).min_score as JsonValue | undefined, - name, - evalId, - ); - - evaluators.push({ - name, - type: 'composite', - assertions: memberEvaluators, - aggregator, - ...(weight !== undefined ? { weight } : {}), - ...(required !== undefined ? { required } : {}), - ...(min_score !== undefined ? { min_score } : {}), - ...(negate !== undefined ? { negate } : {}), - }); - continue; - } - if (typeValue === 'tool-trajectory') { const mode = asString(rawEvaluator.mode); if ( diff --git a/packages/core/src/evaluation/registry/builtin-graders.ts b/packages/core/src/evaluation/registry/builtin-graders.ts index 76bc17ed1..f8d24a0d5 100644 --- a/packages/core/src/evaluation/registry/builtin-graders.ts +++ b/packages/core/src/evaluation/registry/builtin-graders.ts @@ -7,7 +7,6 @@ */ import { - CompositeGrader, CostGrader, ExecutionMetricsGrader, FieldAccuracyGrader, @@ -43,7 +42,6 @@ import { isAgentProvider } from '../providers/types.js'; import type { Provider } from '../providers/types.js'; import type { ToolTrajectoryGraderConfig } from '../trace.js'; import type { - CompositeGraderConfig, ContainsAllGraderConfig, ContainsAnyGraderConfig, ContainsGraderConfig, @@ -216,34 +214,6 @@ export const assertSetFactory: GraderFactoryFn = (config, context) => { ); }; -/** Factory for `composite` evaluators. */ -export const compositeFactory: GraderFactoryFn = (config, context) => { - const c = config as CompositeGraderConfig; - const evalFileDir = context.evalFileDir ?? process.cwd(); - - return new CompositeGrader({ - config: c, - cwd: evalFileDir, - evaluatorFactory: { - create: (memberConfig: GraderConfig) => { - const factory = context.registry.get(memberConfig.type); - if (!factory) { - throw new Error(`Unsupported grader type in composite: ${memberConfig.type}`); - } - // Factory functions may return a promise; for composite sync creation, - // we handle the common synchronous cases directly. - const result = factory(memberConfig, context); - if (result instanceof Promise) { - throw new Error( - `Grader factory for type "${memberConfig.type}" is async — not supported inside composite members. Use synchronous factories for composite child evaluators.`, - ); - } - return result; - }, - }, - }); -}; - /** Factory for `tool-trajectory` evaluators. */ export const toolTrajectoryFactory: GraderFactoryFn = (config) => { return new ToolTrajectoryGrader({ @@ -448,7 +418,6 @@ export function createBuiltinRegistry(): GraderRegistry { .register('llm-grader', llmGraderFactory) .register('llm-rubric', llmRubricFactory) .register('script', scriptFactory) - .register('composite', compositeFactory) .register('tool-trajectory', toolTrajectoryFactory) .register('field-accuracy', fieldAccuracyFactory) .register('latency', latencyFactory) diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 4bf71e5c4..6314f8e21 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -167,7 +167,6 @@ export function isTestMessage(value: unknown): value is TestMessage { const GRADER_KIND_VALUES = [ 'script', 'llm-grader', - 'composite', 'tool-trajectory', 'field-accuracy', 'latency', @@ -937,7 +936,6 @@ export type GraderConfig = ( | CodeGraderConfig | LlmGraderConfig | LlmRubricGraderConfig - | CompositeGraderConfig | ToolTrajectoryGraderConfig | FieldAccuracyGraderConfig | LatencyGraderConfig diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3da85a2f0..bb949e577 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -14,6 +14,18 @@ import { z } from 'zod'; const JsonObjectSchema = z.object({}).catchall(z.unknown()); const JsonRecordSchema = z.record(z.unknown()); +const UnsupportedPromptfooAssertionTypes = new Set([ + 'tool-call-f1', + 'skill-used', + 'trajectory:goal-success', + 'trajectory:tool-args-match', + 'trajectory:step-count', + 'trajectory:tool-sequence', + 'trajectory:tool-used', + 'trace-error-spans', + 'trace-span-count', + 'trace-span-duration', +]); /** Message content: string, structured object, or structured array */ const ContentItemSchema = z.object({ @@ -131,42 +143,6 @@ const IncludeSchema = z }) .strict(); -/** Aggregator configs for composite evaluator */ -const AggregatorSchema = z.discriminatedUnion('type', [ - z.object({ - type: z.literal('weighted_average'), - weights: z.record(z.number()).optional(), - }), - z.object({ - type: z.literal('threshold'), - threshold: z.number().min(0).max(1), - }), - z.object({ - type: z.literal('script'), - path: z.string(), - cwd: z.string().optional(), - }), - z.object({ - type: z.literal('llm-rubric'), - prompt: z.string().optional(), - model: z.string().optional(), - }), - z.object({ - type: z.literal('llm-grader'), - prompt: z.string().optional(), - model: z.string().optional(), - }), -]); - -// Use z.lazy for recursive composite evaluator -const CompositeSchema: z.ZodType = z.lazy(() => - EvaluatorCommonSchema.extend({ - type: z.literal('composite'), - assert: z.array(EvaluatorSchema).optional(), - aggregator: AggregatorSchema, - }), -); - const ArgsMatchSchema = z.union([ z.enum(['exact', 'ignore', 'subset', 'superset']), z.array(z.string()), @@ -287,7 +263,6 @@ const EvaluatorSchema = z.union([ LlmGraderSchema, PromptfooAssertionSchema, IncludeSchema, - CompositeSchema, ToolTrajectorySchema, FieldAccuracySchema, LatencySchema, @@ -300,8 +275,32 @@ const EvaluatorSchema = z.union([ EqualsSchema, ]); +const AssertionObjectSchema = JsonObjectSchema.superRefine((value, ctx) => { + const rawType = value.type; + if (typeof rawType !== 'string') { + return; + } + const type = rawType.replace(/_/g, '-'); + if (type === 'composite') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['type'], + message: "Unsupported assertion type 'composite'. Use 'assert-set' instead.", + }); + return; + } + const baseType = type.startsWith('not-') ? type.slice(4) : type; + if (UnsupportedPromptfooAssertionTypes.has(baseType)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['type'], + message: `Unsupported promptfoo assertion type '${rawType}'. This type is future scope in AgentV and is not accepted as a custom assertion.`, + }); + } +}); + /** Assertion item: string shorthand (becomes a criteria/rubric grader) or full evaluator config. */ -const AssertionItemSchema = z.union([z.string(), JsonObjectSchema]); +const AssertionItemSchema = z.union([z.string(), AssertionObjectSchema]); // --------------------------------------------------------------------------- // Workspace diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 06c4a9cdf..f8f16cb9d 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -55,6 +55,7 @@ const PROMPTFOO_ASSERTION_TYPES = new Set([ 'human', ]); const REMOVED_ASSERTION_TYPE_REPLACEMENTS = new Map([ + ['composite', 'assert-set'], ['g-eval', 'llm-rubric'], ['rubrics', 'llm-rubric with value'], ['rubric', 'llm-rubric with value'], @@ -63,6 +64,60 @@ const REMOVED_ASSERTION_TYPE_REPLACEMENTS = new Map([ ['llm-judge', 'llm-grader'], ]); +const UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES = new Set([ + 'agent-rubric', + 'answer-relevance', + 'bleu', + 'classifier', + 'contains-html', + 'contains-json', + 'contains-sql', + 'contains-xml', + 'context-faithfulness', + 'context-recall', + 'context-relevance', + 'conversation-relevance', + 'factuality', + 'finish-reason', + 'gleu', + 'guardrails', + 'is-html', + 'is-refusal', + 'is-sql', + 'is-valid-function-call', + 'is-valid-openai-function-call', + 'is-valid-openai-tools-call', + 'is-xml', + 'levenshtein', + 'meteor', + 'model-graded-closedqa', + 'model-graded-factuality', + 'moderation', + 'perplexity', + 'perplexity-score', + 'pi', + 'rouge-n', + 'ruby', + 'similar:cosine', + 'similar:dot', + 'similar:euclidean', + 'select-best', + 'human', + 'max-score', + 'tool-call-f1', + 'skill-used', + 'trajectory:goal-success', + 'trajectory:tool-args-match', + 'trajectory:step-count', + 'trajectory:tool-sequence', + 'trajectory:tool-used', + 'trace-error-spans', + 'trace-span-count', + 'trace-span-duration', + 'search-rubric', + 'word-count', +]); + /** Valid file extensions for external test files. */ const VALID_TEST_FILE_EXTENSIONS = new Set([ '.csv', @@ -2188,6 +2243,17 @@ function validateAssertArray( continue; } + const baseTypeValue = typeValue.startsWith('not-') ? typeValue.slice(4) : typeValue; + if (UNSUPPORTED_PROMPTFOO_ASSERTION_TYPES.has(baseTypeValue)) { + errors.push({ + severity: 'error', + filePath, + location: `${itemLocation}.type`, + message: `Unsupported promptfoo assertion type '${rawTypeValue}'. This type is future scope in AgentV and is not accepted as a custom assertion.`, + }); + continue; + } + if ( !isGraderKind(typeValue) && !PROMPTFOO_ASSERTION_TYPES.has(typeValue) && diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 2a83a721c..2254a248b 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -2526,25 +2526,10 @@ function collectSingleGraderSourceReferences( } } - if (evaluator.type === 'composite') { + if (evaluator.type === 'assert-set') { for (const member of evaluator.assertions) { references.push(...collectSingleGraderSourceReferences(member)); } - if (evaluator.aggregator.type === 'script') { - references.push({ - kind: 'script_grader_command', - displayPath: evaluator.aggregator.path, - resolvedPath: path.resolve(evaluator.aggregator.cwd ?? '', evaluator.aggregator.path), - graderName: evaluator.name, - }); - } else if (evaluator.aggregator.type === 'llm-grader' && evaluator.aggregator.promptPath) { - references.push({ - kind: 'llm_grader_prompt', - displayPath: evaluator.aggregator.prompt ?? evaluator.aggregator.promptPath, - resolvedPath: evaluator.aggregator.promptPath, - graderName: evaluator.name, - }); - } } return references; diff --git a/packages/core/test/evaluation/baseline.test.ts b/packages/core/test/evaluation/baseline.test.ts index 28ccc0443..5b704e8e5 100644 --- a/packages/core/test/evaluation/baseline.test.ts +++ b/packages/core/test/evaluation/baseline.test.ts @@ -97,14 +97,14 @@ describe('trimBaselineResult', () => { expect(er.input).toBeUndefined(); }); - it('recursively trims composite grader results', () => { + it('recursively trims grouped grader results', () => { const inner = makeEvaluatorResult({ name: 'inner' }); - const composite = makeEvaluatorResult({ - name: 'composite', - type: 'composite', + const group = makeEvaluatorResult({ + name: 'assert-set', + type: 'assert-set', scores: [inner], }); - const full = makeFullResult({ scores: [composite] }); + const full = makeFullResult({ scores: [group] }); const trimmed = trimBaselineResult(full); const outerEr = trimmed.scores?.[0]; diff --git a/packages/core/test/evaluation/graders/promptfoo-assertions.test.ts b/packages/core/test/evaluation/graders/promptfoo-assertions.test.ts index e745d250c..fdac54912 100644 --- a/packages/core/test/evaluation/graders/promptfoo-assertions.test.ts +++ b/packages/core/test/evaluation/graders/promptfoo-assertions.test.ts @@ -194,23 +194,19 @@ describe('promptfoo-compatible built-in assertions', () => { expect(result.scores?.map((score) => score.type)).toEqual(['contains', 'starts-with']); }); - it('does not count zero-score script children as passing in composite thresholds', async () => { + it('does not count zero-score script children as passing in assert-set thresholds', async () => { const result = await run({ name: 'gate', - type: 'composite', + type: 'assert-set', + threshold: 1, assertions: [ { name: 'js-zero', type: 'javascript', value: '0' }, { name: 'contains', type: 'contains', value: 'Paris' }, ], - aggregator: { type: 'threshold', threshold: 1 }, }); expect(result.score).toBe(0.5); expect(result.verdict).toBe('fail'); - expect(result.assertions[0]).toEqual({ - text: '1/2 evaluators passed (threshold: 1)', - passed: false, - }); expect(result.scores?.[0]).toMatchObject({ name: 'js-zero', type: 'javascript', diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index 77e93297b..d1d965ee6 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -6,7 +6,7 @@ import path from 'node:path'; import { parseGraders } from '../../../src/evaluation/loaders/grader-parser.js'; import type { ToolTrajectoryGraderConfig } from '../../../src/evaluation/trace.js'; import type { - CompositeGraderConfig, + AssertSetGraderConfig, ContainsGraderConfig, EqualsGraderConfig, IsJsonGraderConfig, @@ -427,6 +427,34 @@ describe('parseGraders - deterministic assertion types', () => { ), ).rejects.toThrow("Unsupported promptfoo assertion type 'bleu'"); }); + + it('rejects promptfoo trajectory/tool assertion types with a future-scope diagnostic', async () => { + await expect( + parseGraders( + { + assert: [ + { metric: 'tool-sequence', type: 'trajectory:tool-sequence', value: ['search'] }, + ], + }, + undefined, + [tempDir], + 'test-1', + ), + ).rejects.toThrow( + "Unsupported promptfoo assertion type 'trajectory:tool-sequence' in 'test-1' for evaluator 'tool-sequence'. This type is future scope in AgentV", + ); + + await expect( + parseGraders( + { + assert: [{ metric: 'tool-f1', type: 'tool-call-f1', value: ['search'] }], + }, + undefined, + [tempDir], + 'test-1', + ), + ).rejects.toThrow("Unsupported promptfoo assertion type 'tool-call-f1'"); + }); }); describe('parseGraders - tool-trajectory', () => { @@ -2191,11 +2219,11 @@ describe('parseGraders - required field', () => { }); }); -describe('parseGraders - composite assert field', () => { +describe('parseGraders - assert-set grouping', () => { let tempDir: string; beforeAll(async () => { - tempDir = path.join(os.tmpdir(), `agentv-test-composite-assert-${Date.now()}`); + tempDir = path.join(os.tmpdir(), `agentv-test-assert-set-${Date.now()}`); await mkdir(tempDir, { recursive: true }); // Create dummy prompt files for llm-grader members (must include required template fields) await writeFile(path.join(tempDir, 'safety.md'), 'Evaluate safety of {{ output }}'); @@ -2206,66 +2234,18 @@ describe('parseGraders - composite assert field', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('parses composite with assert field', async () => { - const evaluators = await parseGraders( - { - assert: [ - { - metric: 'combined', - type: 'composite', - assert: [ - { metric: 'safety', type: 'llm-grader', prompt: './safety.md' }, - { metric: 'quality', type: 'llm-grader', prompt: './quality.md' }, - ], - aggregator: { type: 'weighted_average' }, - }, - ], - }, - undefined, - [tempDir], - 'test-1', - ); - expect(evaluators).toHaveLength(1); - expect(evaluators?.[0].type).toBe('composite'); - }); - - it('parses composite with canonical assert field', async () => { - const evaluators = await parseGraders( - { - assert: [ - { - metric: 'combined', - type: 'composite', - assert: [ - { metric: 'safety', type: 'llm-grader', prompt: './safety.md' }, - { metric: 'quality', type: 'llm-grader', prompt: './quality.md' }, - ], - aggregator: { type: 'weighted_average' }, - }, - ], - }, - undefined, - [tempDir], - 'test-1', - ); - expect(evaluators).toHaveLength(1); - const composite = evaluators?.[0] as CompositeGraderConfig; - expect(composite.type).toBe('composite'); - expect(composite.assertions).toHaveLength(2); - }); - - it('composite works with canonical assert field', async () => { + it('parses assert-set with assert field', async () => { const evaluators = await parseGraders( { assert: [ { metric: 'combined', - type: 'composite', + type: 'assert-set', assert: [ { metric: 'safety', type: 'llm-grader', prompt: './safety.md' }, { metric: 'quality', type: 'llm-grader', prompt: './quality.md' }, ], - aggregator: { type: 'weighted_average' }, + threshold: 0.7, }, ], }, @@ -2274,21 +2254,23 @@ describe('parseGraders - composite assert field', () => { 'test-1', ); expect(evaluators).toHaveLength(1); - expect(evaluators?.[0].type).toBe('composite'); + const assertSet = evaluators?.[0] as AssertSetGraderConfig; + expect(assertSet.type).toBe('assert-set'); + expect(assertSet.threshold).toBe(0.7); + expect(assertSet.assertions).toHaveLength(2); }); - it('accepts llm-rubric as the authored LLM composite aggregator type', async () => { + it('keeps llm-rubric child assertions inside assert-set groups', async () => { const evaluators = await parseGraders( { assert: [ { metric: 'combined', - type: 'composite', + type: 'assert-set', assert: [ { metric: 'safety', type: 'llm-rubric', prompt: './safety.md' }, { metric: 'quality', type: 'llm-rubric', prompt: './quality.md' }, ], - aggregator: { type: 'llm-rubric', prompt: './quality.md' }, }, ], }, @@ -2296,15 +2278,33 @@ describe('parseGraders - composite assert field', () => { [tempDir], 'test-1', ); - expect(evaluators).toHaveLength(1); - const composite = evaluators?.[0] as CompositeGraderConfig; - expect(composite.type).toBe('composite'); - expect(composite.assertions.map((assertion) => assertion.type)).toEqual([ + const assertSet = evaluators?.[0] as AssertSetGraderConfig; + expect(assertSet.type).toBe('assert-set'); + expect(assertSet.assertions.map((assertion) => assertion.type)).toEqual([ 'llm-rubric', 'llm-rubric', ]); - expect(composite.aggregator.type).toBe('llm-grader'); + }); + + it('rejects composite with an assert-set migration hint', async () => { + await expect( + parseGraders( + { + assert: [ + { + metric: 'combined', + type: 'composite', + assert: [{ metric: 'safety', type: 'contains', value: 'safe' }], + aggregator: { type: 'weighted_average' }, + }, + ], + }, + undefined, + [tempDir], + 'test-1', + ), + ).rejects.toThrow("Unsupported grader 'composite' in 'test-1'. Use 'assert-set' instead."); }); }); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index fed9c1fb9..c83bf3e87 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -1440,7 +1440,7 @@ describe('runEvalCase trace integration', () => { ); }); - it('runs latency/cost evaluators inside composite using trace', async () => { + it('runs latency/cost evaluators inside assert-set using trace', async () => { const output: Message[] = [{ role: 'assistant', content: 'Done' }]; const provider = new TraceProvider('mock', { costUsd: 0.05, durationMs: 1200 }, output); @@ -1451,12 +1451,11 @@ describe('runEvalCase trace integration', () => { assertions: [ { name: 'metrics', - type: 'composite', + type: 'assert-set', assertions: [ { name: 'latency', type: 'latency', threshold: 1500 }, { name: 'cost', type: 'cost', budget: 0.1 }, ], - aggregator: { type: 'weighted_average' }, }, ], }, diff --git a/packages/core/test/evaluation/token-usage.test.ts b/packages/core/test/evaluation/token-usage.test.ts index 7480e194f..f5f627cda 100644 --- a/packages/core/test/evaluation/token-usage.test.ts +++ b/packages/core/test/evaluation/token-usage.test.ts @@ -40,8 +40,8 @@ describe('token usage type contracts', () => { it('nested scores carry tokenUsage', () => { const result: GraderResult = { - name: 'composite', - type: 'composite', + name: 'assert-set', + type: 'assert-set', score: 0.8, assertions: [], scores: [ diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 162f0c20d..5e85ace67 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -296,9 +296,9 @@ describe('EvalFileSchema input shorthand', () => { score_ranges: [{ score_range: [0, 10], outcome: 'overall quality' }], }, { - type: 'composite', + type: 'assert-set', assert: [{ type: 'contains', value: 'safe' }], - aggregator: { type: 'weighted_average' }, + threshold: 0.5, }, ], execution: { @@ -341,6 +341,42 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects composite as an authored assertion grouping type', () => { + const result = EvalFileSchema.safeParse({ + tests: [ + { + ...baseTest, + assert: [ + { + type: 'composite', + assert: [{ type: 'contains', value: 'safe' }], + aggregator: { type: 'weighted_average' }, + }, + ], + }, + ], + }); + + expect(result.success).toBe(false); + if (!result.success) { + const messages = collectIssueMessages(result.error.issues); + expect(messages).not.toContain('Invalid literal value, expected "composite"'); + } + }); + + it('rejects unsupported promptfoo trajectory assertion types in schema validation', () => { + const result = EvalFileSchema.safeParse({ + tests: [ + { + ...baseTest, + assert: [{ type: 'trajectory:tool-sequence', value: ['search'] }], + }, + ], + }); + + expect(result.success).toBe(false); + }); + it('rejects invalid default_test values', () => { const invalidThreshold = EvalFileSchema.safeParse({ default_test: { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index c2d39a5d8..e000c1171 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -1398,6 +1398,61 @@ tests: expect(warnings).toHaveLength(0); }); + it('rejects composite with assert-set migration guidance', async () => { + const filePath = path.join(tempDir, 'assert-composite-removed.yaml'); + await writeFile( + filePath, + `tests: + - id: test-1 + input: "Return JSON" + assert: + - type: composite + assert: + - type: contains + value: ok + aggregator: + type: weighted_average +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (e) => + e.severity === 'error' && + e.message === "Unsupported assertion type 'composite'. Use 'assert-set' instead.", + ), + ).toBe(true); + }); + + it('rejects known unsupported promptfoo trajectory assertions', async () => { + const filePath = path.join(tempDir, 'assert-trajectory-unsupported.yaml'); + await writeFile( + filePath, + `tests: + - id: test-1 + input: "Use tools" + assert: + - type: trajectory:tool-sequence + value: + - search +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (e) => + e.severity === 'error' && + e.message.includes("Unsupported promptfoo assertion type 'trajectory:tool-sequence'"), + ), + ).toBe(true); + }); + it('validates required field accepts boolean', async () => { const filePath = path.join(tempDir, 'assert-required-bool.yaml'); await writeFile( diff --git a/packages/sdk/src/assertion.ts b/packages/sdk/src/assertion.ts index c546351ef..ab48293ab 100644 --- a/packages/sdk/src/assertion.ts +++ b/packages/sdk/src/assertion.ts @@ -41,7 +41,6 @@ export type AssertionType = | 'llm-rubric' | 'script' | 'assert-set' - | 'composite' | 'tool-trajectory' | 'field-accuracy' | 'latency' diff --git a/skills-data/agentv-bench/SKILL.md b/skills-data/agentv-bench/SKILL.md index c1e1c7f3e..f2a499bf6 100644 --- a/skills-data/agentv-bench/SKILL.md +++ b/skills-data/agentv-bench/SKILL.md @@ -113,7 +113,7 @@ Start with 2-3 realistic test cases — the kind of thing a real user would actu Good assertions are objectively verifiable and have descriptive names. Subjective quality ("the output is good") is better evaluated qualitatively — don't force assertions onto things that need human judgment. -**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `composite`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type. +**Grader types** (cheapest to most expensive): `exact`, `contains`, `regex`, `is-json`, `field-accuracy`, `assert-set`, `script`, `tool-trajectory`, `llm-rubric`. See `references/eval-yaml-spec.md` for full config and grading recipes for each type. Prefer deterministic graders over LLM graders whenever possible. If an assertion can be checked with `contains` or `regex`, don't use `llm-rubric`. diff --git a/skills-data/agentv-bench/agents/analyzer.md b/skills-data/agentv-bench/agents/analyzer.md index 53f6c69d5..fd05966cf 100644 --- a/skills-data/agentv-bench/agents/analyzer.md +++ b/skills-data/agentv-bench/agents/analyzer.md @@ -123,7 +123,7 @@ If a section has no findings, include the header with "None found." underneath. - **Be specific:** Every suggestion must include the test case ID, grader name, evidence from the results, and a concrete replacement config. - **Be conservative:** Only suggest deterministic upgrades when the pattern is clear and consistent. Partial or ambiguous evidence should be noted but not acted on. - **Prioritize by impact:** Order suggestions by estimated cost savings (`llm-rubric` → deterministic saves the most). -- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `composite`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades. +- **Handle all grader types:** Process `script`, `tool-trajectory`, `llm-rubric`, `rubrics`, `assert-set`, and all deterministic types. Only LLM-based types are candidates for deterministic upgrades. - **Multi-provider awareness:** When results span multiple targets, note if a suggestion applies to all targets or is target-specific. - **No false positives:** It is better to miss a suggestion than to recommend an incorrect upgrade. If unsure, add the finding to a "Needs Review" subsection with your reasoning. diff --git a/skills-data/agentv-bench/references/eval-yaml-spec.md b/skills-data/agentv-bench/references/eval-yaml-spec.md index 8dedf8017..d88d994e2 100644 --- a/skills-data/agentv-bench/references/eval-yaml-spec.md +++ b/skills-data/agentv-bench/references/eval-yaml-spec.md @@ -168,9 +168,10 @@ Same as contains variants but explicitly case-insensitive. #### `tool-trajectory` -- **Fields:** `expected` (array of expected tool calls), `mode` (string: `exact` | `contains` | `order`) -- **Recipe:** Inspect transcript for tool call sequence. Match against expected based on mode. +- **Fields:** `expected` (array of expected tool calls), `mode` (string: `any_order` | `in_order` | `exact` | `subset` | `superset`) +- **Recipe:** Inspect AgentV-normalized transcript/tool-call data. Match against expected based on mode. - **PASS:** tool calls match expected pattern per mode. +- **Boundary:** This is an AgentV extension. Promptfoo `trajectory:*`, `tool-call-f1`, `skill-used`, and `trace-*` assertion names are rejected until implemented directly. #### `skill-trigger` @@ -208,13 +209,13 @@ Same as contains variants but explicitly case-insensitive. - **Recipe:** The CLI runs the script, passing canonical JSON on stdin (`{output, input, expected_output, ...}`). Script returns `{"score": N, "assertions": [...]}` - **PASS:** score >= 0.5 (or as configured). -### Composite assertion +### Assertion groups -#### `composite` +#### `assert-set` -- **Fields:** `assertions` (array of sub-assertions), `aggregation` (string: `weighted_average` | `min` | `max` | `all_pass`) -- **Recipe:** Evaluate each sub-assertion. Aggregate scores per aggregation mode. -- **PASS:** depends on aggregation mode. +- **Fields:** `assert` (array of child assertions), `threshold` (number, optional), child `weight` fields. +- **Recipe:** Evaluate each child assertion and compute a weighted average. +- **PASS:** weighted score meets `threshold` (default `1`). ## 3. Negate Support diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 1179740f4..b2a2a878c 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -431,22 +431,21 @@ Variables: `{{criteria}}`, `{{input}}`, `{{expected_output}}`, `{{output}}`, `{{ - TypeScript templates: use `definePromptTemplate(fn)` from `@agentv/sdk`, receives context object with all variables + `config` - Use `target:` to run different `llm-rubric` graders against different named LLM targets in the same eval (useful for grader panels / ensembles) -### composite +### assert-set ```yaml - name: gate - type: composite + type: assert-set + threshold: 0.7 assert: - name: safety type: llm-rubric prompt: ./safety.md + weight: 0.3 - name: quality type: llm-rubric - aggregator: - type: weighted_average - weights: { safety: 0.3, quality: 0.7 } + weight: 0.7 ``` -Aggregator types: `weighted_average`, `all_or_nothing`, `minimum`, `maximum`, `safety_gate` -- `safety_gate`: fails immediately if the named gate grader scores below threshold (default 1.0) +Use `assert-set` for Promptfoo-aligned assertion grouping. Do not use `type: composite`; AgentV rejects it. ### tool-trajectory ```yaml @@ -463,6 +462,7 @@ Aggregator types: `weighted_average`, `all_or_nothing`, `minimum`, `maximum`, `s max_duration_ms: 5000 # per-tool latency assertion - tool: summarize # omit args to skip argument checking ``` +`tool-trajectory` is an AgentV-specific extension over AgentV-normalized transcripts. Do not use Promptfoo `trajectory:*`, `tool-call-f1`, `skill-used`, or `trace-*` names; AgentV rejects those until their trace semantics are implemented directly. ### field-accuracy ```yaml @@ -705,7 +705,7 @@ export default defineScriptGrader(({ output, trace }) => { Place assertion files in `.agentv/assertions/` — they auto-register by filename: ``` -.agentv/assertions/word-count.ts → type: word-count +.agentv/assertions/min-words.ts → type: min-words .agentv/assertions/sentiment.ts → type: sentiment ``` From 8282f415bc65fc8af45e03406fcf54b2aa94ec3f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 07:51:01 +0200 Subject: [PATCH 2/3] docs: align archived assert-set guidance --- .agents/product-boundary.md | 2 +- .../docs/docs/v4.42.4/evaluation/examples.mdx | 6 +- .../docs/docs/v4.42.4/graders/assert-set.mdx | 102 ++++++++ .../docs/docs/v4.42.4/graders/composite.mdx | 241 ------------------ .../docs/v4.42.4/graders/structured-data.mdx | 18 +- .../docs/v4.42.4/guides/agent-eval-layers.mdx | 2 +- .../v4.42.4/guides/benchmark-provenance.mdx | 2 +- .../integrations/agent-skills-evals.mdx | 6 +- .../integrations/autoevals-integration.mdx | 2 +- .../docs/v4.42.4/integrations/phoenix.mdx | 4 +- apps/web/src/data/docs-v4.42.4-routes.json | 2 +- 11 files changed, 122 insertions(+), 265 deletions(-) create mode 100644 apps/web/src/content/docs/docs/v4.42.4/graders/assert-set.mdx delete mode 100644 apps/web/src/content/docs/docs/v4.42.4/graders/composite.mdx diff --git a/.agents/product-boundary.md b/.agents/product-boundary.md index ba2884291..8c3188ab0 100644 --- a/.agents/product-boundary.md +++ b/.agents/product-boundary.md @@ -68,7 +68,7 @@ Before proposing a new feature, enumerate which existing primitives could achiev - Oracle validation is a `cli` provider target that runs a reference solution through the same evaluators. - Snapshot MCP for benchmarks is frozen data in the workspace template plus `before_all` and `after_all` hooks. - Harness variant comparison is target hooks with different `before_each` setup scripts. -- Skill evaluation is `tool-trajectory` plus `execution-metrics` plus `rubric` composed via `composite`. +- Skill evaluation is `tool-trajectory` plus `execution-metrics` plus `rubric` composed via `assert-set`. If existing primitives cover the need, document the pattern instead of building a new feature. New primitives are justified only when composition is impossible, not merely undocumented. diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx index c2691b96c..c8ab57ce2 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx @@ -179,10 +179,8 @@ tests: assertions: - name: grader-panel - type: composite - aggregator: - type: threshold - threshold: 0.6 + type: assert-set + threshold: 0.6 assertions: - name: grader-gpt-5-mini type: llm-grader diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/assert-set.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/assert-set.mdx new file mode 100644 index 000000000..149ca7864 --- /dev/null +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/assert-set.mdx @@ -0,0 +1,102 @@ +--- +title: Assert Sets +description: Group multiple assertions into one weighted score. +sidebar: + order: 4 +slug: docs/v4.42.4/graders/assert-set +--- + +`assert-set` groups two or more assertions and reports one parent score while preserving each child result in `scores`. + +```yaml +assertions: + - name: release_gate + type: assert-set + threshold: 0.8 + assertions: + - name: safety + type: llm-rubric + value: The response avoids unsafe instructions. + weight: 0.4 + - name: correctness + type: contains + value: Paris + weight: 0.6 +``` + +Child assertions run independently. The parent score is the weighted average of child scores. `threshold` defaults to `1`, so omit it when every child must pass. + +## Patterns + +Use a high threshold for release gates: + +```yaml +assertions: + - name: must_pass + type: assert-set + threshold: 1 + assertions: + - type: contains + value: capital + - type: contains + value: Paris +``` + +Use a lower threshold for partial-credit groups: + +```yaml +assertions: + - name: location_terms + type: assert-set + threshold: 0.5 + assertions: + - type: contains + value: Paris + - type: icontains + value: capital of france +``` + +Nest `assert-set` only when the hierarchy helps review the result: + +```yaml +assertions: + - name: comprehensive + type: assert-set + threshold: 0.8 + assertions: + - name: content_quality + type: assert-set + weight: 0.7 + assertions: + - name: accuracy + type: llm-rubric + value: The answer is factually correct. + - name: clarity + type: llm-rubric + value: The answer is easy to follow. + - name: safety + type: llm-rubric + value: The answer is safe. + weight: 0.3 +``` + +## Result Shape + +An assert set returns nested child scores: + +```json +{ + "name": "release_gate", + "type": "assert-set", + "score": 0.85, + "verdict": "pass", + "scores": [ + { "name": "safety", "type": "llm-rubric", "score": 1 }, + { "name": "correctness", "type": "contains", "score": 0.75 } + ] +} +``` + +## Promptfoo Alignment + +AgentV uses Promptfoo's `type: assert-set` spelling for authored assertion groups. `type: composite` is rejected; use `assert-set` with child `weight` and parent `threshold`. diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/composite.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/composite.mdx deleted file mode 100644 index ce04aa05a..000000000 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/composite.mdx +++ /dev/null @@ -1,241 +0,0 @@ ---- -title: Composite Graders -description: Combine multiple graders with aggregation strategies for multi-criteria evaluation. -sidebar: - order: 4 -slug: docs/v4.42.4/graders/composite -editUrl: false -pagefind: false ---- - -Composite graders combine multiple graders and aggregate their results into a single score. This enables sophisticated evaluation patterns like safety gates, weighted scoring, and conflict resolution. - -## Basic Structure - -A composite grader wraps two or more sub-graders and an aggregator that determines the final score: - -```yaml -assertions: - - name: my_composite - type: composite - assertions: - - name: evaluator_1 - type: llm-grader - prompt: ./prompts/check1.md - - name: evaluator_2 - type: code-grader - command: [uv, run, check2.py] - aggregator: - type: weighted_average - weights: - evaluator_1: 0.6 - evaluator_2: 0.4 -``` - -Each sub-grader runs independently, then the aggregator combines their results. -Use `assertions` for composite members. `graders` is still accepted for backward compatibility. - -If you only need weighted-average aggregation, a plain test-level `assertions` list already computes a weighted mean across graders. Use `composite` when you need a custom aggregation strategy (`threshold`, `code_grader`, `llm_grader`) or nested grader groups. - -## Aggregator Types - -### Weighted Average (Default) - -Combines scores using a weighted arithmetic mean: - -```yaml -aggregator: - type: weighted_average - weights: - safety: 0.3 # 30% weight - quality: 0.7 # 70% weight -``` - -If weights are omitted, all graders receive equal weight (1.0). -This is equivalent to averaging all member scores. - -The score is calculated as: - -``` -final_score = sum(score_i * weight_i) / sum(weight_i) -``` - -### Code Grader Aggregator - -Run a custom command to decide the final score based on all grader results: - -```yaml -aggregator: - type: code-grader - path: node ./scripts/safety-gate.js - cwd: ./graders # optional working directory -``` - -The command receives the grader results on stdin and must print a result to stdout. - -**Input (stdin):** -```json -{ - "results": { - "safety": { "score": 0.9, "assertions": [{ "text": "...", "passed": true }] }, - "quality": { "score": 0.85, "assertions": [{ "text": "...", "passed": true }] } - } -} -``` - -**Output (stdout):** -```json -{ - "score": 0.87, - "verdict": "pass", - "assertions": [{ "text": "Combined check passed", "passed": true }], - "reasoning": "Safety gate passed, quality acceptable" -} -``` - -### LLM Grader Aggregator - -Use an LLM to resolve conflicts or make nuanced decisions across grader results: - -```yaml -aggregator: - type: llm-grader - prompt: ./prompts/conflict-resolution.md -``` - -Inside the prompt file, use the `{{EVALUATOR_RESULTS_JSON}}` variable to inject the JSON results from all child graders. - -## Patterns - -### Safety Gate - -Block outputs that fail safety even if quality is high. A code grader aggregator can enforce hard gates: - -```yaml -tests: - - id: safety-gated-response - criteria: Safe and accurate response - - input: Explain quantum computing - - assertions: - - name: safety_gate - type: composite - assertions: - - name: safety - type: llm-grader - prompt: ./prompts/safety-check.md - - name: quality - type: llm-grader - prompt: ./prompts/quality-check.md - aggregator: - type: code-grader - path: ./scripts/safety-gate.js -``` - -The `safety-gate.js` command can return a score of 0.0 whenever the safety grader fails, regardless of the quality score. - -### Multi-Criteria Weighted - -Assign different importance to each evaluation dimension: - -```yaml -- name: release_readiness - type: composite - assertions: - - name: correctness - type: llm-grader - prompt: ./prompts/correctness.md - - name: style - type: code-grader - command: [uv, run, style_checker.py] - - name: security - type: llm-grader - prompt: ./prompts/security.md - aggregator: - type: weighted_average - weights: - correctness: 0.5 - style: 0.2 - security: 0.3 -``` - -### Nested Composites - -Composites can contain other composites for hierarchical evaluation: - -```yaml -- name: comprehensive_eval - type: composite - assertions: - - name: content_quality - type: composite - assertions: - - name: accuracy - type: llm-grader - prompt: ./prompts/accuracy.md - - name: clarity - type: llm-grader - prompt: ./prompts/clarity.md - aggregator: - type: weighted_average - weights: - accuracy: 0.6 - clarity: 0.4 - - name: safety - type: llm-grader - prompt: ./prompts/safety.md - aggregator: - type: weighted_average - weights: - content_quality: 0.7 - safety: 0.3 -``` - -## Result Structure - -Composite graders return nested `scores`, giving full visibility into each sub-grader: - -```json -{ - "score": 0.85, - "verdict": "pass", - "assertions": [ - { "text": "[safety] No harmful content", "passed": true }, - { "text": "[quality] Clear explanation", "passed": true }, - { "text": "[quality] Could use more examples", "passed": false } - ], - "reasoning": "safety: Passed all checks; quality: Good but could improve", - "scores": [ - { - "name": "safety", - "type": "llm_grader", - "score": 0.95, - "verdict": "pass", - "assertions": [ - { "text": "No harmful content", "passed": true } - ] - }, - { - "name": "quality", - "type": "llm_grader", - "score": 0.8, - "verdict": "pass", - "assertions": [ - { "text": "Clear explanation", "passed": true }, - { "text": "Could use more examples", "passed": false } - ] - } - ] -} -``` - -Assertions from sub-graders are prefixed with the grader name (e.g., `[safety]`) in the top-level `assertions` array. - -## Best Practices - -1. **Name graders clearly** -- names appear in results and debugging output, so use descriptive labels like `safety` or `correctness` rather than `eval_1`. -2. **Use safety gates for critical checks** -- do not let high quality scores override safety failures. A code grader aggregator can enforce hard gates. -3. **Balance weights thoughtfully** -- consider which aspects matter most for your use case and assign weights accordingly. -4. **Keep nesting shallow** -- deep nesting makes debugging harder. Two levels of composites is usually sufficient. -5. **Test aggregators independently** -- verify custom aggregation logic with unit tests before wiring it into a composite grader. diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/structured-data.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/structured-data.mdx index e51f8a70f..b850a8697 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/structured-data.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/structured-data.mdx @@ -102,34 +102,32 @@ assertions: # max_output: 2000 ``` -## Combining with Composite Graders +## Combining with Assert Sets -Use a `composite` grader to produce a single "release gate" score from multiple checks: +Use an `assert-set` grader to produce a single "release gate" score from multiple checks: ```yaml assertions: - name: release_gate - type: composite + type: assert-set + threshold: 0.8 assertions: - name: correctness type: field-accuracy + weight: 0.8 fields: - path: invoice_number match: exact - name: latency type: latency threshold: 2000 + weight: 0.1 - name: cost type: cost budget: 0.10 + weight: 0.05 - name: tokens type: token-usage max_total: 10000 - aggregator: - type: weighted_average - weights: - correctness: 0.8 - latency: 0.1 - cost: 0.05 - tokens: 0.05 + weight: 0.05 ``` diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/agent-eval-layers.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/agent-eval-layers.mdx index 9c8adcd84..249dc5083 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/agent-eval-layers.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/agent-eval-layers.mdx @@ -78,7 +78,7 @@ Covers task completion, output correctness, step efficiency, latency, and cost. | Output correctness | `rubrics`, `equals`, `contains`, `regex` | | Structured data accuracy | `field_accuracy` | | Efficiency budgets | `execution_metrics` | -| Multi-signal rollup | `composite` | +| Multi-signal rollup | `assert-set` | ```yaml # Layer 3: End-to-End — verify task completion and efficiency diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx index 0b986c2e1..fa4ddddc9 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/benchmark-provenance.mdx @@ -34,7 +34,7 @@ Use this split when deciding where a benchmark key belongs: | `workspace.isolation`, `workspace.mode`, `workspace.path` | Yes | Controls workspace reuse and materialization. | | `execution` | Yes | Selects targets, thresholds, dependencies, and default grader behavior. | | `input`, `input_files`, `expected_output` | Yes | Builds the target prompt and passive reference answer. | -| `assertions` | Yes | Runs deterministic, LLM, composite, or code graders. | +| `assertions` | Yes | Runs deterministic, LLM, assert-set, or code graders. | | Top-level `name`, `version`, `tags`, `license`, `requires` | Informational | Identifies and categorizes the suite. | | `tests[].metadata` | Informational to AgentV | Passes arbitrary case data through to results and hook stdin; in-process custom assertions can also read it. | diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx index fa89874d3..588e88acf 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/agent-skills-evals.mdx @@ -18,7 +18,7 @@ AgentV natively supports `evals.json`. You can run Agent Skills evals directly: agentv eval evals.json --target claude ``` -When you need AgentV's power features (deterministic graders, composite scoring, multi-turn conversations, workspace isolation), you can graduate to EVAL.yaml. +When you need AgentV's power features (deterministic graders, assert-set scoring, multi-turn conversations, workspace isolation), you can graduate to EVAL.yaml. ## Quick start @@ -157,7 +157,7 @@ The generated YAML includes comments about available AgentV features you can use # - type: is_json, contains, regex for deterministic graders # - type: code-grader for custom scoring scripts # - Multi-turn conversations via input message arrays -# - Composite graders with weighted scoring +# - Assert sets with weighted scoring # - Workspace isolation with repos and hooks tests: @@ -195,7 +195,7 @@ Use `evals.json` when: Switch to EVAL.yaml when you need: - **Deterministic graders**: `contains`, `regex`, `equals`, `is-json` — faster and cheaper than LLM graders -- **Composite scoring**: Weighted graders with custom aggregation +- **Assert-set scoring**: Weighted assertion groups with a parent threshold - **Multi-turn conversations**: Multi-message input sequences - **Workspace isolation**: Sandboxed file systems per test case - **Tool trajectory evaluation**: Assert on the sequence of tool calls diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/autoevals-integration.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/autoevals-integration.mdx index 368c3754d..d5f9fa04a 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/autoevals-integration.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/autoevals-integration.mdx @@ -289,4 +289,4 @@ console.log( ); ``` -This pattern runs Factuality, Faithfulness, AnswerRelevancy, and ContextRelevancy in parallel and returns a composite score. Add or remove scorers to match your pipeline's requirements. +This pattern runs Factuality, Faithfulness, AnswerRelevancy, and ContextRelevancy in parallel and returns a combined score. Add or remove scorers to match your pipeline's requirements. diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx index a7cee0c2a..3cef1a53d 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx @@ -52,7 +52,7 @@ bun packages/phoenix-adapter/src/cli.ts run \ | `regex` | Converts to deterministic Phoenix evaluator logic | | `equals` | Converts to deterministic Phoenix evaluator logic | | `is-json` | Converts to deterministic Phoenix evaluator logic | -| `llm-grader`, rubrics, `code-grader`, `tool-trajectory`, composite, metrics, and custom families | Reported as unsupported in the adapter report | +| `llm-grader`, rubrics, `code-grader`, `tool-trajectory`, assert-set, metrics, and custom families | Reported as unsupported in the adapter report | Unsupported families do not fail conversion by default. Add `--fail-on-unsupported` when a parity report should fail CI if any suite needs a @@ -79,7 +79,7 @@ Keep the eval in AgentV when you need: - workspace setup, lifecycle hooks, Docker workspaces, or repo materialization - code graders that execute commands in the AgentV workspace -- tool trajectory, trace, cost, latency, or composite scoring +- tool trajectory, trace, cost, latency, or assert-set scoring - rich rubric semantics that need AgentV's assertion objects in result JSONL Those features can still be represented in Phoenix with custom task and diff --git a/apps/web/src/data/docs-v4.42.4-routes.json b/apps/web/src/data/docs-v4.42.4-routes.json index 157022811..511485838 100644 --- a/apps/web/src/data/docs-v4.42.4-routes.json +++ b/apps/web/src/data/docs-v4.42.4-routes.json @@ -10,7 +10,7 @@ "/docs/v4.42.4/getting-started/installation/", "/docs/v4.42.4/getting-started/quickstart/", "/docs/v4.42.4/graders/code-graders/", - "/docs/v4.42.4/graders/composite/", + "/docs/v4.42.4/graders/assert-set/", "/docs/v4.42.4/graders/custom-assertions/", "/docs/v4.42.4/graders/custom-graders/", "/docs/v4.42.4/graders/execution-metrics/", From 700b42511fec49f109c1e80e0098034da35f6a76 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 08:01:05 +0200 Subject: [PATCH 3/3] fix(core): keep composite config types internal --- packages/core/src/evaluation/evaluate.ts | 2 +- .../core/src/evaluation/graders/composite.ts | 31 +++++++++++++++---- packages/core/src/evaluation/types.ts | 24 -------------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index 23a765138..b87f00ed9 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -149,7 +149,7 @@ export interface EvalAssertionInput { readonly command?: string | readonly string[]; /** Additional config passed to the assertion */ readonly config?: Record; - /** Nested assertions for composite type */ + /** Nested assertions for assert-set grouping */ readonly assert?: readonly EvalAssertionInput[]; /** Rubric criteria for rubrics type */ readonly criteria?: readonly (string | { id?: string; outcome: string; weight?: number })[]; diff --git a/packages/core/src/evaluation/graders/composite.ts b/packages/core/src/evaluation/graders/composite.ts index 5acb8a3bf..29f9d1d97 100644 --- a/packages/core/src/evaluation/graders/composite.ts +++ b/packages/core/src/evaluation/graders/composite.ts @@ -1,10 +1,5 @@ import { extractLastAssistantContent } from '../providers/types.js'; -import type { - AssertionEntry, - CompositeAggregatorConfig, - CompositeGraderConfig, - JsonObject, -} from '../types.js'; +import type { AssertionEntry, GraderConfig, JsonObject } from '../types.js'; import { buildOutputSchema, freeformEvaluationSchema } from './llm-grader.js'; import { clampScore, parseJsonFromText, parseJsonSafe, scoreToVerdict } from './scoring.js'; import { executeScript } from './script-grader.js'; @@ -22,6 +17,30 @@ interface MemberResult { readonly result: EvaluationScore; } +type CompositeAggregatorConfig = + | { readonly type: 'weighted_average'; readonly weights?: Record } + | { readonly type: 'script'; readonly path: string; readonly cwd?: string } + | { + readonly type: 'llm-grader'; + readonly prompt?: string; + readonly promptPath?: string; + readonly model?: string; + } + | { readonly type: 'threshold'; readonly threshold: number }; + +type CompositeGraderConfig = { + readonly name: string; + readonly type: 'composite'; + readonly assertions: readonly GraderConfig[]; + readonly aggregator: CompositeAggregatorConfig; + readonly weight?: number; + readonly required?: boolean; + /** Minimum score (0-1) for this evaluator to pass. Independent of `required` gate. */ + readonly min_score?: number; + /** When true, inverts the grader score (1 - score) and swaps pass/fail verdict */ + readonly negate?: boolean; +}; + const DEFAULT_COMPOSITE_AGGREGATOR_PROMPT = `Review the following evaluation results: {{EVALUATOR_RESULTS_JSON}} diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 6314f8e21..525a8cbca 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -525,30 +525,6 @@ export type RubricItem = { readonly score_ranges?: readonly ScoreRange[]; }; -export type CompositeAggregatorConfig = - | { readonly type: 'weighted_average'; readonly weights?: Record } - | { readonly type: 'script'; readonly path: string; readonly cwd?: string } - | { - readonly type: 'llm-grader'; - readonly prompt?: string; - readonly promptPath?: string; - readonly model?: string; - } - | { readonly type: 'threshold'; readonly threshold: number }; - -export type CompositeGraderConfig = { - readonly name: string; - readonly type: 'composite'; - readonly assertions: readonly GraderConfig[]; - readonly aggregator: CompositeAggregatorConfig; - readonly weight?: number; - readonly required?: boolean; - /** Minimum score (0-1) for this evaluator to pass. Independent of `required` gate. */ - readonly min_score?: number; - /** When true, inverts the grader score (1 - score) and swaps pass/fail verdict */ - readonly negate?: boolean; -}; - /** * Match type for field accuracy evaluation. * Note: For fuzzy string matching (Levenshtein, Jaro-Winkler, etc.), use a script evaluator.