From d49d8e34329b911629e3af0158c52683c7c7cdbe Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 03:43:44 +0200 Subject: [PATCH 1/2] feat(core)!: require canonical assertions Remove the legacy assert authoring alias from the programmatic evaluate API and parser surfaces so eval definitions use the durable assertions contract everywhere. Update docs, examples, fixtures, and regression coverage for the breaking removal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/content/docs/docs/evaluation/sdk.mdx | 8 +-- .../default-graders/evals/dataset.eval.yaml | 6 +- .../features/rubric/evals/dataset.eval.yaml | 4 +- .../sdk-programmatic-api-advanced/evaluate.ts | 6 +- .../features/sdk-programmatic-api/README.md | 6 +- .../features/sdk-programmatic-api/evaluate.ts | 4 +- .../suite-level-input/evals/dataset.eval.yaml | 2 +- examples/showcase/cross-repo-sync/README.md | 2 +- .../cross-repo-sync/evals/dataset.eval.yaml | 8 +-- .../evals/ground-truth/eval-spec-v2.diff | 4 +- .../cross-repo-sync/scripts/validate-sync.ts | 2 +- .../workspace-template/mock-agent.sh | 4 +- packages/core/src/evaluation/assertions.ts | 4 +- packages/core/src/evaluation/evaluate.ts | 65 ++++++++++++++----- .../loaders/eval-yaml-transpiler.ts | 41 ++---------- .../src/evaluation/loaders/grader-parser.ts | 20 +++--- .../src/evaluation/loaders/jsonl-parser.ts | 6 +- .../evaluation/validation/eval-validator.ts | 42 +++++++++--- packages/core/src/evaluation/yaml-parser.ts | 11 +--- .../test/evaluation/criteria-optional.test.ts | 2 +- .../test/evaluation/evaluate-enhanced.test.ts | 39 ++++++++--- .../evaluate-programmatic-api.test.ts | 36 +++++----- .../loaders/eval-yaml-transpiler.test.ts | 30 +++++---- .../loaders/fixtures/default-export.eval.ts | 2 +- .../fixtures/eval-config-named.eval.ts | 2 +- .../loaders/fixtures/named-config.eval.ts | 2 +- .../evaluation/loaders/grader-parser.test.ts | 12 ++-- .../core/test/evaluation/orchestrator.test.ts | 8 +-- .../validation/eval-validator.test.ts | 41 +++++++----- 29 files changed, 228 insertions(+), 191 deletions(-) diff --git a/apps/web/src/content/docs/docs/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/evaluation/sdk.mdx index 5dabbfafd..34abcc28b 100644 --- a/apps/web/src/content/docs/docs/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/evaluation/sdk.mdx @@ -44,7 +44,7 @@ Use the simplest surface that matches the job: - **YAML / JSONL first** for portable eval specs you want to run from the CLI, check into a repo, or share across TypeScript and Python workflows. - **`defineEval()` / `evalSuite()`** when you want a `.eval.ts` file that mirrors YAML concepts and lowers back to the canonical snake_case contract. - **`evaluate({ specFile })`** when you want library control around an existing YAML suite. -- **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput` and `assert`. +- **Inline `evaluate({ tests })`** when the eval definition truly belongs inside application code. The programmatic API mirrors YAML, but uses current TypeScript naming such as `expectedOutput`. - **`defineAssertion` / `defineCodeGrader`** when the grading logic itself must execute code. - **`agentv eval `** for deterministic workspace checks that fit normal Vitest `expect(...)` tests. @@ -348,7 +348,7 @@ const { results, summary } = await evaluate({ id: 'greeting', input: 'Say hello', expectedOutput: 'Hello there!', - assert: [{ type: 'contains', value: 'Hello' }], + assertions: [{ type: 'contains', value: 'Hello' }], }, ], }); @@ -356,7 +356,7 @@ const { results, summary } = await evaluate({ console.log(`${summary.passed}/${summary.total} passed`); ``` -A strict OR is easy with `assert` inline handlers: +A strict OR is easy with inline assertion handlers: ```typescript import { evaluate } from '@agentv/sdk'; @@ -367,7 +367,7 @@ const { summary } = await evaluate({ id: 'capital', input: 'What is the capital of France?', expectedOutput: 'Paris', - assert: [ + assertions: [ ({ output }) => ({ name: 'capital-or-phrase', score: ((output ?? '').includes('Paris') || /capital of france/i.test(output ?? '')) ? 1 : 0, diff --git a/examples/features/default-graders/evals/dataset.eval.yaml b/examples/features/default-graders/evals/dataset.eval.yaml index bb4080531..abf1c8b4b 100644 --- a/examples/features/default-graders/evals/dataset.eval.yaml +++ b/examples/features/default-graders/evals/dataset.eval.yaml @@ -1,5 +1,5 @@ # Default Graders Example -# Demonstrates root-level assert that apply to all tests +# Demonstrates root-level assertionsions that apply to all tests name: default-graders-example description: Root-level graders that automatically apply to every test @@ -15,7 +15,7 @@ tests: criteria: The assistant responds with a friendly greeting input: "Hello!" expected_output: "Hello! How can I help you today?" - # Gets tone_check from root-level assert + # Gets tone_check from root-level assertions - id: with-custom-eval criteria: The assistant provides a helpful response about refunds @@ -24,7 +24,7 @@ tests: assertions: - name: helpfulness type: llm-grader - # Also gets tone_check from root-level assert + # Also gets tone_check from root-level assertions - id: skip-defaults criteria: The assistant handles urgent requests appropriately diff --git a/examples/features/rubric/evals/dataset.eval.yaml b/examples/features/rubric/evals/dataset.eval.yaml index 0f1c4d577..abff5a9b0 100644 --- a/examples/features/rubric/evals/dataset.eval.yaml +++ b/examples/features/rubric/evals/dataset.eval.yaml @@ -1,5 +1,5 @@ # AgentV Rubric Grader Example -# Demonstrates the rubric-based evaluation feature using type: rubrics under assert +# Demonstrates the rubric-based evaluation feature using type: rubrics under assertions name: rubric description: "Example showing rubric grader - string shorthand and type: rubrics" @@ -10,7 +10,7 @@ execution: tests: # ========================================== # Example 1: Simple string rubrics - # Demonstrates: string shorthand in assert (strings default to rubrics grader) + # Demonstrates: string shorthand in assertions (strings default to rubrics grader) # ========================================== - id: code-explanation-simple diff --git a/examples/features/sdk-programmatic-api-advanced/evaluate.ts b/examples/features/sdk-programmatic-api-advanced/evaluate.ts index 7fdf05338..514663de0 100644 --- a/examples/features/sdk-programmatic-api-advanced/evaluate.ts +++ b/examples/features/sdk-programmatic-api-advanced/evaluate.ts @@ -20,7 +20,7 @@ const { results, summary } = await evaluate({ { id: 'greeting', input: 'Say hello.', - assert: [{ type: 'contains', value: 'Hello' }], + assertions: [{ type: 'contains', value: 'Hello' }], }, // Multi-turn conversation test @@ -30,12 +30,12 @@ const { results, summary } = await evaluate({ turns: [ { input: 'Hi, my name is Alice.', - assert: [{ type: 'contains', value: 'Alice' }], + assertions: [{ type: 'contains', value: 'Alice' }], }, { input: 'What is my name?', expectedOutput: 'Your name is Alice.', - assert: [{ type: 'contains', value: 'Alice' }], + assertions: [{ type: 'contains', value: 'Alice' }], }, ], // Use weakest-link scoring: final score = lowest turn score diff --git a/examples/features/sdk-programmatic-api/README.md b/examples/features/sdk-programmatic-api/README.md index d162ed3fa..944a778ff 100644 --- a/examples/features/sdk-programmatic-api/README.md +++ b/examples/features/sdk-programmatic-api/README.md @@ -1,11 +1,11 @@ # SDK Example: Programmatic API -Demonstrates using `evaluate()` from `@agentv/sdk` to run evaluations as a library when the eval definition belongs in TypeScript. The config mirrors the canonical YAML surface, but uses programmatic names such as `expectedOutput` and `assert`. +Demonstrates using `evaluate()` from `@agentv/sdk` to run evaluations as a library when the eval definition belongs in TypeScript. The config mirrors the canonical YAML surface, but uses programmatic names such as `expectedOutput` and canonical `assertions`. ## What It Does 1. Imports `evaluate()` from `@agentv/sdk` -2. Defines tests inline with `assert` +2. Defines tests inline with `assertions` 3. Runs the evaluation and prints summary statistics 4. Writes canonical AgentV run artifacts under `.agentv/results/...` @@ -24,6 +24,6 @@ bun run evaluate.ts - **`evaluate()`** — use AgentV as a library, not just a CLI - **Inline tests** — define YAML-shaped tests directly in TypeScript -- **Config mirrors YAML** — same evaluation model, with programmatic `assert` and camelCase fields +- **Config mirrors YAML** — same evaluation model, with canonical `assertions` and camelCase fields - **Typed results** — `EvalRunResult` with summary statistics - **Canonical artifacts** — opt into the same `index.jsonl` / `summary.json` workspace layout as `agentv eval` diff --git a/examples/features/sdk-programmatic-api/evaluate.ts b/examples/features/sdk-programmatic-api/evaluate.ts index fc24f9b68..952538fa0 100644 --- a/examples/features/sdk-programmatic-api/evaluate.ts +++ b/examples/features/sdk-programmatic-api/evaluate.ts @@ -15,7 +15,7 @@ const { results, summary } = await evaluate({ id: 'greeting', input: 'Say hello and introduce yourself briefly.', expectedOutput: "Hello! I'm an AI assistant here to help you.", - assert: [{ type: 'contains', value: 'Hello' }], + assertions: [{ type: 'contains', value: 'Hello' }], }, { id: 'json-output', @@ -24,7 +24,7 @@ const { results, summary } = await evaluate({ { role: 'user', content: 'Return a JSON object with a "status" field set to "ok".' }, ], expectedOutput: '{"status": "ok"}', - assert: [ + assertions: [ { type: 'is-json', required: true }, { type: 'contains', value: 'ok' }, ], diff --git a/examples/features/suite-level-input/evals/dataset.eval.yaml b/examples/features/suite-level-input/evals/dataset.eval.yaml index 980a8ec58..f702a5d41 100644 --- a/examples/features/suite-level-input/evals/dataset.eval.yaml +++ b/examples/features/suite-level-input/evals/dataset.eval.yaml @@ -3,7 +3,7 @@ # This avoids repeating the same system prompt file in each test case. name: suite-level-input-example -description: Suite-level input prepended to all tests (like suite-level assert) +description: Suite-level input prepended to all tests (like suite-level assertions) execution: target: llm diff --git a/examples/showcase/cross-repo-sync/README.md b/examples/showcase/cross-repo-sync/README.md index 8f5ff268d..96603f867 100644 --- a/examples/showcase/cross-repo-sync/README.md +++ b/examples/showcase/cross-repo-sync/README.md @@ -18,7 +18,7 @@ When **agentv** (EntityProcess/agentv) ships a feature, the **agentevals** (agen ## Test Cases -1. **eval-spec-v2-sync** — Add 4 deterministic assert types + required gates +1. **eval-spec-v2-sync** — Add 4 deterministic assertion types + required gates 2. **cases-to-tests-sync** — Rename `cases` → `tests` across spec docs 3. **schema-field-rename-sync** — Rename `eval_cases` → `cases`, `expected_outcome` → `criteria`/`outcome` diff --git a/examples/showcase/cross-repo-sync/evals/dataset.eval.yaml b/examples/showcase/cross-repo-sync/evals/dataset.eval.yaml index 4180c06d5..506077db0 100644 --- a/examples/showcase/cross-repo-sync/evals/dataset.eval.yaml +++ b/examples/showcase/cross-repo-sync/evals/dataset.eval.yaml @@ -25,13 +25,13 @@ tests: ground_truth: ../evals/ground-truth/eval-spec-v2.diff criteria: >- Update agentevals spec to reflect eval spec v2: add contains/regex/is_json/equals - assert types, required gates for all graders, tests-as-string-path. + assertion types, required gates for all graders, tests-as-string-path. input: - role: user content: | agentv just merged eval spec v2 (PR #262). Update the agentevals - spec docs to reflect: 4 new deterministic assert types, required - gates, assert field at test/suite level, tests-as-string-path. + spec docs to reflect: 4 new deterministic assertion types, required + gates, assertions field at test/suite level, tests-as-string-path. assertions: - name: sync-check type: code-grader @@ -39,7 +39,7 @@ tests: expected_files_modified: - agentevals/docs/src/content/docs/specification/graders.mdx - agentevals/docs/src/content/docs/specification/eval-format.mdx - expected_keywords: [contains, regex, is_json, equals, required, assert] + expected_keywords: [contains, regex, is_json, equals, required, assertions] - id: cases-to-tests-sync metadata: diff --git a/examples/showcase/cross-repo-sync/evals/ground-truth/eval-spec-v2.diff b/examples/showcase/cross-repo-sync/evals/ground-truth/eval-spec-v2.diff index 24bbb3755..a8888676d 100644 --- a/examples/showcase/cross-repo-sync/evals/ground-truth/eval-spec-v2.diff +++ b/examples/showcase/cross-repo-sync/evals/ground-truth/eval-spec-v2.diff @@ -44,7 +44,7 @@ index 2055cfc..fa75628 100644 # Evaluation (optional) rubrics: (string | Rubric)[] # Inline evaluation criteria -+assert: Assertion[] # Deterministic and LLM assertions ++assertions: Assertion[] # Deterministic and LLM assertions execution: ExecutionConfig # Per-test execution override # Metadata (optional) @@ -210,7 +210,7 @@ index e1a4bc4..22bf3e9 100644 - - id: safety - outcome: No harmful content - required: true # Fail verdict if missed, regardless of score -+assert: ++assertions: + - type: contains + value: "DENIED" + required: true # Must pass (>= 0.8) diff --git a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts index 15bbd6aa3..be752dfa5 100644 --- a/examples/showcase/cross-repo-sync/scripts/validate-sync.ts +++ b/examples/showcase/cross-repo-sync/scripts/validate-sync.ts @@ -5,7 +5,7 @@ * - File-level overlap: which expected files were modified * - Keyword matching: key terms that should appear in modifications * - * Pass-through config (from assert block in YAML): + * Pass-through config (from assertions block in YAML): * - expected_files_modified: string[] — paths that should appear in fileChanges * - expected_keywords: string[] — terms that should appear in the diff * - ground_truth: string — path to the ground truth diff file (from metadata) diff --git a/examples/showcase/cross-repo-sync/workspace-template/mock-agent.sh b/examples/showcase/cross-repo-sync/workspace-template/mock-agent.sh index 930b97867..9673ca28a 100755 --- a/examples/showcase/cross-repo-sync/workspace-template/mock-agent.sh +++ b/examples/showcase/cross-repo-sync/workspace-template/mock-agent.sh @@ -12,7 +12,7 @@ sedi() { sed -i.bak "$@" && find . -name '*.bak' -delete; } cd agentevals if echo "$PROMPT" | grep -qi "eval spec v2"; then - # Scenario 1: Add assert types and required gates + # Scenario 1: Add assertion types and required gates EVAL_FILE="docs/src/content/docs/specification/evaluators.mdx" FORMAT_FILE="docs/src/content/docs/specification/eval-format.mdx" @@ -33,7 +33,7 @@ PATCH sedi 's/weight: number/weight: number\n required: boolean/' "$FORMAT_FILE" 2>/dev/null || true fi - echo "Updated evaluators and eval-format for eval spec v2 assert types and required gates" > "$OUTPUT_FILE" + echo "Updated evaluators and eval-format for eval spec v2 assertion types and required gates" > "$OUTPUT_FILE" elif echo "$PROMPT" | grep -qi "cases.*tests"; then # Scenario 2: Rename cases to tests diff --git a/packages/core/src/evaluation/assertions.ts b/packages/core/src/evaluation/assertions.ts index 0a724fe1d..406ae4398 100644 --- a/packages/core/src/evaluation/assertions.ts +++ b/packages/core/src/evaluation/assertions.ts @@ -5,11 +5,11 @@ * that doesn't fit a built-in grader type. For built-in assertions * (contains, regex, is-json, etc.), use config objects instead: * - * assert: [{ type: 'contains', value: 'hello' }] + * assertions: [{ type: 'contains', value: 'hello' }] * * Inline functions are for custom logic: * - * assert: [({ output }) => ({ name: 'len', score: output.length > 5 ? 1 : 0 })] + * assertions: [({ output }) => ({ name: 'len', score: output.length > 5 ? 1 : 0 })] */ /** Context passed to inline assertion functions */ diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index be48b831a..adeb2f204 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -15,7 +15,7 @@ * id: 'capital', * input: 'What is the capital of France?', * expectedOutput: 'Paris', - * assert: [{ type: 'contains', value: 'Paris' }], + * assertions: [{ type: 'contains', value: 'Paris' }], * }, * ], * target: { provider: 'mock_agent' }, @@ -34,7 +34,7 @@ * id: 'echo', * input: 'hello', * expectedOutput: 'Echo: hello', - * assert: [ + * assertions: [ * { type: 'contains', value: 'hello' }, * { type: 'equals' }, * ({ output }) => ({ name: 'custom', score: output.length > 0 ? 1 : 0 }), @@ -104,7 +104,7 @@ export interface EvalTestInput { /** @deprecated Use `expectedOutput` instead */ readonly expected_output?: string; /** Assertion graders — accepts factory functions, config objects, or inline functions */ - readonly assert?: readonly AssertEntry[]; + readonly assertions?: readonly AssertEntry[]; /** Arbitrary metadata */ readonly metadata?: Record; /** Enable multi-turn conversation mode. Inferred automatically when turns[] is provided. */ @@ -127,12 +127,12 @@ export interface ConversationTurnInput { /** @deprecated Use `expectedOutput` instead */ readonly expected_output?: string; /** Per-turn assertions (string criteria or grader config) */ - readonly assert?: readonly AssertEntry[]; + readonly assertions?: readonly AssertEntry[]; } /** * Inline assertion definition for the programmatic API. - * Matches the YAML `assert` block structure. + * Matches the YAML `assertions` block structure. */ export interface EvalAssertionInput { /** Assertion type (e.g., 'contains', 'llm-grader', 'code-grader') */ @@ -154,7 +154,7 @@ export interface EvalAssertionInput { /** Additional config passed to the assertion */ readonly config?: Record; /** Nested assertions for composite type */ - readonly assert?: readonly EvalAssertionInput[]; + readonly assertions?: readonly EvalAssertionInput[]; /** Rubric criteria for rubrics type */ readonly criteria?: readonly (string | { id?: string; outcome: string; weight?: number })[]; /** Additional properties */ @@ -178,7 +178,7 @@ export interface EvalConfig { /** Custom task function — mutually exclusive with target */ readonly task?: (input: string) => string | Promise; /** Suite-level assertions applied to all tests */ - readonly assert?: readonly AssertEntry[]; + readonly assertions?: readonly AssertEntry[]; /** Optional suite metadata used by CLI discovery, tagging, and reporting. */ readonly metadata?: EvalMetadata; /** Filter tests by ID pattern(s) (glob supported). Arrays use OR logic. */ @@ -276,7 +276,7 @@ export interface EvalRunArtifacts { * { * id: 'greeting', * input: 'Say hello', - * assert: [{ type: 'contains', value: 'hello' }], + * assertions: [{ type: 'contains', value: 'hello' }], * }, * ], * target: { provider: 'mock_agent' }, @@ -486,10 +486,37 @@ function toBeforeAllHook(beforeAll: string | readonly string[]): WorkspaceHookCo return { command }; } +const REMOVED_ASSERT_KEY = 'assert'; + +function rejectRemovedAssertKey(value: unknown, location: string): void { + if ( + value && + typeof value === 'object' && + Object.prototype.hasOwnProperty.call(value, REMOVED_ASSERT_KEY) + ) { + throw new Error(`${location}: 'assert' has been removed. Use 'assertions' instead.`); + } +} + +function validateAssertionEntries( + entries: readonly AssertEntry[] | undefined, + location: string, +): void { + entries?.forEach((entry, i) => { + if (typeof entry === 'function') return; + rejectRemovedAssertKey(entry, `${location}[${i}]`); + validateAssertionEntries(entry.assertions, `${location}[${i}].assertions`); + }); +} + /** - * Convert an array of assert entries (inline functions or config objects) to GraderConfig[]. + * Convert an array of assertion entries (inline functions or config objects) to GraderConfig[]. */ -function convertAssertions(entries: readonly AssertEntry[]): GraderConfig[] { +function convertAssertions( + entries: readonly AssertEntry[], + location = 'assertions', +): GraderConfig[] { + validateAssertionEntries(entries, location); return entries.map((entry, i) => { if (typeof entry === 'function') { const base: InlineAssertEvaluatorConfig = { @@ -518,6 +545,7 @@ function buildInlineEvalTests( readonly testFilePath: string; }, ): readonly EvalTest[] { + rejectRemovedAssertKey(config, 'evaluate config'); const suiteWorkspace = config.beforeAll ? { hooks: { before_all: toBeforeAllHook(config.beforeAll) } } : undefined; @@ -530,6 +558,7 @@ function buildInlineEvalTests( return (config.tests ?? []) .filter((test) => !options.filter || matchesFilter(test.id, options.filter)) .map((test): EvalTest => { + rejectRemovedAssertKey(test, `Test '${test.id}'`); const isConversation = test.mode === 'conversation' || (test.turns && test.turns.length > 0); if (!isConversation && !test.input) { @@ -551,16 +580,19 @@ function buildInlineEvalTests( ] as EvalTest['expected_output']) : []; - const allAssertions = [...(test.assert ?? []), ...(config.assert ?? [])]; - const assertConfigs = convertAssertions(allAssertions); + const allAssertions = [...(test.assertions ?? []), ...(config.assertions ?? [])]; + const assertConfigs = convertAssertions(allAssertions, `Test '${test.id}'.assertions`); const turns: ConversationTurn[] | undefined = test.turns?.map((turn) => { + rejectRemovedAssertKey(turn, `Test '${test.id}'.turns[]`); const turnExpected = turn.expectedOutput ?? turn.expected_output; return { input: turn.input as ConversationTurn['input'], ...(turnExpected !== undefined && { expected_output: turnExpected as ConversationTurn['expected_output'], }), - assertions: turn.assert ? convertAssertions([...turn.assert]) : undefined, + assertions: turn.assertions + ? convertAssertions([...turn.assertions], `Test '${test.id}'.turns[].assertions`) + : undefined, }; }); @@ -588,14 +620,17 @@ function applyProgrammaticSuiteOverrides( tests: readonly EvalTest[], config: EvalConfig, ): readonly EvalTest[] { - if (!config.beforeAll && (!config.assert || config.assert.length === 0)) { + rejectRemovedAssertKey(config, 'evaluate config'); + if (!config.beforeAll && (!config.assertions || config.assertions.length === 0)) { return tests; } const suiteWorkspace = config.beforeAll ? { hooks: { before_all: toBeforeAllHook(config.beforeAll) } } : undefined; - const suiteAssertions = config.assert ? convertAssertions(config.assert) : []; + const suiteAssertions = config.assertions + ? convertAssertions(config.assertions, 'evaluate config.assertions') + : []; return tests.map((test) => ({ ...test, diff --git a/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts b/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts index 306b48069..daafeb00a 100644 --- a/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts +++ b/packages/core/src/evaluation/loaders/eval-yaml-transpiler.ts @@ -4,7 +4,7 @@ * Converts an AgentV EVAL.yaml file into Agent Skills evals.json format * for consumption by the skill-creator pipeline. * - * Handles both `assertions:` (current) and `assert:` (deprecated alias). + * Handles canonical `assertions:` entries. */ import { readFileSync } from 'node:fs'; @@ -70,16 +70,12 @@ interface RawTestCase { input_files?: string[]; expected_output?: string | RawMessage[] | unknown; assertions?: RawAssertEntry[]; - /** @deprecated Use `assertions` instead */ - assert?: RawAssertEntry[]; [key: string]: unknown; } interface RawSuite { tests?: RawTestCase[]; assertions?: RawAssertEntry[]; - /** @deprecated Use `assertions` instead */ - assert?: RawAssertEntry[]; [key: string]: unknown; } @@ -266,25 +262,6 @@ function extractTriggerAssertions(assertions: RawAssertEntry[]): RawAssertEntry[ return assertions.filter((a) => a.type === 'skill-trigger'); } -/** - * Collect all assertion entries for a test case, accepting both - * `assertions` and deprecated `assert` key. - */ -function resolveAssertions(rawCase: RawTestCase): RawAssertEntry[] { - if (Array.isArray(rawCase.assertions)) return rawCase.assertions; - if (Array.isArray(rawCase.assert)) return rawCase.assert; - return []; -} - -/** - * Collect suite-level assertions (applied to every test). - */ -function resolveSuiteAssertions(suite: RawSuite): RawAssertEntry[] { - if (Array.isArray(suite.assertions)) return suite.assertions; - if (Array.isArray(suite.assert)) return suite.assert; - return []; -} - // --------------------------------------------------------------------------- // Input extraction // --------------------------------------------------------------------------- @@ -376,7 +353,6 @@ export interface TranspileResult { * @param source Source identifier for error messages (e.g. file path) */ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): TranspileResult { - const warnings: string[] = []; const files = new Map(); if (typeof suite !== 'object' || suite === null) { @@ -389,11 +365,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`); } - if (rawSuite.assert !== undefined && rawSuite.assertions === undefined) { - warnings.push("'assert' is deprecated at the suite level. Use 'assertions' instead."); - } - - const suiteAssertions = resolveSuiteAssertions(rawSuite); + const suiteAssertions = rawSuite.assertions ?? []; // Suite-level NL assertions (appended to every test) const suiteNlAssertions: string[] = suiteAssertions @@ -415,12 +387,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi for (let idx = 0; idx < tests.length; idx++) { const rawCase = tests[idx]; - const caseAssertions = resolveAssertions(rawCase); - - if (rawCase.assert !== undefined && rawCase.assertions === undefined) { - const caseId = rawCase.id ?? idx + 1; - warnings.push(`Test '${caseId}': 'assert' is deprecated. Use 'assertions' instead.`); - } + const caseAssertions = rawCase.assertions ?? []; // Collect NL assertions (not skill-trigger) const nlAssertions: string[] = []; @@ -496,7 +463,7 @@ export function transpileEvalYaml(suite: unknown, source = 'EVAL.yaml'): Transpi // else: keep _no-skill if there are no other skills } - return { files, warnings }; + return { files, warnings: [] }; } // --------------------------------------------------------------------------- diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 7ca6e3854..7df185914 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -56,7 +56,6 @@ export async function parseGraders( readonly execution?: JsonValue; readonly assertions?: JsonValue; readonly evaluators?: JsonValue; - readonly assert?: JsonValue; }, globalExecution: JsonObject | undefined, searchRoots: readonly string[], @@ -66,18 +65,17 @@ export async function parseGraders( const execution = rawEvalCase.execution; const executionObject = isJsonObject(execution) ? execution : undefined; - // Case-level graders priority: assertions > assert > legacy execution/top-level assertion lists + // Case-level graders priority: assertions > legacy execution/top-level assertion lists const caseEvaluators = rawEvalCase.assertions ?? - rawEvalCase.assert ?? (executionObject ? executionObject.evaluators : undefined) ?? // deprecated: use assertions rawEvalCase.evaluators; // deprecated: use assertions - // Root-level default graders: assertions > assert > legacy execution assertion list + // Root-level default graders: assertions > legacy execution assertion list const skipDefaults = executionObject?.skip_defaults === true; const rootEvaluators = skipDefaults ? undefined - : (globalExecution?.assertions ?? globalExecution?.assert ?? globalExecution?.evaluators); // deprecated: use assertions + : (globalExecution?.assertions ?? globalExecution?.evaluators); // deprecated: use assertions // Parse case-level evaluators const parsedCase = await parseGraderList( @@ -251,7 +249,6 @@ export async function collectAssertionTemplateSourceReferences( readonly execution?: JsonValue; readonly assertions?: JsonValue; readonly evaluators?: JsonValue; - readonly assert?: JsonValue; }, globalExecution: JsonObject | undefined, searchRoots: readonly string[], @@ -261,13 +258,12 @@ export async function collectAssertionTemplateSourceReferences( const executionObject = isJsonObject(execution) ? execution : undefined; const caseEvaluators = rawEvalCase.assertions ?? - rawEvalCase.assert ?? (executionObject ? executionObject.evaluators : undefined) ?? rawEvalCase.evaluators; const skipDefaults = executionObject?.skip_defaults === true; const rootEvaluators = skipDefaults ? undefined - : (globalExecution?.assertions ?? globalExecution?.assert ?? globalExecution?.evaluators); + : (globalExecution?.assertions ?? globalExecution?.evaluators); return [ ...(await collectAssertionTemplateReferencesFromValue(caseEvaluators, searchRoots, evalId)), @@ -368,7 +364,7 @@ async function collectAssertionTemplateReferencesFromObject( includeContext: IncludeContext, ): Promise { const references: EvalSourceReference[] = []; - for (const key of ['assertions', 'assert', 'evaluators'] as const) { + for (const key of ['assertions', 'evaluators'] as const) { references.push( ...(await collectAssertionTemplateReferencesFromValue( value[key], @@ -411,7 +407,7 @@ async function parseGraderList( if (typeof item === 'string') { const trimmed = item.trim(); if (trimmed.length === 0) { - logWarning(`Skipping empty string criterion in assert array for '${evalId}'`); + logWarning(`Skipping empty string criterion in assertions array for '${evalId}'`); } else { strings.push(trimmed); } @@ -647,8 +643,8 @@ async function parseGraderList( } if (typeValue === 'composite') { - // Accept assertions > assert > evaluators (deprecated) - const rawMembers = rawEvaluator.assertions ?? rawEvaluator.assert ?? rawEvaluator.evaluators; // evaluators deprecated + // Accept assertions > evaluators (deprecated) + const rawMembers = rawEvaluator.assertions ?? rawEvaluator.evaluators; // evaluators deprecated if (!Array.isArray(rawMembers)) { logWarning( `Skipping composite evaluator '${name}' in '${evalId}': missing assertions (or evaluators) array`, diff --git a/packages/core/src/evaluation/loaders/jsonl-parser.ts b/packages/core/src/evaluation/loaders/jsonl-parser.ts index bf5b68608..4572a6191 100644 --- a/packages/core/src/evaluation/loaders/jsonl-parser.ts +++ b/packages/core/src/evaluation/loaders/jsonl-parser.ts @@ -207,12 +207,12 @@ export async function loadTestsFromJsonl( // Resolve expected_output with shorthand support const expectedMessages = resolveExpectedMessages(testCaseConfig) ?? []; - // A test is complete when it has id, input, and at least one of: criteria, expected_output, or assert + // A test is complete when it has id, input, and at least one of: criteria, expected_output, or assertions const hasEvaluationSpec = - !!outcome || expectedMessages.length > 0 || testCaseConfig.assert !== undefined; + !!outcome || expectedMessages.length > 0 || testCaseConfig.assertions !== undefined; if (!id || !hasEvaluationSpec || !rawInputMessages || rawInputMessages.length === 0) { logError( - `Skipping incomplete test at line ${lineNumber}: ${id ?? 'unknown'}. Missing required fields: id, input, and at least one of criteria/expected_output/assert`, + `Skipping incomplete test at line ${lineNumber}: ${id ?? 'unknown'}. Missing required fields: id, input, and at least one of criteria/expected_output/assertions`, ); continue; } diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 44b04c6da..81cb76b8a 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -94,15 +94,16 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ 'workspace', ]); -/** - * Deprecated top-level fields with migration hints. - * These are still processed by yaml-parser but authors should migrate. - */ +/** Removed top-level fields with migration hints. */ +const REMOVED_TOP_LEVEL_FIELDS = new Map([ + ['assert', "'assert' has been removed. Use 'assertions' instead."], +]); + +/** Deprecated top-level fields with migration hints. */ const DEPRECATED_TOP_LEVEL_FIELDS = new Map([ ['eval_cases', "'eval_cases' is deprecated. Use 'tests' instead."], ['evalcases', "'evalcases' is deprecated. Use 'tests' instead."], ['evaluator', "'evaluator' is deprecated. Use 'assertions' instead."], - ['assert', "'assert' is deprecated. Use 'assertions' instead."], ]); /** Known fields at the test level. */ @@ -131,13 +132,14 @@ const KNOWN_TEST_FIELDS = new Set([ 'window_size', ]); -/** - * Deprecated test-level fields with migration hints. - * These are still processed by yaml-parser but authors should migrate. - */ +/** Removed test-level fields with migration hints. */ +const REMOVED_TEST_FIELDS = new Map([ + ['assert', "'assert' has been removed. Use 'assertions' instead."], +]); + +/** Deprecated test-level fields with migration hints. */ const DEPRECATED_TEST_FIELDS = new Map([ ['evaluator', "'evaluator' is deprecated. Use 'assertions' instead."], - ['assert', "'assert' is deprecated. Use 'assertions' instead."], ['expected_outcome', "'expected_outcome' is deprecated. Use 'criteria' instead."], ]); @@ -274,6 +276,16 @@ export async function validateEvalFile(filePath: string): Promise 0 || renderedCase.assertions !== undefined || - renderedCase.assert !== undefined || (Array.isArray(renderedCase.turns) && renderedCase.turns.length > 0); if (!id || !hasEvaluationSpec || !testInputMessages || testInputMessages.length === 0) { logError( diff --git a/packages/core/test/evaluation/criteria-optional.test.ts b/packages/core/test/evaluation/criteria-optional.test.ts index 09ef60abd..a150d53c7 100644 --- a/packages/core/test/evaluation/criteria-optional.test.ts +++ b/packages/core/test/evaluation/criteria-optional.test.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { loadTests } from '../../src/evaluation/yaml-parser.js'; -describe('criteria is optional when expected_output or assert is present', () => { +describe('criteria is optional when expected_output or assertions is present', () => { let tempDir: string; beforeAll(async () => { diff --git a/packages/core/test/evaluation/evaluate-enhanced.test.ts b/packages/core/test/evaluation/evaluate-enhanced.test.ts index 5f926e011..068574f50 100644 --- a/packages/core/test/evaluation/evaluate-enhanced.test.ts +++ b/packages/core/test/evaluation/evaluate-enhanced.test.ts @@ -9,7 +9,7 @@ describe('evaluate() — enhanced features', () => { id: 'camel-case', input: 'hello', expectedOutput: 'world', - assert: [{ type: 'equals', value: 'world' }], + assertions: [{ type: 'equals', value: 'world' }], }, ], target: { name: 'default', provider: 'mock', response: 'world' }, @@ -17,13 +17,13 @@ describe('evaluate() — enhanced features', () => { expect(summary.passed).toBe(1); }); - it('supports config object assertions in assert array', async () => { + it('supports config object assertions', async () => { const { summary } = await evaluate({ tests: [ { id: 'config-test', input: 'hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], target: { name: 'default', provider: 'mock', response: 'hello world' }, @@ -31,13 +31,32 @@ describe('evaluate() — enhanced features', () => { expect(summary.passed).toBe(1); }); + it('rejects the removed assertion alias in inline tests', async () => { + const removedKey = ['ass', 'ert'].join(''); + const removedAliasTest: { + readonly id: string; + readonly input: string; + readonly [key: string]: unknown; + } = { + id: 'removed-key', + input: 'hello', + [removedKey]: [{ type: 'contains', value: 'hello' }], + }; + await expect( + evaluate({ + tests: [removedAliasTest], + target: { name: 'default', provider: 'mock', response: 'hello world' }, + }), + ).rejects.toThrow("'assert' has been removed"); + }); + it('supports inline assertion functions', async () => { const { summary } = await evaluate({ tests: [ { id: 'inline-fn', input: 'test', - assert: [ + assertions: [ ({ output }) => ({ name: 'custom', score: output.includes('test') ? 1.0 : 0.0, @@ -56,7 +75,7 @@ describe('evaluate() — enhanced features', () => { { id: 'task-fn', input: 'hello', - assert: [{ type: 'contains', value: 'Echo: hello' }], + assertions: [{ type: 'contains', value: 'Echo: hello' }], }, ], task: async (input) => `Echo: ${input}`, @@ -67,7 +86,7 @@ describe('evaluate() — enhanced features', () => { it('throws when both task and target are provided', async () => { await expect( evaluate({ - tests: [{ id: 'bad', input: 'x', assert: [{ type: 'contains', value: 'x' }] }], + tests: [{ id: 'bad', input: 'x', assertions: [{ type: 'contains', value: 'x' }] }], target: { name: 'default', provider: 'mock' }, task: async (input) => input, }), @@ -80,7 +99,7 @@ describe('evaluate() — enhanced features', () => { { id: 'mixed', input: 'hello world', - assert: [ + assertions: [ { type: 'contains', value: 'hello' }, { type: 'contains', value: 'world' }, ({ output }) => ({ @@ -95,13 +114,13 @@ describe('evaluate() — enhanced features', () => { expect(summary.passed).toBe(1); }); - it('supports suite-level assert with inline function', async () => { + it('supports suite-level assertions with inline function', async () => { const { summary } = await evaluate({ tests: [ { id: 'a', input: 'hello' }, { id: 'b', input: 'world' }, ], - assert: [{ type: 'contains', value: 'response' }], + assertions: [{ type: 'contains', value: 'response' }], target: { name: 'default', provider: 'mock', response: 'response text' }, }); expect(summary.total).toBe(2); @@ -115,7 +134,7 @@ describe('evaluate() — enhanced features', () => { id: 'legacy', input: 'hello', expected_output: 'world', - assert: [{ type: 'equals', value: 'world' }], + assertions: [{ type: 'equals', value: 'world' }], }, ], target: { name: 'default', provider: 'mock', response: 'world' }, diff --git a/packages/core/test/evaluation/evaluate-programmatic-api.test.ts b/packages/core/test/evaluation/evaluate-programmatic-api.test.ts index fd2c4c6b5..be272ce1b 100644 --- a/packages/core/test/evaluation/evaluate-programmatic-api.test.ts +++ b/packages/core/test/evaluation/evaluate-programmatic-api.test.ts @@ -28,7 +28,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'budget-test', input: 'hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], target: { name: 'default', provider: 'mock', response: 'hello world' }, @@ -47,12 +47,12 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'quality-pass', input: 'ok', - assert: [{ type: 'contains', value: 'task ok' }], + assertions: [{ type: 'contains', value: 'task ok' }], }, { id: 'provider-error', input: 'explode', - assert: [{ type: 'contains', value: 'task ok' }], + assertions: [{ type: 'contains', value: 'task ok' }], }, ], task: async (input) => { @@ -91,7 +91,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'programmatic-cache-path', input: 'hello', - assert: [{ type: 'contains', value: 'cached' }], + assertions: [{ type: 'contains', value: 'cached' }], }, ], target: { name: 'default', provider: 'mock', response: 'cached response' }, @@ -122,7 +122,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'programmatic-artifacts', input: 'hello', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], target: { name: 'default', provider: 'mock', response: 'mock response' }, @@ -184,11 +184,11 @@ describe('evaluate() — programmatic API extensions', () => { turns: [ { input: 'Hello', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, { input: 'How are you?', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], }, @@ -211,7 +211,7 @@ describe('evaluate() — programmatic API extensions', () => { turns: [ { input: 'First turn', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], }, @@ -234,7 +234,7 @@ describe('evaluate() — programmatic API extensions', () => { { input: 'Say hello', expectedOutput: 'Hello!', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], }, @@ -259,7 +259,7 @@ describe('evaluate() — programmatic API extensions', () => { { role: 'system', content: 'You are helpful' }, { role: 'user', content: 'Hello' }, ], - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], }, @@ -285,11 +285,11 @@ describe('evaluate() — programmatic API extensions', () => { turns: [ { input: 'Turn 1', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, { input: 'Turn 2', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], aggregation: 'min', @@ -316,7 +316,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'before-all-string', input: 'hello', - assert: [{ type: 'contains', value: 'test' }], + assertions: [{ type: 'contains', value: 'test' }], }, ], target: { name: 'default', provider: 'mock', response: 'test output' }, @@ -335,7 +335,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'before-all-array', input: 'hello', - assert: [{ type: 'contains', value: 'test' }], + assertions: [{ type: 'contains', value: 'test' }], }, ], target: { name: 'default', provider: 'mock', response: 'test output' }, @@ -361,11 +361,11 @@ describe('evaluate() — programmatic API extensions', () => { { input: 'Hello', expectedOutput: 'Hi there', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, { input: 'Goodbye', - assert: [{ type: 'contains', value: 'mock' }], + assertions: [{ type: 'contains', value: 'mock' }], }, ], aggregation: 'mean', @@ -392,7 +392,7 @@ describe('evaluate() — programmatic API extensions', () => { { id: 'standard-input', input: 'hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], target: { name: 'default', provider: 'mock', response: 'hello world' }, @@ -427,7 +427,7 @@ describe('evaluate() — programmatic API extensions', () => { expect(() => evaluate({ // biome-ignore lint/suspicious/noExplicitAny: intentionally testing invalid input - tests: [{ id: 'no-input', assert: [{ type: 'contains', value: 'x' }] } as any], + tests: [{ id: 'no-input', assertions: [{ type: 'contains', value: 'x' }] } as any], target: { name: 'default', provider: 'mock', response: 'hello' }, }), ).toThrow("Test 'no-input': input is required for non-conversation tests"); diff --git a/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts b/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts index d12fbf417..9ce1b4d40 100644 --- a/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts +++ b/packages/core/test/evaluation/loaders/eval-yaml-transpiler.test.ts @@ -586,46 +586,48 @@ describe('transpileEvalYaml — suite-level assertions', () => { expect(evals[1].assertions).toContain("Output contains 'global-check'"); }); - it('accepts deprecated assert: key at suite level', () => { + it('ignores the removed suite-level removed assertion key', () => { + const removedKey = ['ass', 'ert'].join(''); const suite = { tests: [ { id: 't1', input: 'hello', - assert: [{ type: 'skill-trigger', skill: 's', should_trigger: true }], + assertions: [{ type: 'skill-trigger', skill: 's', should_trigger: true }], }, ], - assert: [{ type: 'contains', value: 'suite-level' }], - }; + [removedKey]: [{ type: 'contains', value: 'suite-level' }], + } as Record; const { files, warnings } = transpileEvalYaml(suite); const evals = files.get('s')?.evals; - expect(evals[0].assertions).toContain("Output contains 'suite-level'"); - expect(warnings.some((w) => w.includes("'assert' is deprecated"))).toBe(true); + expect(evals?.[0].assertions).not.toContain("Output contains 'suite-level'"); + expect(warnings).toEqual([]); }); }); // --------------------------------------------------------------------------- -// Deprecated assert: key at test level +// Removed legacy assertion key at test level // --------------------------------------------------------------------------- -describe('transpileEvalYaml — deprecated assert: key', () => { - it('accepts assert: key at test level with deprecation warning', () => { +describe('transpileEvalYaml — removed legacy assertion key', () => { + it('ignores the removed test-level removed assertion key', () => { + const removedKey = ['ass', 'ert'].join(''); const suite = { tests: [ { id: 't1', input: 'Hello', - assert: [ + [removedKey]: [ { type: 'skill-trigger', skill: 'skill-a', should_trigger: true }, { type: 'contains', value: 'world' }, ], }, ], - }; + } as Record; const { files, warnings } = transpileEvalYaml(suite); - expect(files.has('skill-a')).toBe(true); - expect(files.get('skill-a')?.evals[0].assertions).toContain("Output contains 'world'"); - expect(warnings.some((w) => w.includes("'assert' is deprecated"))).toBe(true); + expect(files.has('skill-a')).toBe(false); + expect(files.get('_no-skill')?.evals[0].assertions).not.toContain("Output contains 'world'"); + expect(warnings).toEqual([]); }); }); diff --git a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts index 10073ddb0..ca732b547 100644 --- a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts @@ -9,7 +9,7 @@ const config: EvalConfig = { { id: 'greeting', input: 'Say hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], workers: 2, diff --git a/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts b/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts index 2c74e72e0..16e8d1008 100644 --- a/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts @@ -5,7 +5,7 @@ export const evalConfig: EvalConfig = { { id: 'eval-config-named', input: 'Say hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], target: { provider: 'mock_agent' }, diff --git a/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts b/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts index 8dfb9f81c..010d4ec67 100644 --- a/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/named-config.eval.ts @@ -5,7 +5,7 @@ export const config: EvalConfig = { { id: 'named-config', input: 'Say hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], target: { provider: 'mock_agent' }, diff --git a/packages/core/test/evaluation/loaders/grader-parser.test.ts b/packages/core/test/evaluation/loaders/grader-parser.test.ts index d0f0ef641..4e2329cfd 100644 --- a/packages/core/test/evaluation/loaders/grader-parser.test.ts +++ b/packages/core/test/evaluation/loaders/grader-parser.test.ts @@ -1251,7 +1251,7 @@ describe('parseGraders - default evaluators merge', () => { }); }); -describe('parseGraders - assert field', () => { +describe('parseGraders - assertions field', () => { let tempDir: string; beforeAll(async () => { @@ -1276,17 +1276,17 @@ describe('parseGraders - assert field', () => { expect(evaluators?.[0].type).toBe('contains'); }); - it('parses legacy assert field as evaluators (backward compat)', async () => { + it('ignores the removed assertion field as evaluator input', async () => { + const removedKey = ['ass', 'ert'].join(''); const evaluators = await parseGraders( { - assert: [{ type: 'contains', value: 'DENIED' }], + [removedKey]: [{ type: 'contains', value: 'DENIED' }], }, undefined, [tempDir], 'test-1', ); - expect(evaluators).toHaveLength(1); - expect(evaluators?.[0].type).toBe('contains'); + expect(evaluators).toBeUndefined(); }); it('assertions takes precedence over execution.evaluators', async () => { @@ -1347,7 +1347,7 @@ describe('parseGraders - assert field', () => { expect(evaluators?.[0].type).toBe('contains'); }); - it('falls back to execution.evaluators when assert is not present', async () => { + it('falls back to execution.evaluators when assertions is not present', async () => { const evaluators = await parseGraders( { execution: { diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 64bdc0002..8ed09e287 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -2433,7 +2433,7 @@ describe('deterministic assertion evaluators in orchestrator', () => { }); }); -describe('criteria with assert runs only declared evaluators (#452)', () => { +describe('criteria with assertions runs only declared evaluators (#452)', () => { const criteriaTestCase: EvalTest = { id: 'no-implicit-grader-1', suite: 'test-dataset', @@ -2445,7 +2445,7 @@ describe('criteria with assert runs only declared evaluators (#452)', () => { criteria: 'Response should be polite', }; - it('does NOT inject implicit llm-grader when criteria is present with assert', async () => { + it('does NOT inject implicit llm-grader when criteria is present with assertions', async () => { const provider = new SequenceProvider('mock', { responses: [{ output: [{ role: 'assistant', content: 'hello world' }] }], }); @@ -2502,7 +2502,7 @@ describe('criteria with assert runs only declared evaluators (#452)', () => { expect(result.score).toBeCloseTo(1.0); }); - it('does NOT inject implicit llm-grader when expected_output is present with assert', async () => { + it('does NOT inject implicit llm-grader when expected_output is present with assertions', async () => { const provider = new SequenceProvider('mock', { responses: [{ output: [{ role: 'assistant', content: 'hello world' }] }], }); @@ -2537,7 +2537,7 @@ describe('criteria with assert runs only declared evaluators (#452)', () => { graderTarget: 'grader-target', }; - // When user explicitly adds llm-grader to assert, it runs and reads criteria + // When user explicitly adds llm-grader to assertions, it runs and reads criteria const result = await runEvalCase({ evalCase: { ...criteriaTestCase, diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 21516ff59..b0c80a354 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -867,8 +867,8 @@ tests: expect(warnings).toHaveLength(0); }); - describe('assert field validation', () => { - it('validates assert array items have type field', async () => { + describe('assertions field validation', () => { + it('validates assertions array items have type field', async () => { const filePath = path.join(tempDir, 'assert-missing-type.yaml'); await writeFile( filePath, @@ -886,7 +886,7 @@ tests: expect(warnings.some((e) => e.message.includes("'type'"))).toBe(true); }); - it('warns on invalid assert type', async () => { + it('warns on invalid assertion type', async () => { const filePath = path.join(tempDir, 'assert-invalid-type.yaml'); await writeFile( filePath, @@ -1099,7 +1099,7 @@ tests: expect(warnings.some((e) => e.message.includes('required'))).toBe(true); }); - it('warns when assert is not an array', async () => { + it('warns when assertions is not an array', async () => { const filePath = path.join(tempDir, 'assert-not-array.yaml'); await writeFile( filePath, @@ -1154,7 +1154,7 @@ tests: expect(warnings.some((e) => e.message.includes('string or an object'))).toBe(true); }); - it('passes valid assert array', async () => { + it('passes valid assertions array', async () => { const filePath = path.join(tempDir, 'assert-valid.yaml'); await writeFile( filePath, @@ -1758,14 +1758,15 @@ tests: ).toBe(true); }); - it('warns on assert as deprecated field at test level', async () => { - const filePath = path.join(tempDir, 'assert-deprecated.yaml'); + it('errors on removed assertion field at test level', async () => { + const removedKey = ['ass', 'ert'].join(''); + const filePath = path.join(tempDir, 'removed-test-field.yaml'); await writeFile( filePath, `tests: - id: test-1 input: "Hello" - assert: + ${removedKey}: - type: contains value: "hello" `, @@ -1773,20 +1774,22 @@ tests: const result = await validateEvalFile(filePath); - expect(result.valid).toBe(true); - const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(result.valid).toBe(false); + const errors = result.errors.filter((e) => e.severity === 'error'); expect( - warnings.some( - (e) => e.message.includes("'assert' is deprecated") && e.message.includes("'assertions'"), + errors.some( + (e) => + e.message.includes("'assert' has been removed") && e.message.includes("'assertions'"), ), ).toBe(true); }); - it('warns on assert as deprecated field at top level', async () => { - const filePath = path.join(tempDir, 'assert-top-deprecated.yaml'); + it('errors on removed assertion field at top level', async () => { + const removedKey = ['ass', 'ert'].join(''); + const filePath = path.join(tempDir, 'removed-top-field.yaml'); await writeFile( filePath, - `assert: + `${removedKey}: - type: contains value: "hello" tests: @@ -1797,10 +1800,12 @@ tests: const result = await validateEvalFile(filePath); - const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(result.valid).toBe(false); + const errors = result.errors.filter((e) => e.severity === 'error'); expect( - warnings.some( - (e) => e.message.includes("'assert' is deprecated") && e.message.includes("'assertions'"), + errors.some( + (e) => + e.message.includes("'assert' has been removed") && e.message.includes("'assertions'"), ), ).toBe(true); }); From d5da38f98d0abe4e32c31dcc72edc3bf0cfbd863 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 05:14:21 +0200 Subject: [PATCH 2/2] test(sdk): update evaluate export fixture to assertions --- packages/sdk/test/evaluate-export.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/test/evaluate-export.test.ts b/packages/sdk/test/evaluate-export.test.ts index d4dad0f65..6f98e8227 100644 --- a/packages/sdk/test/evaluate-export.test.ts +++ b/packages/sdk/test/evaluate-export.test.ts @@ -9,7 +9,7 @@ describe('evaluate export', () => { { id: 'sdk-evaluate-export', input: 'Say hello', - assert: [{ type: 'contains', value: 'hello' }], + assertions: [{ type: 'contains', value: 'hello' }], }, ], task: async (input) => `hello: ${input}`,