diff --git a/apps/cli/src/commands/create/commands.ts b/apps/cli/src/commands/create/commands.ts index 743515edd..02cf21488 100644 --- a/apps/cli/src/commands/create/commands.ts +++ b/apps/cli/src/commands/create/commands.ts @@ -12,7 +12,8 @@ export default defineAssertion(({ output }) => { const pass = text.length > 0; return { pass, - assertions: [{ text: pass ? 'Output has content' : 'Output is empty', passed: pass }], + score: pass ? 1 : 0, + reason: pass ? 'Output has content' : 'Output is empty', }; }); `, @@ -26,7 +27,14 @@ export default defineAssertion(({ output }) => { return { pass: score >= 0.5, score, - assertions: [{ text: 'Output has content', passed: score === 1.0 }], + reason: score === 1.0 ? 'Output has content' : 'Output is empty', + checks: [ + { + text: 'Output has content', + pass: score === 1.0, + reason: score === 1.0 ? 'Output is non-empty' : 'Output is empty', + }, + ], }; }); `, diff --git a/apps/cli/test/commands/create/assertion.test.ts b/apps/cli/test/commands/create/assertion.test.ts index 9a001c096..e7f5e8d38 100644 --- a/apps/cli/test/commands/create/assertion.test.ts +++ b/apps/cli/test/commands/create/assertion.test.ts @@ -35,9 +35,11 @@ describe('agentv create assertion', () => { ); expect(content).toContain("import { defineAssertion } from '@agentv/sdk';"); expect(content).toContain("const text = output ?? '';"); - expect(content).toContain( - "assertions: [{ text: pass ? 'Output has content' : 'Output is empty', passed: pass }]", - ); + expect(content).toContain('pass,'); + expect(content).toContain('score: pass ? 1 : 0,'); + expect(content).toContain("reason: pass ? 'Output has content' : 'Output is empty'"); + expect(content).not.toContain('assertions:'); + expect(content).not.toContain('passed:'); expect(content).not.toContain('reasoning:'); expect(content).not.toContain('getMessageText'); } finally { diff --git a/apps/cli/test/commands/eval/vitest.test.ts b/apps/cli/test/commands/eval/vitest.test.ts index c5a7fa939..24a32b27d 100644 --- a/apps/cli/test/commands/eval/vitest.test.ts +++ b/apps/cli/test/commands/eval/vitest.test.ts @@ -107,15 +107,23 @@ process.exit(1); ); const output = JSON.parse(result.stdout); + expect(output.pass).toBe(false); expect(output.score).toBe(0.5); - expect(output.assertions).toEqual([ - { text: 'welcome banner includes ready status', passed: true }, + expect(output.reason).toBe('1/2 Vitest tests passed.'); + expect(output.checks).toEqual([ + { + text: 'welcome banner includes ready status', + pass: true, + reason: 'Vitest test passed.', + }, { text: 'welcome banner links to dashboard', - passed: false, + pass: false, + reason: 'Vitest test failed.', evidence: 'AssertionError: expected link to point at /dashboard', }, ]); + expect(output).not.toHaveProperty('assertions'); expect(output.details).toMatchObject({ vitest_success: false, num_total_tests: 2, diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index c591d5386..e7a721deb 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -223,7 +223,8 @@ export default defineAssertion(({ output }) => { const pass = wordCount >= 3; return { pass, - assertions: [{ text: `Output has ${wordCount} words`, passed: pass }], + score: pass ? 1 : 0, + reason: `Output has ${wordCount} words`, }; }); ``` @@ -241,7 +242,7 @@ export default defineAssertion(({ output, traceSummary }) => { const isEfficient = (traceSummary?.eventCount ?? 0) <= 10 ? 0.5 : 0; return { score: hasContent + isEfficient, - reasoning: 'Checks content exists and is efficient', + reason: 'Checks content exists and is efficient', }; }); ``` @@ -268,16 +269,26 @@ assert: ## Script Graders -Use `defineScriptGrader` from `@agentv/sdk` for full control over scoring with an explicit assertions array: +Use `defineScriptGrader` from `@agentv/sdk` for full control over scoring with optional per-check results: ```typescript import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, traceSummary }) => ({ + pass: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5, score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, - assertions: [ - { text: 'Answer is not empty', passed: (output ?? '').length > 0 }, - { text: 'Efficient tool usage', passed: (traceSummary?.eventCount ?? 0) <= 5 }, + reason: 'Checks answer text and tool usage', + checks: [ + { + text: 'Answer is not empty', + pass: (output ?? '').length > 0, + reason: (output ?? '').length > 0 ? 'Output is non-empty' : 'Output is empty', + }, + { + text: 'Efficient tool usage', + pass: (traceSummary?.eventCount ?? 0) <= 5, + reason: 'Trace event count is within limit', + }, ], })); ``` diff --git a/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx b/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx index 3910714b9..a2f77f270 100644 --- a/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx +++ b/apps/web/src/content/docs/docs/next/graders/python-helpers.mdx @@ -36,7 +36,7 @@ Use canonical fields instead: ## Example ```python -from agentv_py.grader import Assertion, ScriptGraderResult, define_script +from agentv_py.grader import Check, ScriptGraderResult, define_script def evaluate(context): @@ -44,11 +44,16 @@ def evaluate(context): expected = context.expected_output[0]["content"] passed = actual.strip() == expected.strip() return ScriptGraderResult( + pass_=passed, score=1.0 if passed else 0.0, - assertions=[ - Assertion( + reason="Candidate output matches expected output" + if passed + else "Candidate output does not match expected output", + checks=[ + Check( text="Candidate output matches expected output", - passed=passed, + pass_=passed, + reason="Exact string comparison passed" if passed else "Exact string comparison failed", ) ], ) diff --git a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx index 3d3acaaed..0524737f0 100644 --- a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -53,21 +53,25 @@ Emit a JSON object for numeric scores or multi-aspect results: ```json { + "pass": true, "score": 1.0, - "assertions": [ - { "text": "Answer contains correct value (42)", "passed": true } + "reason": "Answer contains the correct value.", + "checks": [ + { "text": "Answer contains correct value (42)", "pass": true, "reason": "42 appears in the output." } ] } ``` | Output Field | Type | Description | |-------------|------|-------------| +| `pass` | `boolean` | Aggregate pass/fail decision | | `score` | `number` | 0.0 to 1.0 | -| `assertions` | `Array<{ text, passed, evidence? }>` | Per-aspect results with verdict and optional evidence | +| `reason` | `string` | Explanation for the aggregate decision | +| `checks` | `Array<{ text, pass, score?, reason, evidence? }>` | Optional per-aspect results with verdict, optional score, reason, and evidence | ### Plain-text output (exit-code convention) -For simple pass/fail checks, skip the JSON protocol entirely. The exit code determines the score and stdout becomes the assertion text: +For simple pass/fail checks, skip the JSON protocol entirely. The exit code determines the score and stdout becomes the check text: | Exit code | Score | Verdict | |-----------|-------|---------| @@ -112,37 +116,42 @@ import json, sys data = json.load(sys.stdin) output = data.get("output") or "" -assertions = [] +checks = [] if "42" in output: - assertions.append({"text": "Output contains correct value (42)", "passed": True}) + checks.append({"text": "Output contains correct value (42)", "pass": True, "reason": "42 appears in the output"}) else: - assertions.append({"text": "Output does not contain expected value (42)", "passed": False}) + checks.append({"text": "Output contains correct value (42)", "pass": False, "reason": "42 is missing from the output"}) -passed = sum(1 for a in assertions if a["passed"]) -score = passed / len(assertions) if assertions else 0.0 +passed = sum(1 for check in checks if check["pass"]) +score = passed / len(checks) if checks else 0.0 print(json.dumps({ + "pass": passed == len(checks), "score": score, - "assertions": assertions, + "reason": f"{passed}/{len(checks)} checks passed", + "checks": checks, })) ``` The repo-local helper in `examples/features/sdk-python/` wraps the same contract for that example checkout: ```python -from agentv_py.grader import Assertion, ScriptGraderResult, define_script +from agentv_py.grader import Check, ScriptGraderResult, define_script def evaluate(context): candidate = context.output or "" passed = "42" in candidate return ScriptGraderResult( + pass_=passed, score=1.0 if passed else 0.0, - assertions=[ - Assertion( + reason="Answer contains the correct value" if passed else "Answer is missing the correct value", + checks=[ + Check( text="Output contains correct value (42)", - passed=passed, + pass_=passed, + reason="42 appears in the output" if passed else "42 is missing from the output", ) ], ) @@ -162,19 +171,21 @@ import { readFileSync } from "fs"; const data = JSON.parse(readFileSync("/dev/stdin", "utf-8")); const output: string = data.output ?? ""; -const assertions: Array<{ text: string; passed: boolean }> = []; +const checks: Array<{ text: string; pass: boolean; reason: string }> = []; if (output.includes("42")) { - assertions.push({ text: "Output contains correct value (42)", passed: true }); + checks.push({ text: "Output contains correct value (42)", pass: true, reason: "42 appears in the output" }); } else { - assertions.push({ text: "Output does not contain expected value (42)", passed: false }); + checks.push({ text: "Output contains correct value (42)", pass: false, reason: "42 is missing from the output" }); } -const passed = assertions.filter(a => a.passed).length; +const passed = checks.filter(check => check.pass).length; console.log(JSON.stringify({ - score: passed > 0 ? 1.0 : 0.0, - assertions, + pass: passed === checks.length, + score: checks.length === 0 ? 0.0 : passed / checks.length, + reason: `${passed}/${checks.length} checks passed`, + checks, })); ``` @@ -197,18 +208,20 @@ import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, criteria }) => { const outputText = output ?? ''; - const assertions: Array<{ text: string; passed: boolean }> = []; + const checks: Array<{ text: string; pass: boolean; reason: string }> = []; if (outputText.includes(criteria)) { - assertions.push({ text: 'Output matches expected outcome', passed: true }); + checks.push({ text: 'Output matches expected outcome', pass: true, reason: 'Criteria text appears in the output.' }); } else { - assertions.push({ text: 'Output does not match expected outcome', passed: false }); + checks.push({ text: 'Output matches expected outcome', pass: false, reason: 'Criteria text is missing from the output.' }); } - const passed = assertions.filter(a => a.passed).length; + const passed = checks.filter(check => check.pass).length; return { - score: assertions.length === 0 ? 0 : passed / assertions.length, - assertions, + pass: checks.length > 0 && passed === checks.length, + score: checks.length === 0 ? 0 : passed / checks.length, + reason: `${passed}/${checks.length} checks passed`, + checks, }; }); ``` @@ -240,7 +253,7 @@ describe('welcome banner', () => { }); ``` -Then use AgentV's built-in Vitest adapter as the `script` command. The adapter copies verifier files into a temporary workspace-local path when needed, runs Vitest in `workspace_path`, reads the JSON reporter output, and maps each test outcome to an AgentV assertion: +Then use AgentV's built-in Vitest adapter as the `script` command. The adapter copies verifier files into a temporary workspace-local path when needed, runs Vitest in `workspace_path`, reads the JSON reporter output, and maps each test outcome to an AgentV check: ```yaml assert: @@ -253,7 +266,7 @@ AgentV infers the Vitest adapter for verifier-looking files such as `*.test.ts`, ### Lower-Level Workspace Helpers -For tiny one-off file checks, `defineWorkspaceGrader` can resolve the workspace path, read files relative to the workspace, build assertions, and aggregate the score: +For tiny one-off file checks, `defineWorkspaceGrader` can resolve the workspace path, read files relative to the workspace, build checks, and aggregate the score: ```typescript #!/usr/bin/env bun @@ -269,7 +282,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ Prefer Vitest verifiers when the checks naturally fit `expect(...)`. Use `defineWorkspaceGrader` when you need a very small custom script, custom weighting, or details that do not map cleanly to individual test outcomes. -**SDK exports:** `defineScriptGrader`, `defineVitestWorkspaceGrader`, `defineWorkspaceGrader`, `Message`, `ToolCall`, `Trace`, `TraceSummary`, `ScriptGraderInput`, `ScriptGraderResult`, `Workspace`, `WorkspaceAssertion` +**SDK exports:** `defineScriptGrader`, `defineVitestWorkspaceGrader`, `defineWorkspaceGrader`, `Message`, `ToolCall`, `Trace`, `TraceSummary`, `ScriptGraderInput`, `ScriptGraderResult`, `ScriptGraderCheck`, `Workspace`, `WorkspaceCheck` ## Target Access @@ -303,7 +316,7 @@ export default defineScriptGrader(async ({ input, output }) => { .join('\n'); const outputText = output ?? ''; const target = createTargetClient(); - if (!target) return { score: 0, assertions: [{ text: 'Target not configured', passed: false }] }; + if (!target) return { pass: false, score: 0, reason: 'Target not configured' }; const response = await target.invoke({ question: `Is this relevant to: ${inputText}? Response: ${outputText}`, @@ -311,7 +324,11 @@ export default defineScriptGrader(async ({ input, output }) => { }); const result = JSON.parse(response.rawText ?? '{}'); - return { score: result.relevant ? 1.0 : 0.0 }; + return { + pass: result.relevant === true, + score: result.relevant === true ? 1.0 : 0.0, + reason: result.relevant === true ? 'Response is relevant' : 'Response is not relevant', + }; }); ``` @@ -392,30 +409,32 @@ import { execFileSync } from "child_process"; const input = JSON.parse(readFileSync("/dev/stdin", "utf-8")); const cwd = input.workspace_path; -const assertions: Array<{ text: string; passed: boolean }> = []; +const checks: Array<{ text: string; pass: boolean; reason: string }> = []; // Stage 1: Install dependencies try { execFileSync("npm", ["install"], { cwd, stdio: "pipe" }); - assertions.push({ text: "npm install passed", passed: true }); -} catch { assertions.push({ text: "npm install failed", passed: false }); } + checks.push({ text: "npm install", pass: true, reason: "npm install passed" }); +} catch { checks.push({ text: "npm install", pass: false, reason: "npm install failed" }); } // Stage 2: Typecheck try { execFileSync("npx", ["tsc", "--noEmit"], { cwd, stdio: "pipe" }); - assertions.push({ text: "typecheck passed", passed: true }); -} catch { assertions.push({ text: "typecheck failed", passed: false }); } + checks.push({ text: "typecheck", pass: true, reason: "typecheck passed" }); +} catch { checks.push({ text: "typecheck", pass: false, reason: "typecheck failed" }); } // Stage 3: Run tests try { execFileSync("npm", ["test"], { cwd, stdio: "pipe" }); - assertions.push({ text: "tests passed", passed: true }); -} catch { assertions.push({ text: "tests failed", passed: false }); } + checks.push({ text: "tests", pass: true, reason: "tests passed" }); +} catch { checks.push({ text: "tests", pass: false, reason: "tests failed" }); } -const passed = assertions.filter(a => a.passed).length; +const passed = checks.filter(check => check.pass).length; console.log(JSON.stringify({ - score: assertions.length > 0 ? passed / assertions.length : 0, - assertions, + pass: checks.length > 0 && passed === checks.length, + score: checks.length > 0 ? passed / checks.length : 0, + reason: `${passed}/${checks.length} checks passed`, + checks, })); ``` diff --git a/examples/README.md b/examples/README.md index 640c4faa8..cfd0a6883 100644 --- a/examples/README.md +++ b/examples/README.md @@ -118,7 +118,8 @@ Then write type-safe script graders: import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output }) => ({ + pass: (output ?? '').includes('expected'), score: (output ?? '').includes('expected') ? 1.0 : 0.0, - assert: [{ text: 'Found expected content', passed: (output ?? '').includes('expected') }], + reason: (output ?? '').includes('expected') ? 'Found expected content' : 'Missing expected content', })); ``` diff --git a/examples/features/script-grader-sdk/README.md b/examples/features/script-grader-sdk/README.md index 5a52c58fb..5dab6f026 100644 --- a/examples/features/script-grader-sdk/README.md +++ b/examples/features/script-grader-sdk/README.md @@ -61,7 +61,8 @@ The `defineScriptGrader` helper: import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, criteria }) => ({ + pass: (output ?? '').includes(criteria), score: (output ?? '').includes(criteria) ? 1.0 : 0.0, - assertions: [{ text: 'Check passed', passed: (output ?? '').includes(criteria) }], + reason: (output ?? '').includes(criteria) ? 'Check passed' : 'Check failed', })); ``` diff --git a/examples/features/script-grader-sdk/scripts/verify-attachments.ts b/examples/features/script-grader-sdk/scripts/verify-attachments.ts index 608dead77..53dffa113 100755 --- a/examples/features/script-grader-sdk/scripts/verify-attachments.ts +++ b/examples/features/script-grader-sdk/scripts/verify-attachments.ts @@ -14,7 +14,7 @@ function fileName(path: string): string { export default defineScriptGrader(({ expectedOutput, output, inputFiles }) => { const outputText = output ?? ''; - const assertions: Array<{ text: string; passed: boolean }> = []; + const checks: Array<{ text: string; pass: boolean; reason: string }> = []; // Check if candidate matches expected message const expectedMessage = expectedOutput[0]; @@ -24,24 +24,42 @@ export default defineScriptGrader(({ expectedOutput, output, inputFiles }) => { : undefined; if (expectedContent && outputText.trim() === expectedContent.trim()) { - assertions.push({ text: 'Candidate output matches expected message', passed: true }); + checks.push({ + text: 'Candidate output matches expected message', + pass: true, + reason: 'Candidate output exactly matched the expected message.', + }); } else { - assertions.push({ text: 'Candidate output does not match expected message', passed: false }); + checks.push({ + text: 'Candidate output matches expected message', + pass: false, + reason: 'Candidate output did not exactly match the expected message.', + }); } // Check if attachments are mentioned const attachmentNames = inputFiles.map(fileName); for (const name of attachmentNames) { if (outputText.includes(name)) { - assertions.push({ text: `Mentions attachment: ${name}`, passed: true }); + checks.push({ + text: `Mentions attachment: ${name}`, + pass: true, + reason: `Candidate output mentions ${name}.`, + }); } else { - assertions.push({ text: `Missing attachment: ${name}`, passed: false }); + checks.push({ + text: `Mentions attachment: ${name}`, + pass: false, + reason: `Candidate output does not mention ${name}.`, + }); } } - const passed = assertions.filter((assertion) => assertion.passed).length; + const passed = checks.filter((check) => check.pass).length; return { - score: assertions.length > 0 ? passed / assertions.length : 0, - assertions, + pass: checks.length > 0 && passed === checks.length, + score: checks.length > 0 ? passed / checks.length : 0, + reason: `${passed}/${checks.length} attachment checks passed.`, + checks, }; }); diff --git a/examples/features/sdk-python/README.md b/examples/features/sdk-python/README.md index 56e4b9a31..2db20f80f 100644 --- a/examples/features/sdk-python/README.md +++ b/examples/features/sdk-python/README.md @@ -4,7 +4,7 @@ This example is the smallest repo-local Python helper surface for AgentV. It is intentionally scoped to two jobs: -- build Python `script-grader` scripts over the existing stdin/stdout contract +- build Python `script-grader` scripts over the stdin/stdout contract with `pass`, `score`, `reason`, and optional `checks` - author AgentV-shaped eval definitions and emit canonical YAML/JSONL It does **not** add a native Python runner. Evaluations still run through the AgentV CLI. diff --git a/examples/features/sdk-python/scripts/check_expected_output.py b/examples/features/sdk-python/scripts/check_expected_output.py index 9794c826f..ce757c19e 100644 --- a/examples/features/sdk-python/scripts/check_expected_output.py +++ b/examples/features/sdk-python/scripts/check_expected_output.py @@ -2,7 +2,7 @@ from __future__ import annotations -from agentv_py.grader import Assertion, ScriptGraderContext, ScriptGraderResult, define_script_grader +from agentv_py.grader import Check, ScriptGraderContext, ScriptGraderResult, define_script_grader def evaluate(context: ScriptGraderContext) -> ScriptGraderResult: @@ -10,11 +10,16 @@ def evaluate(context: ScriptGraderContext) -> ScriptGraderResult: actual = context.output or "" passed = actual.strip() == expected.strip() return ScriptGraderResult( + pass_=passed, score=1.0 if passed else 0.0, - assertions=[ - Assertion( + reason="Candidate output matches expected output" + if passed + else "Candidate output does not match expected output", + checks=[ + Check( text="Candidate output matches expected output", - passed=passed, + pass_=passed, + reason="Exact string comparison passed" if passed else "Exact string comparison failed", ) ], ) diff --git a/examples/features/sdk-python/src/agentv_py/__init__.py b/examples/features/sdk-python/src/agentv_py/__init__.py index 5aeaec85a..40da2fc8e 100644 --- a/examples/features/sdk-python/src/agentv_py/__init__.py +++ b/examples/features/sdk-python/src/agentv_py/__init__.py @@ -1,34 +1,39 @@ """Minimal Python helpers for AgentV script-graders and eval authoring.""" -from .evals import EvalDefinition, EvalTest, JsonlCase, render_eval_yaml, render_jsonl, run_agentv_eval, write_eval_yaml, write_jsonl +from .evals import ( + EvalDefinition, + EvalTest, + JsonlCase, + render_eval_yaml, + render_jsonl, + run_agentv_eval, + write_eval_yaml, + write_jsonl, +) from .grader import ( Assertion, CodeGraderContext, - ScriptGraderResult, + Check, ScriptGraderContext, ScriptGraderResult, TargetClient, define_script_grader, - define_script_grader, emit_grader_result, load_grader_input, run_script_grader, - run_script_grader, ) __all__ = [ "Assertion", + "Check", "ScriptGraderContext", "ScriptGraderResult", "CodeGraderContext", - "ScriptGraderResult", "TargetClient", "define_script_grader", - "define_script_grader", "emit_grader_result", "load_grader_input", "run_script_grader", - "run_script_grader", "EvalDefinition", "EvalTest", "JsonlCase", diff --git a/examples/features/sdk-python/src/agentv_py/grader.py b/examples/features/sdk-python/src/agentv_py/grader.py index 42bda63da..4d32aacd7 100644 --- a/examples/features/sdk-python/src/agentv_py/grader.py +++ b/examples/features/sdk-python/src/agentv_py/grader.py @@ -37,13 +37,20 @@ def _read_output_file(path: str) -> Any: @dataclass(frozen=True) -class Assertion: +class Check: text: str - passed: bool + pass_: bool + reason: str + score: float | None = None evidence: str | None = None + id: str | None = None def to_wire(self) -> dict[str, Any]: - wire = {"text": self.text, "passed": self.passed} + wire = {"text": self.text, "pass": self.pass_, "reason": self.reason} + if self.id is not None: + wire["id"] = self.id + if self.score is not None: + wire["score"] = min(max(float(self.score), 0.0), 1.0) if self.evidence is not None: wire["evidence"] = self.evidence return wire @@ -51,15 +58,19 @@ def to_wire(self) -> dict[str, Any]: @dataclass(frozen=True) class ScriptGraderResult: + pass_: bool score: float - assertions: list[Assertion] = field(default_factory=list) + reason: str + checks: list[Check] = field(default_factory=list) details: Mapping[str, Any] | None = None def to_wire(self) -> dict[str, Any]: score = min(max(float(self.score), 0.0), 1.0) wire: dict[str, Any] = { + "pass": self.pass_, "score": score, - "assertions": [assertion.to_wire() for assertion in self.assertions], + "reason": self.reason, + "checks": [check.to_wire() for check in self.checks], } if self.details is not None: wire["details"] = dict(self.details) @@ -251,8 +262,16 @@ def run_script_grader(handler: ScriptGraderHandler, stdin_text: str | None = Non except Exception as error: emit_grader_result( ScriptGraderResult( + pass_=False, score=0.0, - assertions=[Assertion(text=f"Evaluation failed: {error}", passed=False)], + reason=f"Evaluation failed: {error}", + checks=[ + Check( + text="Script grader execution", + pass_=False, + reason=str(error), + ) + ], ) ) return 1 @@ -263,7 +282,7 @@ def define_script_grader(handler: ScriptGraderHandler) -> None: CodeGraderContext = ScriptGraderContext -ScriptGraderResult = ScriptGraderResult +Assertion = Check CodeGraderHandler = ScriptGraderHandler run_script_grader = run_script_grader define_script_grader = define_script_grader diff --git a/examples/features/sdk-python/tests/test_grader.py b/examples/features/sdk-python/tests/test_grader.py index f8f074f20..8de5330c5 100644 --- a/examples/features/sdk-python/tests/test_grader.py +++ b/examples/features/sdk-python/tests/test_grader.py @@ -5,7 +5,7 @@ import pytest -from agentv_py.grader import Assertion, ScriptGraderContext, ScriptGraderResult, load_grader_input, run_script_grader +from agentv_py.grader import Check, ScriptGraderContext, ScriptGraderResult, load_grader_input, run_script_grader def canonical_payload() -> dict: @@ -62,8 +62,10 @@ def test_load_grader_input_reads_output_path(tmp_path: Path) -> None: def test_run_script_grader_emits_canonical_result(capsys: pytest.CaptureFixture[str]) -> None: def handler(_: ScriptGraderContext) -> ScriptGraderResult: return ScriptGraderResult( + pass_=True, score=1.0, - assertions=[Assertion(text="Exact match", passed=True)], + reason="Exact match", + checks=[Check(text="Exact match", pass_=True, reason="Output matched expected text")], details={"source": "pytest"}, ) @@ -72,7 +74,9 @@ def handler(_: ScriptGraderContext) -> ScriptGraderResult: assert exit_code == 0 emitted = json.loads(capsys.readouterr().out) assert emitted == { + "pass": True, "score": 1.0, - "assertions": [{"text": "Exact match", "passed": True}], + "reason": "Exact match", + "checks": [{"text": "Exact match", "pass": True, "reason": "Output matched expected text"}], "details": {"source": "pytest"}, } diff --git a/packages/core/src/evaluation/graders/script-grader.ts b/packages/core/src/evaluation/graders/script-grader.ts index ca2989df9..cb6e2d849 100644 --- a/packages/core/src/evaluation/graders/script-grader.ts +++ b/packages/core/src/evaluation/graders/script-grader.ts @@ -10,7 +10,12 @@ import { } from '../../runtime/target-proxy.js'; import { serializeSnakeCaseBoundaryPayload } from '../case-conversion.js'; import { type ContentImage, isContentArray } from '../content.js'; -import type { AssertionEntry, JsonObject, TargetAccessConfig } from '../types.js'; +import type { + AssertionEntry, + GraderCheckResult, + JsonObject, + TargetAccessConfig, +} from '../types.js'; import { getRepoCheckoutTargets } from '../workspace/repo-checkout.js'; import { clampScore, isNonEmptyString, parseJsonSafe, scoreToVerdict } from './scoring.js'; import type { EvaluationContext, EvaluationScore, Grader } from './types.js'; @@ -21,6 +26,20 @@ const FILE_BACKED_OUTPUT_THRESHOLD = 50_000; /** Regex matching `data:;base64,` URIs. */ const DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s; +interface ScriptProtocolResult { + readonly pass: boolean; + readonly score: number; + readonly reason?: string; + readonly checks: readonly GraderCheckResult[]; + readonly details?: JsonObject; +} + +interface ScriptProtocolCheckRecord extends Record { + readonly text: string; + readonly pass: boolean; + readonly reason: string; +} + /** * Convert ContentImage blocks in message arrays for script grader consumption. * @@ -95,6 +114,77 @@ export async function materializeContentForGrader( return result; } +function optionalString(record: Record, key: string): string | undefined { + const value = record[key]; + return typeof value === 'string' ? value : undefined; +} + +function optionalScore(record: Record, key: string): number | undefined { + const value = record[key]; + return typeof value === 'number' ? clampScore(value) : undefined; +} + +function parseScriptChecks(value: unknown): readonly GraderCheckResult[] { + if (!Array.isArray(value)) { + return []; + } + + return value + .filter((check): check is ScriptProtocolCheckRecord => { + if (typeof check !== 'object' || check === null || Array.isArray(check)) { + return false; + } + const record = check as Record; + return ( + typeof record.text === 'string' && + typeof record.pass === 'boolean' && + typeof record.reason === 'string' + ); + }) + .map((check) => ({ + ...(typeof check.id === 'string' ? { id: check.id } : {}), + text: check.text, + pass: check.pass, + ...(typeof check.score === 'number' ? { score: clampScore(check.score) } : {}), + reason: check.reason, + ...(typeof check.evidence === 'string' ? { evidence: check.evidence } : {}), + })); +} + +function checksToAssertions(checks: readonly GraderCheckResult[]): AssertionEntry[] { + return checks.map((check) => ({ + text: check.text, + passed: check.pass, + ...(check.evidence !== undefined ? { evidence: check.evidence } : {}), + })); +} + +function normalizeScriptProtocol(parsed: Record): ScriptProtocolResult { + const checks = parseScriptChecks(parsed.checks); + const score = + optionalScore(parsed, 'score') ?? + (checks.length > 0 + ? checks.reduce((sum, check) => sum + (check.score ?? (check.pass ? 1 : 0)), 0) / + checks.length + : typeof parsed.pass === 'boolean' + ? parsed.pass + ? 1 + : 0 + : 0); + const pass = typeof parsed.pass === 'boolean' ? parsed.pass : scoreToVerdict(score) === 'pass'; + const reason = optionalString(parsed, 'reason'); + const details = + parsed.details && typeof parsed.details === 'object' && !Array.isArray(parsed.details) + ? (parsed.details as JsonObject) + : undefined; + + if (typeof parsed.pass !== 'boolean' && typeof parsed.score !== 'number' && checks.length === 0) { + throw new Error('Script evaluator JSON must include pass, score, or checks[]'); + } + + return { pass, score, reason, checks, details }; +} + export interface ScriptGraderOptions { readonly command: readonly string[]; readonly cwd?: string; @@ -277,44 +367,19 @@ export class ScriptGrader implements Grader { rawParsed != null && typeof rawParsed === 'object' && !Array.isArray(rawParsed) ? rawParsed : undefined; - // Plain-text fallback: exit code is pass/fail, stdout is the assertion text. + // Plain-text fallback: exit code is pass/fail, stdout is the check text. // For numeric scores or multi-aspect results, use the JSON protocol instead. const passed = exitCode === 0; + const protocol = parsed != null ? normalizeScriptProtocol(parsed) : undefined; + const checks = protocol?.checks ?? []; const assertions: AssertionEntry[] = - parsed != null && Array.isArray(parsed?.assertions) - ? parsed.assertions - .filter( - (a: unknown): a is { text: string; passed: boolean; evidence?: string } => - typeof a === 'object' && - a !== null && - typeof (a as Record).text === 'string', - ) - .map((a) => ({ - text: String(a.text), - passed: Boolean(a.passed), - ...(typeof a.evidence === 'string' ? { evidence: a.evidence } : {}), - })) - : parsed == null - ? [{ text: stdout.trim() || (passed ? 'exit 0' : `exit ${exitCode}`), passed }] - : []; - // When the script omits `score` but returns `assertions`, derive score as passing/total. - const score = - parsed != null - ? clampScore( - typeof parsed.score === 'number' - ? parsed.score - : assertions.length > 0 - ? assertions.filter((a) => a.passed).length / assertions.length - : 0, - ) - : passed - ? 1 - : 0; - // Capture optional structured details from code judge output - const details = - parsed?.details && typeof parsed.details === 'object' && !Array.isArray(parsed.details) - ? (parsed.details as JsonObject) - : undefined; + protocol != null + ? checksToAssertions(checks) + : [{ text: stdout.trim() || (passed ? 'exit 0' : `exit ${exitCode}`), passed }]; + const score = protocol?.score ?? (passed ? 1 : 0); + const verdict = protocol ? (protocol.pass ? 'pass' : 'fail') : scoreToVerdict(score); + const reason = protocol?.reason; + const details = protocol?.details; // Build evaluator raw request with proxy metadata if used const proxyUsage = getProxyUsage?.(); @@ -333,7 +398,9 @@ export class ScriptGrader implements Grader { return { score, - verdict: scoreToVerdict(score), + verdict, + reason, + checks, assertions, expectedAspectCount: assertions.length || 1, graderRawRequest, @@ -346,6 +413,15 @@ export class ScriptGrader implements Grader { return { score: 0, verdict: 'fail', + reason: `Script evaluator failed: ${message}`, + checks: [ + { + text: 'Script evaluator execution', + pass: false, + score: 0, + reason: message, + }, + ], assertions: [{ text: `Script evaluator failed: ${message}`, passed: false }], expectedAspectCount: 1, graderRawRequest: { diff --git a/packages/core/src/evaluation/graders/types.ts b/packages/core/src/evaluation/graders/types.ts index 995474eb0..8055e7f3d 100644 --- a/packages/core/src/evaluation/graders/types.ts +++ b/packages/core/src/evaluation/graders/types.ts @@ -70,6 +70,8 @@ export interface EvaluationContext { export interface EvaluationScore { readonly score: number; readonly verdict: EvaluationVerdict; + readonly reason?: string; + readonly checks?: readonly import('../types.js').GraderCheckResult[]; readonly assertions: readonly import('../types.js').AssertionEntry[]; readonly expectedAspectCount: number; readonly graderRawRequest?: JsonObject; @@ -88,6 +90,8 @@ export interface ChildGraderResult { readonly score: number; readonly weight?: number; readonly verdict: EvaluationVerdict; + readonly reason?: string; + readonly checks?: readonly import('../types.js').GraderCheckResult[]; readonly assertions: readonly import('../types.js').AssertionEntry[]; readonly graderRawRequest?: JsonObject; readonly scores?: readonly ChildGraderResult[]; diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 4f3d852fb..4e942f531 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -741,6 +741,8 @@ export async function gradePreparedEvalCase( category: evalCase.category, conversationId: evalCase.conversation_id, score: skippedEvaluatorError ? 0 : score.score, + reason: score.reason, + checks: score.checks, assertions: score.assertions, target: target.name, input, @@ -2833,6 +2835,8 @@ async function evaluateCandidate(options: { category: evalCase.category, conversationId: evalCase.conversation_id, score: score.score, + reason: score.reason, + checks: score.checks, assertions: score.assertions, target: target.name, tokenUsage, @@ -3202,6 +3206,8 @@ async function runEvaluatorList(options: { score: score.score, weight, verdict: score.verdict, + reason: score.reason, + checks: score.checks, assertions: score.assertions, input: transformedContext.input ?? score.graderRawRequest, target: score.graderTarget, @@ -4067,6 +4073,8 @@ function mapChildResults( score: child.score, weight: child.weight, verdict: child.verdict, + reason: child.reason, + checks: child.checks, assertions: child.assertions, input: child.graderRawRequest, scores: mapChildResults(child.scores), diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 1a90a79ca..c9e34d24d 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -9,6 +9,16 @@ export interface AssertionEntry { readonly evidence?: string; } +/** A grader-produced check using the public pass/reason result vocabulary. */ +export interface GraderCheckResult { + readonly id?: string; + readonly text: string; + readonly pass: boolean; + readonly score?: number; + readonly reason: string; + readonly evidence?: string; +} + /** * JSON primitive values appearing in AgentV payloads. */ @@ -1255,6 +1265,8 @@ export interface EvaluationResult { readonly category?: string; readonly conversationId?: string; readonly score: number; + readonly reason?: string; + readonly checks?: readonly GraderCheckResult[]; readonly assertions: readonly AssertionEntry[]; readonly target: string; /** Optional explicit comparable variant. Path segments are not authoritative for this value. */ @@ -1342,6 +1354,8 @@ export interface GraderResult { readonly name: string; readonly type: GraderKind; readonly score: number; + readonly reason?: string; + readonly checks?: readonly GraderCheckResult[]; readonly weight?: number; readonly verdict?: EvaluationVerdict; readonly assertions: readonly AssertionEntry[]; diff --git a/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts b/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts index 5f74e9784..5cad599a3 100644 --- a/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts +++ b/packages/core/test/evaluation/graders/script-grader-plain-text.test.ts @@ -2,7 +2,7 @@ * Tests for script-grader plain-text fallback. * * When a script emits non-JSON stdout, the grader uses the exit code as - * pass/fail (0 = score 1, non-zero = score 0) and stdout as the assertion + * pass/fail (0 = score 1, non-zero = score 0) and stdout as the check * text. For numeric scores or multi-aspect results, use the JSON protocol. */ @@ -58,41 +58,51 @@ describe('script-grader plain-text fallback', () => { expect(result.score).toBe(0); }); - it('JSON protocol still works (score + assertions)', async () => { + it('JSON protocol accepts aggregate pass/score/reason without checks', async () => { const result = await grader( - `echo '{"score":0.6,"assertions":[{"text":"ok","passed":true}]}'`, + `echo '{"pass":true,"score":0.6,"reason":"Aggregate script score passed"}'`, ).evaluate(ctx); expect(result.score).toBe(0.6); - expect(result.assertions).toHaveLength(1); - expect(result.assertions[0].text).toBe('ok'); + expect(result.verdict).toBe('pass'); + expect(result.reason).toBe('Aggregate script score passed'); + expect(result.checks).toEqual([]); + expect(result.assertions).toEqual([]); }); - it('assertions without score → derived as passing/total', async () => { + it('JSON protocol preserves checks with scores', async () => { const result = await grader( - `echo '{"assertions":[{"text":"a","passed":true},{"text":"b","passed":false},{"text":"c","passed":true}]}'`, + `echo '{"pass":false,"score":0.4,"reason":"One weighted check failed","checks":[{"id":"a","text":"A","pass":true,"score":1,"reason":"A passed"},{"id":"b","text":"B","pass":false,"score":0.2,"reason":"B failed","evidence":"Observed B"}]}'`, ).evaluate(ctx); - expect(result.score).toBeCloseTo(2 / 3); - expect(result.assertions).toHaveLength(3); + expect(result.score).toBe(0.4); + expect(result.verdict).toBe('fail'); + expect(result.reason).toBe('One weighted check failed'); + expect(result.checks).toEqual([ + { id: 'a', text: 'A', pass: true, score: 1, reason: 'A passed' }, + { + id: 'b', + text: 'B', + pass: false, + score: 0.2, + reason: 'B failed', + evidence: 'Observed B', + }, + ]); + expect(result.assertions).toEqual([ + { text: 'A', passed: true }, + { text: 'B', passed: false, evidence: 'Observed B' }, + ]); }); - it('assertions all passing without score → score 1', async () => { + it('checks without score derive aggregate score from pass ratio', async () => { const result = await grader( - `echo '{"assertions":[{"text":"a","passed":true},{"text":"b","passed":true}]}'`, + `echo '{"reason":"Two checks, one pass","checks":[{"text":"a","pass":true,"reason":"A passed"},{"text":"b","pass":false,"reason":"B failed"}]}'`, ).evaluate(ctx); - expect(result.score).toBe(1); - }); - - it('assertions all failing without score → score 0', async () => { - const result = await grader(`echo '{"assertions":[{"text":"a","passed":false}]}'`).evaluate( - ctx, - ); - expect(result.score).toBe(0); - }); - - it('empty assertions array without score → score 0', async () => { - const result = await grader(`echo '{"assertions":[]}'`).evaluate(ctx); - expect(result.score).toBe(0); - expect(result.assertions).toHaveLength(0); + expect(result.score).toBe(0.5); + expect(result.verdict).toBe('fail'); + expect(result.checks).toEqual([ + { text: 'a', pass: true, reason: 'A passed' }, + { text: 'b', pass: false, reason: 'B failed' }, + ]); }); it('script with stderr on non-zero exit → surfaces as error assertion', async () => { diff --git a/packages/core/test/evaluation/script-grader-file-backed.test.ts b/packages/core/test/evaluation/script-grader-file-backed.test.ts index a30bedd04..6657c1107 100644 --- a/packages/core/test/evaluation/script-grader-file-backed.test.ts +++ b/packages/core/test/evaluation/script-grader-file-backed.test.ts @@ -27,9 +27,14 @@ async function createEchoGrader(dir: string): Promise { `const input = require('fs').readFileSync(0, 'utf8'); const payload = JSON.parse(input); console.log(JSON.stringify({ - hasOutputPath: !!payload.output_path, - outputIsNull: payload.output === null, - outputPath: payload.output_path || null, + pass: true, + score: 1, + reason: 'Payload parsed', + details: { + hasOutputPath: !!payload.output_path, + outputIsNull: payload.output === null, + outputPath: payload.output_path || null, + }, })); `, 'utf8', @@ -42,7 +47,7 @@ async function createScoringGrader(dir: string): Promise { const script = join(dir, 'score-grader.js'); await writeFile( script, - `console.log(JSON.stringify({ score: 1.0, assertions: [{ text: 'ok', passed: true }] })); + `console.log(JSON.stringify({ pass: true, score: 1.0, reason: 'ok', checks: [{ text: 'ok', pass: true, reason: 'ok' }] })); `, 'utf8', ); @@ -56,14 +61,19 @@ async function createPayloadShapeGrader(dir: string): Promise `const input = require('fs').readFileSync(0, 'utf8'); const payload = JSON.parse(input); console.log(JSON.stringify({ + pass: payload.expected_output?.[0]?.content?.answer === 'Paris' && + payload.config?.mode === 'strict' && + payload.input?.[0]?.content === 'Test input', score: payload.expected_output?.[0]?.content?.answer === 'Paris' && payload.config?.mode === 'strict' && payload.input?.[0]?.content === 'Test input' ? 1 : 0, - assertions: [{ + reason: 'Structured stdin preservation check', + checks: [{ text: 'structured stdin preserved', - passed: payload.expected_output?.[0]?.content?.answer === 'Paris' && + pass: payload.expected_output?.[0]?.content?.answer === 'Paris' && payload.config?.mode === 'strict' && - payload.input?.[0]?.content === 'Test input' + payload.input?.[0]?.content === 'Test input', + reason: 'expected_output, config, and input are present' }], details: { expected_output: payload.expected_output, config: payload.config, input: payload.input } })); @@ -113,6 +123,7 @@ describe('ScriptGrader file-backed output', () => { }); expect(result.score).toBe(1.0); + expect(result.checks?.filter((check) => check.pass).map((check) => check.text)).toEqual(['ok']); expect(result.assertions.filter((a) => a.passed).map((a) => a.text)).toEqual(['ok']); // Temp files should be cleaned up @@ -152,6 +163,13 @@ describe('ScriptGrader file-backed output', () => { }); expect(result.score).toBe(1); + expect(result.checks).toEqual([ + { + text: 'structured stdin preserved', + pass: true, + reason: 'expected_output, config, and input are present', + }, + ]); expect(result.assertions).toEqual([{ text: 'structured stdin preserved', passed: true }]); expect(result.details?.expected_output).toEqual([ { role: 'assistant', content: { answer: 'Paris' } }, diff --git a/packages/core/test/evaluation/script-grader-multimodal.test.ts b/packages/core/test/evaluation/script-grader-multimodal.test.ts index 18805a642..d614743d8 100644 --- a/packages/core/test/evaluation/script-grader-multimodal.test.ts +++ b/packages/core/test/evaluation/script-grader-multimodal.test.ts @@ -33,8 +33,10 @@ async function createPayloadEchoGrader(dir: string): Promise `const input = require('fs').readFileSync(0, 'utf8'); const payload = JSON.parse(input); console.log(JSON.stringify({ + pass: true, score: 1.0, - assertions: [{ text: 'ok', passed: true }], + reason: 'Payload captured', + checks: [{ text: 'ok', pass: true, reason: 'Payload captured' }], details: { payload }, })); `, diff --git a/packages/core/test/fixtures/test-define-grader.ts b/packages/core/test/fixtures/test-define-grader.ts index 5483db597..1775c077a 100644 --- a/packages/core/test/fixtures/test-define-grader.ts +++ b/packages/core/test/fixtures/test-define-grader.ts @@ -9,7 +9,7 @@ const input = JSON.parse(readFileSync(0, 'utf8')) as { readonly criteria?: string; }; -const assertions: { text: string; passed: boolean }[] = []; +const checks: { text: string; pass: boolean; reason: string }[] = []; // `output` is the final answer/scored result. Transcript-aware graders should // use messages/trace instead. @@ -22,14 +22,30 @@ const candidateWords = candidateText.toLowerCase().split(/\s+/); for (const word of outcomeWords) { if (word.length > 3 && candidateWords.includes(word)) { - assertions.push({ text: `Contains keyword: ${word}`, passed: true }); + checks.push({ text: `Contains keyword: ${word}`, pass: true, reason: `Found keyword ${word}` }); } } -if (assertions.length === 0) { - assertions.push({ text: 'No matching keywords found', passed: false }); +if (checks.length === 0) { + checks.push({ + text: 'No matching keywords found', + pass: false, + reason: 'No criteria words matched', + }); } -const score = assertions.some((a) => a.passed) ? 1.0 : 0.0; - -console.log(JSON.stringify({ score, assertions }, null, 2)); +const pass = checks.some((check) => check.pass); +const score = pass ? 1.0 : 0.0; + +console.log( + JSON.stringify( + { + pass, + score, + reason: pass ? 'At least one criteria keyword matched' : 'No criteria keywords matched', + checks, + }, + null, + 2, + ), +); diff --git a/packages/core/test/fixtures/test-grader-with-details.cjs b/packages/core/test/fixtures/test-grader-with-details.cjs index ae8604d17..1653ef897 100644 --- a/packages/core/test/fixtures/test-grader-with-details.cjs +++ b/packages/core/test/fixtures/test-grader-with-details.cjs @@ -16,14 +16,19 @@ const candidateText = ? input.output.map((m) => String(m.content ?? '')).join('') : ''; const hasCandidate = candidateText.length > 0; +const pass = hasExpected && hasCandidate; // Emit details with structured metrics console.log( JSON.stringify({ - score: hasExpected && hasCandidate ? 0.75 : 0, - assertions: [ - ...(hasExpected ? [{ text: 'expected_output present', passed: true }] : []), - ...(hasCandidate ? [] : [{ text: 'output missing', passed: false }]), + pass, + score: pass ? 0.75 : 0, + reason: pass + ? 'Expected output and candidate output were present' + : 'Missing required payload data', + checks: [ + { text: 'expected_output present', pass: hasExpected, reason: 'expected_output was present' }, + { text: 'output present', pass: hasCandidate, reason: 'output was present' }, ], details: { metrics: { diff --git a/packages/core/test/fixtures/test-grader-workspace.cjs b/packages/core/test/fixtures/test-grader-workspace.cjs index e812d0d4c..940b85516 100644 --- a/packages/core/test/fixtures/test-grader-workspace.cjs +++ b/packages/core/test/fixtures/test-grader-workspace.cjs @@ -3,31 +3,58 @@ const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); -const assertions = []; +const checks = []; // Check workspace_path in JSON payload if (typeof input.workspace_path === 'string' && input.workspace_path.length > 0) { - assertions.push({ text: 'workspace_path present in payload', passed: true }); + checks.push({ + text: 'workspace_path present in payload', + pass: true, + reason: 'workspace_path is present', + }); } else { - assertions.push({ text: 'workspace_path missing from payload', passed: false }); + checks.push({ + text: 'workspace_path present in payload', + pass: false, + reason: 'workspace_path is missing', + }); } // Check AGENTV_WORKSPACE_PATH env var const envPath = process.env.AGENTV_WORKSPACE_PATH; if (typeof envPath === 'string' && envPath.length > 0) { - assertions.push({ text: 'AGENTV_WORKSPACE_PATH env var set', passed: true }); + checks.push({ + text: 'AGENTV_WORKSPACE_PATH env var set', + pass: true, + reason: 'AGENTV_WORKSPACE_PATH is set', + }); } else { - assertions.push({ text: 'AGENTV_WORKSPACE_PATH env var missing', passed: false }); + checks.push({ + text: 'AGENTV_WORKSPACE_PATH env var set', + pass: false, + reason: 'AGENTV_WORKSPACE_PATH is missing', + }); } // Check that both match when present if (input.workspace_path && envPath && input.workspace_path === envPath) { - assertions.push({ text: 'payload and env var match', passed: true }); + checks.push({ + text: 'payload and env var match', + pass: true, + reason: 'Both workspace paths match', + }); } else if (input.workspace_path && envPath) { - assertions.push({ text: 'payload and env var do not match', passed: false }); + checks.push({ + text: 'payload and env var match', + pass: false, + reason: 'Workspace paths do not match', + }); } -const passed = assertions.filter((a) => a.passed).length; -const score = assertions.every((a) => a.passed) ? 1.0 : passed / assertions.length; +const passed = checks.filter((check) => check.pass).length; +const pass = checks.length > 0 && passed === checks.length; +const score = pass ? 1.0 : passed / checks.length; -console.log(JSON.stringify({ score, assertions })); +console.log( + JSON.stringify({ pass, score, reason: `${passed}/${checks.length} checks passed`, checks }), +); diff --git a/packages/core/test/fixtures/test-grader.cjs b/packages/core/test/fixtures/test-grader.cjs index 74ebf0312..5192502dc 100644 --- a/packages/core/test/fixtures/test-grader.cjs +++ b/packages/core/test/fixtures/test-grader.cjs @@ -23,14 +23,17 @@ try { } catch {} const ok = hasExpected && hasCandidate && candidateDecisionOk; +const checks = [ + { text: 'expected_output present', pass: hasExpected, reason: 'expected_output was present' }, + { text: 'output present', pass: hasCandidate, reason: 'output was present' }, + { text: 'output parses', pass: candidateDecisionOk, reason: 'output JSON has decision ACCEPT' }, +]; console.log( JSON.stringify({ + pass: ok, score: ok ? 1 : 0, - assertions: [ - { text: 'expected_output present', passed: hasExpected }, - { text: 'output present', passed: hasCandidate }, - { text: 'output parses', passed: candidateDecisionOk }, - ].filter((a) => a.passed !== undefined), + reason: ok ? 'All script checks passed' : 'One or more script checks failed', + checks, }), ); diff --git a/packages/core/test/fixtures/test-no-trace-summary.cjs b/packages/core/test/fixtures/test-no-trace-summary.cjs index 1d87078a6..20e6fc9d3 100644 --- a/packages/core/test/fixtures/test-no-trace-summary.cjs +++ b/packages/core/test/fixtures/test-no-trace-summary.cjs @@ -4,12 +4,19 @@ const fs = require('node:fs'); const input = JSON.parse(fs.readFileSync(0, 'utf8')); const hasSummary = input.trace !== null && input.trace !== undefined; +const pass = !hasSummary; console.log( JSON.stringify({ - score: hasSummary ? 0 : 1, - assertions: hasSummary - ? [{ text: 'Expected no summary', passed: false }] - : [{ text: 'Correctly handled missing summary', passed: true }], + pass, + score: pass ? 1 : 0, + reason: pass ? 'Correctly handled missing summary' : 'Expected no summary', + checks: [ + { + text: pass ? 'Correctly handled missing summary' : 'Expected no summary', + pass, + reason: pass ? 'No trace summary was present' : 'Trace summary was present', + }, + ], }), ); diff --git a/packages/core/test/fixtures/test-trace-summary.cjs b/packages/core/test/fixtures/test-trace-summary.cjs index 00ee0e9c8..d2fefbb51 100644 --- a/packages/core/test/fixtures/test-trace-summary.cjs +++ b/packages/core/test/fixtures/test-trace-summary.cjs @@ -8,14 +8,29 @@ const hasEventCount = summary && typeof summary.event_count === 'number'; const hasTokenUsage = input.token_usage && typeof input.token_usage.input === 'number'; const hasCostUsd = typeof input.cost_usd === 'number'; const score = hasEventCount && hasTokenUsage && hasCostUsd ? 1 : 0; +const pass = score === 1; console.log( JSON.stringify({ + pass, score, - assertions: [ - { text: 'eventCount present', passed: !!hasEventCount }, - { text: 'tokenUsage present', passed: !!hasTokenUsage }, - { text: 'costUsd present', passed: !!hasCostUsd }, + reason: pass ? 'Trace summary fields are present' : 'Trace summary fields are missing', + checks: [ + { + text: 'eventCount present', + pass: !!hasEventCount, + reason: hasEventCount ? 'event_count is present' : 'event_count is missing', + }, + { + text: 'tokenUsage present', + pass: !!hasTokenUsage, + reason: hasTokenUsage ? 'token_usage is present' : 'token_usage is missing', + }, + { + text: 'costUsd present', + pass: !!hasCostUsd, + reason: hasCostUsd ? 'cost_usd is present' : 'cost_usd is missing', + }, ], }), ); diff --git a/packages/sdk/README.md b/packages/sdk/README.md index c8baacf96..c76b71556 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -66,7 +66,8 @@ import { defineAssertion } from '@agentv/sdk'; export default defineAssertion(({ output }) => ({ pass: (output ?? '').toLowerCase().includes('hello'), - reasoning: 'Checks for greeting', + score: (output ?? '').toLowerCase().includes('hello') ? 1 : 0, + reason: 'Checks for greeting', })); ``` @@ -79,10 +80,20 @@ Checks support `pass: boolean` for simple checks and `score: number` (0-1) for g import { defineScriptGrader } from '@agentv/sdk'; export default defineScriptGrader(({ output, traceSummary }) => ({ + pass: (output ?? '').length > 0 && traceSummary !== null, score: (output ?? '').length > 0 ? 1.0 : 0.0, - assertions: [ - { text: 'Output received', passed: (output ?? '').length > 0 }, - { text: 'Trace summary available', passed: traceSummary !== null }, + reason: 'Checks output presence and trace availability', + checks: [ + { + text: 'Output received', + pass: (output ?? '').length > 0, + reason: (output ?? '').length > 0 ? 'Output is non-empty' : 'Output is empty', + }, + { + text: 'Trace summary available', + pass: traceSummary !== null, + reason: traceSummary !== null ? 'Trace summary is present' : 'Trace summary is missing', + }, ], })); ``` @@ -113,7 +124,7 @@ assert: command: [agentv, eval, graders/welcome-banner.test.ts] ``` -The command reads the normal script-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV assertion, and computes score as `passed / total`. +The command reads the normal script-grader stdin payload, runs Vitest in `workspace_path`, maps each Vitest test to an AgentV check, and computes score as `passed / total`. Use the explicit `agentv eval vitest` subcommand when you need adapter options such as `--cwd`, `--in-workspace`, or `--vitest-command`. Use `defineVitestWorkspaceGrader` when embedding this adapter in a custom script: @@ -143,7 +154,7 @@ export default defineWorkspaceGrader(async ({ workspace }) => [ ]); ``` -The helper resolves `workspace_path` or `AGENTV_WORKSPACE_PATH`, reads files relative to the workspace, returns AgentV assertion objects, and computes `score` as passed checks divided by total checks. Prefer Vitest verifiers for checks that naturally fit a test file; use this lower-level helper for tiny one-off graders or custom score shaping. +The helper resolves `workspace_path` or `AGENTV_WORKSPACE_PATH`, reads files relative to the workspace, returns AgentV check objects, and computes `score` as passed checks divided by total checks. Prefer Vitest verifiers for checks that naturally fit a test file; use this lower-level helper for tiny one-off graders or custom score shaping. ### defineEval (YAML-aligned `.eval.ts` authoring) diff --git a/packages/sdk/src/assertion.ts b/packages/sdk/src/assertion.ts index ab48293ab..be1ad7f54 100644 --- a/packages/sdk/src/assertion.ts +++ b/packages/sdk/src/assertion.ts @@ -65,25 +65,39 @@ export type AssertionType = | 'similar' | (string & {}); +/** + * Check returned from an assertion handler. + */ +export interface AssertionCheck { + readonly id?: string; + readonly text: string; + readonly pass: boolean; + readonly score?: number; + readonly reason: string; + readonly evidence?: string; +} + /** * Result returned from an assertion handler. * * @example Pass with score * ```ts - * { pass: true, assertions: [{ text: 'Output contains expected keywords', passed: true }] } + * { pass: true, score: 1, reason: 'Output contains expected keywords' } * ``` * - * @example Fail with evidence + * @example Fail with checks * ```ts - * { pass: false, score: 0.3, assertions: [{ text: 'Missing required header', passed: false }] } + * { pass: false, score: 0.3, reason: 'Missing required header', checks: [ + * { text: 'Header present', pass: false, reason: 'No header found' }, + * ] } * ``` * * @example Granular score (0-1) * ```ts - * { score: 0.75, assertions: [ - * { text: 'Format correct', passed: true }, - * { text: 'Content relevant', passed: true }, - * { text: 'Missing citation', passed: false }, + * { score: 0.75, reason: 'Two of three checks passed', checks: [ + * { text: 'Format correct', pass: true, reason: 'Matches expected format' }, + * { text: 'Content relevant', pass: true, reason: 'Addresses the request' }, + * { text: 'Citation present', pass: false, reason: 'Missing citation' }, * ] } * ``` */ @@ -92,12 +106,10 @@ export interface AssertionScore { readonly pass?: boolean; /** Numeric score between 0 and 1. Defaults to 1 if pass=true, 0 if pass=false. */ readonly score?: number; - /** Per-assertion verdicts with optional evidence. */ - readonly assertions?: readonly { - readonly text: string; - readonly passed: boolean; - readonly evidence?: string; - }[]; + /** Explanation for the aggregate pass/fail decision. */ + readonly reason?: string; + /** Per-check verdicts with optional score and evidence. */ + readonly checks?: readonly AssertionCheck[]; /** Optional structured details for domain-specific metrics. */ readonly details?: Record; } @@ -143,10 +155,18 @@ function normalizeScore(result: AssertionScore): ScriptGraderResult { } else { score = 0; } + const pass = result.pass ?? score >= 0.5; return { + pass, score, - assertions: result.assertions ? [...result.assertions] : [], + reason: result.reason ?? (pass ? 'Assertion passed' : 'Assertion failed'), + checks: result.checks + ? result.checks.map((check) => ({ + ...check, + ...(check.score !== undefined ? { score: clampScore(check.score) } : {}), + })) + : [], details: result.details, }; } @@ -189,8 +209,10 @@ export async function runAssertion(handler: AssertionHandler): Promise { } catch (error) { const errorMessage = formatError(error); const errorResult: ScriptGraderResult = { + pass: false, score: 0, - assertions: [{ text: `Assertion failed: ${errorMessage}`, passed: false }], + reason: `Assertion failed: ${errorMessage}`, + checks: [{ text: 'Assertion execution', pass: false, reason: errorMessage }], }; console.log(JSON.stringify(errorResult, null, 2)); process.exit(1); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index c4875ea0e..446d63468 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -12,7 +12,8 @@ * const answer = output ?? ''; * return { * pass: answer.includes('hello'), - * assertions: [{ text: 'Checks greeting', passed: answer.includes('hello') }], + * score: answer.includes('hello') ? 1 : 0, + * reason: answer.includes('hello') ? 'Greeting found' : 'Greeting missing', * }; * })); * ``` @@ -25,9 +26,11 @@ * export default defineScriptGrader(({ output, traceSummary }) => { * return { * score: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5 ? 1.0 : 0.5, - * assertions: [ - * { text: 'Answer is not empty', passed: (output ?? '').length > 0 }, - * { text: 'Efficient tool usage', passed: (traceSummary?.eventCount ?? 0) <= 5 }, + * pass: (output ?? '').length > 0 && (traceSummary?.eventCount ?? 0) <= 5, + * reason: 'Checks answer text and trace size', + * checks: [ + * { text: 'Answer is not empty', pass: (output ?? '').length > 0, reason: 'Output text is present' }, + * { text: 'Efficient tool usage', pass: (traceSummary?.eventCount ?? 0) <= 5, reason: 'Trace event count is within limit' }, * ], * }; * })); @@ -63,6 +66,7 @@ // Re-export schemas and types export { ScriptGraderInputSchema, + ScriptGraderCheckSchema, ScriptGraderResultSchema, CodeGraderInputSchema, CodeGraderResultSchema, @@ -93,6 +97,7 @@ export { ContentFileSchema, ContentSchema, type ScriptGraderInput, + type ScriptGraderCheck, type ScriptGraderResult, type CodeGraderInput, type CodeGraderResult, @@ -215,6 +220,7 @@ export { runWorkspaceGrader, type Workspace, type WorkspaceAssertion, + type WorkspaceCheck, type WorkspaceFile, type WorkspaceFileAssertionOptions, type WorkspaceGraderContext, @@ -236,6 +242,7 @@ export { z } from 'zod'; // Re-export assertion types export type { + AssertionCheck, AssertionContext, AssertionHandler, AssertionScore, @@ -273,13 +280,15 @@ export type { PromptTemplateHandler }; * * export default defineScriptGrader(({ trace }) => { * if (!trace) { - * return { score: 0.5, assertions: [{ text: 'No trace available', passed: false }] }; + * return { pass: false, score: 0.5, reason: 'No trace available' }; * } * * const efficient = trace.eventCount <= 10; * return { + * pass: efficient, * score: efficient ? 1.0 : 0.5, - * assertions: [{ text: efficient ? 'Efficient execution' : 'Too many tool calls', passed: efficient }], + * reason: efficient ? 'Efficient execution' : 'Too many tool calls', + * checks: [{ text: 'Trace event count within limit', pass: efficient, reason: `${trace.eventCount} events observed` }], * }; * }); * ``` @@ -366,7 +375,7 @@ export function definePromptTemplate(handler: PromptTemplateHandler): void { * const text = output ?? ''; * return { * pass: text.toLowerCase().includes('hello'), - * assertions: [{ text: 'Checks for greeting', passed: text.toLowerCase().includes('hello') }], + * reason: text.toLowerCase().includes('hello') ? 'Greeting found' : 'Greeting missing', * }; * })); * ``` @@ -381,9 +390,10 @@ export function definePromptTemplate(handler: PromptTemplateHandler): void { * const isEfficient = (traceSummary?.eventCount ?? 0) <= 5 ? 0.5 : 0; * return { * score: hasContent + isEfficient, - * assertions: [ - * { text: 'Has content', passed: !!hasContent }, - * { text: 'Efficient', passed: !!isEfficient }, + * reason: 'Checks content exists and trace size', + * checks: [ + * { text: 'Has content', pass: !!hasContent, reason: hasContent ? 'Output is non-empty' : 'Output is empty' }, + * { text: 'Efficient', pass: !!isEfficient, reason: isEfficient ? 'Trace is within limit' : 'Trace exceeds limit' }, * ], * }; * })); diff --git a/packages/sdk/src/runtime.ts b/packages/sdk/src/runtime.ts index b4d442d40..cf0259d72 100644 --- a/packages/sdk/src/runtime.ts +++ b/packages/sdk/src/runtime.ts @@ -91,6 +91,10 @@ export async function runScriptGrader(handler: ScriptGraderHandler): Promise ({ + ...check, + ...(check.score !== undefined ? { score: clampScore(check.score) } : {}), + })), }); // 9. Output JSON @@ -99,8 +103,16 @@ export async function runScriptGrader(handler: ScriptGraderHandler): Promise; +export type ScriptGraderCheck = z.infer; export type ScriptGraderResult = z.infer; export type TraceSummary = z.infer; diff --git a/packages/sdk/src/target-client.ts b/packages/sdk/src/target-client.ts index 7c1937f4e..a7e3c26ca 100644 --- a/packages/sdk/src/target-client.ts +++ b/packages/sdk/src/target-client.ts @@ -115,7 +115,7 @@ export class TargetInvocationError extends Error { * * if (!target) { * // Target not available - no target config on this evaluator - * return { score: 0.5, assertions: [{ text: 'Target not available', passed: false }] }; + * return { pass: false, score: 0.5, reason: 'Target not available' }; * } * * const response = await target.invoke({ @@ -124,7 +124,11 @@ export class TargetInvocationError extends Error { * }); * * const result = JSON.parse(response.rawText ?? '{}'); - * return { score: result.correct ? 1.0 : 0.0 }; + * return { + * pass: result.correct === true, + * score: result.correct === true ? 1.0 : 0.0, + * reason: result.correct === true ? 'Target judged the answer correct' : 'Target judged the answer incorrect', + * }; * }); * ``` */ diff --git a/packages/sdk/src/vitest.ts b/packages/sdk/src/vitest.ts index 5030e5a9c..31755afcd 100644 --- a/packages/sdk/src/vitest.ts +++ b/packages/sdk/src/vitest.ts @@ -189,26 +189,35 @@ export function vitestReportToScriptGraderResult( report: VitestJsonReport, options: Pick = {}, ): ScriptGraderResult { - const assertions = (report.testResults ?? []).flatMap((file) => + const checks = (report.testResults ?? []).flatMap((file) => (file.assertionResults ?? []).map((item) => { - const passed = item.status === 'passed'; + const pass = item.status === 'passed'; const evidence = item.failureMessages && item.failureMessages.length > 0 ? truncate(item.failureMessages.join('\n\n')) : undefined; return { text: assertionText(file, item), - passed, + pass, + reason: pass ? 'Vitest test passed.' : 'Vitest test failed.', ...(evidence !== undefined ? { evidence } : {}), }; }), ); - if (assertions.length === 0) { - const passed = options.passWithNoTests === true; + if (checks.length === 0) { + const pass = options.passWithNoTests === true; return ScriptGraderResultSchema.parse({ - score: passed ? 1 : 0, - assertions: [{ text: 'Vitest reported no tests', passed }], + pass, + score: pass ? 1 : 0, + reason: pass ? 'Vitest reported no tests; configured to pass.' : 'Vitest reported no tests.', + checks: [ + { + text: 'Vitest reported no tests', + pass, + reason: pass ? 'passWithNoTests is enabled.' : 'No Vitest tests were discovered.', + }, + ], details: { vitest_success: report.success ?? false, num_total_tests: report.numTotalTests ?? 0, @@ -220,15 +229,18 @@ export function vitestReportToScriptGraderResult( }); } - const passedCount = assertions.filter((item) => item.passed).length; + const passedCount = checks.filter((item) => item.pass).length; + const pass = passedCount === checks.length; return ScriptGraderResultSchema.parse({ - score: passedCount / assertions.length, - assertions, + pass, + score: passedCount / checks.length, + reason: `${passedCount}/${checks.length} Vitest tests passed.`, + checks, details: { - vitest_success: report.success ?? passedCount === assertions.length, - num_total_tests: report.numTotalTests ?? assertions.length, + vitest_success: report.success ?? pass, + num_total_tests: report.numTotalTests ?? checks.length, num_passed_tests: report.numPassedTests ?? passedCount, - num_failed_tests: report.numFailedTests ?? assertions.length - passedCount, + num_failed_tests: report.numFailedTests ?? checks.length - passedCount, num_pending_tests: report.numPendingTests ?? 0, num_todo_tests: report.numTodoTests ?? 0, }, @@ -342,12 +354,14 @@ export async function runVitestWorkspaceGrader( const workspacePath = workspacePathFrom(input); if (!workspacePath) { return { + pass: false, score: 0, - assertions: [ + reason: 'Vitest workspace verifier requires workspace_path.', + checks: [ { text: 'Vitest workspace verifier requires workspace_path', - passed: false, - evidence: 'Configure workspace in the eval YAML so AgentV can pass workspace_path.', + pass: false, + reason: 'Configure workspace in the eval YAML so AgentV can pass workspace_path.', }, ], }; @@ -393,13 +407,16 @@ export async function runVitestWorkspaceGrader( const report = await readVitestReport(result, outputFile); return vitestReportToScriptGraderResult(report, options); } catch (error) { + const reason = error instanceof Error ? error.message : String(error); return { + pass: false, score: 0, - assertions: [ + reason: 'Vitest workspace verifier failed to run.', + checks: [ { text: 'Vitest workspace verifier failed to run', - passed: false, - evidence: error instanceof Error ? error.message : String(error), + pass: false, + reason, }, ], }; diff --git a/packages/sdk/src/workspace.ts b/packages/sdk/src/workspace.ts index 4ab1a32e2..75d30d032 100644 --- a/packages/sdk/src/workspace.ts +++ b/packages/sdk/src/workspace.ts @@ -1,8 +1,8 @@ /** - * Workspace grader helpers for deterministic file assertions. + * Workspace grader helpers for deterministic file checks. * * `defineWorkspaceGrader()` wraps the script-grader runtime with a small - * workspace object so graders can read files and return assertion arrays + * workspace object so graders can read files and return check arrays * without hand-rolling stdin parsing, workspace path fallback, file reads, or * score aggregation. */ @@ -11,23 +11,29 @@ import nodePath from 'node:path'; import { runScriptGrader } from './runtime.js'; import { + type ScriptGraderCheck, type ScriptGraderInput, type ScriptGraderResult, ScriptGraderResultSchema, } from './schemas.js'; -export interface WorkspaceAssertion { +export interface WorkspaceCheck { readonly text: string; - readonly passed: boolean; + readonly pass: boolean; + readonly score?: number; + readonly reason: string; readonly evidence?: string; } +/** @deprecated Use WorkspaceCheck. */ +export type WorkspaceAssertion = WorkspaceCheck; + type Awaitable = T | Promise; export type WorkspaceGraderReturn = | ScriptGraderResult - | WorkspaceAssertion - | readonly Awaitable[]; + | WorkspaceCheck + | readonly Awaitable[]; export interface WorkspaceFileAssertionOptions { readonly text?: string; @@ -37,14 +43,11 @@ export interface WorkspaceFile { readonly path: string; readonly absolutePath?: string; readText(): Promise; - exists(options?: WorkspaceFileAssertionOptions): Promise; - contains(expected: string, options?: WorkspaceFileAssertionOptions): Promise; - notContains( - expected: string, - options?: WorkspaceFileAssertionOptions, - ): Promise; - matches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise; - notMatches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise; + exists(options?: WorkspaceFileAssertionOptions): Promise; + contains(expected: string, options?: WorkspaceFileAssertionOptions): Promise; + notContains(expected: string, options?: WorkspaceFileAssertionOptions): Promise; + matches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise; + notMatches(pattern: RegExp, options?: WorkspaceFileAssertionOptions): Promise; } export interface Workspace { @@ -121,10 +124,11 @@ function resolveWorkspacePath(workspacePath: string | undefined, relativePath: s }; } -function assertion(text: string, passed: boolean, evidence?: string): WorkspaceAssertion { +function check(text: string, pass: boolean, reason: string, evidence?: string): WorkspaceCheck { return { text, - passed, + pass, + reason, ...(evidence !== undefined ? { evidence } : {}), }; } @@ -189,17 +193,17 @@ export function createWorkspace(input: ScriptGraderInput): Workspace { async exists(options: WorkspaceFileAssertionOptions = {}) { const text = options.text ?? `${label} exists`; if (resolved.error) { - return assertion(text, false, resolved.error); + return check(text, false, resolved.error); } try { const fileStat = await stat(resolved.absolutePath as string); if (fileStat.isFile()) { - return assertion(text, true); + return check(text, true, `${label} exists.`); } - return assertion(text, false, `${label} exists but is not a file.`); + return check(text, false, `${label} exists but is not a file.`); } catch { - return assertion(text, false, `${label} does not exist.`); + return check(text, false, `${label} does not exist.`); } }, @@ -207,14 +211,16 @@ export function createWorkspace(input: ScriptGraderInput): Workspace { const text = options.text ?? `${label} contains ${quote(expected)}`; const content = await readFileForAssertion(this); if ('error' in content) { - return assertion(text, false, content.error); + return check(text, false, content.error); } - const passed = content.content.includes(expected); - return assertion( + const pass = content.content.includes(expected); + return check( text, - passed, - passed ? undefined : `${label} is missing ${quote(expected)}.`, + pass, + pass + ? `${label} contains ${quote(expected)}.` + : `${label} is missing ${quote(expected)}.`, ); }, @@ -222,14 +228,16 @@ export function createWorkspace(input: ScriptGraderInput): Workspace { const text = options.text ?? `${label} does not contain ${quote(expected)}`; const content = await readFileForAssertion(this); if ('error' in content) { - return assertion(text, false, content.error); + return check(text, false, content.error); } - const passed = !content.content.includes(expected); - return assertion( + const pass = !content.content.includes(expected); + return check( text, - passed, - passed ? undefined : `${label} contains unexpected text ${quote(expected)}.`, + pass, + pass + ? `${label} does not contain ${quote(expected)}.` + : `${label} contains unexpected text ${quote(expected)}.`, ); }, @@ -237,15 +245,17 @@ export function createWorkspace(input: ScriptGraderInput): Workspace { const text = options.text ?? `${label} matches ${regexLabel(pattern)}`; const content = await readFileForAssertion(this); if ('error' in content) { - return assertion(text, false, content.error); + return check(text, false, content.error); } pattern.lastIndex = 0; - const passed = pattern.test(content.content); - return assertion( + const pass = pattern.test(content.content); + return check( text, - passed, - passed ? undefined : `${label} does not match ${regexLabel(pattern)}.`, + pass, + pass + ? `${label} matches ${regexLabel(pattern)}.` + : `${label} does not match ${regexLabel(pattern)}.`, ); }, @@ -253,15 +263,17 @@ export function createWorkspace(input: ScriptGraderInput): Workspace { const text = options.text ?? `${label} does not match ${regexLabel(pattern)}`; const content = await readFileForAssertion(this); if ('error' in content) { - return assertion(text, false, content.error); + return check(text, false, content.error); } pattern.lastIndex = 0; - const passed = !pattern.test(content.content); - return assertion( + const pass = !pattern.test(content.content); + return check( text, - passed, - passed ? undefined : `${label} matches unexpected pattern ${regexLabel(pattern)}.`, + pass, + pass + ? `${label} does not match ${regexLabel(pattern)}.` + : `${label} matches unexpected pattern ${regexLabel(pattern)}.`, ); }, }; @@ -281,12 +293,18 @@ export async function normalizeWorkspaceGraderResult( return ScriptGraderResultSchema.parse(result); } - const assertions = Array.isArray(result) ? await Promise.all(result) : [result]; - const passed = assertions.filter((item) => item.passed).length; + const checks = Array.isArray(result) ? await Promise.all(result) : [result]; + const passed = checks.filter((item) => item.pass).length; + const score = + checks.length > 0 + ? checks.reduce((sum, item) => sum + (item.score ?? (item.pass ? 1 : 0)), 0) / checks.length + : 0; return ScriptGraderResultSchema.parse({ - score: assertions.length > 0 ? passed / assertions.length : 0, - assertions, + pass: checks.length > 0 && passed === checks.length, + score, + reason: checks.length > 0 ? `${passed}/${checks.length} checks passed.` : 'No checks ran.', + checks: checks satisfies readonly ScriptGraderCheck[], }); } diff --git a/packages/sdk/test/define-script-grader.test.ts b/packages/sdk/test/define-script-grader.test.ts index bd2c1b1b1..5b094b171 100644 --- a/packages/sdk/test/define-script-grader.test.ts +++ b/packages/sdk/test/define-script-grader.test.ts @@ -288,50 +288,56 @@ describe('ScriptGraderInputSchema', () => { describe('ScriptGraderResultSchema', () => { it('parses valid result with all fields', () => { const result: ScriptGraderResult = { + pass: false, score: 0.8, - assertions: [ - { text: 'Correct answer', passed: true }, - { text: 'Missing explanation', passed: false }, + reason: 'One check failed', + checks: [ + { text: 'Correct answer', pass: true, reason: 'Answer matched' }, + { text: 'Missing explanation', pass: false, reason: 'No explanation included' }, ], }; const parsed = ScriptGraderResultSchema.parse(result); + expect(parsed.pass).toBe(false); expect(parsed.score).toBe(0.8); - expect(parsed.assertions).toEqual([ - { text: 'Correct answer', passed: true }, - { text: 'Missing explanation', passed: false }, + expect(parsed.reason).toBe('One check failed'); + expect(parsed.checks).toEqual([ + { text: 'Correct answer', pass: true, reason: 'Answer matched' }, + { text: 'Missing explanation', pass: false, reason: 'No explanation included' }, ]); }); - it('defaults assertions to empty array', () => { - const result = { score: 0.5 }; + it('defaults checks to empty array', () => { + const result = { pass: true, score: 0.5, reason: 'Aggregate only' }; const parsed = ScriptGraderResultSchema.parse(result); - expect(parsed.assertions).toEqual([]); + expect(parsed.checks).toEqual([]); }); - it('defaults assertions to empty array when omitted', () => { - const result = { score: 1.0 }; + it('defaults checks to empty array when omitted', () => { + const result = { pass: true, score: 1.0, reason: 'All good' }; const parsed = ScriptGraderResultSchema.parse(result); - expect(parsed.assertions).toEqual([]); + expect(parsed.checks).toEqual([]); }); it('rejects score below 0', () => { - const result = { score: -0.5 }; + const result = { pass: false, score: -0.5, reason: 'Too low' }; expect(() => ScriptGraderResultSchema.parse(result)).toThrow(); }); it('rejects score above 1', () => { - const result = { score: 1.5 }; + const result = { pass: true, score: 1.5, reason: 'Too high' }; expect(() => ScriptGraderResultSchema.parse(result)).toThrow(); }); it('accepts boundary scores 0 and 1', () => { - expect(ScriptGraderResultSchema.parse({ score: 0 }).score).toBe(0); - expect(ScriptGraderResultSchema.parse({ score: 1 }).score).toBe(1); + expect(ScriptGraderResultSchema.parse({ pass: false, score: 0, reason: 'Fail' }).score).toBe(0); + expect(ScriptGraderResultSchema.parse({ pass: true, score: 1, reason: 'Pass' }).score).toBe(1); }); it('accepts optional details object', () => { const result = { + pass: true, score: 0.75, + reason: 'Metric details included', details: { tp: 5, tn: 2, @@ -353,14 +359,16 @@ describe('ScriptGraderResultSchema', () => { }); it('allows details to be omitted', () => { - const result = { score: 0.5 }; + const result = { pass: true, score: 0.5, reason: 'No details' }; const parsed = ScriptGraderResultSchema.parse(result); expect(parsed.details).toBeUndefined(); }); it('accepts nested details object', () => { const result = { + pass: true, score: 0.8, + reason: 'Nested details included', details: { alignment: [ { expectedIdx: 0, parsedIdx: 1, similarity: 0.95 }, @@ -376,6 +384,23 @@ describe('ScriptGraderResultSchema', () => { expect(parsed.details?.alignment).toHaveLength(2); expect(parsed.details?.metrics).toBeDefined(); }); + + it('accepts checks with and without scores', () => { + const parsed = ScriptGraderResultSchema.parse({ + pass: false, + score: 0.5, + reason: 'One of two checks passed', + checks: [ + { id: 'format', text: 'Format valid', pass: true, score: 1, reason: 'JSON parsed' }, + { text: 'Content complete', pass: false, reason: 'Missing summary' }, + ], + }); + + expect(parsed.checks).toEqual([ + { id: 'format', text: 'Format valid', pass: true, score: 1, reason: 'JSON parsed' }, + { text: 'Content complete', pass: false, reason: 'Missing summary' }, + ]); + }); }); // --------------------------------------------------------------------------- @@ -397,7 +422,7 @@ describe('CodeJudgeInputSchema (backward-compat alias)', () => { describe('CodeJudgeResultSchema (backward-compat alias)', () => { it('parses valid result via deprecated alias', () => { - const result = { score: 0.8, assertions: [{ text: 'ok', passed: true }] }; + const result = { pass: true, score: 0.8, reason: 'ok' }; const parsed = CodeJudgeResultSchema.parse(result); expect(parsed.score).toBe(0.8); }); diff --git a/packages/sdk/test/vitest-workspace-grader.test.ts b/packages/sdk/test/vitest-workspace-grader.test.ts index ef45da4dd..051578451 100644 --- a/packages/sdk/test/vitest-workspace-grader.test.ts +++ b/packages/sdk/test/vitest-workspace-grader.test.ts @@ -67,15 +67,22 @@ describe('Vitest workspace grader adapter', () => { } }); - it('maps individual Vitest test outcomes to AgentV assertions', () => { + it('maps individual Vitest test outcomes to AgentV checks', () => { const result = vitestReportToScriptGraderResult(mixedVitestReport); expect(result.score).toBe(0.5); - expect(result.assertions).toEqual([ - { text: 'welcome banner contains status', passed: true }, + expect(result.pass).toBe(false); + expect(result.reason).toBe('1/2 Vitest tests passed.'); + expect(result.checks).toEqual([ + { + text: 'welcome banner contains status', + pass: true, + reason: 'Vitest test passed.', + }, { text: 'welcome banner links to dashboard', - passed: false, + pass: false, + reason: 'Vitest test failed.', evidence: 'AssertionError: expected href to equal /dashboard', }, ]); @@ -113,7 +120,7 @@ process.exit(1); ); expect(result.score).toBe(0.5); - expect(result.assertions.map((item) => item.text)).toEqual([ + expect(result.checks.map((item) => item.text)).toEqual([ 'welcome banner contains status', 'welcome banner links to dashboard', ]); @@ -191,7 +198,7 @@ process.exit(1); ); expect(result.score).toBe(0.5); - expect(result.assertions[1].passed).toBe(false); + expect(result.checks[1].pass).toBe(false); }); it('returns a failed AgentV result when workspace_path is unavailable', async () => { @@ -203,9 +210,9 @@ process.exit(1); ); expect(result.score).toBe(0); - expect(result.assertions[0]).toMatchObject({ + expect(result.checks[0]).toMatchObject({ text: 'Vitest workspace verifier requires workspace_path', - passed: false, + pass: false, }); }); }); diff --git a/packages/sdk/test/workspace-grader.test.ts b/packages/sdk/test/workspace-grader.test.ts index b7aab2420..1b9720620 100644 --- a/packages/sdk/test/workspace-grader.test.ts +++ b/packages/sdk/test/workspace-grader.test.ts @@ -39,7 +39,7 @@ describe('workspace grader helpers', () => { } }); - it('runs compact workspace file assertions and aggregates passing checks', async () => { + it('runs compact workspace file checks and aggregates passing checks', async () => { writeFileSync( join(tmpDir, 'app/page.tsx'), '
Status: All systems ready Open dashboard
', @@ -56,11 +56,13 @@ describe('workspace grader helpers', () => { ); expect(result.score).toBe(1); - expect(result.assertions).toHaveLength(4); - expect(result.assertions.every((item) => item.passed)).toBe(true); + expect(result.pass).toBe(true); + expect(result.reason).toBe('4/4 checks passed.'); + expect(result.checks).toHaveLength(4); + expect(result.checks.every((item) => item.pass)).toBe(true); }); - it('scores failed file assertions by passed assertion count', async () => { + it('scores failed file checks by passed check count', async () => { writeFileSync(join(tmpDir, 'app/page.tsx'), '
Hello TODO
'); const result = await runWorkspaceGrader( @@ -74,9 +76,10 @@ describe('workspace grader helpers', () => { ); expect(result.score).toBe(0.25); - expect(result.assertions.map((item) => item.passed)).toEqual([true, false, false, false]); - expect(result.assertions[1].evidence).toContain('Open dashboard'); - expect(result.assertions[3].evidence).toContain('no such file'); + expect(result.pass).toBe(false); + expect(result.checks.map((item) => item.pass)).toEqual([true, false, false, false]); + expect(result.checks[1].reason).toContain('Open dashboard'); + expect(result.checks[3].reason).toContain('no such file'); }); it('uses AGENTV_WORKSPACE_PATH when the stdin payload omits workspacePath', async () => { @@ -85,7 +88,11 @@ describe('workspace grader helpers', () => { const result = await runWorkspaceGrader( async ({ workspace }) => [ - { text: 'Workspace env fallback is exposed', passed: workspace.path === tmpDir }, + { + text: 'Workspace env fallback is exposed', + pass: workspace.path === tmpDir, + reason: 'Workspace path came from AGENTV_WORKSPACE_PATH.', + }, await workspace.file('app/page.tsx').contains('Ready'), ], buildInput(), @@ -94,7 +101,7 @@ describe('workspace grader helpers', () => { expect(result.score).toBe(1); }); - it('returns failed assertions instead of requiring manual workspace path checks', async () => { + it('returns failed checks instead of requiring manual workspace path checks', async () => { process.env.AGENTV_WORKSPACE_PATH = undefined; const result = await runWorkspaceGrader( @@ -103,11 +110,11 @@ describe('workspace grader helpers', () => { ); expect(result.score).toBe(0); - expect(result.assertions[0]).toMatchObject({ + expect(result.checks[0]).toMatchObject({ text: 'app/page.tsx contains "Ready"', - passed: false, + pass: false, }); - expect(result.assertions[0].evidence).toContain('Workspace path is not available'); + expect(result.checks[0].reason).toContain('Workspace path is not available'); }); it('rejects file paths outside the workspace', async () => { @@ -118,23 +125,27 @@ describe('workspace grader helpers', () => { ]); expect(result.score).toBe(0); - expect(result.assertions[0].passed).toBe(false); - expect(result.assertions[0].evidence).toContain('inside the workspace'); + expect(result.checks[0].pass).toBe(false); + expect(result.checks[0].reason).toContain('inside the workspace'); }); it('passes through explicit ScriptGraderResult objects', async () => { const result = await runWorkspaceGrader( () => ({ + pass: true, score: 0.75, - assertions: [{ text: 'custom weighted result', passed: true }], + reason: 'Custom weighted result', + checks: [{ text: 'custom weighted result', pass: true, reason: 'Matched custom rule' }], details: { matched: 3, total: 4 }, }), buildInput({ workspacePath: tmpDir }), ); expect(result).toEqual({ + pass: true, score: 0.75, - assertions: [{ text: 'custom weighted result', passed: true }], + reason: 'Custom weighted result', + checks: [{ text: 'custom weighted result', pass: true, reason: 'Matched custom rule' }], details: { matched: 3, total: 4 }, }); });