diff --git a/README.md b/README.md index 30612b03f..262127dc9 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Plain assertion strings are short-form rubric criteria: AgentV groups them into `llm-rubric` and writes each criterion to `grading.json.assertion_results` for the Dashboard. Use explicit `type: llm-rubric` when you need weights, required flags, or `score_ranges`, or when you need a custom grader prompt, grader target, or -preprocessing; use string `value` for free-form rubric checks. Executable +output transforms; use string `value` for free-form rubric checks. Executable graders use `type: script`. The target can be an eval-local object when this eval needs target settings of its own: diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx index 46f881155..5d3e5e265 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx @@ -416,7 +416,7 @@ default_test: ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) : undefined; if (!file) return output; - const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], { + const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], { stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) }); if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); diff --git a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx index e3a050bf4..aa89c8c48 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx @@ -117,7 +117,7 @@ default_test: ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) : undefined; if (!file) return output; - const result = Bun.spawnSync(["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"], { + const result = Bun.spawnSync(["bun", "run", "../scripts/transforms/xlsx-to-csv.ts"], { stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) }); if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); @@ -133,7 +133,7 @@ tests: - Output contains the transformed spreadsheet text including the revenue rows ``` -See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable end-to-end example with a file-producing target and custom grader target. +See [`examples/features/file-transforms/`](../../../../../examples/features/file-transforms/) for a runnable end-to-end example with a file-producing target and custom grader target. ## Tool Trajectory diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 7b870bf2e..2b2459510 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -12,7 +12,7 @@ LLM graders use a language model to evaluate agent responses against custom crit Put semantic grading requirements in `assert`. Plain strings are handled by the built-in `llm-rubric` rubric grader. Use `type: llm-rubric` when you -need a custom prompt, target, or grader-specific preprocessing: +need a custom prompt, target, or grader-specific transform: ```yaml tests: @@ -207,7 +207,7 @@ default_test: ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) : undefined; if (!file) return output; - const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], { + const result = Bun.spawnSync(["bun", "run", "scripts/transforms/xlsx-to-csv.ts"], { stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) }); if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); @@ -235,7 +235,7 @@ Resolution order: - if no transform is configured, AgentV falls back to a UTF-8 text read - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run -See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a spreadsheet conversion script. +See [`examples/features/file-transforms/`](../../../../../examples/features/file-transforms/) for a runnable example with a file-producing target and a spreadsheet conversion script. ## Available Context Fields diff --git a/examples/features/README.md b/examples/features/README.md index 8921a9e48..46504dfc4 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -22,7 +22,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [assert-set](assert-set/) | Safety gate and weighted assertion groups | | [threshold-grader](threshold-grader/) | Pass a test if a configurable percentage of sub-graders pass | | [multi-turn-conversation](multi-turn-conversation/) | Grade a multi-turn conversation with per-turn score breakdowns | -| [preprocessors](preprocessors/) | Convert `ContentFile` outputs with `default_test.options.transform` before `llm-rubric` runs | +| [file-transforms](file-transforms/) | Convert `ContentFile` outputs with `default_test.options.transform` before `llm-rubric` runs | --- @@ -159,7 +159,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [local-cli](local-cli/) | Workspace & targets | | [multi-turn-conversation](multi-turn-conversation/) | LLM grading | | [nlp-metrics](nlp-metrics/) | Deterministic assertions | -| [preprocessors](preprocessors/) | LLM grading | +| [file-transforms](file-transforms/) | LLM grading | | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK | | [repo-lifecycle](repo-lifecycle/) | Workspace & targets | | [rubric](rubric/) | LLM grading | diff --git a/examples/features/preprocessors/.agentv/providers/file-output.ts b/examples/features/file-transforms/.agentv/providers/file-output.ts similarity index 100% rename from examples/features/preprocessors/.agentv/providers/file-output.ts rename to examples/features/file-transforms/.agentv/providers/file-output.ts diff --git a/examples/features/preprocessors/.agentv/providers/grader-check.ts b/examples/features/file-transforms/.agentv/providers/grader-check.ts similarity index 91% rename from examples/features/preprocessors/.agentv/providers/grader-check.ts rename to examples/features/file-transforms/.agentv/providers/grader-check.ts index 130991ffc..4ea2653ce 100644 --- a/examples/features/preprocessors/.agentv/providers/grader-check.ts +++ b/examples/features/file-transforms/.agentv/providers/grader-check.ts @@ -17,7 +17,7 @@ writeFileSync( score: passed ? 1 : 0, assertions: [ { - text: 'preprocessed file content reached the llm grader', + text: 'transformed file content reached the llm grader', passed, evidence: passed ? 'found transformed spreadsheet text in prompt' diff --git a/examples/features/preprocessors/.agentv/targets.yaml b/examples/features/file-transforms/.agentv/targets.yaml similarity index 100% rename from examples/features/preprocessors/.agentv/targets.yaml rename to examples/features/file-transforms/.agentv/targets.yaml diff --git a/examples/features/preprocessors/README.md b/examples/features/file-transforms/README.md similarity index 83% rename from examples/features/preprocessors/README.md rename to examples/features/file-transforms/README.md index de63197fa..13dddebd8 100644 --- a/examples/features/preprocessors/README.md +++ b/examples/features/file-transforms/README.md @@ -13,7 +13,7 @@ Demonstrates how `default_test.options.transform` turns `ContentFile` outputs in ```bash # From repository root -bun apps/cli/src/cli.ts eval examples/features/preprocessors/evals/suite.yaml --target file_output +bun apps/cli/src/cli.ts eval examples/features/file-transforms/evals/suite.yaml --target file_output ``` Expected result: the eval passes because the grader sees the transformed spreadsheet text from `generated/report.xlsx`. @@ -24,4 +24,4 @@ Expected result: the eval passes because the grader sees the transformed spreads - `.agentv/targets.yaml` - custom file-producing target and custom grader target - `.agentv/providers/file-output.ts` - emits a relative `ContentFile` path - `.agentv/providers/grader-check.ts` - passes only when transformed text reaches the grader prompt -- `scripts/preprocessors/xlsx-to-csv.ts` - example spreadsheet conversion script +- `scripts/transforms/xlsx-to-csv.ts` - example spreadsheet conversion script diff --git a/examples/features/preprocessors/evals/suite.yaml b/examples/features/file-transforms/evals/suite.yaml similarity index 90% rename from examples/features/preprocessors/evals/suite.yaml rename to examples/features/file-transforms/evals/suite.yaml index 205e6d0d8..01169ca68 100644 --- a/examples/features/preprocessors/evals/suite.yaml +++ b/examples/features/file-transforms/evals/suite.yaml @@ -21,11 +21,11 @@ default_test: return matchers.some((matcher) => mediaType === matcher || filePath.endsWith(matcher)); }); if (!file || typeof file.path !== "string") return output; - const result = Bun.spawnSync(["bun","run","../scripts/preprocessors/xlsx-to-csv.ts"], { + const result = Bun.spawnSync(["bun","run","../scripts/transforms/xlsx-to-csv.ts"], { stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) }); if (result.exitCode !== 0) { - throw new Error(new TextDecoder().decode(result.stderr).trim() || "preprocessor command failed"); + throw new Error(new TextDecoder().decode(result.stderr).trim() || "transform command failed"); } return new TextDecoder().decode(result.stdout).trim(); })() diff --git a/examples/features/preprocessors/scripts/preprocessors/xlsx-to-csv.ts b/examples/features/file-transforms/scripts/transforms/xlsx-to-csv.ts similarity index 100% rename from examples/features/preprocessors/scripts/preprocessors/xlsx-to-csv.ts rename to examples/features/file-transforms/scripts/transforms/xlsx-to-csv.ts diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 4e0f7f116..61b6b0bbe 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -569,13 +569,7 @@ async function parseGraderList( const pushEvaluator = (config: GraderConfig): void => { evaluators.push(transform !== undefined ? { ...config, transform } : config); }; - const mergedPreprocessors = await parseMergedPreprocessors( - rawEvaluator.preprocessors as JsonValue | undefined, - defaultPreprocessors, - searchRoots, - name, - evalId, - ); + const inheritedInternalPreprocessors = defaultPreprocessors; // Custom assertion types — store with their type name for registry dispatch if (isCustomType) { @@ -788,7 +782,9 @@ async function parseGraderList( ...(min_score !== undefined ? { min_score } : {}), ...(negate !== undefined ? { negate } : {}), ...(Object.keys(mergedConfig).length > 0 ? { config: mergedConfig } : {}), - ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), + ...(inheritedInternalPreprocessors + ? { preprocessors: inheritedInternalPreprocessors } + : {}), ...(targetConfig !== undefined ? { target: targetConfig } : {}), }); continue; @@ -1684,7 +1680,9 @@ async function parseGraderList( ...(finalConfig ? { config: finalConfig } : {}), ...(llmMaxSteps !== undefined ? { max_steps: llmMaxSteps } : {}), ...(llmTemperature !== undefined ? { temperature: llmTemperature } : {}), - ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), + ...(inheritedInternalPreprocessors + ? { preprocessors: inheritedInternalPreprocessors } + : {}), }); continue; } @@ -1705,7 +1703,7 @@ async function parseGraderList( ...(finalConfig ? { config: finalConfig } : {}), ...(llmMaxSteps !== undefined ? { max_steps: llmMaxSteps } : {}), ...(llmTemperature !== undefined ? { temperature: llmTemperature } : {}), - ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), + ...(inheritedInternalPreprocessors ? { preprocessors: inheritedInternalPreprocessors } : {}), }); } diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 8bb595b24..1a90a79ca 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -963,7 +963,7 @@ export interface EvalSourceReference { | 'code_grader_cwd' | 'assertion_template' | 'default_test' - | 'preprocessor_command'; + | 'content_transform_command'; readonly displayPath: string; readonly resolvedPath?: string; readonly graderName?: string; @@ -1063,7 +1063,7 @@ export interface EvalTest { readonly vars?: JsonObject; /** Promptfoo-compatible output transform inherited from default_test.options or tests[].options. */ readonly outputTransform?: TransformSpec; - /** Suite-level preprocessors used by the implicit default llm-grader. */ + /** Internal content conversion commands for file blocks. Authored eval YAML uses transform. */ readonly preprocessors?: readonly ContentPreprocessorConfig[]; /** Promptfoo-style lifecycle extensions inherited from the suite. */ readonly extensions?: readonly AgentVExtensionConfig[]; diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 1a04143e3..b72a93a5c 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -49,7 +49,6 @@ import { coerceEvaluator, collectAssertionTemplateSourceReferences, parseGraders, - parsePreprocessors, warnUnconsumedCriteria, } from './loaders/grader-parser.js'; import { detectFormat, loadTestsFromJsonl } from './loaders/jsonl-parser.js'; @@ -1278,12 +1277,6 @@ async function loadTestsFromParsedYamlValue( const evalFileDir = path.dirname(absoluteTestPath); const globalEvaluator = coerceEvaluator(suite.evaluator, 'global'); - const suitePreprocessors = await parsePreprocessors( - suite.preprocessors, - searchRoots, - '', - absoluteTestPath, - ); const defaultTestRubricPrompt = extractDefaultTestRubricPrompt(suite); const suiteExtensions = parseExtensions(suite.extensions, evalFileDir); @@ -1554,7 +1547,7 @@ async function loadTestsFromParsedYamlValue( globalExecution, searchRoots, id ?? 'unknown', - suitePreprocessors, + undefined, defaultTestRubricPrompt, ); } catch (error) { @@ -1650,7 +1643,6 @@ async function loadTestsFromParsedYamlValue( assertions: evaluators, ...(caseVars ? { vars: caseVars } : {}), ...(outputTransform ? { outputTransform } : {}), - ...(suitePreprocessors ? { preprocessors: suitePreprocessors } : {}), ...(suiteExtensions.length > 0 ? { extensions: suiteExtensions } : {}), workspace: mergedWorkspace, metadata, @@ -2694,7 +2686,7 @@ function collectSingleGraderSourceReferences( for (const preprocessor of preprocessors ?? []) { if (preprocessor.resolvedCommand && preprocessor.resolvedCommand.length > 0) { references.push({ - kind: 'preprocessor_command', + kind: 'content_transform_command', displayPath: preprocessor.resolvedCommand.at(-1) ?? preprocessor.type, resolvedPath: preprocessor.resolvedCommand.at(-1), graderName: evaluator.name, diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 068809aa3..c8baacf96 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -199,7 +199,7 @@ export default defineEval({ }); ``` -The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`. +The helpers return ordinary `assert` entries such as `type: contains`, `type: llm-rubric`, and `type: script`. Use the shared `transform` option for assertion-level output shaping. CamelCase SDK options such as `minScore` and `maxSteps` lower to canonical YAML keys such as `min_score` and `max_steps`. If you are coming from Braintrust `scores` or DeepEval metrics, model reusable checks as small AgentV-native helper factories that return these grader configs. They still lower to the same YAML/runtime contract: diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 686340774..762ff42bb 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -75,12 +75,6 @@ export interface EvalAssertionConfig { readonly [key: string]: unknown; } -export interface EvalPreprocessor { - readonly type: string; - readonly command: string | readonly string[]; - readonly [key: string]: unknown; -} - export interface EvalWorkspaceHook { readonly command?: string | readonly string[]; readonly timeoutMs?: number; @@ -229,7 +223,6 @@ export interface EvalDefinition { readonly threshold?: number; readonly budgetUsd?: number; readonly assert?: readonly EvalAssertionConfig[]; - readonly preprocessors?: readonly EvalPreprocessor[]; readonly workspace?: EvalWorkspace | string; } @@ -314,6 +307,11 @@ function validateTopLevelRuntimeFields(definition: EvalDefinition): void { "defineEval() does not accept top-level 'input'. Use prompts with default test vars or tests[].vars instead.", ); } + if (Object.prototype.hasOwnProperty.call(rawDefinition, 'preprocessors')) { + throw new Error( + "defineEval() does not accept top-level 'preprocessors'. Use defaultTest.options.transform or assertion-level transform instead.", + ); + } if ( Object.prototype.hasOwnProperty.call(rawDefinition, 'experiment') && typeof rawDefinition.experiment !== 'string' diff --git a/packages/sdk/src/graders.ts b/packages/sdk/src/graders.ts index b61142282..c40d9e9dc 100644 --- a/packages/sdk/src/graders.ts +++ b/packages/sdk/src/graders.ts @@ -1,4 +1,4 @@ -import type { EvalAssertionConfig, EvalPreprocessor } from './eval.js'; +import type { EvalAssertionConfig } from './eval.js'; export type GraderCommand = string | readonly string[]; @@ -8,6 +8,7 @@ export interface GraderHelperOptions { readonly required?: boolean; readonly minScore?: number; readonly negate?: boolean; + readonly transform?: string; } export interface GraderCommonConfig { @@ -16,6 +17,7 @@ export interface GraderCommonConfig { readonly required?: boolean; readonly minScore?: number; readonly negate?: boolean; + readonly transform?: string; } export interface ContainsGraderConfig extends EvalAssertionConfig, GraderCommonConfig { @@ -70,7 +72,6 @@ export interface LlmRubricGraderConfig extends EvalAssertionConfig, GraderCommon readonly config?: Readonly>; readonly maxSteps?: number; readonly temperature?: number; - readonly preprocessors?: readonly EvalPreprocessor[]; } export interface GraderPromptScriptConfig { @@ -85,7 +86,6 @@ export interface LlmGraderOptions extends GraderHelperOptions { readonly config?: Readonly>; readonly maxSteps?: number; readonly temperature?: number; - readonly preprocessors?: readonly EvalPreprocessor[]; } export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig { @@ -96,7 +96,6 @@ export interface LlmGraderConfig extends EvalAssertionConfig, GraderCommonConfig readonly config?: Readonly>; readonly maxSteps?: number; readonly temperature?: number; - readonly preprocessors?: readonly EvalPreprocessor[]; } export interface ScriptGraderTargetOptions { @@ -107,7 +106,6 @@ export interface ScriptGraderOptions extends GraderHelperOptions { readonly cwd?: string; readonly target?: true | ScriptGraderTargetOptions; readonly config?: Readonly>; - readonly preprocessors?: readonly EvalPreprocessor[]; } export interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonConfig { @@ -116,7 +114,6 @@ export interface ScriptGraderConfig extends EvalAssertionConfig, GraderCommonCon readonly cwd?: string; readonly target?: true | ScriptGraderTargetOptions; readonly config?: Readonly>; - readonly preprocessors?: readonly EvalPreprocessor[]; } /** @deprecated Use ScriptGraderTargetOptions. */ @@ -148,6 +145,7 @@ function withCommon( ...(options.required !== undefined ? { required: options.required } : {}), ...(options.minScore !== undefined ? { minScore: options.minScore } : {}), ...(options.negate !== undefined ? { negate: options.negate } : {}), + ...(options.transform !== undefined ? { transform: options.transform } : {}), } as T & GraderCommonConfig & EvalAssertionConfig; } @@ -196,7 +194,6 @@ export function llmRubricGrader( readonly config?: Readonly>; readonly maxSteps?: number; readonly temperature?: number; - readonly preprocessors?: readonly EvalPreprocessor[]; } = {}, ): LlmRubricGraderConfig { return withCommon( @@ -208,7 +205,6 @@ export function llmRubricGrader( ...(options.config !== undefined ? { config: options.config } : {}), ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}), ...(options.temperature !== undefined ? { temperature: options.temperature } : {}), - ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}), }, options, ); @@ -224,7 +220,6 @@ export function llmGrader(options: LlmGraderOptions = {}): LlmGraderConfig { ...(options.config !== undefined ? { config: options.config } : {}), ...(options.maxSteps !== undefined ? { maxSteps: options.maxSteps } : {}), ...(options.temperature !== undefined ? { temperature: options.temperature } : {}), - ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}), }, options, ); @@ -249,7 +244,6 @@ export function scriptGrader( ...(options.cwd !== undefined ? { cwd: options.cwd } : {}), ...(options.target !== undefined ? { target: options.target } : {}), ...(options.config !== undefined ? { config: options.config } : {}), - ...(options.preprocessors !== undefined ? { preprocessors: options.preprocessors } : {}), }, options, ); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index d492586b1..c4875ea0e 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -145,7 +145,6 @@ export { type EvalExecution, type EvalMessage, type EvalMessageContent, - type EvalPreprocessor, type EvalRequires, type EvalTargetRef, type EvalTest, diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 3dad0c54a..6fceee2a8 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -261,6 +261,17 @@ describe('YAML-aligned eval authoring helpers', () => { ).toThrow(/tests\[0\]\.input/); }); + it('rejects removed public preprocessor authoring', () => { + expect(() => + defineEval({ + name: 'removed-preprocessors', + prompts: ['{{ input }}'], + preprocessors: [{ type: 'xlsx', command: ['bun', 'run', 'xlsx-to-text.ts'] }], + tests: [{ id: 'hello', vars: { input: 'Say hello' } }], + } as never), + ).toThrow(/top-level 'preprocessors'.*defaultTest\.options\.transform/); + }); + it('rejects removed experiment authoring blocks', () => { expect(() => defineEval({ diff --git a/packages/sdk/test/grader-helpers.test.ts b/packages/sdk/test/grader-helpers.test.ts index b391f11bd..33a40bd49 100644 --- a/packages/sdk/test/grader-helpers.test.ts +++ b/packages/sdk/test/grader-helpers.test.ts @@ -18,11 +18,18 @@ import { describe('grader helper config builders', () => { it('returns existing AgentV assertion/evaluator config shapes', () => { expect(containsGrader('Hello')).toEqual({ type: 'contains', value: 'Hello' }); - expect(equalsGrader('exact answer', { metric: 'exact-answer', minScore: 1 })).toEqual({ + expect( + equalsGrader('exact answer', { + metric: 'exact-answer', + minScore: 1, + transform: 'output.trim()', + }), + ).toEqual({ metric: 'exact-answer', type: 'equals', value: 'exact answer', minScore: 1, + transform: 'output.trim()', }); expect(exactGrader('same answer')).toEqual({ type: 'equals', value: 'same answer' }); expect(regexGrader(/hello\s+world/i, { metric: 'hello-pattern' })).toEqual({ @@ -108,6 +115,7 @@ describe('grader helper config builders', () => { prompt: 'Grade whether the answer is useful.', target: 'grader-target', maxSteps: 2, + transform: 'output.trim()', }), graders.script(['bun', 'run', 'graders/check.ts'], { metric: 'scripted-check', @@ -150,6 +158,7 @@ describe('grader helper config builders', () => { prompt: 'Grade whether the answer is useful.', target: 'grader-target', max_steps: 2, + transform: 'output.trim()', }, { metric: 'scripted-check', @@ -167,6 +176,7 @@ describe('grader helper config builders', () => { expect(yaml).toContain('type: llm-rubric'); expect(yaml).toContain('type: script'); expect(yaml).toContain('max_steps: 2'); + expect(yaml).toContain('transform: output.trim()'); expect(yaml).toContain('max_calls: 2'); expect(yaml).toContain('min_score: 0.8'); expect(yaml).toContain('score_range:'); diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 1a1db5a91..ede8a97f4 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -40,7 +40,7 @@ Use `@agentv/sdk` for TypeScript helper imports. Do not use `@agentv/eval` for n ## Authoring Checklist - Put grading criteria in `assert`, not in test-level `criteria`. Plain assertion strings become an `llm-rubric` grader. -- Prefer plain assertion strings for semantic checks when the default rubric grader can judge them. Use `type: llm-rubric` for structured criteria, custom prompts, custom grader targets, or preprocessing, and `type: script` when grading must execute code. +- Prefer plain assertion strings for semantic checks when the default rubric grader can judge them. Use `type: llm-rubric` for structured criteria, custom prompts, custom grader targets, or assertion-level transforms, and `type: script` when grading must execute code. - Write `expected_output` as a golden/reference answer the target could have produced. Do not write criteria, scoring instructions, or "the agent should..." rubric prose there. - For historical or repo-state evals, materialize the repo under `workspace.repos[]` pinned to the commit under test. Mentioning a SHA only in prompt prose is not enough because the agent needs an actual checkout to inspect.