From cfd94a9f1a6f9a573c5be756af1322932dddcfe5 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 13:40:01 +0200 Subject: [PATCH 1/2] feat(core): add promptfoo transform runtime --- packages/core/src/evaluation/graders/types.ts | 2 + .../src/evaluation/loaders/grader-parser.ts | 73 ++-- .../evaluation/loaders/transform-parser.ts | 55 +++ packages/core/src/evaluation/orchestrator.ts | 179 ++++++++- .../core/src/evaluation/output-transform.ts | 129 +++++++ packages/core/src/evaluation/types.ts | 7 + packages/core/src/evaluation/yaml-parser.ts | 78 ++++ .../test/evaluation/transform-runtime.test.ts | 343 ++++++++++++++++++ 8 files changed, 832 insertions(+), 34 deletions(-) create mode 100644 packages/core/src/evaluation/loaders/transform-parser.ts create mode 100644 packages/core/src/evaluation/output-transform.ts create mode 100644 packages/core/test/evaluation/transform-runtime.test.ts diff --git a/packages/core/src/evaluation/graders/types.ts b/packages/core/src/evaluation/graders/types.ts index f915a27ed..8ccc88476 100644 --- a/packages/core/src/evaluation/graders/types.ts +++ b/packages/core/src/evaluation/graders/types.ts @@ -21,6 +21,8 @@ export type TargetResolver = (targetName: string) => Provider | undefined; export interface EvaluationContext { readonly evalCase: EvalTest; readonly candidate: string; + /** Raw transformed output value before string coercion, for assertion-level transforms. */ + readonly candidateValue?: unknown; readonly target: ResolvedTarget; readonly provider: Provider; readonly attempt: number; diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index c5daaf755..942d3481f 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -17,6 +17,7 @@ import { RUBRIC_OPERATOR_VALUES, isGraderKind } from '../types.js'; import { validateCustomPromptContent } from '../validation/prompt-validator.js'; import { parseYamlValue } from '../yaml-loader.js'; import { resolveFileReference } from './file-resolver.js'; +import { parseTransformSpec } from './transform-parser.js'; const ANSI_YELLOW = '\u001b[33m'; const ANSI_RESET = '\u001b[0m'; @@ -550,6 +551,19 @@ async function parseGraderList( } const negate = rawEvaluator.negate === true ? true : undefined; + if (rawEvaluator.postprocess !== undefined) { + throw new Error( + `Grader '${name}' in '${evalId}': postprocess has been removed. Use transform instead.`, + ); + } + const transform = await parseTransformSpec( + rawEvaluator.transform as JsonValue | undefined, + searchRoots, + `Grader '${name}' in '${evalId}'`, + ); + const pushEvaluator = (config: GraderConfig): void => { + evaluators.push(transform !== undefined ? { ...config, transform } : config); + }; const mergedPreprocessors = await parseMergedPreprocessors( rawEvaluator.preprocessors as JsonValue | undefined, defaultPreprocessors, @@ -568,14 +582,23 @@ async function parseGraderList( evalId, ); // Collect all properties except known meta-keys as pass-through config - const knownProps = new Set(['metric', 'type', 'weight', 'required', 'min_score', 'negate']); + const knownProps = new Set([ + 'metric', + 'type', + 'weight', + 'required', + 'min_score', + 'negate', + 'transform', + 'postprocess', + ]); const config: Record = {}; for (const [key, value] of Object.entries(rawEvaluator)) { if (!knownProps.has(key) && value !== undefined) { config[key] = value as JsonValue; } } - evaluators.push({ + pushEvaluator({ name, type: customTypeName as unknown as GraderKind, ...(weight !== undefined ? { weight } : {}), @@ -623,7 +646,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'assert-set', assertions: parsedMembers, @@ -734,6 +757,8 @@ async function parseGraderList( 'required', 'min_score', 'negate', + 'transform', + 'postprocess', ]); const config: Record = {}; for (const [key, value] of Object.entries(rawEvaluator)) { @@ -746,7 +771,7 @@ async function parseGraderList( : {}; const mergedConfig = { ...config, ...topLevelConfig }; - evaluators.push({ + pushEvaluator({ name, type: 'script', command, @@ -927,7 +952,7 @@ async function parseGraderList( ...(argsMatch !== undefined ? { argsMatch } : {}), }; - evaluators.push(config); + pushEvaluator(config); continue; } @@ -1006,7 +1031,7 @@ async function parseGraderList( evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'field-accuracy', fields, @@ -1036,7 +1061,7 @@ async function parseGraderList( evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'latency', threshold, @@ -1065,7 +1090,7 @@ async function parseGraderList( evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'cost', budget, @@ -1120,7 +1145,7 @@ async function parseGraderList( evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'token-usage', ...validLimits, @@ -1205,7 +1230,7 @@ async function parseGraderList( evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'execution-metrics', ...validThresholds, @@ -1232,7 +1257,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'skill-trigger', skill: skillName, @@ -1265,7 +1290,7 @@ async function parseGraderList( evalId, ); const config = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined; - evaluators.push({ + pushEvaluator({ name, type: typeValue, value, @@ -1303,7 +1328,7 @@ async function parseGraderList( ? rawEvaluator.provider : undefined; const config = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined; - evaluators.push({ + pushEvaluator({ name, type: 'similar', value, @@ -1331,7 +1356,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'contains', value, @@ -1358,7 +1383,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: typeValue, value, @@ -1383,7 +1408,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'icontains', value, @@ -1410,7 +1435,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: typeValue, value, @@ -1435,7 +1460,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: typeValue, value, @@ -1461,7 +1486,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'regex', value, @@ -1482,7 +1507,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'is-json', ...(weight !== undefined ? { weight } : {}), @@ -1506,7 +1531,7 @@ async function parseGraderList( name, evalId, ); - evaluators.push({ + pushEvaluator({ name, type: 'equals', value, @@ -1572,6 +1597,8 @@ async function parseGraderList( 'maxSteps', 'temperature', 'preprocessors', + 'transform', + 'postprocess', ]); const config: Record = {}; for (const [key, value] of Object.entries(rawEvaluator)) { @@ -1633,7 +1660,7 @@ async function parseGraderList( continue; } - evaluators.push({ + pushEvaluator({ name, type: 'llm-rubric', prompt, @@ -1657,7 +1684,7 @@ async function parseGraderList( continue; } - evaluators.push({ + pushEvaluator({ name, type: 'llm-grader', prompt, diff --git a/packages/core/src/evaluation/loaders/transform-parser.ts b/packages/core/src/evaluation/loaders/transform-parser.ts new file mode 100644 index 000000000..37721a407 --- /dev/null +++ b/packages/core/src/evaluation/loaders/transform-parser.ts @@ -0,0 +1,55 @@ +import path from 'node:path'; + +import type { TransformSpec } from '../output-transform.js'; +import type { JsonValue } from '../types.js'; +import { resolveFileReference } from './file-resolver.js'; + +const FILE_PREFIX = 'file://'; + +function splitFileTransform(value: string): { + readonly filePath: string; + readonly functionName?: string; +} { + const rawPath = value.slice(FILE_PREFIX.length); + const lastColon = rawPath.lastIndexOf(':'); + if (lastColon > 1) { + return { + filePath: rawPath.slice(0, lastColon), + functionName: rawPath.slice(lastColon + 1), + }; + } + return { filePath: rawPath }; +} + +export async function parseTransformSpec( + rawValue: JsonValue | undefined, + searchRoots: readonly string[], + label: string, +): Promise { + if (rawValue === undefined) { + return undefined; + } + if (typeof rawValue !== 'string' || rawValue.trim().length === 0) { + throw new Error(`${label}: transform must be a non-empty string`); + } + + const trimmed = rawValue.trim(); + if (!trimmed.startsWith(FILE_PREFIX)) { + return trimmed; + } + + const { filePath, functionName } = splitFileTransform(trimmed); + const resolved = await resolveFileReference(filePath, searchRoots); + if (!resolved.resolvedPath) { + throw new Error( + `${label}: transform file not found: ${resolved.displayPath}${ + resolved.attempted.length > 0 + ? `\n${resolved.attempted.map((attempt) => ` Tried: ${attempt}`).join('\n')}` + : '' + }`, + ); + } + + const absolutePath = path.resolve(resolved.resolvedPath); + return `${FILE_PREFIX}${absolutePath}${functionName ? `:${functionName}` : ''}`; +} diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 2e38d7192..2d97ddbd3 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -16,6 +16,12 @@ import { negateScore, scoreToVerdict, } from './graders.js'; +import { + type TransformContext, + applyTransform, + contentToTransformInput, + stringifyTransformOutput, +} from './output-transform.js'; import { createBuiltinProviderRegistry, createProvider } from './providers/index.js'; import { discoverProviders } from './providers/provider-discovery.js'; import { @@ -659,12 +665,19 @@ export async function gradePreparedEvalCase( const outputMessages: readonly Message[] = preparedTrace?.messages ?? (candidate.length > 0 ? [{ role: 'assistant' as const, content: candidate }] : []); + const preparedCandidate = await prepareCandidateForGrading({ + evalCase, + candidate, + output: outputMessages, + promptInputs, + provider, + }); const resultTrace = preparedTrace ?? buildTraceFromMessages({ input, output: outputMessages, - finalOutput: candidate, + finalOutput: preparedCandidate.candidate, provider: provider.kind, target: target.name, testId: authoredResultTestId(evalCase), @@ -673,9 +686,17 @@ export async function gradePreparedEvalCase( try { const gradeStartedAt = nowFn(); + const gradingOutput = evalCase.outputTransform + ? ([ + { role: 'assistant' as const, content: preparedCandidate.candidate }, + ] satisfies readonly Message[]) + : preparedTrace + ? outputMessages + : undefined; const { score, scores } = await runEvaluatorsForCase({ evalCase, - candidate, + candidate: preparedCandidate.candidate, + candidateValue: preparedCandidate.candidateValue, target, provider, evaluators: evaluatorRegistry, @@ -684,7 +705,7 @@ export async function gradePreparedEvalCase( promptInputs, now: gradeStartedAt, agentTimeoutMs, - output: preparedTrace ? outputMessages : undefined, + output: gradingOutput, trace: preparedTrace ? resultTrace : undefined, costUsd: preparedTrace ? resultTrace.costUsd : undefined, durationMs: preparedTrace ? resultTrace.durationMs : undefined, @@ -722,7 +743,7 @@ export async function gradePreparedEvalCase( assertions: score.assertions, target: target.name, input, - output: candidate, + output: preparedCandidate.candidate, scores, trace: resultTrace, fileChanges, @@ -2521,6 +2542,76 @@ async function runEvalCaseWithTrials( }; } +function lastAssistantTransformInput( + output: readonly Message[] | undefined, + candidate: string, +): unknown { + if (!output || output.length === 0) { + return candidate; + } + for (let index = output.length - 1; index >= 0; index--) { + const message = output[index]; + if (message.role === 'assistant' && message.content !== undefined) { + return contentToTransformInput(message.content); + } + } + return candidate; +} + +function buildTransformContext(options: { + readonly evalCase: EvalTest; + readonly promptInputs: PromptInputs; + readonly provider: Provider; +}): TransformContext { + const { evalCase, promptInputs, provider } = options; + return { + ...(evalCase.vars ? { vars: evalCase.vars } : {}), + prompt: { + ...(evalCase.prompt?.id ? { id: evalCase.prompt.id } : {}), + ...(evalCase.prompt?.label ? { label: evalCase.prompt.label } : {}), + raw: promptInputs.question, + }, + ...(evalCase.metadata ? { metadata: evalCase.metadata } : {}), + provider: { + id: provider.id, + kind: provider.kind, + target: provider.targetName, + }, + }; +} + +async function prepareCandidateForGrading(options: { + readonly evalCase: EvalTest; + readonly candidate: string; + readonly output?: readonly Message[]; + readonly promptInputs: PromptInputs; + readonly provider: Provider; +}): Promise<{ + readonly candidate: string; + readonly candidateValue: unknown; + readonly rawCandidateValue: unknown; +}> { + const rawCandidateValue = lastAssistantTransformInput(options.output, options.candidate); + if (!options.evalCase.outputTransform) { + return { + candidate: options.candidate, + candidateValue: rawCandidateValue, + rawCandidateValue, + }; + } + + const transformed = await applyTransform( + options.evalCase.outputTransform, + rawCandidateValue, + buildTransformContext(options), + ); + return { + candidate: stringifyTransformOutput(transformed.value), + candidateValue: transformed.value, + rawCandidateValue, + }; +} + async function evaluateCandidate(options: { readonly evalCase: EvalTest; readonly candidate: string; @@ -2583,13 +2674,20 @@ async function evaluateCandidate(options: { dependencyResults, } = options; + const preparedCandidate = await prepareCandidateForGrading({ + evalCase, + candidate, + output, + promptInputs, + provider, + }); const input = buildResultInput(promptInputs); const outputMessages = output ?? [{ role: 'assistant' as const, content: candidate }]; const evaluationTrace = buildTraceFromMessages({ input, output: outputMessages, summary: trace, - finalOutput: candidate, + finalOutput: preparedCandidate.candidate, tokenUsage, costUsd, durationMs, @@ -2602,9 +2700,15 @@ async function evaluateCandidate(options: { }); const gradeTimestamp = nowFn(); + const gradingOutput = evalCase.outputTransform + ? ([ + { role: 'assistant' as const, content: preparedCandidate.candidate }, + ] satisfies readonly Message[]) + : output; const { score, scores } = await runEvaluatorsForCase({ evalCase, - candidate, + candidate: preparedCandidate.candidate, + candidateValue: preparedCandidate.candidateValue, target, provider, evaluators, @@ -2614,7 +2718,7 @@ async function evaluateCandidate(options: { now: gradeTimestamp, graderProvider, agentTimeoutMs, - output, + output: gradingOutput, trace: evaluationTrace, costUsd, durationMs, @@ -2684,7 +2788,7 @@ async function evaluateCandidate(options: { endTime, requests, input, - output: candidate, + output: preparedCandidate.candidate, scores: scores, trace: evaluationTrace, rawProviderLogPath, @@ -2696,6 +2800,7 @@ async function evaluateCandidate(options: { async function runEvaluatorsForCase(options: { readonly evalCase: EvalTest; readonly candidate: string; + readonly candidateValue?: unknown; readonly target: ResolvedTarget; readonly provider: Provider; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; @@ -2724,6 +2829,7 @@ async function runEvaluatorsForCase(options: { const { evalCase, candidate, + candidateValue, target, provider, evaluators, @@ -2758,6 +2864,7 @@ async function runEvaluatorsForCase(options: { evalCase, evaluators: evalCase.assertions, candidate, + candidateValue, target, provider, evaluatorRegistry: evaluators, @@ -2814,6 +2921,7 @@ async function runEvaluatorsForCase(options: { const score = await activeEvaluator.evaluate({ evalCase, candidate, + candidateValue, target, provider, attempt, @@ -2852,10 +2960,50 @@ function buildImplicitLlmGraderConfig(evalCase: EvalTest): LlmGraderConfig | und }; } +async function transformEvaluationContextForGrader( + context: import('./graders/types.js').EvaluationContext, + evaluatorConfig: GraderConfig, +): Promise<{ + readonly context: import('./graders/types.js').EvaluationContext; + readonly input?: JsonObject; + readonly details?: JsonObject; +}> { + if (!evaluatorConfig.transform) { + return { context }; + } + + const transformed = await applyTransform( + evaluatorConfig.transform, + context.candidateValue ?? context.candidate, + buildTransformContext({ + evalCase: context.evalCase, + promptInputs: context.promptInputs, + provider: context.provider, + }), + ); + const transformedCandidate = stringifyTransformOutput(transformed.value); + const transformDetails = { + transform: { + input: transformed.input as JsonValue, + output: transformed.value as JsonValue, + }, + } as JsonObject; + return { + context: { + ...context, + candidate: transformedCandidate, + candidateValue: transformed.value, + }, + input: transformDetails, + details: transformDetails, + }; +} + async function runEvaluatorList(options: { readonly evalCase: EvalTest; readonly evaluators: readonly GraderConfig[]; readonly candidate: string; + readonly candidateValue?: unknown; readonly target: ResolvedTarget; readonly provider: Provider; readonly evaluatorRegistry: Partial> & { @@ -2887,6 +3035,7 @@ async function runEvaluatorList(options: { evalCase, evaluators, candidate, + candidateValue, target, provider, evaluatorRegistry, @@ -2926,6 +3075,7 @@ async function runEvaluatorList(options: { const evalContext: import('./graders/types.js').EvaluationContext = { evalCase, candidate, + candidateValue, target, provider, attempt, @@ -2965,7 +3115,11 @@ async function runEvaluatorList(options: { try { // Create evaluator instance via registry const evaluatorInstance = await typeRegistry.create(evaluatorConfig, dispatchContext); - const score = await evaluatorInstance.evaluate(evalContext); + const transformedContext = await transformEvaluationContextForGrader( + evalContext, + evaluatorConfig, + ); + const score = await evaluatorInstance.evaluate(transformedContext.context); const endedAt = new Date(); const weight = evaluatorConfig.weight ?? 1.0; @@ -2987,9 +3141,12 @@ async function runEvaluatorList(options: { weight, verdict: score.verdict, assertions: score.assertions, - input: score.graderRawRequest, + input: transformedContext.input ?? score.graderRawRequest, target: score.graderTarget, - details: score.details, + details: + transformedContext.details && score.details + ? { ...transformedContext.details, ...score.details } + : (transformedContext.details ?? score.details), scores: mapChildResults(score.scores), tokenUsage: score.tokenUsage, durationMs: endedAt.getTime() - startedAt.getTime(), diff --git a/packages/core/src/evaluation/output-transform.ts b/packages/core/src/evaluation/output-transform.ts new file mode 100644 index 000000000..e633545e9 --- /dev/null +++ b/packages/core/src/evaluation/output-transform.ts @@ -0,0 +1,129 @@ +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import type { Content } from './content.js'; +import type { JsonObject } from './types.js'; + +export type TransformSpec = string; +type TransformFunction = (output: unknown, context: TransformContext) => unknown | Promise; + +export interface TransformPromptContext { + readonly id?: string; + readonly label?: string; + readonly raw?: string; +} + +export interface TransformContext { + readonly vars?: JsonObject; + readonly prompt?: TransformPromptContext; + readonly metadata?: Record; + readonly provider?: { + readonly id: string; + readonly kind: string; + readonly target: string; + }; +} + +export interface TransformResult { + readonly value: unknown; + readonly input: unknown; + readonly spec: TransformSpec; +} + +const FILE_PREFIX = 'file://'; +const INLINE_STRING_LABEL_MAX_LENGTH = 80; + +function parseFileTransformReference(spec: string): { + readonly filePath: string; + readonly functionName?: string; +} { + const ref = spec.slice(FILE_PREFIX.length); + const lastColon = ref.lastIndexOf(':'); + if (lastColon > 1) { + return { + filePath: ref.slice(0, lastColon), + functionName: ref.slice(lastColon + 1), + }; + } + return { filePath: ref }; +} + +function isJavaScriptFile(filePath: string): boolean { + return ['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts'].includes(path.extname(filePath)); +} + +async function loadFileTransform(spec: string): Promise { + const { filePath, functionName } = parseFileTransformReference(spec); + const absolutePath = path.isAbsolute(filePath) ? filePath : path.resolve(filePath); + if (!isJavaScriptFile(absolutePath)) { + throw new Error(`Unsupported transform file format: ${spec}`); + } + + const mod = (await import(pathToFileURL(absolutePath).href)) as Record; + const candidate = functionName ? mod[functionName] : (mod.default ?? mod); + if (typeof candidate === 'function') { + return candidate as TransformFunction; + } + throw new Error( + functionName + ? `Transform ${spec} must export function "${functionName}"` + : `Transform ${spec} must export a function or default function`, + ); +} + +function inlineTransformFunction(code: string): TransformFunction { + return new Function( + 'output', + 'context', + code.includes('\n') ? code : `return ${code}`, + ) as TransformFunction; +} + +function transformLabel(spec: TransformSpec): string { + if (spec.startsWith(FILE_PREFIX)) { + return '[file transform]'; + } + const singleLine = spec.replace(/\s+/g, ' ').trim(); + const truncated = + singleLine.length > INLINE_STRING_LABEL_MAX_LENGTH + ? `${singleLine.slice(0, INLINE_STRING_LABEL_MAX_LENGTH - 1)}...` + : singleLine; + return `[inline transform]: ${truncated}`; +} + +export function stringifyTransformOutput(value: unknown): string { + if (value === undefined || value === null) { + return ''; + } + if (typeof value === 'string') { + return value; + } + return JSON.stringify(value, null, 2); +} + +export function contentToTransformInput(content: string | readonly Content[] | undefined): unknown { + if (content === undefined) { + return ''; + } + return content; +} + +export async function applyTransform( + spec: TransformSpec, + input: unknown, + context: TransformContext, +): Promise { + try { + const fn = spec.startsWith(FILE_PREFIX) + ? await loadFileTransform(spec) + : inlineTransformFunction(spec); + const value = await Promise.resolve(fn(input, context)); + if (value === undefined || value === null) { + throw new Error('Transform function did not return a value'); + } + return { value, input, spec }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Transform failed (${transformLabel(spec)}): ${message}`); + } +} diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index d7267805e..73c75fa40 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1,3 +1,4 @@ +import type { TransformSpec } from './output-transform.js'; import type { TargetExecutionEnvelope } from './providers/types.js'; import type { TokenUsage, ToolTrajectoryGraderConfig, Trace } from './trace.js'; @@ -938,6 +939,8 @@ export type GraderConfig = ( ) & { /** Optional promptfoo-style named score key. Scoring aggregation support is layered separately. */ readonly metric?: string; + /** Promptfoo-compatible assertion-level transform. */ + readonly transform?: TransformSpec; }; /** @@ -1055,6 +1058,10 @@ export interface EvalTest { readonly criteria: string; readonly evaluator?: GraderKind; readonly assertions?: readonly GraderConfig[]; + /** Merged Promptfoo-compatible row vars used by transform scripts. */ + readonly vars?: JsonObject; + /** Promptfoo-compatible output transform inherited from default_test.options or tests[].options. */ + readonly outputTransform?: TransformSpec; /** Suite-level preprocessors used by the implicit default llm-grader. */ readonly preprocessors?: readonly ContentPreprocessorConfig[]; /** Promptfoo-style lifecycle extensions inherited from the suite. */ diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 24171ac77..46a8d794d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -60,6 +60,7 @@ import { resolveExpectedMessages, resolveInputMessages, } from './loaders/shorthand-expansion.js'; +import { parseTransformSpec } from './loaders/transform-parser.js'; import { parseMetadata } from './metadata.js'; import { normalizeTargetDefinition } from './providers/targets.js'; import type { TargetDefinition } from './providers/types.js'; @@ -544,6 +545,73 @@ function mergeDefaultTestVarsIntoCases( }); } +function readDefaultTestOptions(defaultTest: JsonValue | undefined): JsonObject | undefined { + if (!isJsonObject(defaultTest) || !isJsonObject(defaultTest.options)) { + return undefined; + } + return defaultTest.options; +} + +function mergeDefaultTestOptionsIntoCases( + rawCases: readonly JsonValue[], + defaultTest: JsonValue | undefined, +): readonly JsonValue[] { + const defaultOptions = readDefaultTestOptions(defaultTest); + if (!defaultOptions || Object.keys(defaultOptions).length === 0) { + return rawCases; + } + + return rawCases.map((rawCase) => { + if (!isJsonObject(rawCase)) { + return rawCase; + } + if (rawCase.options !== undefined && !isJsonObject(rawCase.options)) { + return rawCase; + } + const caseOptions = isJsonObject(rawCase.options) ? rawCase.options : {}; + return { + ...rawCase, + options: { + ...defaultOptions, + ...caseOptions, + }, + }; + }); +} + +function rejectPostprocess(value: unknown, location: string): void { + if (!isJsonObject(value)) { + return; + } + if (value.postprocess !== undefined) { + throw new Error(`${location}.postprocess has been removed. Use ${location}.transform instead.`); + } +} + +function rejectAuthoredPostprocess(suite: RawTestSuite): void { + rejectPostprocess( + isJsonObject(suite.default_test) ? suite.default_test.options : undefined, + 'default_test.options', + ); + if (Array.isArray(suite.assert)) { + suite.assert.forEach((entry, index) => rejectPostprocess(entry, `assert[${index}]`)); + } + if (!Array.isArray(suite.tests)) { + return; + } + suite.tests.forEach((entry, index) => { + if (!isJsonObject(entry)) { + return; + } + rejectPostprocess(entry.options, `tests[${index}].options`); + if (Array.isArray(entry.assert)) { + entry.assert.forEach((assertion, assertionIndex) => + rejectPostprocess(assertion, `tests[${index}].assert[${assertionIndex}]`), + ); + } + }); +} + function isChatPromptArray(value: readonly JsonValue[]): boolean { return value.length > 0 && value.every((entry) => isJsonObject(entry) && isTestMessage(entry)); } @@ -1159,6 +1227,7 @@ async function loadTestsFromParsedYamlValue( ...(interpolated as RawTestSuite), default_test: resolvedDefaultTest.value, } as RawTestSuite; + rejectAuthoredPostprocess(suite); const defaultTestReferences = resolvedDefaultTest.references; const suiteNameFromFile = asString(suite.name)?.trim(); const fallbackSuiteName = @@ -1225,6 +1294,7 @@ async function loadTestsFromParsedYamlValue( } expandedTestCases = mergeDefaultTestVarsIntoCases(expandedTestCases, suite.default_test); + expandedTestCases = mergeDefaultTestOptionsIntoCases(expandedTestCases, suite.default_test); const promptDefinitions = await parseSuitePrompts(suite.prompts, searchRoots); const promptExpansion = expandPromptMatrix(expandedTestCases, promptDefinitions, suite); @@ -1521,6 +1591,12 @@ async function loadTestsFromParsedYamlValue( : undefined; const category = normalizeCategoryPath(suite.category ?? options?.category); + const renderedOptions = isJsonObject(renderedCase.options) ? renderedCase.options : undefined; + const outputTransform = await parseTransformSpec( + renderedOptions?.transform as JsonValue | undefined, + searchRoots, + `test '${id ?? 'unknown'}'.options`, + ); const testCase: EvalTest = { id, @@ -1537,6 +1613,8 @@ async function loadTestsFromParsedYamlValue( criteria: outcome ?? '', evaluator: testCaseEvaluatorKind, assertions: evaluators, + ...(caseVars ? { vars: caseVars } : {}), + ...(outputTransform ? { outputTransform } : {}), ...(suitePreprocessors ? { preprocessors: suitePreprocessors } : {}), ...(suiteExtensions.length > 0 ? { extensions: suiteExtensions } : {}), workspace: mergedWorkspace, diff --git a/packages/core/test/evaluation/transform-runtime.test.ts b/packages/core/test/evaluation/transform-runtime.test.ts new file mode 100644 index 000000000..fdc46943b --- /dev/null +++ b/packages/core/test/evaluation/transform-runtime.test.ts @@ -0,0 +1,343 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { runEvalCase, runEvaluation } from '../../src/evaluation/orchestrator.js'; +import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; +import type { + Provider, + ProviderRequest, + ProviderResponse, +} from '../../src/evaluation/providers/types.js'; +import type { EvalTest } from '../../src/evaluation/types.js'; +import { loadTests } from '../../src/evaluation/yaml-parser.js'; + +class StaticProvider implements Provider { + readonly id: string; + readonly kind = 'mock' as const; + readonly targetName: string; + lastRequest?: ProviderRequest; + + constructor( + targetName: string, + private readonly response: ProviderResponse, + ) { + this.id = `mock:${targetName}`; + this.targetName = targetName; + } + + async invoke(request: ProviderRequest): Promise { + this.lastRequest = request; + return this.response; + } +} + +const target: ResolvedTarget = { + name: 'mock', + kind: 'mock', + config: { response: 'raw' }, +}; + +const evaluatorRegistry = { + 'llm-grader': { + kind: 'llm-grader', + async evaluate() { + return { + score: 1, + verdict: 'pass' as const, + assertions: [{ text: 'default grader', passed: true }], + expectedAspectCount: 1, + }; + }, + }, +}; + +function tempDir(prefix: string): string { + return mkdtempSync(path.join(tmpdir(), prefix)); +} + +function baseEvalCase(overrides: Partial = {}): EvalTest { + return { + id: 'case-1', + suite: 'transform-suite', + question: 'Grade the answer', + input: [{ role: 'user', content: 'Grade the answer' }], + expected_output: [], + reference_answer: '', + file_paths: [], + criteria: 'Output should satisfy assertions', + ...overrides, + }; +} + +describe('Promptfoo-compatible transform runtime', () => { + it('loads default_test options.transform and lets test options.transform override it', async () => { + const dir = tempDir('agentv-transform-loader-'); + try { + const transformPath = path.join(dir, 'default-transform.js'); + writeFileSync(transformPath, 'export default (output) => `${output}|default`;\n', 'utf8'); + const evalPath = path.join(dir, 'suite.eval.yaml'); + writeFileSync( + evalPath, + ` +default_test: + options: + transform: file://default-transform.js +tests: + - id: inherited + input: say hi + assert: + - type: contains + value: default + - id: override + input: say hi + options: + transform: output + "|override" + assert: + - type: contains + value: override +`, + 'utf8', + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests[0]?.outputTransform).toBe(`file://${transformPath}`); + expect(tests[1]?.outputTransform).toBe('output + "|override"'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('rejects deprecated postprocess with transform guidance', async () => { + const dir = tempDir('agentv-transform-postprocess-'); + try { + const evalPath = path.join(dir, 'suite.eval.yaml'); + writeFileSync( + evalPath, + ` +default_test: + options: + postprocess: output.trim() +tests: + - id: postprocess + input: say hi + assert: + - type: contains + value: hi +`, + 'utf8', + ); + + await expect(loadTests(evalPath, dir)).rejects.toThrow( + 'default_test.options.postprocess has been removed. Use default_test.options.transform instead.', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('rejects test and assertion postprocess with transform guidance', async () => { + const dir = tempDir('agentv-transform-test-postprocess-'); + try { + const evalPath = path.join(dir, 'suite.eval.yaml'); + writeFileSync( + evalPath, + ` +tests: + - id: test-postprocess + input: say hi + options: + postprocess: output.trim() + assert: + - type: contains + value: hi +`, + 'utf8', + ); + + await expect(loadTests(evalPath, dir)).rejects.toThrow( + 'tests[0].options.postprocess has been removed. Use tests[0].options.transform instead.', + ); + + writeFileSync( + evalPath, + ` +tests: + - id: assertion-postprocess + input: say hi + assert: + - type: contains + value: hi + postprocess: output.trim() +`, + 'utf8', + ); + + await expect(loadTests(evalPath, dir)).rejects.toThrow( + 'tests[0].assert[0].postprocess has been removed. Use tests[0].assert[0].transform instead.', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('applies test transform once before all assertions and isolates assertion transforms', async () => { + const provider = new StaticProvider('mock', { + output: [{ role: 'assistant', content: 'raw' }], + }); + + const result = await runEvalCase({ + evalCase: baseEvalCase({ + outputTransform: 'output + "|case"', + assertions: [ + { name: 'case-visible', type: 'contains', value: 'raw|case' }, + { + name: 'assert-only', + type: 'contains', + value: 'raw|case|assert', + transform: 'output + "|assert"', + }, + { name: 'assert-isolated', type: 'contains', value: 'raw|case|assert' }, + ], + }), + provider, + target, + evaluators: evaluatorRegistry, + }); + + expect(result.output).toBe('raw|case'); + expect(result.scores?.map((score) => score.score)).toEqual([1, 1, 0]); + expect(result.scores?.[1]?.input).toEqual({ + transform: { input: 'raw|case', output: 'raw|case|assert' }, + }); + }); + + it('runs file transforms over file-style assistant output before llm-rubric grading', async () => { + const dir = tempDir('agentv-transform-xlsx-'); + try { + const reportPath = path.join(dir, 'report.xlsx'); + const transformPath = path.join(dir, 'xlsx-to-markdown.js'); + writeFileSync(reportPath, Buffer.from([0, 159, 146, 150])); + writeFileSync( + transformPath, + ` +export default function transform(output, context) { + if (!Array.isArray(output)) throw new Error('expected content array'); + const file = output.find((block) => block.type === 'file'); + if (!file || !file.path.endsWith('report.xlsx')) throw new Error('missing xlsx file'); + if (context.vars.sheet !== 'revenue') throw new Error('missing vars'); + return '| quarter | revenue |\\n| --- | ---: |\\n| Q1 | 42 |'; +} +`, + 'utf8', + ); + + const answerProvider = new StaticProvider('file-output', { + output: [ + { + role: 'assistant', + content: [ + { + type: 'file', + media_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + path: reportPath, + }, + ], + }, + ], + }); + const graderProvider = new StaticProvider('grader', { + output: [ + { + role: 'assistant', + content: JSON.stringify({ + score: 1, + assertions: [{ text: 'spreadsheet rows visible', passed: true }], + }), + }, + ], + }); + + const [result] = await runEvaluation({ + testFilePath: path.join(dir, 'suite.eval.yaml'), + repoRoot: dir, + target: { ...target, name: 'file-output', graderTarget: 'grader' }, + targets: [ + { name: 'grader', provider: 'mock' }, + { name: 'file-output', provider: 'mock', grader_target: 'grader' }, + ], + providerFactory: (resolved) => + resolved.name === 'grader' ? graderProvider : answerProvider, + evalCases: [ + baseEvalCase({ + id: 'xlsx-transform', + vars: { sheet: 'revenue' }, + outputTransform: `file://${transformPath}`, + assertions: [ + { + name: 'judge-spreadsheet', + type: 'llm-rubric', + value: 'The markdown table includes Q1 revenue.', + }, + ], + }), + ], + }); + + expect(result?.score).toBe(1); + expect(result?.output).toContain('| Q1 | 42 |'); + expect(graderProvider.lastRequest?.question).toContain('| Q1 | 42 |'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('surfaces test transform errors as evaluation failures', async () => { + const provider = new StaticProvider('mock', { + output: [{ role: 'assistant', content: 'raw' }], + }); + + const result = await runEvalCase({ + evalCase: baseEvalCase({ + outputTransform: '(() => { throw new Error("boom") })()', + assertions: [{ name: 'never', type: 'contains', value: 'raw' }], + }), + provider, + target, + evaluators: evaluatorRegistry, + }); + + expect(result.executionStatus).toBe('execution_error'); + expect(result.error).toContain('Transform failed'); + expect(result.error).toContain('boom'); + }); + + it('surfaces assertion transform errors on that assertion only', async () => { + const provider = new StaticProvider('mock', { + output: [{ role: 'assistant', content: 'raw' }], + }); + + const result = await runEvalCase({ + evalCase: baseEvalCase({ + assertions: [ + { name: 'ok', type: 'contains', value: 'raw' }, + { + name: 'bad-transform', + type: 'contains', + value: 'raw', + transform: 'throw new Error("assert boom")', + }, + ], + }), + provider, + target, + evaluators: evaluatorRegistry, + }); + + expect(result.scores?.map((score) => score.score)).toEqual([1, 0]); + expect(result.scores?.[1]?.assertions[0]?.text).toContain('Transform failed'); + expect(result.output).toBe('raw'); + }); +}); From 4ff747348d35a2a1c9531849b55258cd30071d9e Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 13:54:07 +0200 Subject: [PATCH 2/2] test(core): align transform fixtures with prompts --- packages/core/src/evaluation/graders/types.ts | 2 + packages/core/src/evaluation/orchestrator.ts | 66 ++++++++++++++++++- .../test/evaluation/transform-runtime.test.ts | 44 +++++++++---- 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/packages/core/src/evaluation/graders/types.ts b/packages/core/src/evaluation/graders/types.ts index 8ccc88476..995474eb0 100644 --- a/packages/core/src/evaluation/graders/types.ts +++ b/packages/core/src/evaluation/graders/types.ts @@ -23,6 +23,8 @@ export interface EvaluationContext { readonly candidate: string; /** Raw transformed output value before string coercion, for assertion-level transforms. */ readonly candidateValue?: unknown; + /** JSON-safe provider response metadata available to transforms. */ + readonly responseMetadata?: JsonObject; readonly target: ResolvedTarget; readonly provider: Provider; readonly attempt: number; diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 2d97ddbd3..ae10788d1 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -84,6 +84,7 @@ import type { TrialResult, TrialsConfig, } from './types.js'; +import { isJsonValue } from './types.js'; import { cleanupEvalWorkspaces, cleanupWorkspace } from './workspace/manager.js'; import type { RepoManager } from './workspace/repo-manager.js'; import { @@ -1684,6 +1685,7 @@ async function runBatchEvaluation(options: { const startTime = merged?.startTime; const endTime = merged?.endTime; const rawProviderLogPath = extractProviderRawLogPath(providerResponse); + const providerResponseMetadata = buildProviderResponseTransformMetadata(providerResponse); // Extract candidate from last assistant message in output const candidate = extractLastAssistantContent(output); @@ -1710,6 +1712,7 @@ async function runBatchEvaluation(options: { costUsd, durationMs, tokenUsage, + providerResponseMetadata, startTime, endTime, rawProviderLogPath, @@ -2206,6 +2209,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise = {}; + const fields: Array = [ + ['raw', response.raw], + ['usage', response.usage], + ['token_usage', response.tokenUsage], + ['cost_usd', response.costUsd], + ['duration_ms', response.durationMs], + ['start_time', response.startTime], + ['end_time', response.endTime], + ['steps', response.steps], + ]; + for (const [key, value] of fields) { + const jsonValue = toJsonValue(value); + if (jsonValue !== undefined) { + metadata[key] = jsonValue; + } + } + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + async function prepareCandidateForGrading(options: { readonly evalCase: EvalTest; readonly candidate: string; readonly output?: readonly Message[]; readonly promptInputs: PromptInputs; readonly provider: Provider; + readonly responseMetadata?: JsonObject; }): Promise<{ readonly candidate: string; readonly candidateValue: unknown; @@ -2630,6 +2680,7 @@ async function evaluateCandidate(options: { readonly costUsd?: number; readonly durationMs?: number; readonly tokenUsage?: TokenUsage; + readonly providerResponseMetadata?: JsonObject; readonly startTime?: string; readonly endTime?: string; readonly rawProviderLogPath?: string; @@ -2661,6 +2712,7 @@ async function evaluateCandidate(options: { costUsd, durationMs, tokenUsage, + providerResponseMetadata, startTime, endTime, rawProviderLogPath, @@ -2680,6 +2732,7 @@ async function evaluateCandidate(options: { output, promptInputs, provider, + responseMetadata: providerResponseMetadata, }); const input = buildResultInput(promptInputs); const outputMessages = output ?? [{ role: 'assistant' as const, content: candidate }]; @@ -2709,6 +2762,7 @@ async function evaluateCandidate(options: { evalCase, candidate: preparedCandidate.candidate, candidateValue: preparedCandidate.candidateValue, + responseMetadata: providerResponseMetadata, target, provider, evaluators, @@ -2801,6 +2855,7 @@ async function runEvaluatorsForCase(options: { readonly evalCase: EvalTest; readonly candidate: string; readonly candidateValue?: unknown; + readonly responseMetadata?: JsonObject; readonly target: ResolvedTarget; readonly provider: Provider; readonly evaluators: Partial> & { readonly 'llm-grader': Grader }; @@ -2830,6 +2885,7 @@ async function runEvaluatorsForCase(options: { evalCase, candidate, candidateValue, + responseMetadata, target, provider, evaluators, @@ -2865,6 +2921,7 @@ async function runEvaluatorsForCase(options: { evaluators: evalCase.assertions, candidate, candidateValue, + responseMetadata, target, provider, evaluatorRegistry: evaluators, @@ -2922,6 +2979,7 @@ async function runEvaluatorsForCase(options: { evalCase, candidate, candidateValue, + responseMetadata, target, provider, attempt, @@ -2979,6 +3037,7 @@ async function transformEvaluationContextForGrader( evalCase: context.evalCase, promptInputs: context.promptInputs, provider: context.provider, + responseMetadata: context.responseMetadata, }), ); const transformedCandidate = stringifyTransformOutput(transformed.value); @@ -3004,6 +3063,7 @@ async function runEvaluatorList(options: { readonly evaluators: readonly GraderConfig[]; readonly candidate: string; readonly candidateValue?: unknown; + readonly responseMetadata?: JsonObject; readonly target: ResolvedTarget; readonly provider: Provider; readonly evaluatorRegistry: Partial> & { @@ -3036,6 +3096,7 @@ async function runEvaluatorList(options: { evaluators, candidate, candidateValue, + responseMetadata, target, provider, evaluatorRegistry, @@ -3076,6 +3137,7 @@ async function runEvaluatorList(options: { evalCase, candidate, candidateValue, + responseMetadata, target, provider, attempt, diff --git a/packages/core/test/evaluation/transform-runtime.test.ts b/packages/core/test/evaluation/transform-runtime.test.ts index fdc46943b..20652d1e1 100644 --- a/packages/core/test/evaluation/transform-runtime.test.ts +++ b/packages/core/test/evaluation/transform-runtime.test.ts @@ -81,17 +81,21 @@ describe('Promptfoo-compatible transform runtime', () => { writeFileSync( evalPath, ` +prompts: + - "{{ input }}" default_test: + vars: + input: say hi options: transform: file://default-transform.js tests: - id: inherited - input: say hi assert: - type: contains value: default - id: override - input: say hi + vars: + input: say override options: transform: output + "|override" assert: @@ -117,12 +121,15 @@ tests: writeFileSync( evalPath, ` +prompts: + - "{{ input }}" default_test: + vars: + input: say hi options: postprocess: output.trim() tests: - id: postprocess - input: say hi assert: - type: contains value: hi @@ -145,9 +152,12 @@ tests: writeFileSync( evalPath, ` +prompts: + - "{{ input }}" tests: - id: test-postprocess - input: say hi + vars: + input: say hi options: postprocess: output.trim() assert: @@ -164,9 +174,12 @@ tests: writeFileSync( evalPath, ` +prompts: + - "{{ input }}" tests: - id: assertion-postprocess - input: say hi + vars: + input: say hi assert: - type: contains value: hi @@ -186,20 +199,24 @@ tests: it('applies test transform once before all assertions and isolates assertion transforms', async () => { const provider = new StaticProvider('mock', { output: [{ role: 'assistant', content: 'raw' }], + raw: { response_id: 'resp-1' }, + tokenUsage: { input: 2, output: 3 }, + durationMs: 7, }); const result = await runEvalCase({ evalCase: baseEvalCase({ - outputTransform: 'output + "|case"', + outputTransform: + 'output + "|" + context.metadata.raw.response_id + "|" + context.metadata.token_usage.input + "|case"', assertions: [ - { name: 'case-visible', type: 'contains', value: 'raw|case' }, + { name: 'case-visible', type: 'contains', value: 'raw|resp-1|2|case' }, { name: 'assert-only', type: 'contains', - value: 'raw|case|assert', - transform: 'output + "|assert"', + value: 'raw|resp-1|2|case|7|assert', + transform: 'output + "|" + context.metadata.duration_ms + "|assert"', }, - { name: 'assert-isolated', type: 'contains', value: 'raw|case|assert' }, + { name: 'assert-isolated', type: 'contains', value: 'raw|resp-1|2|case|7|assert' }, ], }), provider, @@ -207,10 +224,13 @@ tests: evaluators: evaluatorRegistry, }); - expect(result.output).toBe('raw|case'); + expect(result.output).toBe('raw|resp-1|2|case'); expect(result.scores?.map((score) => score.score)).toEqual([1, 1, 0]); expect(result.scores?.[1]?.input).toEqual({ - transform: { input: 'raw|case', output: 'raw|case|assert' }, + transform: { + input: 'raw|resp-1|2|case', + output: 'raw|resp-1|2|case|7|assert', + }, }); });