Skip to content

feat(core): add Promptfoo-compatible transform runtime #1648

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
christso merged 2 commits into from
Jul 4, 2026
Merged

feat(core): add Promptfoo-compatible transform runtime #1648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cfd94a9
feat(core): add promptfoo transform runtime
christso Jul 4, 2026
4ff7473
test(core): align transform fixtures with prompts
christso Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/core/src/evaluation/graders/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ export type TargetResolver = (targetName: string) => Provider | undefined;
export interface EvaluationContext {
readonly evalCase: EvalTest;
readonly candidate: string;
/** Raw transformed output value before string coercion, for assertion-level transforms. */
readonly candidateValue?: unknown;
/** JSON-safe provider response metadata available to transforms. */
readonly responseMetadata?: JsonObject;
readonly target: ResolvedTarget;
readonly provider: Provider;
readonly attempt: number;
Expand Down
73 changes: 50 additions & 23 deletions packages/core/src/evaluation/loaders/grader-parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { RUBRIC_OPERATOR_VALUES, isGraderKind } from '../types.js';
import { validateCustomPromptContent } from '../validation/prompt-validator.js';
import { parseYamlValue } from '../yaml-loader.js';
import { resolveFileReference } from './file-resolver.js';
import { parseTransformSpec } from './transform-parser.js';

const ANSI_YELLOW = '\u001b[33m';
const ANSI_RESET = '\u001b[0m';
Expand Down Expand Up @@ -550,6 +551,19 @@ async function parseGraderList(
}

const negate = rawEvaluator.negate === true ? true : undefined;
if (rawEvaluator.postprocess !== undefined) {
throw new Error(
`Grader '${name}' in '${evalId}': postprocess has been removed. Use transform instead.`,
);
}
const transform = await parseTransformSpec(
rawEvaluator.transform as JsonValue | undefined,
searchRoots,
`Grader '${name}' in '${evalId}'`,
);
const pushEvaluator = (config: GraderConfig): void => {
evaluators.push(transform !== undefined ? { ...config, transform } : config);
};
const mergedPreprocessors = await parseMergedPreprocessors(
rawEvaluator.preprocessors as JsonValue | undefined,
defaultPreprocessors,
Expand All @@ -568,14 +582,23 @@ async function parseGraderList(
evalId,
);
// Collect all properties except known meta-keys as pass-through config
const knownProps = new Set(['metric', 'type', 'weight', 'required', 'min_score', 'negate']);
const knownProps = new Set([
'metric',
'type',
'weight',
'required',
'min_score',
'negate',
'transform',
'postprocess',
]);
const config: Record<string, JsonValue> = {};
for (const [key, value] of Object.entries(rawEvaluator)) {
if (!knownProps.has(key) && value !== undefined) {
config[key] = value as JsonValue;
}
}
evaluators.push({
pushEvaluator({
name,
type: customTypeName as unknown as GraderKind,
...(weight !== undefined ? { weight } : {}),
Expand Down Expand Up @@ -623,7 +646,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'assert-set',
assertions: parsedMembers,
Expand Down Expand Up @@ -734,6 +757,8 @@ async function parseGraderList(
'required',
'min_score',
'negate',
'transform',
'postprocess',
]);
const config: Record<string, JsonValue> = {};
for (const [key, value] of Object.entries(rawEvaluator)) {
Expand All @@ -746,7 +771,7 @@ async function parseGraderList(
: {};
const mergedConfig = { ...config, ...topLevelConfig };

evaluators.push({
pushEvaluator({
name,
type: 'script',
command,
Expand Down Expand Up @@ -927,7 +952,7 @@ async function parseGraderList(
...(argsMatch !== undefined ? { argsMatch } : {}),
};

evaluators.push(config);
pushEvaluator(config);
continue;
}

Expand Down Expand Up @@ -1006,7 +1031,7 @@ async function parseGraderList(
evalId,
);

evaluators.push({
pushEvaluator({
name,
type: 'field-accuracy',
fields,
Expand Down Expand Up @@ -1036,7 +1061,7 @@ async function parseGraderList(
evalId,
);

evaluators.push({
pushEvaluator({
name,
type: 'latency',
threshold,
Expand Down Expand Up @@ -1065,7 +1090,7 @@ async function parseGraderList(
evalId,
);

evaluators.push({
pushEvaluator({
name,
type: 'cost',
budget,
Expand Down Expand Up @@ -1120,7 +1145,7 @@ async function parseGraderList(
evalId,
);

evaluators.push({
pushEvaluator({
name,
type: 'token-usage',
...validLimits,
Expand Down Expand Up @@ -1205,7 +1230,7 @@ async function parseGraderList(
evalId,
);

evaluators.push({
pushEvaluator({
name,
type: 'execution-metrics',
...validThresholds,
Expand All @@ -1232,7 +1257,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'skill-trigger',
skill: skillName,
Expand Down Expand Up @@ -1265,7 +1290,7 @@ async function parseGraderList(
evalId,
);
const config = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined;
evaluators.push({
pushEvaluator({
name,
type: typeValue,
value,
Expand Down Expand Up @@ -1303,7 +1328,7 @@ async function parseGraderList(
? rawEvaluator.provider
: undefined;
const config = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined;
evaluators.push({
pushEvaluator({
name,
type: 'similar',
value,
Expand Down Expand Up @@ -1331,7 +1356,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'contains',
value,
Expand All @@ -1358,7 +1383,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: typeValue,
value,
Expand All @@ -1383,7 +1408,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'icontains',
value,
Expand All @@ -1410,7 +1435,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: typeValue,
value,
Expand All @@ -1435,7 +1460,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: typeValue,
value,
Expand All @@ -1461,7 +1486,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'regex',
value,
Expand All @@ -1482,7 +1507,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'is-json',
...(weight !== undefined ? { weight } : {}),
Expand All @@ -1506,7 +1531,7 @@ async function parseGraderList(
name,
evalId,
);
evaluators.push({
pushEvaluator({
name,
type: 'equals',
value,
Expand Down Expand Up @@ -1572,6 +1597,8 @@ async function parseGraderList(
'maxSteps',
'temperature',
'preprocessors',
'transform',
'postprocess',
]);
const config: Record<string, JsonValue> = {};
for (const [key, value] of Object.entries(rawEvaluator)) {
Expand Down Expand Up @@ -1633,7 +1660,7 @@ async function parseGraderList(
continue;
}

evaluators.push({
pushEvaluator({
name,
type: 'llm-rubric',
prompt,
Expand All @@ -1657,7 +1684,7 @@ async function parseGraderList(
continue;
}

evaluators.push({
pushEvaluator({
name,
type: 'llm-grader',
prompt,
Expand Down
55 changes: 55 additions & 0 deletions packages/core/src/evaluation/loaders/transform-parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import path from 'node:path';

import type { TransformSpec } from '../output-transform.js';
import type { JsonValue } from '../types.js';
import { resolveFileReference } from './file-resolver.js';

const FILE_PREFIX = 'file://';

function splitFileTransform(value: string): {
readonly filePath: string;
readonly functionName?: string;
} {
const rawPath = value.slice(FILE_PREFIX.length);
const lastColon = rawPath.lastIndexOf(':');
if (lastColon > 1) {
return {
filePath: rawPath.slice(0, lastColon),
functionName: rawPath.slice(lastColon + 1),
};
}
return { filePath: rawPath };
}

export async function parseTransformSpec(
rawValue: JsonValue | undefined,
searchRoots: readonly string[],
label: string,
): Promise<TransformSpec | undefined> {
if (rawValue === undefined) {
return undefined;
}
if (typeof rawValue !== 'string' || rawValue.trim().length === 0) {
throw new Error(`${label}: transform must be a non-empty string`);
}

const trimmed = rawValue.trim();
if (!trimmed.startsWith(FILE_PREFIX)) {
return trimmed;
}

const { filePath, functionName } = splitFileTransform(trimmed);
const resolved = await resolveFileReference(filePath, searchRoots);
if (!resolved.resolvedPath) {
throw new Error(
`${label}: transform file not found: ${resolved.displayPath}${
resolved.attempted.length > 0
? `\n${resolved.attempted.map((attempt) => ` Tried: ${attempt}`).join('\n')}`
: ''
}`,
);
}

const absolutePath = path.resolve(resolved.resolvedPath);
return `${FILE_PREFIX}${absolutePath}${functionName ? `:${functionName}` : ''}`;
}
Loading
Loading