From b6b6ebf81d82d04f6bae078226f813f0b45b1985 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 09:41:21 +0200 Subject: [PATCH] test(promptfoo): add oracle export gate --- .github/workflows/ci.yml | 2 +- scripts/export-promptfoo-config.test.ts | 133 +++++++++++++++++- scripts/export-promptfoo-config.ts | 52 ++++++- .../oracle-grader-provider.cjs | 21 +++ .../oracle-matrix.agentv.yaml | 66 +++++++++ .../oracle-target-provider.cjs | 18 +++ 6 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 scripts/fixtures/promptfoo-export/oracle-grader-provider.cjs create mode 100644 scripts/fixtures/promptfoo-export/oracle-matrix.agentv.yaml create mode 100644 scripts/fixtures/promptfoo-export/oracle-target-provider.cjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2377301a3..5371843fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,7 +183,7 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile - - name: Run Promptfoo export tests + - name: Run Promptfoo export and oracle tests run: bun run validate:promptfoo-export - name: Export Promptfoo validation fixture diff --git a/scripts/export-promptfoo-config.test.ts b/scripts/export-promptfoo-config.test.ts index 78b9d5fbb..73d591dcf 100644 --- a/scripts/export-promptfoo-config.test.ts +++ b/scripts/export-promptfoo-config.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { existsSync, mkdtempSync, readFileSync } from 'node:fs'; +import { copyFileSync, existsSync, mkdtempSync, readFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import YAML from 'yaml'; @@ -7,6 +7,8 @@ import { PromptfooExportDiagnostic, exportPromptfooConfig } from './export-promp const ROOT = path.resolve(import.meta.dir, '..'); const FIXTURE_DIR = path.join(ROOT, 'scripts', 'fixtures', 'promptfoo-export'); +const PROMPTFOO_ORACLE_VERSION = '0.121.15'; +const PROMPTFOO_REFERENCE_CLONE_COMMIT = '6bfc5a0c7f16f9c4717ac731d276b578e63d0769'; function outputPath(name: string): string { return path.join(mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')), name); @@ -16,6 +18,65 @@ function parseYamlFile(filePath: string): Record { return YAML.parse(readFileSync(filePath, 'utf8')) as Record; } +function readJsonlFile(filePath: string): Array> { + return readFileSync(filePath, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +function copyOraclePromptfooFiles(output: string): void { + for (const name of ['oracle-target-provider.cjs', 'oracle-grader-provider.cjs']) { + copyFileSync(path.join(FIXTURE_DIR, name), path.join(path.dirname(output), name)); + } +} + +function runPromptfooOracle(configPath: string, outputPath: string): void { + const result = Bun.spawnSync({ + cmd: [ + 'bunx', + `promptfoo@${PROMPTFOO_ORACLE_VERSION}`, + 'eval', + '-c', + configPath, + '--no-cache', + '--no-table', + '--no-write', + '-o', + outputPath, + ], + stdout: 'pipe', + stderr: 'pipe', + env: { + ...process.env, + CI: 'true', + NO_COLOR: '1', + PROMPTFOO_DISABLE_UPDATE: 'true', + }, + }); + + if (!result.success) { + throw new Error( + [ + `promptfoo@${PROMPTFOO_ORACLE_VERSION} eval failed with exit code ${result.exitCode}`, + result.stdout.toString(), + result.stderr.toString(), + ].join('\n'), + ); + } +} + +function getPath(value: unknown, keys: string[]): unknown { + return keys.reduce( + (current, key) => + current && typeof current === 'object' && !Array.isArray(current) + ? (current as Record)[key] + : undefined, + value, + ); +} + describe('exportPromptfooConfig', () => { it('preserves Promptfoo-native colon provider ids and labels', () => { const output = outputPath('promptfooconfig.yaml'); @@ -81,6 +142,22 @@ describe('exportPromptfooConfig', () => { expect(exported).not.toHaveProperty('evaluate_options'); }); + it('lowers AgentV defaults to Promptfoo defaultTest provider selectors', () => { + const output = outputPath('promptfooconfig.yaml'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'oracle-matrix.agentv.yaml'), + outputPath: output, + }); + + const exported = parseYamlFile(output); + const defaultTest = exported.defaultTest as Record; + const options = defaultTest.options as Record; + + expect(exported).not.toHaveProperty('defaults'); + expect(defaultTest.providers).toEqual(['target-default']); + expect(options.provider).toBe('grader-default'); + }); + it('lowers host environment setup to a generated Promptfoo extension and workdir metadata', () => { const output = outputPath('promptfooconfig.yaml'); exportPromptfooConfig({ @@ -142,4 +219,58 @@ describe('exportPromptfooConfig', () => { expect((error as Error).message).toContain('isolation, image/context, mounts, services'); } }); + + it('executes exported Promptfoo config with deterministic matrix and grader outcomes', () => { + const output = outputPath('promptfooconfig.yaml'); + const resultOutput = path.join(path.dirname(output), 'promptfoo-results.jsonl'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'oracle-matrix.agentv.yaml'), + outputPath: output, + }); + copyOraclePromptfooFiles(output); + + const exported = parseYamlFile(output); + expect(getPath(exported, ['metadata', 'agentv_promptfoo_oracle'])).toEqual({ + promptfoo_version: PROMPTFOO_ORACLE_VERSION, + promptfoo_reference_clone_commit: PROMPTFOO_REFERENCE_CLONE_COMMIT, + }); + + runPromptfooOracle(output, resultOutput); + const rows = readJsonlFile(resultOutput); + + expect(rows).toHaveLength(3); + expect( + rows.map((row) => ({ + caseId: getPath(row, ['vars', 'case_id']), + provider: getPath(row, ['provider', 'label']), + success: row.success, + output: getPath(row, ['response', 'output']), + reasons: ( + getPath(row, ['gradingResult', 'componentResults']) as Array> + ).map((result) => result.reason), + })), + ).toEqual([ + { + caseId: 'default-case', + provider: 'target-default', + success: true, + output: 'DEFAULT:default-case', + reasons: ['graded-by:default', 'Assertion passed'], + }, + { + caseId: 'test-options-case', + provider: 'target-default', + success: true, + output: 'DEFAULT:test-options-case', + reasons: ['graded-by:test-options', 'Assertion passed'], + }, + { + caseId: 'assertion-override-case', + provider: 'target-override', + success: true, + output: 'OVERRIDE:assertion-override-case', + reasons: ['graded-by:default', 'Assertion passed', 'graded-by:assertion'], + }, + ]); + }, 20000); }); diff --git a/scripts/export-promptfoo-config.ts b/scripts/export-promptfoo-config.ts index 27b2a3bee..7e26481e1 100644 --- a/scripts/export-promptfoo-config.ts +++ b/scripts/export-promptfoo-config.ts @@ -345,24 +345,57 @@ function defaultTestWithEnvironment(value: unknown, environment: HostEnvironment }; } +function defaultTestWithAgentVDefaults(value: unknown, defaults: unknown): JsonMap { + if (typeof value === 'string') { + throw new PromptfooExportDiagnostic( + 'unsupported_defaults_default_test_ref', + 'defaults.provider/defaults.grader export cannot merge into a default_test file reference yet. Inline default_test before exporting to Promptfoo.', + ); + } + const defaultTest = + value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonMap) : {}; + const defaultOptions = + defaultTest.options && + typeof defaultTest.options === 'object' && + !Array.isArray(defaultTest.options) + ? (defaultTest.options as JsonMap) + : {}; + const defaultValues = assertRecord(defaults, 'defaults'); + const provider = defaultValues.provider; + const grader = defaultValues.grader; + + return { + ...defaultTest, + ...(typeof provider === 'string' && defaultTest.providers === undefined + ? { providers: [lowerProviderId(provider)] } + : {}), + ...(typeof grader === 'string' && defaultOptions.provider === undefined + ? { options: { ...defaultOptions, provider: lowerProviderId(grader) } } + : {}), + }; +} + function promptfooConfigFromAgentVConfig( config: JsonMap, environment?: HostEnvironmentExport, ): JsonMap { const promptfooConfig: JsonMap = {}; + let defaults: unknown; for (const [key, value] of Object.entries(config)) { if (key === 'environment') { continue; } + if (key === 'defaults') { + defaults = value; + continue; + } const outputKey = TOP_LEVEL_KEY_RENAMES[key] ?? key; if (key === 'providers') { promptfooConfig[outputKey] = lowerProviders(value, environment); continue; } if (key === 'default_test') { - promptfooConfig[outputKey] = environment - ? defaultTestWithEnvironment(value, environment) - : value; + promptfooConfig[outputKey] = value; continue; } if (key === 'evaluate_options') { @@ -371,8 +404,17 @@ function promptfooConfigFromAgentVConfig( } promptfooConfig[outputKey] = value; } - if (environment && !('defaultTest' in promptfooConfig)) { - promptfooConfig.defaultTest = defaultTestWithEnvironment(undefined, environment); + if (defaults !== undefined) { + promptfooConfig.defaultTest = defaultTestWithAgentVDefaults( + promptfooConfig.defaultTest, + defaults, + ); + } + if (environment) { + promptfooConfig.defaultTest = defaultTestWithEnvironment( + promptfooConfig.defaultTest, + environment, + ); } if (environment) { promptfooConfig.metadata = mergeJsonObject(promptfooConfig.metadata, { diff --git a/scripts/fixtures/promptfoo-export/oracle-grader-provider.cjs b/scripts/fixtures/promptfoo-export/oracle-grader-provider.cjs new file mode 100644 index 000000000..2b5b81194 --- /dev/null +++ b/scripts/fixtures/promptfoo-export/oracle-grader-provider.cjs @@ -0,0 +1,21 @@ +module.exports = class OracleGraderProvider { + constructor(options = {}) { + this.label = options.label; + this.config = options.config || {}; + } + + id() { + return this.label || 'oracle-grader'; + } + + async callApi() { + const label = this.config.grade_label || this.config.gradeLabel || this.label || 'unknown'; + return { + output: JSON.stringify({ + pass: true, + score: 1, + reason: `graded-by:${label}`, + }), + }; + } +}; diff --git a/scripts/fixtures/promptfoo-export/oracle-matrix.agentv.yaml b/scripts/fixtures/promptfoo-export/oracle-matrix.agentv.yaml new file mode 100644 index 000000000..a7adacebd --- /dev/null +++ b/scripts/fixtures/promptfoo-export/oracle-matrix.agentv.yaml @@ -0,0 +1,66 @@ +description: Promptfoo oracle matrix export fixture +metadata: + agentv_promptfoo_oracle: + promptfoo_version: "0.121.15" + promptfoo_reference_clone_commit: "6bfc5a0c7f16f9c4717ac731d276b578e63d0769" + +providers: + - id: file://oracle-target-provider.cjs + label: target-default + config: + prefix: DEFAULT + - id: file://oracle-target-provider.cjs + label: target-override + config: + prefix: OVERRIDE + - id: file://oracle-grader-provider.cjs + label: grader-default + config: + grade_label: default + - id: file://oracle-grader-provider.cjs + label: grader-test-options + config: + grade_label: test-options + - id: file://oracle-grader-provider.cjs + label: grader-assertion + config: + grade_label: assertion + +defaults: + provider: target-default + grader: grader-default + +prompts: + - "{{ case_id }}" + +default_test: + assert: + - type: llm-rubric + value: "The answer matches the requested case." + +tests: + - id: default-provider-and-grader + vars: + case_id: default-case + assert: + - type: contains + value: "DEFAULT:default-case" + - id: test-options-grader + vars: + case_id: test-options-case + options: + provider: grader-test-options + assert: + - type: contains + value: "DEFAULT:test-options-case" + - id: assertion-provider-and-target-override + vars: + case_id: assertion-override-case + providers: + - target-override + assert: + - type: contains + value: "OVERRIDE:assertion-override-case" + - type: llm-rubric + value: "The assertion-level grader is used." + provider: grader-assertion diff --git a/scripts/fixtures/promptfoo-export/oracle-target-provider.cjs b/scripts/fixtures/promptfoo-export/oracle-target-provider.cjs new file mode 100644 index 000000000..1e24edcfe --- /dev/null +++ b/scripts/fixtures/promptfoo-export/oracle-target-provider.cjs @@ -0,0 +1,18 @@ +module.exports = class OracleTargetProvider { + constructor(options = {}) { + this.label = options.label; + this.config = options.config || {}; + } + + id() { + return this.label || 'oracle-target'; + } + + async callApi(_prompt, context = {}) { + const vars = context.vars || {}; + const prefix = this.config.prefix || this.label || 'TARGET'; + return { + output: `${prefix}:${vars.case_id}`, + }; + } +};