test(promptfoo): add oracle export gate

.github/workflows/ci.yml

@@ -183,7 +183,7 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
-      - name: Run Promptfoo export tests
+      - name: Run Promptfoo export and oracle tests
         run: bun run validate:promptfoo-export
 
       - name: Export Promptfoo validation fixture

scripts/export-promptfoo-config.test.ts

@@ -1,12 +1,14 @@
 import { describe, expect, it } from 'bun:test';
-import { existsSync, mkdtempSync, readFileSync } from 'node:fs';
+import { copyFileSync, existsSync, mkdtempSync, readFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import YAML from 'yaml';
 import { PromptfooExportDiagnostic, exportPromptfooConfig } from './export-promptfoo-config';
 
 const ROOT = path.resolve(import.meta.dir, '..');
 const FIXTURE_DIR = path.join(ROOT, 'scripts', 'fixtures', 'promptfoo-export');
+const PROMPTFOO_ORACLE_VERSION = '0.121.15';
+const PROMPTFOO_REFERENCE_CLONE_COMMIT = '6bfc5a0c7f16f9c4717ac731d276b578e63d0769';
 
 function outputPath(name: string): string {
   return path.join(mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')), name);
@@ -16,6 +18,65 @@ function parseYamlFile(filePath: string): Record<string, unknown> {
   return YAML.parse(readFileSync(filePath, 'utf8')) as Record<string, unknown>;
 }
 
+function readJsonlFile(filePath: string): Array<Record<string, unknown>> {
+  return readFileSync(filePath, 'utf8')
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line) as Record<string, unknown>);
+}
+
+function copyOraclePromptfooFiles(output: string): void {
+  for (const name of ['oracle-target-provider.cjs', 'oracle-grader-provider.cjs']) {
+    copyFileSync(path.join(FIXTURE_DIR, name), path.join(path.dirname(output), name));
+  }
+}
+
+function runPromptfooOracle(configPath: string, outputPath: string): void {
+  const result = Bun.spawnSync({
+    cmd: [
+      'bunx',
+      `promptfoo@${PROMPTFOO_ORACLE_VERSION}`,
+      'eval',
+      '-c',
+      configPath,
+      '--no-cache',
+      '--no-table',
+      '--no-write',
+      '-o',
+      outputPath,
+    ],
+    stdout: 'pipe',
+    stderr: 'pipe',
+    env: {
+      ...process.env,
+      CI: 'true',
+      NO_COLOR: '1',
+      PROMPTFOO_DISABLE_UPDATE: 'true',
+    },
+  });
+
+  if (!result.success) {
+    throw new Error(
+      [
+        `promptfoo@${PROMPTFOO_ORACLE_VERSION} eval failed with exit code ${result.exitCode}`,
+        result.stdout.toString(),
+        result.stderr.toString(),
+      ].join('\n'),
+    );
+  }
+}
+
+function getPath(value: unknown, keys: string[]): unknown {
+  return keys.reduce<unknown>(
+    (current, key) =>
+      current && typeof current === 'object' && !Array.isArray(current)
+        ? (current as Record<string, unknown>)[key]
+        : undefined,
+    value,
+  );
+}
+
 describe('exportPromptfooConfig', () => {
   it('preserves Promptfoo-native colon provider ids and labels', () => {
     const output = outputPath('promptfooconfig.yaml');
@@ -81,6 +142,22 @@ describe('exportPromptfooConfig', () => {
     expect(exported).not.toHaveProperty('evaluate_options');
   });
 
+  it('lowers AgentV defaults to Promptfoo defaultTest provider selectors', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'oracle-matrix.agentv.yaml'),
+      outputPath: output,
+    });
+
+    const exported = parseYamlFile(output);
+    const defaultTest = exported.defaultTest as Record<string, unknown>;
+    const options = defaultTest.options as Record<string, unknown>;
+
+    expect(exported).not.toHaveProperty('defaults');
+    expect(defaultTest.providers).toEqual(['target-default']);
+    expect(options.provider).toBe('grader-default');
+  });
+
   it('lowers host environment setup to a generated Promptfoo extension and workdir metadata', () => {
     const output = outputPath('promptfooconfig.yaml');
     exportPromptfooConfig({
@@ -142,4 +219,58 @@ describe('exportPromptfooConfig', () => {
       expect((error as Error).message).toContain('isolation, image/context, mounts, services');
     }
   });
+
+  it('executes exported Promptfoo config with deterministic matrix and grader outcomes', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    const resultOutput = path.join(path.dirname(output), 'promptfoo-results.jsonl');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'oracle-matrix.agentv.yaml'),
+      outputPath: output,
+    });
+    copyOraclePromptfooFiles(output);
+
+    const exported = parseYamlFile(output);
+    expect(getPath(exported, ['metadata', 'agentv_promptfoo_oracle'])).toEqual({
+      promptfoo_version: PROMPTFOO_ORACLE_VERSION,
+      promptfoo_reference_clone_commit: PROMPTFOO_REFERENCE_CLONE_COMMIT,
+    });
+
+    runPromptfooOracle(output, resultOutput);
+    const rows = readJsonlFile(resultOutput);
+
+    expect(rows).toHaveLength(3);
+    expect(
+      rows.map((row) => ({
+        caseId: getPath(row, ['vars', 'case_id']),
+        provider: getPath(row, ['provider', 'label']),
+        success: row.success,
+        output: getPath(row, ['response', 'output']),
+        reasons: (
+          getPath(row, ['gradingResult', 'componentResults']) as Array<Record<string, unknown>>
+        ).map((result) => result.reason),
+      })),
+    ).toEqual([
+      {
+        caseId: 'default-case',
+        provider: 'target-default',
+        success: true,
+        output: 'DEFAULT:default-case',
+        reasons: ['graded-by:default', 'Assertion passed'],
+      },
+      {
+        caseId: 'test-options-case',
+        provider: 'target-default',
+        success: true,
+        output: 'DEFAULT:test-options-case',
+        reasons: ['graded-by:test-options', 'Assertion passed'],
+      },
+      {
+        caseId: 'assertion-override-case',
+        provider: 'target-override',
+        success: true,
+        output: 'OVERRIDE:assertion-override-case',
+        reasons: ['graded-by:default', 'Assertion passed', 'graded-by:assertion'],
+      },
+    ]);
+  }, 20000);
 });

scripts/export-promptfoo-config.ts

@@ -345,24 +345,57 @@ function defaultTestWithEnvironment(value: unknown, environment: HostEnvironment
   };
 }
 
+function defaultTestWithAgentVDefaults(value: unknown, defaults: unknown): JsonMap {
+  if (typeof value === 'string') {
+    throw new PromptfooExportDiagnostic(
+      'unsupported_defaults_default_test_ref',
+      'defaults.provider/defaults.grader export cannot merge into a default_test file reference yet. Inline default_test before exporting to Promptfoo.',
+    );
+  }
+  const defaultTest =
+    value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonMap) : {};
+  const defaultOptions =
+    defaultTest.options &&
+    typeof defaultTest.options === 'object' &&
+    !Array.isArray(defaultTest.options)
+      ? (defaultTest.options as JsonMap)
+      : {};
+  const defaultValues = assertRecord(defaults, 'defaults');
+  const provider = defaultValues.provider;
+  const grader = defaultValues.grader;
+
+  return {
+    ...defaultTest,
+    ...(typeof provider === 'string' && defaultTest.providers === undefined
+      ? { providers: [lowerProviderId(provider)] }
+      : {}),
+    ...(typeof grader === 'string' && defaultOptions.provider === undefined
+      ? { options: { ...defaultOptions, provider: lowerProviderId(grader) } }
+      : {}),
+  };
+}
+
 function promptfooConfigFromAgentVConfig(
   config: JsonMap,
   environment?: HostEnvironmentExport,
 ): JsonMap {
   const promptfooConfig: JsonMap = {};
+  let defaults: unknown;
   for (const [key, value] of Object.entries(config)) {
     if (key === 'environment') {
       continue;
     }
+    if (key === 'defaults') {
+      defaults = value;
+      continue;
+    }
     const outputKey = TOP_LEVEL_KEY_RENAMES[key] ?? key;
     if (key === 'providers') {
       promptfooConfig[outputKey] = lowerProviders(value, environment);
       continue;
     }
     if (key === 'default_test') {
-      promptfooConfig[outputKey] = environment
-        ? defaultTestWithEnvironment(value, environment)
-        : value;
+      promptfooConfig[outputKey] = value;
       continue;
     }
     if (key === 'evaluate_options') {
@@ -371,8 +404,17 @@ function promptfooConfigFromAgentVConfig(
     }
     promptfooConfig[outputKey] = value;
   }
-  if (environment && !('defaultTest' in promptfooConfig)) {
-    promptfooConfig.defaultTest = defaultTestWithEnvironment(undefined, environment);
+  if (defaults !== undefined) {
+    promptfooConfig.defaultTest = defaultTestWithAgentVDefaults(
+      promptfooConfig.defaultTest,
+      defaults,
+    );
+  }
+  if (environment) {
+    promptfooConfig.defaultTest = defaultTestWithEnvironment(
+      promptfooConfig.defaultTest,
+      environment,
+    );
   }
   if (environment) {
     promptfooConfig.metadata = mergeJsonObject(promptfooConfig.metadata, {

scripts/fixtures/promptfoo-export/oracle-grader-provider.cjs

@@ -0,0 +1,21 @@
+module.exports = class OracleGraderProvider {
+  constructor(options = {}) {
+    this.label = options.label;
+    this.config = options.config || {};
+  }
+
+  id() {
+    return this.label || 'oracle-grader';
+  }
+
+  async callApi() {
+    const label = this.config.grade_label || this.config.gradeLabel || this.label || 'unknown';
+    return {
+      output: JSON.stringify({
+        pass: true,
+        score: 1,
+        reason: `graded-by:${label}`,
+      }),
+    };
+  }
+};

scripts/fixtures/promptfoo-export/oracle-matrix.agentv.yaml

@@ -0,0 +1,66 @@
+description: Promptfoo oracle matrix export fixture
+metadata:
+  agentv_promptfoo_oracle:
+    promptfoo_version: "0.121.15"
+    promptfoo_reference_clone_commit: "6bfc5a0c7f16f9c4717ac731d276b578e63d0769"
+
+providers:
+  - id: file://oracle-target-provider.cjs
+    label: target-default
+    config:
+      prefix: DEFAULT
+  - id: file://oracle-target-provider.cjs
+    label: target-override
+    config:
+      prefix: OVERRIDE
+  - id: file://oracle-grader-provider.cjs
+    label: grader-default
+    config:
+      grade_label: default
+  - id: file://oracle-grader-provider.cjs
+    label: grader-test-options
+    config:
+      grade_label: test-options
+  - id: file://oracle-grader-provider.cjs
+    label: grader-assertion
+    config:
+      grade_label: assertion
+
+defaults:
+  provider: target-default
+  grader: grader-default
+
+prompts:
+  - "{{ case_id }}"
+
+default_test:
+  assert:
+    - type: llm-rubric
+      value: "The answer matches the requested case."
+
+tests:
+  - id: default-provider-and-grader
+    vars:
+      case_id: default-case
+    assert:
+      - type: contains
+        value: "DEFAULT:default-case"
+  - id: test-options-grader
+    vars:
+      case_id: test-options-case
+    options:
+      provider: grader-test-options
+    assert:
+      - type: contains
+        value: "DEFAULT:test-options-case"
+  - id: assertion-provider-and-target-override
+    vars:
+      case_id: assertion-override-case
+    providers:
+      - target-override
+    assert:
+      - type: contains
+        value: "OVERRIDE:assertion-override-case"
+      - type: llm-rubric
+        value: "The assertion-level grader is used."
+        provider: grader-assertion

scripts/fixtures/promptfoo-export/oracle-target-provider.cjs

@@ -0,0 +1,18 @@
+module.exports = class OracleTargetProvider {
+  constructor(options = {}) {
+    this.label = options.label;
+    this.config = options.config || {};
+  }
+
+  id() {
+    return this.label || 'oracle-target';
+  }
+
+  async callApi(_prompt, context = {}) {
+    const vars = context.vars || {};
+    const prefix = this.config.prefix || this.label || 'TARGET';
+    return {
+      output: `${prefix}:${vars.case_id}`,
+    };
+  }
+};