From c08d2bd89a1700c4179a9e99c3ed38fbad117fcc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 10:12:04 +0200 Subject: [PATCH 1/2] feat(evaluation): support scenario file references --- .../docs/docs/next/evaluation/eval-files.mdx | 31 + examples/features/README.md | 2 + examples/features/scenarios/README.md | 79 + .../scenarios/evals/scenarios/french.yaml | 12 + .../scenarios/evals/scenarios/spanish.yaml | 19 + examples/features/scenarios/evals/suite.yaml | 19 + .../scenarios/scripts/translation-target.mjs | 16 + examples/features/scenarios/targets.yaml | 4 + .../loaders/scenario-file-loader.ts | 129 + .../evaluation/validation/eval-file.schema.ts | 10 +- .../evaluation/validation/eval-validator.ts | 53 +- packages/core/src/evaluation/yaml-parser.ts | 12 +- .../evaluation/loaders/jsonl-parser.test.ts | 95 + .../validation/eval-file-schema.test.ts | 15 + .../validation/eval-validator.test.ts | 86 + .../references/eval.schema.json | 5646 +++++++++-------- 16 files changed, 3402 insertions(+), 2826 deletions(-) create mode 100644 examples/features/scenarios/README.md create mode 100644 examples/features/scenarios/evals/scenarios/french.yaml create mode 100644 examples/features/scenarios/evals/scenarios/spanish.yaml create mode 100644 examples/features/scenarios/evals/suite.yaml create mode 100644 examples/features/scenarios/scripts/translation-target.mjs create mode 100644 examples/features/scenarios/targets.yaml create mode 100644 packages/core/src/evaluation/loaders/scenario-file-loader.ts diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index a551d3423..e3583933b 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -185,6 +185,7 @@ tests: | `env` | Promptfoo-compatible provider/eval environment-variable overrides and load-time template inputs. | | `extensions` | Lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. | | `tests` | Inline raw tests, a string path to an external raw-case file or directory, or a list mixing inline tests with raw-case file refs. | +| `scenarios` | Promptfoo-style scenario matrix rows. Each inline scenario crosses `config` rows with `tests` rows; entries can also be `file://` refs or globs that load scenario arrays. | | `assert` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test | `environment` is what the agent can inspect or modify through tools, not prompt @@ -282,6 +283,36 @@ External raw-case files referenced through `tests: file://...` may still contain raw internal `input` rows for compatibility. Keep that compatibility out of normal eval YAML authoring. +### Scenarios + +Use `scenarios` when a small set of configuration rows should cross with a set +of test rows. AgentV flattens inline scenario objects and `file://` scenario +refs or globs into one scenario list, then lowers each scenario as +`scenarios[].config x scenarios[].tests` before prompt expansion. + +```yaml +prompts: + - "Translate '{{ phrase }}' to {{ language }}." + +scenarios: + - config: + - vars: + language: Spanish + tests: + - id: spanish-hello-world + vars: + phrase: hello world + expected_translation: hola mundo + assert: + - type: equals + value: "{{ expected_translation }}" + - file://scenarios/*.yaml +``` + +External scenario files must contain a YAML or JSON scenario array, or an +object with a `scenarios` array. Scenario file refs are field-local to +`scenarios`; AgentV does not expand arbitrary YAML fields as files. + ### Lifecycle Extensions `extensions` uses AgentV lifecycle names. File hooks are local diff --git a/examples/features/README.md b/examples/features/README.md index c4999bdf3..3c68c7bfc 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -70,6 +70,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | Example | Description | |---------|-------------| | [external-datasets](external-datasets/) | Load test cases from YAML/JSONL files using `file://` references and globs | +| [scenarios](scenarios/) | Build Promptfoo-style `scenarios[].config x scenarios[].tests` matrices with inline and file-backed scenarios | | [input-files-shorthand](input-files-shorthand/) | Attach files to every test using a compact shorthand | | [suite-level-input](suite-level-input/) | Prepend a shared system prompt to every test in the suite | | [suite-level-input-files](suite-level-input-files/) | Share file attachments across every test in the suite | @@ -163,6 +164,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK | | [repo-lifecycle](repo-lifecycle/) | Workspace & targets | | [rubric](rubric/) | LLM grading | +| [scenarios](scenarios/) | Dataset & prompt templates | | [sdk-config-file](sdk-config-file/) | TypeScript SDK | | [sdk-custom-assertion](sdk-custom-assertion/) | TypeScript SDK | | [sdk-eval-authoring](sdk-eval-authoring/) | TypeScript SDK | diff --git a/examples/features/scenarios/README.md b/examples/features/scenarios/README.md new file mode 100644 index 000000000..e6bf50057 --- /dev/null +++ b/examples/features/scenarios/README.md @@ -0,0 +1,79 @@ +# Scenarios Example + +Demonstrates Promptfoo-style `scenarios` authoring with AgentV's current contract: +top-level `prompts`, inline `scenarios`, `scenarios[].config`, +`scenarios[].tests`, reference answers in `vars`, and explicit assertions. + +## What This Shows + +- Crossing each scenario `config` row with each scenario `tests` row +- Mixing inline scenario objects with `file://` scenario refs +- Loading scenario files through a glob +- Keeping reference answers in `vars.expected_translation` +- Consuming reference answers with explicit `equals` assertions +- Running against a deterministic local CLI target + +## Expansion + +The main eval contains one inline Portuguese scenario and one file glob: + +```yaml +scenarios: + - description: Inline Portuguese scenario + config: + - vars: + language: Portuguese + tests: + - id: inline-portuguese-hello + vars: + phrase: hello + expected_translation: ola + assert: + - type: equals + value: "{{ expected_translation }}" + - file://scenarios/*.yaml +``` + +The glob loads `scenarios/french.yaml` and `scenarios/spanish.yaml`. AgentV +flattens those files into the top-level scenario list before lowering each +scenario as `config x tests`. + +For example, the Spanish scenario has one config row and two tests: + +```yaml +config: + - vars: + language: Spanish +tests: + - id: spanish-hello-world + vars: + phrase: hello world + expected_translation: hola mundo +``` + +That row renders the prompt: + +```text +Translate 'hello world' to Spanish. +``` + +The deterministic local CLI target returns `hola mundo`, and the assertion +compares it to the reference answer from `vars.expected_translation`. + +## Running + +```bash +# From repository root +bun apps/cli/src/cli.ts validate examples/features/scenarios/evals/suite.yaml +bun apps/cli/src/cli.ts eval run examples/features/scenarios/evals/suite.yaml \ + --targets examples/features/scenarios/targets.yaml \ + --target translation-cli +``` + +## Key Files + +- `evals/suite.yaml` - Main eval with inline and file-backed scenarios +- `evals/scenarios/french.yaml` - Scenario file loaded by glob +- `evals/scenarios/spanish.yaml` - Scenario file loaded by glob +- `targets.yaml` - Deterministic CLI target for running the example +- `scripts/translation-target.mjs` - Prompt-to-translation target script diff --git a/examples/features/scenarios/evals/scenarios/french.yaml b/examples/features/scenarios/evals/scenarios/french.yaml new file mode 100644 index 000000000..6a4f1bb3f --- /dev/null +++ b/examples/features/scenarios/evals/scenarios/french.yaml @@ -0,0 +1,12 @@ +- description: French translation scenario + config: + - vars: + language: French + tests: + - id: french-hello + vars: + phrase: hello + expected_translation: bonjour + assert: + - type: equals + value: "{{ expected_translation }}" diff --git a/examples/features/scenarios/evals/scenarios/spanish.yaml b/examples/features/scenarios/evals/scenarios/spanish.yaml new file mode 100644 index 000000000..71fd22f1f --- /dev/null +++ b/examples/features/scenarios/evals/scenarios/spanish.yaml @@ -0,0 +1,19 @@ +- description: Spanish translation scenario + config: + - vars: + language: Spanish + tests: + - id: spanish-hello-world + vars: + phrase: hello world + expected_translation: hola mundo + assert: + - type: equals + value: "{{ expected_translation }}" + - id: spanish-thank-you + vars: + phrase: thank you + expected_translation: gracias + assert: + - type: equals + value: "{{ expected_translation }}" diff --git a/examples/features/scenarios/evals/suite.yaml b/examples/features/scenarios/evals/suite.yaml new file mode 100644 index 000000000..60b8d568d --- /dev/null +++ b/examples/features/scenarios/evals/suite.yaml @@ -0,0 +1,19 @@ +name: scenarios-demo +description: Demonstrates Promptfoo-style scenario matrices with inline and file-backed scenarios. +version: "1.0" +prompts: + - "Translate '{{ phrase }}' to {{ language }}." +scenarios: + - description: Inline Portuguese scenario + config: + - vars: + language: Portuguese + tests: + - id: inline-portuguese-hello + vars: + phrase: hello + expected_translation: ola + assert: + - type: equals + value: "{{ expected_translation }}" + - file://scenarios/*.yaml diff --git a/examples/features/scenarios/scripts/translation-target.mjs b/examples/features/scenarios/scripts/translation-target.mjs new file mode 100644 index 000000000..8b944d686 --- /dev/null +++ b/examples/features/scenarios/scripts/translation-target.mjs @@ -0,0 +1,16 @@ +import { readFileSync, writeFileSync } from 'node:fs'; + +const promptFile = process.argv[2]; +const outputFile = process.argv[3]; + +const translations = new Map([ + ["Translate 'hello' to Portuguese.", 'ola'], + ["Translate 'hello' to French.", 'bonjour'], + ["Translate 'hello world' to Spanish.", 'hola mundo'], + ["Translate 'thank you' to Spanish.", 'gracias'], +]); + +const prompt = readFileSync(promptFile, 'utf8').trim(); +const text = translations.get(prompt) ?? `unexpected prompt: ${prompt}`; + +writeFileSync(outputFile, JSON.stringify({ text })); diff --git a/examples/features/scenarios/targets.yaml b/examples/features/scenarios/targets.yaml new file mode 100644 index 000000000..f8b5d7e76 --- /dev/null +++ b/examples/features/scenarios/targets.yaml @@ -0,0 +1,4 @@ +targets: + - id: translation-cli + provider: cli + command: "node ../scripts/translation-target.mjs {PROMPT_FILE} {OUTPUT_FILE}" diff --git a/packages/core/src/evaluation/loaders/scenario-file-loader.ts b/packages/core/src/evaluation/loaders/scenario-file-loader.ts new file mode 100644 index 000000000..f4400933e --- /dev/null +++ b/packages/core/src/evaluation/loaders/scenario-file-loader.ts @@ -0,0 +1,129 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import fg from 'fast-glob'; + +import { interpolateEnv } from '../interpolation.js'; +import type { JsonObject, JsonValue } from '../types.js'; +import { isJsonObject } from '../types.js'; +import { parseYamlValue } from '../yaml-loader.js'; + +const ANSI_YELLOW = '\u001b[33m'; +const ANSI_RESET = '\u001b[0m'; +const FILE_PROTOCOL = 'file://'; + +export function isScenarioFileReference(value: JsonValue): value is string { + return typeof value === 'string' && value.startsWith(FILE_PROTOCOL); +} + +function stripFileProtocol(value: string): string { + return value.startsWith(FILE_PROTOCOL) ? value.slice(FILE_PROTOCOL.length) : value; +} + +function isGlobPattern(filePath: string): boolean { + return filePath.includes('*') || filePath.includes('?') || filePath.includes('{'); +} + +function parseScenarioFileValue(value: unknown, filePath: string): JsonObject[] { + const parsed = interpolateEnv(value, process.env); + const rawScenarios = Array.isArray(parsed) + ? parsed + : isJsonObject(parsed) && Array.isArray(parsed.scenarios) + ? parsed.scenarios + : undefined; + + if (!rawScenarios) { + throw new Error( + `External scenario file must contain a scenario array or an object with a scenarios array: ${filePath}`, + ); + } + + return rawScenarios.map((scenario, index) => { + if (!isJsonObject(scenario)) { + throw new Error( + `External scenario file contains non-object entry at index ${index}: ${filePath}`, + ); + } + return scenario; + }); +} + +export async function loadScenariosFromFile(filePath: string): Promise { + let content: string; + try { + content = await readFile(filePath, 'utf8'); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Cannot read external scenario file: ${filePath}\n ${message}`); + } + + if (content.trim() === '') { + console.warn( + `${ANSI_YELLOW}Warning: External scenario file is empty, skipping: ${filePath}${ANSI_RESET}`, + ); + return []; + } + + const ext = path.extname(filePath).toLowerCase(); + if (ext === '.yaml' || ext === '.yml') { + return parseScenarioFileValue(parseYamlValue(content), filePath); + } + if (ext === '.json') { + try { + return parseScenarioFileValue(JSON.parse(content) as unknown, filePath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Malformed JSON scenario file: ${message}\n File: ${filePath}`); + } + } + + throw new Error( + `Unsupported external scenario file format '${ext}': ${filePath}. Supported: .json, .yaml, .yml`, + ); +} + +export async function resolveScenarioFileReference( + ref: string, + evalFileDir: string, +): Promise { + const rawPath = stripFileProtocol(ref); + const absolutePattern = path.resolve(evalFileDir, rawPath); + + if (isGlobPattern(rawPath)) { + const matches = ( + await fg(absolutePattern.replaceAll('\\', '/'), { + onlyFiles: true, + absolute: true, + }) + ).sort(); + + if (matches.length === 0) { + console.warn( + `${ANSI_YELLOW}Warning: Glob pattern matched no scenario files: ${ref} (resolved to ${absolutePattern})${ANSI_RESET}`, + ); + return []; + } + + const scenarios: JsonObject[] = []; + for (const match of matches) { + scenarios.push(...(await loadScenariosFromFile(match))); + } + return scenarios; + } + + return loadScenariosFromFile(absolutePattern); +} + +export async function expandScenarioReferences( + scenarios: readonly JsonValue[], + evalFileDir: string, +): Promise { + const expanded: JsonValue[] = []; + for (const scenario of scenarios) { + if (isScenarioFileReference(scenario)) { + expanded.push(...(await resolveScenarioFileReference(scenario, evalFileDir))); + } else { + expanded.push(scenario); + } + } + return expanded; +} diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index bc56c77b6..3632143e1 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -845,6 +845,12 @@ const ScenarioSchema = z tests: z.array(EvalTestSchema), }) .strict(); +const ScenarioReferenceSchema = z + .string() + .min(1) + .refine((value) => value.startsWith('file://'), { + message: 'Scenario references must use file://...', + }); const DerivedMetricSchema = z .object({ @@ -859,7 +865,7 @@ const TagsSchema = z.union([ ]); const TOP_LEVEL_IMPORTS_MESSAGE = - "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds."; + "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds."; // --------------------------------------------------------------------------- // Top-level eval file @@ -930,7 +936,7 @@ export const EvalFileSchemaInput: z.ZodType = z.object({ threshold: z.number().min(0).max(1).optional(), default_test: z.union([DefaultTestReferenceSchema, DefaultTestSchema]).optional(), environment: EnvironmentSchema.optional(), - scenarios: z.array(ScenarioSchema).optional(), + scenarios: z.array(z.union([ScenarioSchema, ScenarioReferenceSchema])).optional(), derived_metrics: z.array(DerivedMetricSchema).optional(), output_path: z.union([z.string().min(1), z.array(z.string().min(1))]).optional(), env: z.record(z.string()).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 42fa11b78..27f0fd63e 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -7,6 +7,11 @@ import { loadCasesFromDirectory, loadCasesFromFile } from '../loaders/case-file- import { resolveEnvironmentRecipe } from '../loaders/environment-recipe.js'; import { buildSearchRoots } from '../loaders/file-resolver.js'; import { loadPromptMdFallback } from '../loaders/prompt-md-fallback.js'; +import { + isScenarioFileReference, + resolveScenarioFileReference, +} from '../loaders/scenario-file-loader.js'; +import type { JsonValue as EvalJsonValue } from '../types.js'; import { isGraderKind } from '../types.js'; import { parseYamlValue } from '../yaml-loader.js'; import type { ValidationError, ValidationResult } from './types.js'; @@ -289,7 +294,7 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ const REMOVED_TOP_LEVEL_FIELDS = new Map([ [ 'imports', - "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.", + "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.", ], [ 'expected_output', @@ -589,10 +594,15 @@ export async function validateEvalFile(filePath: string): Promise 0; - const hasScenarios = Array.isArray(parsed.scenarios); + const hasScenarios = Array.isArray(expandedScenarios); // tests can be a string path (external file/directory reference) or an array if (typeof cases === 'string') { @@ -1560,6 +1570,43 @@ async function validateScenarios( } } +async function expandScenariosForValidation( + scenarios: JsonValue | undefined, + filePath: string, + errors: ValidationError[], +): Promise { + if (!Array.isArray(scenarios)) { + return scenarios; + } + + const expanded: JsonValue[] = []; + for (let scenarioIndex = 0; scenarioIndex < scenarios.length; scenarioIndex++) { + const scenario = scenarios[scenarioIndex]; + if (!isScenarioFileReference(scenario as EvalJsonValue)) { + expanded.push(scenario); + continue; + } + + try { + const externalScenarios = await resolveScenarioFileReference( + scenario, + path.dirname(filePath), + ); + expanded.push(...(externalScenarios as JsonValue[])); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + errors.push({ + severity: 'error', + filePath, + location: `scenarios[${scenarioIndex}]`, + message, + }); + } + } + + return expanded; +} + async function validateScenarioTestLikeRow( row: JsonValue | undefined, location: string, diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index cea22a086..7c8461980 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -55,6 +55,7 @@ import { import { detectFormat, loadTestsFromJsonl } from './loaders/jsonl-parser.js'; import { processExpectedMessages, processMessages } from './loaders/message-processor.js'; import { loadPromptMdFallback } from './loaders/prompt-md-fallback.js'; +import { expandScenarioReferences } from './loaders/scenario-file-loader.js'; import { expandInputShorthand, resolveExpectedMessages, @@ -807,7 +808,7 @@ function rejectAuthoredPostprocess(suite: RawTestSuite): void { function rejectTopLevelImports(suite: JsonObject): void { if (suite.imports !== undefined) { throw new Error( - "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.", + "Top-level 'imports' is not supported. Run eval files directly with CLI multi-file selection and tags for grouping. For raw case files, use tests: file://... or string entries under tests. For reusable scenarios, use scenarios: [file://...]. For reusable config, use prompts: file://..., default_test: file://..., and environment: file://... for coding-agent testbeds.", ); } } @@ -1494,7 +1495,14 @@ async function loadTestsFromParsedYamlValue( throw new Error(`Invalid test file format: ${evalFilePath} - missing 'tests' field`); } - const scenarioTestCases = lowerScenariosIntoTests(suite, evalFilePath); + const expandedScenarios = Array.isArray(suite.scenarios) + ? await expandScenarioReferences(suite.scenarios, evalFileDir) + : suite.scenarios; + const scenarioSuite = + expandedScenarios === undefined + ? suite + : ({ ...suite, scenarios: expandedScenarios } as RawTestSuite); + const scenarioTestCases = lowerScenariosIntoTests(scenarioSuite, evalFilePath); if (scenarioTestCases.length > 0) { expandedTestCases = [...expandedTestCases, ...scenarioTestCases]; } diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index d8d34a055..0bc527185 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -1071,6 +1071,82 @@ scenarios: expect(cases.map((test) => test.question)).toEqual(['Alpha critical fix', 'Beta critical fix']); }); + it('expands inline scenarios and scenario file glob refs before lowering', async () => { + const yamlPath = path.join(tempDir, 'scenario-file-refs.yaml'); + const scenarioDir = path.join(tempDir, 'scenarios'); + await mkdir(scenarioDir, { recursive: true }); + await writeFile( + path.join(scenarioDir, 'french.yaml'), + `- description: French translations + config: + - vars: + language: French + tests: + - id: translate-french + vars: + phrase: hello + provider_output: bonjour + assert: + - type: equals + value: bonjour +`, + ); + await writeFile( + path.join(scenarioDir, 'spanish.yaml'), + `- description: Spanish translations + config: + - vars: + language: Spanish + tests: + - id: translate-spanish + vars: + phrase: goodbye + provider_output: adios + assert: + - type: equals + value: adios +`, + ); + await writeFile( + yamlPath, + `prompts: + - "Translate {{ phrase }} to {{ language }}" +scenarios: + - description: Italian translations + config: + - vars: + language: Italian + tests: + - id: translate-italian + vars: + phrase: hello + provider_output: ciao + assert: + - type: equals + value: ciao + - file://scenarios/*.yaml +`, + ); + + const cases = await loadTests(yamlPath, tempDir); + + expect(cases.map((test) => test.id)).toEqual([ + 'translate-italian', + 'translate-french', + 'translate-spanish', + ]); + expect(cases.map((test) => test.question)).toEqual([ + 'Translate hello to Italian', + 'Translate hello to French', + 'Translate goodbye to Spanish', + ]); + expect(cases.map((test) => test.vars)).toEqual([ + { language: 'Italian', phrase: 'hello' }, + { language: 'French', phrase: 'hello' }, + { language: 'Spanish', phrase: 'goodbye' }, + ]); + }); + it('rejects malformed scenarios at load time', async () => { const yamlPath = path.join(tempDir, 'malformed-scenario.yaml'); await writeFile( @@ -1092,6 +1168,25 @@ scenarios: ); }); + it('rejects malformed external scenario files at load time', async () => { + const yamlPath = path.join(tempDir, 'malformed-scenario-file-ref.yaml'); + const scenarioDir = path.join(tempDir, 'malformed-scenarios'); + await mkdir(scenarioDir, { recursive: true }); + await writeFile(path.join(scenarioDir, 'broken.yaml'), 'not: a scenario array\n'); + await writeFile( + yamlPath, + `prompts: + - "Review {{ diff }}" +scenarios: + - file://malformed-scenarios/broken.yaml +`, + ); + + await expect(loadTests(yamlPath, tempDir)).rejects.toThrow( + 'External scenario file must contain a scenario array', + ); + }); + it('hard-errors removed fields inside scenarios at load time', async () => { const yamlPath = path.join(tempDir, 'scenario-removed-field.yaml'); await writeFile( diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 279cab109..346584d83 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -77,6 +77,21 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('accepts scenario file references in top-level scenarios arrays', () => { + const result = EvalFileSchema.safeParse({ + prompts: ['Translate {{ phrase }} to {{ language }}'], + scenarios: [ + { + config: [{ vars: { language: 'Spanish' } }], + tests: [{ vars: { phrase: 'hello' }, provider_output: 'hola' }], + }, + 'file://scenarios/*.yaml', + ], + }); + + expect(result.success).toBe(true); + }); + it('requires scenario config and tests arrays', () => { const result = EvalFileSchema.safeParse({ prompts: ['Classify {{ request }}'], diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index b02493db4..7d080ed0e 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -551,6 +551,66 @@ scenarios: expect(result.errors).toHaveLength(0); }); + it('validates top-level scenario file refs and globs', async () => { + const filePath = path.join(tempDir, 'scenario-file-refs.yaml'); + const scenarioDir = path.join(tempDir, 'scenario-file-refs'); + await mkdir(scenarioDir, { recursive: true }); + await writeFile( + path.join(scenarioDir, 'french.yaml'), + `- description: French translations + config: + - vars: + language: French + tests: + - vars: + phrase: hello + provider_output: bonjour + assert: + - type: equals + value: bonjour +`, + ); + await writeFile( + path.join(scenarioDir, 'spanish.yaml'), + `- description: Spanish translations + config: + - vars: + language: Spanish + tests: + - vars: + phrase: goodbye + provider_output: adios + assert: + - type: equals + value: adios +`, + ); + await writeFile( + filePath, + `prompts: + - "Translate {{ vars.phrase }} to {{ vars.language }}" +scenarios: + - description: inline Italian translation + config: + - vars: + language: Italian + tests: + - vars: + phrase: hello + provider_output: ciao + assert: + - type: equals + value: ciao + - file://scenario-file-refs/*.yaml +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + it('rejects scenario tests without input when only some config rows supply prompts', async () => { const filePath = path.join(tempDir, 'scenario-mixed-config-prompts.yaml'); await writeFile( @@ -622,6 +682,32 @@ scenarios: ); }); + it('rejects malformed external scenario files', async () => { + const filePath = path.join(tempDir, 'malformed-scenario-file-ref.yaml'); + const scenarioDir = path.join(tempDir, 'malformed-scenario-file-ref'); + await mkdir(scenarioDir, { recursive: true }); + await writeFile(path.join(scenarioDir, 'broken.yaml'), '- not-an-object\n'); + await writeFile( + filePath, + `prompts: + - "Review {{ vars.diff }}" +scenarios: + - file://malformed-scenario-file-ref/broken.yaml +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'scenarios[0]', + message: expect.stringContaining('External scenario file contains non-object entry'), + }), + ); + }); + it('rejects removed fields inside scenarios', async () => { const filePath = path.join(tempDir, 'scenario-removed-fields.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 305863939..ac36b0f08 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -4761,65 +4761,236 @@ "scenarios": { "type": "array", "items": { - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "config": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "vars": { + "anyOf": [ + { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "config": { + "type": "array", + "items": { "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "provider": { - "anyOf": [ - { + "properties": { + "id": { "type": "string", "minLength": 1 }, - { + "description": { + "type": "string" + }, + "vars": { "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { + "properties": {}, + "additionalProperties": {} + }, + "provider": { + "anyOf": [ + { "type": "string", "minLength": 1 }, - "config": { + { "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "extends": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "transform": { "anyOf": [ { "type": "string" }, { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { + "type": "string", + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { "type": "object", "properties": { "command": { @@ -4835,334 +5006,334 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, "additionalProperties": false }, - { + "before_each": { "type": "object", "properties": { - "id": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "label": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "raw": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "function": { + "cwd": { "type": "string" }, - "function_file": { - "type": "string" + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "path": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "prefix": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "suffix": { + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } + } + ] }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } + "timeout_ms": { + "type": "number" }, - "additionalProperties": true - } - ] + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + } }, - "minItems": 1 + "additionalProperties": false } - ] - }, - "transform": { + }, + "additionalProperties": true + } + ] + }, + "providers": { + "anyOf": [ + { "anyOf": [ { - "type": "string" + "type": "string", + "minLength": 1 }, { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { "type": "object", "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" + "id": { + "type": "string", + "minLength": 1 }, - "timeoutMs": { - "type": "number" + "extends": { + "type": "string", + "minLength": 1 }, - "cwd": { - "type": "string" + "provider": { + "type": "string", + "minLength": 1 }, - "reset": { + "model": { "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" + "minLength": 1 }, - "cwd": { - "type": "string" + "config": { + "type": "object", + "additionalProperties": {} }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { + "prompts": { "anyOf": [ { - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] }, { "type": "array", "items": { - "type": "string" - } + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 } ] }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { + "transform": { "anyOf": [ { "type": "string" }, { - "type": "array", - "items": { - "type": "string" - } + "type": "object", + "properties": {}, + "additionalProperties": {} } ] }, - "timeout_ms": { - "type": "number" + "delay": { + "type": "number", + "minimum": 0 }, - "timeoutMs": { - "type": "number" + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } }, - "cwd": { - "type": "string" + "environment": { + "not": {} }, - "reset": { + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - "providers": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { "type": "object", "properties": { "command": { @@ -5178,334 +5349,398 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "required": ["command"], "additionalProperties": false }, - { + "before_each": { "type": "object", "properties": { - "id": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "label": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "raw": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "function": { + "cwd": { "type": "string" }, - "function_file": { - "type": "string" + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "path": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "prefix": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "suffix": { + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } + } + ] }, - "required": ["command"], - "additionalProperties": false + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "extends": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { + "anyOf": [ + { "type": "string" }, - "suffix": { - "type": "string" + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false }, - "config": { + { "type": "object", - "additionalProperties": {} + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] }, - "additionalProperties": true + "minItems": 1 } ] }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { + "transform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] } }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { + "type": "string", + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" }, - { - "type": "array", - "items": { + "additionalProperties": false + }, + "before_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" + "additionalProperties": false }, - { + "after_each": { "type": "object", "properties": { "command": { @@ -5521,546 +5756,269 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "required": ["command"], "additionalProperties": false }, - { + "after_all": { "type": "object", "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "path": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "prefix": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "suffix": { + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true + "additionalProperties": false } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} + "additionalProperties": false } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} } }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } }, - "reasoning_effort": { - "type": "string", - "minLength": 1 + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" }, - "hooks": { + { "type": "object", "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { + "command": { + "anyOf": [ + { "type": "string" }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] + { + "type": "array", + "items": { + "type": "string" + } } - }, - "additionalProperties": false + ] }, - "before_each": { + "config": { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "label": { + "type": "string" }, - "after_all": { + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "additionalProperties": {} } }, - "additionalProperties": false + "additionalProperties": true } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "prompts": { - "anyOf": [ - { + ] + }, + "minItems": 1 + } + ] + }, + "provider_output": { "anyOf": [ { "type": "string" }, { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false + "properties": {}, + "additionalProperties": {} }, { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { + "type": "array", + "items": { "type": "object", "properties": { - "command": { + "role": { + "type": "string", + "enum": ["system", "user", "assistant", "tool"] + }, + "content": { "anyOf": [ { "type": "string" }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false } } ] - }, - "config": { - "type": "object", - "additionalProperties": {} } }, - "required": ["command"], + "required": ["role", "content"], "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true } - ] - }, - "minItems": 1 - } - ] - }, - "provider_output": { - "anyOf": [ - { - "type": "string" + } + ] }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} + "input": { + "not": {} }, - { + "input_files": { "type": "array", "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, - "input": { - "not": {} - }, - "input_files": { - "type": "array", - "items": { - "type": "string" - } - }, - "expected_output": { - "not": {} - }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} } - ] - } - }, - "assert_scoring_function": { - "anyOf": [ - { - "type": "string", - "minLength": 1 }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "options": { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "execution": { - "type": "object", - "properties": { - "workers": { + "expected_output": { "not": {} }, "assert": { @@ -6078,459 +6036,672 @@ ] } }, - "skip_defaults": { - "type": "boolean" - }, - "cache": { - "type": "boolean" - }, - "trials": { - "not": {} - }, - "budget_usd": { - "type": "number", - "minimum": 0 - }, - "budgetUsd": { - "type": "number", - "minimum": 0 - }, - "fail_on_error": { - "type": "boolean" + "assert_scoring_function": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] }, - "failOnError": { - "type": "boolean" + "options": { + "type": "object", + "properties": {}, + "additionalProperties": {} }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - } - }, - "additionalProperties": false - }, - "run": { - "type": "object", - "properties": { "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, - "repeat": { + "execution": { "type": "object", "properties": { - "count": { - "type": "integer", - "minimum": 1 + "workers": { + "not": {} }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } }, - "early_exit": { + "skip_defaults": { "type": "boolean" }, - "cost_limit_usd": { + "cache": { + "type": "boolean" + }, + "trials": { + "not": {} + }, + "budget_usd": { "type": "number", "minimum": 0 + }, + "budgetUsd": { + "type": "number", + "minimum": 0 + }, + "fail_on_error": { + "type": "boolean" + }, + "failOnError": { + "type": "boolean" + }, + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 } }, - "required": ["count"], "additionalProperties": false }, - "timeout_seconds": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "budget_usd": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "additionalProperties": false - }, - "environment": { - "anyOf": [ - { - "type": "string", - "pattern": "^\\s*file:\\/\\/" - }, - { + "run": { "type": "object", "properties": { - "workdir": { - "type": "string", - "minLength": 1 + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "required": ["command"], - "additionalProperties": false + "repeat": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "minimum": 1 + }, + "strategy": { + "type": "string", + "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] + }, + "early_exit": { + "type": "boolean" + }, + "cost_limit_usd": { + "type": "number", + "minimum": 0 } - ] + }, + "required": ["count"], + "additionalProperties": false }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 }, - "type": { - "type": "string", - "const": "host" + "budget_usd": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 } }, - "required": ["workdir", "type"], "additionalProperties": false }, - { - "type": "object", - "properties": { - "workdir": { - "type": "string", - "minLength": 1 - }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "required": ["command"], - "additionalProperties": false - } - ] - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "docker" - }, - "context": { - "type": "string", - "minLength": 1 - }, - "dockerfile": { - "type": "string", - "minLength": 1 - }, - "image": { + "environment": { + "anyOf": [ + { "type": "string", - "minLength": 1 + "pattern": "^\\s*file:\\/\\/" }, - "resources": { + { "type": "object", "properties": { - "cpus": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "memory": { - "type": "string", - "minLength": 1 - }, - "disk": { + "workdir": { "type": "string", "minLength": 1 }, - "gpu": { - "anyOf": [ - { - "type": "boolean" - }, + "setup": { + "allOf": [ + {}, { - "type": "string", - "minLength": 1 + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false } ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" } }, + "required": ["workdir", "type"], "additionalProperties": false }, - "mounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source": { - "type": "string", - "minLength": 1 - }, - "target": { - "type": "string", - "minLength": 1 - }, - "access": { - "type": "string", - "enum": ["ro", "rw"] - }, - "read_only": { - "type": "boolean" - } - }, - "required": ["source", "target"], - "additionalProperties": false - } - }, - "secrets": { + { "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "required": ["workdir", "type"], - "additionalProperties": false - } - ] - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "conversation_id": { - "type": "string" - }, - "suite": { - "type": "string" - }, - "depends_on": { - "type": "array", - "items": { - "type": "string" - } - }, - "on_dependency_failure": { - "type": "string", - "enum": ["skip", "fail", "run"] - }, - "mode": { - "type": "string", - "enum": ["conversation"] - }, - "turns": { - "type": "array", - "items": { - "type": "object", - "properties": { - "input": { - "anyOf": [ - { - "type": "string" - }, - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { "type": "object", "properties": { - "type": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { "type": "string", - "enum": ["text", "file", "image"] + "minLength": 1 }, - "value": { - "type": "string" + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 } }, - "required": ["type", "value"], + "required": ["command"], "additionalProperties": false } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" } - ] - } - ] - }, - "expected_output": { - "anyOf": [ - { - "type": "string" + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } }, - { + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "conversation_id": { + "type": "string" + }, + "suite": { + "type": "string" + }, + "depends_on": { + "type": "array", + "items": { + "type": "string" + } + }, + "on_dependency_failure": { + "type": "string", + "enum": ["skip", "fail", "run"] + }, + "mode": { + "type": "string", + "enum": ["conversation"] + }, + "turns": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": { "anyOf": [ { "type": "string" }, { - "type": "object", - "properties": {}, - "additionalProperties": {} + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] + } + ] + }, + "expected_output": { + "anyOf": [ + { + "type": "string" }, { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } + "anyOf": [ + { + "type": "string" }, - "required": ["type", "value"], - "additionalProperties": false - } + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] } ] + }, + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } } - ] + }, + "required": ["input"], + "additionalProperties": false }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - } + "minItems": 1 }, - "required": ["input"], - "additionalProperties": false + "aggregation": { + "type": "string", + "enum": ["mean", "min", "max"] + }, + "on_turn_failure": { + "type": "string", + "enum": ["continue", "stop"] + }, + "window_size": { + "type": "integer", + "minimum": 1 + } }, - "minItems": 1 - }, - "aggregation": { - "type": "string", - "enum": ["mean", "min", "max"] - }, - "on_turn_failure": { - "type": "string", - "enum": ["continue", "stop"] - }, - "window_size": { - "type": "integer", - "minimum": 1 + "additionalProperties": false } }, - "additionalProperties": false - } - }, - "tests": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "vars": { + "tests": { + "type": "array", + "items": { "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "provider": { - "anyOf": [ - { + "properties": { + "id": { "type": "string", "minLength": 1 }, - { + "description": { + "type": "string" + }, + "vars": { "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { + "properties": {}, + "additionalProperties": {} + }, + "provider": { + "anyOf": [ + { "type": "string", "minLength": 1 }, - "config": { + { "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "extends": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "transform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { + "type": "string", + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" }, { "type": "array", @@ -6540,334 +6711,334 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "required": ["command"], "additionalProperties": false }, - { + "before_each": { "type": "object", "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "function_file": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "path": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "prefix": { + "cwd": { "type": "string" }, - "suffix": { + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } + } + ] }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } + "timeout_ms": { + "type": "number" }, - "additionalProperties": true - } - ] + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + } }, - "minItems": 1 + "additionalProperties": false } - ] - }, - "transform": { + }, + "additionalProperties": true + } + ] + }, + "providers": { + "anyOf": [ + { "anyOf": [ { - "type": "string" + "type": "string", + "minLength": 1 }, { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { "type": "object", "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" + "id": { + "type": "string", + "minLength": 1 }, - "reset": { + "extends": { "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] + "minLength": 1 }, - "timeout_ms": { - "type": "number" + "provider": { + "type": "string", + "minLength": 1 }, - "timeoutMs": { - "type": "number" + "model": { + "type": "string", + "minLength": 1 }, - "cwd": { - "type": "string" + "config": { + "type": "object", + "additionalProperties": {} }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { + "prompts": { "anyOf": [ { - "type": "string" - }, + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, { "type": "array", "items": { - "type": "string" - } + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 } ] }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { + "transform": { "anyOf": [ { "type": "string" }, { - "type": "array", - "items": { - "type": "string" - } + "type": "object", + "properties": {}, + "additionalProperties": {} } ] }, - "timeout_ms": { - "type": "number" + "delay": { + "type": "number", + "minimum": 0 }, - "timeoutMs": { - "type": "number" + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } }, - "cwd": { - "type": "string" + "environment": { + "not": {} }, - "reset": { + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - "providers": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { - "anyOf": [ - { - "type": "string" - }, - { + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { "type": "object", "properties": { "command": { @@ -6883,334 +7054,334 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "required": ["command"], "additionalProperties": false }, - { + "before_each": { "type": "object", "properties": { - "id": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "label": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "raw": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "function": { + "cwd": { "type": "string" }, - "function_file": { - "type": "string" + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "path": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "prefix": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "suffix": { + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } + } + ] }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } + "timeout_ms": { + "type": "number" }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 - }, - "hooks": { - "type": "object", - "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" + "timeoutMs": { + "type": "number" }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "before_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "cwd": { "type": "string" }, - { - "type": "array", - "items": { - "type": "string" - } + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] + }, + "additionalProperties": false } }, "additionalProperties": false - }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { + } + }, + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "extends": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false - }, - "after_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "additionalProperties": true + } + ] }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } + "minItems": 1 + } + ] }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, - "provider": { - "type": "string", - "minLength": 1 - }, - "model": { - "type": "string", - "minLength": 1 - }, - "config": { - "type": "object", - "additionalProperties": {} - }, - "prompts": { - "anyOf": [ - { + "transform": { "anyOf": [ { "type": "string" }, { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + }, + "delay": { + "type": "number", + "minimum": 0 + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, + "reasoning_effort": { + "type": "string", + "minLength": 1 + }, + "hooks": { + "type": "object", + "properties": { + "before_all": { "type": "object", "properties": { "command": { @@ -7226,959 +7397,796 @@ } ] }, - "config": { - "type": "object", - "additionalProperties": {} + "timeout_ms": { + "type": "number" + }, + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "required": ["command"], "additionalProperties": false }, - { + "before_each": { "type": "object", "properties": { - "id": { - "type": "string" + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "label": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "raw": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "function": { + "cwd": { "type": "string" }, - "function_file": { - "type": "string" + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + }, + "after_each": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] }, - "path": { - "type": "string" + "timeout_ms": { + "type": "number" }, - "prefix": { - "type": "string" + "timeoutMs": { + "type": "number" }, - "suffix": { + "cwd": { "type": "string" }, - "config": { - "type": "object", - "additionalProperties": {} + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] } }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { + "additionalProperties": false + }, + "after_all": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } + } + ] }, - "required": ["command"], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } + "timeout_ms": { + "type": "number" }, - "additionalProperties": true - } - ] + "timeoutMs": { + "type": "number" + }, + "cwd": { + "type": "string" + }, + "reset": { + "type": "string", + "enum": ["none", "fast", "strict"] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + ] + }, + "minItems": 1 + } + ] + }, + "prompts": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string" }, - "minItems": 1 - } - ] - }, - "transform": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "delay": { - "type": "number", - "minimum": 0 + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "config": { + "type": "object", + "additionalProperties": {} + } }, - "env": { - "type": "object", - "additionalProperties": { + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { "type": "string" + }, + "config": { + "type": "object", + "additionalProperties": {} } }, - "environment": { - "not": {} - }, - "container": { - "not": {} - }, - "install": { - "not": {} - }, - "reasoning_effort": { - "type": "string", - "minLength": 1 + "additionalProperties": true + } + ] + }, + { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" }, - "hooks": { + { "type": "object", "properties": { - "before_all": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { + "command": { + "anyOf": [ + { "type": "string" }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] + { + "type": "array", + "items": { + "type": "string" + } } - }, - "additionalProperties": false + ] }, - "before_each": { + "config": { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "additionalProperties": {} + } + }, + "required": ["command"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" }, - "after_each": { - "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "label": { + "type": "string" }, - "after_all": { + "raw": { + "type": "string" + }, + "function": { + "type": "string" + }, + "function_file": { + "type": "string" + }, + "path": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "suffix": { + "type": "string" + }, + "config": { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "timeout_ms": { - "type": "number" - }, - "timeoutMs": { - "type": "number" - }, - "cwd": { - "type": "string" - }, - "reset": { - "type": "string", - "enum": ["none", "fast", "strict"] - } - }, - "additionalProperties": false + "additionalProperties": {} } }, - "additionalProperties": false + "additionalProperties": true } - }, - "additionalProperties": true - } - ] - }, - "minItems": 1 - } - ] - }, - "prompts": { - "anyOf": [ - { + ] + }, + "minItems": 1 + } + ] + }, + "provider_output": { "anyOf": [ { "type": "string" }, { "type": "object", - "properties": { - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "required": ["command"], - "additionalProperties": false + "properties": {}, + "additionalProperties": {} }, { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true - } - ] - }, - { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { + "type": "array", + "items": { "type": "object", "properties": { - "command": { + "role": { + "type": "string", + "enum": ["system", "user", "assistant", "tool"] + }, + "content": { "anyOf": [ { "type": "string" }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false } } ] - }, - "config": { - "type": "object", - "additionalProperties": {} } }, - "required": ["command"], + "required": ["role", "content"], "additionalProperties": false - }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "label": { - "type": "string" - }, - "raw": { - "type": "string" - }, - "function": { - "type": "string" - }, - "function_file": { - "type": "string" - }, - "path": { - "type": "string" - }, - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - }, - "config": { - "type": "object", - "additionalProperties": {} - } - }, - "additionalProperties": true } - ] - }, - "minItems": 1 - } - ] - }, - "provider_output": { - "anyOf": [ - { - "type": "string" + } + ] }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} + "input": { + "not": {} }, - { + "input_files": { "type": "array", "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, - "input": { - "not": {} - }, - "input_files": { - "type": "array", - "items": { - "type": "string" - } - }, - "expected_output": { - "not": {} - }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} } - ] - } - }, - "assert_scoring_function": { - "anyOf": [ - { - "type": "string", - "minLength": 1 }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - }, - "options": { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "execution": { - "type": "object", - "properties": { - "workers": { + "expected_output": { "not": {} }, "assert": { "type": "array", "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - }, - "skip_defaults": { - "type": "boolean" - }, - "cache": { - "type": "boolean" - }, - "trials": { - "not": {} - }, - "budget_usd": { - "type": "number", - "minimum": 0 - }, - "budgetUsd": { - "type": "number", - "minimum": 0 + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } }, - "fail_on_error": { - "type": "boolean" + "assert_scoring_function": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] }, - "failOnError": { - "type": "boolean" + "options": { + "type": "object", + "properties": {}, + "additionalProperties": {} }, - "threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - } - }, - "additionalProperties": false - }, - "run": { - "type": "object", - "properties": { "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, - "repeat": { + "execution": { "type": "object", "properties": { - "count": { - "type": "integer", - "minimum": 1 + "workers": { + "not": {} }, - "strategy": { - "type": "string", - "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } + }, + "skip_defaults": { + "type": "boolean" }, - "early_exit": { + "cache": { "type": "boolean" }, - "cost_limit_usd": { + "trials": { + "not": {} + }, + "budget_usd": { + "type": "number", + "minimum": 0 + }, + "budgetUsd": { "type": "number", "minimum": 0 + }, + "fail_on_error": { + "type": "boolean" + }, + "failOnError": { + "type": "boolean" + }, + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 } }, - "required": ["count"], "additionalProperties": false }, - "timeout_seconds": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - }, - "budget_usd": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "additionalProperties": false - }, - "environment": { - "anyOf": [ - { - "type": "string", - "pattern": "^\\s*file:\\/\\/" - }, - { + "run": { "type": "object", "properties": { - "workdir": { - "type": "string", - "minLength": 1 + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 - } - }, - "required": ["command"], - "additionalProperties": false + "repeat": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "minimum": 1 + }, + "strategy": { + "type": "string", + "enum": ["pass_any", "pass_all", "mean", "confidence_interval"] + }, + "early_exit": { + "type": "boolean" + }, + "cost_limit_usd": { + "type": "number", + "minimum": 0 } - ] + }, + "required": ["count"], + "additionalProperties": false }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 }, - "type": { - "type": "string", - "const": "host" + "budget_usd": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 } }, - "required": ["workdir", "type"], "additionalProperties": false }, - { - "type": "object", - "properties": { - "workdir": { + "environment": { + "anyOf": [ + { "type": "string", - "minLength": 1 + "pattern": "^\\s*file:\\/\\/" }, - "setup": { - "allOf": [ - {}, - { - "type": "object", - "properties": { - "command": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } }, - "minItems": 1 - }, - "cwd": { - "type": "string", - "minLength": 1 - }, - "timeout_ms": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 + "required": ["command"], + "additionalProperties": false } - }, - "required": ["command"], - "additionalProperties": false + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" } - ] - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "docker" - }, - "context": { - "type": "string", - "minLength": 1 - }, - "dockerfile": { - "type": "string", - "minLength": 1 - }, - "image": { - "type": "string", - "minLength": 1 + }, + "required": ["workdir", "type"], + "additionalProperties": false }, - "resources": { + { "type": "object", "properties": { - "cpus": { - "type": "number", - "exclusiveMinimum": true, - "minimum": 0 + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "allOf": [ + {}, + { + "type": "object", + "properties": { + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "cwd": { + "type": "string", + "minLength": 1 + }, + "timeout_ms": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + } + ] + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } }, - "memory": { + "type": { + "type": "string", + "const": "docker" + }, + "context": { "type": "string", "minLength": 1 }, - "disk": { + "dockerfile": { "type": "string", "minLength": 1 }, - "gpu": { - "anyOf": [ - { - "type": "boolean" + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 }, - { + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { "type": "string", "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] } - ] - } - }, - "additionalProperties": false - }, - "mounts": { - "type": "array", - "items": { - "type": "object", - "properties": { - "source": { - "type": "string", - "minLength": 1 - }, - "target": { - "type": "string", - "minLength": 1 }, - "access": { - "type": "string", - "enum": ["ro", "rw"] - }, - "read_only": { - "type": "boolean" + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false } }, - "required": ["source", "target"], - "additionalProperties": false - } - }, - "secrets": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "required": ["workdir", "type"], - "additionalProperties": false - } - ] - }, - "metadata": { - "type": "object", - "additionalProperties": {} - }, - "conversation_id": { - "type": "string" - }, - "suite": { - "type": "string" - }, - "depends_on": { - "type": "array", - "items": { - "type": "string" - } - }, - "on_dependency_failure": { - "type": "string", - "enum": ["skip", "fail", "run"] - }, - "mode": { - "type": "string", - "enum": ["conversation"] - }, - "turns": { - "type": "array", - "items": { - "type": "object", - "properties": { - "input": { - "anyOf": [ - { - "type": "string" + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } }, - { + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "metadata": { + "type": "object", + "additionalProperties": {} + }, + "conversation_id": { + "type": "string" + }, + "suite": { + "type": "string" + }, + "depends_on": { + "type": "array", + "items": { + "type": "string" + } + }, + "on_dependency_failure": { + "type": "string", + "enum": ["skip", "fail", "run"] + }, + "mode": { + "type": "string", + "enum": ["conversation"] + }, + "turns": { + "type": "array", + "items": { + "type": "object", + "properties": { + "input": { "anyOf": [ { "type": "string" }, { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } + "anyOf": [ + { + "type": "string" }, - "required": ["type", "value"], - "additionalProperties": false - } + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] } ] - } - ] - }, - "expected_output": { - "anyOf": [ - { - "type": "string" }, - { + "expected_output": { "anyOf": [ { "type": "string" }, { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } + "anyOf": [ + { + "type": "string" }, - "required": ["type", "value"], - "additionalProperties": false - } + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["text", "file", "image"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + } + ] } ] + }, + "assert": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + } + ] + } } - ] + }, + "required": ["input"], + "additionalProperties": false }, - "assert": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - } - ] - } - } + "minItems": 1 }, - "required": ["input"], - "additionalProperties": false + "aggregation": { + "type": "string", + "enum": ["mean", "min", "max"] + }, + "on_turn_failure": { + "type": "string", + "enum": ["continue", "stop"] + }, + "window_size": { + "type": "integer", + "minimum": 1 + } }, - "minItems": 1 - }, - "aggregation": { - "type": "string", - "enum": ["mean", "min", "max"] - }, - "on_turn_failure": { - "type": "string", - "enum": ["continue", "stop"] - }, - "window_size": { - "type": "integer", - "minimum": 1 + "additionalProperties": false } - }, - "additionalProperties": false - } + } + }, + "required": ["config", "tests"], + "additionalProperties": false + }, + { + "type": "string", + "minLength": 1 } - }, - "required": ["config", "tests"], - "additionalProperties": false + ] } }, "derived_metrics": { From 253aa6d6a186f270c346740707557a56668fd564 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Jul 2026 10:29:08 +0200 Subject: [PATCH 2/2] fix(evaluation): reject authored provider output --- .../docs/docs/next/evaluation/eval-files.mdx | 12 +- examples/features/external-datasets/README.md | 7 +- .../external-datasets/evals/cases/magic.csv | 4 +- .../evaluation/loaders/case-file-loader.ts | 6 +- .../evaluation/validation/eval-file.schema.ts | 75 +++--- .../evaluation/validation/eval-validator.ts | 30 ++- packages/core/src/evaluation/yaml-parser.ts | 29 +++ .../loaders/case-file-loader.test.ts | 21 +- .../evaluation/loaders/jsonl-parser.test.ts | 3 - .../validation/eval-file-schema.test.ts | 20 +- .../validation/eval-validator.test.ts | 59 ++++- .../references/eval.schema.json | 226 +----------------- 12 files changed, 195 insertions(+), 297 deletions(-) diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index e3583933b..71975aeee 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -566,11 +566,13 @@ mini-DSL (`contains:*`, `icontains:*`, `contains-any:*`, `contains-all:*`, `file://*.py`; file paths inside CSV cells are resolved relative to the CSV file). Unsupported assertion forms such as `similar:*` are rejected during validation instead of being skipped at runtime. -`__provider_output` becomes first-class `expected_output` reference data, -`__metric` names the generated assertions, `__threshold` sets the test threshold, -`__metadata:` adds metadata, and `__config:__expectedN:threshold` sets an -assertion `min_score`. Ordinary columns become `vars`, so CSV rows can rely on -suite-level `prompts` that interpolate those variables. +`__provider_output` is rejected; use an explicit deterministic target such as +`provider: cli` for fixed outputs, or a replay/fixture target for captured +provider responses. `__metric` names the generated assertions, `__threshold` +sets the test threshold, `__metadata:` adds metadata, and +`__config:__expectedN:threshold` sets an assertion `min_score`. Ordinary +columns become `vars`, so CSV rows can rely on suite-level `prompts` that +interpolate those variables. String shorthand is raw-case-only. Use a direct raw-case path or file ref when you want case data to run in the parent suite context: diff --git a/examples/features/external-datasets/README.md b/examples/features/external-datasets/README.md index f1a0e7eb2..5f294f455 100644 --- a/examples/features/external-datasets/README.md +++ b/examples/features/external-datasets/README.md @@ -50,15 +50,16 @@ One JSON test object per line: CSV files use ordinary columns for `id`, `input`, and `vars`, plus promptfoo-style magic columns for assertions and metadata: ```csv -id,input,__expected,__provider_output,__metric,__threshold,__metadata:source,locale -csv-test,Reply with a greeting,icontains:hello,Hello there,greeting,0.8,csv,en-US +id,input,__expected,__metric,__threshold,__metadata:source,locale +csv-test,Reply with a greeting,icontains:hello,greeting,0.8,csv,en-US ``` `__expected` and `__expectedN` become AgentV assertions for the supported CSV mini-DSL. `latency()`, `cost()`, and `file://*.py` map to runnable AgentV graders, with CSV file paths resolved relative to the CSV file; unsupported promptfoo forms such as `similar:*` are rejected during validation. -`__provider_output` becomes AgentV `expected_output`; ordinary non-magic +Use an explicit deterministic target such as `provider: cli` for fixed outputs, +or a replay/fixture target for captured provider responses. Ordinary non-magic columns such as `locale` become `vars` and can be interpolated by suite-level `input`. diff --git a/examples/features/external-datasets/evals/cases/magic.csv b/examples/features/external-datasets/evals/cases/magic.csv index 2ee757062..16c8098ff 100644 --- a/examples/features/external-datasets/evals/cases/magic.csv +++ b/examples/features/external-datasets/evals/cases/magic.csv @@ -1,2 +1,2 @@ -id,vars.input,__expected,__provider_output,__metric,__threshold,__metadata:source,locale -csv-magic-greeting,Reply with a short greeting,icontains:hello,Hello there,greeting,0.8,csv,en-US +id,vars.input,__expected,__metric,__threshold,__metadata:source,locale +csv-magic-greeting,Reply with a short greeting,icontains:hello,greeting,0.8,csv,en-US diff --git a/packages/core/src/evaluation/loaders/case-file-loader.ts b/packages/core/src/evaluation/loaders/case-file-loader.ts index 43f4d5842..1ec10f748 100644 --- a/packages/core/src/evaluation/loaders/case-file-loader.ts +++ b/packages/core/src/evaluation/loaders/case-file-loader.ts @@ -357,7 +357,6 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] { let prefix = ''; let suffix = ''; let criteria: string | undefined; - let expectedOutput: string | undefined; let metric: string | undefined; let threshold: number | undefined; @@ -379,7 +378,9 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] { } else if (key === '__description') { criteria = value; } else if (key === '__provider_output' || key === '__providerOutput') { - expectedOutput = value; + throw new Error( + `${key} has been removed from CSV imports. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.`, + ); } else if (key === '__metric') { metric = value; } else if (key === '__threshold') { @@ -435,7 +436,6 @@ function parseCsvCases(content: string, filePath: string): JsonObject[] { id: id && id.trim() !== '' ? id : `row-${rowIndex + 1}`, ...(caseInput !== undefined ? { input: caseInput } : {}), ...(criteria ? { criteria } : {}), - ...(expectedOutput ? { expected_output: expectedOutput } : {}), ...(assertions.length > 0 ? { assert: assertions } : {}), ...(threshold !== undefined ? { threshold } : {}), ...(threshold !== undefined ? { execution: { threshold } } : {}), diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3632143e1..6ce12750c 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -700,7 +700,6 @@ const DefaultTestSchema = z provider: EvalTargetSchema.optional(), providers: EvalTargetsSchema.optional(), prompts: PromptsSchema.optional(), - provider_output: ExpectedOutputSchema.optional(), expected_output: z.never().optional(), assert: z.array(AssertionItemSchema).optional(), assert_scoring_function: z.union([z.string().min(1), JsonObjectSchema]).optional(), @@ -741,35 +740,49 @@ const ConversationTurnSchema = z.object({ const TestExecutionSchema = ExecutionSchema.omit({ target: true, targets: true }).strict(); -const EvalTestSchema = z.object({ - id: z.string().min(1).optional(), - description: z.string().optional(), - vars: JsonObjectSchema.optional(), - provider: EvalTargetSchema.optional(), - providers: EvalTargetsSchema.optional(), - prompts: PromptsSchema.optional(), - provider_output: ExpectedOutputSchema.optional(), - input: z.never().optional(), - input_files: z.array(z.string()).optional(), - expected_output: z.never().optional(), - assert: z.array(AssertionItemSchema).optional(), - assert_scoring_function: z.union([z.string().min(1), JsonObjectSchema]).optional(), - options: JsonObjectSchema.optional(), - threshold: z.number().min(0).max(1).optional(), - execution: TestExecutionSchema.optional(), - run: RunOverrideSchema.optional(), - environment: EnvironmentSchema.optional(), - metadata: z.record(z.unknown()).optional(), - conversation_id: z.string().optional(), - suite: z.string().optional(), - depends_on: z.array(z.string()).optional(), - on_dependency_failure: z.enum(['skip', 'fail', 'run']).optional(), - mode: z.enum(['conversation']).optional(), - turns: z.array(ConversationTurnSchema).min(1).optional(), - aggregation: z.enum(['mean', 'min', 'max']).optional(), - on_turn_failure: z.enum(['continue', 'stop']).optional(), - window_size: z.number().int().min(1).optional(), -}); +const EvalTestBaseSchema = z + .object({ + id: z.string().min(1).optional(), + description: z.string().optional(), + vars: JsonObjectSchema.optional(), + provider: EvalTargetSchema.optional(), + providers: EvalTargetsSchema.optional(), + prompts: PromptsSchema.optional(), + input: z.never().optional(), + input_files: z.array(z.string()).optional(), + expected_output: z.never().optional(), + assert: z.array(AssertionItemSchema).optional(), + assert_scoring_function: z.union([z.string().min(1), JsonObjectSchema]).optional(), + options: JsonObjectSchema.optional(), + threshold: z.number().min(0).max(1).optional(), + execution: TestExecutionSchema.optional(), + run: RunOverrideSchema.optional(), + environment: EnvironmentSchema.optional(), + metadata: z.record(z.unknown()).optional(), + conversation_id: z.string().optional(), + suite: z.string().optional(), + depends_on: z.array(z.string()).optional(), + on_dependency_failure: z.enum(['skip', 'fail', 'run']).optional(), + mode: z.enum(['conversation']).optional(), + turns: z.array(ConversationTurnSchema).min(1).optional(), + aggregation: z.enum(['mean', 'min', 'max']).optional(), + on_turn_failure: z.enum(['continue', 'stop']).optional(), + window_size: z.number().int().min(1).optional(), + }) + .passthrough(); + +function rejectProviderOutput(value: Record, context: z.RefinementCtx): void { + if (Object.prototype.hasOwnProperty.call(value, 'provider_output')) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['provider_output'], + message: + 'tests[].provider_output is not supported in authored AgentV YAML. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.', + }); + } +} + +const EvalTestSchema = EvalTestBaseSchema.superRefine(rejectProviderOutput); const SelectPatternSchema = z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]); const SelectMetadataValueSchema = z.union([ @@ -836,7 +849,7 @@ const ConfigDefaultsSchema = z }) .strict(); -const ScenarioConfigSchema = EvalTestSchema.partial(); +const ScenarioConfigSchema = EvalTestBaseSchema.partial().superRefine(rejectProviderOutput); const ScenarioSchema = z .object({ diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 27f0fd63e..166a01f32 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -257,7 +257,6 @@ const KNOWN_DEFAULT_TEST_FIELDS = new Set([ 'provider', 'providers', 'prompts', - 'provider_output', 'assert', 'assert_scoring_function', 'options', @@ -358,7 +357,6 @@ const KNOWN_TEST_FIELDS = new Set([ 'provider', 'providers', 'prompts', - 'provider_output', 'input', 'input_files', 'assert', @@ -401,6 +399,10 @@ REMOVED_TEST_FIELDS.set( 'input', "tests[].input has been removed from authored eval YAML. Put prompt text or chat/system/user messages in top-level 'prompts' and put row-specific data in tests[].vars.", ); +REMOVED_TEST_FIELDS.set( + 'provider_output', + 'tests[].provider_output is not supported in authored AgentV YAML. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.', +); REMOVED_TEST_FIELDS.set( 'preprocessors', 'tests[].preprocessors has been removed from authored eval YAML. Use tests[].options.transform or assertion-level transform instead.', @@ -756,10 +758,7 @@ export async function validateEvalFile(filePath: string): Promise 0 && - scenarioConfigs.every( - (config) => - isObject(config) && - (config.prompts !== undefined || config.provider_output !== undefined), - ); + scenarioConfigs.every((config) => isObject(config) && config.prompts !== undefined); const requireInput = isObject(test) && parsed.prompts === undefined && test.prompts === undefined && - test.provider_output === undefined && !everyConfigProvidesInput; await validateScenarioTestLikeRow( test, diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 7c8461980..3bdeae831 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -299,6 +299,11 @@ function rejectRemovedScenarioRowFields(row: JsonObject, location: string): void if (row.evalcases !== undefined) { throw new Error(`${location}.evalcases has been removed. Use ${location} fields directly.`); } + if (row.provider_output !== undefined) { + throw new Error( + `${location}.provider_output is not supported in authored AgentV YAML. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.`, + ); + } if (row.input !== undefined) { throw new Error( `${location}.input has been removed from authored eval YAML. Put prompt text or chat/system/user messages in top-level 'prompts' and put row-specific data in ${location}.vars.`, @@ -1428,6 +1433,7 @@ async function loadTestsFromParsedYamlValue( if (options?.allowInternalExpectedOutput !== true) { rejectAuthoredExpectedOutput(interpolated); } + rejectAuthoredProviderOutput(interpolated); const rawSuite = rawParsed as RawTestSuite; const resolvedDefaultTest = await resolveDefaultTestValue( @@ -1884,6 +1890,7 @@ function buildEvalSuiteResult( if (options?.allowInternalExpectedOutput !== true) { rejectAuthoredExpectedOutput(parsed); } + rejectAuthoredProviderOutput(parsed); const metadata = parseMetadata(parsed); const failOnError = extractFailOnError(parsed); const threshold = extractThreshold(parsed); @@ -2054,6 +2061,28 @@ function rejectAuthoredExpectedOutput(parsed: JsonObject): void { } } +function rejectAuthoredProviderOutput(parsed: JsonObject): void { + if (isJsonObject(parsed.default_test) && parsed.default_test.provider_output !== undefined) { + throw new Error( + 'default_test.provider_output is not supported in authored AgentV YAML. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.', + ); + } + + if (!Array.isArray(parsed.tests)) { + return; + } + + for (let index = 0; index < parsed.tests.length; index++) { + const entry = parsed.tests[index]; + if (!isJsonObject(entry) || entry.provider_output === undefined) { + continue; + } + throw new Error( + `tests[${index}].provider_output is not supported in authored AgentV YAML. Use an explicit deterministic target such as provider: cli for fixed outputs, or use a replay/fixture target for captured provider responses.`, + ); + } +} + function collectWorkersLocations(raw: unknown, location: string, locations: string[]): void { if (!isJsonObject(raw)) { return; diff --git a/packages/core/test/evaluation/loaders/case-file-loader.test.ts b/packages/core/test/evaluation/loaders/case-file-loader.test.ts index ae6cf2ee5..b09669bc7 100644 --- a/packages/core/test/evaluation/loaders/case-file-loader.test.ts +++ b/packages/core/test/evaluation/loaders/case-file-loader.test.ts @@ -99,8 +99,8 @@ describe('resolveFileReference', () => { await writeFile( path.join(tempDir, 'cases', 'tests.csv'), [ - 'id,input,__expected,__expected2,__prefix,__suffix,__description,__provider_output,__metric,__threshold,__metadata:category,__metadata:tags[],__config:__expected2:threshold,locale', - 'csv-1,"What is 2+2?",equals:4,icontains:four,"Answer briefly:"," Thanks","Arithmetic case","4","accuracy",0.7,math,"smoke,regression",0.6,en-US', + 'id,input,__expected,__expected2,__prefix,__suffix,__description,__metric,__threshold,__metadata:category,__metadata:tags[],__config:__expected2:threshold,locale', + 'csv-1,"What is 2+2?",equals:4,icontains:four,"Answer briefly:"," Thanks","Arithmetic case","accuracy",0.7,math,"smoke,regression",0.6,en-US', ].join('\n'), ); @@ -110,7 +110,6 @@ describe('resolveFileReference', () => { expect(cases[0]).toMatchObject({ id: 'csv-1', input: 'Answer briefly:What is 2+2? Thanks', - expected_output: '4', criteria: 'Arithmetic case', threshold: 0.7, vars: { locale: 'en-US' }, @@ -122,6 +121,17 @@ describe('resolveFileReference', () => { }); }); + it('rejects removed promptfoo CSV provider output columns', async () => { + await writeFile( + path.join(tempDir, 'cases', 'provider-output.csv'), + ['id,input,__expected,__provider_output', 'csv-1,Hello,contains:Hi,Hi there'].join('\n'), + ); + + await expect(resolveFileReference('file://cases/provider-output.csv', tempDir)).rejects.toThrow( + /__provider_output has been removed from CSV imports/, + ); + }); + it('maps supported promptfoo expected DSL forms to runnable AgentV assertions', async () => { const graderPath = path.join(tempDir, 'cases', 'grader.py'); await writeFile(graderPath, 'print("ok")\n'); @@ -526,8 +536,8 @@ tests: await writeFile( path.join(tempDir, 'magic-cases.csv'), [ - 'id,input,__expected,__metric,__threshold,__metadata:category,__provider_output', - 'magic-csv,Hello,contains:Hi,greeting,0.9,smoke,Hi there', + 'id,input,__expected,__metric,__threshold,__metadata:category', + 'magic-csv,Hello,contains:Hi,greeting,0.9,smoke', ].join('\n'), ); await writeFile( @@ -543,7 +553,6 @@ tests: file://magic-cases.csv expect(tests[0].id).toBe('magic-csv'); expect(tests[0].threshold).toBe(0.9); expect(tests[0].metadata).toMatchObject({ category: 'smoke' }); - expect(tests[0].reference_answer).toBe('Hi there'); expect(tests[0].assertions?.[0]).toMatchObject({ name: 'greeting', type: 'contains', diff --git a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts index 0bc527185..c85f361a1 100644 --- a/packages/core/test/evaluation/loaders/jsonl-parser.test.ts +++ b/packages/core/test/evaluation/loaders/jsonl-parser.test.ts @@ -1085,7 +1085,6 @@ scenarios: - id: translate-french vars: phrase: hello - provider_output: bonjour assert: - type: equals value: bonjour @@ -1101,7 +1100,6 @@ scenarios: - id: translate-spanish vars: phrase: goodbye - provider_output: adios assert: - type: equals value: adios @@ -1120,7 +1118,6 @@ scenarios: - id: translate-italian vars: phrase: hello - provider_output: ciao assert: - type: equals value: ciao diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 346584d83..c79cb849d 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -83,7 +83,7 @@ describe('EvalFileSchema input shorthand', () => { scenarios: [ { config: [{ vars: { language: 'Spanish' } }], - tests: [{ vars: { phrase: 'hello' }, provider_output: 'hola' }], + tests: [{ vars: { phrase: 'hello' }, assert: [{ type: 'equals', value: 'hola' }] }], }, 'file://scenarios/*.yaml', ], @@ -415,11 +415,10 @@ describe('EvalFileSchema input shorthand', () => { }, tests: [ { - description: 'grades a fixed provider output', + description: 'grades rendered target output', vars: { diff: 'change', }, - provider_output: 'Looks safe.', assert: [ { type: 'contains', @@ -483,6 +482,21 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('rejects authored provider_output fields', () => { + const result = EvalFileSchema.safeParse({ + prompts: ['Review {{ diff }}'], + tests: [ + { + vars: { diff: 'change' }, + provider_output: 'Looks safe.', + assert: [{ type: 'contains', value: 'safe' }], + }, + ], + }); + + expect(result.success).toBe(false); + }); + it('rejects composite as an authored assertion grouping type', () => { const result = EvalFileSchema.safeParse({ tests: [ diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 7d080ed0e..15a2fdf94 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -455,14 +455,13 @@ evaluate_options: max_eval_time_ms: 120000 filter_range: [0, 10] tests: - - description: fixed output row + - description: target output row vars: diff: change options: repeat: count: 3 strategy: mean - provider_output: "Looks safe." assert: - type: contains value: safe @@ -564,7 +563,6 @@ scenarios: tests: - vars: phrase: hello - provider_output: bonjour assert: - type: equals value: bonjour @@ -579,7 +577,6 @@ scenarios: tests: - vars: phrase: goodbye - provider_output: adios assert: - type: equals value: adios @@ -597,7 +594,6 @@ scenarios: tests: - vars: phrase: hello - provider_output: ciao assert: - type: equals value: ciao @@ -611,6 +607,34 @@ scenarios: expect(result.errors).toHaveLength(0); }); + it('rejects authored provider_output with explicit target guidance', async () => { + const filePath = path.join(tempDir, 'provider-output-removed.yaml'); + await writeFile( + filePath, + `prompts: + - "Review {{ diff }}" +tests: + - vars: + diff: change + provider_output: "Looks safe." + assert: + - type: contains + value: safe +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].provider_output', + message: expect.stringContaining('Use an explicit deterministic target'), + }), + ); + }); + it('rejects scenario tests without input when only some config rows supply prompts', async () => { const filePath = path.join(tempDir, 'scenario-mixed-config-prompts.yaml'); await writeFile( @@ -2549,6 +2573,31 @@ tests: file://suite-input-cases.csv }), ); }); + + it('rejects removed promptfoo CSV provider output columns during validation', async () => { + await writeFile( + path.join(tempDir, 'provider-output-cases.csv'), + 'id,input,__expected,__provider_output\ncase,Hello,contains:Hi,Hi there\n', + ); + + const filePath = path.join(tempDir, 'provider-output-csv.yaml'); + await writeFile( + filePath, + `tests: file://provider-output-cases.csv +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests', + message: expect.stringContaining('__provider_output has been removed from CSV imports'), + }), + ); + }); }); describe('suite-level input validation', () => { diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index ac36b0f08..571c0bd38 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -1397,61 +1397,6 @@ } ] }, - "provider_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, "input": { "not": {} }, @@ -1913,7 +1858,7 @@ "minimum": 1 } }, - "additionalProperties": false + "additionalProperties": true }, { "type": "object", @@ -4472,61 +4417,6 @@ } ] }, - "provider_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, "expected_output": { "not": {} }, @@ -5954,61 +5844,6 @@ } ] }, - "provider_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, "input": { "not": {} }, @@ -6470,7 +6305,7 @@ "minimum": 1 } }, - "additionalProperties": false + "additionalProperties": true } }, "tests": { @@ -7659,61 +7494,6 @@ } ] }, - "provider_output": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "role": { - "type": "string", - "enum": ["system", "user", "assistant", "tool"] - }, - "content": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["text", "file", "image"] - }, - "value": { - "type": "string" - } - }, - "required": ["type", "value"], - "additionalProperties": false - } - } - ] - } - }, - "required": ["role", "content"], - "additionalProperties": false - } - } - ] - }, "input": { "not": {} }, @@ -8175,7 +7955,7 @@ "minimum": 1 } }, - "additionalProperties": false + "additionalProperties": true } } },