From 91a4a27d618bc6b3019fde0b9b17aa1d55aa6056 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 12:30:32 +0200 Subject: [PATCH 1/3] fix(schema): reject authored eval YAML workers --- apps/cli/src/commands/eval/commands/bundle.ts | 5 -- apps/cli/src/commands/eval/run-eval.ts | 17 +---- apps/cli/test/eval.integration.test.ts | 23 +++--- .../docs/docs/evaluation/eval-files.mdx | 1 + .../docs/docs/evaluation/experiments.mdx | 6 +- .../docs/docs/guides/benchmark-provenance.mdx | 2 +- ...arate-experiments-from-eval-definitions.md | 6 +- .../evals/code-grader-contract.eval.yaml | 4 +- .../contract/evals/release-gate.eval.yaml | 4 +- .../evals/repo-materialization.eval.yaml | 4 +- .../evals/docker-example.EVAL.yaml | 5 +- .../repo-lifecycle/evals/pool-e2e.eval.yaml | 4 +- packages/core/src/evaluation/evaluate.ts | 2 +- packages/core/src/evaluation/experiment.ts | 16 ++-- .../src/evaluation/loaders/config-loader.ts | 25 ++----- .../src/evaluation/loaders/ts-eval-loader.ts | 1 - .../evaluation/validation/eval-file.schema.ts | 2 +- .../evaluation/validation/eval-validator.ts | 75 ++++++++++++++++++- .../core/src/evaluation/workspace/setup.ts | 7 +- packages/core/src/evaluation/yaml-parser.ts | 58 ++++++++++++-- .../evaluation/eval-inline-experiment.test.ts | 72 +++++++++++++++++- .../core/test/evaluation/experiment.test.ts | 7 +- .../evaluation/loaders/config-loader.test.ts | 16 ++++ .../loaders/fixtures/default-export.eval.ts | 1 - .../loaders/fixtures/sdk-define-eval.eval.ts | 2 - .../evaluation/loaders/ts-eval-loader.test.ts | 2 - .../validation/eval-validator.test.ts | 40 +++++++++- packages/sdk/src/eval.ts | 1 - packages/sdk/test/eval-authoring.test.ts | 2 - .../references/eval-schema.json | 15 ---- 30 files changed, 301 insertions(+), 124 deletions(-) diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index c0f544784..49e824ce5 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -100,7 +100,6 @@ function definitionsWithEvalTargetRefs( function buildBundleExecution(options: { readonly targetNames: readonly string[]; readonly targetRefs?: readonly EvalTargetRef[]; - readonly workers?: number; readonly cache?: boolean; readonly cachePath?: string; readonly budgetUsd?: number; @@ -121,9 +120,6 @@ function buildBundleExecution(options: { targets: options.targetNames.map((name) => serializeTargetRef(name)), }; - if (options.workers !== undefined) { - execution.workers = options.workers; - } if (options.cache !== undefined) { execution.cache = options.cache; } @@ -224,7 +220,6 @@ export const evalBundleCommand = command({ execution: buildBundleExecution({ targetNames, targetRefs: suite.targetRefs, - workers: suite.workers, cache: suite.cacheConfig?.enabled, cachePath: suite.cacheConfig?.cachePath, budgetUsd: suite.budgetUsd, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index c6d4b599f..77d99201d 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -387,7 +387,7 @@ function normalizeOptions( yamlExecution?: ExecutionDefaults, ): NormalizedOptions { const cliWorkers = normalizeOptionalNumber(rawOptions.workers); - const configWorkers = config?.execution?.workers; + const configWorkers = config?.execution?.workers ?? yamlExecution?.workers; const workers = cliWorkers ?? configWorkers ?? 0; const cliOutputDir = normalizeString(rawOptions.output); @@ -759,7 +759,6 @@ function applyExperimentOptions( ...options, target: options.target, agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds, - workers: options.workers ?? experiment.workers, workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, @@ -1110,7 +1109,6 @@ async function prepareFileMetadata(params: { readonly selections: readonly { selection: TargetSelection; inlineTargetLabel: string }[]; readonly trialsConfig?: TrialsConfig; readonly suiteTargets?: readonly string[]; - readonly yamlWorkers?: number; readonly yamlCache?: boolean; readonly yamlCachePath?: string; readonly budgetUsd?: number; @@ -1160,7 +1158,6 @@ async function prepareFileMetadata(params: { selections: [], trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, - yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, @@ -1318,7 +1315,6 @@ async function prepareFileMetadata(params: { selections, trialsConfig: effectiveOptions.experimentTrialsConfig, suiteTargets, - yamlWorkers: suite.workers, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, @@ -1360,7 +1356,6 @@ async function runSingleEvalFile(params: { readonly cache?: EvaluationCache; readonly evaluationRunner: typeof defaultRunEvaluation; readonly workersOverride?: number; - readonly yamlWorkers?: number; readonly progressReporter: ProgressReporter; readonly seenTestCases: Set; readonly displayIdTracker: { getOrAssign(testCaseKey: string): number }; @@ -1388,7 +1383,6 @@ async function runSingleEvalFile(params: { cache, evaluationRunner, workersOverride, - yamlWorkers, progressReporter, seenTestCases, displayIdTracker, @@ -1439,13 +1433,10 @@ async function runSingleEvalFile(params: { const agentTimeoutMs = agentTimeoutSeconds != null ? Math.max(0, agentTimeoutSeconds) * 1000 : undefined; - // Resolve workers: CLI flag > eval YAML execution.workers > target setting > default + // Resolve workers: CLI/config > target setting > default const workerPreference = workersOverride ?? options.workers; let resolvedWorkers = - workerPreference ?? - yamlWorkers ?? - resolvedTargetSelection.resolvedTarget.workers ?? - DEFAULT_WORKERS; + workerPreference ?? resolvedTargetSelection.resolvedTarget.workers ?? DEFAULT_WORKERS; if (resolvedWorkers < 1 || resolvedWorkers > 50) { throw new Error(`Workers must be between 1 and 50, got: ${resolvedWorkers}`); } @@ -1931,7 +1922,6 @@ export async function runEvalCommand( }[]; readonly trialsConfig?: TrialsConfig; readonly suiteTargets?: readonly string[]; - readonly yamlWorkers?: number; readonly yamlCache?: boolean; readonly yamlCachePath?: string; readonly budgetUsd?: number; @@ -2301,7 +2291,6 @@ export async function runEvalCommand( cache, evaluationRunner, workersOverride: fileOptions.workers, - yamlWorkers: targetPrep.yamlWorkers, progressReporter, seenTestCases, displayIdTracker, diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 96487ad93..5faae713b 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -616,8 +616,6 @@ describe('agentv eval CLI', () => { 'name: native-exp', 'target: codex-target', 'model: gpt-5-codex', - 'execution:', - ' workers: 4', 'policy:', ' timeout_seconds: 12', ' threshold: 0.8', @@ -639,7 +637,7 @@ describe('agentv eval CLI', () => { 'utf8', ); - const { stdout, exitCode } = await runCli(fixture, ['eval', wrapperPath]); + const { stdout, exitCode } = await runCli(fixture, ['eval', wrapperPath, '--workers', '4']); expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); @@ -669,7 +667,6 @@ describe('agentv eval CLI', () => { model: 'gpt-5-codex', runs: 2, timeout_seconds: 12, - workers: 4, }); expect( (benchmark.metadata?.experiment_config as Record).fingerprint, @@ -677,7 +674,7 @@ describe('agentv eval CLI', () => { expect(benchmark.metadata?.runtime_source).toMatchObject({ schema_version: 'agentv.runtime_source.v1', kind: 'wrapper_eval', - config_source: 'inline_experiment', + config_source: 'mixed', experiment_namespace: 'native-exp', experiment_namespace_source: 'eval_metadata', eval_files: ['native-exp.eval.yaml'], @@ -689,7 +686,7 @@ describe('agentv eval CLI', () => { } }, 30_000); - it('keeps inline runtime policy isolated across multiple eval files', async () => { + it('keeps non-concurrency runtime policy isolated across multiple eval files', async () => { const fixture = await createFixture(); try { const firstPath = path.join(fixture.suiteDir, 'first.eval.yaml'); @@ -699,8 +696,6 @@ describe('agentv eval CLI', () => { [ 'name: first', 'target: cli-target', - 'execution:', - ' workers: 1', 'policy:', ' timeout_seconds: 11', ' budget_usd: 0.11', @@ -717,8 +712,6 @@ describe('agentv eval CLI', () => { [ 'name: second', 'target: file-target', - 'execution:', - ' workers: 2', 'policy:', ' timeout_seconds: 22', ' budget_usd: 0.22', @@ -731,7 +724,13 @@ describe('agentv eval CLI', () => { 'utf8', ); - const { stdout, exitCode } = await runCli(fixture, ['eval', firstPath, secondPath]); + const { stdout, exitCode } = await runCli(fixture, [ + 'eval', + firstPath, + secondPath, + '--workers', + '2', + ]); expect(exitCode).toBe(0); const outputPath = extractOutputPath(stdout); @@ -743,7 +742,7 @@ describe('agentv eval CLI', () => { expect(calls[0]).toMatchObject({ target: 'cli-target', agentTimeoutMs: 11_000, - maxConcurrency: 1, + maxConcurrency: 2, budgetUsd: 0.11, runBudgetCapUsd: 0.11, evalCaseIds: ['first-case'], diff --git a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx index 2a473dbb2..791b013c5 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-files.mdx @@ -8,6 +8,7 @@ sidebar: Evaluation files define the test cases, graders, workspace lifecycle, and run policy for an evaluation run. Top-level `name` is the experiment/result namespace, top-level `target` identifies the system under test, and top-level [`policy`](/docs/evaluation/experiments/) owns thresholds, budgets, timeouts, and repeat runs. Workspace reuse belongs under `workspace.isolation`; Docker/container binding belongs under `workspace.docker`. Install, build, and reset commands belong under `workspace.hooks`; runner-specific setup belongs under `targets[].hooks`. AgentV supports two eval data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. +Eval files describe the task, target binding, and run policy. Concurrency is an operator/run setting: pass `--workers` or set `execution.workers` in `agentv.config.*` / `.agentv/config.yaml` instead of authoring `workers` in eval YAML. ## Authoring Shapes diff --git a/apps/web/src/content/docs/docs/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/evaluation/experiments.mdx index 6c88d809a..dd8c19787 100644 --- a/apps/web/src/content/docs/docs/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/evaluation/experiments.mdx @@ -11,6 +11,9 @@ top-level `target` for the system under test, and top-level `policy` for runtime and gating controls such as run count, timeout, budgets, and thresholds. AgentV does not have a separate `experiment.yaml` file, top-level `run_group`, or schema-significant `experiments/` directory. +Concurrency is intentionally outside eval YAML. Use `agentv eval --workers N` +or project config defaults such as `agentv.config.*` / `.agentv/config.yaml` +`execution.workers` for operator-side parallelism. ```yaml name: support-regression @@ -37,7 +40,8 @@ tests: `model`, and runtime controls to top-level `policy` with `runs`, `timeout_seconds`, `threshold`, and `budget_usd`. `execution:` is accepted only as a legacy top-level runtime alias for existing -eval files and target matrices. +eval files and target matrices. It does not accept `workers`; use `--workers` +or project config for concurrency. ## Layout Conventions diff --git a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx index 1bfdcd739..338417962 100644 --- a/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx +++ b/apps/web/src/content/docs/docs/guides/benchmark-provenance.mdx @@ -29,7 +29,7 @@ Use this split when deciding where a benchmark key belongs: | `workspace.template` | Yes | Copies a workspace template into the run workspace. | | `workspace.hooks` | Yes | Runs lifecycle commands with workspace and case context on stdin. | | `workspace.isolation` | Yes | Controls shared vs per-case folder isolation. Runtime workspace paths are machine-local config/CLI bindings, not benchmark provenance. | -| `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, workers, and default grader behavior. | +| `experiment` | Yes | Selects targets, thresholds, repeat policy, budgets, and default grader behavior. Concurrency is an operator/run setting from `--workers` or project config. | | `input`, `input_files`, `expected_output` | Yes | Builds the target prompt and passive reference answer. | | `assertions` | Yes | Runs deterministic, LLM, composite, or code graders. | | Top-level `name`, `version`, `tags`, `license`, `requires` | Informational | Identifies and categorizes the suite. | diff --git a/docs/adr/0006-separate-experiments-from-eval-definitions.md b/docs/adr/0006-separate-experiments-from-eval-definitions.md index de53e15d2..abb68ba46 100644 --- a/docs/adr/0006-separate-experiments-from-eval-definitions.md +++ b/docs/adr/0006-separate-experiments-from-eval-definitions.md @@ -70,8 +70,6 @@ fields: name: cargowise-sql-migration-codex target: agent model: gpt-5-codex -execution: - workers: 4 policy: threshold: 0.8 runs: 3 @@ -109,7 +107,6 @@ already-existing eval files and target matrices. New surfaces should not teach The old experiment runtime fields are ported into the parent eval file: - target or target matrix -- workers - thresholds - repeated run count through `policy.runs` - timeout @@ -129,7 +126,8 @@ Parent-versus-child is not the main composition rule. Contract ownership is: | Task prompt | `input`, `input_files`, shared prompt defaults | Imported child suite | | Task environment | `workspace`, `workspace.repos[]`, templates, workspace hooks | Imported child suite | | Scoring | `assertions`, graders, expected references | Imported child suite | -| Run policy | `experiment`, CLI target flags, workers, repeat, gates, budget | Parent wrapper eval or CLI | +| Run policy | `experiment`, CLI target flags, repeat, gates, budget | Parent wrapper eval or CLI | +| Run concurrency | `--workers`, project config defaults, target/provider caps | Operator or selected target | | Target runtime | selected target config and `targets[].hooks` | Selected target | `workspace` can influence what an agent perceives through tools, but it is not diff --git a/examples/contract/evals/code-grader-contract.eval.yaml b/examples/contract/evals/code-grader-contract.eval.yaml index 4a1145599..7bfae4ea1 100644 --- a/examples/contract/evals/code-grader-contract.eval.yaml +++ b/examples/contract/evals/code-grader-contract.eval.yaml @@ -1,9 +1,7 @@ name: code-grader-contract description: Release gate verifying the code-grader stdin payload contract. -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: code-grader-stdin-payload diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 228b361f4..c740d2e20 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -4,9 +4,7 @@ description: Lightweight release gate for npm latest promotion. workspace: template: ../workspace-template -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: json-contract diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 033bad68d..c8c6f8320 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -7,9 +7,7 @@ workspace: repo: EntityProcess/agentv-contract-fixture commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb -execution: - target: github-models-contract - workers: 1 +target: github-models-contract tests: - id: repo-materializes-previous-commit diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index c43903554..80d7e333a 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -10,6 +10,7 @@ name: docker-workspace-example description: Example eval using Docker workspace for grading +target: mock_agent workspace: docker: @@ -18,10 +19,6 @@ workspace: memory: 2g cpus: 1 -execution: - target: mock_agent - workers: 1 - tests: - id: hello-world input: "Write a Python function that returns 'hello world'" diff --git a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml index d9ab18042..75470d9ac 100644 --- a/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml +++ b/examples/features/repo-lifecycle/evals/pool-e2e.eval.yaml @@ -1,6 +1,7 @@ description: >- E2E test for workspace pooling. No pool config in YAML — pooling is enabled by default for shared workspaces with repos. + Run with --workers 2 to exercise multiple pool slots. workspace: repos: @@ -8,9 +9,6 @@ workspace: repo: https://github.com/EntityProcess/agentv.git commit: main -execution: - workers: 2 - tags: [agent] tests: diff --git a/packages/core/src/evaluation/evaluate.ts b/packages/core/src/evaluation/evaluate.ts index adeb2f204..ea28b1157 100644 --- a/packages/core/src/evaluation/evaluate.ts +++ b/packages/core/src/evaluation/evaluate.ts @@ -425,7 +425,7 @@ export async function materializeEvalConfig( return { testFilePath, tests, - workers: config.workers ?? suite.workers, + workers: config.workers, cache: config.cache ?? suite.cacheConfig?.enabled, cachePath: config.cachePath ?? suite.cacheConfig?.cachePath, budgetUsd: config.budgetUsd ?? suite.budgetUsd, diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index 03a423be8..0f2c7514f 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -42,7 +42,6 @@ export type ExperimentConfigWire = { readonly runs?: number; readonly early_exit?: boolean; readonly timeout_seconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budget_usd?: number; readonly workspace?: never; @@ -59,7 +58,6 @@ export type ExperimentConfig = { readonly runs?: number; readonly earlyExit?: boolean; readonly timeoutSeconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budgetUsd?: number; readonly fingerprint?: string; @@ -80,7 +78,6 @@ export type ExperimentArtifactMetadata = { readonly runs?: number; readonly early_exit?: boolean; readonly timeout_seconds?: number; - readonly workers?: number; readonly threshold?: number; readonly budget_usd?: number; }; @@ -123,7 +120,7 @@ export function normalizeExperimentConfig(rawConfig: unknown): ExperimentConfig rawConfig.timeout_seconds ?? rawConfig.timeoutSeconds, 'timeout_seconds', ); - const workers = readOptionalPositiveInteger(rawConfig.workers, 'workers'); + rejectExperimentWorkers(rawConfig.workers); const threshold = readOptionalThreshold(rawConfig.threshold); const budgetUsd = readOptionalPositiveNumber( rawConfig.budget_usd ?? rawConfig.budgetUsd, @@ -142,7 +139,6 @@ export function normalizeExperimentConfig(rawConfig: unknown): ExperimentConfig ...(runs !== undefined && { runs }), ...(earlyExit !== undefined && { earlyExit }), ...(timeoutSeconds !== undefined && { timeoutSeconds }), - ...(workers !== undefined && { workers }), ...(threshold !== undefined && { threshold }), ...(budgetUsd !== undefined && { budgetUsd }), }; @@ -217,7 +213,6 @@ export function buildExperimentArtifactMetadata( ...(config.runs !== undefined && { runs: config.runs }), ...(config.earlyExit !== undefined && { early_exit: config.earlyExit }), ...(config.timeoutSeconds !== undefined && { timeout_seconds: config.timeoutSeconds }), - ...(config.workers !== undefined && { workers: config.workers }), ...(config.threshold !== undefined && { threshold: config.threshold }), ...(config.budgetUsd !== undefined && { budget_usd: config.budgetUsd }), }; @@ -379,6 +374,15 @@ function rejectExperimentWorkspace(raw: unknown): void { ); } +function rejectExperimentWorkers(raw: unknown): void { + if (raw === undefined) { + return; + } + throw new Error( + 'Experiment workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', + ); +} + function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 4ad3b3310..39863f2e7 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -32,6 +32,7 @@ export const DEFAULT_EVAL_PATTERNS: readonly string[] = [ ]; export type ExecutionDefaults = { + readonly workers?: number; readonly verbose?: boolean; readonly keep_workspaces?: boolean; readonly workspace_mode?: 'pooled' | 'temp' | 'static'; @@ -407,23 +408,6 @@ function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined { }; } -/** - * Extract workers count from suite-level execution block. - */ -export function extractWorkersFromSuite(suite: JsonObject): number | undefined { - const runtime = getSuiteRuntimeBlock(suite); - if (!runtime) { - return undefined; - } - - const workers = runtime.workers; - if (typeof workers === 'number' && Number.isInteger(workers) && workers >= 1 && workers <= 50) { - return workers; - } - - return undefined; -} - /** * Cache configuration parsed from execution block. */ @@ -578,6 +562,13 @@ export function parseExecutionDefaults( const obj = raw as Record; const result: Record = {}; + const workers = obj.workers; + if (typeof workers === 'number' && Number.isInteger(workers) && workers >= 1 && workers <= 50) { + result.workers = workers; + } else if (workers !== undefined) { + logWarning(`Invalid execution.workers in ${configPath}, expected integer 1-50`); + } + if (typeof obj.verbose === 'boolean') { result.verbose = obj.verbose; } else if (obj.verbose !== undefined) { diff --git a/packages/core/src/evaluation/loaders/ts-eval-loader.ts b/packages/core/src/evaluation/loaders/ts-eval-loader.ts index b5e20e119..8a1e8e529 100644 --- a/packages/core/src/evaluation/loaders/ts-eval-loader.ts +++ b/packages/core/src/evaluation/loaders/ts-eval-loader.ts @@ -92,7 +92,6 @@ export async function loadTsEvalSuite( return { tests: materialized.tests, - ...(materialized.workers !== undefined && { workers: materialized.workers }), ...(materialized.cache !== undefined && { cacheConfig: { enabled: materialized.cache, diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 038eacc3b..0f9fff3e3 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -360,7 +360,7 @@ const FailOnErrorSchema = z.boolean(); const ExecutionSchema = z.object({ target: z.string().optional(), targets: z.array(z.union([z.string(), EvalTargetRefSchema])).optional(), - workers: z.number().int().min(1).max(50).optional(), + workers: z.never().optional(), assertions: z.array(EvaluatorSchema).optional(), evaluators: z.array(EvaluatorSchema).optional(), skip_defaults: z.boolean().optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index ebdf46fe7..114e4d85d 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -82,7 +82,6 @@ const KNOWN_IMPORT_FIELDS = new Set(['path', 'select', 'run']); const KNOWN_RUN_OVERRIDE_FIELDS = new Set(['threshold', 'repeat', 'timeout_seconds', 'budget_usd']); const KNOWN_REPEAT_STRATEGIES = new Set(['pass_at_k', 'pass_all', 'mean', 'confidence_interval']); const KNOWN_TEST_EXECUTION_FIELDS = new Set([ - 'workers', 'assertions', 'evaluators', 'skip_defaults', @@ -99,6 +98,10 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ /** Removed top-level fields with migration hints. */ const REMOVED_TOP_LEVEL_FIELDS = new Map([ ['assert', "'assert' has been removed. Use 'assertions' instead."], + [ + 'workers', + "'workers' has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.", + ], [ 'experiment', "Top-level 'experiment' has been removed. Move experiment.target to top-level 'target', experiment.model to top-level 'model', and runtime controls to top-level 'policy' with runs, timeout_seconds, threshold, and budget_usd.", @@ -305,6 +308,7 @@ export async function validateEvalFile(filePath: string): Promise { + if (!isObject(entry)) { + return; + } + rejectWorkersField(entry.execution, `tests[${index}].execution`, filePath, errors); + }); + } +} + +function rejectWorkersField( + raw: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (!isObject(raw)) { + return; + } + if (raw.workers !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.workers`, + message: `${location}.workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + }); + } + rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors); +} + +function rejectTargetWorkers( + rawTargets: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (!Array.isArray(rawTargets)) { + return; + } + rawTargets.forEach((target, index) => { + if (!isObject(target) || target.workers === undefined) { + return; + } + errors.push({ + severity: 'error', + filePath, + location: `${location}[${index}].workers`, + message: `${location}[${index}].workers has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + }); + }); +} + function rejectRuntimeWorkspaceConfig( workspace: JsonValue | undefined, filePath: string, diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 660db6a84..d7f385921 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -468,12 +468,7 @@ export async function prepareSharedWorkspaceSetup( [ `Warning: This eval uses a shared workspace with ${workers} workers.`, 'If the agent under test makes file edits, concurrent runs may corrupt each other.', - 'To limit concurrency, add this to your eval YAML:', - '', - ' execution:', - ' workers: 1', - '', - 'Or pass --workers 1 on the command line.', + 'To limit concurrency, pass --workers 1 on the command line or set execution.workers in agentv.config.* / .agentv/config.yaml.', ].join('\n'), ); } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 73c2c1418..1f3b56783 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -26,7 +26,6 @@ import { extractTargetRefsFromSuite, extractTargetsFromSuite, extractThreshold, - extractWorkersFromSuite, loadConfig, } from './loaders/config-loader.js'; import { buildSearchRoots, resolveToAbsolutePath } from './loaders/file-resolver.js'; @@ -84,7 +83,6 @@ export { extractTargetRefsFromSuite, extractTargetsFromSuite, extractThreshold, - extractWorkersFromSuite, loadConfig, } from './loaders/config-loader.js'; export type { AgentVConfig, CacheConfig, ExecutionDefaults } from './loaders/config-loader.js'; @@ -111,7 +109,6 @@ type SuiteImportStackEntry = { }; const KNOWN_TEST_EXECUTION_FIELDS = new Set([ - 'workers', 'assertions', 'evaluators', 'skip_defaults', @@ -331,8 +328,6 @@ export type EvalSuiteResult = { readonly targets?: readonly string[]; /** Suite-level target refs with hooks from execution.targets (object form) */ readonly targetRefs?: readonly import('./types.js').EvalTargetRef[]; - /** Suite-level workers from execution.workers */ - readonly workers?: number; /** Suite-level cache config from execution.cache */ readonly cacheConfig?: import('./loaders/config-loader.js').CacheConfig; /** Suite-level metadata (name, description, version, etc.) */ @@ -464,6 +459,7 @@ async function loadTestsFromParsedYamlValue( if (!isJsonObject(interpolated)) { throw new Error(`Invalid test file format: ${evalFilePath}`); } + rejectAuthoredWorkers(interpolated); const suite = interpolated as RawTestSuite; const suiteNameFromFile = asString(suite.name)?.trim(); @@ -849,6 +845,7 @@ async function loadTestsFromParsedYamlValue( } function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): EvalSuiteResult { + rejectAuthoredWorkers(parsed); const metadata = parseMetadata(parsed); const failOnError = extractFailOnError(parsed); const threshold = extractThreshold(parsed); @@ -858,7 +855,6 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E tests, targets: extractTargetsFromSuite(parsed), targetRefs: extractTargetRefsFromSuite(parsed), - workers: extractWorkersFromSuite(parsed), cacheConfig: extractCacheConfig(parsed), budgetUsd: extractBudgetUsd(parsed), ...(metadata !== undefined && { metadata }), @@ -868,6 +864,56 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E }; } +function rejectAuthoredWorkers(parsed: JsonObject): void { + const locations: string[] = []; + if (parsed.workers !== undefined) { + locations.push('workers'); + } + collectWorkersLocations(parsed.execution, 'execution', locations); + collectWorkersLocations(parsed.experiment, 'experiment', locations); + if (Array.isArray(parsed.tests)) { + parsed.tests.forEach((entry, index) => { + if (!isJsonObject(entry)) { + return; + } + collectWorkersLocations(entry.execution, `tests[${index}].execution`, locations); + }); + } + + if (locations.length === 0) { + return; + } + + throw new Error( + `${locations[0]} has been removed from eval YAML. Set concurrency with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + ); +} + +function collectWorkersLocations(raw: unknown, location: string, locations: string[]): void { + if (!isJsonObject(raw)) { + return; + } + if (raw.workers !== undefined) { + locations.push(`${location}.workers`); + } + collectTargetWorkersLocations(raw.targets, `${location}.targets`, locations); +} + +function collectTargetWorkersLocations( + rawTargets: unknown, + location: string, + locations: string[], +): void { + if (!Array.isArray(rawTargets)) { + return; + } + rawTargets.forEach((target, index) => { + if (isJsonObject(target) && target.workers !== undefined) { + locations.push(`${location}[${index}].workers`); + } + }); +} + type IncludeEntryType = 'suite' | 'tests'; type ExpandedInlineTestEntries = { diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 17e708485..3bef0aa67 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -49,7 +49,72 @@ describe('eval.yaml runtime policy and tests imports', () => { budgetUsd: 1.5, }); expect(suite.targets).toBeUndefined(); - expect(suite.workers).toBeUndefined(); + }); + + it('rejects authored workers in eval YAML runtime blocks', async () => { + const cases = [ + { + file: 'top-level.eval.yaml', + body: ['workers: 2', 'tests:', ' - id: one', ' input: hello', ' criteria: ok'], + message: /workers has been removed from eval YAML/, + }, + { + file: 'execution.eval.yaml', + body: [ + 'execution:', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /execution\.workers has been removed from eval YAML/, + }, + { + file: 'experiment.eval.yaml', + body: [ + 'experiment:', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /experiment\.workers has been removed from eval YAML/, + }, + { + file: 'target-ref.eval.yaml', + body: [ + 'execution:', + ' targets:', + ' - name: codex', + ' workers: 2', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ], + message: /execution\.targets\[0\]\.workers has been removed from eval YAML/, + }, + { + file: 'test-execution.eval.yaml', + body: [ + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + ' execution:', + ' workers: 2', + ], + message: /tests\[0\]\.execution\.workers has been removed from eval YAML/, + }, + ]; + + for (const testCase of cases) { + const evalPath = path.join(tempDir, testCase.file); + await writeFile(evalPath, `${testCase.body.join('\n')}\n`); + await expect(loadTestSuite(evalPath, tempDir)).rejects.toThrow(testCase.message); + } }); it('rejects repeat strategy config under top-level policy', async () => { @@ -471,7 +536,6 @@ describe('eval.yaml runtime policy and tests imports', () => { 'name: child-suite', 'execution:', ' target: child-target', - ' workers: 1', ' threshold: 0.2', ' repeat:', ' count: 5', @@ -771,7 +835,7 @@ describe('eval.yaml runtime policy and tests imports', () => { [ 'name: child-a', 'execution:', - ' workers: 2', + ' threshold: 0.2', 'tests:', ' - id: a', ' input: a', @@ -784,7 +848,7 @@ describe('eval.yaml runtime policy and tests imports', () => { [ 'name: child-b', 'execution:', - ' workers: 4', + ' threshold: 0.4', 'tests:', ' - id: b', ' input: b', diff --git a/packages/core/test/evaluation/experiment.test.ts b/packages/core/test/evaluation/experiment.test.ts index b2334b113..763931772 100644 --- a/packages/core/test/evaluation/experiment.test.ts +++ b/packages/core/test/evaluation/experiment.test.ts @@ -17,7 +17,6 @@ describe('inline experiment config', () => { runs: 3, early_exit: false, timeout_seconds: 900, - workers: 4, threshold: 0.8, budget_usd: 1.25, }); @@ -32,7 +31,6 @@ describe('inline experiment config', () => { runs: 3, earlyExit: false, timeoutSeconds: 900, - workers: 4, budgetUsd: 1.25, }); expect(config.fingerprint).toMatch(/^[a-f0-9]{64}$/); @@ -93,6 +91,9 @@ describe('inline experiment config', () => { expect(() => normalizeExperimentConfig({ workspace: { repos: [{ repo: 'acme/support-app' }] } }), ).toThrow(/Experiment workspace has been removed from eval YAML/); + expect(() => normalizeExperimentConfig({ workers: 3 })).toThrow( + /Experiment workers has been removed from eval YAML/, + ); }); it('builds safe snake_case artifact metadata without agent options', () => { @@ -103,7 +104,6 @@ describe('inline experiment config', () => { repeat: { count: 2, strategy: 'mean', cost_limit_usd: 0.5 }, early_exit: true, timeout_seconds: 120, - workers: 3, }); const metadata = buildExperimentArtifactMetadata(config); @@ -118,7 +118,6 @@ describe('inline experiment config', () => { }, early_exit: true, timeout_seconds: 120, - workers: 3, }); expect(metadata).not.toHaveProperty('agent_options'); expect(metadata).not.toHaveProperty('setup'); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 9c4dc46b2..686dc15b1 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -837,6 +837,22 @@ describe('parseExecutionDefaults', () => { expect(result?.verbose).toBe(true); }); + it('parses workers as an operator-side execution default', () => { + const result = parseExecutionDefaults({ workers: 4 }, '/test/config.yaml'); + expect(result?.workers).toBe(4); + }); + + it('ignores invalid workers defaults', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = parseExecutionDefaults({ workers: 0 }, '/test/config.yaml'); + expect(result).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('execution.workers')); + } finally { + warnSpy.mockRestore(); + } + }); + it('parses keep_workspaces boolean', () => { const result = parseExecutionDefaults({ keep_workspaces: true }, '/test/config.yaml'); expect(result?.keep_workspaces).toBe(true); diff --git a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts index ca732b547..bcf339591 100644 --- a/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts @@ -12,7 +12,6 @@ const config: EvalConfig = { assertions: [{ type: 'contains', value: 'hello' }], }, ], - workers: 2, cache: false, cachePath: '.agentv/ts-eval-cache', budgetUsd: 1.5, diff --git a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts index ab6a7f0ab..d89394160 100644 --- a/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts +++ b/packages/core/test/evaluation/loaders/fixtures/sdk-define-eval.eval.ts @@ -7,7 +7,6 @@ const suite = { tags: ['sdk', 'typescript', 'yaml'], execution: { targets: ['mock-target'], - workers: 2, skipDefaults: true, budgetUsd: 2, threshold: 0.75, @@ -49,7 +48,6 @@ export default Object.defineProperties(suite, { tags: suite.tags, execution: { targets: ['mock-target'], - workers: 2, skip_defaults: true, budget_usd: 2, threshold: 0.75, diff --git a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts index 2298ec0f7..e19710c22 100644 --- a/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts +++ b/packages/core/test/evaluation/loaders/ts-eval-loader.test.ts @@ -56,7 +56,6 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].suite).toBe('default-export-suite'); expect(suite.tests[0].category).toBe('sdk'); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript']); - expect(suite.workers).toBe(2); expect(suite.cacheConfig?.enabled).toBe(false); expect(suite.cacheConfig?.cachePath).toBe('.agentv/ts-eval-cache'); expect(suite.budgetUsd).toBe(1.5); @@ -79,7 +78,6 @@ describe('loadTsEvalFile', () => { expect(suite.tests[0].workspace?.hooks?.before_each?.command).toEqual(['echo', 'case-setup']); expect(suite.tests[0].workspace?.hooks?.before_each?.timeout_ms).toBe(1_000); expect(suite.targets).toEqual(['mock-target']); - expect(suite.workers).toBe(2); expect(suite.budgetUsd).toBe(2); expect(suite.threshold).toBe(0.75); expect(suite.metadata?.tags).toEqual(['sdk', 'typescript', 'yaml']); diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 3ba05bb1c..76df2203e 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -358,12 +358,50 @@ tests: ).toBe(true); }); + it('rejects authored workers in eval YAML', async () => { + const filePath = path.join(tempDir, 'authored-workers.eval.yaml'); + await writeFile( + filePath, + `workers: 2 +execution: + workers: 3 + targets: + - name: codex + workers: 4 +tests: + - id: test-1 + criteria: Goal + input: Query + execution: + workers: 5 +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + [ + 'workers', + 'execution.workers', + 'execution.targets[0].workers', + 'tests[0].execution.workers', + ].every((location) => + result.errors.some( + (error) => + error.severity === 'error' && + error.location === location && + error.message.includes('has been removed from eval YAML'), + ), + ), + ).toBe(true); + }); + it('warns that imported child legacy execution is ignored by wrapper composition', async () => { await writeFile( path.join(tempDir, 'composition-child-experiment.eval.yaml'), `execution: target: child-target - workers: 2 threshold: 0.9 tests: - id: child-case diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 06abfe49e..f0064828f 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -147,7 +147,6 @@ export interface EvalPolicy { export interface EvalExecution { readonly target?: string; readonly targets?: readonly (string | EvalTargetRef)[]; - readonly workers?: number; readonly assertions?: readonly EvalAssertionConfig[]; readonly skipDefaults?: boolean; readonly cache?: boolean; diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 051c9253b..2be1525a5 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -28,7 +28,6 @@ describe('YAML-aligned eval authoring helpers', () => { }, }, ], - workers: 2, skipDefaults: true, budgetUsd: 1.5, failOnError: true, @@ -119,7 +118,6 @@ describe('YAML-aligned eval authoring helpers', () => { }, }, ], - workers: 2, skip_defaults: true, budget_usd: 1.5, fail_on_error: true, diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval-schema.json index 047712c35..fca3e7720 100644 --- a/skills-data/agentv-eval-writer/references/eval-schema.json +++ b/skills-data/agentv-eval-writer/references/eval-schema.json @@ -3231,11 +3231,6 @@ "execution": { "type": "object", "properties": { - "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 - }, "assertions": { "type": "array", "items": { @@ -9861,11 +9856,6 @@ "execution": { "type": "object", "properties": { - "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 - }, "assertions": { "type": "array", "items": { @@ -14070,11 +14060,6 @@ }, "minItems": 1 }, - "workers": { - "type": "integer", - "minimum": 1, - "maximum": 50 - }, "assertions": { "type": "array", "items": { From e15eb3c9775c5eede49a430cf16629ab1498e2da Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 12:55:16 +0200 Subject: [PATCH 2/3] fix(schema): sync generated eval schema --- packages/core/scripts/generate-eval-schema.ts | 2 +- .../evaluation/validation/eval-file.schema.ts | 4 +- .../validation/eval-schema-sync.test.ts | 4 +- skills-data/agentv-eval-writer/SKILL.md | 8 +- ...{config-schema.json => config.schema.json} | 0 .../{eval-schema.json => eval.schema.json} | 3215 +++++++++++++---- 6 files changed, 2597 insertions(+), 636 deletions(-) rename skills-data/agentv-eval-writer/references/{config-schema.json => config.schema.json} (100%) rename skills-data/agentv-eval-writer/references/{eval-schema.json => eval.schema.json} (87%) diff --git a/packages/core/scripts/generate-eval-schema.ts b/packages/core/scripts/generate-eval-schema.ts index 71d323b01..73b22a69b 100644 --- a/packages/core/scripts/generate-eval-schema.ts +++ b/packages/core/scripts/generate-eval-schema.ts @@ -44,5 +44,5 @@ await writeSchema({ name: 'EvalFile', title: 'AgentV Eval File', description: 'Schema for AgentV evaluation YAML files (.eval.yaml)', - outputFile: 'eval-schema.json', + outputFile: 'eval.schema.json', }); diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 0f9fff3e3..a2aeab493 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -1,10 +1,10 @@ /** * Zod schema for eval YAML file format. - * Used to generate eval-schema.json for AI agent reference. + * Used to generate eval.schema.json for AI agent reference. * * IMPORTANT: This schema describes the YAML input format, not the parsed runtime types. * When adding new eval features, update this schema AND run `bun run generate:schema` - * to regenerate eval-schema.json. The sync test will fail if they diverge. + * to regenerate eval.schema.json. The sync test will fail if they diverge. */ import { z } from 'zod'; diff --git a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts index 89e410861..bfffc49cf 100644 --- a/packages/core/test/evaluation/validation/eval-schema-sync.test.ts +++ b/packages/core/test/evaluation/validation/eval-schema-sync.test.ts @@ -5,11 +5,11 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; import { EvalFileSchema } from '../../../src/evaluation/validation/eval-file.schema.js'; describe('generated schema sync', () => { - it('keeps eval-schema.json synced with the Zod schema', async () => { + it('keeps eval.schema.json synced with the Zod schema', async () => { const repoRoot = path.resolve(import.meta.dirname, '../../../../..'); const schemaPath = path.join( repoRoot, - 'skills-data/agentv-eval-writer/references/eval-schema.json', + 'skills-data/agentv-eval-writer/references/eval.schema.json', ); // Read committed schema diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index c1ec9580c..183434bf3 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -812,21 +812,21 @@ Do not invent a separate Opik-specific eval surface. Keep the eval definition in ## Schemas -- Eval file: `references/eval-schema.json` -- Config: `references/config-schema.json` +- Eval file: `references/eval.schema.json` +- Config: `references/config.schema.json` ## Accessing reference files To load a specific reference without pulling the entire skill into context: ```bash -agentv skills get agentv-eval-writer --ref eval-schema.json +agentv skills get agentv-eval-writer --ref eval.schema.json ``` Or resolve the skill directory and read files directly: ```bash -cat $(agentv skills path agentv-eval-writer)/references/eval-schema.json +cat $(agentv skills path agentv-eval-writer)/references/eval.schema.json ``` Use `--full` to retrieve every file in the skill at once. diff --git a/skills-data/agentv-eval-writer/references/config-schema.json b/skills-data/agentv-eval-writer/references/config.schema.json similarity index 100% rename from skills-data/agentv-eval-writer/references/config-schema.json rename to skills-data/agentv-eval-writer/references/config.schema.json diff --git a/skills-data/agentv-eval-writer/references/eval-schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json similarity index 87% rename from skills-data/agentv-eval-writer/references/eval-schema.json rename to skills-data/agentv-eval-writer/references/eval.schema.json index fca3e7720..44c4d1cdc 100644 --- a/skills-data/agentv-eval-writer/references/eval-schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -54,7 +54,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -73,20 +78,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false }, { @@ -105,7 +120,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -124,20 +144,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -234,7 +264,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -263,7 +297,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -274,7 +313,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -291,7 +332,9 @@ "additionalProperties": false } }, - "required": ["path"], + "required": [ + "path" + ], "additionalProperties": false }, { @@ -381,7 +424,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -410,7 +457,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -421,7 +473,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -438,7 +492,9 @@ "additionalProperties": false } }, - "required": ["path"], + "required": [ + "path" + ], "additionalProperties": false } ] @@ -525,7 +581,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -554,7 +614,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -565,7 +630,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -582,7 +649,9 @@ "additionalProperties": false } }, - "required": ["path"], + "required": [ + "path" + ], "additionalProperties": false }, { @@ -672,7 +741,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -701,7 +774,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -712,7 +790,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -729,7 +809,9 @@ "additionalProperties": false } }, - "required": ["path"], + "required": [ + "path" + ], "additionalProperties": false } ] @@ -768,7 +850,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -787,20 +874,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false }, { @@ -819,7 +916,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -838,20 +940,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -880,7 +992,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -899,20 +1016,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -956,7 +1083,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -1030,12 +1160,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -1072,7 +1208,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -1130,7 +1269,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -1171,7 +1313,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -1222,12 +1367,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1238,7 +1388,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -1301,7 +1453,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1317,7 +1471,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -1334,7 +1491,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -1351,13 +1511,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -1394,11 +1559,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -1439,7 +1613,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1453,7 +1632,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1464,7 +1648,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -1472,7 +1658,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1486,7 +1677,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1497,7 +1693,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -1534,7 +1733,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -1546,7 +1748,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -1568,17 +1774,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -1622,7 +1837,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -1666,7 +1884,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -1703,7 +1924,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -1718,7 +1942,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1755,7 +1981,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -1787,7 +2016,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1830,7 +2061,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -1873,7 +2107,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -1910,10 +2147,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1956,7 +2198,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -2014,7 +2259,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -2055,7 +2303,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -2067,7 +2318,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -2111,7 +2365,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -2185,12 +2442,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -2227,7 +2490,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -2285,7 +2551,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -2326,7 +2595,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -2377,12 +2649,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2393,7 +2670,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -2456,7 +2735,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2472,7 +2753,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -2489,7 +2773,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -2506,13 +2793,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -2549,11 +2841,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -2594,7 +2895,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2608,7 +2914,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2619,7 +2930,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -2627,7 +2940,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2641,7 +2959,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2652,7 +2975,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -2689,7 +3015,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -2701,7 +3030,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -2723,17 +3056,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -2777,7 +3119,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -2821,7 +3166,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -2858,7 +3206,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -2873,7 +3224,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2910,7 +3263,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -2942,7 +3298,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2985,7 +3343,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3028,7 +3389,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3065,10 +3429,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3111,7 +3480,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3169,7 +3541,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -3210,7 +3585,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -3222,7 +3600,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -3231,6 +3612,9 @@ "execution": { "type": "object", "properties": { + "workers": { + "not": {} + }, "assertions": { "type": "array", "items": { @@ -3269,7 +3653,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -3343,12 +3730,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -3385,7 +3778,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -3443,7 +3839,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -3484,7 +3883,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -3535,12 +3937,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3551,7 +3958,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -3614,7 +4023,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3630,7 +4041,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -3647,7 +4061,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -3664,13 +4081,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -3707,7 +4129,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -3758,7 +4183,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3772,7 +4202,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3783,7 +4218,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -3791,7 +4228,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3805,7 +4247,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3816,7 +4263,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -3853,7 +4303,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -3865,7 +4318,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -3887,17 +4344,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -3941,7 +4407,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -3985,7 +4454,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -4022,7 +4494,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -4037,7 +4512,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4074,7 +4551,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -4106,7 +4586,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4149,7 +4631,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4192,7 +4677,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4229,10 +4717,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4275,7 +4768,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4333,7 +4829,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -4374,7 +4873,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -4386,7 +4888,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -4430,7 +4935,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -4504,12 +5012,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -4546,7 +5060,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -4604,7 +5121,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -4645,7 +5165,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -4696,12 +5219,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4712,7 +5240,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -4775,7 +5305,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4791,7 +5323,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -4808,7 +5343,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -4825,13 +5363,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -4868,7 +5411,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -4919,7 +5465,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -4933,7 +5484,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -4944,7 +5500,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -4952,7 +5510,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -4966,7 +5529,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -4977,7 +5545,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -5014,7 +5585,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -5026,7 +5600,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -5048,17 +5626,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -5102,7 +5689,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -5146,7 +5736,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -5183,7 +5776,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -5198,7 +5794,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5235,7 +5833,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -5267,7 +5868,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5310,7 +5913,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5353,7 +5959,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5390,10 +5999,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5436,7 +6050,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5494,7 +6111,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -5535,7 +6155,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -5547,7 +6170,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -5604,7 +6230,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -5615,7 +6246,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -5639,7 +6272,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_case"] + "enum": [ + "shared", + "per_case" + ] }, "repos": { "type": "array", @@ -5721,7 +6357,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -5766,7 +6406,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -5811,7 +6455,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -5856,7 +6504,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -5882,7 +6534,9 @@ "minimum": 0.1 } }, - "required": ["image"], + "required": [ + "image" + ], "additionalProperties": false }, "env": { @@ -5926,11 +6580,17 @@ }, "on_dependency_failure": { "type": "string", - "enum": ["skip", "fail", "run"] + "enum": [ + "skip", + "fail", + "run" + ] }, "mode": { "type": "string", - "enum": ["conversation"] + "enum": [ + "conversation" + ] }, "turns": { "type": "array", @@ -5959,13 +6619,20 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } @@ -5995,13 +6662,20 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } @@ -6052,7 +6726,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -6126,12 +6803,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -6168,7 +6851,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -6226,7 +6912,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -6267,7 +6956,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -6318,12 +7010,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6334,7 +7031,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -6397,7 +7096,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6413,7 +7114,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -6430,7 +7134,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -6447,13 +7154,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -6490,7 +7202,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -6576,7 +7291,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -6584,7 +7301,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6598,7 +7320,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6609,7 +7336,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -6646,7 +7376,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -6658,7 +7391,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -6680,17 +7417,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -6734,7 +7480,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -6778,7 +7527,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -6815,7 +7567,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -6830,7 +7585,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6867,7 +7624,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -6899,7 +7659,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6942,7 +7704,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -6985,7 +7750,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -7022,10 +7790,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7068,7 +7841,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -7126,7 +7902,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -7167,7 +7946,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -7179,7 +7961,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -7188,25 +7973,36 @@ } } }, - "required": ["input"], + "required": [ + "input" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["mean", "min", "max"] + "enum": [ + "mean", + "min", + "max" + ] }, "on_turn_failure": { "type": "string", - "enum": ["continue", "stop"] + "enum": [ + "continue", + "stop" + ] }, "window_size": { "type": "integer", "minimum": 1 } }, - "required": ["id"], + "required": [ + "id" + ], "additionalProperties": false }, { @@ -7218,7 +8014,10 @@ }, "type": { "type": "string", - "enum": ["suite", "tests"] + "enum": [ + "suite", + "tests" + ] }, "select": { "anyOf": [ @@ -7289,7 +8088,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -7318,7 +8121,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -7329,7 +8137,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -7346,7 +8156,10 @@ "additionalProperties": false } }, - "required": ["include", "type"], + "required": [ + "include", + "type" + ], "additionalProperties": false }, { @@ -7393,7 +8206,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -7412,20 +8230,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false }, { @@ -7444,7 +8272,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -7463,20 +8296,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -7505,7 +8348,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -7524,20 +8372,30 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -7581,7 +8439,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -7655,12 +8516,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -7697,7 +8564,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -7755,7 +8625,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -7796,7 +8669,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -7847,12 +8723,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7863,7 +8744,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -7926,7 +8809,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7942,7 +8827,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -7959,7 +8847,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -7976,13 +8867,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -8019,11 +8915,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -8064,7 +8969,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8078,7 +8988,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8089,7 +9004,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -8097,7 +9014,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8111,7 +9033,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8122,7 +9049,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -8159,7 +9089,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -8171,7 +9104,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -8193,17 +9130,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -8247,7 +9193,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -8291,7 +9240,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -8328,7 +9280,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -8343,7 +9298,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8380,7 +9337,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -8412,7 +9372,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8455,7 +9417,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -8498,7 +9463,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -8535,10 +9503,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8581,7 +9554,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -8639,7 +9615,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -8680,7 +9659,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -8692,7 +9674,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -8736,7 +9721,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -8810,12 +9798,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -8852,7 +9846,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -8910,7 +9907,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -8951,7 +9951,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -9002,12 +10005,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9018,7 +10026,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -9081,7 +10091,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9097,7 +10109,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -9114,7 +10129,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -9131,13 +10149,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -9174,11 +10197,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -9219,7 +10251,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9233,7 +10270,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9244,7 +10286,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -9252,7 +10296,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9266,7 +10315,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9277,7 +10331,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -9314,7 +10371,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -9326,7 +10386,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -9348,17 +10412,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -9402,7 +10475,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -9446,7 +10522,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -9483,7 +10562,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -9498,7 +10580,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9535,7 +10619,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -9567,7 +10654,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9610,7 +10699,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9653,7 +10745,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9690,10 +10785,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9736,7 +10836,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9794,7 +10897,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -9835,7 +10941,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -9847,7 +10956,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -9856,6 +10968,9 @@ "execution": { "type": "object", "properties": { + "workers": { + "not": {} + }, "assertions": { "type": "array", "items": { @@ -9894,7 +11009,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -9968,12 +11086,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -10010,7 +11134,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -10068,7 +11195,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -10109,7 +11239,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -10160,12 +11293,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10176,7 +11314,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -10239,7 +11379,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10255,7 +11397,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -10272,7 +11417,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -10289,13 +11437,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -10332,7 +11485,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -10383,7 +11539,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10397,7 +11558,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10408,7 +11574,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -10416,7 +11584,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10430,7 +11603,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10441,7 +11619,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -10478,7 +11659,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -10490,7 +11674,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -10512,17 +11700,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -10566,7 +11763,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -10610,7 +11810,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -10647,7 +11850,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -10662,7 +11868,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10699,7 +11907,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -10731,7 +11942,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10774,7 +11987,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10817,7 +12033,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10854,10 +12073,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10900,7 +12124,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10958,7 +12185,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -10999,7 +12229,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -11011,7 +12244,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -11055,7 +12291,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -11129,12 +12368,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -11171,7 +12416,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -11229,7 +12477,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -11270,7 +12521,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -11321,12 +12575,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11337,7 +12596,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -11400,7 +12661,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11416,7 +12679,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -11433,7 +12699,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -11450,13 +12719,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -11493,7 +12767,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -11544,7 +12821,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11558,7 +12840,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11569,7 +12856,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -11577,7 +12866,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11591,7 +12885,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11602,7 +12901,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -11639,7 +12941,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -11651,7 +12956,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -11673,17 +12982,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -11727,7 +13045,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -11771,7 +13092,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -11808,7 +13132,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -11823,7 +13150,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11860,7 +13189,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -11892,7 +13224,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11935,7 +13269,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -11978,7 +13315,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -12015,10 +13355,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12061,7 +13406,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -12119,7 +13467,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -12160,7 +13511,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -12172,7 +13526,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -12229,7 +13586,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -12240,7 +13602,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -12264,7 +13628,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_case"] + "enum": [ + "shared", + "per_case" + ] }, "repos": { "type": "array", @@ -12346,7 +13713,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12391,7 +13762,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12436,7 +13811,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12481,7 +13860,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12507,7 +13890,9 @@ "minimum": 0.1 } }, - "required": ["image"], + "required": [ + "image" + ], "additionalProperties": false }, "env": { @@ -12551,11 +13936,17 @@ }, "on_dependency_failure": { "type": "string", - "enum": ["skip", "fail", "run"] + "enum": [ + "skip", + "fail", + "run" + ] }, "mode": { "type": "string", - "enum": ["conversation"] + "enum": [ + "conversation" + ] }, "turns": { "type": "array", @@ -12584,13 +13975,20 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } @@ -12620,13 +14018,20 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file", "image"] + "enum": [ + "text", + "file", + "image" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } @@ -12677,7 +14082,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -12751,12 +14159,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -12793,7 +14207,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -12851,7 +14268,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -12892,7 +14312,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -12943,12 +14366,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12959,7 +14387,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -13022,7 +14452,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -13038,7 +14470,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -13055,7 +14490,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -13072,13 +14510,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -13115,7 +14558,10 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", @@ -13201,7 +14647,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -13209,7 +14657,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -13223,7 +14676,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -13234,7 +14692,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -13271,7 +14732,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -13283,7 +14747,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -13305,17 +14773,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -13359,7 +14836,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -13403,7 +14883,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -13440,7 +14923,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -13455,7 +14941,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -13492,7 +14980,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -13524,7 +15015,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -13567,7 +15060,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -13610,7 +15106,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -13647,10 +15146,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -13693,7 +15197,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -13751,7 +15258,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -13792,7 +15302,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -13804,7 +15317,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -13813,25 +15329,36 @@ } } }, - "required": ["input"], + "required": [ + "input" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["mean", "min", "max"] + "enum": [ + "mean", + "min", + "max" + ] }, "on_turn_failure": { "type": "string", - "enum": ["continue", "stop"] + "enum": [ + "continue", + "stop" + ] }, "window_size": { "type": "integer", "minimum": 1 } }, - "required": ["id"], + "required": [ + "id" + ], "additionalProperties": false }, { @@ -13843,7 +15370,10 @@ }, "type": { "type": "string", - "enum": ["suite", "tests"] + "enum": [ + "suite", + "tests" + ] }, "select": { "anyOf": [ @@ -13914,7 +15444,11 @@ { "type": "array", "items": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, "minItems": 1 } @@ -13943,7 +15477,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -13954,7 +15493,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "timeout_seconds": { @@ -13971,7 +15512,10 @@ "additionalProperties": false } }, - "required": ["include", "type"], + "required": [ + "include", + "type" + ], "additionalProperties": false }, { @@ -14053,13 +15597,18 @@ "additionalProperties": {} } }, - "required": ["name"], + "required": [ + "name" + ], "additionalProperties": false } ] }, "minItems": 1 }, + "workers": { + "not": {} + }, "assertions": { "type": "array", "items": { @@ -14098,7 +15647,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -14172,12 +15724,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -14214,7 +15772,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -14272,7 +15833,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -14313,7 +15877,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -14364,12 +15931,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -14380,7 +15952,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -14443,7 +16017,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -14459,7 +16035,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -14476,7 +16055,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -14493,13 +16075,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -14536,11 +16123,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -14581,7 +16177,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -14595,7 +16196,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -14606,7 +16212,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -14614,7 +16222,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -14628,7 +16241,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -14639,7 +16257,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -14676,7 +16297,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -14688,7 +16312,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -14710,17 +16338,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -14764,7 +16401,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -14808,7 +16448,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -14845,7 +16488,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -14860,7 +16506,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -14897,7 +16545,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -14929,7 +16580,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -14972,7 +16625,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -15015,7 +16671,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -15052,10 +16711,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -15098,7 +16762,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -15156,7 +16823,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -15197,7 +16867,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -15209,7 +16882,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -15253,7 +16929,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -15327,12 +17006,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -15369,7 +17054,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -15427,7 +17115,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -15468,7 +17159,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -15519,12 +17213,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -15535,7 +17234,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -15598,7 +17299,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -15614,7 +17317,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -15631,7 +17337,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -15648,13 +17357,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -15691,11 +17405,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -15736,7 +17459,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -15750,7 +17478,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -15761,7 +17494,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -15769,7 +17504,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -15783,7 +17523,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -15794,7 +17539,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -15831,7 +17579,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -15843,7 +17594,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -15865,17 +17620,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -15919,7 +17683,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -15963,7 +17730,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -16000,7 +17770,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -16015,7 +17788,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -16052,7 +17827,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -16084,7 +17862,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -16127,7 +17907,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -16170,7 +17953,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -16207,10 +17993,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -16253,7 +18044,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -16311,7 +18105,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -16352,7 +18149,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -16364,7 +18164,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -16427,7 +18230,12 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "pass_all", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -16438,7 +18246,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "runs": { @@ -16497,7 +18307,10 @@ }, "type": { "type": "string", - "enum": ["code-grader", "code_grader"] + "enum": [ + "code-grader", + "code_grader" + ] }, "command": { "anyOf": [ @@ -16571,12 +18384,18 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -16613,7 +18432,10 @@ }, "type": { "type": "string", - "enum": ["llm-grader", "llm_grader"] + "enum": [ + "llm-grader", + "llm_grader" + ] }, "prompt": { "anyOf": [ @@ -16671,7 +18493,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -16712,7 +18537,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -16763,12 +18591,17 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -16779,7 +18612,9 @@ "minLength": 1 } }, - "required": ["include"], + "required": [ + "include" + ], "additionalProperties": false }, { @@ -16842,7 +18677,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -16858,7 +18695,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -16875,7 +18715,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -16892,13 +18735,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -16935,11 +18783,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -16980,7 +18837,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -16994,7 +18856,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -17005,7 +18872,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -17013,7 +18882,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -17027,7 +18901,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -17038,7 +18917,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -17075,7 +18957,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -17087,7 +18972,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -17109,17 +18998,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -17163,7 +19061,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -17207,7 +19108,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -17244,7 +19148,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -17259,7 +19166,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -17296,7 +19205,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -17328,7 +19240,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -17371,7 +19285,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -17414,7 +19331,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -17451,10 +19371,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -17497,7 +19422,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -17555,7 +19483,10 @@ }, "operator": { "type": "string", - "enum": ["correctness", "contradiction"] + "enum": [ + "correctness", + "contradiction" + ] }, "weight": { "type": "number" @@ -17596,7 +19527,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -17608,7 +19542,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -17637,7 +19574,10 @@ ] } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false } }, @@ -17651,7 +19591,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_case"] + "enum": [ + "shared", + "per_case" + ] }, "repos": { "type": "array", @@ -17733,7 +19676,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -17778,7 +19725,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -17823,7 +19774,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -17868,7 +19823,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -17894,7 +19853,9 @@ "minimum": 0.1 } }, - "required": ["image"], + "required": [ + "image" + ], "additionalProperties": false }, "env": { From a90048cb6404db69515e147a80a53bbf4f805c9b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 30 Jun 2026 12:59:12 +0200 Subject: [PATCH 3/3] fix(schema): format generated eval schema --- packages/core/scripts/generate-eval-schema.ts | 20 + .../references/eval.schema.json | 3206 ++++------------- 2 files changed, 647 insertions(+), 2579 deletions(-) diff --git a/packages/core/scripts/generate-eval-schema.ts b/packages/core/scripts/generate-eval-schema.ts index 73b22a69b..90fe690e4 100644 --- a/packages/core/scripts/generate-eval-schema.ts +++ b/packages/core/scripts/generate-eval-schema.ts @@ -1,4 +1,5 @@ #!/usr/bin/env bun +import { spawn } from 'node:child_process'; /** * Generates AgentV JSON schemas from Zod schemas. * Run: bun run generate:schema (from packages/core) @@ -9,6 +10,24 @@ import path from 'node:path'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { EvalFileSchema } from '../src/evaluation/validation/eval-file.schema.js'; +async function formatWithBiome(filePath: string): Promise { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['x', 'biome', 'format', '--write', filePath], { + stdio: 'inherit', + }); + + child.on('error', reject); + child.on('exit', (code) => { + if (code === 0) { + resolve(); + return; + } + + reject(new Error(`Biome exited with code ${code}`)); + }); + }); +} + async function writeSchema(options: { readonly schema: Parameters[0]; readonly name: string; @@ -36,6 +55,7 @@ async function writeSchema(options: { ); await writeFile(outputPath, `${JSON.stringify(schema, null, 2)}\n`); + await formatWithBiome(outputPath); console.log(`Generated: ${outputPath}`); } diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 44c4d1cdc..4c852e7b5 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -54,12 +54,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -78,30 +73,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false }, { @@ -120,12 +105,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -144,30 +124,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -264,11 +234,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -297,12 +263,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -313,9 +274,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -332,9 +291,7 @@ "additionalProperties": false } }, - "required": [ - "path" - ], + "required": ["path"], "additionalProperties": false }, { @@ -424,11 +381,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -457,12 +410,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -473,9 +421,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -492,9 +438,7 @@ "additionalProperties": false } }, - "required": [ - "path" - ], + "required": ["path"], "additionalProperties": false } ] @@ -581,11 +525,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -614,12 +554,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -630,9 +565,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -649,9 +582,7 @@ "additionalProperties": false } }, - "required": [ - "path" - ], + "required": ["path"], "additionalProperties": false }, { @@ -741,11 +672,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -774,12 +701,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -790,9 +712,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -809,9 +729,7 @@ "additionalProperties": false } }, - "required": [ - "path" - ], + "required": ["path"], "additionalProperties": false } ] @@ -850,12 +768,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -874,30 +787,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false }, { @@ -916,12 +819,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -940,30 +838,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -992,12 +880,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -1016,30 +899,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -1083,10 +956,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -1160,18 +1030,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -1208,10 +1072,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -1269,10 +1130,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -1313,10 +1171,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -1367,17 +1222,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1388,9 +1238,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -1453,9 +1301,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1471,10 +1317,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -1491,10 +1334,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -1511,18 +1351,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -1559,20 +1394,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -1613,12 +1439,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1632,12 +1453,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1648,9 +1464,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -1658,12 +1472,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1677,12 +1486,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1693,10 +1497,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -1733,10 +1534,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -1748,11 +1546,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -1774,26 +1568,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -1837,10 +1622,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -1884,10 +1666,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -1924,10 +1703,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -1942,9 +1718,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1981,10 +1755,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -2016,9 +1787,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2061,10 +1830,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2107,10 +1873,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2147,15 +1910,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2198,10 +1956,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2259,10 +2014,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -2303,10 +2055,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -2318,10 +2067,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -2365,10 +2111,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -2442,18 +2185,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -2490,10 +2227,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -2551,10 +2285,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -2595,10 +2326,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -2649,17 +2377,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2670,9 +2393,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -2735,9 +2456,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2753,10 +2472,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -2773,10 +2489,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -2793,18 +2506,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -2841,20 +2549,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -2895,12 +2594,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -2914,12 +2608,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -2930,9 +2619,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -2940,12 +2627,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -2959,12 +2641,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -2975,10 +2652,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -3015,10 +2689,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -3030,11 +2701,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -3056,26 +2723,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -3119,10 +2777,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -3166,10 +2821,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -3206,10 +2858,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -3224,9 +2873,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3263,10 +2910,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -3298,9 +2942,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3343,10 +2985,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3389,10 +3028,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3429,15 +3065,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3480,10 +3111,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3541,10 +3169,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -3585,10 +3210,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -3600,10 +3222,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -3653,10 +3272,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -3730,18 +3346,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -3778,10 +3388,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -3839,10 +3446,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -3883,10 +3487,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -3937,17 +3538,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3958,9 +3554,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -4023,9 +3617,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4041,10 +3633,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -4061,10 +3650,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -4081,18 +3667,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -4129,10 +3710,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -4183,12 +3761,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4202,12 +3775,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4218,9 +3786,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -4228,12 +3794,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4247,12 +3808,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4263,10 +3819,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -4303,10 +3856,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -4318,11 +3868,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -4344,26 +3890,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -4407,10 +3944,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -4454,10 +3988,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -4494,10 +4025,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -4512,9 +4040,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4551,10 +4077,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -4586,9 +4109,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4631,10 +4152,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4677,10 +4195,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4717,15 +4232,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4768,10 +4278,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4829,10 +4336,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -4873,10 +4377,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -4888,10 +4389,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -4935,10 +4433,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -5012,18 +4507,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -5060,10 +4549,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -5121,10 +4607,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -5165,10 +4648,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -5219,17 +4699,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5240,9 +4715,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -5305,9 +4778,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5323,10 +4794,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -5343,10 +4811,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -5363,18 +4828,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -5411,10 +4871,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -5465,12 +4922,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5484,12 +4936,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5500,9 +4947,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -5510,12 +4955,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5529,12 +4969,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5545,10 +4980,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -5585,10 +5017,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -5600,11 +5029,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -5626,26 +5051,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -5689,10 +5105,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -5736,10 +5149,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -5776,10 +5186,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -5794,9 +5201,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5833,10 +5238,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -5868,9 +5270,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5913,10 +5313,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -5959,10 +5356,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -5999,15 +5393,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6050,10 +5439,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -6111,10 +5497,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -6155,10 +5538,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -6170,10 +5550,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -6230,12 +5607,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -6246,9 +5618,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -6272,10 +5642,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_case" - ] + "enum": ["shared", "per_case"] }, "repos": { "type": "array", @@ -6357,11 +5724,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -6406,11 +5769,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -6455,11 +5814,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -6504,11 +5859,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -6534,9 +5885,7 @@ "minimum": 0.1 } }, - "required": [ - "image" - ], + "required": ["image"], "additionalProperties": false }, "env": { @@ -6580,17 +5929,11 @@ }, "on_dependency_failure": { "type": "string", - "enum": [ - "skip", - "fail", - "run" - ] + "enum": ["skip", "fail", "run"] }, "mode": { "type": "string", - "enum": [ - "conversation" - ] + "enum": ["conversation"] }, "turns": { "type": "array", @@ -6619,20 +5962,13 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } @@ -6662,20 +5998,13 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } @@ -6726,10 +6055,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -6803,18 +6129,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -6851,10 +6171,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -6912,10 +6229,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -6956,10 +6270,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -7010,17 +6321,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7031,9 +6337,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -7096,9 +6400,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7114,10 +6416,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -7134,10 +6433,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -7154,18 +6450,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -7202,10 +6493,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -7291,9 +6579,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -7301,12 +6587,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7320,12 +6601,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7336,10 +6612,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -7376,10 +6649,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -7391,11 +6661,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -7417,26 +6683,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -7480,10 +6737,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -7527,10 +6781,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -7567,10 +6818,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -7585,9 +6833,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7624,10 +6870,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -7659,9 +6902,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7704,10 +6945,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7750,10 +6988,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7790,15 +7025,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7841,10 +7071,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7902,10 +7129,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -7946,10 +7170,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -7961,10 +7182,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -7973,36 +7191,25 @@ } } }, - "required": [ - "input" - ], + "required": ["input"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "mean", - "min", - "max" - ] + "enum": ["mean", "min", "max"] }, "on_turn_failure": { "type": "string", - "enum": [ - "continue", - "stop" - ] + "enum": ["continue", "stop"] }, "window_size": { "type": "integer", "minimum": 1 } }, - "required": [ - "id" - ], + "required": ["id"], "additionalProperties": false }, { @@ -8014,10 +7221,7 @@ }, "type": { "type": "string", - "enum": [ - "suite", - "tests" - ] + "enum": ["suite", "tests"] }, "select": { "anyOf": [ @@ -8088,11 +7292,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -8121,12 +7321,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -8137,9 +7332,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -8156,10 +7349,7 @@ "additionalProperties": false } }, - "required": [ - "include", - "type" - ], + "required": ["include", "type"], "additionalProperties": false }, { @@ -8206,12 +7396,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -8230,30 +7415,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false }, { @@ -8272,12 +7447,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -8296,30 +7466,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -8348,12 +7508,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -8372,30 +7527,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -8439,10 +7584,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -8516,18 +7658,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -8564,10 +7700,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -8625,10 +7758,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -8669,10 +7799,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -8723,17 +7850,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8744,9 +7866,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -8809,9 +7929,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8827,10 +7945,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -8847,10 +7962,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -8867,18 +7979,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -8915,20 +8022,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -8969,12 +8067,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -8988,12 +8081,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9004,9 +8092,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -9014,12 +8100,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9033,12 +8114,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9049,10 +8125,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -9089,10 +8162,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -9104,11 +8174,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -9130,26 +8196,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -9193,10 +8250,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -9240,10 +8294,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -9280,10 +8331,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -9298,9 +8346,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9337,10 +8383,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -9372,9 +8415,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9417,10 +8458,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -9463,10 +8501,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -9503,15 +8538,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9554,10 +8584,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -9615,10 +8642,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -9659,10 +8683,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -9674,10 +8695,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -9721,10 +8739,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -9798,18 +8813,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -9846,10 +8855,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -9907,10 +8913,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -9951,10 +8954,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -10005,17 +9005,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10026,9 +9021,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -10091,9 +9084,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10109,10 +9100,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -10129,10 +9117,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -10149,18 +9134,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -10197,20 +9177,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -10251,12 +9222,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -10270,12 +9236,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -10286,9 +9247,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -10296,12 +9255,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -10315,12 +9269,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -10331,10 +9280,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -10371,10 +9317,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -10386,11 +9329,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -10412,26 +9351,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -10475,10 +9405,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -10522,10 +9449,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -10562,10 +9486,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -10580,9 +9501,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10619,10 +9538,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -10654,9 +9570,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10699,10 +9613,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10745,10 +9656,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10785,15 +9693,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10836,10 +9739,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10897,10 +9797,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -10941,10 +9838,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -10956,10 +9850,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -11009,10 +9900,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -11086,18 +9974,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -11134,10 +10016,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -11195,10 +10074,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -11239,10 +10115,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -11293,17 +10166,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11314,9 +10182,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -11379,9 +10245,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11397,10 +10261,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -11417,10 +10278,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -11437,18 +10295,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -11485,10 +10338,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -11539,12 +10389,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11558,12 +10403,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11574,9 +10414,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -11584,12 +10422,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11603,12 +10436,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11619,10 +10447,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -11659,10 +10484,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -11674,11 +10496,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -11700,26 +10518,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -11763,10 +10572,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -11810,10 +10616,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -11850,10 +10653,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -11868,9 +10668,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11907,10 +10705,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -11942,9 +10737,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11987,10 +10780,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12033,10 +10823,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12073,15 +10860,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12124,10 +10906,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12185,10 +10964,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -12229,10 +11005,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -12244,10 +11017,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -12291,10 +11061,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -12368,18 +11135,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -12416,10 +11177,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -12477,10 +11235,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -12521,10 +11276,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -12575,17 +11327,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12596,9 +11343,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -12661,9 +11406,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12679,10 +11422,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -12699,10 +11439,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -12719,18 +11456,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -12767,10 +11499,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -12821,12 +11550,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12840,12 +11564,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12856,9 +11575,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -12866,12 +11583,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12885,12 +11597,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12901,10 +11608,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -12941,10 +11645,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -12956,11 +11657,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -12982,26 +11679,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -13045,10 +11733,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -13092,10 +11777,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -13132,10 +11814,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -13150,9 +11829,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13189,10 +11866,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -13224,9 +11898,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13269,10 +11941,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -13315,10 +11984,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -13355,15 +12021,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13406,10 +12067,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -13467,10 +12125,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -13511,10 +12166,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -13526,10 +12178,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -13586,12 +12235,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -13602,9 +12246,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -13628,10 +12270,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_case" - ] + "enum": ["shared", "per_case"] }, "repos": { "type": "array", @@ -13713,11 +12352,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -13762,11 +12397,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -13811,11 +12442,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -13860,11 +12487,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -13890,9 +12513,7 @@ "minimum": 0.1 } }, - "required": [ - "image" - ], + "required": ["image"], "additionalProperties": false }, "env": { @@ -13936,17 +12557,11 @@ }, "on_dependency_failure": { "type": "string", - "enum": [ - "skip", - "fail", - "run" - ] + "enum": ["skip", "fail", "run"] }, "mode": { "type": "string", - "enum": [ - "conversation" - ] + "enum": ["conversation"] }, "turns": { "type": "array", @@ -13975,20 +12590,13 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } @@ -14018,20 +12626,13 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file", - "image" - ] + "enum": ["text", "file", "image"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } @@ -14082,10 +12683,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -14159,18 +12757,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -14207,10 +12799,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -14268,10 +12857,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -14312,10 +12898,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -14366,17 +12949,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -14387,9 +12965,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -14452,9 +13028,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -14470,10 +13044,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -14490,10 +13061,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -14510,18 +13078,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -14558,10 +13121,7 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", @@ -14647,9 +13207,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -14657,12 +13215,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -14676,12 +13229,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -14692,10 +13240,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -14732,10 +13277,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -14747,11 +13289,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -14773,26 +13311,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -14836,10 +13365,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -14883,10 +13409,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -14923,10 +13446,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -14941,9 +13461,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -14980,10 +13498,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -15015,9 +13530,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -15060,10 +13573,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -15106,10 +13616,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -15146,15 +13653,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -15197,10 +13699,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -15258,10 +13757,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -15302,10 +13798,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -15317,10 +13810,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -15329,36 +13819,25 @@ } } }, - "required": [ - "input" - ], + "required": ["input"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "mean", - "min", - "max" - ] + "enum": ["mean", "min", "max"] }, "on_turn_failure": { "type": "string", - "enum": [ - "continue", - "stop" - ] + "enum": ["continue", "stop"] }, "window_size": { "type": "integer", "minimum": 1 } }, - "required": [ - "id" - ], + "required": ["id"], "additionalProperties": false }, { @@ -15370,10 +13849,7 @@ }, "type": { "type": "string", - "enum": [ - "suite", - "tests" - ] + "enum": ["suite", "tests"] }, "select": { "anyOf": [ @@ -15444,11 +13920,7 @@ { "type": "array", "items": { - "type": [ - "string", - "number", - "boolean" - ] + "type": ["string", "number", "boolean"] }, "minItems": 1 } @@ -15477,12 +13949,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -15493,9 +13960,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "timeout_seconds": { @@ -15512,10 +13977,7 @@ "additionalProperties": false } }, - "required": [ - "include", - "type" - ], + "required": ["include", "type"], "additionalProperties": false }, { @@ -15597,9 +14059,7 @@ "additionalProperties": {} } }, - "required": [ - "name" - ], + "required": ["name"], "additionalProperties": false } ] @@ -15647,10 +14107,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -15724,18 +14181,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -15772,10 +14223,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -15833,10 +14281,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -15877,10 +14322,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -15931,17 +14373,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -15952,9 +14389,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -16017,9 +14452,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -16035,10 +14468,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -16055,10 +14485,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -16075,18 +14502,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -16123,20 +14545,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -16177,12 +14590,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -16196,12 +14604,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -16212,9 +14615,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -16222,12 +14623,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -16241,12 +14637,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -16257,10 +14648,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -16297,10 +14685,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -16312,11 +14697,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -16338,26 +14719,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -16401,10 +14773,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -16448,10 +14817,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -16488,10 +14854,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -16506,9 +14869,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -16545,10 +14906,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -16580,9 +14938,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -16625,10 +14981,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -16671,10 +15024,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -16711,15 +15061,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -16762,10 +15107,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -16823,10 +15165,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -16867,10 +15206,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -16882,10 +15218,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -16929,10 +15262,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -17006,18 +15336,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -17054,10 +15378,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -17115,10 +15436,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -17159,10 +15477,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -17213,17 +15528,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -17234,9 +15544,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -17299,9 +15607,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -17317,10 +15623,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -17337,10 +15640,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -17357,18 +15657,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -17405,20 +15700,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -17459,12 +15745,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -17478,12 +15759,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -17494,9 +15770,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -17504,12 +15778,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -17523,12 +15792,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -17539,10 +15803,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -17579,10 +15840,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -17594,11 +15852,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -17620,26 +15874,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -17683,10 +15928,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -17730,10 +15972,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -17770,10 +16009,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -17788,9 +16024,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -17827,10 +16061,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -17862,9 +16093,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -17907,10 +16136,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -17953,10 +16179,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -17993,15 +16216,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -18044,10 +16262,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -18105,10 +16320,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -18149,10 +16361,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -18164,10 +16373,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -18230,12 +16436,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "pass_all", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "pass_all", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -18246,9 +16447,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "runs": { @@ -18307,10 +16506,7 @@ }, "type": { "type": "string", - "enum": [ - "code-grader", - "code_grader" - ] + "enum": ["code-grader", "code_grader"] }, "command": { "anyOf": [ @@ -18384,18 +16580,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -18432,10 +16622,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-grader", - "llm_grader" - ] + "enum": ["llm-grader", "llm_grader"] }, "prompt": { "anyOf": [ @@ -18493,10 +16680,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -18537,10 +16721,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -18591,17 +16772,12 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -18612,9 +16788,7 @@ "minLength": 1 } }, - "required": [ - "include" - ], + "required": ["include"], "additionalProperties": false }, { @@ -18677,9 +16851,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -18695,10 +16867,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -18715,10 +16884,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -18735,18 +16901,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -18783,20 +16944,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -18837,12 +16989,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -18856,12 +17003,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -18872,9 +17014,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -18882,12 +17022,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -18901,12 +17036,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -18917,10 +17047,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -18957,10 +17084,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -18972,11 +17096,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -18998,26 +17118,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -19061,10 +17172,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -19108,10 +17216,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -19148,10 +17253,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -19166,9 +17268,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -19205,10 +17305,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -19240,9 +17337,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -19285,10 +17380,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -19331,10 +17423,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -19371,15 +17460,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -19422,10 +17506,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -19483,10 +17564,7 @@ }, "operator": { "type": "string", - "enum": [ - "correctness", - "contradiction" - ] + "enum": ["correctness", "contradiction"] }, "weight": { "type": "number" @@ -19527,10 +17605,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -19542,10 +17617,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -19574,10 +17646,7 @@ ] } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false } }, @@ -19591,10 +17660,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_case" - ] + "enum": ["shared", "per_case"] }, "repos": { "type": "array", @@ -19676,11 +17742,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -19725,11 +17787,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -19774,11 +17832,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -19823,11 +17877,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -19853,9 +17903,7 @@ "minimum": 0.1 } }, - "required": [ - "image" - ], + "required": ["image"], "additionalProperties": false }, "env": {