diff --git a/README.md b/README.md index 1afe02603..6090e3f0b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Test AI targets on real repo tasks and measure what actually works. - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts. - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by name from `targets.yaml` or with an eval-local target object. - **Experiment** is the run/result grouping label being measured over that corpus, such as `backend-with-skills` or `backend-without-skills`. -- **Run controls** configure repeats, timeouts, budgets, thresholds, and completion hooks with fields such as `repeat`, `timeout_seconds`, `budget_usd`, `threshold`, and `on_run_complete`. +- **Per-test defaults / run controls** configure inherited score cutoffs, repeats, timeouts, budgets, and completion hooks with fields such as `default_test.threshold`, `repeat`, `timeout_seconds`, `budget_usd`, and `on_run_complete`. - **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. ```mermaid @@ -65,7 +65,8 @@ repeat: strategy: pass_any early_exit: false timeout_seconds: 600 -threshold: 0.8 +default_test: + threshold: 0.8 budget_usd: 5 workspace: @@ -97,7 +98,8 @@ repeat: count: 2 strategy: pass_any timeout_seconds: 900 -threshold: 0.85 +default_test: + threshold: 0.85 tests: - id: fizzbuzz @@ -106,6 +108,8 @@ tests: `target: codex-gpt5` resolves the named target from `.agentv/targets.yaml` or `targets.yaml` and uses its default provider, model, hooks, and provider settings. The object form above starts from `codex-gpt5`, then applies the eval-local fields for this eval. If `extends` is omitted, the object defines the full target inline and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. +Use `default_test.threshold` for the inherited per-test pass cutoff. Existing eval files with a top-level `threshold` still load during migration, and `--threshold` on the CLI still overrides YAML thresholds for a run. + **4. Run it:** ```bash agentv eval evals/my-eval.yaml diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index f78854391..88af2f0be 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -1107,6 +1107,7 @@ async function prepareFileMetadata(params: { effectiveOptions.cliBudgetUsd === undefined ? (effectiveOptions.budgetUsd ?? suite.budgetUsd) : suite.budgetUsd; + const suiteDefaultThreshold = suite.defaultTest?.threshold ?? suite.threshold; if (testCases.length === 0) { return { @@ -1120,7 +1121,7 @@ async function prepareFileMetadata(params: { yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, failOnError: suite.failOnError, - threshold: suite.threshold, + threshold: suiteDefaultThreshold, tags: suite.metadata?.tags, providerFactory: suite.providerFactory, }; @@ -1280,7 +1281,7 @@ async function prepareFileMetadata(params: { yamlCachePath: suite.cacheConfig?.cachePath, budgetUsd: defaultBudgetUsd, failOnError: suite.failOnError, - threshold: suite.threshold, + threshold: suiteDefaultThreshold, tags: suite.metadata?.tags, providerFactory: suite.providerFactory, }; @@ -2076,9 +2077,10 @@ export async function runEvalCommand( }); const hasPerFileRuntimeThresholds = options.cliThreshold === undefined && - activeTestFiles.some( - (activeTestFile) => fileMetadata.get(activeTestFile)?.options.threshold !== undefined, - ); + activeTestFiles.some((activeTestFile) => { + const metadata = fileMetadata.get(activeTestFile); + return metadata?.options.threshold !== undefined || metadata?.threshold !== undefined; + }); // --transcript: create a shared TranscriptProvider and validate entry count let transcriptProviderFactory: @@ -2228,7 +2230,7 @@ export async function runEvalCommand( tests: filteredTestCases, options: fileOptions, defaultTrialsConfig: fileOptions.transcript ? undefined : targetPrep.trialsConfig, - defaultThreshold: fileOptions.threshold ?? targetPrep.threshold, + defaultThreshold: targetPrep.threshold ?? fileOptions.threshold, defaultTimeoutSeconds: fileOptions.agentTimeoutSeconds, defaultBudgetUsd: targetPrep.budgetUsd, }); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 4050f8002..3908a9e9c 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -684,6 +684,101 @@ describe('agentv eval CLI', () => { } }, 30_000); + it('resolves default_test threshold below CLI and per-test run overrides but above legacy threshold', async () => { + const fixture = await createFixture(); + try { + const evalPath = path.join(fixture.suiteDir, 'default-threshold.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: default-threshold', + 'target: file-target', + 'threshold: 0.9', + 'default_test:', + ' threshold: 0.6', + 'tests:', + ' - id: default-case', + ' input: default', + ' criteria: ok', + ' - id: strict-case', + ' input: strict', + ' criteria: ok', + ' run:', + ' threshold: 1.0', + '', + ].join('\n'), + 'utf8', + ); + + const firstRun = await runCli(fixture, ['eval', evalPath]); + expect(firstRun.exitCode).toBe(0); + const firstDiagnostics = await readDiagnostics(fixture); + expect(firstDiagnostics.calls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ evalCaseIds: ['default-case'], threshold: 0.6 }), + expect.objectContaining({ evalCaseIds: ['strict-case'], threshold: 1 }), + ]), + ); + + await rm(fixture.diagnosticsPath, { force: true }); + + const cliRun = await runCli(fixture, ['eval', evalPath, '--threshold', '0.4']); + expect(cliRun.exitCode).toBe(0); + const cliDiagnostics = await readDiagnostics(fixture); + expect(cliDiagnostics).toMatchObject({ + evalCaseIds: ['default-case', 'strict-case'], + threshold: 0.4, + }); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + }, 30_000); + + it('summarizes multi-file default_test thresholds from per-result execution status', async () => { + const fixture = await createFixture(); + try { + const firstPath = path.join(fixture.suiteDir, 'first-default-threshold.eval.yaml'); + const secondPath = path.join(fixture.suiteDir, 'second-default-threshold.eval.yaml'); + await writeFile( + firstPath, + [ + 'name: first-default-threshold', + 'target: file-target', + 'default_test:', + ' threshold: 0.6', + 'tests:', + ' - id: first-default-case', + ' input: first', + ' criteria: ok', + '', + ].join('\n'), + 'utf8', + ); + await writeFile( + secondPath, + [ + 'name: second-default-threshold', + 'target: file-target', + 'default_test:', + ' threshold: 0.7', + 'tests:', + ' - id: second-default-case', + ' input: second', + ' criteria: ok', + '', + ].join('\n'), + 'utf8', + ); + + const { stdout, exitCode } = await runCli(fixture, ['eval', firstPath, secondPath]); + + expect(exitCode).toBe(0); + expect(stdout).toContain('scored >= configured threshold(s)'); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + }, 30_000); + it('keeps non-concurrency run controls isolated across multiple eval files', async () => { const fixture = await createFixture(); try { diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 9da28b9f2..b3fe62b41 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -444,7 +444,7 @@ export function extractFailOnError(suite: JsonObject): FailOnError | undefined { } /** - * Extract top-level suite quality threshold. + * Extract the legacy top-level suite quality threshold. * Accepts a number in [0, 1] range. * Returns undefined when not specified. */ @@ -457,6 +457,34 @@ export function extractThreshold(suite: JsonObject): number | undefined { ); } +/** + * Extract the preferred inherited per-test default threshold. + * Accepts default_test.threshold as a number in [0, 1] range. + * Returns undefined when not specified. + */ +export function extractDefaultTestThreshold(suite: JsonObject): number | undefined { + rejectAuthoredRuntimeContainers(suite); + const rawDefaultTest = suite.default_test; + if (rawDefaultTest === undefined || rawDefaultTest === null) { + return undefined; + } + if (!isJsonObject(rawDefaultTest)) { + logWarning(`Invalid default_test: ${rawDefaultTest}. Ignoring.`); + return undefined; + } + const rawThreshold = rawDefaultTest.threshold; + if (rawThreshold === undefined || rawThreshold === null) { + return undefined; + } + if (typeof rawThreshold === 'number' && rawThreshold >= 0 && rawThreshold <= 1) { + return rawThreshold; + } + logWarning( + `Invalid default_test.threshold. Must be a number between 0 and 1: ${rawThreshold}. Ignoring.`, + ); + return undefined; +} + export function parseExecutionDefaults( raw: unknown, configPath: string, diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index bfaddbbf9..c7fc5b1f6 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -404,6 +404,12 @@ const RunOverrideSchema = z }) .strict(); +const DefaultTestSchema = z + .object({ + threshold: z.number().min(0).max(1).optional(), + }) + .strict(); + /** Per-turn assertion: string shorthand (becomes rubric) or full evaluator config */ const TurnAssertionSchema = z.union([z.string(), EvaluatorSchema]); @@ -534,6 +540,7 @@ export const EvalFileSchema = z timeout_seconds: z.number().gt(0).optional(), budget_usd: z.number().gt(0).optional(), threshold: z.number().min(0).max(1).optional(), + default_test: DefaultTestSchema.optional(), on_run_complete: z.union([z.string().min(1), z.array(z.string().min(1))]).optional(), policy: z.never().optional(), execution: z.never().optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 66c522745..95e6f8da1 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -74,6 +74,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'timeout_seconds', 'budget_usd', 'threshold', + 'default_test', 'on_run_complete', 'assertions', 'evaluators', @@ -333,6 +334,7 @@ export async function validateEvalFile(filePath: string): Promise 1) + ) { + errors.push({ + severity: 'error', + filePath, + location: 'default_test.threshold', + message: "Invalid 'default_test.threshold' field (must be a number between 0 and 1)", + }); + } +} + function validateRepeatOverride( repeat: JsonValue | undefined, location: string, diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index bd216e4e2..6c5f56881 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -21,6 +21,7 @@ import { import { extractBudgetUsd, extractCacheConfig, + extractDefaultTestThreshold, extractFailOnError, extractTargetFromSuite, extractTargetRefsFromSuite, @@ -82,6 +83,7 @@ export { buildPromptInputs, type PromptInputs } from './formatting/prompt-builde export { DEFAULT_EVAL_PATTERNS, extractCacheConfig, + extractDefaultTestThreshold, extractFailOnError, extractTargetFromSuite, extractTargetRefsFromSuite, @@ -187,6 +189,7 @@ type RawTestSuite = JsonObject & { readonly timeout_seconds?: JsonValue; readonly budget_usd?: JsonValue; readonly threshold?: JsonValue; + readonly default_test?: JsonValue; readonly workspace?: JsonValue; readonly assertions?: JsonValue; readonly preprocessors?: JsonValue; @@ -354,6 +357,8 @@ export type EvalSuiteResult = { readonly failOnError?: import('./types.js').FailOnError; /** Suite-level quality threshold (0-1) — suite fails if mean score is below */ readonly threshold?: number; + /** Preferred inherited per-test defaults from default_test. */ + readonly defaultTest?: EvalDefaultTestDefaults; /** Internal normalized run controls derived from flat eval YAML. */ readonly experimentConfig?: ExperimentConfig; /** Inline target definition from a TS eval config. */ @@ -362,6 +367,10 @@ export type EvalSuiteResult = { readonly providerFactory?: import('./providers/provider-registry.js').ProviderFactoryFn; }; +export type EvalDefaultTestDefaults = { + readonly threshold?: number; +}; + export type EvalTargetSpec = { readonly name: string; readonly extends?: string; @@ -868,6 +877,9 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E const metadata = parseMetadata(parsed); const failOnError = extractFailOnError(parsed); const threshold = extractThreshold(parsed); + const defaultTestThreshold = extractDefaultTestThreshold(parsed); + const defaultTest = + defaultTestThreshold !== undefined ? { threshold: defaultTestThreshold } : undefined; const experimentConfig = normalizeSuiteExperimentConfig(parsed); return { @@ -881,6 +893,7 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E ...(metadata !== undefined && { metadata }), ...(failOnError !== undefined && { failOnError }), ...(threshold !== undefined && { threshold }), + ...(defaultTest !== undefined && { defaultTest }), ...(experimentConfig !== undefined && { experimentConfig }), }; } diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 547cef847..7ccd122f3 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -65,6 +65,31 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.targets).toBeUndefined(); }); + it('parses default_test.threshold separately from legacy top-level threshold', async () => { + const evalPath = path.join(tempDir, 'default-test-threshold.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: threshold-suite', + 'target: codex', + 'threshold: 0.9', + 'default_test:', + ' threshold: 0.6', + 'tests:', + ' - id: one', + ' input: hello', + ' criteria: ok', + '', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.defaultTest).toEqual({ threshold: 0.6 }); + expect(suite.threshold).toBe(0.9); + expect(suite.experimentConfig?.threshold).toBe(0.9); + }); + it('rejects authored workers in eval YAML runtime blocks', async () => { const cases = [ { diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index e418f5ca6..f12719062 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -136,6 +136,36 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(true); }); + it('accepts default_test.threshold as the preferred inherited test threshold', () => { + const result = EvalFileSchema.safeParse({ + default_test: { + threshold: 0.6, + }, + tests: [baseTest], + }); + + expect(result.success).toBe(true); + }); + + it('rejects invalid default_test values', () => { + const invalidThreshold = EvalFileSchema.safeParse({ + default_test: { + threshold: 1.2, + }, + tests: [baseTest], + }); + const unknownDefault = EvalFileSchema.safeParse({ + default_test: { + threshold: 0.6, + assertions: [], + }, + tests: [baseTest], + }); + + expect(invalidThreshold.success).toBe(false); + expect(unknownDefault.success).toBe(false); + }); + it('rejects authored policy blocks', () => { const result = EvalFileSchema.safeParse({ target: 'codex', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 313eafda7..5d11f38f9 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -76,6 +76,54 @@ imports: expect(result.errors).toHaveLength(0); }); + it('validates default_test.threshold', async () => { + const filePath = path.join(tempDir, 'default-test-threshold.yaml'); + await writeFile( + filePath, + `default_test: + threshold: 0.6 +tests: + - id: test-1 + criteria: Goal + input: Query +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('rejects invalid default_test threshold values and unsupported default fields', async () => { + const filePath = path.join(tempDir, 'invalid-default-test-threshold.yaml'); + await writeFile( + filePath, + `default_test: + threshold: 1.2 + assertions: [] +tests: + - id: test-1 + criteria: Goal + input: Query +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => error.severity === 'error' && error.location === 'default_test.threshold', + ), + ).toBe(true); + expect( + result.errors.some( + (error) => error.severity === 'error' && error.location === 'default_test.assertions', + ), + ).toBe(true); + }); + it('rejects removed top-level runs and early_exit with migration guidance', async () => { const filePath = path.join(tempDir, 'removed-repeat-fields.yaml'); await writeFile( diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 207c4e4bf..0eee22f5d 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -14255,6 +14255,17 @@ "minimum": 0, "maximum": 1 }, + "default_test": { + "type": "object", + "properties": { + "threshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "additionalProperties": false + }, "on_run_complete": { "anyOf": [ {