feat(evals): support default_test threshold

README.md

@@ -19,7 +19,7 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
 - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by name from `targets.yaml` or with an eval-local target object.
 - **Experiment** is the run/result grouping label being measured over that corpus, such as `backend-with-skills` or `backend-without-skills`.
-- **Run controls** configure repeats, timeouts, budgets, thresholds, and completion hooks with fields such as `repeat`, `timeout_seconds`, `budget_usd`, `threshold`, and `on_run_complete`.
+- **Per-test defaults / run controls** configure inherited score cutoffs, repeats, timeouts, budgets, and completion hooks with fields such as `default_test.threshold`, `repeat`, `timeout_seconds`, `budget_usd`, and `on_run_complete`.
 - **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
 ```mermaid
@@ -65,7 +65,8 @@ repeat:
   strategy: pass_any
   early_exit: false
 timeout_seconds: 600
-threshold: 0.8
+default_test:
+  threshold: 0.8
 budget_usd: 5
 
 workspace:
@@ -97,7 +98,8 @@ repeat:
   count: 2
   strategy: pass_any
 timeout_seconds: 900
-threshold: 0.85
+default_test:
+  threshold: 0.85
 
 tests:
   - id: fizzbuzz
@@ -106,6 +108,8 @@ tests:
 
 `target: codex-gpt5` resolves the named target from `.agentv/targets.yaml` or `targets.yaml` and uses its default provider, model, hooks, and provider settings. The object form above starts from `codex-gpt5`, then applies the eval-local fields for this eval. If `extends` is omitted, the object defines the full target inline and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed.
 
+Use `default_test.threshold` for the inherited per-test pass cutoff. Existing eval files with a top-level `threshold` still load during migration, and `--threshold` on the CLI still overrides YAML thresholds for a run.
+
 **4. Run it:**
 ```bash
 agentv eval evals/my-eval.yaml

apps/cli/src/commands/eval/run-eval.ts

@@ -1107,6 +1107,7 @@ async function prepareFileMetadata(params: {
     effectiveOptions.cliBudgetUsd === undefined
       ? (effectiveOptions.budgetUsd ?? suite.budgetUsd)
       : suite.budgetUsd;
+  const suiteDefaultThreshold = suite.defaultTest?.threshold ?? suite.threshold;
 
   if (testCases.length === 0) {
     return {
@@ -1120,7 +1121,7 @@ async function prepareFileMetadata(params: {
       yamlCachePath: suite.cacheConfig?.cachePath,
       budgetUsd: defaultBudgetUsd,
       failOnError: suite.failOnError,
-      threshold: suite.threshold,
+      threshold: suiteDefaultThreshold,
       tags: suite.metadata?.tags,
       providerFactory: suite.providerFactory,
     };
@@ -1280,7 +1281,7 @@ async function prepareFileMetadata(params: {
     yamlCachePath: suite.cacheConfig?.cachePath,
     budgetUsd: defaultBudgetUsd,
     failOnError: suite.failOnError,
-    threshold: suite.threshold,
+    threshold: suiteDefaultThreshold,
     tags: suite.metadata?.tags,
     providerFactory: suite.providerFactory,
   };
@@ -2076,9 +2077,10 @@ export async function runEvalCommand(
   });
   const hasPerFileRuntimeThresholds =
     options.cliThreshold === undefined &&
-    activeTestFiles.some(
-      (activeTestFile) => fileMetadata.get(activeTestFile)?.options.threshold !== undefined,
-    );
+    activeTestFiles.some((activeTestFile) => {
+      const metadata = fileMetadata.get(activeTestFile);
+      return metadata?.options.threshold !== undefined || metadata?.threshold !== undefined;
+    });
 
   // --transcript: create a shared TranscriptProvider and validate entry count
   let transcriptProviderFactory:
@@ -2228,7 +2230,7 @@ export async function runEvalCommand(
               tests: filteredTestCases,
               options: fileOptions,
               defaultTrialsConfig: fileOptions.transcript ? undefined : targetPrep.trialsConfig,
-              defaultThreshold: fileOptions.threshold ?? targetPrep.threshold,
+              defaultThreshold: targetPrep.threshold ?? fileOptions.threshold,
               defaultTimeoutSeconds: fileOptions.agentTimeoutSeconds,
               defaultBudgetUsd: targetPrep.budgetUsd,
             });

apps/cli/test/eval.integration.test.ts

@@ -684,6 +684,101 @@ describe('agentv eval CLI', () => {
     }
   }, 30_000);
 
+  it('resolves default_test threshold below CLI and per-test run overrides but above legacy threshold', async () => {
+    const fixture = await createFixture();
+    try {
+      const evalPath = path.join(fixture.suiteDir, 'default-threshold.eval.yaml');
+      await writeFile(
+        evalPath,
+        [
+          'name: default-threshold',
+          'target: file-target',
+          'threshold: 0.9',
+          'default_test:',
+          '  threshold: 0.6',
+          'tests:',
+          '  - id: default-case',
+          '    input: default',
+          '    criteria: ok',
+          '  - id: strict-case',
+          '    input: strict',
+          '    criteria: ok',
+          '    run:',
+          '      threshold: 1.0',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+
+      const firstRun = await runCli(fixture, ['eval', evalPath]);
+      expect(firstRun.exitCode).toBe(0);
+      const firstDiagnostics = await readDiagnostics(fixture);
+      expect(firstDiagnostics.calls).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({ evalCaseIds: ['default-case'], threshold: 0.6 }),
+          expect.objectContaining({ evalCaseIds: ['strict-case'], threshold: 1 }),
+        ]),
+      );
+
+      await rm(fixture.diagnosticsPath, { force: true });
+
+      const cliRun = await runCli(fixture, ['eval', evalPath, '--threshold', '0.4']);
+      expect(cliRun.exitCode).toBe(0);
+      const cliDiagnostics = await readDiagnostics(fixture);
+      expect(cliDiagnostics).toMatchObject({
+        evalCaseIds: ['default-case', 'strict-case'],
+        threshold: 0.4,
+      });
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it('summarizes multi-file default_test thresholds from per-result execution status', async () => {
+    const fixture = await createFixture();
+    try {
+      const firstPath = path.join(fixture.suiteDir, 'first-default-threshold.eval.yaml');
+      const secondPath = path.join(fixture.suiteDir, 'second-default-threshold.eval.yaml');
+      await writeFile(
+        firstPath,
+        [
+          'name: first-default-threshold',
+          'target: file-target',
+          'default_test:',
+          '  threshold: 0.6',
+          'tests:',
+          '  - id: first-default-case',
+          '    input: first',
+          '    criteria: ok',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+      await writeFile(
+        secondPath,
+        [
+          'name: second-default-threshold',
+          'target: file-target',
+          'default_test:',
+          '  threshold: 0.7',
+          'tests:',
+          '  - id: second-default-case',
+          '    input: second',
+          '    criteria: ok',
+          '',
+        ].join('\n'),
+        'utf8',
+      );
+
+      const { stdout, exitCode } = await runCli(fixture, ['eval', firstPath, secondPath]);
+
+      expect(exitCode).toBe(0);
+      expect(stdout).toContain('scored >= configured threshold(s)');
+    } finally {
+      await rm(fixture.baseDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
   it('keeps non-concurrency run controls isolated across multiple eval files', async () => {
     const fixture = await createFixture();
     try {

packages/core/src/evaluation/loaders/config-loader.ts

@@ -444,7 +444,7 @@ export function extractFailOnError(suite: JsonObject): FailOnError | undefined {
 }
 
 /**
- * Extract top-level suite quality threshold.
+ * Extract the legacy top-level suite quality threshold.
  * Accepts a number in [0, 1] range.
  * Returns undefined when not specified.
  */
@@ -457,6 +457,34 @@ export function extractThreshold(suite: JsonObject): number | undefined {
   );
 }
 
+/**
+ * Extract the preferred inherited per-test default threshold.
+ * Accepts default_test.threshold as a number in [0, 1] range.
+ * Returns undefined when not specified.
+ */
+export function extractDefaultTestThreshold(suite: JsonObject): number | undefined {
+  rejectAuthoredRuntimeContainers(suite);
+  const rawDefaultTest = suite.default_test;
+  if (rawDefaultTest === undefined || rawDefaultTest === null) {
+    return undefined;
+  }
+  if (!isJsonObject(rawDefaultTest)) {
+    logWarning(`Invalid default_test: ${rawDefaultTest}. Ignoring.`);
+    return undefined;
+  }
+  const rawThreshold = rawDefaultTest.threshold;
+  if (rawThreshold === undefined || rawThreshold === null) {
+    return undefined;
+  }
+  if (typeof rawThreshold === 'number' && rawThreshold >= 0 && rawThreshold <= 1) {
+    return rawThreshold;
+  }
+  logWarning(
+    `Invalid default_test.threshold. Must be a number between 0 and 1: ${rawThreshold}. Ignoring.`,
+  );
+  return undefined;
+}
+
 export function parseExecutionDefaults(
   raw: unknown,
   configPath: string,

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -404,6 +404,12 @@ const RunOverrideSchema = z
   })
   .strict();
 
+const DefaultTestSchema = z
+  .object({
+    threshold: z.number().min(0).max(1).optional(),
+  })
+  .strict();
+
 /** Per-turn assertion: string shorthand (becomes rubric) or full evaluator config */
 const TurnAssertionSchema = z.union([z.string(), EvaluatorSchema]);
 
@@ -534,6 +540,7 @@ export const EvalFileSchema = z
     timeout_seconds: z.number().gt(0).optional(),
     budget_usd: z.number().gt(0).optional(),
     threshold: z.number().min(0).max(1).optional(),
+    default_test: DefaultTestSchema.optional(),
     on_run_complete: z.union([z.string().min(1), z.array(z.string().min(1))]).optional(),
     policy: z.never().optional(),
     execution: z.never().optional(),

packages/core/src/evaluation/validation/eval-validator.ts

@@ -74,6 +74,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([
   'timeout_seconds',
   'budget_usd',
   'threshold',
+  'default_test',
   'on_run_complete',
   'assertions',
   'evaluators',
@@ -333,6 +334,7 @@ export async function validateEvalFile(filePath: string): Promise<ValidationResu
   await validateSuiteWorkspaceConfigs(parsed, absolutePath, errors);
   validateAuthoredWorkers(parsed, absolutePath, errors);
   validateRepeatOverride(parsed.repeat, 'repeat', absolutePath, errors);
+  validateDefaultTest(parsed.default_test, absolutePath, errors);
   await validateImportsField(parsed.imports, absolutePath, errors);
 
   const cases: JsonValue | undefined = parsed.tests;
@@ -883,6 +885,7 @@ const WRAPPER_RUNTIME_CONTROL_FIELDS = [
   'timeout_seconds',
   'budget_usd',
   'threshold',
+  'default_test',
   'on_run_complete',
 ] as const;
 
@@ -1049,6 +1052,49 @@ function validateRunOverride(
   validateRepeatOverride(run.repeat, `${location}.repeat`, filePath, errors);
 }
 
+function validateDefaultTest(
+  defaultTest: JsonValue | undefined,
+  filePath: string,
+  errors: ValidationError[],
+): void {
+  if (defaultTest === undefined) {
+    return;
+  }
+  if (!isObject(defaultTest)) {
+    errors.push({
+      severity: 'error',
+      filePath,
+      location: 'default_test',
+      message: "Invalid 'default_test' field (must be an object)",
+    });
+    return;
+  }
+
+  for (const key of Object.keys(defaultTest)) {
+    if (key !== 'threshold') {
+      errors.push({
+        severity: 'error',
+        filePath,
+        location: `default_test.${key}`,
+        message: 'Invalid default_test field. Supported fields: threshold.',
+      });
+    }
+  }
+
+  const threshold = defaultTest.threshold;
+  if (
+    threshold !== undefined &&
+    (typeof threshold !== 'number' || threshold < 0 || threshold > 1)
+  ) {
+    errors.push({
+      severity: 'error',
+      filePath,
+      location: 'default_test.threshold',
+      message: "Invalid 'default_test.threshold' field (must be a number between 0 and 1)",
+    });
+  }
+}
+
 function validateRepeatOverride(
   repeat: JsonValue | undefined,
   location: string,

packages/core/src/evaluation/yaml-parser.ts

@@ -21,6 +21,7 @@ import {
 import {
   extractBudgetUsd,
   extractCacheConfig,
+  extractDefaultTestThreshold,
   extractFailOnError,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
@@ -82,6 +83,7 @@ export { buildPromptInputs, type PromptInputs } from './formatting/prompt-builde
 export {
   DEFAULT_EVAL_PATTERNS,
   extractCacheConfig,
+  extractDefaultTestThreshold,
   extractFailOnError,
   extractTargetFromSuite,
   extractTargetRefsFromSuite,
@@ -187,6 +189,7 @@ type RawTestSuite = JsonObject & {
   readonly timeout_seconds?: JsonValue;
   readonly budget_usd?: JsonValue;
   readonly threshold?: JsonValue;
+  readonly default_test?: JsonValue;
   readonly workspace?: JsonValue;
   readonly assertions?: JsonValue;
   readonly preprocessors?: JsonValue;
@@ -354,6 +357,8 @@ export type EvalSuiteResult = {
   readonly failOnError?: import('./types.js').FailOnError;
   /** Suite-level quality threshold (0-1) — suite fails if mean score is below */
   readonly threshold?: number;
+  /** Preferred inherited per-test defaults from default_test. */
+  readonly defaultTest?: EvalDefaultTestDefaults;
   /** Internal normalized run controls derived from flat eval YAML. */
   readonly experimentConfig?: ExperimentConfig;
   /** Inline target definition from a TS eval config. */
@@ -362,6 +367,10 @@ export type EvalSuiteResult = {
   readonly providerFactory?: import('./providers/provider-registry.js').ProviderFactoryFn;
 };
 
+export type EvalDefaultTestDefaults = {
+  readonly threshold?: number;
+};
+
 export type EvalTargetSpec = {
   readonly name: string;
   readonly extends?: string;
@@ -868,6 +877,9 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E
   const metadata = parseMetadata(parsed);
   const failOnError = extractFailOnError(parsed);
   const threshold = extractThreshold(parsed);
+  const defaultTestThreshold = extractDefaultTestThreshold(parsed);
+  const defaultTest =
+    defaultTestThreshold !== undefined ? { threshold: defaultTestThreshold } : undefined;
   const experimentConfig = normalizeSuiteExperimentConfig(parsed);
 
   return {
@@ -881,6 +893,7 @@ function buildEvalSuiteResult(parsed: JsonObject, tests: readonly EvalTest[]): E
     ...(metadata !== undefined && { metadata }),
     ...(failOnError !== undefined && { failOnError }),
     ...(threshold !== undefined && { threshold }),
+    ...(defaultTest !== undefined && { defaultTest }),
     ...(experimentConfig !== undefined && { experimentConfig }),
   };
 }