Implement recursive grading result artifacts

apps/cli/src/commands/pipeline/bench.ts

@@ -26,13 +26,35 @@ interface EvaluatorScore {
   readonly assertions: readonly { text: string; passed: boolean; evidence?: string }[];
 }
 
-function toAssertionResult(assertion: { text: string; passed: boolean; evidence?: string }) {
+function toComponentResult(
+  assertion: { text: string; passed: boolean; evidence?: string },
+  evaluator?: Pick<EvaluatorScore, 'name' | 'type'>,
+) {
   return {
-    text: assertion.text,
-    passed: assertion.passed,
-    evidence: assertion.evidence ?? '',
+    pass: assertion.passed,
     score: assertion.passed ? 1 : 0,
-    verdict: assertion.passed ? 'pass' : 'fail',
+    reason: assertion.evidence ?? assertion.text,
+    assertion: {
+      ...(evaluator ? { name: evaluator.name, type: evaluator.type } : {}),
+      value: assertion.text,
+    },
+  };
+}
+
+function evaluatorComponent(evaluator: EvaluatorScore) {
+  const pass = evaluator.score >= DEFAULT_THRESHOLD;
+  return {
+    pass,
+    score: evaluator.score,
+    reason: pass ? 'Grader passed.' : 'Grader failed.',
+    assertion: {
+      name: evaluator.name,
+      type: evaluator.type,
+      weight: evaluator.weight,
+    },
+    component_results: evaluator.assertions.map((assertion) =>
+      toComponentResult(assertion, evaluator),
+    ),
   };
 }
 
@@ -139,20 +161,21 @@ export const evalBenchCommand = command({
       allPassRates.push(passRate);
 
       // Write grading.json
+      const pass = weightedScore >= DEFAULT_THRESHOLD;
       const grading = {
+        pass,
         score: Math.round(weightedScore * 1000) / 1000,
-        verdict: weightedScore >= DEFAULT_THRESHOLD ? 'pass' : 'fail',
-        assertion_results: allAssertions.map(toAssertionResult),
-        summary: { passed, failed, total: allAssertions.length, pass_rate: passRate },
-        graders: evaluators.map((e) => ({
-          name: e.name,
-          type: e.type,
-          score: e.score,
-          verdict: e.score >= DEFAULT_THRESHOLD ? 'pass' : 'fail',
-          reasoning: '',
-          weight: e.weight,
-          assertion_results: e.assertions.map(toAssertionResult),
-        })),
+        reason: pass ? 'All grading components passed.' : 'One or more grading components failed.',
+        component_results:
+          evaluators.length > 0
+            ? evaluators.map(evaluatorComponent)
+            : allAssertions.map((assertion) => toComponentResult(assertion)),
+        metadata: {
+          pass_count: passed,
+          fail_count: failed,
+          sample_count: allAssertions.length,
+          pass_rate: passRate,
+        },
       };
       await writeFile(
         join(testDir, 'grading.json'),
@@ -164,14 +187,10 @@ export const evalBenchCommand = command({
       const scores = evaluators.map((e) => ({
         name: e.name,
         type: e.type,
+        pass: e.score >= DEFAULT_THRESHOLD,
         score: e.score,
         weight: e.weight,
-        verdict: e.score >= 0.5 ? 'pass' : 'fail',
-        assertions: e.assertions.map((a) => ({
-          text: a.text,
-          passed: a.passed,
-          evidence: a.evidence ?? '',
-        })),
+        reason: e.score >= DEFAULT_THRESHOLD ? 'Grader passed.' : 'Grader failed.',
       }));
 
       // Read execution_status from metrics.json (written by pipeline run)

apps/cli/src/commands/results/manifest.ts

@@ -152,6 +152,9 @@ function readGradingAssertionResults(
 }
 
 function readNestedGradingScores(record: Record<string, unknown>): unknown {
+  if (Array.isArray(record.component_results)) {
+    return record.component_results;
+  }
   if (Array.isArray(record.scores)) {
     return record.scores;
   }
@@ -164,21 +167,74 @@ function readNestedGradingScores(record: Record<string, unknown>): unknown {
   return undefined;
 }
 
+function componentLabel(component: Record<string, unknown>): string {
+  const assertion = component.assertion;
+  if (assertion && typeof assertion === 'object' && !Array.isArray(assertion)) {
+    const record = assertion as Record<string, unknown>;
+    for (const key of ['value', 'name', 'id', 'type']) {
+      const value = record[key];
+      if (typeof value === 'string' && value.trim().length > 0) {
+        return value;
+      }
+    }
+  }
+  return typeof component.reason === 'string' ? component.reason : 'grading component';
+}
+
+function mapComponentAssertion(
+  component: Record<string, unknown>,
+): EvaluationResult['assertions'][number] {
+  return {
+    text: componentLabel(component),
+    passed: component.pass === true,
+    evidence: typeof component.reason === 'string' ? component.reason : undefined,
+  };
+}
+
+function collectComponentAssertions(value: unknown): NonNullable<EvaluationResult['assertions']> {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((component) => {
+    if (!component || typeof component !== 'object' || Array.isArray(component)) {
+      return [];
+    }
+    const record = component as Record<string, unknown>;
+    const nested = collectComponentAssertions(record.component_results);
+    return nested.length > 0 ? nested : [mapComponentAssertion(record)];
+  });
+}
+
 function mapGradingEvaluator(evaluator: Record<string, unknown>): HydratedScore {
-  const verdict =
-    evaluator.verdict === 'pass' || evaluator.verdict === 'fail' || evaluator.verdict === 'skip'
-      ? evaluator.verdict
-      : undefined;
+  const pass =
+    typeof evaluator.pass === 'boolean'
+      ? evaluator.pass
+      : evaluator.verdict === 'pass' ||
+        (typeof evaluator.score === 'number' && evaluator.score >= 0.8);
+  const verdict = pass ? ('pass' as const) : ('fail' as const);
   const details =
     evaluator.details && typeof evaluator.details === 'object' && !Array.isArray(evaluator.details)
       ? (evaluator.details as HydratedScore['details'])
       : undefined;
+  const assertion =
+    evaluator.assertion &&
+    typeof evaluator.assertion === 'object' &&
+    !Array.isArray(evaluator.assertion)
+      ? (evaluator.assertion as Record<string, unknown>)
+      : undefined;
 
   return {
-    name: String(evaluator.name ?? ''),
-    type: String(evaluator.type ?? '') as HydratedScore['type'],
+    name: String(assertion?.name ?? assertion?.id ?? evaluator.name ?? componentLabel(evaluator)),
+    type: String(assertion?.type ?? evaluator.type ?? 'llm-grader') as HydratedScore['type'],
     score: typeof evaluator.score === 'number' ? evaluator.score : 0,
-    assertions: readGradingAssertionResults(evaluator) ?? [],
+    reason: typeof evaluator.reason === 'string' ? evaluator.reason : undefined,
+    assertions: (() => {
+      const nestedAssertions = collectComponentAssertions(evaluator.component_results);
+      if (nestedAssertions.length > 0) {
+        return nestedAssertions;
+      }
+      return readGradingAssertionResults(evaluator) ?? [mapComponentAssertion(evaluator)];
+    })(),
     scores: mapGradingEvaluators(readNestedGradingScores(evaluator)),
     weight: typeof evaluator.weight === 'number' ? evaluator.weight : undefined,
     verdict,
@@ -338,12 +394,16 @@ function hydrateManifestRecord(
   const timing = metrics ?? readOptionalJson<LegacyTimingArtifact>(baseDir, record.timing_path);
   const testId = record.test_id ?? 'unknown';
   const gradingAssertions = grading
-    ? readGradingAssertionResults(grading as unknown as Record<string, unknown>)
+    ? collectComponentAssertions((grading as unknown as Record<string, unknown>).component_results)
     : undefined;
+  const gradingRecord = grading as
+    | (GradingArtifact & {
+        graders?: readonly Record<string, unknown>[];
+        evaluators?: readonly Record<string, unknown>[];
+      })
+    | undefined;
   const gradingScores = mapGradingEvaluators(
-    grading?.graders ??
-      (grading as (GradingArtifact & { evaluators?: GradingArtifact['graders'] }) | undefined)
-        ?.evaluators,
+    gradingRecord?.component_results ?? gradingRecord?.graders ?? gradingRecord?.evaluators,
   );
 
   return {

apps/cli/src/commands/results/summary.ts

@@ -39,10 +39,19 @@ export function formatSummary(
   let passRate: number;
 
   if (grading) {
-    // Use pre-computed assertion-level counts from grading artifact
-    passed = grading.summary.passed;
-    failed = grading.summary.failed;
-    passRate = grading.summary.pass_rate;
+    const metadata = grading.metadata ?? {};
+    passed =
+      typeof metadata.pass_count === 'number' ? metadata.pass_count : grading.pass ? total : 0;
+    failed =
+      typeof metadata.sample_count === 'number' && typeof metadata.pass_count === 'number'
+        ? metadata.sample_count - metadata.pass_count
+        : total - passed;
+    passRate =
+      typeof metadata.pass_rate === 'number'
+        ? metadata.pass_rate
+        : total > 0
+          ? Math.round((passed / total) * 1000) / 1000
+          : 0;
   } else {
     // Fall back to computing from per-test scores
     passed = results.filter((r) => r.score >= 1.0).length;

apps/cli/src/commands/results/validate.ts

@@ -54,6 +54,66 @@ interface IndexEntry {
   readonly [key: string]: unknown;
 }
 
+const LEGACY_GRADING_FIELDS = [
+  'assertion_results',
+  'assertions',
+  'passed',
+  'evidence',
+  'verdict',
+  'graders',
+  'checks',
+] as const;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function validateGradingNode(value: unknown, pathLabel: string): string[] {
+  if (!isRecord(value)) {
+    return [`${pathLabel} must be an object`];
+  }
+
+  const errors: string[] = [];
+  if (
+    typeof value.pass !== 'boolean' ||
+    typeof value.score !== 'number' ||
+    typeof value.reason !== 'string'
+  ) {
+    errors.push(`${pathLabel} must include pass, score, and reason`);
+  }
+  if (value.component_results !== undefined) {
+    if (!Array.isArray(value.component_results)) {
+      errors.push(`${pathLabel}.component_results must be an array when present`);
+    } else {
+      value.component_results.forEach((component, index) => {
+        errors.push(...validateGradingNode(component, `${pathLabel}.component_results[${index}]`));
+      });
+    }
+  }
+  return errors;
+}
+
+function validateNoLegacyGradingFields(value: unknown, pathLabel: string): string[] {
+  if (Array.isArray(value)) {
+    return value.flatMap((entry, index) =>
+      validateNoLegacyGradingFields(entry, `${pathLabel}[${index}]`),
+    );
+  }
+  if (!isRecord(value)) {
+    return [];
+  }
+
+  const errors: string[] = [];
+  const legacyFields = LEGACY_GRADING_FIELDS.filter((field) => Object.hasOwn(value, field));
+  if (legacyFields.length > 0) {
+    errors.push(`${pathLabel} uses legacy field(s): ${legacyFields.join(', ')}`);
+  }
+  for (const [key, entry] of Object.entries(value)) {
+    errors.push(...validateNoLegacyGradingFields(entry, `${pathLabel}.${key}`));
+  }
+  return errors;
+}
+
 // ── Checks ───────────────────────────────────────────────────────────────
 
 function checkDirectoryNaming(runDir: string): Diagnostic[] {
@@ -204,7 +264,8 @@ function checkIndexJsonl(runDir: string): { diagnostics: Diagnostic[]; entries:
           if (typeof s.name !== 'string') missing.push('name');
           if (typeof s.type !== 'string') missing.push('type');
           if (typeof s.score !== 'number') missing.push('score');
-          if (typeof s.verdict !== 'string') missing.push('verdict');
+          if (typeof s.pass !== 'boolean') missing.push('pass');
+          if (typeof s.reason !== 'string') missing.push('reason');
           if (missing.length > 0) {
             diagnostics.push({
               severity: 'warning',
@@ -285,23 +346,13 @@ function checkArtifactFiles(runDir: string, entries: IndexEntry[]): Diagnostic[]
       } else {
         try {
           const grading = JSON.parse(readFileSync(gradingPath, 'utf8'));
-          if (Array.isArray(grading.assertion_results)) {
-            // Current grading sidecar contract.
-          } else if (Array.isArray(grading.assertions)) {
-            diagnostics.push({
-              severity: 'warning',
-              message: `${testId}: grading.json uses legacy 'assertions' array; rewrite the run to emit 'assertion_results'`,
-            });
-          } else {
+          for (const error of [
+            ...validateGradingNode(grading, 'grading.json'),
+            ...validateNoLegacyGradingFields(grading, 'grading.json'),
+          ]) {
             diagnostics.push({
               severity: 'error',
-              message: `${testId}: grading.json missing 'assertion_results' array`,
-            });
-          }
-          if (!grading.summary) {
-            diagnostics.push({
-              severity: 'warning',
-              message: `${testId}: grading.json missing 'summary' object`,
+              message: `${testId}: ${error}`,
             });
           }
         } catch {

apps/cli/test/commands/eval/aggregate.test.ts

@@ -295,7 +295,8 @@ describe('writePerTestArtifacts', () => {
     const grading1 = JSON.parse(
       readFileSync(rowRunPath(tmpDir, 'test-1', 'sample-1', 'grading.json'), 'utf8'),
     );
-    expect(grading1.assertion_results).toHaveLength(1);
+    expect(grading1.component_results).toHaveLength(1);
+    expect(grading1).not.toHaveProperty('assertion_results');
 
     const metrics1 = JSON.parse(
       readFileSync(rowRunPath(tmpDir, 'test-1', 'sample-1', 'metrics.json'), 'utf8'),
@@ -305,7 +306,8 @@ describe('writePerTestArtifacts', () => {
     const grading2 = JSON.parse(
       readFileSync(rowRunPath(tmpDir, 'test-2', 'sample-1', 'grading.json'), 'utf8'),
     );
-    expect(grading2.assertion_results).toHaveLength(1);
+    expect(grading2.component_results).toHaveLength(1);
+    expect(grading2).not.toHaveProperty('assertion_results');
   });
 
   it('writes outputs/answer.md for results with output', async () => {