(MOT-4368) feat(e2e): surface semantic validation health

.github/benchmark-site/README.md

@@ -3,7 +3,7 @@
 This static shell replaces the generic benchmark-action index at
 `dev/harness-e2e/`. The workflow-generated `data.js` remains the source of truth
 for metric trends. `executions.js` indexes workflow attempts, and
-`runs/<execution-id>.json` supplies the complete retained reports.
+`runs/<execution-id>.json` supplies compact retained diagnostics.
 
 Import the default local report and serve the real dashboard from the repository
 root:
@@ -45,22 +45,23 @@ Metric names are stable identifiers:
 ```
 
 The execution index retains 100 workflow attempts. The latest 30 also retain the
-complete structured `results.json` content, including prompts, transcripts,
-session ids, gates, criteria, failures, retries, usage, and traces. The UI loads
-those reports only on the detail page and renders transcript-heavy sections only
-when expanded. Each run opens its transcript in a read-only dialog patterned
-after the Harness chat, with message and error filters, paired function calls
-and results, and recovered errors expanded by default. Diagnostic logs, stack
-files, and credentials remain in access-controlled Actions artifacts.
+allowlisted diagnostic projection: execution identity, scenario outcomes,
+scores, metrics, cost, duration, retries, hard gates, and failure messages.
+Prompts, transcripts, model responses, criteria, traces, and tool payloads are
+never copied into Pages. They remain in access-controlled Actions artifacts
+alongside diagnostic logs and stack files.
+Each publish also rewrites retained schema 2 detail files through the same
+allowlist and removes unreferenced run files before deploying Pages.
 
 Each full execution summary also carries compact per-scenario averages for
 tokens, wall time, cost, function calls, function-call errors, sessions, and
 turns. Tokens mean input plus output; cache-read tokens are already represented
 in input usage and are not added again. The execution table also exposes exact
-total tokens and function calls for every retained full report.
+total tokens and function calls for every retained diagnostic report.
 
-Efficiency is the primary overview. Its cards show the current operational suite
-totals, while deltas use only successful scenarios with the same subject,
+Operational health is the primary overview. Efficiency appears after the latest
+status, completeness, first actionable failure, KPIs, and scenario matrix. Its
+cards show current suite totals, while deltas use only successful scenarios with the same subject,
 scenario id, and behavioral contract fingerprint. New and changed scenarios
 collect five comparable executions before receiving a trend verdict. Removed
 scenarios remain visible as historical rows and never count as an efficiency

.github/benchmark-site/execution-data.js

@@ -114,13 +114,56 @@
     return [...scenarios].sort();
   }
 
-  function normalizeStatus(value) {
-    if (["passed", "failed", "incomplete", "cancelled"].includes(value)) return value;
+  function normalizeStatus(value, execution = {}) {
+    const semanticStatuses = [
+      "passed",
+      "quality_advisory",
+      "hard_gate_failed",
+      "technical_failed",
+      "infra_failed",
+      "incomplete",
+      "cancelled",
+      "running",
+    ];
+    if (semanticStatuses.includes(value)) return value;
     if (value === "pass" || value === "success") return "passed";
-    if (value === "fail" || value === "failure") return "failed";
+    if (value === "cancelled") return "cancelled";
+
+    // Schema 2 collapsed every complete non-pass into `failed`. Reconstruct
+    // the semantic outcome from its retained blocking counters.
+    if (value === "fail" || value === "failed" || value === "failure") {
+      const totals = execution?.totals || {};
+      if (Number(totals.missing_reports || 0) > 0) return "incomplete";
+      if (Number(totals.technical_failures || 0) > 0) return "technical_failed";
+      if (Number(totals.hard_gate_failures || 0) > 0) return "hard_gate_failed";
+      if (String(execution?.conclusion || "") !== "success") return "infra_failed";
+      const subjects = Array.isArray(execution?.subjects) ? execution.subjects : [];
+      if (subjects.length && subjects.some((subject) => !subject?.passed)) {
+        return "quality_advisory";
+      }
+      return "infra_failed";
+    }
     return "incomplete";
   }
 
+  function normalizeScenarioStatus(value) {
+    const scenario = value && typeof value === "object" ? value : {};
+    const status = String(scenario.status || "");
+    if (status === "cancelled") return "cancelled";
+    if (status === "running") return "running";
+    if (status === "missing_report" || status === "incomplete") return "incomplete";
+    if (status === "technical_failed" || Number(scenario.technical_failures || 0) > 0) {
+      return "technical_failed";
+    }
+    if (status === "hard_gate_failed" || Number(scenario.hard_gate_failures || 0) > 0) {
+      return "hard_gate_failed";
+    }
+    if (status === "infra_failed") return "infra_failed";
+    if (status === "quality_advisory") return "quality_advisory";
+    if (scenario.passed || status === "passed" || status === "success") return "passed";
+    return "quality_advisory";
+  }
+
   function normalizeExecution(entry) {
     const execution = entry && typeof entry === "object" ? entry : {};
     const subjects = Array.isArray(execution.subjects)
@@ -131,7 +174,7 @@
       id: String(execution.id || ""),
       run_id: String(execution.run_id || ""),
       attempt: Number(execution.attempt) || 1,
-      status: normalizeStatus(execution.status),
+      status: normalizeStatus(execution.status, execution),
       conclusion: String(execution.conclusion || ""),
       event: String(execution.event || ""),
       actor: String(execution.actor || ""),
@@ -166,7 +209,7 @@
     const scenarios = listLegacyScenarios(subject).map((scenarioId) => {
       const score = subject.metrics?.quality?.[scenarioId]?.median_score;
       const passRate = subject.metrics?.quality?.[scenarioId]?.pass_rate;
-      return {
+      const scenario = {
         id: scenarioId,
         status: score?.status || passRate?.status || "unknown",
         passed: score?.passed ?? passRate?.passed ?? false,
@@ -185,6 +228,7 @@
         wall_time_seconds:
           metricValue(subject, "efficiency", scenarioId, "wall_time_seconds"),
       };
+      return { ...scenario, status: normalizeScenarioStatus(scenario) };
     });
     return {
       id: subject.id,
@@ -194,7 +238,7 @@
       engine_revision: subject.engineRevision || "",
       passed: Boolean(subject.passed),
       expected_reports: scenarios.length,
-      received_reports: scenarios.filter((scenario) => scenario.status !== "missing_report")
+      received_reports: scenarios.filter((scenario) => scenario.status !== "incomplete")
         .length,
       scenario_pass_rate:
         (metricValue(subject, "quality", "suite", "scenario_pass_rate") ?? 0) / 100,
@@ -340,6 +384,32 @@
     });
   }
 
+  function latestHealthModel(entry) {
+    const execution = normalizeExecution(entry);
+    const release = execution.release || {};
+    const releaseIdentity = [release.worker, release.version]
+      .filter(Boolean)
+      .join("@");
+    const firstFailure =
+      execution.first_failure && typeof execution.first_failure === "object"
+        ? execution.first_failure
+        : null;
+    return {
+      status: execution.status,
+      lane: String(execution.lane || "daily"),
+      identity:
+        releaseIdentity ||
+        String(release.tag || "") ||
+        String(execution.source?.sha || "").slice(0, 12) ||
+        "Unknown",
+      expectedReports: Number(execution.totals?.expected_reports || 0),
+      receivedReports: Number(execution.totals?.received_reports || 0),
+      availability: execution.availability,
+      firstFailure,
+      workflowUrl: execution.workflow_url,
+    };
+  }
+
   function executionsWithinDays(executions, days, now = Date.now()) {
     const windowDays = Number(days);
     if (!Number.isFinite(windowDays) || windowDays <= 0) return [...(executions || [])];
@@ -384,18 +454,17 @@
     const subject = execution?.subjects?.find((item) => item.id === row.subjectId);
     const scenario = subject?.scenarios?.find((item) => item.id === row.scenarioId);
     if (!scenario) return null;
-    const status =
-      scenario.status === "missing_report"
-        ? "incomplete"
-        : scenario.passed
-          ? "passed"
-          : "failed";
+    const status = normalizeScenarioStatus(scenario);
     return { ...scenario, status };
   }
 
   function matrixCellLabel(cell, status) {
-    if (status === "failed") return "×";
+    if (["failed", "hard_gate_failed", "technical_failed", "infra_failed"].includes(status)) {
+      return "×";
+    }
+    if (status === "quality_advisory") return "!";
     if (status === "cancelled") return "○";
+    if (status === "running") return "•";
     if (status !== "passed") return "–";
 
     const score = numberOrNull(cell?.median_score);
@@ -527,7 +596,7 @@
     const technicalFailures = Number(scenario.technical_failures) || 0;
     return {
       passed: Boolean(scenario.passed),
-      complete: scenario.status !== "missing_report",
+      complete: normalizeScenarioStatus(scenario) !== "incomplete",
       hardGateFailures,
       technicalFailures,
       score: numberOrNull(scenario.median_score),
@@ -936,12 +1005,14 @@
     filterExecutions,
     findExecution,
     groupRunFailures,
+    latestHealthModel,
     legacyExecution,
     matrixCell,
     matrixCellLabel,
     matrixRows,
     mergeExecutionHistory,
     normalizeExecution,
+    normalizeScenarioStatus,
     normalizeStatus,
     scenarioMetricSeries,
     scenarioMetricRows,

.github/benchmark-site/execution-data.test.cjs

@@ -10,11 +10,13 @@ const {
   filterExecutions,
   findExecution,
   groupRunFailures,
+  latestHealthModel,
   matrixCell,
   matrixCellLabel,
   matrixRows,
   mergeExecutionHistory,
   normalizeExecution,
+  normalizeScenarioStatus,
   scenarioMetricSeries,
   scenarioMetricRows,
   scenarioMetricsFromDetail,
@@ -62,10 +64,74 @@ test("normalizes execution status and availability", () => {
   });
 
   assert.equal(normalized.id, "99");
-  assert.equal(normalized.status, "failed");
+  assert.equal(normalized.status, "infra_failed");
   assert.equal(normalized.availability, "unavailable");
 });
 
+test("normalizes schema 2 failures without turning workflow failures into advisories", () => {
+  assert.equal(
+    normalizeExecution({
+      status: "failed",
+      conclusion: "failure",
+      totals: {},
+      subjects: [{ passed: false }],
+    }).status,
+    "infra_failed",
+  );
+  assert.equal(
+    normalizeExecution({
+      status: "failed",
+      conclusion: "success",
+      totals: { technical_failures: 1, hard_gate_failures: 1 },
+      subjects: [{ passed: false }],
+    }).status,
+    "technical_failed",
+  );
+  assert.equal(
+    normalizeExecution({
+      status: "failed",
+      conclusion: "failure",
+      totals: { hard_gate_failures: 1 },
+      subjects: [{ passed: false }],
+    }).status,
+    "hard_gate_failed",
+  );
+  assert.equal(
+    normalizeExecution({
+      status: "failed",
+      conclusion: "success",
+      totals: {},
+      subjects: [{ passed: false }],
+    }).status,
+    "quality_advisory",
+  );
+});
+
+test("builds the latest health identity, completeness, and compact first failure", () => {
+  const model = latestHealthModel(
+    execution({
+      status: "infra_failed",
+      lane: "daily",
+      release: { worker: "harness", version: "1.7.3" },
+      totals: { expected_reports: 19, received_reports: 0 },
+      first_failure: {
+        kind: "job",
+        job_name: "harness e2e build",
+        step_name: "Validate E2E manifests and lockfiles",
+        message: "provider-deepseek/Cargo.lock needs to be updated",
+      },
+    }),
+  );
+
+  assert.equal(model.status, "infra_failed");
+  assert.equal(model.lane, "daily");
+  assert.equal(model.identity, "harness@1.7.3");
+  assert.equal(model.expectedReports, 19);
+  assert.equal(model.receivedReports, 0);
+  assert.equal(model.firstFailure.step_name, "Validate E2E manifests and lockfiles");
+  assert.equal(model.workflowUrl, "");
+});
+
 test("merges manifest executions and finds a retained detail", () => {
   const history = mergeExecutionHistory(
     {
@@ -168,6 +234,9 @@ test("builds subject and scenario matrix rows with result cells", () => {
   assert.equal(cell.median_score, 92);
   assert.equal(matrixCellLabel(cell, cell.status), "92%");
   assert.equal(matrixCellLabel({ passed: false }, "failed"), "×");
+  assert.equal(matrixCellLabel(null, "infra_failed"), "×");
+  assert.equal(matrixCellLabel(null, "quality_advisory"), "!");
+  assert.equal(matrixCellLabel(null, "running"), "•");
   assert.equal(matrixCellLabel(null, "incomplete"), "–");
   assert.equal(matrixCellLabel(null, "cancelled"), "○");
   assert.equal(
@@ -176,6 +245,35 @@ test("builds subject and scenario matrix rows with result cells", () => {
   );
 });
 
+test("normalizes scenario outcomes with the shared blocking precedence", () => {
+  assert.equal(normalizeScenarioStatus({ status: "missing_report" }), "incomplete");
+  assert.equal(
+    normalizeScenarioStatus({ status: "cancelled", technical_failures: 1 }),
+    "cancelled",
+  );
+  assert.equal(
+    normalizeScenarioStatus({
+      passed: false,
+      hard_gate_failures: 1,
+      technical_failures: 1,
+    }),
+    "technical_failed",
+  );
+  assert.equal(
+    normalizeScenarioStatus({ passed: false, hard_gate_failures: 1 }),
+    "hard_gate_failed",
+  );
+  assert.equal(
+    normalizeScenarioStatus({ passed: false, hard_gate_failures: 0 }),
+    "quality_advisory",
+  );
+  assert.equal(
+    normalizeScenarioStatus({ passed: true, hard_gate_failures: 1 }),
+    "hard_gate_failed",
+  );
+  assert.equal(normalizeScenarioStatus({ passed: true }), "passed");
+});
+
 test("derives aggregate legacy entries from benchmark snapshots", () => {
   const history = mergeExecutionHistory(null, {
     lastUpdate: 1,