From b2f84d23e17083e7a014ed1d128e0e65a512de42 Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Thu, 6 Aug 2026 12:44:59 -0300 Subject: [PATCH] (MOT-4368) feat(e2e): surface semantic validation health --- .github/benchmark-site/README.md | 23 +- .github/benchmark-site/execution-data.js | 99 ++- .../benchmark-site/execution-data.test.cjs | 100 ++- .github/benchmark-site/execution.html | 35 +- .github/benchmark-site/execution.js | 85 +-- .github/benchmark-site/index.html | 162 +++-- .../overview-structure.test.cjs | 51 ++ .github/benchmark-site/overview.js | 65 +- .github/benchmark-site/sample-executions.js | 344 +--------- .github/benchmark-site/styles.css | 124 +++- .../scripts/collect_harness_e2e_benchmarks.py | 32 +- .../scripts/publish_harness_e2e_dashboard.py | 586 +++++++++++++++++- .../scripts/serve_harness_e2e_dashboard.py | 10 +- .../test_collect_harness_e2e_benchmarks.py | 80 +++ .../test_publish_harness_e2e_dashboard.py | 234 +++++++ .../tests/test_serve_harness_e2e_dashboard.py | 28 +- .../test_validate_harness_e2e_manifests.py | 144 +++++ .../scripts/validate_harness_e2e_manifests.py | 248 ++++++++ .github/workflows/_harness-e2e.yml | 20 +- .github/workflows/harness-e2e-benchmark.yml | 17 + 20 files changed, 1945 insertions(+), 542 deletions(-) create mode 100644 .github/benchmark-site/overview-structure.test.cjs create mode 100644 .github/scripts/tests/test_validate_harness_e2e_manifests.py create mode 100644 .github/scripts/validate_harness_e2e_manifests.py diff --git a/.github/benchmark-site/README.md b/.github/benchmark-site/README.md index b83c178a6..1210d3193 100644 --- a/.github/benchmark-site/README.md +++ b/.github/benchmark-site/README.md @@ -3,7 +3,7 @@ This static shell replaces the generic benchmark-action index at `dev/harness-e2e/`. The workflow-generated `data.js` remains the source of truth for metric trends. `executions.js` indexes workflow attempts, and -`runs/.json` supplies the complete retained reports. +`runs/.json` supplies compact retained diagnostics. Import the default local report and serve the real dashboard from the repository root: @@ -45,22 +45,23 @@ Metric names are stable identifiers: ``` The execution index retains 100 workflow attempts. The latest 30 also retain the -complete structured `results.json` content, including prompts, transcripts, -session ids, gates, criteria, failures, retries, usage, and traces. The UI loads -those reports only on the detail page and renders transcript-heavy sections only -when expanded. Each run opens its transcript in a read-only dialog patterned -after the Harness chat, with message and error filters, paired function calls -and results, and recovered errors expanded by default. Diagnostic logs, stack -files, and credentials remain in access-controlled Actions artifacts. +allowlisted diagnostic projection: execution identity, scenario outcomes, +scores, metrics, cost, duration, retries, hard gates, and failure messages. +Prompts, transcripts, model responses, criteria, traces, and tool payloads are +never copied into Pages. They remain in access-controlled Actions artifacts +alongside diagnostic logs and stack files. +Each publish also rewrites retained schema 2 detail files through the same +allowlist and removes unreferenced run files before deploying Pages. Each full execution summary also carries compact per-scenario averages for tokens, wall time, cost, function calls, function-call errors, sessions, and turns. Tokens mean input plus output; cache-read tokens are already represented in input usage and are not added again. The execution table also exposes exact -total tokens and function calls for every retained full report. +total tokens and function calls for every retained diagnostic report. -Efficiency is the primary overview. Its cards show the current operational suite -totals, while deltas use only successful scenarios with the same subject, +Operational health is the primary overview. Efficiency appears after the latest +status, completeness, first actionable failure, KPIs, and scenario matrix. Its +cards show current suite totals, while deltas use only successful scenarios with the same subject, scenario id, and behavioral contract fingerprint. New and changed scenarios collect five comparable executions before receiving a trend verdict. Removed scenarios remain visible as historical rows and never count as an efficiency diff --git a/.github/benchmark-site/execution-data.js b/.github/benchmark-site/execution-data.js index 59af9f2c4..8c9278d52 100644 --- a/.github/benchmark-site/execution-data.js +++ b/.github/benchmark-site/execution-data.js @@ -114,13 +114,56 @@ return [...scenarios].sort(); } - function normalizeStatus(value) { - if (["passed", "failed", "incomplete", "cancelled"].includes(value)) return value; + function normalizeStatus(value, execution = {}) { + const semanticStatuses = [ + "passed", + "quality_advisory", + "hard_gate_failed", + "technical_failed", + "infra_failed", + "incomplete", + "cancelled", + "running", + ]; + if (semanticStatuses.includes(value)) return value; if (value === "pass" || value === "success") return "passed"; - if (value === "fail" || value === "failure") return "failed"; + if (value === "cancelled") return "cancelled"; + + // Schema 2 collapsed every complete non-pass into `failed`. Reconstruct + // the semantic outcome from its retained blocking counters. + if (value === "fail" || value === "failed" || value === "failure") { + const totals = execution?.totals || {}; + if (Number(totals.missing_reports || 0) > 0) return "incomplete"; + if (Number(totals.technical_failures || 0) > 0) return "technical_failed"; + if (Number(totals.hard_gate_failures || 0) > 0) return "hard_gate_failed"; + if (String(execution?.conclusion || "") !== "success") return "infra_failed"; + const subjects = Array.isArray(execution?.subjects) ? execution.subjects : []; + if (subjects.length && subjects.some((subject) => !subject?.passed)) { + return "quality_advisory"; + } + return "infra_failed"; + } return "incomplete"; } + function normalizeScenarioStatus(value) { + const scenario = value && typeof value === "object" ? value : {}; + const status = String(scenario.status || ""); + if (status === "cancelled") return "cancelled"; + if (status === "running") return "running"; + if (status === "missing_report" || status === "incomplete") return "incomplete"; + if (status === "technical_failed" || Number(scenario.technical_failures || 0) > 0) { + return "technical_failed"; + } + if (status === "hard_gate_failed" || Number(scenario.hard_gate_failures || 0) > 0) { + return "hard_gate_failed"; + } + if (status === "infra_failed") return "infra_failed"; + if (status === "quality_advisory") return "quality_advisory"; + if (scenario.passed || status === "passed" || status === "success") return "passed"; + return "quality_advisory"; + } + function normalizeExecution(entry) { const execution = entry && typeof entry === "object" ? entry : {}; const subjects = Array.isArray(execution.subjects) @@ -131,7 +174,7 @@ id: String(execution.id || ""), run_id: String(execution.run_id || ""), attempt: Number(execution.attempt) || 1, - status: normalizeStatus(execution.status), + status: normalizeStatus(execution.status, execution), conclusion: String(execution.conclusion || ""), event: String(execution.event || ""), actor: String(execution.actor || ""), @@ -166,7 +209,7 @@ const scenarios = listLegacyScenarios(subject).map((scenarioId) => { const score = subject.metrics?.quality?.[scenarioId]?.median_score; const passRate = subject.metrics?.quality?.[scenarioId]?.pass_rate; - return { + const scenario = { id: scenarioId, status: score?.status || passRate?.status || "unknown", passed: score?.passed ?? passRate?.passed ?? false, @@ -185,6 +228,7 @@ wall_time_seconds: metricValue(subject, "efficiency", scenarioId, "wall_time_seconds"), }; + return { ...scenario, status: normalizeScenarioStatus(scenario) }; }); return { id: subject.id, @@ -194,7 +238,7 @@ engine_revision: subject.engineRevision || "", passed: Boolean(subject.passed), expected_reports: scenarios.length, - received_reports: scenarios.filter((scenario) => scenario.status !== "missing_report") + received_reports: scenarios.filter((scenario) => scenario.status !== "incomplete") .length, scenario_pass_rate: (metricValue(subject, "quality", "suite", "scenario_pass_rate") ?? 0) / 100, @@ -340,6 +384,32 @@ }); } + function latestHealthModel(entry) { + const execution = normalizeExecution(entry); + const release = execution.release || {}; + const releaseIdentity = [release.worker, release.version] + .filter(Boolean) + .join("@"); + const firstFailure = + execution.first_failure && typeof execution.first_failure === "object" + ? execution.first_failure + : null; + return { + status: execution.status, + lane: String(execution.lane || "daily"), + identity: + releaseIdentity || + String(release.tag || "") || + String(execution.source?.sha || "").slice(0, 12) || + "Unknown", + expectedReports: Number(execution.totals?.expected_reports || 0), + receivedReports: Number(execution.totals?.received_reports || 0), + availability: execution.availability, + firstFailure, + workflowUrl: execution.workflow_url, + }; + } + function executionsWithinDays(executions, days, now = Date.now()) { const windowDays = Number(days); if (!Number.isFinite(windowDays) || windowDays <= 0) return [...(executions || [])]; @@ -384,18 +454,17 @@ const subject = execution?.subjects?.find((item) => item.id === row.subjectId); const scenario = subject?.scenarios?.find((item) => item.id === row.scenarioId); if (!scenario) return null; - const status = - scenario.status === "missing_report" - ? "incomplete" - : scenario.passed - ? "passed" - : "failed"; + const status = normalizeScenarioStatus(scenario); return { ...scenario, status }; } function matrixCellLabel(cell, status) { - if (status === "failed") return "×"; + if (["failed", "hard_gate_failed", "technical_failed", "infra_failed"].includes(status)) { + return "×"; + } + if (status === "quality_advisory") return "!"; if (status === "cancelled") return "○"; + if (status === "running") return "•"; if (status !== "passed") return "–"; const score = numberOrNull(cell?.median_score); @@ -527,7 +596,7 @@ const technicalFailures = Number(scenario.technical_failures) || 0; return { passed: Boolean(scenario.passed), - complete: scenario.status !== "missing_report", + complete: normalizeScenarioStatus(scenario) !== "incomplete", hardGateFailures, technicalFailures, score: numberOrNull(scenario.median_score), @@ -936,12 +1005,14 @@ filterExecutions, findExecution, groupRunFailures, + latestHealthModel, legacyExecution, matrixCell, matrixCellLabel, matrixRows, mergeExecutionHistory, normalizeExecution, + normalizeScenarioStatus, normalizeStatus, scenarioMetricSeries, scenarioMetricRows, diff --git a/.github/benchmark-site/execution-data.test.cjs b/.github/benchmark-site/execution-data.test.cjs index c249c0b6c..30a649835 100644 --- a/.github/benchmark-site/execution-data.test.cjs +++ b/.github/benchmark-site/execution-data.test.cjs @@ -10,11 +10,13 @@ const { filterExecutions, findExecution, groupRunFailures, + latestHealthModel, matrixCell, matrixCellLabel, matrixRows, mergeExecutionHistory, normalizeExecution, + normalizeScenarioStatus, scenarioMetricSeries, scenarioMetricRows, scenarioMetricsFromDetail, @@ -62,10 +64,74 @@ test("normalizes execution status and availability", () => { }); assert.equal(normalized.id, "99"); - assert.equal(normalized.status, "failed"); + assert.equal(normalized.status, "infra_failed"); assert.equal(normalized.availability, "unavailable"); }); +test("normalizes schema 2 failures without turning workflow failures into advisories", () => { + assert.equal( + normalizeExecution({ + status: "failed", + conclusion: "failure", + totals: {}, + subjects: [{ passed: false }], + }).status, + "infra_failed", + ); + assert.equal( + normalizeExecution({ + status: "failed", + conclusion: "success", + totals: { technical_failures: 1, hard_gate_failures: 1 }, + subjects: [{ passed: false }], + }).status, + "technical_failed", + ); + assert.equal( + normalizeExecution({ + status: "failed", + conclusion: "failure", + totals: { hard_gate_failures: 1 }, + subjects: [{ passed: false }], + }).status, + "hard_gate_failed", + ); + assert.equal( + normalizeExecution({ + status: "failed", + conclusion: "success", + totals: {}, + subjects: [{ passed: false }], + }).status, + "quality_advisory", + ); +}); + +test("builds the latest health identity, completeness, and compact first failure", () => { + const model = latestHealthModel( + execution({ + status: "infra_failed", + lane: "daily", + release: { worker: "harness", version: "1.7.3" }, + totals: { expected_reports: 19, received_reports: 0 }, + first_failure: { + kind: "job", + job_name: "harness e2e build", + step_name: "Validate E2E manifests and lockfiles", + message: "provider-deepseek/Cargo.lock needs to be updated", + }, + }), + ); + + assert.equal(model.status, "infra_failed"); + assert.equal(model.lane, "daily"); + assert.equal(model.identity, "harness@1.7.3"); + assert.equal(model.expectedReports, 19); + assert.equal(model.receivedReports, 0); + assert.equal(model.firstFailure.step_name, "Validate E2E manifests and lockfiles"); + assert.equal(model.workflowUrl, ""); +}); + test("merges manifest executions and finds a retained detail", () => { const history = mergeExecutionHistory( { @@ -168,6 +234,9 @@ test("builds subject and scenario matrix rows with result cells", () => { assert.equal(cell.median_score, 92); assert.equal(matrixCellLabel(cell, cell.status), "92%"); assert.equal(matrixCellLabel({ passed: false }, "failed"), "×"); + assert.equal(matrixCellLabel(null, "infra_failed"), "×"); + assert.equal(matrixCellLabel(null, "quality_advisory"), "!"); + assert.equal(matrixCellLabel(null, "running"), "•"); assert.equal(matrixCellLabel(null, "incomplete"), "–"); assert.equal(matrixCellLabel(null, "cancelled"), "○"); assert.equal( @@ -176,6 +245,35 @@ test("builds subject and scenario matrix rows with result cells", () => { ); }); +test("normalizes scenario outcomes with the shared blocking precedence", () => { + assert.equal(normalizeScenarioStatus({ status: "missing_report" }), "incomplete"); + assert.equal( + normalizeScenarioStatus({ status: "cancelled", technical_failures: 1 }), + "cancelled", + ); + assert.equal( + normalizeScenarioStatus({ + passed: false, + hard_gate_failures: 1, + technical_failures: 1, + }), + "technical_failed", + ); + assert.equal( + normalizeScenarioStatus({ passed: false, hard_gate_failures: 1 }), + "hard_gate_failed", + ); + assert.equal( + normalizeScenarioStatus({ passed: false, hard_gate_failures: 0 }), + "quality_advisory", + ); + assert.equal( + normalizeScenarioStatus({ passed: true, hard_gate_failures: 1 }), + "hard_gate_failed", + ); + assert.equal(normalizeScenarioStatus({ passed: true }), "passed"); +}); + test("derives aggregate legacy entries from benchmark snapshots", () => { const history = mergeExecutionHistory(null, { lastUpdate: 1, diff --git a/.github/benchmark-site/execution.html b/.github/benchmark-site/execution.html index d4d948cba..dc837da0b 100644 --- a/.github/benchmark-site/execution.html +++ b/.github/benchmark-site/execution.html @@ -5,7 +5,7 @@ Harness E2E execution detail @@ -130,18 +130,18 @@

Configuration

-
Complete result set
+
Compact result set

Scenarios and runs

-
Source record
-

Raw data

+
Public diagnostic record
+

Diagnostic JSON

- The structured views above preserve the source report. Use the raw record for fields - that do not yet have a dedicated visualization. + This allowlisted record contains metrics, gates, and failure messages. Prompts, + transcripts, model responses, and tool payloads are not published.

@@ -153,28 +153,6 @@

Raw data

- -
-
-
-
Session transcript
-

Execution conversation

-

-
- -
-
-
-