diff --git a/.github/workflows/monitor-freshness.yml b/.github/workflows/monitor-freshness.yml new file mode 100644 index 0000000..6863d28 --- /dev/null +++ b/.github/workflows/monitor-freshness.yml @@ -0,0 +1,61 @@ +name: monitor-freshness + +# THE WATCHER FOR THE WATCHERS — the answer to #258. +# +# `rekor-monitor.yml` last completed on 2026-08-21 and was cancelled on every +# run for the thirteen days after. Nobody was told. The reusable workflow's +# notification job fires on `failure`, and a CANCELLED run is not a FAILED one, +# so the Actions tab stayed green while the Sigstore identity monitoring — the +# control that workflow's own header credits with catching the Shai-Hulud worm — +# was doing nothing at all. +# +# The lesson generalises past that one workflow: **the status of the last run +# tells you nothing about whether a monitor is working.** A monitor can be +# cancelled, skipped, disabled, or have its schedule quietly dropped by GitHub, +# and in every case the newest row still looks recent. The one signal that +# survives all of those is the AGE OF THE LAST SUCCESS, which is what this +# checks. +# +# It is deliberately not part of `npm run check`: that suite is hermetic, and +# this needs the network. A monitor is a schedule, not a build gate — the same +# reasoning the `link-check.yml` header records. +# +# Its own test suite runs in `npm run check`, stubbed, so the decision logic is +# gated hermetically even though the check itself cannot be. + +on: + schedule: + # Every 6h, off the mark — matched to the tightest threshold the script + # declares (rekor-monitor at 6h), so an alert is never more than one period + # behind the condition it reports. + - cron: "17 */6 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "Print the freshness table and file nothing" + type: boolean + default: false + +permissions: + contents: read + +jobs: + freshness: + runs-on: ubuntu-latest + permissions: + contents: read + issues: write # file one issue per stuck monitor + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + # No `npm ci`: the script is dependency-free on purpose, so the watcher + # cannot be taken down by the thing it would then fail to report. + - name: Every monitor has completed within its threshold + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + node scripts/check-monitor-freshness.mjs \ + ${{ inputs.dry_run && '--dry-run' || '' }} diff --git a/package.json b/package.json index 4e29859..75362f9 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "node scripts/run-pipeline.mjs hermetic stamped local", "conformance": "node scripts/gen-conformance.mjs", - "check": "node scripts/verify-vendor.mjs && node scripts/gen-seams.mjs --check && node scripts/gen-blog.mjs --check && node scripts/gen-strings.mjs --check && node scripts/check-emphasis.mjs && node scripts/check-outline.mjs && node scripts/check-inline-purity.mjs && node scripts/check-copy-coverage.mjs && node scripts/check-jargon.mjs && node scripts/gen-claims.mjs --check && node scripts/claims-registry.mjs --check && node scripts/check-evidence-edges.mjs && node scripts/check-evidence-pinned.mjs && node scripts/check-overclaim.mjs && node scripts/check-ledger.mjs && node scripts/gen-ledger.mjs --check && node scripts/gen-map.mjs --check && node scripts/emit-catalog.mjs --check && node scripts/check-reader.mjs && node scripts/check-seo.mjs && node scripts/check-node-uniqueness.mjs && node scripts/check-license.mjs && node scripts/check-repetition.mjs && node --test scripts/legibility/coldread.test.mjs", + "check": "node scripts/verify-vendor.mjs && node scripts/gen-seams.mjs --check && node scripts/gen-blog.mjs --check && node scripts/gen-strings.mjs --check && node scripts/check-emphasis.mjs && node scripts/check-outline.mjs && node scripts/check-inline-purity.mjs && node scripts/check-copy-coverage.mjs && node scripts/check-jargon.mjs && node scripts/gen-claims.mjs --check && node scripts/claims-registry.mjs --check && node scripts/check-evidence-edges.mjs && node scripts/check-evidence-pinned.mjs && node scripts/check-overclaim.mjs && node scripts/check-ledger.mjs && node scripts/gen-ledger.mjs --check && node scripts/gen-map.mjs --check && node scripts/emit-catalog.mjs --check && node scripts/check-reader.mjs && node scripts/check-seo.mjs && node scripts/check-node-uniqueness.mjs && node scripts/check-license.mjs && node scripts/check-repetition.mjs && node --test scripts/legibility/coldread.test.mjs scripts/check-monitor-freshness.test.mjs", "strings": "node scripts/gen-strings.mjs", "claims-registry": "node scripts/claims-registry.mjs --check", "seams": "node scripts/gen-seams.mjs", diff --git a/scripts/check-monitor-freshness.mjs b/scripts/check-monitor-freshness.mjs new file mode 100644 index 0000000..03404cc --- /dev/null +++ b/scripts/check-monitor-freshness.mjs @@ -0,0 +1,256 @@ +#!/usr/bin/env node +// check-monitor-freshness — the watcher for the watchers. +// +// A scheduled MONITOR is only doing its job while it COMPLETES. A monitor that +// starts hourly and never finishes looks identical, on the Actions tab, to one +// that is fine — the runs are there, the schedule is firing, and the newest row +// is minutes old. What it is not doing is finishing, and nothing says so. +// +// That is not hypothetical here. `rekor-monitor.yml` last completed on +// 2026-08-21 and was then cancelled on every subsequent run for thirteen days +// (#258). Nobody was told, because the reusable workflow's notification job +// fires on `failure` and a cancelled run is not a failed one. The Sigstore +// identity monitoring the workflow header calls "the control that actually +// caught the Shai-Hulud worm" was down for two weeks behind a green-looking +// tab. +// +// So this checks the one thing the run status cannot: HOW OLD IS THE NEWEST +// SUCCESSFUL RUN. Age of last success is the only signal that survives every +// way a monitor can stop working — cancelled, skipped, disabled, a schedule +// silently dropped by GitHub, a repo gone quiet. It is deliberately NOT a check +// on the monitor's findings; it asks whether the instrument ran, not what it +// saw. +// +// It files ONE issue per stuck monitor and then leaves it alone. A watcher that +// opens a fresh issue every day is its own kind of silence. +// +// node scripts/check-monitor-freshness.mjs # check + file +// node scripts/check-monitor-freshness.mjs --dry-run # print the table only +// +// Exits non-zero when any monitor is stale, so the run is red as well as filed. + +const API = "https://api.github.com"; + +// The monitors this repo relies on, and how long a gap is tolerable for each. +// A threshold is a multiple of the monitor's own period, not a guess: it must +// absorb one dropped slot without crying wolf, and catch a genuine stop well +// before it becomes routine. GitHub drops scheduled runs under load, so 1x a +// period would be noise. +export const MONITORS = [ + { + // hourly (`41 * * * *`); 6h tolerates five dropped or overrunning slots. + workflow: "rekor-monitor.yml", + maxAgeHours: 6, + what: "Sigstore identity monitoring — watches the public Rekor log for certs minted for this repo's Actions identities", + }, + { + // weekly (`23 6 * * 1`); 10d tolerates one dropped slot plus a margin. + workflow: "link-check.yml", + maxAgeHours: 24 * 10, + what: "external-link liveness", + }, +]; + +export const ISSUE_LABEL = "monitor-stale"; + +// --- pure helpers (unit-tested) ---------------------------------------------- + +// The whole decision, as a function of two timestamps and a threshold, so the +// hard part is testable without a network. +export function assess(newestSuccessIso, nowMs, maxAgeHours) { + if (!newestSuccessIso) { + return { stale: true, ageHours: null, reason: "no successful run on record" }; + } + const t = Date.parse(newestSuccessIso); + if (!Number.isFinite(t)) { + return { stale: true, ageHours: null, reason: `unparseable run timestamp: ${newestSuccessIso}` }; + } + const ageHours = (nowMs - t) / 3_600_000; + return { + stale: ageHours > maxAgeHours, + ageHours, + reason: ageHours > maxAgeHours + ? `last success ${fmtAge(ageHours)} ago, threshold ${fmtAge(maxAgeHours)}` + : `last success ${fmtAge(ageHours)} ago`, + }; +} + +export function fmtAge(hours) { + if (hours == null) return "never"; + if (hours < 1) return `${Math.round(hours * 60)}m`; + if (hours < 48) return `${Math.round(hours)}h`; + return `${(hours / 24).toFixed(1)}d`; +} + +// Stable per-monitor title, so a stuck monitor maps to exactly one open issue. +export function issueTitle(workflow) { + return `[monitor-stale]: ${workflow} has not completed successfully`; +} + +export function issueBody({ workflow, what, assessment, maxAgeHours, repo, runUrl }) { + return [ + `\`${workflow}\` has not recorded a successful run.`, + "", + `| | |`, + `|---|---|`, + `| Monitor | \`${workflow}\` |`, + `| Watches | ${what} |`, + `| Last success | ${assessment.ageHours == null ? "none on record" : `${fmtAge(assessment.ageHours)} ago`} |`, + `| Threshold | ${fmtAge(maxAgeHours)} |`, + "", + "**A monitor that does not complete is not monitoring.** Its runs may still be", + "appearing on the Actions tab — cancelled, skipped, or timing out — which is why", + "this check reads the age of the last SUCCESS rather than the status of the last run.", + "", + `Runs: https://github.com/${repo}/actions/workflows/${workflow}`, + "", + "This issue is filed once and then left alone; it will not be reopened or", + "duplicated while it stays open. Close it once the monitor completes again —", + "the next check will re-file if it stops a second time.", + "", + runUrl ? `Filed by [\`check-monitor-freshness\`](${runUrl}).` : "Filed by `check-monitor-freshness`.", + ].join("\n"); +} + +// --- GitHub API -------------------------------------------------------------- + +async function gh(path, { token, method = "GET", body, fetchImpl = fetch } = {}) { + const res = await fetchImpl(`${API}${path}`, { + method, + headers: { + accept: "application/vnd.github+json", + "x-github-api-version": "2022-11-28", + "user-agent": "bounded-systems-site/check-monitor-freshness", + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(body ? { "content-type": "application/json" } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + if (!res.ok) { + const err = new Error(`GitHub ${method} ${path} → ${res.status} ${res.statusText}: ${(await res.text()).slice(0, 300)}`); + err.status = res.status; + throw err; + } + return res.status === 204 ? null : res.json(); +} + +// Newest run that actually SUCCEEDED, or null. `status=success` filters +// server-side, so this is one page of one, not a scan. +async function newestSuccess(repo, workflow, token, fetchImpl) { + const q = new URLSearchParams({ status: "success", per_page: "1" }); + const data = await gh(`/repos/${repo}/actions/workflows/${workflow}/runs?${q}`, { token, fetchImpl }); + const run = data.workflow_runs?.[0]; + return run ? { at: run.updated_at || run.created_at, url: run.html_url } : null; +} + +async function openStaleIssue(repo, workflow, token, fetchImpl) { + const data = await gh( + `/repos/${repo}/issues?${new URLSearchParams({ state: "open", labels: ISSUE_LABEL, per_page: "100" })}`, + { token, fetchImpl }, + ); + return data.find((i) => i.title === issueTitle(workflow)) ?? null; +} + +// --- run --------------------------------------------------------------------- + +// The clock and the network are parameters, so the whole decision path is +// exercisable offline. Returns counts rather than exiting, so a test can assert +// on them and main() owns the exit code. +export async function run({ + repo, + token, + dryRun = false, + now = Date.now(), + runUrl = null, + fetchImpl = fetch, + log = console.log, + logErr = console.error, +} = {}) { + if (!repo) throw new Error("GITHUB_REPOSITORY is required (owner/name)"); + if (!token && !dryRun) throw new Error("GITHUB_TOKEN is required unless --dry-run"); + + let staleCount = 0; + let brokenCount = 0; + let filed = 0; + + for (const m of MONITORS) { + let last; + try { + last = await newestSuccess(repo, m.workflow, token, fetchImpl); + } catch (e) { + if (e.status === 404) { + // The workflow is genuinely not in this repo. Benign: skip it. + log(`skip ${m.workflow.padEnd(22)} not present in ${repo}`); + continue; + } + // Anything else — 401, 403, a 5xx, a network fault — means THIS CHECK + // failed, which is not the same as the monitor being healthy. Reporting + // "all fresh" here would reproduce, inside the watcher, the exact bug it + // exists to catch: absence of a signal read as the presence of health. + logErr(`BROKEN ${m.workflow.padEnd(22)} could not be checked: ${e.message}`); + brokenCount++; + continue; + } + + const a = assess(last?.at ?? null, now, m.maxAgeHours); + log(`${a.stale ? "STALE " : "fresh "} ${m.workflow.padEnd(22)} ${a.reason}`); + if (!a.stale) continue; + staleCount++; + + if (dryRun) { + log(` (dry run — would ensure an issue titled "${issueTitle(m.workflow)}")`); + continue; + } + + const existing = await openStaleIssue(repo, m.workflow, token, fetchImpl); + if (existing) { + log(` already filed: #${existing.number}`); + continue; + } + const created = await gh(`/repos/${repo}/issues`, { + token, + fetchImpl, + method: "POST", + body: { + title: issueTitle(m.workflow), + body: issueBody({ workflow: m.workflow, what: m.what, assessment: a, maxAgeHours: m.maxAgeHours, repo, runUrl }), + labels: [ISSUE_LABEL, "bug"], + }, + }); + filed++; + log(` filed: #${created.number}`); + } + + if (brokenCount) { + logErr(`\n${brokenCount} monitor(s) could not be checked — this run proves nothing about them.`); + } + if (staleCount) { + logErr(`${staleCount} monitor(s) stale.`); + } + if (!staleCount && !brokenCount) { + log("\nAll monitors have completed within their thresholds."); + } + return { staleCount, brokenCount, filed }; +} + +// --- main -------------------------------------------------------------------- + +async function main() { + const repo = process.env.GITHUB_REPOSITORY; + const { staleCount, brokenCount } = await run({ + repo, + token: process.env.GITHUB_TOKEN, + dryRun: process.argv.includes("--dry-run"), + runUrl: process.env.GITHUB_RUN_ID && repo + ? `https://github.com/${repo}/actions/runs/${process.env.GITHUB_RUN_ID}` + : null, + }); + if (staleCount || brokenCount) process.exitCode = 1; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((e) => { + console.error(e.message); + process.exitCode = 1; + }); +} diff --git a/scripts/check-monitor-freshness.test.mjs b/scripts/check-monitor-freshness.test.mjs new file mode 100644 index 0000000..2105ce7 --- /dev/null +++ b/scripts/check-monitor-freshness.test.mjs @@ -0,0 +1,222 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { assess, fmtAge, issueTitle, issueBody, MONITORS } from "./check-monitor-freshness.mjs"; + +const NOW = Date.parse("2026-09-03T12:00:00Z"); +const hoursAgo = (h) => new Date(NOW - h * 3_600_000).toISOString(); + +test("a recent success is fresh", () => { + const a = assess(hoursAgo(2), NOW, 6); + assert.equal(a.stale, false); + assert.ok(Math.abs(a.ageHours - 2) < 1e-6); +}); + +test("a success older than the threshold is stale", () => { + const a = assess(hoursAgo(7), NOW, 6); + assert.equal(a.stale, true); + assert.match(a.reason, /threshold/); +}); + +test("the boundary is not stale — a threshold is a ceiling, not a trigger", () => { + assert.equal(assess(hoursAgo(6), NOW, 6).stale, false); +}); + +test("no successful run on record is stale, not fresh", () => { + // The failure that motivated this: absence must never read as health. + const a = assess(null, NOW, 6); + assert.equal(a.stale, true); + assert.equal(a.ageHours, null); + assert.match(a.reason, /no successful run/); +}); + +test("an unparseable timestamp is stale, not fresh", () => { + assert.equal(assess("not-a-date", NOW, 6).stale, true); +}); + +test("the real #258 gap trips the rekor-monitor threshold", () => { + // rekor-monitor last completed 2026-08-21; checked 2026-09-03. + const m = MONITORS.find((x) => x.workflow === "rekor-monitor.yml"); + const a = assess("2026-08-21T11:56:20Z", NOW, m.maxAgeHours); + assert.equal(a.stale, true); + assert.ok(a.ageHours > 24 * 12, `expected >12d, got ${a.ageHours}h`); +}); + +test("a weekly monitor is not stale one day after it ran", () => { + const m = MONITORS.find((x) => x.workflow === "link-check.yml"); + assert.equal(assess(hoursAgo(24), NOW, m.maxAgeHours).stale, false); +}); + +test("every monitor threshold comfortably exceeds one period", () => { + // Guards the wolf-crying failure: a threshold at 1x the period turns a single + // dropped GitHub slot into an issue. + for (const m of MONITORS) { + assert.ok(m.maxAgeHours > 1, `${m.workflow}: threshold must exceed an hour`); + assert.ok(m.what && m.what.length > 10, `${m.workflow}: needs a plain description`); + } +}); + +test("fmtAge reads plainly at each scale", () => { + assert.equal(fmtAge(null), "never"); + assert.equal(fmtAge(0.5), "30m"); + assert.equal(fmtAge(6), "6h"); + assert.equal(fmtAge(24 * 13), "13.0d"); +}); + +test("the issue title is stable per monitor, so one stuck monitor means one issue", () => { + assert.equal(issueTitle("rekor-monitor.yml"), issueTitle("rekor-monitor.yml")); + assert.notEqual(issueTitle("rekor-monitor.yml"), issueTitle("link-check.yml")); + assert.match(issueTitle("rekor-monitor.yml"), /rekor-monitor\.yml/); +}); + +test("the issue body names the monitor, the age, and where to look", () => { + const body = issueBody({ + workflow: "rekor-monitor.yml", + what: "Sigstore identity monitoring", + assessment: assess("2026-08-21T11:56:20Z", NOW, 6), + maxAgeHours: 6, + repo: "bounded-systems/site", + runUrl: "https://example.invalid/run", + }); + assert.match(body, /rekor-monitor\.yml/); + assert.match(body, /13\.0d ago/); + assert.match(body, /actions\/workflows\/rekor-monitor\.yml/); + assert.match(body, /does not complete is not monitoring/); +}); + +test("a body with no run URL still renders", () => { + const body = issueBody({ + workflow: "link-check.yml", + what: "external-link liveness", + assessment: assess(null, NOW, 240), + maxAgeHours: 240, + repo: "bounded-systems/site", + runUrl: null, + }); + assert.match(body, /none on record/); + assert.doesNotMatch(body, /undefined|null/); +}); + +// --- run(), with the network stubbed ---------------------------------------- +// The clock and fetch are injected, so these exercise the real decision path +// offline. No live API call: a monitor's own tests must not be hostage to the +// service it monitors. + +import { run } from "./check-monitor-freshness.mjs"; + +const quiet = { log: () => {}, logErr: () => {} }; + +// Builds a fake GitHub. `runs` maps workflow file -> ISO of newest success (or +// null for "no successful run"); `issues` is the list an issue query returns. +function fakeGitHub({ runs = {}, issues = [], status = {} } = {}) { + const created = []; + const fetchImpl = async (url, opts = {}) => { + const path = url.replace("https://api.github.com", ""); + const wf = path.match(/workflows\/([^/]+)\/runs/)?.[1]; + if (wf) { + if (status[wf]) return { ok: false, status: status[wf], statusText: "x", text: async () => "" }; + const at = runs[wf]; + return { + ok: true, status: 200, + json: async () => ({ workflow_runs: at ? [{ updated_at: at, html_url: "u" }] : [] }), + }; + } + if (path.startsWith("/repos/") && path.includes("/issues?")) { + return { ok: true, status: 200, json: async () => issues }; + } + if (opts.method === "POST" && path.endsWith("/issues")) { + const body = JSON.parse(opts.body); + created.push(body); + return { ok: true, status: 201, json: async () => ({ number: 900 + created.length }) }; + } + throw new Error(`unexpected request: ${opts.method || "GET"} ${path}`); + }; + return { fetchImpl, created }; +} + +const FRESH = { + "rekor-monitor.yml": hoursAgo(1), + "link-check.yml": hoursAgo(24), +}; + +test("all monitors fresh: nothing stale, nothing filed", async () => { + const { fetchImpl, created } = fakeGitHub({ runs: FRESH }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.deepEqual(r, { staleCount: 0, brokenCount: 0, filed: 0 }); + assert.equal(created.length, 0); +}); + +test("a stale monitor files exactly one issue, labelled and titled for it", async () => { + const { fetchImpl, created } = fakeGitHub({ + runs: { ...FRESH, "rekor-monitor.yml": "2026-08-21T11:56:20Z" }, + }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.equal(r.staleCount, 1); + assert.equal(r.filed, 1); + assert.equal(created.length, 1); + assert.equal(created[0].title, issueTitle("rekor-monitor.yml")); + assert.ok(created[0].labels.includes("monitor-stale")); + assert.match(created[0].body, /13\.0d ago/); +}); + +test("it does not duplicate: an already-open issue means file nothing", async () => { + const { fetchImpl, created } = fakeGitHub({ + runs: { ...FRESH, "rekor-monitor.yml": "2026-08-21T11:56:20Z" }, + issues: [{ number: 258, title: issueTitle("rekor-monitor.yml") }], + }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.equal(r.staleCount, 1); + assert.equal(r.filed, 0, "must not re-file over an open issue"); + assert.equal(created.length, 0); +}); + +test("--dry-run detects staleness and writes nothing", async () => { + const { fetchImpl, created } = fakeGitHub({ + runs: { ...FRESH, "rekor-monitor.yml": "2026-08-21T11:56:20Z" }, + }); + const r = await run({ repo: "o/r", dryRun: true, now: NOW, fetchImpl, ...quiet }); + assert.equal(r.staleCount, 1); + assert.equal(r.filed, 0); + assert.equal(created.length, 0); +}); + +test("REGRESSION: an API failure never reports health", async () => { + // The bug this script exists to prevent, once reproduced inside the script + // itself: a 401 was swallowed and the run printed "all monitors fresh" and + // exited 0. Absence of a signal must never read as the presence of health. + for (const code of [401, 403, 500]) { + const { fetchImpl, created } = fakeGitHub({ + runs: FRESH, + status: { "rekor-monitor.yml": code }, + }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.equal(r.brokenCount, 1, `${code} must count as unchecked`); + assert.equal(r.staleCount, 0, `${code} is not staleness`); + assert.equal(created.length, 0, `${code} must not file an issue`); + } +}); + +test("a 404 is a genuinely absent workflow, not an alarm", async () => { + const { fetchImpl, created } = fakeGitHub({ + runs: FRESH, + status: { "rekor-monitor.yml": 404 }, + }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.deepEqual(r, { staleCount: 0, brokenCount: 0, filed: 0 }); + assert.equal(created.length, 0); +}); + +test("a monitor with no successful run at all is stale", async () => { + const { fetchImpl, created } = fakeGitHub({ runs: { ...FRESH, "link-check.yml": null } }); + const r = await run({ repo: "o/r", token: "t", now: NOW, fetchImpl, ...quiet }); + assert.equal(r.staleCount, 1); + assert.match(created[0].body, /none on record/); +}); + +test("run() refuses to write without a token", async () => { + const { fetchImpl } = fakeGitHub({ runs: FRESH }); + await assert.rejects(() => run({ repo: "o/r", now: NOW, fetchImpl, ...quiet }), /GITHUB_TOKEN/); +}); + +test("run() requires a repo", async () => { + await assert.rejects(() => run({ token: "t", ...quiet }), /GITHUB_REPOSITORY/); +});