Freshness: know when a refresh mechanism has stopped

ops/freshness-monitor.md

@@ -0,0 +1,82 @@
+# Dataset freshness monitor
+
+Answers the one question an unattended pipeline cannot answer about itself:
+**has a refresh mechanism stopped running?**
+
+A flow on the Jetson dies quietly. The box reboots, n8n is switched off, a PAT
+expires. Nothing errors, because nothing is watching. The only symptom is a date
+that stops moving, which is how `interconnection-queue.json` reached 77 days
+without anyone noticing.
+
+## What it is
+
+- `server/freshness-registry.ts` declares every dataset: where it lives, how to
+  read its timestamp, how old it is allowed to get, and what is supposed to
+  refresh it. Adding a dataset here is the whole integration.
+- `server/freshness.ts` is the pure classifier. No fs, no clock, no env.
+- `GET /api/admin/freshness` returns the full report.
+- `GET /api/admin/freshness/check` is the deadman: 200 when nothing is stale,
+  503 listing offenders when something is.
+
+Both routes are admin-gated. Freshness is the floor, not a feature; a public
+"look how current we are" page advertises the bare minimum.
+
+## Statuses
+
+| status | meaning | trips the alarm |
+|---|---|---|
+| `ok` | within its declared cadence | no |
+| `aging` | overdue but under 2x cadence, ie. one missed run | no |
+| `stale` | past 2x cadence, the mechanism has probably stopped | **yes** |
+| `manual` | hand-curated, no cadence declared | no |
+| `unknown` | no readable timestamp in the file | no |
+
+`aging` is deliberately a 200. If a single missed run pages you, the alert stops
+meaning anything within a month.
+
+`unknown` deliberately does not alarm either: it marks a gap in instrumentation,
+not evidence of failure. `datacenters.json` is the current example, a bare array
+with no date field anywhere, so its age cannot be checked at all.
+
+## Wiring the deadman
+
+**The watchdog must not run on the Jetson.** A watchdog hosted on the box it
+watches dies with it, which is the exact failure this exists to catch. Use the
+same cron-job.org account already firing the daily tweet.
+
+1. New cron job, URL `https://gridtilt.com/api/admin/freshness/check`.
+2. Add request header `x-admin-key` with the production `ADMIN_API_KEY`.
+3. Schedule daily. Hourly is pointless: every cadence here is measured in days.
+4. Enable the job's failure notification. cron-job.org alerts on any non-2xx,
+   which is exactly the 503.
+5. Save, run once manually, and confirm you get the failure mail while datasets
+   are still stale. **Verify the alarm fires before trusting the silence.**
+
+Nothing else to configure. The response body lists each stale dataset with its
+mechanism, so the alert tells you which flow to go restart.
+
+## Why the dates can disagree with the site
+
+Two different questions, deliberately two different fields:
+
+- `lastRefreshed` is when a value last **changed**.
+- `lastChecked` is when something last **looked**, changed or not.
+
+Staleness follows `lastChecked` where it exists, falling back to `lastRefreshed`.
+Otherwise a dataset that is checked daily and correctly unchanged would read as
+abandoned, and you could not tell that apart from the scanner never running.
+
+So once the scanner is scheduled, the monitor can be green while the homepage
+still shows an older "as of" date on that stat. Both are honest and they answer
+different questions. Do not "fix" this by pointing them at the same field.
+
+`POST /api/admin/scan-news-now` stamps `lastChecked` on every run. The n8n
+`cluster-refresh` flow achieves the same thing differently: it commits even on a
+zero-change pass so only `lastRefreshed` moves.
+
+## Current state
+
+As of the last check, three datasets are stale purely because their flows were
+built but never mounted on the Jetson: `clusters`, `interconnection-queue`, and
+`gpu-rental-prices`. See `ops/n8n/README.md` for mounting instructions. The
+monitor does not fix staleness, it makes staleness loud.

ops/n8n/README.md

@@ -3,6 +3,11 @@
 Importable n8n workflows that keep GridTilt data fresh from the Jetson homelab.
 These run on the self-hosted n8n instance, not inside the app.
 
+> **Nothing here runs until you mount it.** If a flow is never imported, or the
+> Jetson is off, the data silently stops moving. `ops/freshness-monitor.md` is
+> the watchdog that makes that failure loud; it deliberately runs off-Jetson,
+> because a watchdog on the box it watches dies with it.
+
 ## gpu-price-refresh.json
 
 Two independent flows in one workflow file:

server/__tests__/auth-boundary.test.ts

@@ -50,6 +50,8 @@ async function startTestServer(): Promise<{ url: string; close: () => Promise<vo
 const PROTECTED_ROUTES: Array<[string, string]> = [
   ["GET", "/api/admin/subscribers"],
   ["GET", "/api/admin/gpu-history"],
+  ["GET", "/api/admin/freshness"],
+  ["GET", "/api/admin/freshness/check"],
   ["GET", "/api/newsletter/preview"], // SEC-1: was public, leaked subscriber count
   ["POST", "/api/social/generate"], // SEC-2: was public, burned Yahoo quota
   ["DELETE", "/api/admin/subscribers/x@y.com"],

server/__tests__/freshness.test.ts

@@ -0,0 +1,188 @@
+// The freshness monitor exists to catch a stopped pipeline. These tests pin the
+// behaviours that make it trustworthy: it never reports data as fresher than it
+// can prove, it never alarms on data nobody promised to refresh, and a dataset
+// that cannot report its age says so instead of guessing.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import {
+  computeFreshness,
+  parseStamp,
+  readStamp,
+  type FileContents,
+} from "../freshness.js";
+import { DATASET_REGISTRY, type DatasetSpec } from "../freshness-registry.js";
+
+const NOW = Date.parse("2026-08-05T12:00:00Z");
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+const spec = (over: Partial<DatasetSpec> = {}): DatasetSpec => ({
+  id: "t",
+  label: "Test",
+  file: "t.json",
+  read: { kind: "envelope", fields: ["lastRefreshed"] },
+  expectedMaxAgeHours: 48,
+  mechanism: "test mechanism",
+  ...over,
+});
+
+const dayBefore = (days: number) => new Date(NOW - days * DAY_MS).toISOString().slice(0, 10);
+
+// ─── parseStamp ────────────────────────────────────────────────────────────
+
+test("a bare date parses as the start of that UTC day, never the end", () => {
+  // Conservative on purpose: a file stamped today must not read as 0h old.
+  assert.equal(parseStamp("2026-08-05"), Date.parse("2026-08-05T00:00:00Z"));
+});
+
+test("parseStamp rejects junk rather than coercing it to a date", () => {
+  for (const bad of ["", "   ", "not-a-date", null, undefined, 42, {}, []]) {
+    assert.equal(parseStamp(bad), null, `should reject ${JSON.stringify(bad)}`);
+  }
+});
+
+// ─── readStamp ─────────────────────────────────────────────────────────────
+
+test("envelope strategy reads its declared field", () => {
+  assert.equal(
+    readStamp({ lastRefreshed: "2026-06-26", other: "x" }, { kind: "envelope", fields: ["lastRefreshed"] }),
+    "2026-06-26",
+  );
+});
+
+test("envelope prefers the earlier field: 'did we look' beats 'did it change'", () => {
+  // The scanner stamps lastChecked every run and lastRefreshed only on a real
+  // change. Staleness must follow lastChecked, or a dataset checked daily and
+  // correctly unchanged reads as abandoned.
+  const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] };
+  assert.equal(
+    readStamp({ lastChecked: "2026-08-05", lastRefreshed: "2026-05-20" }, strategy),
+    "2026-08-05",
+  );
+});
+
+test("envelope falls back when the preferred field is absent", () => {
+  // Before the scanner has ever run, only lastRefreshed exists.
+  const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] };
+  assert.equal(readStamp({ lastRefreshed: "2026-05-20" }, strategy), "2026-05-20");
+});
+
+test("envelope strategy does not accept an array", () => {
+  assert.equal(readStamp([{ lastRefreshed: "2026-06-26" }], { kind: "envelope", fields: ["lastRefreshed"] }), null);
+});
+
+test("series-max takes the newest row, not the last row", () => {
+  const rows = [{ date: "2026-08-03" }, { date: "2026-07-01" }, { date: "2026-08-01" }];
+  assert.equal(readStamp(rows, { kind: "series-max", field: "date" }), "2026-08-03");
+});
+
+test("series-max ignores malformed rows instead of treating them as epoch", () => {
+  // One bad row must not make a live series look dead.
+  const rows = [{ date: "2026-08-03" }, { date: "garbage" }, { nope: 1 }, null, "x"];
+  assert.equal(readStamp(rows as unknown[], { kind: "series-max", field: "date" }), "2026-08-03");
+});
+
+test("the none strategy never invents a stamp", () => {
+  assert.equal(readStamp({ lastRefreshed: "2026-08-05" }, { kind: "none" }), null);
+});
+
+// ─── classification ────────────────────────────────────────────────────────
+
+test("within cadence is ok", () => {
+  const r = computeFreshness({ t: { lastRefreshed: dayBefore(1) } }, NOW, [spec()]);
+  assert.equal(r.datasets[0].status, "ok");
+  assert.equal(r.healthy, true);
+});
+
+test("one missed run reads as aging, not stale", () => {
+  // 3d old against a 2d cadence: overdue but under 2x.
+  const r = computeFreshness({ t: { lastRefreshed: dayBefore(3) } }, NOW, [spec()]);
+  assert.equal(r.datasets[0].status, "aging");
+  assert.deepEqual(r.aging, ["t"]);
+  // Aging alone must not trip the deadman, or it cries wolf on every hiccup.
+  assert.equal(r.healthy, true);
+  assert.deepEqual(r.stale, []);
+});
+
+test("past double the cadence reads as stale and trips the deadman", () => {
+  const r = computeFreshness({ t: { lastRefreshed: dayBefore(9) } }, NOW, [spec()]);
+  assert.equal(r.datasets[0].status, "stale");
+  assert.deepEqual(r.stale, ["t"]);
+  assert.equal(r.healthy, false);
+  assert.match(r.datasets[0].detail, /test mechanism has probably stopped/);
+});
+
+test("hand-curated data is reported but can never alarm", () => {
+  // A curated file going quiet is a decision, not a failure.
+  const r = computeFreshness(
+    { t: { lastRefreshed: dayBefore(400) } },
+    NOW,
+    [spec({ expectedMaxAgeHours: null })],
+  );
+  assert.equal(r.datasets[0].status, "manual");
+  assert.equal(r.healthy, true);
+});
+
+test("a dataset with no readable timestamp is unknown, not fresh and not stale", () => {
+  const r = computeFreshness({ t: [{ name: "x" }] }, NOW, [spec({ read: { kind: "none" } })]);
+  assert.equal(r.datasets[0].status, "unknown");
+  assert.equal(r.datasets[0].asOf, null);
+  assert.equal(r.datasets[0].ageHours, null);
+  // Unknown must not alarm: it is a gap in instrumentation, not evidence of failure.
+  assert.equal(r.healthy, true);
+});
+
+test("a missing file is unknown rather than a crash", () => {
+  const r = computeFreshness({} as FileContents, NOW, [spec()]);
+  assert.equal(r.datasets[0].status, "unknown");
+  assert.equal(r.healthy, true);
+});
+
+test("a future timestamp clamps to zero age instead of going negative", () => {
+  const r = computeFreshness({ t: { lastRefreshed: "2027-01-01" } }, NOW, [spec()]);
+  assert.equal(r.datasets[0].ageHours, 0);
+  assert.equal(r.datasets[0].status, "ok");
+});
+
+test("one stale dataset makes the whole report unhealthy", () => {
+  const r = computeFreshness(
+    { a: { lastRefreshed: dayBefore(1) }, b: { lastRefreshed: dayBefore(30) } },
+    NOW,
+    [spec({ id: "a" }), spec({ id: "b" })],
+  );
+  assert.equal(r.healthy, false);
+  assert.deepEqual(r.stale, ["b"]);
+});
+
+// ─── the registry itself ───────────────────────────────────────────────────
+
+test("every registered dataset points at a file that exists", () => {
+  // A typo in the registry would otherwise show up as a permanent "unknown",
+  // which reads as an instrumentation gap rather than the mistake it is.
+  const missing = DATASET_REGISTRY.filter(
+    (d) => !existsSync(join(process.cwd(), "server", "data", d.file)),
+  ).map((d) => `${d.id} -> server/data/${d.file}`);
+  assert.deepEqual(missing, [], `registry points at files that do not exist:\n${missing.join("\n")}`);
+});
+
+test("registry ids are unique", () => {
+  const ids = DATASET_REGISTRY.map((d) => d.id);
+  assert.equal(new Set(ids).size, ids.length);
+});
+
+test("every declared read strategy actually resolves against the real file", () => {
+  // Guards the case that matters most: a dataset whose shape changed under the
+  // registry would silently report "unknown" forever and never alarm again.
+  const broken: string[] = [];
+  for (const d of DATASET_REGISTRY) {
+    if (d.read.kind === "none") continue;
+    const path = join(process.cwd(), "server", "data", d.file);
+    if (!existsSync(path)) continue;
+    const contents = JSON.parse(readFileSync(path, "utf-8"));
+    if (readStamp(contents, d.read) == null) {
+      broken.push(`${d.id}: ${d.read.kind} strategy found nothing in ${d.file}`);
+    }
+  }
+  assert.deepEqual(broken, [], `registry read strategies no longer match the data:\n${broken.join("\n")}`);
+});