From bc55289a6403386df1fde13f2e1896efc812fa0f Mon Sep 17 00:00:00 2001 From: aurph Date: Wed, 5 Aug 2026 09:53:13 -0400 Subject: [PATCH] Freshness: know when a refresh mechanism has stopped The interconnection queue was 77 days old and nothing anywhere knew. That is not a data problem, it is a missing sense: an unattended pipeline cannot tell you it stopped running. A flow on the Jetson dies quietly when the box reboots, n8n is switched off, or a PAT expires. Nothing errors, because nothing watches. The only symptom is a date that stops moving. server/freshness-registry.ts declares every dataset: file, how to read its timestamp, how old it may get, and what is supposed to refresh it. That expectation previously lived only in ops/n8n/README.md and in my head. Adding a dataset here is the whole integration. server/freshness.ts classifies. Pure and injected like indices.ts and clusters.ts, so the clock and the IO stay in the route layer: - ok: within cadence - aging: overdue but under 2x, which is one missed run - stale: past 2x, the mechanism has probably stopped - manual: hand-curated on no schedule, reported but never alarms - unknown: no readable timestamp, an instrumentation gap not a failure GET /api/admin/freshness returns the report. GET /api/admin/freshness/check is the deadman: 200 when nothing is stale, 503 naming the offenders and the exact mechanism to restart. Both admin-gated and both added to the auth-boundary route table. Freshness is the floor, not a feature, and publishing a page about how current we are would be advertising the bare minimum. The watchdog is pointed at deliberately from off the Jetson (cron-job.org, which already fires the daily tweet and alerts on non-2xx). A watchdog hosted on the box it watches dies with it, which is the failure this exists to catch. Setup in ops/freshness-monitor.md. Two supporting honesty fixes: - The news scanner now stamps lastChecked on every run and lastRefreshed only on a real change. Staleness follows lastChecked, otherwise a dataset checked daily and correctly unchanged reads as abandoned and you cannot tell that apart from the scanner never running. - Dates are read only from fields the file actually carries. Never file mtime, which would report a fresh checkout as fresh data. A bare date parses as the start of its UTC day, so nothing is ever reported fresher than it can prove. Against real data today it returns 503 on clusters (40d), interconnection-queue (77d), and gpu-rental-prices (39d), all stale because their flows were built but never mounted. datacenters.json reports unknown: it is a bare array with no date field, so its age genuinely cannot be checked. tsc clean, 307 tests pass, build passes. Verified live end to end: 401 without a key, 503 with real data, and 200 after temporarily stamping the three files current, so the alarm is proven to go green and not just stay red. --- ops/freshness-monitor.md | 82 +++++++++++ ops/n8n/README.md | 5 + server/__tests__/auth-boundary.test.ts | 2 + server/__tests__/freshness.test.ts | 188 ++++++++++++++++++++++++ server/freshness-registry.ts | 126 ++++++++++++++++ server/freshness.ts | 195 +++++++++++++++++++++++++ server/routes.ts | 73 ++++++++- 7 files changed, 669 insertions(+), 2 deletions(-) create mode 100644 ops/freshness-monitor.md create mode 100644 server/__tests__/freshness.test.ts create mode 100644 server/freshness-registry.ts create mode 100644 server/freshness.ts diff --git a/ops/freshness-monitor.md b/ops/freshness-monitor.md new file mode 100644 index 0000000..df85070 --- /dev/null +++ b/ops/freshness-monitor.md @@ -0,0 +1,82 @@ +# Dataset freshness monitor + +Answers the one question an unattended pipeline cannot answer about itself: +**has a refresh mechanism stopped running?** + +A flow on the Jetson dies quietly. The box reboots, n8n is switched off, a PAT +expires. Nothing errors, because nothing is watching. The only symptom is a date +that stops moving, which is how `interconnection-queue.json` reached 77 days +without anyone noticing. + +## What it is + +- `server/freshness-registry.ts` declares every dataset: where it lives, how to + read its timestamp, how old it is allowed to get, and what is supposed to + refresh it. Adding a dataset here is the whole integration. +- `server/freshness.ts` is the pure classifier. No fs, no clock, no env. +- `GET /api/admin/freshness` returns the full report. +- `GET /api/admin/freshness/check` is the deadman: 200 when nothing is stale, + 503 listing offenders when something is. + +Both routes are admin-gated. Freshness is the floor, not a feature; a public +"look how current we are" page advertises the bare minimum. + +## Statuses + +| status | meaning | trips the alarm | +|---|---|---| +| `ok` | within its declared cadence | no | +| `aging` | overdue but under 2x cadence, ie. one missed run | no | +| `stale` | past 2x cadence, the mechanism has probably stopped | **yes** | +| `manual` | hand-curated, no cadence declared | no | +| `unknown` | no readable timestamp in the file | no | + +`aging` is deliberately a 200. If a single missed run pages you, the alert stops +meaning anything within a month. + +`unknown` deliberately does not alarm either: it marks a gap in instrumentation, +not evidence of failure. `datacenters.json` is the current example, a bare array +with no date field anywhere, so its age cannot be checked at all. + +## Wiring the deadman + +**The watchdog must not run on the Jetson.** A watchdog hosted on the box it +watches dies with it, which is the exact failure this exists to catch. Use the +same cron-job.org account already firing the daily tweet. + +1. New cron job, URL `https://gridtilt.com/api/admin/freshness/check`. +2. Add request header `x-admin-key` with the production `ADMIN_API_KEY`. +3. Schedule daily. Hourly is pointless: every cadence here is measured in days. +4. Enable the job's failure notification. cron-job.org alerts on any non-2xx, + which is exactly the 503. +5. Save, run once manually, and confirm you get the failure mail while datasets + are still stale. **Verify the alarm fires before trusting the silence.** + +Nothing else to configure. The response body lists each stale dataset with its +mechanism, so the alert tells you which flow to go restart. + +## Why the dates can disagree with the site + +Two different questions, deliberately two different fields: + +- `lastRefreshed` is when a value last **changed**. +- `lastChecked` is when something last **looked**, changed or not. + +Staleness follows `lastChecked` where it exists, falling back to `lastRefreshed`. +Otherwise a dataset that is checked daily and correctly unchanged would read as +abandoned, and you could not tell that apart from the scanner never running. + +So once the scanner is scheduled, the monitor can be green while the homepage +still shows an older "as of" date on that stat. Both are honest and they answer +different questions. Do not "fix" this by pointing them at the same field. + +`POST /api/admin/scan-news-now` stamps `lastChecked` on every run. The n8n +`cluster-refresh` flow achieves the same thing differently: it commits even on a +zero-change pass so only `lastRefreshed` moves. + +## Current state + +As of the last check, three datasets are stale purely because their flows were +built but never mounted on the Jetson: `clusters`, `interconnection-queue`, and +`gpu-rental-prices`. See `ops/n8n/README.md` for mounting instructions. The +monitor does not fix staleness, it makes staleness loud. diff --git a/ops/n8n/README.md b/ops/n8n/README.md index 8e39c13..6e7b04d 100644 --- a/ops/n8n/README.md +++ b/ops/n8n/README.md @@ -3,6 +3,11 @@ Importable n8n workflows that keep GridTilt data fresh from the Jetson homelab. These run on the self-hosted n8n instance, not inside the app. +> **Nothing here runs until you mount it.** If a flow is never imported, or the +> Jetson is off, the data silently stops moving. `ops/freshness-monitor.md` is +> the watchdog that makes that failure loud; it deliberately runs off-Jetson, +> because a watchdog on the box it watches dies with it. + ## gpu-price-refresh.json Two independent flows in one workflow file: diff --git a/server/__tests__/auth-boundary.test.ts b/server/__tests__/auth-boundary.test.ts index f6e460a..34ef638 100644 --- a/server/__tests__/auth-boundary.test.ts +++ b/server/__tests__/auth-boundary.test.ts @@ -50,6 +50,8 @@ async function startTestServer(): Promise<{ url: string; close: () => Promise = [ ["GET", "/api/admin/subscribers"], ["GET", "/api/admin/gpu-history"], + ["GET", "/api/admin/freshness"], + ["GET", "/api/admin/freshness/check"], ["GET", "/api/newsletter/preview"], // SEC-1: was public, leaked subscriber count ["POST", "/api/social/generate"], // SEC-2: was public, burned Yahoo quota ["DELETE", "/api/admin/subscribers/x@y.com"], diff --git a/server/__tests__/freshness.test.ts b/server/__tests__/freshness.test.ts new file mode 100644 index 0000000..b01e2d7 --- /dev/null +++ b/server/__tests__/freshness.test.ts @@ -0,0 +1,188 @@ +// The freshness monitor exists to catch a stopped pipeline. These tests pin the +// behaviours that make it trustworthy: it never reports data as fresher than it +// can prove, it never alarms on data nobody promised to refresh, and a dataset +// that cannot report its age says so instead of guessing. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, existsSync } from "fs"; +import { join } from "path"; +import { + computeFreshness, + parseStamp, + readStamp, + type FileContents, +} from "../freshness.js"; +import { DATASET_REGISTRY, type DatasetSpec } from "../freshness-registry.js"; + +const NOW = Date.parse("2026-08-05T12:00:00Z"); +const DAY_MS = 24 * 60 * 60 * 1000; + +const spec = (over: Partial = {}): DatasetSpec => ({ + id: "t", + label: "Test", + file: "t.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 48, + mechanism: "test mechanism", + ...over, +}); + +const dayBefore = (days: number) => new Date(NOW - days * DAY_MS).toISOString().slice(0, 10); + +// ─── parseStamp ──────────────────────────────────────────────────────────── + +test("a bare date parses as the start of that UTC day, never the end", () => { + // Conservative on purpose: a file stamped today must not read as 0h old. + assert.equal(parseStamp("2026-08-05"), Date.parse("2026-08-05T00:00:00Z")); +}); + +test("parseStamp rejects junk rather than coercing it to a date", () => { + for (const bad of ["", " ", "not-a-date", null, undefined, 42, {}, []]) { + assert.equal(parseStamp(bad), null, `should reject ${JSON.stringify(bad)}`); + } +}); + +// ─── readStamp ───────────────────────────────────────────────────────────── + +test("envelope strategy reads its declared field", () => { + assert.equal( + readStamp({ lastRefreshed: "2026-06-26", other: "x" }, { kind: "envelope", fields: ["lastRefreshed"] }), + "2026-06-26", + ); +}); + +test("envelope prefers the earlier field: 'did we look' beats 'did it change'", () => { + // The scanner stamps lastChecked every run and lastRefreshed only on a real + // change. Staleness must follow lastChecked, or a dataset checked daily and + // correctly unchanged reads as abandoned. + const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] }; + assert.equal( + readStamp({ lastChecked: "2026-08-05", lastRefreshed: "2026-05-20" }, strategy), + "2026-08-05", + ); +}); + +test("envelope falls back when the preferred field is absent", () => { + // Before the scanner has ever run, only lastRefreshed exists. + const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] }; + assert.equal(readStamp({ lastRefreshed: "2026-05-20" }, strategy), "2026-05-20"); +}); + +test("envelope strategy does not accept an array", () => { + assert.equal(readStamp([{ lastRefreshed: "2026-06-26" }], { kind: "envelope", fields: ["lastRefreshed"] }), null); +}); + +test("series-max takes the newest row, not the last row", () => { + const rows = [{ date: "2026-08-03" }, { date: "2026-07-01" }, { date: "2026-08-01" }]; + assert.equal(readStamp(rows, { kind: "series-max", field: "date" }), "2026-08-03"); +}); + +test("series-max ignores malformed rows instead of treating them as epoch", () => { + // One bad row must not make a live series look dead. + const rows = [{ date: "2026-08-03" }, { date: "garbage" }, { nope: 1 }, null, "x"]; + assert.equal(readStamp(rows as unknown[], { kind: "series-max", field: "date" }), "2026-08-03"); +}); + +test("the none strategy never invents a stamp", () => { + assert.equal(readStamp({ lastRefreshed: "2026-08-05" }, { kind: "none" }), null); +}); + +// ─── classification ──────────────────────────────────────────────────────── + +test("within cadence is ok", () => { + const r = computeFreshness({ t: { lastRefreshed: dayBefore(1) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "ok"); + assert.equal(r.healthy, true); +}); + +test("one missed run reads as aging, not stale", () => { + // 3d old against a 2d cadence: overdue but under 2x. + const r = computeFreshness({ t: { lastRefreshed: dayBefore(3) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "aging"); + assert.deepEqual(r.aging, ["t"]); + // Aging alone must not trip the deadman, or it cries wolf on every hiccup. + assert.equal(r.healthy, true); + assert.deepEqual(r.stale, []); +}); + +test("past double the cadence reads as stale and trips the deadman", () => { + const r = computeFreshness({ t: { lastRefreshed: dayBefore(9) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "stale"); + assert.deepEqual(r.stale, ["t"]); + assert.equal(r.healthy, false); + assert.match(r.datasets[0].detail, /test mechanism has probably stopped/); +}); + +test("hand-curated data is reported but can never alarm", () => { + // A curated file going quiet is a decision, not a failure. + const r = computeFreshness( + { t: { lastRefreshed: dayBefore(400) } }, + NOW, + [spec({ expectedMaxAgeHours: null })], + ); + assert.equal(r.datasets[0].status, "manual"); + assert.equal(r.healthy, true); +}); + +test("a dataset with no readable timestamp is unknown, not fresh and not stale", () => { + const r = computeFreshness({ t: [{ name: "x" }] }, NOW, [spec({ read: { kind: "none" } })]); + assert.equal(r.datasets[0].status, "unknown"); + assert.equal(r.datasets[0].asOf, null); + assert.equal(r.datasets[0].ageHours, null); + // Unknown must not alarm: it is a gap in instrumentation, not evidence of failure. + assert.equal(r.healthy, true); +}); + +test("a missing file is unknown rather than a crash", () => { + const r = computeFreshness({} as FileContents, NOW, [spec()]); + assert.equal(r.datasets[0].status, "unknown"); + assert.equal(r.healthy, true); +}); + +test("a future timestamp clamps to zero age instead of going negative", () => { + const r = computeFreshness({ t: { lastRefreshed: "2027-01-01" } }, NOW, [spec()]); + assert.equal(r.datasets[0].ageHours, 0); + assert.equal(r.datasets[0].status, "ok"); +}); + +test("one stale dataset makes the whole report unhealthy", () => { + const r = computeFreshness( + { a: { lastRefreshed: dayBefore(1) }, b: { lastRefreshed: dayBefore(30) } }, + NOW, + [spec({ id: "a" }), spec({ id: "b" })], + ); + assert.equal(r.healthy, false); + assert.deepEqual(r.stale, ["b"]); +}); + +// ─── the registry itself ─────────────────────────────────────────────────── + +test("every registered dataset points at a file that exists", () => { + // A typo in the registry would otherwise show up as a permanent "unknown", + // which reads as an instrumentation gap rather than the mistake it is. + const missing = DATASET_REGISTRY.filter( + (d) => !existsSync(join(process.cwd(), "server", "data", d.file)), + ).map((d) => `${d.id} -> server/data/${d.file}`); + assert.deepEqual(missing, [], `registry points at files that do not exist:\n${missing.join("\n")}`); +}); + +test("registry ids are unique", () => { + const ids = DATASET_REGISTRY.map((d) => d.id); + assert.equal(new Set(ids).size, ids.length); +}); + +test("every declared read strategy actually resolves against the real file", () => { + // Guards the case that matters most: a dataset whose shape changed under the + // registry would silently report "unknown" forever and never alarm again. + const broken: string[] = []; + for (const d of DATASET_REGISTRY) { + if (d.read.kind === "none") continue; + const path = join(process.cwd(), "server", "data", d.file); + if (!existsSync(path)) continue; + const contents = JSON.parse(readFileSync(path, "utf-8")); + if (readStamp(contents, d.read) == null) { + broken.push(`${d.id}: ${d.read.kind} strategy found nothing in ${d.file}`); + } + } + assert.deepEqual(broken, [], `registry read strategies no longer match the data:\n${broken.join("\n")}`); +}); diff --git a/server/freshness-registry.ts b/server/freshness-registry.ts new file mode 100644 index 0000000..b2ea774 --- /dev/null +++ b/server/freshness-registry.ts @@ -0,0 +1,126 @@ +// ─── Dataset freshness registry ────────────────────────────────────────── +// +// One declared place for the question "how old is this dataset allowed to get +// before something is wrong?". Before this file that expectation lived only in +// ops/n8n/README.md and in the owner's head, which is how the interconnection +// queue reached 77 days without anything noticing. +// +// Nothing here is public. The report this feeds is admin-gated: freshness is +// the floor, not a feature, and a service that advertises its own currency is +// advertising the bare minimum. +// +// Adding a dataset here is the whole integration. server/freshness.ts consumes +// this and needs no per-dataset code. + +/** + * How to find a dataset's own timestamp. Deliberately explicit per dataset: + * inferring from file mtime would report a fresh checkout as fresh data, which + * is the exact lie this module exists to prevent. + */ +export type ReadStrategy = + /** + * Envelope object carrying one of these fields, e.g. + * { lastRefreshed: "2026-06-26", ... }. Ordered: the first field present + * wins, so a dataset can prefer "when did we last look" (lastChecked) over + * "when did a value last change" (lastRefreshed). Staleness is about whether + * the mechanism is alive, which is the first question, not the second. + */ + | { kind: "envelope"; fields: string[] } + /** Bare array of rows; freshness is the newest value of `field` across rows. */ + | { kind: "series-max"; field: string } + /** No timestamp exists in the file. Reports "unknown", never alarms. */ + | { kind: "none" }; + +export interface DatasetSpec { + id: string; + /** Human label for the admin report. */ + label: string; + /** Path under server/data/. */ + file: string; + read: ReadStrategy; + /** + * Hours before the dataset is considered overdue. null means the dataset is + * hand-curated on no schedule: it is reported but can never trip the alarm, + * because a curated file going quiet is a decision, not a failure. + */ + expectedMaxAgeHours: number | null; + /** What is supposed to refresh this, in words, for the alert to be actionable. */ + mechanism: string; +} + +const DAY = 24; + +/** + * Cadences are set to roughly twice the mechanism's own period, so a single + * missed run is tolerated and a stopped mechanism is not. The n8n flows and + * their schedules are documented in ops/n8n/README.md. + */ +export const DATASET_REGISTRY: DatasetSpec[] = [ + { + id: "clusters", + label: "Compute Frontier clusters", + file: "clusters.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 2 * DAY, + mechanism: "n8n cluster-refresh, daily 06:30 (commits even on a zero-change pass)", + }, + { + id: "interconnection-queue", + label: "Power deals / interconnection queue", + file: "interconnection-queue.json", + read: { kind: "envelope", fields: ["lastChecked", "lastRefreshed"] }, + expectedMaxAgeHours: 7 * DAY, + mechanism: "POST /api/admin/scan-news-now, not yet scheduled", + }, + { + id: "gpu-rental-prices", + label: "GPU rental prices (curated)", + file: "gpu-rental-prices.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 10 * DAY, + mechanism: "n8n gpu-price-refresh, weekly Mon 06:00", + }, + { + id: "gpu-price-history", + label: "GPU price history (recorder)", + file: "gpu-price-history.json", + read: { kind: "series-max", field: "date" }, + expectedMaxAgeHours: 3 * DAY, + mechanism: "daily recorder ping to /api/gpu-prices/metrics, n8n 05:00", + }, + { + id: "hyperscaler-capex", + label: "Hyperscaler capex", + file: "hyperscaler-capex.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated, quarterly with earnings", + }, + { + id: "inference-prices", + label: "Frontier inference prices", + file: "inference-prices.json", + read: { kind: "envelope", fields: ["asOf"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated from provider pricing pages", + }, + { + id: "frontier-models", + label: "Frontier model registry", + file: "frontier-models.json", + read: { kind: "envelope", fields: ["asOf"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated on model releases", + }, + { + // No per-row date field and no envelope: the ingester appends rows without + // stamping anything, so the file cannot report its own age. Reports + // "unknown" until the ingester stamps a timestamp. + id: "datacenters", + label: "Data center facilities", + file: "datacenters.json", + read: { kind: "none" }, + expectedMaxAgeHours: 2 * DAY, + mechanism: "in-process ingester every 6h (unreliable on Replit autoscale)", + }, +]; diff --git a/server/freshness.ts b/server/freshness.ts new file mode 100644 index 0000000..d36fc18 --- /dev/null +++ b/server/freshness.ts @@ -0,0 +1,195 @@ +// ─── Dataset freshness (pure) ──────────────────────────────────────────── +// +// Turns the declared registry plus raw file contents into a per-dataset verdict. +// Pure and injected on purpose (same discipline as indices.ts / clusters.ts / +// gpu-index.ts): no fs, no Date.now, no env. The route layer does the IO. +// +// The point of this module is not to display dates. It is to answer one +// question an unattended pipeline cannot answer about itself: has a mechanism +// stopped running? A flow on a homelab box dies quietly (reboot, expired token, +// n8n switched off) and the only symptom is a date that stops moving. That is +// exactly how the interconnection queue reached 77 days unnoticed. + +import { DATASET_REGISTRY, type DatasetSpec, type ReadStrategy } from "./freshness-registry.js"; + +export type FreshnessStatus = + /** Within its expected cadence. */ + | "ok" + /** Overdue, but under 2x. One missed run looks like this. */ + | "aging" + /** Past 2x its cadence. The mechanism has almost certainly stopped. */ + | "stale" + /** Hand-curated with no declared cadence. Reported, never alarms. */ + | "manual" + /** No readable timestamp, or the file is missing/malformed. Never alarms. */ + | "unknown"; + +export interface DatasetFreshness { + id: string; + label: string; + file: string; + /** ISO date the dataset reports for itself, or null when unreadable. */ + asOf: string | null; + ageHours: number | null; + expectedMaxAgeHours: number | null; + status: FreshnessStatus; + mechanism: string; + /** Why the status is what it is, in words, so an alert is actionable. */ + detail: string; +} + +export interface FreshnessReport { + generatedAt: string; + datasets: DatasetFreshness[]; + /** Datasets at "stale". These are what trip the deadman. */ + stale: string[]; + /** Datasets at "aging". Reported but do not trip the deadman on their own. */ + aging: string[]; + /** True when nothing is stale. The whole point of the check endpoint. */ + healthy: boolean; +} + +/** Raw file contents by dataset id. undefined means unreadable or missing. */ +export type FileContents = Record; + +const HOUR_MS = 60 * 60 * 1000; + +/** + * Parse a date that may be "2026-06-26" or a full ISO timestamp. + * + * A bare date is treated as the START of that UTC day, never the end. A dataset + * stamped "today" therefore reads as up to 24h old rather than 0h old, which + * keeps the age estimate conservative: this module should never report data as + * fresher than it can prove. + */ +export function parseStamp(value: unknown): number | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed) return null; + if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) { + const ms = Date.parse(`${trimmed}T00:00:00Z`); + return Number.isNaN(ms) ? null : ms; + } + const ms = Date.parse(trimmed); + return Number.isNaN(ms) ? null : ms; +} + +/** Pull the dataset's self-reported stamp out of its contents, per strategy. */ +export function readStamp(contents: unknown, strategy: ReadStrategy): string | null { + if (contents == null) return null; + + if (strategy.kind === "none") return null; + + if (strategy.kind === "envelope") { + if (typeof contents !== "object" || Array.isArray(contents)) return null; + const obj = contents as Record; + // First declared field that is actually present wins, so "did we look" + // takes precedence over "did anything change" where both are recorded. + for (const field of strategy.fields) { + const raw = obj[field]; + if (typeof raw === "string" && raw.trim()) return raw.trim(); + } + return null; + } + + // series-max: newest row wins. Rows without the field are ignored rather + // than treated as epoch, so one malformed row cannot make a live series + // look dead. + if (!Array.isArray(contents)) return null; + let best: string | null = null; + let bestMs = -Infinity; + for (const row of contents) { + if (row == null || typeof row !== "object") continue; + const raw = (row as Record)[strategy.field]; + if (typeof raw !== "string") continue; + const ms = parseStamp(raw); + if (ms == null) continue; + if (ms > bestMs) { + bestMs = ms; + best = raw.trim(); + } + } + return best; +} + +function classify( + spec: DatasetSpec, + asOf: string | null, + ageHours: number | null, +): { status: FreshnessStatus; detail: string } { + if (asOf == null || ageHours == null) { + return { + status: "unknown", + detail: + spec.read.kind === "none" + ? "file carries no timestamp, so its age cannot be checked" + : "timestamp missing or unparseable", + }; + } + + if (spec.expectedMaxAgeHours == null) { + return { + status: "manual", + detail: `hand-curated, ${Math.floor(ageHours / 24)}d old, no cadence to miss`, + }; + } + + const days = Math.floor(ageHours / 24); + const limit = spec.expectedMaxAgeHours; + + if (ageHours <= limit) { + return { status: "ok", detail: `${days}d old, within ${Math.round(limit / 24)}d` }; + } + if (ageHours <= limit * 2) { + return { + status: "aging", + detail: `${days}d old, past its ${Math.round(limit / 24)}d cadence (one missed run looks like this)`, + }; + } + return { + status: "stale", + detail: `${days}d old, more than double its ${Math.round(limit / 24)}d cadence: ${spec.mechanism} has probably stopped`, + }; +} + +/** + * Build the report. + * + * `nowMs` is injected so the tests are deterministic and so the caller owns the + * clock, matching how the rest of the server's pure modules are written. + */ +export function computeFreshness( + contentsById: FileContents, + nowMs: number, + registry: DatasetSpec[] = DATASET_REGISTRY, +): FreshnessReport { + const datasets: DatasetFreshness[] = registry.map((spec) => { + const asOf = readStamp(contentsById[spec.id], spec.read); + const stampMs = parseStamp(asOf); + const ageHours = stampMs == null ? null : Math.max(0, (nowMs - stampMs) / HOUR_MS); + const { status, detail } = classify(spec, asOf, ageHours); + + return { + id: spec.id, + label: spec.label, + file: spec.file, + asOf, + ageHours: ageHours == null ? null : Math.round(ageHours * 10) / 10, + expectedMaxAgeHours: spec.expectedMaxAgeHours, + status, + mechanism: spec.mechanism, + detail, + }; + }); + + const stale = datasets.filter((d) => d.status === "stale").map((d) => d.id); + const aging = datasets.filter((d) => d.status === "aging").map((d) => d.id); + + return { + generatedAt: new Date(nowMs).toISOString(), + datasets, + stale, + aging, + healthy: stale.length === 0, + }; +} diff --git a/server/routes.ts b/server/routes.ts index 1507e9d..b0dac64 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -48,6 +48,8 @@ import { composeBrief, renderBriefText, type BriefInput } from "./brief"; import { computeGpuEconomics, TRAINING_PRESETS } from "./gpu-economics"; import { readFrontierRegistry, summarizeFrontierRegistry } from "./frontier-models"; import { readInferencePrices, buildInferencePriceView } from "./inference-prices"; +import { computeFreshness, type FileContents } from "./freshness"; +import { DATASET_REGISTRY } from "./freshness-registry"; import { buildBuildoutTweet, buildGpuRentalTweet, @@ -846,10 +848,20 @@ async function scanNewsForBacklogUpdates(news: NewsItem[]): Promise<{ applied: n } } + // Two different questions, two different fields. + // + // lastRefreshed answers "when did a number here last change" and so only + // moves on a real change. lastChecked answers "when did anything last look + // at this", and must move on every run including a clean one. Conflating + // them is why a dataset that is watched daily and correctly unchanged would + // still look abandoned, and why nothing could tell that apart from the + // scanner never running at all. + const today = new Date().toISOString().slice(0, 10); + data.lastChecked = today; if (applied > 0) { - data.lastRefreshed = new Date().toISOString().slice(0, 10); - writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n"); + data.lastRefreshed = today; } + writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n"); return { applied, flagged, checked }; } @@ -1027,7 +1039,14 @@ interface BacklogProject { notes?: string; } interface BacklogDataset { + /** When a value in this dataset last changed. */ lastRefreshed: string; + /** + * When the news scanner last ran against this dataset, changed or not. + * Optional because files written before the scanner started stamping it + * will not carry one; freshness falls back to lastRefreshed in that case. + */ + lastChecked?: string; headline: { trackedProjects: number; trackedCapacityGW: number; @@ -2188,6 +2207,56 @@ export async function registerRoutes( res.json(readGpuHistory()); }); + // ─── Dataset freshness (admin only) ────────────────────────────────────── + // + // Answers the one question an unattended pipeline cannot answer about + // itself: has a refresh mechanism stopped? Both routes are gated. Freshness + // is the floor, not a feature, and publishing a "how current are we" page + // advertises the bare minimum. + + /** Read every registered dataset off disk. Unreadable files stay absent. */ + function readRegisteredDatasets(): FileContents { + const contents: FileContents = {}; + for (const spec of DATASET_REGISTRY) { + try { + const path = join(process.cwd(), "server", "data", spec.file); + contents[spec.id] = JSON.parse(readFileSync(path, "utf-8")); + } catch { + // Leave it out: computeFreshness reports "unknown" rather than + // guessing, and a missing file must never read as fresh. + } + } + return contents; + } + + app.get("/api/admin/freshness", (req: Request, res) => { + if (!requireAdmin(req, res)) return; + res.json(computeFreshness(readRegisteredDatasets(), Date.now())); + }); + + // The deadman. An external pinger (cron-job.org, same account already firing + // the daily tweet) hits this with x-admin-key on a schedule and alerts on any + // non-2xx. It deliberately lives off the Jetson: a watchdog hosted on the box + // it watches dies with it, which is the failure that let the interconnection + // queue rot for 77 days. + // + // Path is /freshness/check and never /health so the platform cannot mistake + // a stale-data 503 for an unhealthy instance and start recycling it. + app.get("/api/admin/freshness/check", (req: Request, res) => { + if (!requireAdmin(req, res)) return; + const report = computeFreshness(readRegisteredDatasets(), Date.now()); + // "aging" is intentionally a 200: one missed run should not page anyone, + // or the alert stops meaning anything. + res.status(report.healthy ? 200 : 503).json({ + healthy: report.healthy, + stale: report.datasets + .filter((d) => d.status === "stale") + .map((d) => ({ id: d.id, asOf: d.asOf, detail: d.detail, mechanism: d.mechanism })), + aging: report.aging, + generatedAt: report.generatedAt, + }); + }); + app.post("/api/subscribe", subscribeLimiter, async (req: Request, res) => { try { const { email, intent, context } = req.body;