diff --git a/ops/freshness-monitor.md b/ops/freshness-monitor.md new file mode 100644 index 0000000..df85070 --- /dev/null +++ b/ops/freshness-monitor.md @@ -0,0 +1,82 @@ +# Dataset freshness monitor + +Answers the one question an unattended pipeline cannot answer about itself: +**has a refresh mechanism stopped running?** + +A flow on the Jetson dies quietly. The box reboots, n8n is switched off, a PAT +expires. Nothing errors, because nothing is watching. The only symptom is a date +that stops moving, which is how `interconnection-queue.json` reached 77 days +without anyone noticing. + +## What it is + +- `server/freshness-registry.ts` declares every dataset: where it lives, how to + read its timestamp, how old it is allowed to get, and what is supposed to + refresh it. Adding a dataset here is the whole integration. +- `server/freshness.ts` is the pure classifier. No fs, no clock, no env. +- `GET /api/admin/freshness` returns the full report. +- `GET /api/admin/freshness/check` is the deadman: 200 when nothing is stale, + 503 listing offenders when something is. + +Both routes are admin-gated. Freshness is the floor, not a feature; a public +"look how current we are" page advertises the bare minimum. + +## Statuses + +| status | meaning | trips the alarm | +|---|---|---| +| `ok` | within its declared cadence | no | +| `aging` | overdue but under 2x cadence, ie. one missed run | no | +| `stale` | past 2x cadence, the mechanism has probably stopped | **yes** | +| `manual` | hand-curated, no cadence declared | no | +| `unknown` | no readable timestamp in the file | no | + +`aging` is deliberately a 200. If a single missed run pages you, the alert stops +meaning anything within a month. + +`unknown` deliberately does not alarm either: it marks a gap in instrumentation, +not evidence of failure. `datacenters.json` is the current example, a bare array +with no date field anywhere, so its age cannot be checked at all. + +## Wiring the deadman + +**The watchdog must not run on the Jetson.** A watchdog hosted on the box it +watches dies with it, which is the exact failure this exists to catch. Use the +same cron-job.org account already firing the daily tweet. + +1. New cron job, URL `https://gridtilt.com/api/admin/freshness/check`. +2. Add request header `x-admin-key` with the production `ADMIN_API_KEY`. +3. Schedule daily. Hourly is pointless: every cadence here is measured in days. +4. Enable the job's failure notification. cron-job.org alerts on any non-2xx, + which is exactly the 503. +5. Save, run once manually, and confirm you get the failure mail while datasets + are still stale. **Verify the alarm fires before trusting the silence.** + +Nothing else to configure. The response body lists each stale dataset with its +mechanism, so the alert tells you which flow to go restart. + +## Why the dates can disagree with the site + +Two different questions, deliberately two different fields: + +- `lastRefreshed` is when a value last **changed**. +- `lastChecked` is when something last **looked**, changed or not. + +Staleness follows `lastChecked` where it exists, falling back to `lastRefreshed`. +Otherwise a dataset that is checked daily and correctly unchanged would read as +abandoned, and you could not tell that apart from the scanner never running. + +So once the scanner is scheduled, the monitor can be green while the homepage +still shows an older "as of" date on that stat. Both are honest and they answer +different questions. Do not "fix" this by pointing them at the same field. + +`POST /api/admin/scan-news-now` stamps `lastChecked` on every run. The n8n +`cluster-refresh` flow achieves the same thing differently: it commits even on a +zero-change pass so only `lastRefreshed` moves. + +## Current state + +As of the last check, three datasets are stale purely because their flows were +built but never mounted on the Jetson: `clusters`, `interconnection-queue`, and +`gpu-rental-prices`. See `ops/n8n/README.md` for mounting instructions. The +monitor does not fix staleness, it makes staleness loud. diff --git a/ops/n8n/README.md b/ops/n8n/README.md index 8e39c13..6e7b04d 100644 --- a/ops/n8n/README.md +++ b/ops/n8n/README.md @@ -3,6 +3,11 @@ Importable n8n workflows that keep GridTilt data fresh from the Jetson homelab. These run on the self-hosted n8n instance, not inside the app. +> **Nothing here runs until you mount it.** If a flow is never imported, or the +> Jetson is off, the data silently stops moving. `ops/freshness-monitor.md` is +> the watchdog that makes that failure loud; it deliberately runs off-Jetson, +> because a watchdog on the box it watches dies with it. + ## gpu-price-refresh.json Two independent flows in one workflow file: diff --git a/server/__tests__/auth-boundary.test.ts b/server/__tests__/auth-boundary.test.ts index f6e460a..34ef638 100644 --- a/server/__tests__/auth-boundary.test.ts +++ b/server/__tests__/auth-boundary.test.ts @@ -50,6 +50,8 @@ async function startTestServer(): Promise<{ url: string; close: () => Promise = [ ["GET", "/api/admin/subscribers"], ["GET", "/api/admin/gpu-history"], + ["GET", "/api/admin/freshness"], + ["GET", "/api/admin/freshness/check"], ["GET", "/api/newsletter/preview"], // SEC-1: was public, leaked subscriber count ["POST", "/api/social/generate"], // SEC-2: was public, burned Yahoo quota ["DELETE", "/api/admin/subscribers/x@y.com"], diff --git a/server/__tests__/freshness.test.ts b/server/__tests__/freshness.test.ts new file mode 100644 index 0000000..b01e2d7 --- /dev/null +++ b/server/__tests__/freshness.test.ts @@ -0,0 +1,188 @@ +// The freshness monitor exists to catch a stopped pipeline. These tests pin the +// behaviours that make it trustworthy: it never reports data as fresher than it +// can prove, it never alarms on data nobody promised to refresh, and a dataset +// that cannot report its age says so instead of guessing. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, existsSync } from "fs"; +import { join } from "path"; +import { + computeFreshness, + parseStamp, + readStamp, + type FileContents, +} from "../freshness.js"; +import { DATASET_REGISTRY, type DatasetSpec } from "../freshness-registry.js"; + +const NOW = Date.parse("2026-08-05T12:00:00Z"); +const DAY_MS = 24 * 60 * 60 * 1000; + +const spec = (over: Partial = {}): DatasetSpec => ({ + id: "t", + label: "Test", + file: "t.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 48, + mechanism: "test mechanism", + ...over, +}); + +const dayBefore = (days: number) => new Date(NOW - days * DAY_MS).toISOString().slice(0, 10); + +// ─── parseStamp ──────────────────────────────────────────────────────────── + +test("a bare date parses as the start of that UTC day, never the end", () => { + // Conservative on purpose: a file stamped today must not read as 0h old. + assert.equal(parseStamp("2026-08-05"), Date.parse("2026-08-05T00:00:00Z")); +}); + +test("parseStamp rejects junk rather than coercing it to a date", () => { + for (const bad of ["", " ", "not-a-date", null, undefined, 42, {}, []]) { + assert.equal(parseStamp(bad), null, `should reject ${JSON.stringify(bad)}`); + } +}); + +// ─── readStamp ───────────────────────────────────────────────────────────── + +test("envelope strategy reads its declared field", () => { + assert.equal( + readStamp({ lastRefreshed: "2026-06-26", other: "x" }, { kind: "envelope", fields: ["lastRefreshed"] }), + "2026-06-26", + ); +}); + +test("envelope prefers the earlier field: 'did we look' beats 'did it change'", () => { + // The scanner stamps lastChecked every run and lastRefreshed only on a real + // change. Staleness must follow lastChecked, or a dataset checked daily and + // correctly unchanged reads as abandoned. + const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] }; + assert.equal( + readStamp({ lastChecked: "2026-08-05", lastRefreshed: "2026-05-20" }, strategy), + "2026-08-05", + ); +}); + +test("envelope falls back when the preferred field is absent", () => { + // Before the scanner has ever run, only lastRefreshed exists. + const strategy = { kind: "envelope" as const, fields: ["lastChecked", "lastRefreshed"] }; + assert.equal(readStamp({ lastRefreshed: "2026-05-20" }, strategy), "2026-05-20"); +}); + +test("envelope strategy does not accept an array", () => { + assert.equal(readStamp([{ lastRefreshed: "2026-06-26" }], { kind: "envelope", fields: ["lastRefreshed"] }), null); +}); + +test("series-max takes the newest row, not the last row", () => { + const rows = [{ date: "2026-08-03" }, { date: "2026-07-01" }, { date: "2026-08-01" }]; + assert.equal(readStamp(rows, { kind: "series-max", field: "date" }), "2026-08-03"); +}); + +test("series-max ignores malformed rows instead of treating them as epoch", () => { + // One bad row must not make a live series look dead. + const rows = [{ date: "2026-08-03" }, { date: "garbage" }, { nope: 1 }, null, "x"]; + assert.equal(readStamp(rows as unknown[], { kind: "series-max", field: "date" }), "2026-08-03"); +}); + +test("the none strategy never invents a stamp", () => { + assert.equal(readStamp({ lastRefreshed: "2026-08-05" }, { kind: "none" }), null); +}); + +// ─── classification ──────────────────────────────────────────────────────── + +test("within cadence is ok", () => { + const r = computeFreshness({ t: { lastRefreshed: dayBefore(1) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "ok"); + assert.equal(r.healthy, true); +}); + +test("one missed run reads as aging, not stale", () => { + // 3d old against a 2d cadence: overdue but under 2x. + const r = computeFreshness({ t: { lastRefreshed: dayBefore(3) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "aging"); + assert.deepEqual(r.aging, ["t"]); + // Aging alone must not trip the deadman, or it cries wolf on every hiccup. + assert.equal(r.healthy, true); + assert.deepEqual(r.stale, []); +}); + +test("past double the cadence reads as stale and trips the deadman", () => { + const r = computeFreshness({ t: { lastRefreshed: dayBefore(9) } }, NOW, [spec()]); + assert.equal(r.datasets[0].status, "stale"); + assert.deepEqual(r.stale, ["t"]); + assert.equal(r.healthy, false); + assert.match(r.datasets[0].detail, /test mechanism has probably stopped/); +}); + +test("hand-curated data is reported but can never alarm", () => { + // A curated file going quiet is a decision, not a failure. + const r = computeFreshness( + { t: { lastRefreshed: dayBefore(400) } }, + NOW, + [spec({ expectedMaxAgeHours: null })], + ); + assert.equal(r.datasets[0].status, "manual"); + assert.equal(r.healthy, true); +}); + +test("a dataset with no readable timestamp is unknown, not fresh and not stale", () => { + const r = computeFreshness({ t: [{ name: "x" }] }, NOW, [spec({ read: { kind: "none" } })]); + assert.equal(r.datasets[0].status, "unknown"); + assert.equal(r.datasets[0].asOf, null); + assert.equal(r.datasets[0].ageHours, null); + // Unknown must not alarm: it is a gap in instrumentation, not evidence of failure. + assert.equal(r.healthy, true); +}); + +test("a missing file is unknown rather than a crash", () => { + const r = computeFreshness({} as FileContents, NOW, [spec()]); + assert.equal(r.datasets[0].status, "unknown"); + assert.equal(r.healthy, true); +}); + +test("a future timestamp clamps to zero age instead of going negative", () => { + const r = computeFreshness({ t: { lastRefreshed: "2027-01-01" } }, NOW, [spec()]); + assert.equal(r.datasets[0].ageHours, 0); + assert.equal(r.datasets[0].status, "ok"); +}); + +test("one stale dataset makes the whole report unhealthy", () => { + const r = computeFreshness( + { a: { lastRefreshed: dayBefore(1) }, b: { lastRefreshed: dayBefore(30) } }, + NOW, + [spec({ id: "a" }), spec({ id: "b" })], + ); + assert.equal(r.healthy, false); + assert.deepEqual(r.stale, ["b"]); +}); + +// ─── the registry itself ─────────────────────────────────────────────────── + +test("every registered dataset points at a file that exists", () => { + // A typo in the registry would otherwise show up as a permanent "unknown", + // which reads as an instrumentation gap rather than the mistake it is. + const missing = DATASET_REGISTRY.filter( + (d) => !existsSync(join(process.cwd(), "server", "data", d.file)), + ).map((d) => `${d.id} -> server/data/${d.file}`); + assert.deepEqual(missing, [], `registry points at files that do not exist:\n${missing.join("\n")}`); +}); + +test("registry ids are unique", () => { + const ids = DATASET_REGISTRY.map((d) => d.id); + assert.equal(new Set(ids).size, ids.length); +}); + +test("every declared read strategy actually resolves against the real file", () => { + // Guards the case that matters most: a dataset whose shape changed under the + // registry would silently report "unknown" forever and never alarm again. + const broken: string[] = []; + for (const d of DATASET_REGISTRY) { + if (d.read.kind === "none") continue; + const path = join(process.cwd(), "server", "data", d.file); + if (!existsSync(path)) continue; + const contents = JSON.parse(readFileSync(path, "utf-8")); + if (readStamp(contents, d.read) == null) { + broken.push(`${d.id}: ${d.read.kind} strategy found nothing in ${d.file}`); + } + } + assert.deepEqual(broken, [], `registry read strategies no longer match the data:\n${broken.join("\n")}`); +}); diff --git a/server/freshness-registry.ts b/server/freshness-registry.ts new file mode 100644 index 0000000..b2ea774 --- /dev/null +++ b/server/freshness-registry.ts @@ -0,0 +1,126 @@ +// ─── Dataset freshness registry ────────────────────────────────────────── +// +// One declared place for the question "how old is this dataset allowed to get +// before something is wrong?". Before this file that expectation lived only in +// ops/n8n/README.md and in the owner's head, which is how the interconnection +// queue reached 77 days without anything noticing. +// +// Nothing here is public. The report this feeds is admin-gated: freshness is +// the floor, not a feature, and a service that advertises its own currency is +// advertising the bare minimum. +// +// Adding a dataset here is the whole integration. server/freshness.ts consumes +// this and needs no per-dataset code. + +/** + * How to find a dataset's own timestamp. Deliberately explicit per dataset: + * inferring from file mtime would report a fresh checkout as fresh data, which + * is the exact lie this module exists to prevent. + */ +export type ReadStrategy = + /** + * Envelope object carrying one of these fields, e.g. + * { lastRefreshed: "2026-06-26", ... }. Ordered: the first field present + * wins, so a dataset can prefer "when did we last look" (lastChecked) over + * "when did a value last change" (lastRefreshed). Staleness is about whether + * the mechanism is alive, which is the first question, not the second. + */ + | { kind: "envelope"; fields: string[] } + /** Bare array of rows; freshness is the newest value of `field` across rows. */ + | { kind: "series-max"; field: string } + /** No timestamp exists in the file. Reports "unknown", never alarms. */ + | { kind: "none" }; + +export interface DatasetSpec { + id: string; + /** Human label for the admin report. */ + label: string; + /** Path under server/data/. */ + file: string; + read: ReadStrategy; + /** + * Hours before the dataset is considered overdue. null means the dataset is + * hand-curated on no schedule: it is reported but can never trip the alarm, + * because a curated file going quiet is a decision, not a failure. + */ + expectedMaxAgeHours: number | null; + /** What is supposed to refresh this, in words, for the alert to be actionable. */ + mechanism: string; +} + +const DAY = 24; + +/** + * Cadences are set to roughly twice the mechanism's own period, so a single + * missed run is tolerated and a stopped mechanism is not. The n8n flows and + * their schedules are documented in ops/n8n/README.md. + */ +export const DATASET_REGISTRY: DatasetSpec[] = [ + { + id: "clusters", + label: "Compute Frontier clusters", + file: "clusters.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 2 * DAY, + mechanism: "n8n cluster-refresh, daily 06:30 (commits even on a zero-change pass)", + }, + { + id: "interconnection-queue", + label: "Power deals / interconnection queue", + file: "interconnection-queue.json", + read: { kind: "envelope", fields: ["lastChecked", "lastRefreshed"] }, + expectedMaxAgeHours: 7 * DAY, + mechanism: "POST /api/admin/scan-news-now, not yet scheduled", + }, + { + id: "gpu-rental-prices", + label: "GPU rental prices (curated)", + file: "gpu-rental-prices.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: 10 * DAY, + mechanism: "n8n gpu-price-refresh, weekly Mon 06:00", + }, + { + id: "gpu-price-history", + label: "GPU price history (recorder)", + file: "gpu-price-history.json", + read: { kind: "series-max", field: "date" }, + expectedMaxAgeHours: 3 * DAY, + mechanism: "daily recorder ping to /api/gpu-prices/metrics, n8n 05:00", + }, + { + id: "hyperscaler-capex", + label: "Hyperscaler capex", + file: "hyperscaler-capex.json", + read: { kind: "envelope", fields: ["lastRefreshed"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated, quarterly with earnings", + }, + { + id: "inference-prices", + label: "Frontier inference prices", + file: "inference-prices.json", + read: { kind: "envelope", fields: ["asOf"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated from provider pricing pages", + }, + { + id: "frontier-models", + label: "Frontier model registry", + file: "frontier-models.json", + read: { kind: "envelope", fields: ["asOf"] }, + expectedMaxAgeHours: null, + mechanism: "hand-curated on model releases", + }, + { + // No per-row date field and no envelope: the ingester appends rows without + // stamping anything, so the file cannot report its own age. Reports + // "unknown" until the ingester stamps a timestamp. + id: "datacenters", + label: "Data center facilities", + file: "datacenters.json", + read: { kind: "none" }, + expectedMaxAgeHours: 2 * DAY, + mechanism: "in-process ingester every 6h (unreliable on Replit autoscale)", + }, +]; diff --git a/server/freshness.ts b/server/freshness.ts new file mode 100644 index 0000000..d36fc18 --- /dev/null +++ b/server/freshness.ts @@ -0,0 +1,195 @@ +// ─── Dataset freshness (pure) ──────────────────────────────────────────── +// +// Turns the declared registry plus raw file contents into a per-dataset verdict. +// Pure and injected on purpose (same discipline as indices.ts / clusters.ts / +// gpu-index.ts): no fs, no Date.now, no env. The route layer does the IO. +// +// The point of this module is not to display dates. It is to answer one +// question an unattended pipeline cannot answer about itself: has a mechanism +// stopped running? A flow on a homelab box dies quietly (reboot, expired token, +// n8n switched off) and the only symptom is a date that stops moving. That is +// exactly how the interconnection queue reached 77 days unnoticed. + +import { DATASET_REGISTRY, type DatasetSpec, type ReadStrategy } from "./freshness-registry.js"; + +export type FreshnessStatus = + /** Within its expected cadence. */ + | "ok" + /** Overdue, but under 2x. One missed run looks like this. */ + | "aging" + /** Past 2x its cadence. The mechanism has almost certainly stopped. */ + | "stale" + /** Hand-curated with no declared cadence. Reported, never alarms. */ + | "manual" + /** No readable timestamp, or the file is missing/malformed. Never alarms. */ + | "unknown"; + +export interface DatasetFreshness { + id: string; + label: string; + file: string; + /** ISO date the dataset reports for itself, or null when unreadable. */ + asOf: string | null; + ageHours: number | null; + expectedMaxAgeHours: number | null; + status: FreshnessStatus; + mechanism: string; + /** Why the status is what it is, in words, so an alert is actionable. */ + detail: string; +} + +export interface FreshnessReport { + generatedAt: string; + datasets: DatasetFreshness[]; + /** Datasets at "stale". These are what trip the deadman. */ + stale: string[]; + /** Datasets at "aging". Reported but do not trip the deadman on their own. */ + aging: string[]; + /** True when nothing is stale. The whole point of the check endpoint. */ + healthy: boolean; +} + +/** Raw file contents by dataset id. undefined means unreadable or missing. */ +export type FileContents = Record; + +const HOUR_MS = 60 * 60 * 1000; + +/** + * Parse a date that may be "2026-06-26" or a full ISO timestamp. + * + * A bare date is treated as the START of that UTC day, never the end. A dataset + * stamped "today" therefore reads as up to 24h old rather than 0h old, which + * keeps the age estimate conservative: this module should never report data as + * fresher than it can prove. + */ +export function parseStamp(value: unknown): number | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed) return null; + if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) { + const ms = Date.parse(`${trimmed}T00:00:00Z`); + return Number.isNaN(ms) ? null : ms; + } + const ms = Date.parse(trimmed); + return Number.isNaN(ms) ? null : ms; +} + +/** Pull the dataset's self-reported stamp out of its contents, per strategy. */ +export function readStamp(contents: unknown, strategy: ReadStrategy): string | null { + if (contents == null) return null; + + if (strategy.kind === "none") return null; + + if (strategy.kind === "envelope") { + if (typeof contents !== "object" || Array.isArray(contents)) return null; + const obj = contents as Record; + // First declared field that is actually present wins, so "did we look" + // takes precedence over "did anything change" where both are recorded. + for (const field of strategy.fields) { + const raw = obj[field]; + if (typeof raw === "string" && raw.trim()) return raw.trim(); + } + return null; + } + + // series-max: newest row wins. Rows without the field are ignored rather + // than treated as epoch, so one malformed row cannot make a live series + // look dead. + if (!Array.isArray(contents)) return null; + let best: string | null = null; + let bestMs = -Infinity; + for (const row of contents) { + if (row == null || typeof row !== "object") continue; + const raw = (row as Record)[strategy.field]; + if (typeof raw !== "string") continue; + const ms = parseStamp(raw); + if (ms == null) continue; + if (ms > bestMs) { + bestMs = ms; + best = raw.trim(); + } + } + return best; +} + +function classify( + spec: DatasetSpec, + asOf: string | null, + ageHours: number | null, +): { status: FreshnessStatus; detail: string } { + if (asOf == null || ageHours == null) { + return { + status: "unknown", + detail: + spec.read.kind === "none" + ? "file carries no timestamp, so its age cannot be checked" + : "timestamp missing or unparseable", + }; + } + + if (spec.expectedMaxAgeHours == null) { + return { + status: "manual", + detail: `hand-curated, ${Math.floor(ageHours / 24)}d old, no cadence to miss`, + }; + } + + const days = Math.floor(ageHours / 24); + const limit = spec.expectedMaxAgeHours; + + if (ageHours <= limit) { + return { status: "ok", detail: `${days}d old, within ${Math.round(limit / 24)}d` }; + } + if (ageHours <= limit * 2) { + return { + status: "aging", + detail: `${days}d old, past its ${Math.round(limit / 24)}d cadence (one missed run looks like this)`, + }; + } + return { + status: "stale", + detail: `${days}d old, more than double its ${Math.round(limit / 24)}d cadence: ${spec.mechanism} has probably stopped`, + }; +} + +/** + * Build the report. + * + * `nowMs` is injected so the tests are deterministic and so the caller owns the + * clock, matching how the rest of the server's pure modules are written. + */ +export function computeFreshness( + contentsById: FileContents, + nowMs: number, + registry: DatasetSpec[] = DATASET_REGISTRY, +): FreshnessReport { + const datasets: DatasetFreshness[] = registry.map((spec) => { + const asOf = readStamp(contentsById[spec.id], spec.read); + const stampMs = parseStamp(asOf); + const ageHours = stampMs == null ? null : Math.max(0, (nowMs - stampMs) / HOUR_MS); + const { status, detail } = classify(spec, asOf, ageHours); + + return { + id: spec.id, + label: spec.label, + file: spec.file, + asOf, + ageHours: ageHours == null ? null : Math.round(ageHours * 10) / 10, + expectedMaxAgeHours: spec.expectedMaxAgeHours, + status, + mechanism: spec.mechanism, + detail, + }; + }); + + const stale = datasets.filter((d) => d.status === "stale").map((d) => d.id); + const aging = datasets.filter((d) => d.status === "aging").map((d) => d.id); + + return { + generatedAt: new Date(nowMs).toISOString(), + datasets, + stale, + aging, + healthy: stale.length === 0, + }; +} diff --git a/server/routes.ts b/server/routes.ts index 1507e9d..b0dac64 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -48,6 +48,8 @@ import { composeBrief, renderBriefText, type BriefInput } from "./brief"; import { computeGpuEconomics, TRAINING_PRESETS } from "./gpu-economics"; import { readFrontierRegistry, summarizeFrontierRegistry } from "./frontier-models"; import { readInferencePrices, buildInferencePriceView } from "./inference-prices"; +import { computeFreshness, type FileContents } from "./freshness"; +import { DATASET_REGISTRY } from "./freshness-registry"; import { buildBuildoutTweet, buildGpuRentalTweet, @@ -846,10 +848,20 @@ async function scanNewsForBacklogUpdates(news: NewsItem[]): Promise<{ applied: n } } + // Two different questions, two different fields. + // + // lastRefreshed answers "when did a number here last change" and so only + // moves on a real change. lastChecked answers "when did anything last look + // at this", and must move on every run including a clean one. Conflating + // them is why a dataset that is watched daily and correctly unchanged would + // still look abandoned, and why nothing could tell that apart from the + // scanner never running at all. + const today = new Date().toISOString().slice(0, 10); + data.lastChecked = today; if (applied > 0) { - data.lastRefreshed = new Date().toISOString().slice(0, 10); - writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n"); + data.lastRefreshed = today; } + writeFileSync(filePath, JSON.stringify(data, null, 2) + "\n"); return { applied, flagged, checked }; } @@ -1027,7 +1039,14 @@ interface BacklogProject { notes?: string; } interface BacklogDataset { + /** When a value in this dataset last changed. */ lastRefreshed: string; + /** + * When the news scanner last ran against this dataset, changed or not. + * Optional because files written before the scanner started stamping it + * will not carry one; freshness falls back to lastRefreshed in that case. + */ + lastChecked?: string; headline: { trackedProjects: number; trackedCapacityGW: number; @@ -2188,6 +2207,56 @@ export async function registerRoutes( res.json(readGpuHistory()); }); + // ─── Dataset freshness (admin only) ────────────────────────────────────── + // + // Answers the one question an unattended pipeline cannot answer about + // itself: has a refresh mechanism stopped? Both routes are gated. Freshness + // is the floor, not a feature, and publishing a "how current are we" page + // advertises the bare minimum. + + /** Read every registered dataset off disk. Unreadable files stay absent. */ + function readRegisteredDatasets(): FileContents { + const contents: FileContents = {}; + for (const spec of DATASET_REGISTRY) { + try { + const path = join(process.cwd(), "server", "data", spec.file); + contents[spec.id] = JSON.parse(readFileSync(path, "utf-8")); + } catch { + // Leave it out: computeFreshness reports "unknown" rather than + // guessing, and a missing file must never read as fresh. + } + } + return contents; + } + + app.get("/api/admin/freshness", (req: Request, res) => { + if (!requireAdmin(req, res)) return; + res.json(computeFreshness(readRegisteredDatasets(), Date.now())); + }); + + // The deadman. An external pinger (cron-job.org, same account already firing + // the daily tweet) hits this with x-admin-key on a schedule and alerts on any + // non-2xx. It deliberately lives off the Jetson: a watchdog hosted on the box + // it watches dies with it, which is the failure that let the interconnection + // queue rot for 77 days. + // + // Path is /freshness/check and never /health so the platform cannot mistake + // a stale-data 503 for an unhealthy instance and start recycling it. + app.get("/api/admin/freshness/check", (req: Request, res) => { + if (!requireAdmin(req, res)) return; + const report = computeFreshness(readRegisteredDatasets(), Date.now()); + // "aging" is intentionally a 200: one missed run should not page anyone, + // or the alert stops meaning anything. + res.status(report.healthy ? 200 : 503).json({ + healthy: report.healthy, + stale: report.datasets + .filter((d) => d.status === "stale") + .map((d) => ({ id: d.id, asOf: d.asOf, detail: d.detail, mechanism: d.mechanism })), + aging: report.aging, + generatedAt: report.generatedAt, + }); + }); + app.post("/api/subscribe", subscribeLimiter, async (req: Request, res) => { try { const { email, intent, context } = req.body;