diff --git a/analytics/.env.example b/analytics/.env.example index eaf5f2f3a..06c58828b 100644 --- a/analytics/.env.example +++ b/analytics/.env.example @@ -1,3 +1,9 @@ # Scaleway S3 credentials (required) SCW_ACCESS_KEY=SCWXXXXXXXXXXXXXXXXX SCW_SECRET_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +# Wallet allowed to read GET /stats. A public address, not a secret — it is +# only ever compared against the signer recovered from the request. Locally you +# can set this to any address you hold a key for; in production it is the owner +# wallet, the same value scw_js uses. +OWNER_ETH_ADDRESS=0x0000000000000000000000000000000000000000 diff --git a/analytics/.gitignore b/analytics/.gitignore index ec2ca21b5..80140f637 100644 --- a/analytics/.gitignore +++ b/analytics/.gitignore @@ -6,3 +6,8 @@ dist/ *.log .DS_Store .serverless/ + +# Analytics exports (Umami dumps carry session ids, geo and device columns — +# never commit them; see notebooks/umami_backfill.py) +*.zip +*.csv diff --git a/analytics/README.md b/analytics/README.md index c2ac2cc4e..492353d5e 100644 --- a/analytics/README.md +++ b/analytics/README.md @@ -11,13 +11,28 @@ as JSON objects in Scaleway S3. No sessions, no visitor IDs, no PII. | **Aggregate counters** | One JSON object per site per UTC hour — no per-event rows | | **Per-page breakdown** | Hourly object also tracks a `pages` map, capped at 200 distinct paths | | **Anonymous writes** | `POST /hit` is unauthenticated and increment-only | -| **Private counters** | Stored objects carry no public-read ACL; reads are authorized separately | +| **Private counters** | Stored objects carry no public-read ACL; `GET /stats` gates reads on an owner wallet signature | | **Conditional writes** | Compare-and-swap via `storage.ts`'s `HitStorage`, retried up to 3x on conflict — real S3 (`@fretchen/s3-utils`) in production, local files for `npm run dev` | | **Input sanitisation** | `path` must start with `/`, safe URL-path characters only, max 200 chars | ## API -**Base URL** `https://analytics.fretchen.eu` +Two Scaleway functions. `analytics` serves both HTTP endpoints under one URL +with path-based routing (the repo convention — see `x402_facilitator`); +`rollup` is cron-only, because a scheduled invocation has no path to route on. +`npm run info` prints the URLs. + +| Function | Trigger | Auth | +| ----------- | ----------------------- | ----------------------- | +| `analytics` | `POST /hit` | none — anonymous writes | +| `analytics` | `GET /stats` | owner wallet signature | +| `rollup` | cron, Mondays 00:30 UTC | n/a | + +`analytics.ts` matches routes **exactly**, unlike `x402_facilitator`'s +`path.includes()`. That matters here because an anonymous write sits next to an +owner-gated read: no URL shape may reach `/stats` handling without its auth +check, and none may reach `/hit`'s unauthenticated write while looking like +`/stats`. ### `POST /hit` @@ -37,22 +52,132 @@ control: CORS is browser-enforced only, and the pageview beacon is a `sendBeacon` simple request that triggers no preflight at all. Write abuse is bounded by path validation and the 200-entry `pages` cap instead. +### `GET /stats` + +Serves the `/analytics` dashboard. Requires `Authorization: Bearer ` where the payload is `{address, signature, message}` and `message` is +`analytics-api:`, signed by `OWNER_ETH_ADDRESS`. Tokens expire after +five minutes; anything else returns `401`. + +Both halves of that scheme — building the message and verifying it — live in +`@fretchen/chain-utils` (`auth-protocol.ts`), shared with the Growth API. The +`analytics-api` prefix is what scopes a token to this service. + +```json +{ + "site": "fretchen.eu", + "from": "2025-08-12", + "to": "2026-08-11", + "days": { + "2026-08-10": { "hits": 240, "pages": { "/": 200 }, "source": "beacon" } + } +} +``` + +**No range parameter, by design.** The endpoint always returns the trailing +year, and `days` is sparse — a day with no traffic is absent, not a zero row. +Measured against real data that is ~25KB, about 3KB gzipped, so windowing the +response server-side bought nothing and cost a round trip per view. The +dashboard fetches once and slices client-side +(`website/utils/analyticsBuckets.ts` owns the totals, the top-pages list and +the daily/weekly/monthly bucketing). + +For a token from the terminal: + +```bash +OWNER_PRIVATE_KEY=0x... node --input-type=module -e ' +import { privateKeyToAccount } from "viem/accounts"; +const a = privateKeyToAccount(process.env.OWNER_PRIVATE_KEY); +const message = `analytics-api:${Math.floor(Date.now() / 1000)}`; +const signature = await a.signMessage({ message }); +const payload = { address: a.address, signature, message }; +console.log("Bearer " + Buffer.from(JSON.stringify(payload)).toString("base64")); +' +``` + ## Data model +Two layers. The endpoint only ever writes the first. + +**Write layer — one object per UTC hour:** + ``` counts/{site}/{YYYY-MM-DDTHH}.json +{ "hits": 42, "pages": { "/": 30, "/blog/foo/": 12 } } ``` -```json -{ "hits": 42, "pages": { "/": 30, "/blog/foo": 12 } } +**Read layer — one object per month, a per-day rollup:** + +``` +rollup/{site}/{YYYY-MM}.json +{ "site": "fretchen.eu", "month": "2026-03", "days": { + "2026-03-04": { "hits": 18, "pages": { "/": 9 }, "source": "umami" } } } ``` +The rollup layer exists because reads can't use the hourly one. `listObjects` +(`shared/s3-utils`) issues a single un-paginated ListObjectsV2 — max 1000 keys, +silently truncated — and hourly objects accrue at 8760/year; a 30-day window +would also mean 720 sequential GETs. Rollup keys are **computed** from a date +range rather than listed, so there is no ceiling and a month costs one GET. + +**Hourly buckets are the source of truth and are never deleted.** The `rollup` +cron is a compaction step, and `GET /stats` falls back to the hourly buckets +for recent days that aren't rolled up yet. That is what makes a weekly cadence +safe: a late or missed run changes what a query costs, never what it returns. + +Two things keep that fallback cheap, because rebuilding a day costs 24 GETs: + +- **Only days after the newest compacted one are probed.** Compaction runs in + date order, so everything up to that point is settled — present means + traffic, absent means none. Without this, every quiet day inside the window + would be re-read on every load. `HOURLY_FALLBACK_DAYS` (14) still caps it for + a cold start. +- **`/stats` writes back what it rebuilds.** A complete day reconstructed from + hourly buckets is stored via the same CAS `writeDay` the cron uses, so the + next load reads it as one rollup GET. Today is never written back — it is + still being counted. Write failures are swallowed: warming a cache must not + fail a read. + +In practice a warm load is **37 GETs** — 13 monthly rollups plus today's 24 +hours — regardless of which range the dashboard is showing. + +**Nothing deletes the hourly buckets, but they can stop being reachable.** The +bucket has no lifecycle configuration and no code path deletes under `counts/` +(the only S3 deletes in the repo are scoped to `channels/` and `growth-agent`), +so the objects are permanent — ~9MB/year, never listed, so no truncation limit +applies. What _is_ lossy is visibility: if the cron stops for longer than +`HOURLY_FALLBACK_DAYS`, `/stats` stops probing those days and renders them as +no-traffic while the data sits there intact. To pull such a gap back in, set +`ROLLUP_WINDOW_DAYS` wide enough to cover it and invoke `rollup` once. + +`source` is per day, not per month, because the changeover month holds both +kinds and they are not the same measurement: Umami filtered bots and +sessionised, the beacon counts every hydration and client-side navigation. + +**Path form.** `pageContext.urlPathname` is what the beacon sends, and Vike +derives it from `urlLogical` — set by `website/pages/+onBeforeRoute.ts`. So +recorded paths are in canonical `sitemap.xml` form: locale prefix stripped, +trailing slash on every non-root path, no query, no fragment. One consequence: +German pages are indistinguishable from their English counterparts, since the +beacon never sees the locale. + ## Reading the data Counters are **private** — no public-read ACL is set, so they are not -fetchable from the bucket URL without credentials. A future authorized -`GET /stats` endpoint (owner wallet signature, as in `scw_js/growth_api.ts`) -will serve reads. +fetchable from the bucket URL without credentials. Two ways in: + +- **`/analytics` on the website** — owner-gated dashboard over `GET /stats`, + with three views: 30 days by day, 90 days by week, one year by month. + Switching between them re-slices the single cached response rather than + refetching. Linked from the nav bar alongside `/growth`, but only once the + owner wallet is connected (`OwnerNavLinks` in `website/layouts/LayoutDefault.tsx`). +- **`notebooks/02_readout.ipynb`** — direct S3 reads, for anything the + dashboard doesn't show. + +Jan–Aug 2026 predates the counter and was backfilled from the Umami export — +see `notebooks/03_umami_backfill.ipynb`. Those days are marked +`"source": "umami"` and the dashboard greys them out: Umami filtered bots and +sessionised, so they are not comparable with the beacon's counts. ## Development @@ -64,18 +189,32 @@ npm run lint # eslint npm run build # tsup → dist/ ``` -### Local server +### Local servers ```bash -npm run dev # localhost:8086, file storage (notebooks/state/) — no credentials needed -npm run dev:live # localhost:8086, real S3 (needs analytics/.env) — pre-deploy sanity check +npm run dev # :8086 /hit + /stats, file storage (notebooks/state/) — no credentials +npm run dev:live # :8086 /hit + /stats, real S3 (needs analytics/.env) +npm run dev:rollup # :8088 rollup, file storage ``` -`npm run dev` writes counters to `notebooks/state/` instead of S3 -(`ANALYTICS_STORAGE=file` selects `FileHitStorage` over `S3HitStorage` in -`hit.ts`/`storage.ts`) — safe to hammer repeatedly with no risk to -production data. See `notebooks/01_smoke_test.ipynb` for a driver that -exercises either target. +One server for both endpoints, because they are one function. The +file-storage variants (`ANALYTICS_STORAGE=file` selects `FileHitStorage` over +`S3HitStorage`) read and write `notebooks/state/` instead of S3 — safe to +hammer repeatedly with no risk to production data. See +`notebooks/01_smoke_test.ipynb` for a driver that exercises either target. + +**Driving the website from a local function.** `website/.env` sets +`PUBLIC_ENV__ANALYTICS_URL=http://localhost:8086`; with that in place, `npm run +dev` in `website/` (port 3000, already on the CORS whitelist) plus `npm run +dev:live` here gives the real `/analytics` page over real data, and the beacon +lands locally too. `OWNER_ETH_ADDRESS` must be set in `analytics/.env` to the +wallet you connect with, or every `/stats` request comes back `401 Address +mismatch`. Comment the variable out of `website/.env` to go back to the +deployed function. + +Unlike `/hit`, `/stats` is genuinely gated by CORS — it is a `GET` carrying an +`Authorization` header, so the browser preflights it and an origin missing from +`ALLOWED_ORIGINS` blocks the page outright. `vike dev` serves on **3000**. ## Deployment @@ -84,11 +223,25 @@ Console. ```bash npm run deploy # serverless deploy +npm run info # per-function URLs ``` +After deploying, paste the `analytics` function's URL into the fallback in +`website/utils/analyticsApi.ts` — one string, shared by the beacon and the +dashboard. That fallback is what production uses: `.github/workflows/pages.yml` +sets no `PUBLIC_ENV__*` variables, so there is no CI mechanism to swap it. + +**Then delete the old `hit` function in the Scaleway Console.** It predates the +merge into `analytics` and `serverless deploy` does not remove functions that +have been dropped from the config, so it would otherwise keep running (and +keep collecting beacons from any stale client) forever. + ## Environment variables -| Variable | Scope | Description | -| ---------------- | ------ | ---------------------------- | -| `SCW_ACCESS_KEY` | secret | Scaleway API / S3 credential | -| `SCW_SECRET_KEY` | secret | Scaleway API / S3 credential | +| Variable | Scope | Description | +| -------------------- | ------ | ----------------------------------------------------------- | +| `SCW_ACCESS_KEY` | secret | Scaleway API / S3 credential | +| `SCW_SECRET_KEY` | secret | Scaleway API / S3 credential | +| `OWNER_ETH_ADDRESS` | env | Wallet allowed to read `GET /stats` | +| `ANALYTICS_STORAGE` | env | `file` selects local storage (dev only) | +| `ROLLUP_WINDOW_DAYS` | env | Days the cron compacts (default 14); widen to recover a gap | diff --git a/analytics/analytics.ts b/analytics/analytics.ts new file mode 100644 index 000000000..581ca5ee7 --- /dev/null +++ b/analytics/analytics.ts @@ -0,0 +1,96 @@ +/** + * The analytics function: one Scaleway function, two paths. + * + * POST /hit anonymous pageview counter (hit.ts) + * GET /stats owner-gated readout (stats.ts) + * + * Matches the repo convention — one HTTP function per package with path-based + * routing, as `x402_facilitator` does for `/verify`, `/settle`, `/supported`. + * The weekly compaction cron stays a separate function (`rollup.ts`): a cron + * invocation has no path to route on. + * + * **Exact matching, deliberately.** `x402_facilitator` can get away with + * `path.includes()` because all of its endpoints are unauthenticated. Here an + * anonymous write sits next to an owner-gated read, so a substring or prefix + * match would be a way to reach `/stats` handling through a `/hit`-shaped URL. + * Anything that isn't exactly `/hit` or `/stats` gets a 404 and never reaches a + * handler. + */ +import { handleHit } from "./hit.js"; +import { handleStats } from "./stats.js"; + +// Same whitelist as hit.ts/stats.ts, kept local rather than shared — each handler in this +// package owns its own CORS policy. Used only by the 404 fallback below; /hit and /stats +// each already echo the origin themselves once a route matches. +const ALLOWED_ORIGINS = ["https://www.fretchen.eu", "http://localhost:3000", "http://localhost:5173"]; + +export interface AnalyticsEvent { + httpMethod: string; + path?: string; + headers?: Record; + body?: string | Record; + queryStringParameters?: Record; +} + +export interface HandlerResponse { + statusCode: number; + headers: Record; + body: string; +} + +/** + * Collapses only what a router should: a query string (Scaleway passes it + * separately as `queryStringParameters`, but a proxy that folds it into `path` + * must not change which route matches) plus repeated and trailing slashes. + * + * No `..` resolution, no case folding, no percent-decoding — each of those + * would widen what counts as a match, and a path needing them is not one of + * ours. + */ +function normalizeRoute(rawPath: string | undefined): string { + const collapsed = (rawPath ?? "") + .split("?")[0] + .replace(/\/{2,}/g, "/") + .replace(/\/+$/, ""); + return collapsed === "" ? "/" : collapsed; +} + +export async function handle(event: AnalyticsEvent, context: unknown): Promise { + const route = normalizeRoute(event.path); + + if (route === "/hit") { + return handleHit(event, context); + } + if (route === "/stats") { + return handleStats(event, context); + } + + const origin = event.headers?.origin ?? event.headers?.Origin; + return { + statusCode: 404, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": ALLOWED_ORIGINS.includes(origin ?? "") ? origin! : "https://www.fretchen.eu", + }, + body: JSON.stringify({ error: "Not found. Use POST /hit or GET /stats" }), + }; +} + +/* Local dev server — only when run directly: npm run dev / npm run dev:live */ +const isEntrypoint = + typeof process.argv[1] === "string" && import.meta.url.endsWith(process.argv[1].replace(/.*\//, "")); + +if (isEntrypoint && process.env.NODE_ENV === "test") { + (async () => { + const dotenvModule = await import("dotenv"); + dotenvModule.config(); + + const scw = await import("@scaleway/serverless-functions"); + // AnalyticsEvent's stricter headers type (Record) isn't + // structurally assignable to the package's own looser Event type — same + // cast scw_js/llm_x402_cron.ts uses for the identical mismatch. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument + scw.serveHandler(handle as any, 8086); + console.log("analytics dev server on :8086 — POST /hit, GET /stats"); + })().catch((err) => console.error("Error starting local server", err)); +} diff --git a/analytics/buckets.ts b/analytics/buckets.ts new file mode 100644 index 000000000..0bf3be82a --- /dev/null +++ b/analytics/buckets.ts @@ -0,0 +1,267 @@ +/** + * The read layer: hourly buckets in, monthly rollups out. + * + * `hit.ts` writes one object per UTC hour. That layout can't be read directly + * for a dashboard — `listObjects` (`@fretchen/s3-utils`) issues a single + * un-paginated ListObjectsV2 (max 1000 keys, silently truncated) and hourly + * objects accrue at 8760/year, so a 30-day window would also be 720 GETs. + * + * So reads go through `rollup/{site}/{YYYY-MM}.json`: one object per month + * holding a per-day `{hits, pages, source}`. Every key here is **computed** + * from a date range, never listed, which is what removes the ceiling. + * + * Hourly buckets stay the source of truth and are never deleted — `stats.ts` + * falls back to them for recent days the weekly `rollup.ts` cron hasn't + * compacted yet, which is what makes that cadence safe. + */ +import type { HitStorage } from "./storage.js"; + +/** Caps a rolled-up day, mirroring `hit.ts`'s per-hour `MAX_PAGES_PER_BUCKET`. */ +const MAX_PAGES_PER_DAY = 500; + +const MAX_CAS_ATTEMPTS = 3; + +/** + * How far back `stats.ts` will reconstruct a day from its hourly buckets when + * it's missing from the rollups, and how far back the cron looks for holes to + * fill. Deliberately double the weekly cron cadence, so one missed run + * self-heals on the next; beyond it, a gap is treated as a no-traffic day. + */ +export const HOURLY_FALLBACK_DAYS = 14; + +export interface HourBucket { + hits: number; + pages: Record; +} + +export interface DayBucket { + hits: number; + pages: Record; + /** `"beacon"` for anything this service counted; `"umami"` for backfilled history. */ + source: string; +} + +export interface MonthRollup { + site: string; + month: string; + days: Record; +} + +// ===== Date helpers (UTC throughout — the hour keys are UTC, so these must be too) ===== + +/** `YYYY-MM-DD` for a Date, in UTC. */ +export function toIsoDate(date: Date): string { + return date.toISOString().slice(0, 10); +} + +function fromIsoDate(day: string): Date { + return new Date(`${day}T00:00:00Z`); +} + +export function addDays(day: string, delta: number): string { + const date = fromIsoDate(day); + date.setUTCDate(date.getUTCDate() + delta); + return toIsoDate(date); +} + +/** Every day from `from` to `to`, inclusive. */ +export function daysInRange(from: string, to: string): string[] { + const days: string[] = []; + for (let day = from; day <= to; day = addDays(day, 1)) { + days.push(day); + } + return days; +} + +/** Every `YYYY-MM` touched by the range, inclusive. */ +function monthsInRange(from: string, to: string): string[] { + const months: string[] = []; + let cursor = `${from.slice(0, 7)}-01`; + while (cursor.slice(0, 7) <= to.slice(0, 7)) { + months.push(cursor.slice(0, 7)); + cursor = addDays(cursor, 32).slice(0, 7) + "-01"; + } + return months; +} + +// ===== Keys ===== + +export function rollupKey(site: string, month: string): string { + return `rollup/${site}/${month}.json`; +} + +/** The 24 hour-bucket keys for one UTC day — computed, so no listing is needed. */ +function hourKeys(site: string, day: string): string[] { + return Array.from({ length: 24 }, (_, hour) => `counts/${site}/${day}T${String(hour).padStart(2, "0")}.json`); +} + +// ===== Reads ===== + +function mergePages(target: Record, source: Record): void { + for (const [path, count] of Object.entries(source)) { + target[path] = (target[path] ?? 0) + count; + } +} + +/** Sorts descending by count and truncates to `limit`. */ +function topPages(pages: Record, limit = MAX_PAGES_PER_DAY): Record { + return Object.fromEntries( + Object.entries(pages) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, limit), + ); +} + +/** + * Reconstructs one day from its 24 hourly buckets. Returns null when the day + * has no objects at all — a day with no traffic is absent, never a zero row, + * so the cron can tell "nothing happened" from "not compacted yet". + */ +export async function readDayFromHourly(store: HitStorage, site: string, day: string): Promise { + const results = await Promise.all(hourKeys(site, day).map((key) => store.getWithMeta(key))); + + let hits = 0; + let found = false; + const pages: Record = {}; + + for (const result of results) { + if (!result) { + continue; + } + found = true; + const bucket = JSON.parse(result.body) as HourBucket; + hits += bucket.hits ?? 0; + mergePages(pages, bucket.pages ?? {}); + } + + return found ? { hits, pages: topPages(pages), source: "beacon" } : null; +} + +async function readRollup(store: HitStorage, site: string, month: string): Promise { + const existing = await store.getWithMeta(rollupKey(site, month)); + return existing ? (JSON.parse(existing.body) as MonthRollup) : null; +} + +/** Every stored day across the months a range touches, keyed by `YYYY-MM-DD`. */ +export async function readRollupDays( + store: HitStorage, + site: string, + from: string, + to: string, +): Promise> { + const rollups = await Promise.all(monthsInRange(from, to).map((month) => readRollup(store, site, month))); + + const days: Record = {}; + for (const rollup of rollups) { + for (const [day, bucket] of Object.entries(rollup?.days ?? {})) { + if (day >= from && day <= to) { + days[day] = bucket; + } + } + } + return days; +} + +// ===== Writes ===== + +export type WriteDayResult = "written" | "exists" | "conflict"; + +/** + * Adds one day to its month rollup, compare-and-swap, same loop shape as + * `hit.ts`'s `incrementHit`. + * + * **An already-stored day always wins.** That makes the cron idempotent and, + * more importantly, means it can never overwrite a day backfilled from the + * Umami export (see `notebooks/umami_backfill.py`, whose `merge_into_existing` + * follows the same rule). + */ +export async function writeDay( + store: HitStorage, + site: string, + day: string, + bucket: DayBucket, +): Promise { + const month = day.slice(0, 7); + const key = rollupKey(site, month); + + for (let attempt = 1; attempt <= MAX_CAS_ATTEMPTS; attempt++) { + const existing = await store.getWithMeta(key); + const rollup: MonthRollup = existing ? (JSON.parse(existing.body) as MonthRollup) : { site, month, days: {} }; + + if (rollup.days[day]) { + return "exists"; + } + + rollup.days = Object.fromEntries(Object.entries({ ...rollup.days, [day]: bucket }).sort()); + + const result = await store.putConditional(key, JSON.stringify(rollup), { + ...(existing ? { ifMatch: existing.etag } : { ifNoneMatch: "*" }), + }); + if (result.ok) { + return "written"; + } + // 412: another writer won the race — retry from a fresh read. + } + + return "conflict"; +} + +export interface RebuildResult { + /** The days that had traffic, whether or not storing them succeeded. */ + rebuilt: Record; + written: string[]; + /** Complete days with no hourly objects at all — nothing happened, nothing stored. */ + empty: string[]; + failed: string[]; +} + +/** + * Rebuilds each candidate day from its 24 hourly buckets and compacts the + * complete ones into their monthly rollup. + * + * The single implementation behind both callers: `rollup.ts` runs it over a + * trailing window on a schedule, `stats.ts` runs it over whatever the dashboard + * asked for that isn't compacted yet. They differ only in how they pick + * candidates. + * + * `today` is rebuilt when asked for but never stored — it is still being + * written to, and compacting it would freeze a partial count. Storage failures + * land in `failed` rather than throwing: for `stats.ts` this is a cache warm, + * and a cache that didn't warm must not fail the read. + */ +export async function rebuildDays( + store: HitStorage, + site: string, + candidates: string[], + today: string, +): Promise { + const result: RebuildResult = { rebuilt: {}, written: [], empty: [], failed: [] }; + + const buckets = await Promise.all(candidates.map((day) => readDayFromHourly(store, site, day))); + + const writes = candidates.map(async (day, index) => { + const bucket = buckets[index]; + if (!bucket) { + result.empty.push(day); + return; + } + result.rebuilt[day] = bucket; + if (day === today) { + return; + } + try { + const outcome = await writeDay(store, site, day, bucket); + if (outcome === "written") { + result.written.push(day); + } else if (outcome === "conflict") { + result.failed.push(day); + } + // outcome === "exists": the cron or a backfill got there first — not a failure. + } catch { + result.failed.push(day); + } + }); + await Promise.all(writes); + + return result; +} diff --git a/analytics/hit.ts b/analytics/hit.ts index c0c06aa29..a1a5933ce 100644 --- a/analytics/hit.ts +++ b/analytics/hit.ts @@ -1,8 +1,10 @@ /** - * Anonymous pageview hit counter — one JSON object per site per UTC hour, - * incremented via a conditional-write compare-and-swap loop against storage. + * `POST /hit` — anonymous pageview counter. One JSON object per site per UTC + * hour, incremented via a conditional-write compare-and-swap loop. + * + * Routed from `analytics.ts`, which owns the function entrypoint. */ -import { type HitStorage, S3HitStorage, FileHitStorage } from "./storage.js"; +import { type HitStorage, defaultStorage } from "./storage.js"; export interface ScalewayEvent { httpMethod: string; @@ -21,11 +23,8 @@ const MAX_PATH_LENGTH = 200; const MAX_PAGES_PER_BUCKET = 200; // caps distinct paths tracked per hour bucket const MAX_CAS_ATTEMPTS = 3; -const ALLOWED_ORIGINS = ["https://www.fretchen.eu", "http://localhost:5173"]; - -// Local dev/sandbox (`npm run dev`) uses a file store with no credentials; -// production and `npm test` (both leave ANALYTICS_STORAGE unset) use real S3. -const storage: HitStorage = process.env.ANALYTICS_STORAGE === "file" ? new FileHitStorage() : new S3HitStorage(); +// `vike dev` serves on 3000; 5173 covers a plain `vite dev` fallback. +const ALLOWED_ORIGINS = ["https://www.fretchen.eu", "http://localhost:3000", "http://localhost:5173"]; /** * Note: this whitelist is a consistency/defence-in-depth measure, not a spam @@ -94,7 +93,7 @@ async function incrementHit(store: HitStorage, site: string, path: string): Prom } } -export async function handle(event: ScalewayEvent, _context: unknown): Promise { +export async function handleHit(event: ScalewayEvent, _context: unknown): Promise { const origin = event.headers?.origin ?? event.headers?.Origin; const corsHeaders = getCorsHeaders(origin); @@ -138,25 +137,7 @@ export async function handle(event: ScalewayEvent, _context: unknown): Promise { - const dotenvModule = await import("dotenv"); - dotenvModule.config(); - - const scw = await import("@scaleway/serverless-functions"); - // ScalewayEvent's stricter headers type (Record) isn't - // structurally assignable to the package's own looser Event type — same - // cast scw_js/llm_x402_cron.ts uses for the identical mismatch. - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument - scw.serveHandler(handle as any, 8086); - })().catch((err) => console.error("Error starting local server", err)); -} diff --git a/analytics/notebooks/02_readout.ipynb b/analytics/notebooks/02_readout.ipynb index 464a7b2ff..1a9140c81 100644 --- a/analytics/notebooks/02_readout.ipynb +++ b/analytics/notebooks/02_readout.ipynb @@ -8,10 +8,10 @@ "# Basic readout: aggregating `analytics` hourly buckets\n", "\n", "There is no `GET /stats` endpoint yet (see `scw_js/analytics-implementation-plan.md`\n", - "\u2014 reads will be owner-signature-gated, like the Growth API). This notebook is a\n", + "— reads will be owner-signature-gated, like the Growth API). This notebook is a\n", "throwaway prototype of the aggregation logic: list one **day's** hourly objects by\n", "prefix, sum `hits`, merge `pages`. It also doubles as a rough draft for the future\n", - "growth-agent rollup (`growth-agent/website_analytics.json`, scoped separately) \u2014\n", + "growth-agent rollup (`growth-agent/website_analytics.json`, scoped separately) —\n", "not production code, not wired into any endpoint.\n", "\n", "**Day-prefix, not the bare site prefix**: `list_keys`/`listObjects` is one\n", @@ -21,8 +21,21 @@ }, { "cell_type": "code", + "execution_count": 1, "id": "9f0926e2", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import os\n", "from datetime import date, datetime, timezone\n", @@ -32,14 +45,14 @@ "from storage import LocalStorage, S3Storage\n", "\n", "load_dotenv()" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 2, "id": "b0734202", "metadata": {}, + "outputs": [], "source": [ "def aggregate_day(storage, site: str, day: date) -> tuple[int, dict[str, int]]:\n", " \"\"\"Sum hits and merge pages across one UTC day's hourly buckets.\"\"\"\n", @@ -54,16 +67,14 @@ " for path, count in bucket.get(\"pages\", {}).items():\n", " pages[path] = pages.get(path, 0) + count\n", " return total_hits, pages" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", "id": "a169a1c8", "metadata": {}, "source": [ - "## 1. Fixture pass \u2014 validate the logic against known data\n", + "## 1. Fixture pass — validate the logic against known data\n", "\n", "No credentials needed. Writes three synthetic hourly buckets, then checks\n", "`aggregate_day` produces the hand-computed totals." @@ -71,8 +82,18 @@ }, { "cell_type": "code", + "execution_count": 3, "id": "3c11d37d", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11 {'/a': 4, '/b': 6, '/c': 1}\n" + ] + } + ], "source": [ "local = LocalStorage()\n", "fixture_day = date(2026, 8, 10)\n", @@ -86,9 +107,7 @@ "\n", "assert fixture_hits == 11 # 5 + 2 + 4\n", "assert fixture_pages == {\"/a\": 4, \"/b\": 6, \"/c\": 1}" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -103,8 +122,18 @@ }, { "cell_type": "code", + "execution_count": 4, "id": "65dbaa87", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-08-10: 3 hits across 2 distinct pages\n" + ] + } + ], "source": [ "s3 = S3Storage(\n", " access_key=os.environ[\"SCW_ACCESS_KEY\"],\n", @@ -114,28 +143,356 @@ "\n", "total_hits, pages = aggregate_day(s3, \"fretchen.eu\", today)\n", "print(f\"{today}: {total_hits} hits across {len(pages)} distinct pages\")" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 5, "id": "1679179a", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 2 /notebook-smoke-test\n", + " 1 /dev-live-credential-check\n" + ] + } + ], "source": [ "# top pages, most-hit first\n", "for path, count in sorted(pages.items(), key=lambda item: -item[1])[:10]:\n", " print(f\"{count:>6} {path}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 3. Range readout from monthly rollups\n", + "\n", + "Section 2 above reads one day out of the hourly buckets. That does not scale to\n", + "a dashboard: `listObjects` is a single un-paginated ListObjectsV2 (max 1000\n", + "keys) and a 30-day window would be 720 sequential GETs.\n", + "\n", + "The rollup layer fixes both. `rollup/{site}/{YYYY-MM}.json` holds a per-day\n", + "`{hits, pages, source}` for the whole month; the keys for a date range are\n", + "**computed**, never listed, so a range costs one GET per month spanned.\n", + "`03_umami_backfill.ipynb` populated Jan–Aug 2026 from the Umami export.\n", + "\n", + "This section is the read prototype for the eventual owner-gated `GET /stats`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import date, timedelta\n", + "\n", + "\n", + "def month_keys(site: str, start: date, end: date) -> list[str]:\n", + " \"\"\"Rollup keys spanning [start, end] — computed, so no listing, no truncation.\"\"\"\n", + " keys = []\n", + " cursor = start.replace(day=1)\n", + " while cursor <= end:\n", + " keys.append(f\"rollup/{site}/{cursor:%Y-%m}.json\")\n", + " cursor = (cursor.replace(day=28) + timedelta(days=4)).replace(day=1)\n", + " return keys\n", + "\n", + "\n", + "def read_range(storage, site: str, start: date, end: date) -> dict[str, dict]:\n", + " \"\"\"Per-day buckets for [start, end], inclusive, from the monthly rollups.\"\"\"\n", + " days: dict[str, dict] = {}\n", + " lo, hi = start.isoformat(), end.isoformat()\n", + " for key in month_keys(site, start, end):\n", + " rollup = storage.read(key)\n", + " if not rollup:\n", + " continue\n", + " for day, bucket in rollup.get(\"days\", {}).items():\n", + " if lo <= day <= hi:\n", + " days[day] = bucket\n", + " return dict(sorted(days.items()))\n", + "\n", + "\n", + "def merge_pages(days: dict[str, dict]) -> dict[str, int]:\n", + " pages: dict[str, int] = {}\n", + " for bucket in days.values():\n", + " for path, count in bucket.get(\"pages\", {}).items():\n", + " pages[path] = pages.get(path, 0) + count\n", + " return dict(sorted(pages.items(), key=lambda item: -item[1]))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fixture pass\n", + "\n", + "Same shape as section 1 — validate against hand-computed totals, no credentials.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "range readout OK: {'/a/': 5, '/': 1}\n" + ] + } ], - "execution_count": null, - "outputs": [] + "source": [ + "local.write(\n", + " \"rollup/fixture-site/2026-07.json\",\n", + " {\n", + " \"site\": \"fixture-site\",\n", + " \"month\": \"2026-07\",\n", + " \"days\": {\n", + " \"2026-07-30\": {\"hits\": 3, \"pages\": {\"/\": 3}, \"source\": \"umami\"},\n", + " \"2026-07-31\": {\"hits\": 2, \"pages\": {\"/a/\": 2}, \"source\": \"umami\"},\n", + " },\n", + " },\n", + ")\n", + "local.write(\n", + " \"rollup/fixture-site/2026-08.json\",\n", + " {\n", + " \"site\": \"fixture-site\",\n", + " \"month\": \"2026-08\",\n", + " \"days\": {\"2026-08-01\": {\"hits\": 4, \"pages\": {\"/\": 1, \"/a/\": 3}, \"source\": \"beacon\"}},\n", + " },\n", + ")\n", + "\n", + "spanning = read_range(local, \"fixture-site\", date(2026, 7, 31), date(2026, 8, 1))\n", + "assert list(spanning) == [\"2026-07-31\", \"2026-08-01\"] # crosses the month boundary\n", + "assert merge_pages(spanning) == {\"/a/\": 5, \"/\": 1}\n", + "print(\"range readout OK:\", merge_pages(spanning))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Real readout\n", + "\n", + "`source` is per-day on purpose. Umami filtered bots and sessionised; the beacon\n", + "counts every hydration and every client-side navigation, unfiltered. The two\n", + "are not the same measurement, so the seam is labelled rather than smoothed over.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "86 days with traffic, 649 hits\n", + "by source: {'umami': 649}\n" + ] + } + ], + "source": [ + "today = datetime.now(timezone.utc).date()\n", + "window = read_range(s3, \"fretchen.eu\", today - timedelta(days=90), today)\n", + "\n", + "by_source: dict[str, int] = {}\n", + "for bucket in window.values():\n", + " src = bucket.get(\"source\", \"unknown\")\n", + " by_source[src] = by_source.get(src, 0) + bucket[\"hits\"]\n", + "\n", + "print(f\"{len(window)} days with traffic, {sum(by_source.values())} hits\")\n", + "print(\"by source:\", by_source)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-05-12 1 █\n", + "2026-05-13 30 █████████████████████\n", + "2026-05-14 15 ███████████\n", + "2026-05-15 8 ██████\n", + "2026-05-16 1 █\n", + "2026-05-17 4 ███\n", + "2026-05-18 3 ██\n", + "2026-05-20 1 █\n", + "2026-05-21 10 ███████\n", + "2026-05-22 1 █\n", + "2026-05-23 7 █████\n", + "2026-05-24 1 █\n", + "2026-05-25 32 ███████████████████████\n", + "2026-05-26 10 ███████\n", + "2026-05-27 4 ███\n", + "2026-05-28 5 ████\n", + "2026-05-29 7 █████\n", + "2026-05-30 4 ███\n", + "2026-05-31 10 ███████\n", + "2026-06-01 3 ██\n", + "2026-06-02 4 ███\n", + "2026-06-03 21 ███████████████\n", + "2026-06-04 7 █████\n", + "2026-06-05 11 ████████\n", + "2026-06-06 21 ███████████████\n", + "2026-06-07 10 ███████\n", + "2026-06-08 4 ███\n", + "2026-06-09 5 ████\n", + "2026-06-10 4 ███\n", + "2026-06-11 8 ██████\n", + "2026-06-12 1 █\n", + "2026-06-13 12 █████████\n", + "2026-06-14 12 █████████\n", + "2026-06-15 17 ████████████\n", + "2026-06-16 4 ███\n", + "2026-06-17 4 ███\n", + "2026-06-18 5 ████\n", + "2026-06-19 9 ██████\n", + "2026-06-20 4 ███\n", + "2026-06-21 7 █████\n", + "2026-06-22 4 ███\n", + "2026-06-23 2 █\n", + "2026-06-25 6 ████\n", + "2026-06-26 4 ███\n", + "2026-06-27 5 ████\n", + "2026-06-28 4 ███\n", + "2026-06-29 1 █\n", + "2026-06-30 4 ███\n", + "2026-07-01 11 ████████\n", + "2026-07-02 9 ██████\n", + "2026-07-03 3 ██\n", + "2026-07-04 8 ██████\n", + "2026-07-05 2 █\n", + "2026-07-06 3 ██\n", + "2026-07-08 5 ████\n", + "2026-07-09 1 █\n", + "2026-07-11 1 █\n", + "2026-07-12 2 █\n", + "2026-07-13 6 ████\n", + "2026-07-14 20 ██████████████\n", + "2026-07-15 2 █\n", + "2026-07-16 12 █████████\n", + "2026-07-17 4 ███\n", + "2026-07-18 5 ████\n", + "2026-07-19 4 ███\n", + "2026-07-20 6 ████\n", + "2026-07-21 4 ███\n", + "2026-07-22 6 ████\n", + "2026-07-23 6 ████\n", + "2026-07-24 4 ███\n", + "2026-07-25 5 ████\n", + "2026-07-27 3 ██\n", + "2026-07-28 7 █████\n", + "2026-07-29 4 ███\n", + "2026-07-30 4 ███\n", + "2026-07-31 8 ██████\n", + "2026-08-01 8 ██████\n", + "2026-08-02 6 ████\n", + "2026-08-03 7 █████\n", + "2026-08-04 3 ██\n", + "2026-08-05 9 ██████\n", + "2026-08-06 2 █\n", + "2026-08-07 4 ███\n", + "2026-08-08 56 ████████████████████████████████████████\n", + "2026-08-09 39 ████████████████████████████\n", + "2026-08-10 3 ██\n" + ] + } + ], + "source": [ + "# daily sparkline over the window\n", + "peak = max((b[\"hits\"] for b in window.values()), default=1)\n", + "for day, bucket in window.items():\n", + " bar = \"█\" * max(1, round(bucket[\"hits\"] * 40 / peak))\n", + " print(f\"{day} {bucket['hits']:>4} {bar}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 216 /\n", + " 29 /blog/16/\n", + " 22 /amo/11/\n", + " 19 /amo/13/\n", + " 18 /blog/27/\n", + " 16 /blog/\n", + " 16 /amo/10/\n", + " 15 /x402/\n", + " 15 /blog/28/\n", + " 14 /blog/22/\n", + " 9 /quantum/\n", + " 9 /blog/12/\n", + " 9 /blog/29/\n", + " 8 /quantum/amo/18/\n", + " 7 /blog/26/\n", + " 7 /quantum/amo/\n", + " 7 /blog/13/\n", + " 7 /quantum/hardware/2/\n", + " 6 /quantum/qml/1/\n", + " 6 /amo/5/\n" + ] + } + ], + "source": [ + "# top pages across the window\n", + "for path, count in list(merge_pages(window).items())[:20]:\n", + " print(f\"{count:>6} {path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Today, from the hourly buckets\n", + "\n", + "The current day is not rolled up yet, so it still comes from `aggregate_day`\n", + "above — 24 GETs at most. A `GET /stats` endpoint would do exactly this: rollups\n", + "for whole days, hourly for today.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-08-10: 3 hits across 2 pages (live, not yet rolled up)\n" + ] + } + ], + "source": [ + "today_hits, today_pages = aggregate_day(s3, \"fretchen.eu\", today)\n", + "print(f\"{today}: {today_hits} hits across {len(today_pages)} pages (live, not yet rolled up)\")\n" + ] } ], "metadata": { "kernelspec": { - "display_name": "analytics-notebooks", + "display_name": "Python (growth-agent)", "language": "python", - "name": "analytics-notebooks" + "name": "growth-agent" }, "language_info": { "codemirror_mode": { @@ -147,7 +504,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.6" + "version": "3.14.5" } }, "nbformat": 4, diff --git a/analytics/notebooks/03_umami_backfill.ipynb b/analytics/notebooks/03_umami_backfill.ipynb new file mode 100644 index 000000000..cc9f45631 --- /dev/null +++ b/analytics/notebooks/03_umami_backfill.ipynb @@ -0,0 +1,401 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Backfill: Umami export → monthly rollups\n", + "\n", + "Umami is gone and the `analytics` counter starts from zero, so ~7 months of\n", + "history only exists in the cloud.umami.is data export. This notebook folds that\n", + "export into the **monthly rollup** layout the readout reads:\n", + "\n", + "```\n", + "rollup/{site}/{YYYY-MM}.json\n", + "```\n", + "\n", + "It never touches the hourly `counts/` prefix the live counter writes, and it is\n", + "idempotent — a day already present in a stored rollup is never overwritten.\n", + "\n", + "**Why monthly and not hourly.** `listObjects`\n", + "(`shared/s3-utils/src/index.ts`) issues a single un-paginated ListObjectsV2 —\n", + "max 1000 keys, silently truncated — and hourly objects accrue at 8760/year.\n", + "Replaying this export as hourly objects would add 874 keys and put the\n", + "`counts/fretchen.eu/` prefix within ~10% of that ceiling on day one. Rollup keys\n", + "are *computed* from a date range instead of listed, so there is no ceiling and a\n", + "month of traffic costs one GET.\n", + "\n", + "**Privacy.** `website_event.csv` carries `session_id`, city, region, country,\n", + "device, screen and language. `umami_backfill.py` projects to (day, path) → count\n", + "and drops everything else, so the rollups hold no more than the live counter\n", + "would have recorded. Delete the export when you're done — `analytics/.gitignore`\n", + "covers `*.zip`/`*.csv` so it can't be committed by accident.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "from storage import LocalStorage, S3Storage\n", + "from umami_backfill import (\n", + " SITE,\n", + " backfill,\n", + " normalize_path,\n", + " read_pageviews,\n", + " to_monthly_rollups,\n", + ")\n", + "\n", + "load_dotenv() # searches upward — finds ../.env (analytics/.env)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Unpack the export\n", + "\n", + "The zip is whatever cloud.umami.is named it (a website-id UUID). Only\n", + "`website_event.csv` is used: `event_data.csv` is custom-event payloads and\n", + "`session_data.csv` is empty.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['event_data.csv', 'session_data.csv', 'website_event.csv']\n" + ] + } + ], + "source": [ + "EXPORT_ZIP = Path(\"../95618d92-18ca-46f4-8a9e-b0556f54bab8.zip\")\n", + "EXPORT_DIR = Path(\"export\")\n", + "\n", + "with zipfile.ZipFile(EXPORT_ZIP) as zf:\n", + " zf.extractall(EXPORT_DIR)\n", + "\n", + "CSV_PATH = EXPORT_DIR / \"website_event.csv\"\n", + "print(sorted(p.name for p in EXPORT_DIR.iterdir()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Normalisation — the part that decides whether old and new data line up\n", + "\n", + "Umami logged the raw browser pathname. The beacon reports\n", + "`pageContext.urlPathname`, which Vike derives from `urlLogical` — set by\n", + "`website/pages/+onBeforeRoute.ts` via `extractLocale()`. That means the live\n", + "counter records the **canonical sitemap form**:\n", + "\n", + "| Rule | Umami logged | Beacon records |\n", + "| --- | --- | --- |\n", + "| trailing slash forced on non-root paths | `/amo/11` | `/amo/11/` |\n", + "| locale prefix stripped | `/de/blog/25/` | `/blog/25/` |\n", + "| fragment dropped | `/blog/22/#eine-interpretation` | `/blog/22/` |\n", + "| query dropped | `/x402/?ref=x` | `/x402/` |\n", + "\n", + "`generateSitemap.ts` applies the identical rule (`getLocaleInfo` strips the\n", + "locale, `filePathToUrlPath` forces the trailing slash), so normalising the\n", + "import to sitemap form makes the backfilled and live series directly\n", + "comparable — and both line up with `sitemap.xml`.\n", + "\n", + "**Consequence worth knowing:** German pages are *not* distinguishable. `/de/`\n", + "traffic folds into the English path both historically and going forward,\n", + "because the beacon never sees the locale. Fixing that means sending the locale\n", + "as a separate field — deliberately out of scope for now.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8 normalisation cases OK\n" + ] + } + ], + "source": [ + "checks = {\n", + " \"/\": \"/\",\n", + " \"/blog/25/\": \"/blog/25/\",\n", + " \"/amo/11\": \"/amo/11/\",\n", + " \"/de/\": \"/\",\n", + " \"/de/blog/25/\": \"/blog/25/\",\n", + " \"/blog/22/#user-content-fnref-1\": \"/blog/22/\",\n", + " \"/x402/?ref=x\": \"/x402/\",\n", + " \"not-a-path\": None,\n", + "}\n", + "\n", + "for raw, expected in checks.items():\n", + " got = normalize_path(raw)\n", + " assert got == expected, f\"{raw!r} -> {got!r}, expected {expected!r}\"\n", + "print(f\"{len(checks)} normalisation cases OK\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Dry run — aggregate, don't write anywhere real\n", + "\n", + "`LocalStorage()` writes to `state/`, the same directory `npm run dev`'s\n", + "`FileHitStorage` uses. Safe to re-run.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1612 pageviews -> 8 monthly objects\n", + "\n", + " 2026-01 29 days 199 hits\n", + " 2026-02 28 days 184 hits\n", + " 2026-03 31 days 315 hits\n", + " 2026-04 29 days 202 hits\n", + " 2026-05 30 days 217 hits\n", + " 2026-06 29 days 203 hits\n", + " 2026-07 28 days 155 hits\n", + " 2026-08 10 days 137 hits\n" + ] + } + ], + "source": [ + "pageviews = read_pageviews(CSV_PATH)\n", + "rollups = to_monthly_rollups(pageviews)\n", + "\n", + "print(f\"{len(pageviews)} pageviews -> {len(rollups)} monthly objects\\n\")\n", + "for month, rollup in sorted(rollups.items()):\n", + " hits = sum(day[\"hits\"] for day in rollup[\"days\"].values())\n", + " print(f\" {month} {len(rollup['days']):>2} days {hits:>5} hits\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rollup/fretchen.eu/2026-01.json 0 days, 199 hits (skipped 29 existing)\n", + "rollup/fretchen.eu/2026-02.json 0 days, 184 hits (skipped 28 existing)\n", + "rollup/fretchen.eu/2026-03.json 0 days, 315 hits (skipped 31 existing)\n", + "rollup/fretchen.eu/2026-04.json 0 days, 202 hits (skipped 29 existing)\n", + "rollup/fretchen.eu/2026-05.json 0 days, 217 hits (skipped 30 existing)\n", + "rollup/fretchen.eu/2026-06.json 0 days, 203 hits (skipped 29 existing)\n", + "rollup/fretchen.eu/2026-07.json 0 days, 155 hits (skipped 28 existing)\n", + "rollup/fretchen.eu/2026-08.json 0 days, 137 hits (skipped 10 existing)\n" + ] + } + ], + "source": [ + "dry = LocalStorage()\n", + "report = backfill(dry, CSV_PATH)\n", + "\n", + "for month, entry in report.items():\n", + " skipped = f\" (skipped {len(entry['days_skipped'])} existing)\" if entry[\"days_skipped\"] else \"\"\n", + " print(f\"{entry['key']} {entry['days_written']} days, {entry['hits']} hits{skipped}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Sanity-check the paths against `sitemap.xml`\n", + "\n", + "Anything not in the sitemap is either a page that has since moved or a real\n", + "404 someone hit. Both are legitimate history — this is a look-at-it check, not\n", + "a filter.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "114 canonical paths, 81 in sitemap, 35 unknown\n", + "\n", + " 30 /amo/11/\n", + " 26 /amo/13/\n", + " 24 /amo/10/\n", + " 7 /amo/8/\n", + " 6 /amo/12/\n", + " 6 /amo/5/\n", + " 3 /amo/16/\n", + " 3 /amo/2/\n", + " 3 /amo/18/\n", + " 2 /blog/11/7/\n", + " 2 /daniel-website/\n", + " 2 /AutoIncentive/\n", + " 2 /blog/quantum/\n", + " 2 /quantum/qml/1/qml102/\n", + " 1 /quantum/hardware/qhw2/\n", + " 1 /amo/6/\n", + " 1 /blog/9/6/\n", + " 1 /quantum/qml/2/1/\n", + " 1 /quantum/qml/qml101/\n", + " 1 /quantum/qml/3/2/\n" + ] + } + ], + "source": [ + "SITEMAP = Path(\"../../website/build/sitemap.xml\") # produced by `npm run build`\n", + "\n", + "sitemap_paths = set(re.findall(r\"https://www\\.fretchen\\.eu(.*?)\", SITEMAP.read_text()))\n", + "counts = {}\n", + "for _, path in pageviews:\n", + " counts[path] = counts.get(path, 0) + 1\n", + "\n", + "unknown = sorted(set(counts) - sitemap_paths, key=lambda p: -counts[p])\n", + "print(f\"{len(counts)} canonical paths, {len(sitemap_paths)} in sitemap, {len(unknown)} unknown\\n\")\n", + "for path in unknown[:20]:\n", + " print(f\"{counts[path]:>5} {path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. The real write\n", + "\n", + "Flip the flag. Eight new objects under a prefix nothing else writes to;\n", + "re-running is a no-op because existing days always win.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rollup/fretchen.eu/2026-01.json 29 days, 199 hits\n", + "rollup/fretchen.eu/2026-02.json 28 days, 184 hits\n", + "rollup/fretchen.eu/2026-03.json 31 days, 315 hits\n", + "rollup/fretchen.eu/2026-04.json 29 days, 202 hits\n", + "rollup/fretchen.eu/2026-05.json 30 days, 217 hits\n", + "rollup/fretchen.eu/2026-06.json 29 days, 203 hits\n", + "rollup/fretchen.eu/2026-07.json 28 days, 155 hits\n", + "rollup/fretchen.eu/2026-08.json 10 days, 137 hits\n" + ] + } + ], + "source": [ + "WRITE_TO_S3 = True # flip to True to write for real\n", + "\n", + "if WRITE_TO_S3:\n", + " s3 = S3Storage(\n", + " access_key=os.environ[\"SCW_ACCESS_KEY\"],\n", + " secret_key=os.environ[\"SCW_SECRET_KEY\"],\n", + " )\n", + " report = backfill(s3, CSV_PATH)\n", + " for month, entry in report.items():\n", + " print(f\"{entry['key']} {entry['days_written']} days, {entry['hits']} hits\")\n", + "else:\n", + " print(\"dry run only — nothing written to S3\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Read it back\n", + "\n", + "Confirms the objects landed and the totals survived the round trip. The full\n", + "range readout lives in `02_readout.ipynb`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-01: 29 days, 199 hits\n", + "2026-02: 28 days, 184 hits\n", + "2026-03: 31 days, 315 hits\n", + "2026-04: 29 days, 202 hits\n", + "2026-05: 30 days, 217 hits\n", + "2026-06: 29 days, 203 hits\n", + "2026-07: 28 days, 155 hits\n", + "2026-08: 10 days, 137 hits\n" + ] + } + ], + "source": [ + "if WRITE_TO_S3:\n", + " for month in sorted(rollups):\n", + " stored = s3.read(f\"rollup/{SITE}/{month}.json\")\n", + " hits = sum(day[\"hits\"] for day in stored[\"days\"].values())\n", + " print(f\"{month}: {len(stored['days'])} days, {hits} hits\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (growth-agent)", + "language": "python", + "name": "growth-agent" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/analytics/notebooks/README.md b/analytics/notebooks/README.md index 277933786..771b156f9 100644 --- a/analytics/notebooks/README.md +++ b/analytics/notebooks/README.md @@ -9,7 +9,16 @@ service, kept here rather than in the repo's general-purpose root | Notebook | What it does | | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `01_smoke_test.ipynb` | POSTs real hits to `/hit` (valid and invalid), then reads the resulting object back to confirm the write landed. `LOCAL` toggle at the top switches between `npm run dev` (file storage, no credentials) and the deployed service (real S3). | -| `02_readout.ipynb` | Aggregates one day's hourly buckets (sum `hits`, merge `pages`) — first against a local fixture to validate the logic, then against real data. Prototype only; not wired into any endpoint. | +| `02_readout.ipynb` | Aggregates one day's hourly buckets (sum `hits`, merge `pages`), then reads a date range out of the monthly rollups — each half validated against a local fixture first. Prototype for the eventual `GET /stats`; not wired into any endpoint. | +| `03_umami_backfill.ipynb` | One-off: folds the cloud.umami.is data export into monthly rollups, so the ~7 months predating the hit counter aren't lost. Dry-runs to `state/` by default; `WRITE_TO_S3` flips it to the real write. | + +`umami_backfill.py` holds that import's logic — CSV parsing, path +normalisation, monthly grouping, and an idempotent merge where days already +stored always win. It projects the export down to (day, path) → count and +drops `session_id`/geo/device entirely, so the rollups carry no more than the +live counter would have recorded. Delete the export when you're done; +`analytics/.gitignore` covers `*.zip`/`*.csv` so it can't be committed by +accident. `storage.py` provides `LocalStorage` (JSON files on disk — fixture-driven, no credentials needed) and `S3Storage` (real Scaleway Object Storage via diff --git a/analytics/notebooks/storage.py b/analytics/notebooks/storage.py index a850cc99f..66e5a9ad2 100644 --- a/analytics/notebooks/storage.py +++ b/analytics/notebooks/storage.py @@ -56,6 +56,14 @@ def read(self, key: str) -> dict | None: except self.s3.exceptions.NoSuchKey: return None + def write(self, key: str, data: dict) -> None: + self.s3.put_object( + Bucket=self.bucket, + Key=key, + Body=json.dumps(data).encode(), + ContentType="application/json", + ) + def list_keys(self, prefix: str = "") -> list[str]: response = self.s3.list_objects_v2(Bucket=self.bucket, Prefix=prefix) return [obj["Key"] for obj in response.get("Contents", [])] diff --git a/analytics/notebooks/umami_backfill.py b/analytics/notebooks/umami_backfill.py new file mode 100644 index 000000000..1d26d84c5 --- /dev/null +++ b/analytics/notebooks/umami_backfill.py @@ -0,0 +1,158 @@ +"""One-off backfill: Umami CSV export -> monthly rollup objects. + +Umami was replaced by the `analytics` hit counter, which starts from zero. This +module folds a Umami data export (`website_event.csv` from the cloud.umami.is +"Export data" button) into the same monthly rollup layout the readout reads: + + rollup/{site}/{YYYY-MM}.json + +Nothing here touches the hourly `counts/` prefix the live counter writes. + +**Why monthly, not hourly**: `listObjects` in `shared/s3-utils/src/index.ts` +issues a single un-paginated ListObjectsV2 (max 1000 keys, silently truncated), +and hourly objects accrue at 8760/year. Rollup keys are *computed* from a date +range, never listed, so the read path has no truncation limit and a month costs +one GET. + +**Privacy**: the export carries `session_id`, city, region, country, device, +screen and language columns. This module projects down to (day, path) -> count +and drops every other column on the floor — the rollups it writes contain no +more information than the live counter would have recorded. +""" + +import csv +import re +from collections import defaultdict +from pathlib import Path + +SITE = "fretchen.eu" + +# Mirrors analytics/hit.ts's sanitizePath, applied after normalisation. +SAFE_PATH = re.compile(r"^/[\w/.\-~%]*$") +MAX_PATH_LENGTH = 200 + +# website/locales/locales.ts +LOCALES = ("en", "de") + +# Umami's event_type: 1 = pageview, 2 = custom event. Only pageviews are hits. +PAGEVIEW = "1" + + +def normalize_path(raw: str) -> str | None: + """Normalise a Umami `url_path` to the site's canonical URL form. + + Canonical == what `sitemap.xml` emits, which is also exactly what the + beacon records, so backfilled and live days are directly comparable: + + - query and fragment dropped (Vike's `urlPathname` carries neither); + - locale prefix stripped (`website/pages/+onBeforeRoute.ts` routes on the + locale-less path, so `/de/blog/25/` is recorded as `/blog/25/`); + - trailing slash on every non-root path (GitHub Pages convention, forced + by `website/locales/extractLocale.ts` and by `generateSitemap.ts`). + + Returns None for anything the live endpoint would have rejected. + """ + path = raw.split("?")[0].split("#")[0].strip() + if not path.startswith("/"): + return None + + segments = path.split("/") + if len(segments) > 1 and segments[1] in LOCALES: + path = "/" + "/".join(segments[2:]) + + if path in ("", "//"): + path = "/" + if path != "/" and not path.endswith("/"): + path += "/" + + if len(path) > MAX_PATH_LENGTH or not SAFE_PATH.match(path): + return None + return path + + +def read_pageviews(csv_path: str | Path, hostname: str = f"www.{SITE}") -> list[tuple[str, str]]: + """Extract (day, canonical_path) pairs from a Umami `website_event.csv`. + + Drops custom events, other hostnames (localhost dev traffic), and any path + that fails normalisation. Returns one tuple per pageview — no dedup, no + sessionisation, matching how the beacon counts. + """ + pageviews: list[tuple[str, str]] = [] + with open(csv_path, newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + if row["event_type"] != PAGEVIEW or row["hostname"] != hostname: + continue + path = normalize_path(row["url_path"]) + if path is None: + continue + pageviews.append((row["created_at"][:10], path)) + return pageviews + + +def to_monthly_rollups(pageviews: list[tuple[str, str]], source: str = "umami") -> dict[str, dict]: + """Group (day, path) pairs into `{month_key: rollup_object}`. + + Rollup shape — a per-day version of the hourly bucket, plus provenance: + + {"site": ..., "month": "2026-03", "days": { + "2026-03-04": {"hits": 18, "pages": {"/": 9}, "source": "umami"}}} + + `source` is per *day*, not per month: the changeover month holds both + Umami-derived and beacon-derived days, and the two are not comparable + (Umami filtered bots and sessionised; the beacon counts every hydration). + """ + days: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + for day, path in pageviews: + days[day][path] += 1 + + rollups: dict[str, dict] = {} + for day in sorted(days): + month = day[:7] + rollup = rollups.setdefault(month, {"site": SITE, "month": month, "days": {}}) + pages = dict(sorted(days[day].items(), key=lambda item: (-item[1], item[0]))) + rollup["days"][day] = { + "hits": sum(pages.values()), + "pages": pages, + "source": source, + } + return rollups + + +def merge_into_existing(new: dict, existing: dict | None) -> tuple[dict, list[str]]: + """Merge a generated rollup into whatever is already stored for that month. + + Existing days always win — a re-run is idempotent and can never clobber + days the live counter already rolled up. Returns the merged object and the + list of days that were skipped because they already existed. + """ + if not existing: + return new, [] + + merged = {**existing, "days": dict(existing.get("days", {}))} + skipped = [day for day in new["days"] if day in merged["days"]] + for day, bucket in new["days"].items(): + merged["days"].setdefault(day, bucket) + merged["days"] = dict(sorted(merged["days"].items())) + return merged, skipped + + +def backfill(storage, csv_path: str | Path, source: str = "umami") -> dict: + """Write the rollups to `storage`. Returns a per-month report. + + `storage` is a `LocalStorage` (dry run) or `S3Storage` (real) from + `storage.py`. + """ + rollups = to_monthly_rollups(read_pageviews(csv_path), source=source) + + report: dict[str, dict] = {} + for month, rollup in sorted(rollups.items()): + key = f"rollup/{SITE}/{month}.json" + merged, skipped = merge_into_existing(rollup, storage.read(key)) + storage.write(key, merged) + report[month] = { + "key": key, + "days_written": len(rollup["days"]) - len(skipped), + "days_skipped": skipped, + "hits": sum(day["hits"] for day in rollup["days"].values()), + } + return report diff --git a/analytics/package-lock.json b/analytics/package-lock.json index ea26807ae..fd06f14c8 100644 --- a/analytics/package-lock.json +++ b/analytics/package-lock.json @@ -9,7 +9,9 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@fretchen/s3-utils": "file:../shared/s3-utils" + "@fretchen/chain-utils": "file:../shared/chain-utils", + "@fretchen/s3-utils": "file:../shared/s3-utils", + "viem": "^2.54.2" }, "devDependencies": { "@eslint/js": "^10.0.0", @@ -30,6 +32,27 @@ "node": ">=18.0.0" } }, + "../shared/chain-utils": { + "name": "@fretchen/chain-utils", + "version": "1.0.0", + "devDependencies": { + "@eslint/js": "^10.0.0", + "@vitest/coverage-v8": "^4.1.8", + "eslint": "^10.0.0", + "prettier": "^3.8.1", + "typescript": "^5.0.0", + "typescript-eslint": "^8.60.0", + "viem": "^2.54.2", + "vitest": "^4.1.10" + }, + "optionalDependencies": { + "@rollup/rollup-linux-x64-gnu": "*", + "@rollup/rollup-linux-x64-musl": "*" + }, + "peerDependencies": { + "viem": "^2.0.0" + } + }, "../shared/s3-utils": { "name": "@fretchen/s3-utils", "version": "1.0.0", @@ -44,6 +67,12 @@ "vitest": "^4.1.8" } }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -772,6 +801,10 @@ "fastify-plugin": "^4.0.0" } }, + "node_modules/@fretchen/chain-utils": { + "resolved": "../shared/chain-utils", + "link": true + }, "node_modules/@fretchen/s3-utils": { "resolved": "../shared/s3-utils", "link": true @@ -1033,6 +1066,45 @@ "node": "^22.20 || ^24.12 || >=25" } }, + "node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/curves": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", + "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@oxc-project/types": { "version": "0.143.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.143.0.tgz", @@ -1807,6 +1879,42 @@ "dev": true, "license": "MIT" }, + "node_modules/@scure/base": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", + "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", + "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", + "license": "MIT", + "dependencies": { + "@noble/curves": "~1.9.0", + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", + "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@serverless/utils": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/@serverless/utils/-/utils-6.15.0.tgz", @@ -2378,6 +2486,27 @@ "dev": true, "license": "ISC" }, + "node_modules/abitype": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", + "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -4207,6 +4336,12 @@ "node": ">=6" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -5384,6 +5519,21 @@ "dev": true, "license": "ISC" }, + "node_modules/isows": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.7.tgz", + "integrity": "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", @@ -6454,6 +6604,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ox": { + "version": "0.14.33", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.33.tgz", + "integrity": "sha512-rooA/4o7bBof4Ge2VH/eovfNPb/AEEYyrNj03wggc55g5HZD8Pjs/OeWhttgjic3dDcqn0r29bDuvQEdTiUemQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.11.0", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "1.9.1", + "@noble/hashes": "^1.8.0", + "@scure/bip32": "^1.7.0", + "@scure/bip39": "^1.6.0", + "abitype": "^1.2.3", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/p-cancelable": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", @@ -8633,7 +8813,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -8730,6 +8910,36 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/viem": { + "version": "2.55.13", + "resolved": "https://registry.npmjs.org/viem/-/viem-2.55.13.tgz", + "integrity": "sha512-Rt1NAsdtTdvJgqaCxsv2TuO55xv2d2dKEuRuzrg34xMGxvTSFOX0iIFvEfymPvh+fwN2tbgEU2JwN0YHOVM+Bg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@noble/curves": "1.9.1", + "@noble/hashes": "1.8.0", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "abitype": "1.2.3", + "isows": "1.0.7", + "ox": "0.14.33", + "ws": "8.21.0" + }, + "peerDependencies": { + "typescript": ">=5.0.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/vite": { "version": "8.2.1", "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.1.tgz", @@ -9047,6 +9257,27 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/analytics/package.json b/analytics/package.json index fcca3a30a..478d980f0 100644 --- a/analytics/package.json +++ b/analytics/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Anonymous pageview hit counter, stored in Scaleway S3", "type": "module", - "main": "dist/hit.js", + "main": "dist/analytics.js", "scripts": { "build": "tsup", "test": "vitest run", @@ -15,12 +15,14 @@ "format:check": "prettier --check \"**/*.{ts,js,json,md}\"", "typecheck": "tsc --noEmit", "check": "npm run lint && npm run format:check && npm run typecheck && npm run test:coverage", - "dev": "NODE_ENV=test ANALYTICS_STORAGE=file npx tsx hit.ts", - "dev:live": "NODE_ENV=test npx tsx hit.ts", + "dev": "NODE_ENV=test ANALYTICS_STORAGE=file npx tsx analytics.ts", + "dev:live": "NODE_ENV=test npx tsx analytics.ts", + "dev:rollup": "NODE_ENV=test ANALYTICS_STORAGE=file npx tsx rollup.ts", "predeploy": "npm run build", "deploy": "serverless deploy", "info": "serverless info", - "logs": "serverless logs -f hit", + "logs": "serverless logs -f analytics", + "logs:rollup": "serverless logs -f rollup", "remove": "serverless remove" }, "keywords": [ @@ -32,7 +34,9 @@ "author": "", "license": "MIT", "dependencies": { - "@fretchen/s3-utils": "file:../shared/s3-utils" + "@fretchen/chain-utils": "file:../shared/chain-utils", + "@fretchen/s3-utils": "file:../shared/s3-utils", + "viem": "^2.54.2" }, "devDependencies": { "@eslint/js": "^10.0.0", diff --git a/analytics/rollup.ts b/analytics/rollup.ts new file mode 100644 index 000000000..ef1580882 --- /dev/null +++ b/analytics/rollup.ts @@ -0,0 +1,101 @@ +/** + * Weekly compaction cron: folds complete days of hourly buckets into their + * monthly rollup. + * + * This is a *compaction* step, not the source of truth. `stats.ts` reads recent + * days straight from the hourly buckets when they aren't rolled up yet, so the + * cadence only affects read cost — a late run changes what the dashboard costs + * to render, never what it shows. + * + * It is still load-bearing, though: `stats.ts` only looks back + * `HOURLY_FALLBACK_DAYS`, so days that go uncompacted for longer drop out of + * the dashboard even though their hourly objects are still in the bucket. + * `ROLLUP_WINDOW_DAYS` widens the window for exactly that recovery — set it and + * invoke the function once to pull a gap back in. + */ +import { type HitStorage, defaultStorage } from "./storage.js"; +import { HOURLY_FALLBACK_DAYS, addDays, daysInRange, readRollupDays, rebuildDays, toIsoDate } from "./buckets.js"; + +const SITE = "fretchen.eu"; + +/** + * Days back to compact. Defaults to double the weekly cadence so one missed run + * self-heals on the next; raise it to recover a longer gap. + */ +function windowDays(): number { + const configured = Number.parseInt(process.env.ROLLUP_WINDOW_DAYS ?? "", 10); + return Number.isFinite(configured) && configured > 0 ? configured : HOURLY_FALLBACK_DAYS; +} + +export interface RollupSummary { + from: string; + to: string; + written: string[]; + empty: string[]; + skipped: string[]; + failed: string[]; +} + +/** + * Compacts every complete day in the window that isn't in a rollup yet. Today + * is excluded — it is still being written to. + */ +export async function rollupRecentDays( + store: HitStorage, + site: string, + now: Date = new Date(), + days: number = windowDays(), +): Promise { + const to = addDays(toIsoDate(now), -1); // yesterday: the most recent complete UTC day + const from = addDays(to, -(days - 1)); + + const compacted = await readRollupDays(store, site, from, to); + const all = daysInRange(from, to); + const candidates = all.filter((day) => !compacted[day]); + + const { written, empty, failed } = await rebuildDays(store, site, candidates, toIsoDate(now)); + + return { + from, + to, + written, + empty, + skipped: all.filter((day) => compacted[day]), + failed, + }; +} + +export async function handle( + _event: unknown, + _context: unknown, +): Promise<{ statusCode: number; headers: Record; body: string }> { + const headers = { "Content-Type": "application/json" }; + + try { + const summary = await rollupRecentDays(defaultStorage, SITE); + console.log("rollup complete", JSON.stringify(summary)); + return { + statusCode: summary.failed.length > 0 ? 500 : 200, + headers, + body: JSON.stringify(summary), + }; + } catch (err) { + console.error("rollup failed", err); + return { statusCode: 500, headers, body: JSON.stringify({ error: (err as Error).message }) }; + } +} + +/* Local dev server — only when run directly: npm run dev:rollup */ +const isEntrypoint = + typeof process.argv[1] === "string" && import.meta.url.endsWith(process.argv[1].replace(/.*\//, "")); + +if (isEntrypoint && process.env.NODE_ENV === "test") { + (async () => { + const dotenvModule = await import("dotenv"); + dotenvModule.config(); + + const scw = await import("@scaleway/serverless-functions"); + + scw.serveHandler(handle, 8088); + })().catch((err) => console.error("Error starting local server", err)); +} diff --git a/analytics/serverless.yml b/analytics/serverless.yml index 5389224ff..1fa00e3fa 100644 --- a/analytics/serverless.yml +++ b/analytics/serverless.yml @@ -8,6 +8,9 @@ provider: runtime: node22 env: NODE_ENV: production + # Public address, not a secret — only ever compared against a recovered + # signer in stats.ts. Same value scw_js's growth API uses. + OWNER_ETH_ADDRESS: ${env:OWNER_ETH_ADDRESS} secret: SCW_SECRET_KEY: ${env:SCW_SECRET_KEY} SCW_ACCESS_KEY: ${env:SCW_ACCESS_KEY} @@ -29,9 +32,21 @@ package: - "!dist/**/*.map" functions: - hit: - handler: dist/hit.handle - description: "Pageview hit counter - anonymous, stored in S3" - # No custom domain for now — deploys to Scaleway's auto-generated function - # URL. Add `custom_domains: [analytics.fretchen.eu]` back once DNS/cert - # is set up. + # One HTTP function, path-routed (POST /hit, GET /stats) — same shape as + # x402_facilitator. No custom domain for now, so it deploys to Scaleway's + # auto-generated URL; add `custom_domains: [analytics.fretchen.eu]` once + # DNS/cert is set up. + analytics: + handler: dist/analytics.handle + description: "Pageview counter (POST /hit) and owner-gated readout (GET /stats)" + rollup: + handler: dist/rollup.handle + description: "Weekly compaction of hourly hit buckets into monthly rollups" + events: + - schedule: + # Scaleway's cron validator (serverless-scaleway-functions/shared/validate.js) + # rejects the standard */N step syntax, so keep every field literal. + # Mondays 00:30 UTC. Cadence is a cost choice only — stats.ts reads + # recent days straight from the hourly buckets, so a late or missed + # run never changes what the dashboard shows. + rate: "30 0 * * 1" diff --git a/analytics/stats.ts b/analytics/stats.ts new file mode 100644 index 000000000..d3c86bd86 --- /dev/null +++ b/analytics/stats.ts @@ -0,0 +1,148 @@ +/** + * `GET /stats` — the owner's read endpoint for the dashboard. Routed from + * `analytics.ts`, which owns the function entrypoint. + * + * Counters are private (no public-read ACL), so this is the only way to see + * them without S3 credentials. Gated by an EIP-191 owner signature, the same + * bearer scheme the Growth API uses — see `auth.ts`. + * + * **Always serves the trailing year, unwindowed.** A full year of daily counts + * with per-day page maps measures ~15KB (3KB gzipped) against real data, so + * range parameters were never worth their complexity: the dashboard fetches + * once and slices client-side. Days are returned as a sparse map — the client + * walks a calendar anyway to build weekly/monthly buckets, so it fills the + * gaps itself. + * + * Reads in two passes: monthly rollups for the range, then hourly buckets for + * any recent day the weekly cron hasn't compacted yet. The second pass is what + * decouples this endpoint from the cron's cadence. + */ +import { type HitStorage, defaultStorage } from "./storage.js"; +import { parseBearerToken, verifySignedMessage } from "@fretchen/chain-utils"; +import { + HOURLY_FALLBACK_DAYS, + addDays, + daysInRange, + readRollupDays, + rebuildDays, + toIsoDate, + type DayBucket, +} from "./buckets.js"; + +const SITE = "fretchen.eu"; + +/** Scopes a token to this service — one minted for the growth API won't work here. */ +const AUTH_PREFIX = "analytics-api"; + +/** The window served, in days. One year, always. */ +const WINDOW_DAYS = 365; + +// `vike dev` serves on 3000 (matching comment_service's list); 5173 covers a +// plain `vite dev` fallback. Unlike hit.ts's whitelist this one actually gates +// the browser: /stats is a preflighted GET with an Authorization header, so an +// origin missing here fails the preflight and the dashboard shows nothing. +const ALLOWED_ORIGINS = ["https://www.fretchen.eu", "http://localhost:3000", "http://localhost:5173"]; + +export interface StatsEvent { + httpMethod: string; + headers?: Record; + queryStringParameters?: Record; +} + +export interface StatsResponse { + site: string; + from: string; + to: string; + /** Sparse — days with no traffic are absent, not zero rows. */ + days: Record; +} + +/** Same whitelist as `hit.ts`, plus the Authorization header this endpoint needs. */ +function getCorsHeaders(origin?: string): Record { + const allowedOrigin = ALLOWED_ORIGINS.includes(origin ?? "") ? origin! : "https://www.fretchen.eu"; + return { + "Access-Control-Allow-Origin": allowedOrigin, + "Access-Control-Allow-Headers": "Content-Type, Authorization", + "Access-Control-Allow-Methods": "GET, OPTIONS", + "Content-Type": "application/json", + }; +} + +/** + * Collects the range, preferring rollups and falling back to hourly buckets + * for recent days that aren't compacted yet. + * + * **Which days get probed.** Compaction runs in date order, so every day up to + * the newest one present in the rollups is settled: present means traffic, + * absent means none. Only days after that need their 24 hourly keys read — + * without this the fallback would re-probe every no-traffic day on every load, + * and a quiet week would cost 168 GETs forever. `HOURLY_FALLBACK_DAYS` still + * caps it, for the cold-start case where the rollups are empty. + * + * Whatever it has to rebuild, it also compacts — see `rebuildDays`. Without + * that, every dashboard load would repeat 24 GETs per day the cron hasn't + * reached yet. + */ +export async function collectRange( + store: HitStorage, + site: string, + from: string, + to: string, + now: Date = new Date(), +): Promise> { + const days = await readRollupDays(store, site, from, to); + + const today = toIsoDate(now); + const windowStart = addDays(today, -(HOURLY_FALLBACK_DAYS - 1)); + const newestCompacted = Object.keys(days).sort().at(-1); + const probeFrom = newestCompacted && newestCompacted >= windowStart ? addDays(newestCompacted, 1) : windowStart; + + const candidates = daysInRange(from, to).filter((day) => !days[day] && day >= probeFrom); + const { rebuilt } = await rebuildDays(store, site, candidates, today); + + return { ...days, ...rebuilt }; +} + +export async function buildStats(store: HitStorage, site: string, now: Date = new Date()): Promise { + const to = toIsoDate(now); + const from = addDays(to, -(WINDOW_DAYS - 1)); + + return { site, from, to, days: await collectRange(store, site, from, to, now) }; +} + +export async function handleStats( + event: StatsEvent, + _context: unknown, +): Promise<{ statusCode: number; headers: Record; body: string }> { + const origin = event.headers?.origin ?? event.headers?.Origin; + const headers = getCorsHeaders(origin); + + if (event.httpMethod === "OPTIONS") { + return { statusCode: 200, headers, body: "" }; + } + + if (event.httpMethod !== "GET") { + return { statusCode: 405, headers, body: JSON.stringify({ error: "Method not allowed" }) }; + } + + const token = parseBearerToken(event.headers?.authorization ?? event.headers?.Authorization); + if (!token) { + return { statusCode: 401, headers, body: JSON.stringify({ error: "Missing or invalid Authorization header" }) }; + } + + const ownerAddress = process.env.OWNER_ETH_ADDRESS; + const authError = ownerAddress + ? await verifySignedMessage(token.address, token.signature, token.message, AUTH_PREFIX, ownerAddress) + : "Owner address not configured"; + if (authError) { + return { statusCode: 401, headers, body: JSON.stringify({ error: authError }) }; + } + + try { + const stats = await buildStats(defaultStorage, SITE); + return { statusCode: 200, headers, body: JSON.stringify(stats) }; + } catch (err) { + console.error("stats failed", err); + return { statusCode: 500, headers, body: JSON.stringify({ error: "Internal server error" }) }; + } +} diff --git a/analytics/storage.ts b/analytics/storage.ts index 3ce4f7786..80ac4c8ef 100644 --- a/analytics/storage.ts +++ b/analytics/storage.ts @@ -78,3 +78,14 @@ export class FileHitStorage implements HitStorage { return { ok: true, etag: this.etagOf(body) }; } } + +/** + * The one store the deployed function uses. `ANALYTICS_STORAGE=file` selects + * the local file store (no credentials); production and `npm test` both leave + * it unset and get real S3. + * + * Module-level, so `/hit` and `/stats` — now served by the same function — + * share a single instance instead of constructing one each. + */ +export const defaultStorage: HitStorage = + process.env.ANALYTICS_STORAGE === "file" ? new FileHitStorage() : new S3HitStorage(); diff --git a/analytics/test/analytics.test.ts b/analytics/test/analytics.test.ts new file mode 100644 index 000000000..4ce72bec4 --- /dev/null +++ b/analytics/test/analytics.test.ts @@ -0,0 +1,141 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const { mockGetS3ObjectWithMeta, mockPutS3ObjectConditional } = vi.hoisted(() => ({ + mockGetS3ObjectWithMeta: vi.fn(), + mockPutS3ObjectConditional: vi.fn(), +})); + +vi.mock("@fretchen/s3-utils", () => ({ + getS3ObjectWithMeta: mockGetS3ObjectWithMeta, + putS3ObjectConditional: mockPutS3ObjectConditional, +})); + +import { handle, type AnalyticsEvent } from "../analytics.js"; + +const OWNER = "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"; + +function makeEvent(overrides: Partial = {}): AnalyticsEvent { + return { httpMethod: "GET", ...overrides }; +} + +describe("analytics router", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetS3ObjectWithMeta.mockResolvedValue(null); + mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"e"' }); + process.env.OWNER_ETH_ADDRESS = OWNER; + }); + + afterEach(() => { + delete process.env.OWNER_ETH_ADDRESS; + }); + + it("routes POST /hit to the counter", async () => { + const res = await handle( + makeEvent({ + httpMethod: "POST", + path: "/hit", + body: JSON.stringify({ site: "fretchen.eu", path: "/blog/" }), + }), + {}, + ); + + expect(res.statusCode).toBe(204); + expect(mockPutS3ObjectConditional).toHaveBeenCalled(); + }); + + it("routes GET /stats to the readout, which demands a token", async () => { + const res = await handle(makeEvent({ path: "/stats" }), {}); + + expect(res.statusCode).toBe(401); + expect(JSON.parse(res.body).error).toMatch(/Authorization/i); + }); + + it("tolerates trailing and duplicated slashes", async () => { + for (const path of ["/hit/", "//hit", "/hit//"]) { + const res = await handle( + makeEvent({ httpMethod: "POST", path, body: JSON.stringify({ site: "fretchen.eu", path: "/blog/" }) }), + {}, + ); + expect(res.statusCode, path).toBe(204); + } + }); + + it("404s anything that is not a known route", async () => { + for (const path of ["", "/", "/unknown", "/hits", "/stat", "/hit/extra", "/api/hit"]) { + const res = await handle(makeEvent({ path }), {}); + expect(res.statusCode, path).toBe(404); + } + }); + + it("echoes a whitelisted dev origin on a 404, instead of always the production origin", async () => { + const res = await handle(makeEvent({ path: "/unknown", headers: { origin: "http://localhost:3000" } }), {}); + expect(res.statusCode).toBe(404); + expect(res.headers["Access-Control-Allow-Origin"]).toBe("http://localhost:3000"); + }); + + it("falls back to the production origin on a 404 from an unknown origin", async () => { + const res = await handle(makeEvent({ path: "/unknown", headers: { origin: "https://evil.com" } }), {}); + expect(res.headers["Access-Control-Allow-Origin"]).toBe("https://www.fretchen.eu"); + }); + + // Scaleway passes the query separately, but a proxy that folds it into `path` + // must not change which route matches. + it("ignores a query string when matching", async () => { + const res = await handle(makeEvent({ path: "/stats?days=7" }), {}); + expect(res.statusCode).toBe(401); // reached /stats, then failed auth + }); + + // The reason this router matches exactly instead of using path.includes(): + // /hit is an anonymous write sitting next to an owner-gated read, so no + // URL shape may reach /stats handling without passing its auth check, and + // none may reach /hit's unauthenticated write while looking like /stats. + it("never serves stats data without a token, whatever the path is spelled like", async () => { + const spellings = [ + "/stats", + "/stats/", + "//stats", + "/stats?days=7", + "/hit/../stats", + "/STATS", + "/stats/../hit", + "/hit/stats", + "/stats/extra", + ]; + + for (const path of spellings) { + const res = await handle(makeEvent({ path }), {}); + expect([401, 404, 405], `${path} -> ${res.statusCode}`).toContain(res.statusCode); + expect(res.body, path).not.toMatch(/"days"/); + } + }); + + it("does not let a stats-shaped path reach the unauthenticated write", async () => { + const res = await handle( + makeEvent({ + httpMethod: "POST", + path: "/stats/hit", + body: JSON.stringify({ site: "fretchen.eu", path: "/blog/" }), + }), + {}, + ); + + expect(res.statusCode).toBe(404); + expect(mockPutS3ObjectConditional).not.toHaveBeenCalled(); + }); + + it("passes OPTIONS through to the matched handler's CORS block", async () => { + const hit = await handle(makeEvent({ httpMethod: "OPTIONS", path: "/hit" }), {}); + const stats = await handle(makeEvent({ httpMethod: "OPTIONS", path: "/stats" }), {}); + + expect(hit.statusCode).toBe(200); + expect(hit.headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(stats.statusCode).toBe(200); + expect(stats.headers["Access-Control-Allow-Headers"]).toContain("Authorization"); + }); + + it("rejects the wrong method on a known route", async () => { + expect((await handle(makeEvent({ httpMethod: "GET", path: "/hit" }), {})).statusCode).toBe(405); + expect((await handle(makeEvent({ httpMethod: "POST", path: "/stats" }), {})).statusCode).toBe(405); + }); +}); diff --git a/analytics/test/buckets.test.ts b/analytics/test/buckets.test.ts new file mode 100644 index 000000000..df2795246 --- /dev/null +++ b/analytics/test/buckets.test.ts @@ -0,0 +1,228 @@ +import { describe, it, expect } from "vitest"; +import { + addDays, + daysInRange, + readDayFromHourly, + readRollupDays, + rebuildDays, + rollupKey, + toIsoDate, + writeDay, + type MonthRollup, +} from "../buckets.js"; +import { MemoryHitStorage, hourBucket } from "./memoryStorage.js"; + +const SITE = "fretchen.eu"; + +describe("date helpers", () => { + it("walks days across a month boundary", () => { + expect(addDays("2026-01-31", 1)).toBe("2026-02-01"); + expect(addDays("2026-03-01", -1)).toBe("2026-02-28"); + }); + + it("is UTC, not local — a late-evening local time still reports the UTC day", () => { + expect(toIsoDate(new Date("2026-08-10T23:30:00Z"))).toBe("2026-08-10"); + expect(toIsoDate(new Date("2026-08-11T00:30:00Z"))).toBe("2026-08-11"); + }); + + it("enumerates an inclusive day range", () => { + expect(daysInRange("2026-08-09", "2026-08-11")).toEqual(["2026-08-09", "2026-08-10", "2026-08-11"]); + expect(daysInRange("2026-08-09", "2026-08-09")).toEqual(["2026-08-09"]); + }); +}); + +describe("key computation", () => { + it("reads a day as 24 zero-padded hour keys, and never lists a prefix", async () => { + const store = new MemoryHitStorage(); + + await readDayFromHourly(store, SITE, "2026-08-10"); + + expect(store.gets).toHaveLength(24); + expect(store.gets[0]).toBe("counts/fretchen.eu/2026-08-10T00.json"); + expect(store.gets[23]).toBe("counts/fretchen.eu/2026-08-10T23.json"); + }); + + it("reads one rollup object per month a range touches, across a year boundary", async () => { + const store = new MemoryHitStorage(); + + await readRollupDays(store, SITE, "2025-11-20", "2026-02-03"); + + expect(store.gets).toEqual([ + rollupKey(SITE, "2025-11"), + rollupKey(SITE, "2025-12"), + rollupKey(SITE, "2026-01"), + rollupKey(SITE, "2026-02"), + ]); + }); +}); + +describe("readDayFromHourly", () => { + it("sums hits and merges pages across the day's buckets, most-hit first", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-10T00.json": hourBucket(5, { "/": 3, "/blog/": 2 }), + "counts/fretchen.eu/2026-08-10T13.json": hourBucket(2, { "/": 1, "/x402/": 1 }), + }); + + const day = await readDayFromHourly(store, SITE, "2026-08-10"); + + expect(day).toEqual({ hits: 7, pages: { "/": 4, "/blog/": 2, "/x402/": 1 }, source: "beacon" }); + expect(Object.keys(day!.pages)).toEqual(["/", "/blog/", "/x402/"]); + }); + + it("returns null for a day with no objects, so no-traffic is distinguishable from not-compacted", async () => { + const store = new MemoryHitStorage(); + expect(await readDayFromHourly(store, SITE, "2026-08-10")).toBeNull(); + }); + + it("ignores buckets belonging to other days", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T23.json": hourBucket(9, { "/": 9 }), + "counts/fretchen.eu/2026-08-10T00.json": hourBucket(1, { "/": 1 }), + }); + expect((await readDayFromHourly(store, SITE, "2026-08-10"))?.hits).toBe(1); + }); +}); + +describe("readRollupDays", () => { + it("collects days across the months a range spans, clipped to the range", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-07")]: { + site: SITE, + month: "2026-07", + days: { + "2026-07-30": { hits: 3, pages: { "/": 3 }, source: "umami" }, + "2026-07-31": { hits: 2, pages: { "/a/": 2 }, source: "umami" }, + }, + }, + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { "2026-08-01": { hits: 4, pages: { "/": 4 }, source: "beacon" } }, + }, + }); + + const days = await readRollupDays(store, SITE, "2026-07-31", "2026-08-01"); + + expect(Object.keys(days)).toEqual(["2026-07-31", "2026-08-01"]); + }); + + it("tolerates months with no rollup object", async () => { + const store = new MemoryHitStorage(); + expect(await readRollupDays(store, SITE, "2026-07-01", "2026-08-01")).toEqual({}); + }); +}); + +describe("writeDay", () => { + const bucket = { hits: 7, pages: { "/": 7 }, source: "beacon" }; + + it("creates the month object when it does not exist", async () => { + const store = new MemoryHitStorage(); + + expect(await writeDay(store, SITE, "2026-08-09", bucket)).toBe("written"); + expect(store.read(rollupKey(SITE, "2026-08"))).toEqual({ + site: SITE, + month: "2026-08", + days: { "2026-08-09": bucket }, + }); + }); + + it("leaves an already-stored day untouched, so backfilled Umami days survive", async () => { + const umamiDay = { hits: 42, pages: { "/": 42 }, source: "umami" }; + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { site: SITE, month: "2026-08", days: { "2026-08-09": umamiDay } }, + }); + + expect(await writeDay(store, SITE, "2026-08-09", bucket)).toBe("exists"); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual(umamiDay); + }); + + it("keeps days sorted as it appends", async () => { + const store = new MemoryHitStorage(); + await writeDay(store, SITE, "2026-08-09", bucket); + await writeDay(store, SITE, "2026-08-02", bucket); + + expect(Object.keys(store.read(rollupKey(SITE, "2026-08"))!.days)).toEqual([ + "2026-08-02", + "2026-08-09", + ]); + }); + + it("retries from a fresh read after a 412 and still lands the write", async () => { + const store = new MemoryHitStorage(); + store.failNextPuts = 2; + + expect(await writeDay(store, SITE, "2026-08-09", bucket)).toBe("written"); + }); + + it("gives up after three conflicts rather than looping", async () => { + const store = new MemoryHitStorage(); + store.failNextPuts = 99; + + expect(await writeDay(store, SITE, "2026-08-09", bucket)).toBe("conflict"); + }); +}); + +// The single compaction path behind both rollup.ts and stats.ts. +describe("rebuildDays", () => { + const TODAY = "2026-08-10"; + + it("rebuilds a complete day and compacts it", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + + const result = await rebuildDays(store, SITE, ["2026-08-09"], TODAY); + + expect(result.written).toEqual(["2026-08-09"]); + expect(result.rebuilt["2026-08-09"]?.hits).toBe(3); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]?.hits).toBe(3); + }); + + it("returns today's numbers but never compacts them", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(2, { "/": 2 }), + }); + + const result = await rebuildDays(store, SITE, [TODAY], TODAY); + + expect(result.rebuilt[TODAY]?.hits).toBe(2); + expect(result.written).toEqual([]); + expect(store.read(rollupKey(SITE, "2026-08"))).toBeNull(); + }); + + it("reports days with no traffic as empty rather than storing zero rows", async () => { + const store = new MemoryHitStorage(); + + const result = await rebuildDays(store, SITE, ["2026-08-08", "2026-08-09"], TODAY); + + expect(result.empty).toEqual(expect.arrayContaining(["2026-08-08", "2026-08-09"])); + expect(result.written).toEqual([]); + expect(store.objects.size).toBe(0); + }); + + it("treats an already-compacted day as neither written nor failed", async () => { + const existing = { hits: 42, pages: { "/": 42 }, source: "umami" }; + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { site: SITE, month: "2026-08", days: { "2026-08-09": existing } }, + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + + const result = await rebuildDays(store, SITE, ["2026-08-09"], TODAY); + + expect(result.written).toEqual([]); + expect(result.failed).toEqual([]); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual(existing); + }); + + it("records a storage failure without throwing — the data still comes back", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + store.throwOnPut = true; + + const result = await rebuildDays(store, SITE, ["2026-08-09"], TODAY); + + expect(result.failed).toEqual(["2026-08-09"]); + expect(result.rebuilt["2026-08-09"]?.hits).toBe(3); + }); +}); diff --git a/analytics/test/hit.test.ts b/analytics/test/hit.test.ts index 491eeba7d..969f34dcc 100644 --- a/analytics/test/hit.test.ts +++ b/analytics/test/hit.test.ts @@ -14,7 +14,7 @@ vi.mock("@fretchen/s3-utils", () => ({ // ===== Import after mocks ===== -import { handle, type ScalewayEvent } from "../hit.js"; +import { handleHit, type ScalewayEvent } from "../hit.js"; // ===== Helpers ===== @@ -34,22 +34,22 @@ describe("hit handler", () => { }); it("responds to OPTIONS with CORS headers and no body", async () => { - const res = await handle(makeEvent({ httpMethod: "OPTIONS", body: undefined }), {}); + const res = await handleHit(makeEvent({ httpMethod: "OPTIONS", body: undefined }), {}); expect(res.statusCode).toBe(200); expect(res.body).toBe(""); expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); }); - it("echoes back a whitelisted origin", async () => { - const res = await handle( - makeEvent({ httpMethod: "OPTIONS", body: undefined, headers: { origin: "http://localhost:5173" } }), - {}, - ); - expect(res.headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5173"); - }); + it.each(["http://localhost:3000", "http://localhost:5173"])( + "echoes back the whitelisted origin %s", + async (origin) => { + const res = await handleHit(makeEvent({ httpMethod: "OPTIONS", body: undefined, headers: { origin } }), {}); + expect(res.headers["Access-Control-Allow-Origin"]).toBe(origin); + }, + ); it("falls back to the canonical origin for an unknown origin", async () => { - const res = await handle( + const res = await handleHit( makeEvent({ httpMethod: "OPTIONS", body: undefined, headers: { origin: "https://evil.com" } }), {}, ); @@ -57,23 +57,23 @@ describe("hit handler", () => { }); it("rejects non-POST/OPTIONS methods with 405", async () => { - const res = await handle(makeEvent({ httpMethod: "GET" }), {}); + const res = await handleHit(makeEvent({ httpMethod: "GET" }), {}); expect(res.statusCode).toBe(405); }); it("rejects a missing body with 400", async () => { - const res = await handle(makeEvent({ body: undefined }), {}); + const res = await handleHit(makeEvent({ body: undefined }), {}); expect(res.statusCode).toBe(400); expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); }); it("rejects malformed JSON with 400", async () => { - const res = await handle(makeEvent({ body: "{not json" }), {}); + const res = await handleHit(makeEvent({ body: "{not json" }), {}); expect(res.statusCode).toBe(400); }); it("rejects a wrong or missing site with 400", async () => { - const res = await handle(makeEvent({ body: JSON.stringify({ site: "evil.com", path: "/x" }) }), {}); + const res = await handleHit(makeEvent({ body: JSON.stringify({ site: "evil.com", path: "/x" }) }), {}); expect(res.statusCode).toBe(400); expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); }); @@ -84,7 +84,7 @@ describe("hit handler", () => { ["empty string", ""], ["not a string", 123], ])("rejects an invalid path (%s) with 400", async (_label, path) => { - const res = await handle(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path }) }), {}); + const res = await handleHit(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path }) }), {}); expect(res.statusCode).toBe(400); expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); }); @@ -94,7 +94,7 @@ describe("hit handler", () => { mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"new-etag"' }); const overlong = `/${"a".repeat(300)}`; - const res = await handle(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path: overlong }) }), {}); + const res = await handleHit(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path: overlong }) }), {}); expect(res.statusCode).toBe(204); const [, body] = mockPutS3ObjectConditional.mock.calls[0]; @@ -107,7 +107,7 @@ describe("hit handler", () => { mockGetS3ObjectWithMeta.mockResolvedValue(null); mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"new-etag"' }); - const res = await handle(makeEvent(), {}); + const res = await handleHit(makeEvent(), {}); expect(res.statusCode).toBe(204); const [key, body, opts] = mockPutS3ObjectConditional.mock.calls[0]; @@ -124,7 +124,7 @@ describe("hit handler", () => { mockGetS3ObjectWithMeta.mockResolvedValue(null); mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"new-etag"' }); - await handle(makeEvent(), {}); + await handleHit(makeEvent(), {}); const [, , opts] = mockPutS3ObjectConditional.mock.calls[0]; expect(opts).not.toHaveProperty("acl"); @@ -137,7 +137,7 @@ describe("hit handler", () => { }); mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"etag-2"' }); - const res = await handle(makeEvent(), {}); + const res = await handleHit(makeEvent(), {}); expect(res.statusCode).toBe(204); const [, body, opts] = mockPutS3ObjectConditional.mock.calls[0]; @@ -156,7 +156,7 @@ describe("hit handler", () => { }); mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"etag-2"' }); - const res = await handle(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path: "/new-page" }) }), {}); + const res = await handleHit(makeEvent({ body: JSON.stringify({ site: "fretchen.eu", path: "/new-page" }) }), {}); expect(res.statusCode).toBe(204); const [, body] = mockPutS3ObjectConditional.mock.calls[0]; @@ -177,7 +177,7 @@ describe("hit handler", () => { }); mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"etag-2"' }); - const res = await handle(makeEvent(), {}); + const res = await handleHit(makeEvent(), {}); expect(res.statusCode).toBe(204); const [, body] = mockPutS3ObjectConditional.mock.calls[0]; @@ -193,7 +193,7 @@ describe("hit handler", () => { .mockResolvedValueOnce({ ok: false, status: 412 }) .mockResolvedValueOnce({ ok: true, etag: '"final"' }); - const res = await handle(makeEvent(), {}); + const res = await handleHit(makeEvent(), {}); expect(res.statusCode).toBe(204); expect(mockGetS3ObjectWithMeta).toHaveBeenCalledTimes(2); @@ -206,7 +206,7 @@ describe("hit handler", () => { mockGetS3ObjectWithMeta.mockResolvedValue({ body: JSON.stringify({ hits: 1, pages: {} }), etag: '"e"' }); mockPutS3ObjectConditional.mockResolvedValue({ ok: false, status: 412 }); - const res = await handle(makeEvent(), {}); + const res = await handleHit(makeEvent(), {}); expect(res.statusCode).toBe(204); expect(mockGetS3ObjectWithMeta).toHaveBeenCalledTimes(3); diff --git a/analytics/test/memoryStorage.ts b/analytics/test/memoryStorage.ts new file mode 100644 index 000000000..4571cb3d8 --- /dev/null +++ b/analytics/test/memoryStorage.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import type { GetResult, HitStorage, PutOpts, PutResult } from "../storage.js"; + +/** + * In-memory `HitStorage` for the read-layer tests. Same MD5 ETag and + * compare-and-swap semantics as `FileHitStorage`, so `writeDay`'s CAS loop is + * exercised for real rather than against a mock that always succeeds. + */ +export class MemoryHitStorage implements HitStorage { + readonly objects = new Map(); + /** Every key read, in order — lets tests assert how much work a call did. */ + readonly gets: string[] = []; + /** Forces the next N `putConditional` calls to report a 412, to drive the retry path. */ + failNextPuts = 0; + /** Makes every `putConditional` throw, to prove a failed cache warm can't fail a read. */ + throwOnPut = false; + + constructor(seed: Record = {}) { + for (const [key, value] of Object.entries(seed)) { + this.objects.set(key, JSON.stringify(value)); + } + } + + private etagOf(body: string): string { + return createHash("md5").update(body).digest("hex"); + } + + async getWithMeta(key: string): Promise { + this.gets.push(key); + const body = this.objects.get(key); + return body === undefined ? null : { body, etag: this.etagOf(body) }; + } + + async putConditional(key: string, body: string, opts: PutOpts): Promise { + if (this.throwOnPut) { + throw new Error("S3 write failed"); + } + if (this.failNextPuts > 0) { + this.failNextPuts -= 1; + return { ok: false, status: 412 }; + } + const existing = this.objects.get(key); + if (opts.ifNoneMatch === "*" && existing !== undefined) { + return { ok: false, status: 412 }; + } + if (opts.ifMatch && (existing === undefined || this.etagOf(existing) !== opts.ifMatch)) { + return { ok: false, status: 412 }; + } + this.objects.set(key, body); + return { ok: true, etag: this.etagOf(body) }; + } + + read(key: string): T | null { + const body = this.objects.get(key); + return body === undefined ? null : (JSON.parse(body) as T); + } +} + +/** Seeds one hour bucket, in the layout `hit.ts` writes. */ +export function hourBucket(hits: number, pages: Record) { + return { hits, pages }; +} diff --git a/analytics/test/rollup.test.ts b/analytics/test/rollup.test.ts new file mode 100644 index 000000000..73b020665 --- /dev/null +++ b/analytics/test/rollup.test.ts @@ -0,0 +1,158 @@ +import { describe, it, expect } from "vitest"; +import { rollupRecentDays } from "../rollup.js"; +import { rollupKey, type MonthRollup } from "../buckets.js"; +import { MemoryHitStorage, hourBucket } from "./memoryStorage.js"; + +const SITE = "fretchen.eu"; + +// Fixed "now" so the 14-day window is deterministic: yesterday is 2026-08-09, +// and the window runs 2026-07-27 .. 2026-08-09. +const NOW = new Date("2026-08-10T06:00:00Z"); + +describe("rollupRecentDays", () => { + it("compacts a day of hourly buckets into its month rollup", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 2, "/blog/": 1 }), + "counts/fretchen.eu/2026-08-09T17.json": hourBucket(1, { "/blog/": 1 }), + }); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.written).toEqual(["2026-08-09"]); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual({ + hits: 4, + pages: { "/blog/": 2, "/": 2 }, + source: "beacon", + }); + }); + + it("never touches today — it is still being written to", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(9, { "/": 9 }), + }); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.to).toBe("2026-08-09"); + expect(summary.written).toEqual([]); + expect(store.read(rollupKey(SITE, "2026-08"))).toBeNull(); + }); + + it("is idempotent — a second run writes nothing", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + + await rollupRecentDays(store, SITE, NOW); + const second = await rollupRecentDays(store, SITE, NOW); + + expect(second.written).toEqual([]); + expect(second.skipped).toContain("2026-08-09"); + }); + + it("cannot clobber a backfilled Umami day", async () => { + const umamiDay = { hits: 42, pages: { "/": 42 }, source: "umami" }; + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { site: SITE, month: "2026-08", days: { "2026-08-09": umamiDay } }, + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.written).toEqual([]); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual(umamiDay); + }); + + it("fills only the hole when a run was missed mid-window", async () => { + const day = { hits: 1, pages: { "/": 1 }, source: "beacon" }; + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { "2026-08-05": day, "2026-08-07": day }, + }, + "counts/fretchen.eu/2026-08-06T10.json": hourBucket(5, { "/x402/": 5 }), + }); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.written).toEqual(["2026-08-06"]); + expect(summary.skipped).toEqual(expect.arrayContaining(["2026-08-05", "2026-08-07"])); + }); + + it("records days with no traffic as empty rather than writing zero rows", async () => { + const store = new MemoryHitStorage(); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.written).toEqual([]); + expect(summary.empty).toHaveLength(14); + expect(store.objects.size).toBe(0); + }); + + it("spans the month boundary inside the window", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-07-28T12.json": hourBucket(2, { "/": 2 }), + "counts/fretchen.eu/2026-08-02T12.json": hourBucket(4, { "/": 4 }), + }); + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.from).toBe("2026-07-27"); + expect(summary.written).toEqual(["2026-07-28", "2026-08-02"]); + expect(store.read(rollupKey(SITE, "2026-07"))?.days["2026-07-28"]?.hits).toBe(2); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-02"]?.hits).toBe(4); + }); + + it("reports a day it could not write after repeated conflicts", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T08.json": hourBucket(3, { "/": 3 }), + }); + store.failNextPuts = 99; + + const summary = await rollupRecentDays(store, SITE, NOW); + + expect(summary.failed).toEqual(["2026-08-09"]); + }); + + it("widens the window from ROLLUP_WINDOW_DAYS", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-07-15T10.json": hourBucket(8, { "/": 8 }), + }); + process.env.ROLLUP_WINDOW_DAYS = "40"; + try { + const summary = await rollupRecentDays(store, SITE, NOW); + expect(summary.from).toBe("2026-07-01"); + expect(summary.written).toEqual(["2026-07-15"]); + } finally { + delete process.env.ROLLUP_WINDOW_DAYS; + } + }); + + it("ignores a nonsensical ROLLUP_WINDOW_DAYS and keeps the default", async () => { + const store = new MemoryHitStorage(); + for (const bad of ["abc", "0", "-5", ""]) { + process.env.ROLLUP_WINDOW_DAYS = bad; + const summary = await rollupRecentDays(store, SITE, NOW); + expect(summary.from).toBe("2026-07-27"); // the default 14-day window + } + delete process.env.ROLLUP_WINDOW_DAYS; + }); + + // The recovery path for a gap that has aged past stats.ts's fallback window: + // the hourly objects are still there, they just stopped being reachable. + it("recovers a day older than the default window when given a wider one", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-07-15T10.json": hourBucket(8, { "/": 8 }), + }); + + const ignored = await rollupRecentDays(store, SITE, NOW); + expect(ignored.written).toEqual([]); + + const recovered = await rollupRecentDays(store, SITE, NOW, 40); + + expect(recovered.from).toBe("2026-07-01"); + expect(recovered.written).toEqual(["2026-07-15"]); + expect(store.read(rollupKey(SITE, "2026-07"))?.days["2026-07-15"]?.hits).toBe(8); + }); +}); diff --git a/analytics/test/stats.test.ts b/analytics/test/stats.test.ts new file mode 100644 index 000000000..81f052cff --- /dev/null +++ b/analytics/test/stats.test.ts @@ -0,0 +1,367 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { privateKeyToAccount } from "viem/accounts"; + +// The handler's module-level storage is S3HitStorage; the range logic below is +// tested directly against MemoryHitStorage instead. +const { mockGetS3ObjectWithMeta, mockPutS3ObjectConditional } = vi.hoisted(() => ({ + mockGetS3ObjectWithMeta: vi.fn(), + mockPutS3ObjectConditional: vi.fn(), +})); + +vi.mock("@fretchen/s3-utils", () => ({ + getS3ObjectWithMeta: mockGetS3ObjectWithMeta, + putS3ObjectConditional: mockPutS3ObjectConditional, +})); + +import { buildStats, collectRange, handleStats, type StatsEvent, type StatsResponse } from "../stats.js"; +import { rollupKey, type MonthRollup } from "../buckets.js"; +import { MemoryHitStorage, hourBucket } from "./memoryStorage.js"; + +const SITE = "fretchen.eu"; +const NOW = new Date("2026-08-10T06:00:00Z"); + +// Anvil account #0 — a well-known test key, never used for anything real. +const OWNER_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; +const owner = privateKeyToAccount(OWNER_KEY); +const OTHER_KEY = "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"; +const other = privateKeyToAccount(OTHER_KEY); + +async function bearer(account: typeof owner, timestamp = Math.floor(Date.now() / 1000)): Promise { + const message = `analytics-api:${timestamp}`; + const signature = await account.signMessage({ message }); + return `Bearer ${Buffer.from(JSON.stringify({ address: account.address, signature, message })).toString("base64")}`; +} + +function makeEvent(overrides: Partial = {}): StatsEvent { + return { httpMethod: "GET", ...overrides }; +} + +describe("collectRange", () => { + it("prefers the rollups and falls back to hourly only for the gaps", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { "2026-08-08": { hits: 11, pages: { "/": 11 }, source: "beacon" } }, + }, + // 2026-08-09 was never compacted; 2026-08-10 is today. + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(2, { "/x402/": 2 }), + }); + + const days = await collectRange(store, SITE, "2026-08-08", "2026-08-10", NOW); + + expect(Object.keys(days).sort()).toEqual(["2026-08-08", "2026-08-09", "2026-08-10"]); + expect(days["2026-08-09"]).toEqual({ hits: 4, pages: { "/blog/": 4 }, source: "beacon" }); + }); + + it("returns the same days whether or not they have been rolled up", async () => { + const hourly = { + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + "counts/fretchen.eu/2026-08-08T09.json": hourBucket(6, { "/": 6 }), + }; + const uncompacted = new MemoryHitStorage(hourly); + const compacted = new MemoryHitStorage({ + ...hourly, + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { + "2026-08-08": { hits: 6, pages: { "/": 6 }, source: "beacon" }, + "2026-08-09": { hits: 4, pages: { "/blog/": 4 }, source: "beacon" }, + }, + }, + }); + + const before = await collectRange(uncompacted, SITE, "2026-08-04", "2026-08-10", NOW); + const after = await collectRange(compacted, SITE, "2026-08-04", "2026-08-10", NOW); + + expect(Object.values(before).reduce((sum, d) => sum + d.hits, 0)).toBe(10); + expect(after).toEqual(before); + }); + + it("does not reach past the 14-day fallback window on a cold start", async () => { + const store = new MemoryHitStorage({ + // 15 days before "now" — the cron has already had its chance at this day. + "counts/fretchen.eu/2026-07-27T09.json": hourBucket(50, { "/": 50 }), + "counts/fretchen.eu/2026-07-28T09.json": hourBucket(7, { "/": 7 }), + }); + + const days = await collectRange(store, SITE, "2026-07-20", "2026-08-10", NOW); + + expect(days["2026-07-27"]).toBeUndefined(); + expect(days["2026-07-28"]?.hits).toBe(7); + }); + + it("only probes days after the newest compacted one — quiet days are settled, not re-read", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { "2026-08-07": { hits: 1, pages: { "/": 1 }, source: "beacon" } }, + }, + // 08-08 and 08-09 were quiet; there is nothing to find and nothing to store. + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(2, { "/": 2 }), + }); + + await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + // 08-08, 08-09, 08-10 — the three days after the newest compacted one. + expect(store.gets.filter((key) => key.startsWith("counts/")).length).toBe(3 * 24); + expect(store.gets.some((key) => key.startsWith("counts/fretchen.eu/2026-08-06"))).toBe(false); + }); +}); + +describe("collectRange write-back", () => { + it("compacts a complete day it had to rebuild from hourly buckets", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + }); + + await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual({ + hits: 4, + pages: { "/blog/": 4 }, + source: "beacon", + }); + }); + + it("never compacts today — it is still being written to", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(2, { "/": 2 }), + }); + + await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-10"]).toBeUndefined(); + }); + + it("cannot clobber a backfilled Umami day", async () => { + const umamiDay = { hits: 42, pages: { "/": 42 }, source: "umami" }; + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { site: SITE, month: "2026-08", days: { "2026-08-09": umamiDay } }, + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + }); + + const days = await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + expect(days["2026-08-09"]).toEqual(umamiDay); + expect(store.read(rollupKey(SITE, "2026-08"))?.days["2026-08-09"]).toEqual(umamiDay); + }); + + it("makes the second call cheap — rollup GETs instead of 24 hourly ones", async () => { + const seed = { + "counts/fretchen.eu/2026-08-08T09.json": hourBucket(6, { "/": 6 }), + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + }; + const store = new MemoryHitStorage(seed); + + await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + const firstCallGets = store.gets.length; + store.gets.length = 0; + + const second = await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + // Only today still needs its 24 hourly keys; the two rebuilt days are now rollup reads. + expect(store.gets.filter((key) => key.startsWith("counts/")).length).toBe(24); + expect(store.gets.length).toBeLessThan(firstCallGets); + expect(second["2026-08-09"]).toEqual({ hits: 4, pages: { "/blog/": 4 }, source: "beacon" }); + }); + + it("still serves the data when the write-back fails", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-09T09.json": hourBucket(4, { "/blog/": 4 }), + }); + store.throwOnPut = true; + + const days = await collectRange(store, SITE, "2026-08-01", "2026-08-10", NOW); + + expect(days["2026-08-09"]?.hits).toBe(4); + }); +}); + +describe("buildStats", () => { + it("serves a trailing year, sparse — no zero rows", async () => { + const store = new MemoryHitStorage({ + "counts/fretchen.eu/2026-08-10T05.json": hourBucket(2, { "/": 2 }), + }); + + const stats = await buildStats(store, SITE, NOW); + + expect(stats.from).toBe("2025-08-11"); + expect(stats.to).toBe("2026-08-10"); + expect(stats.days).toEqual({ "2026-08-10": { hits: 2, pages: { "/": 2 }, source: "beacon" } }); + }); + + it("carries per-day pages and source so the client can slice any range", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { + "2026-08-01": { hits: 5, pages: { "/": 3, "/blog/": 2 }, source: "umami" }, + "2026-08-02": { hits: 4, pages: { "/blog/": 4 }, source: "beacon" }, + }, + }, + }); + + const stats = await buildStats(store, SITE, NOW); + + expect(stats.days["2026-08-01"]).toEqual({ hits: 5, pages: { "/": 3, "/blog/": 2 }, source: "umami" }); + expect(stats.days["2026-08-02"]?.source).toBe("beacon"); + }); + + it("spans every month the year touches", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2025-09")]: { + site: SITE, + month: "2025-09", + days: { "2025-09-20": { hits: 3, pages: { "/": 3 }, source: "umami" } }, + }, + [rollupKey(SITE, "2026-08")]: { + site: SITE, + month: "2026-08", + days: { "2026-08-01": { hits: 4, pages: { "/": 4 }, source: "beacon" } }, + }, + }); + + const stats = await buildStats(store, SITE, NOW); + + expect(Object.keys(stats.days).sort()).toEqual(["2025-09-20", "2026-08-01"]); + }); + + it("drops days that fall outside the year", async () => { + const store = new MemoryHitStorage({ + [rollupKey(SITE, "2025-08")]: { + site: SITE, + month: "2025-08", + days: { + "2025-08-10": { hits: 99, pages: { "/": 99 }, source: "umami" }, // a day too old + "2025-08-11": { hits: 1, pages: { "/": 1 }, source: "umami" }, // first day in range + }, + }, + }); + + const stats = await buildStats(store, SITE, NOW); + + expect(stats.days["2025-08-10"]).toBeUndefined(); + expect(stats.days["2025-08-11"]?.hits).toBe(1); + }); +}); + +describe("stats handler", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetS3ObjectWithMeta.mockResolvedValue(null); + mockPutS3ObjectConditional.mockResolvedValue({ ok: true, etag: '"e"' }); + process.env.OWNER_ETH_ADDRESS = owner.address; + }); + + afterEach(() => { + delete process.env.OWNER_ETH_ADDRESS; + }); + + it("responds to OPTIONS with CORS headers, unauthenticated", async () => { + const res = await handleStats(makeEvent({ httpMethod: "OPTIONS" }), {}); + expect(res.statusCode).toBe(200); + expect(res.headers["Access-Control-Allow-Headers"]).toContain("Authorization"); + expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); + }); + + // The preflight is real here — /stats is a GET carrying Authorization — so an + // origin missing from the whitelist blocks the dashboard outright. + it.each(["http://localhost:3000", "http://localhost:5173", "https://www.fretchen.eu"])( + "passes preflight for %s", + async (origin) => { + const res = await handleStats(makeEvent({ httpMethod: "OPTIONS", headers: { origin } }), {}); + expect(res.headers["Access-Control-Allow-Origin"]).toBe(origin); + }, + ); + + it("does not echo an unknown origin", async () => { + const res = await handleStats(makeEvent({ httpMethod: "OPTIONS", headers: { origin: "https://evil.com" } }), {}); + expect(res.headers["Access-Control-Allow-Origin"]).toBe("https://www.fretchen.eu"); + }); + + it("rejects non-GET methods with 405", async () => { + const res = await handleStats(makeEvent({ httpMethod: "POST" }), {}); + expect(res.statusCode).toBe(405); + }); + + it("rejects a missing Authorization header with 401", async () => { + const res = await handleStats(makeEvent(), {}); + expect(res.statusCode).toBe(401); + expect(mockGetS3ObjectWithMeta).not.toHaveBeenCalled(); + }); + + it("rejects a malformed bearer token with 401", async () => { + const res = await handleStats(makeEvent({ headers: { authorization: "Bearer not-base64-json" } }), {}); + expect(res.statusCode).toBe(401); + }); + + it("rejects a signature from a wallet that is not the owner", async () => { + const res = await handleStats(makeEvent({ headers: { authorization: await bearer(other) } }), {}); + expect(res.statusCode).toBe(401); + expect(JSON.parse(res.body).error).toBe("Address mismatch"); + }); + + it("rejects a stale token, so a captured one cannot be replayed", async () => { + const tenMinutesAgo = Math.floor(Date.now() / 1000) - 600; + const res = await handleStats(makeEvent({ headers: { authorization: await bearer(owner, tenMinutesAgo) } }), {}); + expect(res.statusCode).toBe(401); + expect(JSON.parse(res.body).error).toBe("Token expired"); + }); + + it("rejects a message that is not an analytics-api challenge", async () => { + const message = `growth-api:${Math.floor(Date.now() / 1000)}`; + const signature = await owner.signMessage({ message }); + const token = Buffer.from(JSON.stringify({ address: owner.address, signature, message })).toString("base64"); + + const res = await handleStats(makeEvent({ headers: { authorization: `Bearer ${token}` } }), {}); + + expect(res.statusCode).toBe(401); + }); + + it("refuses to serve when no owner address is configured", async () => { + delete process.env.OWNER_ETH_ADDRESS; + const res = await handleStats(makeEvent({ headers: { authorization: await bearer(owner) } }), {}); + expect(res.statusCode).toBe(401); + expect(JSON.parse(res.body).error).toBe("Owner address not configured"); + }); + + it("serves the owner a year-wide envelope", async () => { + const res = await handleStats(makeEvent({ headers: { authorization: await bearer(owner) } }), {}); + + expect(res.statusCode).toBe(200); + const body = JSON.parse(res.body) as StatsResponse; + expect(body.site).toBe(SITE); + expect(body.days).toEqual({}); + // 365 days inclusive of both ends. + const span = (Date.parse(`${body.to}T00:00:00Z`) - Date.parse(`${body.from}T00:00:00Z`)) / 86_400_000; + expect(span).toBe(364); + }); + + it("ignores query parameters — there is no window to ask for", async () => { + const auth = await bearer(owner); + + const plain = JSON.parse( + (await handleStats(makeEvent({ headers: { authorization: auth } }), {})).body, + ) as StatsResponse; + const withParam = JSON.parse( + (await handleStats(makeEvent({ headers: { authorization: auth }, queryStringParameters: { days: "7" } }), {})) + .body, + ) as StatsResponse; + + expect(withParam).toEqual(plain); + }); + + it("returns 500 without leaking the underlying error", async () => { + mockGetS3ObjectWithMeta.mockRejectedValue(new Error("S3 exploded")); + + const res = await handleStats(makeEvent({ headers: { authorization: await bearer(owner) } }), {}); + + expect(res.statusCode).toBe(500); + expect(res.body).not.toContain("S3 exploded"); + }); +}); diff --git a/analytics/tsup.config.js b/analytics/tsup.config.js index cd4e8b8e4..7c70f8541 100644 --- a/analytics/tsup.config.js +++ b/analytics/tsup.config.js @@ -2,7 +2,7 @@ import { defineConfig } from "tsup"; import { builtinModules } from "module"; export default defineConfig({ - entry: ["hit.ts"], + entry: ["analytics.ts", "rollup.ts"], format: ["esm"], platform: "node", target: "node22", @@ -26,7 +26,7 @@ export default defineConfig({ // install` on its own — not a blanket [/.*/]. noExternal wins over // `external` when both would match a package, so this has to be a narrow // allowlist, not "everything except the two entries above". - noExternal: ["@fretchen/s3-utils"], + noExternal: ["@fretchen/s3-utils", "@fretchen/chain-utils"], banner: { js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`, }, diff --git a/scw_js/auth_utils.ts b/scw_js/auth_utils.ts deleted file mode 100644 index ce19448e5..000000000 --- a/scw_js/auth_utils.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { verifyMessage } from "viem"; -import { AUTH_TOKEN_MAX_AGE_MS } from "@fretchen/chain-utils"; - -export interface BearerPayload { - address: `0x${string}`; - signature: string; - message: string; -} - -/** - * Parses a `Bearer ` Authorization header into its decoded payload. - * Returns null if the header is missing, malformed, or fails JSON parsing. - */ -export function parseBearerToken(authHeader: string | undefined): BearerPayload | null { - if (!authHeader) { - return null; - } - const match = authHeader.match(/^Bearer\s+(.+)$/i); - if (!match) { - return null; - } - try { - const decoded = JSON.parse(Buffer.from(match[1], "base64").toString("utf-8")) as BearerPayload; - const { address, signature, message } = decoded; - if ( - typeof address === "string" && - address.startsWith("0x") && - typeof signature === "string" && - signature.startsWith("0x") && - typeof message === "string" && - message.length > 0 - ) { - return { address: address as `0x${string}`, signature, message }; - } - return null; - } catch { - return null; - } -} - -/** - * Verifies a parsed wallet signature payload against an expected message prefix and address. - * Returns null on success, or an error string describing the failure. - * - * Checks (in order): message format, timestamp freshness, address match, signature validity. - */ -export async function verifySignedMessage( - address: string, - signature: string, - message: string, - expectedPrefix: string, - expectedAddress: string, -): Promise { - const match = message.match(new RegExp(`^${expectedPrefix}:(\\d+)$`)); - if (!match) { - return "Unauthorized"; - } - - const ts = parseInt(match[1], 10); - if (ts > 9_999_999_999) { - return "Unauthorized"; - } // guard against year >2286 / integer overflow - if (Math.abs(Date.now() - ts * 1000) > AUTH_TOKEN_MAX_AGE_MS) { - return "Token expired"; - } - - if (address.toLowerCase() !== expectedAddress.toLowerCase()) { - return "Address mismatch"; - } - - try { - const isValid = await verifyMessage({ - address: address as `0x${string}`, - message, - signature: signature as `0x${string}`, - }); - if (!isValid) { - return "Invalid signature"; - } - } catch { - return "Invalid signature"; - } - - return null; -} diff --git a/scw_js/growth_api.ts b/scw_js/growth_api.ts index d8bf58250..29d7503a0 100644 --- a/scw_js/growth_api.ts +++ b/scw_js/growth_api.ts @@ -11,7 +11,7 @@ import { AuthError, type ContentQueue, } from "./growth_service.js"; -import { parseBearerToken } from "./auth_utils.js"; +import { parseBearerToken } from "@fretchen/chain-utils"; import { parseJsonBody } from "./utils.js"; const logger = pino({ level: process.env.LOG_LEVEL || "info" }); diff --git a/scw_js/growth_service.ts b/scw_js/growth_service.ts index e67ff1a49..bd2541ca0 100644 --- a/scw_js/growth_service.ts +++ b/scw_js/growth_service.ts @@ -1,6 +1,6 @@ import { getS3Object, putS3Object } from "@fretchen/s3-utils"; import pino from "pino"; -import { verifySignedMessage } from "./auth_utils.js"; +import { verifySignedMessage } from "@fretchen/chain-utils"; const logger = pino({ level: process.env.LOG_LEVEL || "info" }); diff --git a/scw_js/test/auth_utils.test.ts b/scw_js/test/auth_utils.test.ts deleted file mode 100644 index a48876300..000000000 --- a/scw_js/test/auth_utils.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; - -// ===== Mocks ===== - -const { mockVerifyMessage } = vi.hoisted(() => ({ - mockVerifyMessage: vi.fn(), -})); - -vi.mock("viem", () => ({ - verifyMessage: mockVerifyMessage, -})); - -// ===== Imports ===== - -import { parseBearerToken, verifySignedMessage } from "../auth_utils.js"; - -// ===== Helpers ===== - -const VALID_ADDRESS = "0x1234567890abcdef1234567890abcdef12345678"; -const VALID_SIGNATURE = "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab"; - -function makeToken(payload: Record): string { - return `Bearer ${Buffer.from(JSON.stringify(payload)).toString("base64")}`; -} - -function freshTs(): number { - return Math.floor(Date.now() / 1000); -} - -// ===== parseBearerToken ===== - -describe("parseBearerToken", () => { - it("returns parsed payload for a valid Bearer token", () => { - const payload = { - address: VALID_ADDRESS, - signature: VALID_SIGNATURE, - message: "leaf-history:1234", - }; - const result = parseBearerToken(makeToken(payload)); - expect(result).toEqual(payload); - }); - - it("returns null for undefined header", () => { - expect(parseBearerToken(undefined)).toBeNull(); - }); - - it("returns null for empty string", () => { - expect(parseBearerToken("")).toBeNull(); - }); - - it("returns null for Basic auth (not Bearer)", () => { - expect(parseBearerToken("Basic abc123")).toBeNull(); - }); - - it("returns null for invalid base64 content", () => { - expect(parseBearerToken("Bearer !!!notbase64!!!")).toBeNull(); - }); - - it("returns null when base64 decodes to non-JSON", () => { - const token = `Bearer ${Buffer.from("hello world").toString("base64")}`; - expect(parseBearerToken(token)).toBeNull(); - }); - - it("returns null when address field is missing", () => { - expect(parseBearerToken(makeToken({ signature: VALID_SIGNATURE, message: "m" }))).toBeNull(); - }); - - it("returns null when address is not a string", () => { - expect( - parseBearerToken(makeToken({ address: 123, signature: VALID_SIGNATURE, message: "m" })), - ).toBeNull(); - }); - - it("returns null when address lacks 0x prefix", () => { - expect( - parseBearerToken( - makeToken({ address: "notanaddress", signature: VALID_SIGNATURE, message: "m" }), - ), - ).toBeNull(); - }); - - it("returns null when signature lacks 0x prefix", () => { - expect( - parseBearerToken( - makeToken({ address: VALID_ADDRESS, signature: "nosigprefix", message: "m" }), - ), - ).toBeNull(); - }); - - it("returns null when signature is not a string", () => { - expect( - parseBearerToken(makeToken({ address: VALID_ADDRESS, signature: true, message: "m" })), - ).toBeNull(); - }); - - it("returns null when message is empty string", () => { - expect( - parseBearerToken( - makeToken({ address: VALID_ADDRESS, signature: VALID_SIGNATURE, message: "" }), - ), - ).toBeNull(); - }); - - it("returns null when message is not a string", () => { - expect( - parseBearerToken( - makeToken({ address: VALID_ADDRESS, signature: VALID_SIGNATURE, message: ["array"] }), - ), - ).toBeNull(); - }); - - it("is safe against __proto__ injection — Object.prototype is not modified", () => { - // A crafted raw JSON string attempting prototype pollution - const rawJson = `{"__proto__":{"isAdmin":true},"address":"${VALID_ADDRESS}","signature":"${VALID_SIGNATURE}","message":"leaf-history:1234"}`; - const token = `Bearer ${Buffer.from(rawJson).toString("base64")}`; - parseBearerToken(token); - // Object.prototype must remain unmodified regardless of what JSON.parse does - expect((Object.prototype as Record).isAdmin).toBeUndefined(); - }); -}); - -// ===== verifySignedMessage ===== - -describe("verifySignedMessage", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("returns null for a fully valid payload", async () => { - mockVerifyMessage.mockResolvedValue(true); - const ts = freshTs(); - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${ts}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBeNull(); - }); - - it("returns 'Unauthorized' for a timestamp far in the future (overflow guard)", async () => { - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - "leaf-history:99999999999", - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Unauthorized"); - }); - - it("returns 'Unauthorized' for wrong message prefix", async () => { - const ts = freshTs(); - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `wrong-prefix:${ts}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Unauthorized"); - }); - - it("returns 'Unauthorized' for completely malformed message", async () => { - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - "not-a-valid-message", - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Unauthorized"); - }); - - it("returns 'Token expired' for timestamp 6 minutes in the past", async () => { - const staleTs = freshTs() - 360; - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${staleTs}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Token expired"); - }); - - it("returns 'Token expired' for timestamp 6 minutes in the future", async () => { - const futureTs = freshTs() + 360; - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${futureTs}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Token expired"); - }); - - it("returns 'Address mismatch' when payload address differs from expected", async () => { - const ts = freshTs(); - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${ts}`, - "leaf-history", - "0xdeadbeef00000000000000000000000000000000", - ); - expect(result).toBe("Address mismatch"); - }); - - it("passes case-insensitive address comparison", async () => { - mockVerifyMessage.mockResolvedValue(true); - const ts = freshTs(); - const upper = VALID_ADDRESS.toUpperCase().replace("0X", "0x"); - const lower = VALID_ADDRESS.toLowerCase(); - const result = await verifySignedMessage( - upper, - VALID_SIGNATURE, - `leaf-history:${ts}`, - "leaf-history", - lower, - ); - expect(result).toBeNull(); - }); - - it("returns 'Invalid signature' when verifyMessage returns false", async () => { - mockVerifyMessage.mockResolvedValue(false); - const ts = freshTs(); - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${ts}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Invalid signature"); - }); - - it("returns 'Invalid signature' when verifyMessage throws", async () => { - mockVerifyMessage.mockRejectedValue(new Error("RPC error")); - const ts = freshTs(); - const result = await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - `leaf-history:${ts}`, - "leaf-history", - VALID_ADDRESS, - ); - expect(result).toBe("Invalid signature"); - }); - - it("calls verifyMessage with the exact address, message, and signature", async () => { - mockVerifyMessage.mockResolvedValue(true); - const ts = freshTs(); - const message = `leaf-history:${ts}`; - await verifySignedMessage( - VALID_ADDRESS, - VALID_SIGNATURE, - message, - "leaf-history", - VALID_ADDRESS, - ); - expect(mockVerifyMessage).toHaveBeenCalledWith({ - address: VALID_ADDRESS, - message, - signature: VALID_SIGNATURE, - }); - }); -}); diff --git a/scw_js/test/growth_api.test.ts b/scw_js/test/growth_api.test.ts index 9f2b06631..642ed77a0 100644 --- a/scw_js/test/growth_api.test.ts +++ b/scw_js/test/growth_api.test.ts @@ -9,14 +9,19 @@ vi.mock("@fretchen/s3-utils", () => ({ putS3Object: mockPutS3Object, })); -const mockVerifyMessage = vi.fn(); -vi.mock("viem", () => ({ - verifyMessage: mockVerifyMessage, -})); +// Signatures here are real, not mocked. `verifySignedMessage` now lives in +// @fretchen/chain-utils, which resolves its own copy of viem through the +// symlinked workspace package — a `vi.mock("viem")` in this package cannot +// reach it. Signing for real is also the stronger test. +import { privateKeyToAccount } from "viem/accounts"; // ===== Test data ===== -const OWNER_ADDRESS = "0xAAEBC1441323B8ad6Bdf6793A8428166b510239C"; +// Anvil account #0 — a well-known test key, never used for anything real. +const owner = privateKeyToAccount( + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", +); +const OWNER_ADDRESS = owner.address; const sampleQueue = { drafts: [ @@ -85,13 +90,14 @@ const samplePerformance = { // ===== Helpers ===== -function makeAuthHeader(timestamp?: number): string { +/** A freshly signed owner token, rebuilt each test so it never ages out. */ +let validAuth: string; + +async function makeAuthHeader(timestamp?: number): Promise { const ts = timestamp ?? Math.floor(Date.now() / 1000); - const payload = { - address: OWNER_ADDRESS, - signature: "0xvalidsignature", - message: `growth-api:${ts}`, - }; + const message = `growth-api:${ts}`; + const signature = await owner.signMessage({ message }); + const payload = { address: OWNER_ADDRESS, signature, message }; return `Bearer ${Buffer.from(JSON.stringify(payload)).toString("base64")}`; } @@ -104,7 +110,7 @@ function makeEvent( auth?: string | null; } = {}, ) { - const auth = options.auth === null ? undefined : (options.auth ?? makeAuthHeader()); + const auth = options.auth === null ? undefined : (options.auth ?? validAuth); return { httpMethod: method, path: `/${path}`, @@ -135,8 +141,7 @@ describe("growth_api", () => { vi.resetModules(); mockGetS3Object.mockReset(); mockPutS3Object.mockReset(); - mockVerifyMessage.mockReset(); - mockVerifyMessage.mockResolvedValue(true); + validAuth = await makeAuthHeader(); process.env.OWNER_ETH_ADDRESS = OWNER_ADDRESS; process.env.SCW_ACCESS_KEY = "test-key"; @@ -163,18 +168,33 @@ describe("growth_api", () => { }); test("returns 401 when signature is invalid", async () => { - mockVerifyMessage.mockResolvedValueOnce(false); - const event = makeEvent("GET", "drafts"); + // Claims the owner's address but signed by someone else — the recovered + // signer won't match, which is the case a mocked verifier can't prove. + const impostor = privateKeyToAccount( + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + ); + const message = `growth-api:${Math.floor(Date.now() / 1000)}`; + const payload = { + address: OWNER_ADDRESS, + signature: await impostor.signMessage({ message }), + message, + }; + const auth = `Bearer ${Buffer.from(JSON.stringify(payload)).toString("base64")}`; + const event = makeEvent("GET", "drafts", { auth }); const res = (await handle(event, {})) as { statusCode: number; body: string }; expect(res.statusCode).toBe(401); expect(JSON.parse(res.body).error).toMatch(/Invalid wallet signature/i); }); test("returns 401 when address is not the owner", async () => { + const message = `growth-api:${Math.floor(Date.now() / 1000)}`; + const other = privateKeyToAccount( + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + ); const payload = { - address: "0x1111111111111111111111111111111111111111", - signature: "0xvalidsignature", - message: `growth-api:${Math.floor(Date.now() / 1000)}`, + address: other.address, + signature: await other.signMessage({ message }), + message, }; const auth = `Bearer ${Buffer.from(JSON.stringify(payload)).toString("base64")}`; const event = makeEvent("GET", "drafts", { auth }); @@ -185,7 +205,7 @@ describe("growth_api", () => { test("returns 401 when message timestamp is expired", async () => { const oldTimestamp = Math.floor(Date.now() / 1000) - 600; // 10 min ago - const event = makeEvent("GET", "drafts", { auth: makeAuthHeader(oldTimestamp) }); + const event = makeEvent("GET", "drafts", { auth: await makeAuthHeader(oldTimestamp) }); const res = (await handle(event, {})) as { statusCode: number; body: string }; expect(res.statusCode).toBe(401); expect(JSON.parse(res.body).error).toMatch(/expired/i); diff --git a/shared/chain-utils/src/auth-protocol.ts b/shared/chain-utils/src/auth-protocol.ts index f86393b32..f56be22bb 100644 --- a/shared/chain-utils/src/auth-protocol.ts +++ b/shared/chain-utils/src/auth-protocol.ts @@ -1,13 +1,124 @@ +/** + * Wallet-signature auth, both halves. + * + * A client signs `":"` with its wallet and sends + * `Authorization: Bearer `; a service + * decodes that, checks the timestamp is fresh and the recovered signer is who + * it expects. The `prefix` scopes a token to one service, so a token minted + * for the growth API cannot be replayed against another. + * + * Signing and verification live together because they are one wire format: + * changing the message shape or the freshness window on one side silently + * breaks the other. + * + * Clients: `website/hooks/useWalletAuth.ts`. + * Services: `scw_js/growth_service.ts`, `analytics/stats.ts`. + */ +import { verifyMessage } from "viem"; + +/** + * Declared rather than pulled in via `@types/node`, matching `env-utils.ts` — + * and `atob` rather than `Buffer` because this package is consumed by the + * browser too, where `Buffer` does not exist. + */ +declare const atob: (data: string) => string; + /** Maximum age of a wallet-signed auth token in milliseconds (5 minutes). */ export const AUTH_TOKEN_MAX_AGE_MS = 5 * 60 * 1000; +export interface BearerPayload { + address: `0x${string}`; + signature: string; + message: string; +} + /** * Build the message a wallet should sign for authentication. * Format: ":" - * - * Server-side: verified by scw_js/auth_utils.ts verifySignedMessage() - * Client-side: built by website/hooks/useWalletAuth.ts */ export function buildAuthMessage(prefix: string): string { return `${prefix}:${Math.floor(Date.now() / 1000)}`; } + +/** + * Parses a `Bearer ` Authorization header into its decoded payload. + * Returns null if the header is missing, malformed, or fails JSON parsing. + * + * `atob` yields a binary string, so a payload containing non-ASCII would + * decode wrong — harmlessly, since a mangled message cannot then satisfy + * `verifySignedMessage`'s prefix check or verify against its signature. Every + * real token is ASCII: hex address, hex signature, `:`. + */ +export function parseBearerToken(authHeader: string | undefined): BearerPayload | null { + if (!authHeader) { + return null; + } + const match = authHeader.match(/^Bearer\s+(.+)$/i); + if (!match) { + return null; + } + try { + const decoded = JSON.parse(atob(match[1])) as BearerPayload; + const { address, signature, message } = decoded; + if ( + typeof address === "string" && + address.startsWith("0x") && + typeof signature === "string" && + signature.startsWith("0x") && + typeof message === "string" && + message.length > 0 + ) { + return { address, signature, message }; + } + return null; + } catch { + return null; + } +} + +/** + * Verifies a parsed wallet signature payload against an expected message prefix and address. + * Returns null on success, or an error string describing the failure. + * + * Checks (in order): message format, timestamp freshness, address match, signature validity. + * The timestamp check is what stops a captured token being replayed indefinitely. + */ +export async function verifySignedMessage( + address: string, + signature: string, + message: string, + expectedPrefix: string, + expectedAddress: string +): Promise { + const match = message.match(new RegExp(`^${expectedPrefix}:(\\d+)$`)); + if (!match) { + return "Unauthorized"; + } + + const ts = parseInt(match[1], 10); + if (ts > 9_999_999_999) { + return "Unauthorized"; + } // guard against year >2286 / integer overflow + if (Math.abs(Date.now() - ts * 1000) > AUTH_TOKEN_MAX_AGE_MS) { + return "Token expired"; + } + + if (address.toLowerCase() !== expectedAddress.toLowerCase()) { + return "Address mismatch"; + } + + try { + const isValid = await verifyMessage({ + address: address as `0x${string}`, + message, + signature: signature as `0x${string}`, + }); + if (!isValid) { + return "Invalid signature"; + } + } catch { + return "Invalid signature"; + } + + return null; +} diff --git a/shared/chain-utils/src/index.ts b/shared/chain-utils/src/index.ts index f6874813f..21a6557b1 100644 --- a/shared/chain-utils/src/index.ts +++ b/shared/chain-utils/src/index.ts @@ -155,4 +155,10 @@ export { loadPrivateKey, getRpcUrl } from "./env-utils"; // Auth Protocol (isomorphic — shared by scw_js and website) // ═══════════════════════════════════════════════════════════════ -export { AUTH_TOKEN_MAX_AGE_MS, buildAuthMessage } from "./auth-protocol"; +export { + AUTH_TOKEN_MAX_AGE_MS, + buildAuthMessage, + parseBearerToken, + verifySignedMessage, + type BearerPayload, +} from "./auth-protocol"; diff --git a/shared/chain-utils/test/auth-protocol.test.ts b/shared/chain-utils/test/auth-protocol.test.ts index 740ba0b28..b692f56e8 100644 --- a/shared/chain-utils/test/auth-protocol.test.ts +++ b/shared/chain-utils/test/auth-protocol.test.ts @@ -1,5 +1,30 @@ -import { describe, test, expect, vi, afterEach } from "vitest"; -import { AUTH_TOKEN_MAX_AGE_MS, buildAuthMessage } from "../src/auth-protocol"; +import { describe, test, expect, vi, afterEach, beforeEach } from "vitest"; + +const { mockVerifyMessage } = vi.hoisted(() => ({ + mockVerifyMessage: vi.fn(), +})); + +vi.mock("viem", () => ({ + verifyMessage: mockVerifyMessage, +})); + +import { + AUTH_TOKEN_MAX_AGE_MS, + buildAuthMessage, + parseBearerToken, + verifySignedMessage, +} from "../src/auth-protocol"; + +const VALID_ADDRESS = "0x1234567890abcdef1234567890abcdef12345678"; +const VALID_SIGNATURE = "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab"; + +function makeToken(payload: Record): string { + return `Bearer ${Buffer.from(JSON.stringify(payload)).toString("base64")}`; +} + +function freshTs(): number { + return Math.floor(Date.now() / 1000); +} describe("AUTH_TOKEN_MAX_AGE_MS", () => { test("is exactly 5 minutes in milliseconds", () => { @@ -41,3 +66,255 @@ describe("buildAuthMessage", () => { expect(msg).toMatch(/^test:\d+$/); }); }); + +describe("round trip", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + test("a message built here verifies here — the two halves agree on the wire format", async () => { + mockVerifyMessage.mockResolvedValue(true); + const message = buildAuthMessage("analytics-api"); + + const token = parseBearerToken( + makeToken({ address: VALID_ADDRESS, signature: VALID_SIGNATURE, message }) + ); + expect(token).not.toBeNull(); + + const err = await verifySignedMessage( + token!.address, + token!.signature, + token!.message, + "analytics-api", + VALID_ADDRESS + ); + expect(err).toBeNull(); + }); +}); + +describe("parseBearerToken", () => { + test("returns parsed payload for a valid Bearer token", () => { + const payload = { + address: VALID_ADDRESS, + signature: VALID_SIGNATURE, + message: "leaf-history:1234", + }; + expect(parseBearerToken(makeToken(payload))).toEqual(payload); + }); + + test("returns null for undefined header", () => { + expect(parseBearerToken(undefined)).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(parseBearerToken("")).toBeNull(); + }); + + test("returns null for Basic auth (not Bearer)", () => { + expect(parseBearerToken("Basic abc123")).toBeNull(); + }); + + test("returns null for invalid base64 content", () => { + expect(parseBearerToken("Bearer !!!notbase64!!!")).toBeNull(); + }); + + test("returns null when base64 decodes to non-JSON", () => { + const token = `Bearer ${Buffer.from("hello world").toString("base64")}`; + expect(parseBearerToken(token)).toBeNull(); + }); + + test("returns null when address field is missing", () => { + expect(parseBearerToken(makeToken({ signature: VALID_SIGNATURE, message: "m" }))).toBeNull(); + }); + + test("returns null when address is not a string", () => { + expect( + parseBearerToken(makeToken({ address: 123, signature: VALID_SIGNATURE, message: "m" })) + ).toBeNull(); + }); + + test("returns null when address lacks 0x prefix", () => { + expect( + parseBearerToken( + makeToken({ address: "notanaddress", signature: VALID_SIGNATURE, message: "m" }) + ) + ).toBeNull(); + }); + + test("returns null when signature lacks 0x prefix", () => { + expect( + parseBearerToken( + makeToken({ address: VALID_ADDRESS, signature: "nosigprefix", message: "m" }) + ) + ).toBeNull(); + }); + + test("returns null when signature is not a string", () => { + expect( + parseBearerToken(makeToken({ address: VALID_ADDRESS, signature: true, message: "m" })) + ).toBeNull(); + }); + + test("returns null when message is empty string", () => { + expect( + parseBearerToken( + makeToken({ address: VALID_ADDRESS, signature: VALID_SIGNATURE, message: "" }) + ) + ).toBeNull(); + }); + + test("returns null when message is not a string", () => { + expect( + parseBearerToken( + makeToken({ address: VALID_ADDRESS, signature: VALID_SIGNATURE, message: ["array"] }) + ) + ).toBeNull(); + }); + + test("is safe against __proto__ injection — Object.prototype is not modified", () => { + // A crafted raw JSON string attempting prototype pollution + const rawJson = `{"__proto__":{"isAdmin":true},"address":"${VALID_ADDRESS}","signature":"${VALID_SIGNATURE}","message":"leaf-history:1234"}`; + const token = `Bearer ${Buffer.from(rawJson).toString("base64")}`; + parseBearerToken(token); + // Object.prototype must remain unmodified regardless of what JSON.parse does + expect((Object.prototype as Record).isAdmin).toBeUndefined(); + }); +}); + +describe("verifySignedMessage", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("returns null for a fully valid payload", async () => { + mockVerifyMessage.mockResolvedValue(true); + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs()}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBeNull(); + }); + + test("returns 'Unauthorized' for a timestamp far in the future (overflow guard)", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + "leaf-history:99999999999", + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Unauthorized"); + }); + + test("returns 'Unauthorized' for wrong message prefix", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `wrong-prefix:${freshTs()}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Unauthorized"); + }); + + test("returns 'Unauthorized' for completely malformed message", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + "not-a-valid-message", + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Unauthorized"); + }); + + test("returns 'Token expired' for timestamp 6 minutes in the past", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs() - 360}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Token expired"); + }); + + test("returns 'Token expired' for timestamp 6 minutes in the future", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs() + 360}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Token expired"); + }); + + test("returns 'Address mismatch' when payload address differs from expected", async () => { + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs()}`, + "leaf-history", + "0xdeadbeef00000000000000000000000000000000" + ); + expect(result).toBe("Address mismatch"); + }); + + test("passes case-insensitive address comparison", async () => { + mockVerifyMessage.mockResolvedValue(true); + const upper = VALID_ADDRESS.toUpperCase().replace("0X", "0x"); + const result = await verifySignedMessage( + upper, + VALID_SIGNATURE, + `leaf-history:${freshTs()}`, + "leaf-history", + VALID_ADDRESS.toLowerCase() + ); + expect(result).toBeNull(); + }); + + test("returns 'Invalid signature' when verifyMessage returns false", async () => { + mockVerifyMessage.mockResolvedValue(false); + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs()}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Invalid signature"); + }); + + test("returns 'Invalid signature' when verifyMessage throws", async () => { + mockVerifyMessage.mockRejectedValue(new Error("RPC error")); + const result = await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + `leaf-history:${freshTs()}`, + "leaf-history", + VALID_ADDRESS + ); + expect(result).toBe("Invalid signature"); + }); + + test("calls verifyMessage with the exact address, message, and signature", async () => { + mockVerifyMessage.mockResolvedValue(true); + const message = `leaf-history:${freshTs()}`; + await verifySignedMessage( + VALID_ADDRESS, + VALID_SIGNATURE, + message, + "leaf-history", + VALID_ADDRESS + ); + expect(mockVerifyMessage).toHaveBeenCalledWith({ + address: VALID_ADDRESS, + message, + signature: VALID_SIGNATURE, + }); + }); +}); diff --git a/website/hooks/useAnalyticsStats.ts b/website/hooks/useAnalyticsStats.ts new file mode 100644 index 000000000..0bbb9a072 --- /dev/null +++ b/website/hooks/useAnalyticsStats.ts @@ -0,0 +1,40 @@ +import { useQuery } from "@tanstack/react-query"; +import { useAccount } from "wagmi"; +import { useWalletAuth } from "./useWalletAuth"; +import { ANALYTICS_URL } from "../utils/analyticsApi"; +import type { Stats } from "../types/analytics"; + +/** Wakes the function up while the owner is still deciding on a range. */ +export function prewarmAnalyticsApi(): void { + fetch(`${ANALYTICS_URL}/stats`, { method: "OPTIONS" }).catch(() => {}); +} + +/** + * One query for the whole year — the range selector is a display concern and + * never refetches. + * + * `staleTime` overrides the 60s global default in `pages/+config.ts`: a refetch + * would need a fresh signature once `useWalletAuth`'s 4-minute token cache has + * lapsed, so a short window means switching ranges could re-prompt the wallet. + */ +export function useAnalyticsStats(enabled: boolean) { + const { address } = useAccount(); + const getAuth = useWalletAuth("analytics-api"); + + return useQuery({ + queryKey: ["analyticsStats", address], + queryFn: async () => { + const auth = await getAuth(); + const res = await fetch(`${ANALYTICS_URL}/stats`, { + headers: { Authorization: auth }, + }); + if (!res.ok) { + const body = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(body.error ?? `Stats request failed (${res.status})`); + } + return res.json() as Promise; + }, + enabled: enabled && !!address, + staleTime: 5 * 60_000, + }); +} diff --git a/website/layouts/LayoutDefault.tsx b/website/layouts/LayoutDefault.tsx index 07398528c..690278421 100644 --- a/website/layouts/LayoutDefault.tsx +++ b/website/layouts/LayoutDefault.tsx @@ -61,7 +61,7 @@ export default function LayoutDefault({ children }: { children: React.ReactNode Blog Quantum Lab - +
@@ -113,13 +113,21 @@ function Content({ children }: { children: React.ReactNode }) { ); } -function GrowthNavLink() { +/** + * Nav entries only the site owner sees — one gate for all of them, not one per link. + * + * Cosmetic only: both pages check ownership themselves, and `GET /stats` verifies a wallet + * signature server-side. `isConnected` is reconnect-aware, so these never flash in before + * wagmi has finished reconnecting. + */ +function OwnerNavLinks() { const { address, isConnected } = useWalletConnection(); const isOwner = isConnected && address?.toLowerCase() === OWNER_ADDRESS.toLowerCase(); if (!isOwner) return null; return ( -
- Growth -
+ <> + Growth + Analytics + ); } diff --git a/website/pages/analytics/+Page.tsx b/website/pages/analytics/+Page.tsx new file mode 100644 index 000000000..012372116 --- /dev/null +++ b/website/pages/analytics/+Page.tsx @@ -0,0 +1,239 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { css } from "../../styled-system/css"; +import { titleBar, tabs as tabStyles } from "../../layouts/shared"; +import { button } from "../../styled-system/recipes"; +import { Tab } from "../../components/Tab"; +import { useAnalyticsStats, prewarmAnalyticsApi } from "../../hooks/useAnalyticsStats"; +import { useWalletConnection } from "../../hooks/useWalletConnection"; +import { OWNER_ADDRESS } from "../../utils/getChain"; +import { SITE_CONFIG } from "../../utils/siteConfig"; +import { RANGES, sliceStats, type Bucket } from "../../utils/analyticsBuckets"; + +// ===== Styles ===== + +const container = css({ maxWidth: "900px", mx: "auto", px: "md", pt: "md" }); + +const infoBox = css({ padding: "lg", textAlign: "center", color: "gray.600", fontSize: "md" }); + +const errorBanner = css({ + padding: "sm", + marginBottom: "md", + borderRadius: "sm", + backgroundColor: "dangerSurface", + color: "danger", + fontSize: "sm", +}); + +const headline = css({ fontSize: "3xl", fontWeight: "bold", color: "text", lineHeight: "tight" }); + +const subline = css({ fontSize: "sm", color: "textMuted", marginBottom: "lg" }); + +const chart = css({ + display: "flex", + alignItems: "flex-end", + gap: "1px", + height: "120px", + marginBottom: "xs", + borderBottom: "1px solid token(colors.border)", +}); + +const barSlot = css({ flex: 1, height: "100%", display: "flex", alignItems: "flex-end" }); + +const bar = css({ width: "100%", minHeight: "1px", backgroundColor: "brand", borderRadius: "1px" }); + +// Backfilled Umami days, greyed so the seam is visible rather than implied. +const barHistoric = css({ width: "100%", minHeight: "1px", backgroundColor: "gray.400", borderRadius: "1px" }); + +const axis = css({ + display: "flex", + justifyContent: "space-between", + fontSize: "xs", + color: "textMuted", + marginBottom: "xl", +}); + +const table = css({ width: "100%", borderCollapse: "collapse", fontSize: "sm" }); + +const th = css({ + textAlign: "left", + paddingY: "xs", + borderBottom: "1px solid token(colors.border)", + fontSize: "xs", + textTransform: "uppercase", + letterSpacing: "wide", + color: "textMuted", + fontWeight: "semibold", +}); + +const thRight = css({ + textAlign: "right", + paddingY: "xs", + borderBottom: "1px solid token(colors.border)", + fontSize: "xs", + textTransform: "uppercase", + letterSpacing: "wide", + color: "textMuted", + fontWeight: "semibold", +}); + +const td = css({ paddingY: "xs", borderBottom: "1px solid token(colors.border)" }); + +const tdRight = css({ + paddingY: "xs", + borderBottom: "1px solid token(colors.border)", + textAlign: "right", + fontVariantNumeric: "tabular-nums", + color: "textMuted", +}); + +const pathLink = css({ color: "brand", textDecoration: "none", _hover: { textDecoration: "underline" } }); + +const footnote = css({ fontSize: "xs", color: "textMuted", marginTop: "lg" }); + +// ===== Helpers ===== + +function formatDay(date: string): string { + return new Date(`${date}T00:00:00Z`).toLocaleDateString(undefined, { + month: "short", + day: "numeric", + timeZone: "UTC", + }); +} + +function barClass(bucket: Bucket): string { + return bucket.historic ? barHistoric : bar; +} + +// ===== Page ===== + +export default function Page() { + const { address, hasMounted, isConnected, connectWallet } = useWalletConnection(); + const [rangeIndex, setRangeIndex] = useState(0); + + useEffect(() => { + prewarmAnalyticsApi(); + }, []); + + // isConnected is reconnect-aware + hydration-safe, so the owner check never + // trusts `address` before wagmi's reconnect completes. + const isOwner = isConnected && address?.toLowerCase() === OWNER_ADDRESS.toLowerCase(); + + // One query for the whole year; the range selector only re-slices it, so + // switching is instant and never refetches. + const { data: stats, isPending, error } = useAnalyticsStats(!!isOwner); + + const range = RANGES[rangeIndex]; + const view = useMemo(() => (stats ? sliceStats(stats, range) : null), [stats, range]); + const peak = useMemo(() => Math.max(1, ...(view?.buckets ?? []).map((b) => b.hits)), [view?.buckets]); + + if (!hasMounted) { + return ( +
+

Analytics

+

Loading...

+
+ ); + } + + if (!isConnected) { + return ( +
+

Analytics

+
+
+ +
+
+
+ ); + } + + if (!isOwner) { + return ( +
+

Analytics

+

This page is restricted to the site owner.

+
+ ); + } + + return ( +
+

Analytics

+ +
+ {RANGES.map((option, index) => ( + setRangeIndex(index)} + /> + ))} +
+ + {error &&
{error instanceof Error ? error.message : "Failed to load stats"}
} + + {isPending && !view ? ( +

Loading stats...

+ ) : view ? ( + <> +
{view.totalHits.toLocaleString()} views
+
+ across {view.pages.length} {view.pages.length === 1 ? "page" : "pages"} · {formatDay(view.from)} –{" "} + {formatDay(view.to)} +
+ +
+ {view.buckets.map((bucket) => ( +
+
+
+ ))} +
+
+ {formatDay(view.from)} + {formatDay(view.to)} +
+ + + + + + + + + + {view.pages.map((page) => ( + + + + + ))} + +
PageViews
+ + {page.path} + + {page.hits.toLocaleString()}
+ + {view.pages.length === 0 &&

No traffic recorded in this range.

} + + {view.hasHistoric && ( +

+ Grey bars predate the hit counter and were backfilled from Umami, which filtered bots and counted sessions + rather than pageviews — the two are not directly comparable. +

+ )} + + ) : null} +
+ ); +} diff --git a/website/test/AnalyticsPage.test.tsx b/website/test/AnalyticsPage.test.tsx new file mode 100644 index 000000000..5f71a78c9 --- /dev/null +++ b/website/test/AnalyticsPage.test.tsx @@ -0,0 +1,183 @@ +import React from "react"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen, fireEvent, cleanup, within } from "@testing-library/react"; +import { useAccount, useConnect } from "wagmi"; +import { OWNER_ADDRESS } from "../utils/getChain"; +import { buildAccountData, buildConnectData } from "./setup"; +import type { Stats } from "../types/analytics"; + +const mockUseAnalyticsStats = vi.fn(); +const mockPrewarmAnalyticsApi = vi.fn(); + +vi.mock("../hooks/useAnalyticsStats", () => ({ + useAnalyticsStats: (...args: unknown[]) => mockUseAnalyticsStats(...args), + prewarmAnalyticsApi: () => mockPrewarmAnalyticsApi(), +})); + +vi.mock("../styled-system/css", () => ({ + css: () => "mock-css-class", +})); + +import Page from "../pages/analytics/+Page"; + +const TODAY = "2026-08-11"; + +const sampleStats: Stats = { + site: "fretchen.eu", + from: "2025-08-12", + to: TODAY, + days: { + // Inside the year but outside 30 days — only the 1-year view should see it. + "2026-02-14": { hits: 500, pages: { "/old/": 500 }, source: "umami" }, + "2026-08-10": { hits: 240, pages: { "/": 200, "/x402/": 40 }, source: "beacon" }, + "2026-08-11": { hits: 507, pages: { "/": 312, "/x402/": 22, "/blog/": 173 }, source: "beacon" }, + }, +}; + +function connectAs(address: string | undefined) { + vi.mocked(useAccount).mockReturnValue( + buildAccountData({ + address: address as `0x${string}` | undefined, + isConnected: address !== undefined, + status: address !== undefined ? "connected" : "disconnected", + }), + ); + vi.mocked(useConnect).mockReturnValue(buildConnectData({ connectors: [{ name: "MetaMask" }] })); +} + +describe("Analytics Page", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseAnalyticsStats.mockReturnValue({ data: sampleStats, isPending: false, error: null }); + }); + + afterEach(() => { + cleanup(); + }); + + it("prompts to connect when no wallet is attached", () => { + connectAs(undefined); + render(); + expect(screen.getByText("Connect Wallet")).toBeInTheDocument(); + }); + + it("prewarms the stats function on mount, before any wallet is connected", () => { + connectAs(undefined); + render(); + expect(mockPrewarmAnalyticsApi).toHaveBeenCalledOnce(); + }); + + it("refuses a connected wallet that is not the owner", () => { + connectAs("0x1111111111111111111111111111111111111111"); + render(); + expect(screen.getByText("This page is restricted to the site owner.")).toBeInTheDocument(); + expect(screen.queryByText(/views/)).not.toBeInTheDocument(); + }); + + it("does not request stats for a non-owner", () => { + connectAs("0x1111111111111111111111111111111111111111"); + render(); + expect(mockUseAnalyticsStats).toHaveBeenCalledWith(false); + }); + + it("offers the three granularities and no 7-day view", () => { + connectAs(OWNER_ADDRESS); + render(); + expect(screen.getByText("30 days")).toBeInTheDocument(); + expect(screen.getByText("90 days")).toBeInTheDocument(); + expect(screen.getByText("1 year")).toBeInTheDocument(); + expect(screen.queryByText("7 days")).not.toBeInTheDocument(); + }); + + it("defaults to 30 days, one bar per day, totals scoped to that window", () => { + connectAs(OWNER_ADDRESS); + const { container } = render(); + + expect(screen.getByText("747 views")).toBeInTheDocument(); // 240 + 507, not the 500 from February + expect(container.querySelectorAll("[title$='views']")).toHaveLength(30); + }); + + // The whole point of the rework: the range selector re-slices one payload. + it("never refetches when the range changes", () => { + connectAs(OWNER_ADDRESS); + render(); + const callsBefore = mockUseAnalyticsStats.mock.calls.length; + + fireEvent.click(screen.getByText("1 year")); + + expect(mockUseAnalyticsStats.mock.calls.slice(callsBefore).every(([enabled]) => enabled === true)).toBe(true); + // The hook takes no range argument at all, so there is nothing to key a refetch on. + expect(mockUseAnalyticsStats).toHaveBeenLastCalledWith(true); + }); + + it("switches 90 days to weekly buckets", () => { + connectAs(OWNER_ADDRESS); + const { container } = render(); + + fireEvent.click(screen.getByText("90 days")); + + const bars = container.querySelectorAll("[title$='views']"); + expect(bars.length).toBeGreaterThanOrEqual(13); + expect(bars.length).toBeLessThanOrEqual(14); + expect(container.querySelector("[title^='Week of ']")).toBeInTheDocument(); + }); + + it("switches 1 year to monthly buckets and picks up the older data", () => { + connectAs(OWNER_ADDRESS); + const { container } = render(); + + fireEvent.click(screen.getByText("1 year")); + + const bars = container.querySelectorAll("[title$='views']"); + expect(bars.length).toBeGreaterThanOrEqual(12); + expect(bars.length).toBeLessThanOrEqual(13); + expect(container.querySelector("[title='Feb 2026: 500 views']")).toBeInTheDocument(); + expect(screen.getByText("1,247 views")).toBeInTheDocument(); // now including February + }); + + it("rescopes the top-pages table to the selected range", () => { + connectAs(OWNER_ADDRESS); + render(); + expect(screen.queryByRole("link", { name: "/old/" })).not.toBeInTheDocument(); + + fireEvent.click(screen.getByText("1 year")); + + expect(screen.getByRole("link", { name: "/old/" })).toBeInTheDocument(); + }); + + it("lists the top pages with links to the live site", () => { + connectAs(OWNER_ADDRESS); + render(); + const row = screen.getByRole("link", { name: "/x402/" }); + expect(row).toHaveAttribute("href", "https://www.fretchen.eu/x402/"); + expect(within(row.closest("tr")!).getByText("62")).toBeInTheDocument(); + }); + + it("flags the Umami seam only when the range reaches into it", () => { + connectAs(OWNER_ADDRESS); + render(); + expect(screen.queryByText(/backfilled from Umami/)).not.toBeInTheDocument(); + + fireEvent.click(screen.getByText("1 year")); + + expect(screen.getByText(/backfilled from Umami/)).toBeInTheDocument(); + }); + + it("surfaces an auth failure from the endpoint", () => { + connectAs(OWNER_ADDRESS); + mockUseAnalyticsStats.mockReturnValue({ data: undefined, isPending: false, error: new Error("Token expired") }); + render(); + expect(screen.getByText("Token expired")).toBeInTheDocument(); + }); + + it("says so when the range has no traffic", () => { + connectAs(OWNER_ADDRESS); + mockUseAnalyticsStats.mockReturnValue({ + data: { ...sampleStats, days: {} }, + isPending: false, + error: null, + }); + render(); + expect(screen.getByText("No traffic recorded in this range.")).toBeInTheDocument(); + }); +}); diff --git a/website/test/analyticsBuckets.test.ts b/website/test/analyticsBuckets.test.ts new file mode 100644 index 000000000..bceb43742 --- /dev/null +++ b/website/test/analyticsBuckets.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect } from "vitest"; +import { RANGES, bucketize, sliceStats } from "../utils/analyticsBuckets"; +import type { DayBucket, Stats } from "../types/analytics"; + +const TODAY = "2026-08-11"; // a Tuesday + +function day(hits: number, pages: Record = { "/": hits }, source = "beacon"): DayBucket { + return { hits, pages, source }; +} + +const range = (days: number) => RANGES.find((r) => r.days === days)!; + +describe("date handling", () => { + it("walks the window in UTC across month and year boundaries", () => { + // 30 days back from 2026-01-05 crosses into the previous year. + const { buckets } = bucketize({ "2025-12-31": day(4) }, range(30), "2026-01-05"); + expect(buckets).toHaveLength(30); + expect(buckets[0].key).toBe("2025-12-07"); + expect(buckets.find((b) => b.key === "2025-12-31")?.hits).toBe(4); + }); +}); + +describe("RANGES", () => { + it("offers one range per granularity — 7 days was noise, 90 daily bars unreadable", () => { + expect(RANGES.map((r) => [r.days, r.granularity])).toEqual([ + [30, "day"], + [90, "week"], + [365, "month"], + ]); + }); +}); + +describe("bucketize", () => { + it("gives one bucket per day over 30 days", () => { + const { buckets, from, to } = bucketize({}, range(30), TODAY); + expect(buckets).toHaveLength(30); + expect(from).toBe("2026-07-13"); + expect(to).toBe(TODAY); + }); + + it("groups 90 days into Monday-start weeks", () => { + const { buckets } = bucketize({ [TODAY]: day(5) }, range(90), TODAY); + + // 2026-08-11 is a Tuesday, so its week starts Monday 2026-08-10. + const last = buckets[buckets.length - 1]; + expect(last.key).toBe("2026-08-10"); + expect(last.label).toBe("Week of Aug 10"); + expect(last.hits).toBe(5); + expect(buckets.length).toBeGreaterThanOrEqual(13); + expect(buckets.length).toBeLessThanOrEqual(14); + }); + + it("sums a whole week into one bucket", () => { + const days = { + "2026-08-03": day(1), // Monday + "2026-08-06": day(2), + "2026-08-09": day(4), // Sunday — same week + "2026-08-10": day(8), // next Monday — different bucket + }; + + const { buckets } = bucketize(days, range(90), TODAY); + const byKey = Object.fromEntries(buckets.map((b) => [b.key, b.hits])); + + expect(byKey["2026-08-03"]).toBe(7); + expect(byKey["2026-08-10"]).toBe(8); + }); + + it("groups a year into 12 or 13 calendar months", () => { + const { buckets } = bucketize({ "2026-03-04": day(18) }, range(365), TODAY); + + const march = buckets.find((b) => b.key === "2026-03-01"); + expect(march?.hits).toBe(18); + expect(march?.label).toBe("Mar 2026"); + expect(buckets.length).toBeGreaterThanOrEqual(12); + expect(buckets.length).toBeLessThanOrEqual(13); + }); + + it("keeps empty buckets so the axis stays evenly spaced", () => { + const { buckets } = bucketize({ [TODAY]: day(3) }, range(30), TODAY); + expect(buckets.filter((b) => b.hits === 0)).toHaveLength(29); + }); + + it("ignores days outside the window", () => { + const days = { "2026-01-01": day(999), [TODAY]: day(2) }; + const { buckets } = bucketize(days, range(30), TODAY); + expect(buckets.reduce((sum, b) => sum + b.hits, 0)).toBe(2); + }); + + it("marks a bucket historic when any day in it predates the counter", () => { + const days = { + "2026-08-10": day(4, { "/": 4 }, "umami"), + "2026-08-11": day(6, { "/": 6 }, "beacon"), + }; + + const { buckets } = bucketize(days, range(90), TODAY); + const seam = buckets.find((b) => b.key === "2026-08-10"); + + // Both days fall in the same week — mixed, so not comparable, so flagged. + expect(seam?.hits).toBe(10); + expect(seam?.historic).toBe(true); + }); + + it("leaves a purely beacon-sourced bucket unflagged", () => { + const { buckets } = bucketize({ [TODAY]: day(6) }, range(30), TODAY); + expect(buckets.some((b) => b.historic)).toBe(false); + }); +}); + +describe("totals and top pages", () => { + const stats: Stats = { + site: "fretchen.eu", + from: "2025-08-12", + to: TODAY, + days: { + "2026-08-09": day(5, { "/": 3, "/blog/": 2 }), + "2026-08-10": day(4, { "/blog/": 4 }), + "2026-02-01": day(99, { "/old/": 99 }), + }, + }; + + it("sums only the selected window", () => { + expect(sliceStats(stats, range(30)).totalHits).toBe(9); + expect(sliceStats(stats, range(365)).totalHits).toBe(108); + }); + + it("merges page counts across the window, most-hit first", () => { + expect(sliceStats(stats, range(30)).pages).toEqual([ + { path: "/blog/", hits: 6 }, + { path: "/", hits: 3 }, + ]); + }); + + it("excludes pages whose traffic falls outside the window", () => { + expect(sliceStats(stats, range(30)).pages.map((p) => p.path)).not.toContain("/old/"); + expect(sliceStats(stats, range(365)).pages.map((p) => p.path)).toContain("/old/"); + }); +}); + +describe("sliceStats", () => { + const stats: Stats = { + site: "fretchen.eu", + from: "2025-08-12", + to: TODAY, + days: { + "2026-02-14": day(20, { "/old/": 20 }, "umami"), + "2026-08-10": day(5, { "/": 5 }), + "2026-08-11": day(7, { "/blog/": 7 }), + }, + }; + + it("narrows the totals and pages as the range shrinks", () => { + const year = sliceStats(stats, range(365)); + const month = sliceStats(stats, range(30)); + + expect(year.totalHits).toBe(32); + expect(month.totalHits).toBe(12); + expect(year.pages.map((p) => p.path)).toContain("/old/"); + expect(month.pages.map((p) => p.path)).not.toContain("/old/"); + }); + + it("reports historic data only when the range reaches back into it", () => { + expect(sliceStats(stats, range(365)).hasHistoric).toBe(true); + expect(sliceStats(stats, range(30)).hasHistoric).toBe(false); + }); + + it("anchors the window on the response's `to`, not the local clock", () => { + const slice = sliceStats(stats, range(30)); + expect(slice.to).toBe(TODAY); + expect(slice.from).toBe("2026-07-13"); + }); +}); diff --git a/website/types/analytics.ts b/website/types/analytics.ts new file mode 100644 index 000000000..f36e04026 --- /dev/null +++ b/website/types/analytics.ts @@ -0,0 +1,26 @@ +/** Response shape of `GET /stats` — keep in sync with `analytics/stats.ts`. */ + +export interface DayBucket { + hits: number; + /** Path → count for that day. Summed client-side over whatever range is shown. */ + pages: Record; + /** + * `"beacon"` for anything the hit counter recorded, `"umami"` for days + * backfilled from the old Umami export. The two are not the same + * measurement — Umami filtered bots and sessionised — so the page labels the + * seam rather than smoothing it over. + */ + source: string; +} + +export interface Stats { + site: string; + from: string; + to: string; + /** + * Sparse, keyed `YYYY-MM-DD`: days with no traffic are absent. Always the + * trailing year — the endpoint takes no range, because the whole thing is + * ~3KB gzipped and the page slices it locally. + */ + days: Record; +} diff --git a/website/utils/analyticsApi.ts b/website/utils/analyticsApi.ts new file mode 100644 index 000000000..95da1bdfe --- /dev/null +++ b/website/utils/analyticsApi.ts @@ -0,0 +1,14 @@ +/** + * Base URL of the `analytics` Scaleway function, which serves both endpoints: + * `POST /hit` (anonymous beacon) and `GET /stats` (owner-gated readout). + * + * One constant because it is one function — get the value from `npm run info` + * in `analytics/` after deploying. + * + * The fallback is what production actually uses: `.github/workflows/pages.yml` + * sets no `PUBLIC_ENV__*` variables, so the env var is a local-dev override + * only (point it at `npm run dev` on localhost:8086). + */ +export const ANALYTICS_URL = + (import.meta.env.PUBLIC_ENV__ANALYTICS_URL as string | undefined) ?? + "https://analyticsserviceebp8thpt-analytics.functions.fnc.fr-par.scw.cloud"; diff --git a/website/utils/analyticsBuckets.ts b/website/utils/analyticsBuckets.ts new file mode 100644 index 000000000..fc2afec8a --- /dev/null +++ b/website/utils/analyticsBuckets.ts @@ -0,0 +1,173 @@ +/** + * Slicing and bucketing for the `/analytics` dashboard. + * + * `GET /stats` returns the trailing year as a sparse day map and nothing else — + * no windowing, no totals, no top-pages list. All of that is range-dependent, + * and the range lives here, in the browser. These are pure functions over that + * map so the page stays a rendering concern. + * + * Buckets are **calendar-aligned** (Monday-start ISO weeks, calendar months) + * rather than trailing N-day chunks: labels then stay stable across reloads, + * and monthly buckets line up with the `rollup/{site}/{YYYY-MM}.json` objects + * the data came from. Leading and trailing partial buckets are kept as they + * are — the current week or month is genuinely incomplete, and hiding it would + * be worse than showing it short. + */ +import type { DayBucket, Stats } from "../types/analytics"; + +export type Granularity = "day" | "week" | "month"; + +export interface Range { + days: number; + granularity: Granularity; + label: string; +} + +/** + * 7 days is noise at this traffic level, and 90 days of daily bars is + * unreadable — hence one range per granularity rather than one per duration. + */ +export const RANGES: Range[] = [ + { days: 30, granularity: "day", label: "30 days" }, + { days: 90, granularity: "week", label: "90 days" }, + { days: 365, granularity: "month", label: "1 year" }, +]; + +export interface Bucket { + /** Stable identity for React keys and tests, e.g. `2026-08-03`. */ + key: string; + label: string; + hits: number; + /** True when any day in the bucket predates the counter — see `DayBucket.source`. */ + historic: boolean; +} + +// ===== Date helpers (UTC — the day keys are UTC, so these must be too) ===== + +function parseDay(day: string): Date { + return new Date(`${day}T00:00:00Z`); +} + +function toIsoDate(date: Date): string { + return date.toISOString().slice(0, 10); +} + +function addDays(day: string, delta: number): string { + const date = parseDay(day); + date.setUTCDate(date.getUTCDate() + delta); + return toIsoDate(date); +} + +/** Inclusive. */ +function daysInRange(from: string, to: string): string[] { + const days: string[] = []; + for (let day = from; day <= to; day = addDays(day, 1)) { + days.push(day); + } + return days; +} + +/** The Monday on or before `day`. */ +function weekStart(day: string): string { + const date = parseDay(day); + const dow = (date.getUTCDay() + 6) % 7; // Monday = 0 + return addDays(day, -dow); +} + +// ===== Slicing ===== + +/** The inclusive window a range covers, ending today. */ +function windowFor(range: Range, today: string): { from: string; to: string } { + return { from: addDays(today, -(range.days - 1)), to: today }; +} + +function totalHits(days: Record, from: string, to: string): number { + return Object.entries(days).reduce((sum, [day, bucket]) => (day >= from && day <= to ? sum + bucket.hits : sum), 0); +} + +/** Merged path counts over the window, most-hit first. Ties broken by path. */ +function topPages( + days: Record, + from: string, + to: string, + limit = 50, +): { path: string; hits: number }[] { + const pages: Record = {}; + for (const [day, bucket] of Object.entries(days)) { + if (day < from || day > to) { + continue; + } + for (const [path, count] of Object.entries(bucket.pages)) { + pages[path] = (pages[path] ?? 0) + count; + } + } + return Object.entries(pages) + .map(([path, hits]) => ({ path, hits })) + .sort((a, b) => b.hits - a.hits || a.path.localeCompare(b.path)) + .slice(0, limit); +} + +function bucketKeyFor(day: string, granularity: Granularity): string { + if (granularity === "week") { + return weekStart(day); + } + if (granularity === "month") { + return `${day.slice(0, 7)}-01`; + } + return day; +} + +function labelFor(key: string, granularity: Granularity): string { + const date = parseDay(key); + if (granularity === "month") { + return date.toLocaleDateString(undefined, { month: "short", year: "numeric", timeZone: "UTC" }); + } + const short = date.toLocaleDateString(undefined, { month: "short", day: "numeric", timeZone: "UTC" }); + return granularity === "week" ? `Week of ${short}` : short; +} + +/** + * Groups the window into calendar buckets, including empty ones — a gap in + * traffic is information, so the axis stays evenly spaced. + */ +export function bucketize( + days: Record, + range: Range, + today: string, +): { buckets: Bucket[]; from: string; to: string } { + const { from, to } = windowFor(range, today); + + const buckets: Bucket[] = []; + const byKey = new Map(); + + for (const day of daysInRange(from, to)) { + const key = bucketKeyFor(day, range.granularity); + let bucket = byKey.get(key); + if (!bucket) { + bucket = { key, label: labelFor(key, range.granularity), hits: 0, historic: false }; + byKey.set(key, bucket); + buckets.push(bucket); + } + + const entry = days[day]; + if (entry) { + bucket.hits += entry.hits; + bucket.historic ||= entry.source === "umami"; + } + } + + return { buckets, from, to }; +} + +/** Convenience for the page: everything it needs for one selected range. */ +export function sliceStats(stats: Stats, range: Range) { + const { buckets, from, to } = bucketize(stats.days, range, stats.to); + return { + buckets, + from, + to, + totalHits: totalHits(stats.days, from, to), + pages: topPages(stats.days, from, to), + hasHistoric: buckets.some((bucket) => bucket.historic), + }; +} diff --git a/website/utils/hitTracker.ts b/website/utils/hitTracker.ts index 0ec0e37e8..d68782127 100644 --- a/website/utils/hitTracker.ts +++ b/website/utils/hitTracker.ts @@ -1,6 +1,4 @@ -const ANALYTICS_URL = - (import.meta.env.PUBLIC_ENV__ANALYTICS_URL as string | undefined) ?? - "https://analyticsserviceebp8thpt-hit.functions.fnc.fr-par.scw.cloud"; +import { ANALYTICS_URL } from "./analyticsApi"; export function trackHit(path: string) { navigator.sendBeacon(`${ANALYTICS_URL}/hit`, JSON.stringify({ site: "fretchen.eu", path }));