Better analytics

analytics/buckets.ts

@@ -31,11 +31,20 @@ export const HOURLY_FALLBACK_DAYS = 14;
 
 export interface HourBucket {
   hits: number;
+  /** Fresh page loads only (onHydrationEnd), not in-app navigations — see hit.ts. */
+  landings: number;
   pages: Record<string, number>;
 }
 
 export interface DayBucket {
   hits: number;
+  /**
+   * Absent on any day written before this field existed — every "umami" day
+   * and any older "beacon" day. Not retrofittable (see
+   * analytics/notebooks/umami_backfill.py's scoping note); treat a missing
+   * value as unknown, not zero.
+   */
+  landings?: number;
   pages: Record<string, number>;
   /** `"beacon"` for anything this service counted; `"umami"` for backfilled history. */
   source: string;
@@ -121,6 +130,7 @@ export async function readDayFromHourly(store: HitStorage, site: string, day: st
   const results = await Promise.all(hourKeys(site, day).map((key) => store.getWithMeta(key)));
 
   let hits = 0;
+  let landings = 0;
   let found = false;
   const pages: Record<string, number> = {};
 
@@ -131,10 +141,11 @@ export async function readDayFromHourly(store: HitStorage, site: string, day: st
     found = true;
     const bucket = JSON.parse(result.body) as HourBucket;
     hits += bucket.hits ?? 0;
+    landings += bucket.landings ?? 0; // 0 for hours written before this field existed
     mergePages(pages, bucket.pages ?? {});
   }
 
-  return found ? { hits, pages: topPages(pages), source: "beacon" } : null;
+  return found ? { hits, landings, pages: topPages(pages), source: "beacon" } : null;
 }
 
 async function readRollup(store: HitStorage, site: string, month: string): Promise<MonthRollup | null> {

analytics/hit.ts

@@ -23,6 +23,37 @@ const MAX_PATH_LENGTH = 200;
 const MAX_PAGES_PER_BUCKET = 200; // caps distinct paths tracked per hour bucket
 const MAX_CAS_ATTEMPTS = 3;
 
+/**
+ * Self-identifying crawlers only — catches honest bots, not the chronic
+ * evasive crawler found in analytics/notebooks/05_traffic_bursts.ipynb (that
+ * one never announces itself; see the notebook's "Tier 2" note on why an IP/CIDR
+ * approach was parked instead of built speculatively). Nothing is stored: the
+ * UA is inspected per-request to decide whether to write, then discarded —
+ * same privacy posture as everything else here.
+ */
+const BOT_USER_AGENTS = [
+  "googlebot",
+  "bingbot",
+  "ahrefsbot",
+  "semrushbot",
+  "mj12bot",
+  "gptbot",
+  "ccbot",
+  "claudebot",
+  "perplexitybot",
+  "yandexbot",
+  "petalbot",
+  "bytespider",
+];
+
+function isKnownBot(userAgent: string | undefined): boolean {
+  if (!userAgent) {
+    return false;
+  }
+  const lower = userAgent.toLowerCase();
+  return BOT_USER_AGENTS.some((bot) => lower.includes(bot));
+}
+
 // `vike dev` serves on 3000; 5173 covers a plain `vite dev` fallback.
 const ALLOWED_ORIGINS = ["https://www.fretchen.eu", "http://localhost:3000", "http://localhost:5173"];
 
@@ -45,6 +76,8 @@ function getCorsHeaders(origin?: string): Record<string, string> {
 
 interface HourBucket {
   hits: number;
+  /** Fresh page loads only (onHydrationEnd), not in-app navigations — see hitTracker.ts. */
+  landings: number;
   pages: Record<string, number>;
 }
 
@@ -71,14 +104,25 @@ function hourKey(site: string, now: Date = new Date()): string {
  * read on a CAS conflict; after MAX_CAS_ATTEMPTS gives up silently — a lost
  * count is fine, never worth failing the request over.
  */
-async function incrementHit(store: HitStorage, site: string, path: string): Promise<void> {
+async function incrementHit(store: HitStorage, site: string, path: string, landing: boolean): Promise<void> {
   const key = hourKey(site);
 
   for (let attempt = 1; attempt <= MAX_CAS_ATTEMPTS; attempt++) {
     const existing = await store.getWithMeta(key);
-    const bucket: HourBucket = existing ? (JSON.parse(existing.body) as HourBucket) : { hits: 0, pages: {} };
+    // Stored data can predate `landings` even though the type says it's
+    // always there — read it as partial and default explicitly, rather than
+    // asserting the parsed JSON matches HourBucket outright.
+    const parsed = existing ? (JSON.parse(existing.body) as Partial<HourBucket>) : null;
+    const bucket: HourBucket = {
+      hits: parsed?.hits ?? 0,
+      landings: parsed?.landings ?? 0,
+      pages: parsed?.pages ?? {},
+    };
 
     bucket.hits += 1;
+    if (landing) {
+      bucket.landings += 1;
+    }
     if (bucket.pages[path] !== undefined || Object.keys(bucket.pages).length < MAX_PAGES_PER_BUCKET) {
       bucket.pages[path] = (bucket.pages[path] ?? 0) + 1;
     }
@@ -109,6 +153,14 @@ export async function handleHit(event: ScalewayEvent, _context: unknown): Promis
     };
   }
 
+  if (isKnownBot(event.headers?.["user-agent"] ?? event.headers?.["User-Agent"])) {
+    return {
+      statusCode: 400,
+      headers: corsHeaders,
+      body: JSON.stringify({ error: "Not tracked" }),
+    };
+  }
+
   if (!event.body) {
     return {
       statusCode: 400,
@@ -137,7 +189,12 @@ export async function handleHit(event: ScalewayEvent, _context: unknown): Promis
     };
   }
 
-  await incrementHit(defaultStorage, ALLOWED_SITE, path);
+  // Defaults to false rather than rejecting the request: an old cached client
+  // bundle without this field should keep counting hits, just without the
+  // landing/navigation split.
+  const landing = parsed.landing === true;
+
+  await incrementHit(defaultStorage, ALLOWED_SITE, path, landing);
 
   return { statusCode: 204, headers: corsHeaders, body: "" };
 }