Add first-party analytics gateway

README.md

@@ -34,7 +34,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow. The cross-
 
 ## First-party event measurement
 
-Cloudflare D1 stores four anonymous event types:
+Cloudflare D1 stores four anonymous website event types:
 
 - `page_viewed`
 - `download_clicked`
@@ -43,7 +43,7 @@ Cloudflare D1 stores four anonymous event types:
 
 The implementation intentionally stores no IP address, cookie, fingerprint, referrer, or persistent visitor identifier. Counts therefore represent events rather than unique visitors. `download_failed` is limited to a failure in ScientFactory's redirect service; the website cannot observe a transfer failure after GitHub begins serving an installer.
 
-The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` D1 database. Apply new migrations before deploying code that depends on them:
+The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` D1 database. New events use the shared `analytics_events` table; the earlier `site_events` table remains as read-only historical data. Apply new migrations before deploying code that depends on them:
 
 ```sh
 bun run db:migrate
@@ -56,3 +56,22 @@ bun run analytics:report
 ```
 
 Local and Cloudflare preview hosts do not write events, which keeps production counts free of development traffic.
+
+## Analytics gateway
+
+The Worker under `workers/events` is ScientFactory's first-party telemetry gateway. Desktop clients submit bounded event batches to `https://events.scientfactory.com/v1/events`; the Worker stores them in D1 first and can then forward anonymous copies to the ScientFactory EU PostHog project. PostHog is an optional analysis layer rather than the primary event store.
+
+Generate binding types and validate the Worker with:
+
+```sh
+bun run events:types
+bun run events:typecheck
+```
+
+Deploy the Worker only from an approved production change:
+
+```sh
+bun run events:deploy
+```
+
+`POSTHOG_PROJECT_TOKEN` is a Cloudflare Worker secret and must never be committed. If it is absent, ingestion continues and events remain queued in D1 for later delivery.

functions/_lib/events.ts

@@ -30,17 +30,15 @@ export interface SiteEventContext {
 }
 
 const INSERT_EVENT = `
-  INSERT INTO site_events (
+  INSERT OR IGNORE INTO analytics_events (
+    event_id,
     event_name,
-    page_path,
-    asset_key,
-    release_tag,
-    asset_name,
-    destination_host,
-    destination_path,
-    failure_stage,
-    failure_reason
-  ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    source,
+    privacy_level,
+    occurred_at,
+    distinct_id,
+    properties_json
+  ) VALUES (?, ?, 'website', ?, ?, ?, ?)
 `;
 
 const PRODUCTION_HOSTS = new Set(["scientfactory.com", "www.scientfactory.com"]);
@@ -82,18 +80,29 @@ export function shouldPersistSiteEvents(request: Request): boolean {
 }
 
 export async function insertSiteEvent(db: D1Database, event: SiteEvent): Promise<void> {
+  const eventId = crypto.randomUUID();
+  const properties = Object.fromEntries(
+    Object.entries({
+      page_path: cleanPath(event.pagePath),
+      asset_key: limited(event.assetKey, 64),
+      release_tag: limited(event.releaseTag, 80),
+      asset_name: limited(event.assetName, 255),
+      destination_host: limited(event.destinationHost, 253),
+      destination_path: cleanPath(event.destinationPath),
+      failure_stage: limited(event.failureStage, 80),
+      failure_reason: limited(event.failureReason, 120),
+    }).filter((entry): entry is [string, string] => entry[1] !== null),
+  );
+
   await db
     .prepare(INSERT_EVENT)
     .bind(
+      eventId,
       event.eventName,
-      cleanPath(event.pagePath),
-      limited(event.assetKey, 64),
-      limited(event.releaseTag, 80),
-      limited(event.assetName, 255),
-      limited(event.destinationHost, 253),
-      cleanPath(event.destinationPath),
-      limited(event.failureStage, 80),
-      limited(event.failureReason, 120),
+      event.eventName === "download_failed" ? "diagnostic" : "product",
+      new Date().toISOString(),
+      `web-event:${eventId}`,
+      JSON.stringify(properties),
     )
     .run();
 }

functions/api/download/[asset].test.ts

@@ -59,15 +59,20 @@ describe("tracked download redirect", () => {
     expect(context.waitUntil).toHaveBeenCalledTimes(1);
     await context.waitUntil.mock.calls[0]?.[0];
     expect(db.bind).toHaveBeenCalledWith(
+      expect.any(String),
       "download_clicked",
-      "/download",
-      "macArm64",
-      "v0.5.7",
-      "Scient-0.5.7-arm64.dmg",
-      "github.com",
-      "/ScientFactory/scient-desktop/releases/download/v0.5.7/Scient-0.5.7-arm64.dmg",
-      null,
-      null,
+      "product",
+      expect.any(String),
+      expect.stringMatching(/^web-event:/),
+      JSON.stringify({
+        page_path: "/download",
+        asset_key: "macArm64",
+        release_tag: "v0.5.7",
+        asset_name: "Scient-0.5.7-arm64.dmg",
+        destination_host: "github.com",
+        destination_path:
+          "/ScientFactory/scient-desktop/releases/download/v0.5.7/Scient-0.5.7-arm64.dmg",
+      }),
     );
   });
 
@@ -97,15 +102,17 @@ describe("tracked download redirect", () => {
     expect(response.status).toBe(503);
     await context.waitUntil.mock.calls[0]?.[0];
     expect(db.bind).toHaveBeenCalledWith(
+      expect.any(String),
       "download_failed",
-      "/download",
-      "macArm64",
-      null,
-      null,
-      null,
-      null,
-      "release_fetch",
-      "upstream_unavailable",
+      "diagnostic",
+      expect.any(String),
+      expect.stringMatching(/^web-event:/),
+      JSON.stringify({
+        page_path: "/download",
+        asset_key: "macArm64",
+        failure_stage: "release_fetch",
+        failure_reason: "upstream_unavailable",
+      }),
     );
   });
 

functions/api/events.test.ts

@@ -49,15 +49,12 @@ describe("browser event endpoint", () => {
     expect(response.status).toBe(204);
     await context.waitUntil.mock.calls[0]?.[0];
     expect(database.bind).toHaveBeenCalledWith(
+      expect.any(String),
       "page_viewed",
-      "/docs",
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
-      null,
+      "product",
+      expect.any(String),
+      expect.stringMatching(/^web-event:/),
+      JSON.stringify({ page_path: "/docs" }),
     );
   });
 
@@ -73,15 +70,16 @@ describe("browser event endpoint", () => {
     expect(response.status).toBe(204);
     await context.waitUntil.mock.calls[0]?.[0];
     expect(database.bind).toHaveBeenCalledWith(
+      expect.any(String),
       "outbound_link_clicked",
-      "/about",
-      null,
-      null,
-      null,
-      "github.com",
-      "/ScientFactory/scient-desktop",
-      null,
-      null,
+      "product",
+      expect.any(String),
+      expect.stringMatching(/^web-event:/),
+      JSON.stringify({
+        page_path: "/about",
+        destination_host: "github.com",
+        destination_path: "/ScientFactory/scient-desktop",
+      }),
     );
   });
 

migrations/0002_analytics_gateway.sql

@@ -0,0 +1,25 @@
+CREATE TABLE analytics_events (
+  event_id TEXT PRIMARY KEY,
+  event_name TEXT NOT NULL,
+  source TEXT NOT NULL CHECK (source IN ('website', 'desktop')),
+  privacy_level TEXT NOT NULL CHECK (
+    privacy_level IN ('essential', 'product', 'diagnostic', 'contribution')
+  ),
+  occurred_at TEXT NOT NULL,
+  received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  distinct_id TEXT NOT NULL,
+  properties_json TEXT NOT NULL,
+  posthog_state TEXT NOT NULL DEFAULT 'pending' CHECK (posthog_state IN ('pending', 'sent')),
+  posthog_attempts INTEGER NOT NULL DEFAULT 0,
+  posthog_last_error TEXT,
+  posthog_sent_at TEXT
+);
+
+CREATE INDEX analytics_events_name_occurred_at
+  ON analytics_events (event_name, occurred_at);
+
+CREATE INDEX analytics_events_source_occurred_at
+  ON analytics_events (source, occurred_at);
+
+CREATE INDEX analytics_events_posthog_queue
+  ON analytics_events (posthog_state, received_at);

package.json

@@ -13,9 +13,13 @@
     "test": "vitest run",
     "db:migrate": "wrangler d1 migrations apply scientfactory-downloads --remote",
     "analytics:report": "node scripts/analytics-report.mjs",
+    "events:types": "wrangler types workers/events/worker-configuration.d.ts --config workers/events/wrangler.jsonc --env-interface AnalyticsWorkerBindings --include-runtime false",
+    "events:types:check": "wrangler types workers/events/worker-configuration.d.ts --config workers/events/wrangler.jsonc --env-interface AnalyticsWorkerBindings --include-runtime false --check",
+    "events:typecheck": "tsc --noEmit --project workers/events/tsconfig.json",
+    "events:deploy": "wrangler deploy --config workers/events/wrangler.jsonc",
     "format": "oxfmt",
     "format:check": "oxfmt --check",
-    "check": "bun run format:check && bun run typecheck && bun run test && bun run build && bun run build:edge"
+    "check": "bun run format:check && bun run typecheck && bun run events:types:check && bun run events:typecheck && bun run test && bun run build && bun run build:edge"
   },
   "dependencies": {
     "@fontsource-variable/dm-sans": "5.3.0",

scripts/analytics-report.mjs

@@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process";
 
 const query = `
   SELECT
-    'all_time_event' AS report_section,
+    'legacy_all_time_event' AS report_section,
     event_name AS item,
     COUNT(*) AS event_count,
     MAX(occurred_at) AS latest_event
@@ -11,8 +11,30 @@ const query = `
 
   UNION ALL
 
+  SELECT
+    'all_time_event' AS report_section,
+    source || ':' || event_name AS item,
+    COUNT(*) AS event_count,
+    MAX(occurred_at) AS latest_event
+  FROM analytics_events
+  GROUP BY source, event_name
+
+  UNION ALL
+
   SELECT
     '30_day_download' AS report_section,
+    COALESCE(json_extract(properties_json, '$.asset_key'), 'unknown') AS item,
+    COUNT(*) AS event_count,
+    MAX(occurred_at) AS latest_event
+  FROM analytics_events
+  WHERE event_name = 'download_clicked'
+    AND occurred_at >= datetime('now', '-30 days')
+  GROUP BY json_extract(properties_json, '$.asset_key')
+
+  UNION ALL
+
+  SELECT
+    'legacy_30_day_download' AS report_section,
     COALESCE(asset_key, 'unknown') AS item,
     COUNT(*) AS event_count,
     MAX(occurred_at) AS latest_event
@@ -25,6 +47,21 @@ const query = `
 
   SELECT
     '30_day_outbound' AS report_section,
+    COALESCE(json_extract(properties_json, '$.destination_host'), 'unknown') ||
+      COALESCE(json_extract(properties_json, '$.destination_path'), '/') AS item,
+    COUNT(*) AS event_count,
+    MAX(occurred_at) AS latest_event
+  FROM analytics_events
+  WHERE event_name = 'outbound_link_clicked'
+    AND occurred_at >= datetime('now', '-30 days')
+  GROUP BY
+    json_extract(properties_json, '$.destination_host'),
+    json_extract(properties_json, '$.destination_path')
+
+  UNION ALL
+
+  SELECT
+    'legacy_30_day_outbound' AS report_section,
     COALESCE(destination_host, 'unknown') || COALESCE(destination_path, '/') AS item,
     COUNT(*) AS event_count,
     MAX(occurred_at) AS latest_event
@@ -37,6 +74,21 @@ const query = `
 
   SELECT
     '30_day_download_failure' AS report_section,
+    COALESCE(json_extract(properties_json, '$.failure_stage'), 'unknown') || ':' ||
+      COALESCE(json_extract(properties_json, '$.failure_reason'), 'unknown') AS item,
+    COUNT(*) AS event_count,
+    MAX(occurred_at) AS latest_event
+  FROM analytics_events
+  WHERE event_name = 'download_failed'
+    AND occurred_at >= datetime('now', '-30 days')
+  GROUP BY
+    json_extract(properties_json, '$.failure_stage'),
+    json_extract(properties_json, '$.failure_reason')
+
+  UNION ALL
+
+  SELECT
+    'legacy_30_day_download_failure' AS report_section,
     COALESCE(failure_stage, 'unknown') || ':' || COALESCE(failure_reason, 'unknown') AS item,
     COUNT(*) AS event_count,
     MAX(occurred_at) AS latest_event
@@ -45,6 +97,16 @@ const query = `
     AND occurred_at >= datetime('now', '-30 days')
   GROUP BY failure_stage, failure_reason
 
+  UNION ALL
+
+  SELECT
+    'posthog_delivery' AS report_section,
+    posthog_state AS item,
+    COUNT(*) AS event_count,
+    MAX(received_at) AS latest_event
+  FROM analytics_events
+  GROUP BY posthog_state
+
   ORDER BY report_section, event_count DESC, item
 `;
 

src/pages/privacy.astro

@@ -24,6 +24,9 @@ import { REPO_URL } from "../lib/releases";
         <p>
           The event database stores the time, page path, and only the details needed for the event—for example, the installer type or the destination website and path. Query strings are discarded. We do not add an IP address, cookie, browser fingerprint, advertising identifier, referrer, or persistent visitor ID to these event records. This means the counts describe events, not unique people or a person's journey through the site.
         </p>
+        <p>
+          ScientFactory stores these events first in its Cloudflare database. Anonymous copies may also be processed in ScientFactory's EU-hosted PostHog project so we can analyze product usage. PostHog receives the same limited event details and a new event-specific identifier—not a persistent website visitor profile.
+        </p>
         <p>
           ScientFactory does not use advertising trackers, behavioral session recording, or cross-site tracking on this website.
         </p>

tsconfig.json

@@ -1,4 +1,4 @@
 {
   "extends": "astro/tsconfigs/strict",
-  "exclude": ["dist", "functions", "node_modules"]
+  "exclude": ["dist", "functions", "node_modules", "workers"]
 }