From 16c82da260e3656b4fe7cbcaa49bae8ef97bdf60 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 20 Jul 2026 23:34:14 +0300 Subject: [PATCH] Add consent-aware identity analytics --- README.md | 29 +- functions/_lib/events.ts | 70 +++-- functions/_lib/identity.ts | 124 +++++++++ functions/api/consent.test.ts | 95 +++++++ functions/api/consent.ts | 58 ++++ functions/api/download/[asset].test.ts | 14 +- functions/api/events.test.ts | 46 +++- functions/api/events.ts | 7 +- migrations/0003_identity_tracking.sql | 109 ++++++++ package.json | 1 + scripts/analytics-report.mjs | 34 +++ scripts/link-analytics-identity.mjs | 54 ++++ src/layouts/Layout.astro | 218 ++++++++++++++- src/pages/privacy.astro | 19 +- workers/events/src/index.test.ts | 191 +++++++++++++- workers/events/src/index.ts | 350 +++++++++++++++++++++++-- 16 files changed, 1361 insertions(+), 58 deletions(-) create mode 100644 functions/_lib/identity.ts create mode 100644 functions/api/consent.test.ts create mode 100644 functions/api/consent.ts create mode 100644 migrations/0003_identity_tracking.sql create mode 100644 scripts/link-analytics-identity.mjs diff --git a/README.md b/README.md index 7d8a224..484a497 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,14 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow. The cross- ## First-party event measurement -Cloudflare D1 stores four anonymous website event types: +Cloudflare D1 stores four website event types: - `page_viewed` - `download_clicked` - `download_failed` - `outbound_link_clicked` -The implementation intentionally stores no IP address, cookie, fingerprint, referrer, or persistent visitor identifier. Counts therefore represent events rather than unique visitors. `download_failed` is limited to a failure in ScientFactory's redirect service; the website cannot observe a transfer failure after GitHub begins serving an installer. +Before a visitor chooses analytics, and after choosing **Essential only**, every event receives a new event-specific identifier. Those counts represent events rather than unique people. After explicit **Allow analytics** consent, the site sets a random first-party visitor identifier and creates a session identifier so visits, downloads, sessions, and return behavior can be measured. The identity is not derived from an IP address, browser fingerprint, email address, advertising identifier, referrer, or third-party account. `download_failed` is limited to a failure in ScientFactory's redirect service; the website cannot observe a transfer failure after GitHub begins serving an installer. The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` D1 database. New events use the shared `analytics_events` table; the earlier `site_events` table remains as read-only historical data. Apply new migrations before deploying code that depends on them: @@ -49,7 +49,7 @@ The production binding is `DOWNLOAD_DB`, backed by the `scientfactory-downloads` bun run db:migrate ``` -To view the lifetime event summary plus 30-day download, outbound-link, and failure breakdowns: +To view the lifetime event summary plus 30-day identity, consent, session, download, outbound-link, and failure breakdowns: ```sh bun run analytics:report @@ -59,7 +59,9 @@ Local and Cloudflare preview hosts do not write events, which keeps production c ## Analytics gateway -The Worker under `workers/events` is ScientFactory's first-party telemetry gateway. Desktop clients submit bounded event batches to `https://events.scientfactory.com/v1/events`; the Worker stores them in D1 first and can then forward anonymous copies to the ScientFactory EU PostHog project. PostHog is an optional analysis layer rather than the primary event store. +The Worker under `workers/events` is ScientFactory's first-party telemetry and identity gateway. Desktop clients submit bounded event batches to `https://events.scientfactory.com/v1/events`; the Worker stores them in D1 first and can then forward pseudonymous copies to the ScientFactory EU PostHog project. PostHog is an optional analysis layer rather than the primary event store. + +Website visitors, desktop installations, sessions, and future Scient accounts use separate opaque identifiers. The service-authenticated `POST /v1/identity/link` endpoint can connect a visitor or installation to an account after Scient's account service has authenticated that user. Browser and desktop clients cannot call this endpoint directly or claim an account identifier. Linking updates first-party historical events without changing the user's analytics choice; the corresponding anonymous-to-account PostHog identity event is forwarded only for product-or-higher consent. Generate binding types and validate the Worker with: @@ -74,4 +76,21 @@ Deploy the Worker only from an approved production change: bun run events:deploy ``` -`POSTHOG_PROJECT_TOKEN` is a Cloudflare Worker secret and must never be committed. If it is absent, ingestion continues and events remain queued in D1 for later delivery. +`POSTHOG_PROJECT_TOKEN` and `IDENTITY_LINK_TOKEN` are Cloudflare Worker secrets and must never be committed. If the PostHog token is absent, ingestion continues and events remain queued in D1 for later delivery. If the identity-link token is absent, account linking returns `503` while ordinary ingestion continues. + +The identity-link token is service-to-service authority. Rotate it if it is exposed, and never embed it in website or desktop bundles: + +```sh +wrangler secret put IDENTITY_LINK_TOKEN --config workers/events/wrangler.jsonc +``` + +After an account service has authenticated a user and obtained their opaque account, installation, or visitor IDs, an authorized operator can exercise the same service endpoint with: + +```sh +SCIENT_IDENTITY_LINK_TOKEN=... bun run identity:link \ + --account account: \ + --identity installation: \ + --identity visitor: +``` + +This command is an operational bridge, not a substitute for account authentication. The eventual account service should call the endpoint server-to-server after sign-in; no link token belongs in a client bundle. diff --git a/functions/_lib/events.ts b/functions/_lib/events.ts index 745f4e4..ddfc1d8 100644 --- a/functions/_lib/events.ts +++ b/functions/_lib/events.ts @@ -1,7 +1,9 @@ // FILE: events.ts -// Purpose: Stores a deliberately small, anonymous set of first-party website events. +// Purpose: Stores a deliberately small set of consent-aware first-party website events. // Layer: Cloudflare Pages Function utility +import { cleanSessionId, websiteIdentity, websiteSessionId } from "./identity"; + export const SITE_EVENT_NAMES = [ "page_viewed", "download_clicked", @@ -21,6 +23,7 @@ export interface SiteEvent { readonly destinationPath?: string | null; readonly failureStage?: string | null; readonly failureReason?: string | null; + readonly sessionId?: string | null; } export interface SiteEventContext { @@ -37,8 +40,12 @@ const INSERT_EVENT = ` privacy_level, occurred_at, distinct_id, - properties_json - ) VALUES (?, ?, 'website', ?, ?, ?, ?) + properties_json, + identity_type, + canonical_id, + session_id, + consent_level + ) VALUES (?, ?, 'website', ?, ?, ?, ?, ?, ?, ?, ?) `; const PRODUCTION_HOSTS = new Set(["scientfactory.com", "www.scientfactory.com"]); @@ -79,8 +86,18 @@ export function shouldPersistSiteEvents(request: Request): boolean { return PRODUCTION_HOSTS.has(new URL(request.url).hostname); } -export async function insertSiteEvent(db: D1Database, event: SiteEvent): Promise { +export async function insertSiteEvent( + db: D1Database, + event: SiteEvent, + request?: Request, +): Promise { const eventId = crypto.randomUUID(); + const identity = request ? websiteIdentity(request) : null; + const distinctId = identity?.identityId ?? `web-event:${eventId}`; + const occurredAt = new Date().toISOString(); + const sessionId = identity + ? (websiteSessionId(request!) ?? cleanSessionId(event.sessionId)) + : null; const properties = Object.fromEntries( Object.entries({ page_path: cleanPath(event.pagePath), @@ -94,17 +111,38 @@ export async function insertSiteEvent(db: D1Database, event: SiteEvent): Promise }).filter((entry): entry is [string, string] => entry[1] !== null), ); - await db - .prepare(INSERT_EVENT) - .bind( - eventId, - event.eventName, - event.eventName === "download_failed" ? "diagnostic" : "product", - new Date().toISOString(), - `web-event:${eventId}`, - JSON.stringify(properties), - ) - .run(); + const statements: D1PreparedStatement[] = []; + if (identity) { + statements.push( + db + .prepare( + `INSERT INTO analytics_identities ( + identity_id, identity_type, canonical_id, consent_level, first_seen_at, last_seen_at + ) VALUES (?, 'web_visitor', ?, 'product', ?, ?) + ON CONFLICT(identity_id) DO UPDATE SET + last_seen_at = excluded.last_seen_at, + consent_level = 'product'`, + ) + .bind(identity.identityId, identity.canonicalId, occurredAt, occurredAt), + ); + } + statements.push( + db + .prepare(INSERT_EVENT) + .bind( + eventId, + event.eventName, + event.eventName === "download_failed" ? "diagnostic" : "product", + occurredAt, + distinctId, + JSON.stringify(properties), + identity?.identityType ?? "event", + identity?.canonicalId ?? distinctId, + sessionId, + identity?.consentLevel ?? "essential", + ), + ); + await db.batch(statements); } export function queueSiteEvent(context: SiteEventContext, event: SiteEvent): void { @@ -116,7 +154,7 @@ export function queueSiteEvent(context: SiteEventContext, event: SiteEvent): voi return; } - const write = insertSiteEvent(db, event).catch((error: unknown) => { + const write = insertSiteEvent(db, event, context.request).catch((error: unknown) => { console.error( JSON.stringify({ message: "Site event write failed", diff --git a/functions/_lib/identity.ts b/functions/_lib/identity.ts new file mode 100644 index 0000000..a2df4c7 --- /dev/null +++ b/functions/_lib/identity.ts @@ -0,0 +1,124 @@ +// FILE: identity.ts +// Purpose: Resolve consent-aware first-party website identity without browser fingerprinting. +// Layer: Cloudflare Pages Function utility + +export const ANALYTICS_NOTICE_VERSION = "2026-07-identity-v1"; +export const ANALYTICS_CONSENT_COOKIE = "sf_analytics"; +export const ANALYTICS_VISITOR_COOKIE = "sf_visitor"; +export const ANALYTICS_SESSION_COOKIE = "sf_session"; + +export type WebsiteConsentLevel = "essential" | "product"; + +export interface WebsiteIdentity { + readonly identityId: string; + readonly identityType: "web_visitor"; + readonly canonicalId: string; + readonly consentLevel: "product"; +} + +const VISITOR_ID_PATTERN = + /^visitor:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const SESSION_ID_PATTERN = + /^session:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +function cookieMap(request: Request): ReadonlyMap { + const entries = (request.headers.get("Cookie") ?? "") + .split(";") + .map((part) => part.trim()) + .filter(Boolean) + .flatMap((part): ReadonlyArray => { + const separator = part.indexOf("="); + if (separator < 1) return []; + const key = part.slice(0, separator); + const value = part.slice(separator + 1); + try { + return [[key, decodeURIComponent(value)]]; + } catch { + return []; + } + }); + return new Map(entries); +} + +export function websiteConsentLevel(request: Request): WebsiteConsentLevel | null { + const value = cookieMap(request).get(ANALYTICS_CONSENT_COOKIE); + return value === "essential" || value === "product" ? value : null; +} + +export function websiteIdentity(request: Request): WebsiteIdentity | null { + if (websiteConsentLevel(request) !== "product") return null; + const identityId = cookieMap(request).get(ANALYTICS_VISITOR_COOKIE); + if (!identityId || !VISITOR_ID_PATTERN.test(identityId)) return null; + return { + identityId, + identityType: "web_visitor", + canonicalId: identityId, + consentLevel: "product", + }; +} + +export function cleanSessionId(value: string | null | undefined): string | null { + return value && SESSION_ID_PATTERN.test(value) ? value : null; +} + +export function websiteSessionId(request: Request): string | null { + return cleanSessionId(cookieMap(request).get(ANALYTICS_SESSION_COOKIE)); +} + +export function newVisitorId(): string { + return `visitor:${crypto.randomUUID()}`; +} + +function secureCookie(request: Request): string { + return new URL(request.url).protocol === "https:" ? "; Secure" : ""; +} + +export function consentCookies( + request: Request, + level: WebsiteConsentLevel, + visitorId?: string, +): ReadonlyArray { + const common = `Path=/; Max-Age=${180 * 24 * 60 * 60}; SameSite=Lax${secureCookie(request)}`; + const cookies = [`${ANALYTICS_CONSENT_COOKIE}=${level}; ${common}`]; + if (level === "product" && visitorId) { + cookies.push(`${ANALYTICS_VISITOR_COOKIE}=${visitorId}; ${common}; HttpOnly`); + } else { + cookies.push( + `${ANALYTICS_VISITOR_COOKIE}=; Path=/; Max-Age=0; SameSite=Lax${secureCookie(request)}; HttpOnly`, + ); + } + return cookies; +} + +export async function persistWebsiteConsent( + database: D1Database, + level: WebsiteConsentLevel, + recordedAt: string, + identityId: string | null, +): Promise { + const statements: D1PreparedStatement[] = []; + if (identityId) { + statements.push( + database + .prepare( + `INSERT INTO analytics_identities ( + identity_id, identity_type, canonical_id, consent_level, first_seen_at, last_seen_at + ) VALUES (?, 'web_visitor', ?, ?, ?, ?) + ON CONFLICT(identity_id) DO UPDATE SET + consent_level = excluded.consent_level, + last_seen_at = excluded.last_seen_at`, + ) + .bind(identityId, identityId, level, recordedAt, recordedAt), + ); + } + statements.push( + database + .prepare( + `INSERT INTO analytics_consents ( + consent_id, identity_id, source, consent_level, notice_version, recorded_at + ) VALUES (?, ?, 'website', ?, ?, ?)`, + ) + .bind(crypto.randomUUID(), identityId, level, ANALYTICS_NOTICE_VERSION, recordedAt), + ); + await database.batch(statements); +} diff --git a/functions/api/consent.test.ts b/functions/api/consent.test.ts new file mode 100644 index 0000000..cc96c1d --- /dev/null +++ b/functions/api/consent.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { onRequestPost } from "./consent"; + +function createDatabase() { + const run = vi.fn().mockResolvedValue({ success: true }); + const bind = vi.fn(() => ({ run })); + const prepare = vi.fn(() => ({ bind })); + const batch = vi.fn().mockResolvedValue([]); + return { database: { prepare, batch } as unknown as D1Database, bind, prepare, batch }; +} + +function createContext(level: unknown, cookie?: string) { + const database = createDatabase(); + const request = new Request("https://scientfactory.com/api/consent", { + method: "POST", + headers: { + "Content-Type": "application/json", + Origin: "https://scientfactory.com", + ...(cookie ? { Cookie: cookie } : {}), + }, + body: JSON.stringify({ level }), + }); + return { + context: { request, env: { DOWNLOAD_DB: database.database } } as Parameters< + typeof onRequestPost + >[0], + database, + }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("website analytics consent", () => { + it("creates a random first-party visitor after product consent", async () => { + const { context, database } = createContext("product"); + const response = await onRequestPost(context); + + expect(response.status).toBe(200); + const cookies = response.headers.getSetCookie(); + expect(cookies).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^sf_analytics=product;/), + expect.stringMatching(/^sf_visitor=visitor:[0-9a-f-]+;/), + ]), + ); + expect(database.batch).toHaveBeenCalledOnce(); + expect(database.bind).toHaveBeenCalledWith( + expect.stringMatching(/^visitor:/), + expect.stringMatching(/^visitor:/), + "product", + expect.any(String), + expect.any(String), + ); + }); + + it("removes persistent identity when essential-only is selected", async () => { + const { context, database } = createContext( + "essential", + "sf_analytics=product; sf_visitor=visitor:16ace444-e7c3-4b26-893f-98713188ae52", + ); + const response = await onRequestPost(context); + + expect(response.status).toBe(200); + expect(response.headers.getSetCookie()).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^sf_analytics=essential;/), + expect.stringContaining("sf_visitor=; Path=/; Max-Age=0"), + ]), + ); + expect(database.batch).toHaveBeenCalledOnce(); + expect(database.bind).toHaveBeenCalledWith( + expect.any(String), + "visitor:16ace444-e7c3-4b26-893f-98713188ae52", + "essential", + "2026-07-identity-v1", + expect.any(String), + ); + }); + + it("rejects a cross-origin consent submission", async () => { + const { context, database } = createContext("product"); + Object.defineProperty(context, "request", { + value: new Request("https://scientfactory.com/api/consent", { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://attacker.example" }, + body: JSON.stringify({ level: "product" }), + }), + }); + + const response = await onRequestPost(context); + expect(response.status).toBe(403); + expect(database.prepare).not.toHaveBeenCalled(); + }); +}); diff --git a/functions/api/consent.ts b/functions/api/consent.ts new file mode 100644 index 0000000..9ee61e1 --- /dev/null +++ b/functions/api/consent.ts @@ -0,0 +1,58 @@ +// FILE: consent.ts +// Purpose: Persist the visitor's website analytics choice and issue first-party cookies. +// Layer: Cloudflare Pages Function + +import { + consentCookies, + newVisitorId, + persistWebsiteConsent, + websiteIdentity, + type WebsiteConsentLevel, +} from "../_lib/identity"; +import { shouldPersistSiteEvents } from "../_lib/events"; + +const RESPONSE_HEADERS = { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + "X-Content-Type-Options": "nosniff", +}; + +function response(body: unknown, status = 200, cookies: ReadonlyArray = []): Response { + const headers = new Headers(RESPONSE_HEADERS); + for (const cookie of cookies) headers.append("Set-Cookie", cookie); + return Response.json(body, { status, headers }); +} + +function isSameOrigin(request: Request): boolean { + const origin = request.headers.get("Origin"); + return origin !== null && origin === new URL(request.url).origin; +} + +export const onRequestPost: PagesFunction = async (context) => { + if (!isSameOrigin(context.request)) return response({ error: "Forbidden" }, 403); + if (!context.request.headers.get("Content-Type")?.startsWith("application/json")) { + return response({ error: "Content-Type must be application/json" }, 415); + } + + let level: WebsiteConsentLevel; + try { + const body = (await context.request.json()) as { level?: unknown }; + if (body.level !== "essential" && body.level !== "product") { + return response({ error: "Invalid consent level" }, 400); + } + level = body.level; + } catch { + return response({ error: "Invalid JSON" }, 400); + } + + const existing = websiteIdentity(context.request)?.identityId; + const visitorId = level === "product" ? (existing ?? newVisitorId()) : null; + const consentIdentityId = visitorId ?? existing ?? null; + const recordedAt = new Date().toISOString(); + if (shouldPersistSiteEvents(context.request)) { + if (!context.env.DOWNLOAD_DB) return response({ error: "Analytics storage unavailable" }, 503); + await persistWebsiteConsent(context.env.DOWNLOAD_DB, level, recordedAt, consentIdentityId); + } + + return response({ level }, 200, consentCookies(context.request, level, visitorId ?? undefined)); +}; diff --git a/functions/api/download/[asset].test.ts b/functions/api/download/[asset].test.ts index 8bb319b..4319b1c 100644 --- a/functions/api/download/[asset].test.ts +++ b/functions/api/download/[asset].test.ts @@ -22,7 +22,11 @@ const releaseFixture = { function createDatabase(run = vi.fn().mockResolvedValue({ success: true })) { const bind = vi.fn(() => ({ run })); const prepare = vi.fn(() => ({ bind })); - return { database: { prepare } as unknown as D1Database, prepare, bind, run }; + const batch = vi.fn(async (statements: ReadonlyArray<{ run?: () => Promise }>) => { + await Promise.all(statements.map((statement) => statement.run?.())); + return []; + }); + return { database: { prepare, batch } as unknown as D1Database, prepare, bind, run, batch }; } function createContext( @@ -73,6 +77,10 @@ describe("tracked download redirect", () => { destination_path: "/ScientFactory/scient-desktop/releases/download/v0.5.7/Scient-0.5.7-arm64.dmg", }), + "event", + expect.stringMatching(/^web-event:/), + null, + "essential", ); }); @@ -113,6 +121,10 @@ describe("tracked download redirect", () => { failure_stage: "release_fetch", failure_reason: "upstream_unavailable", }), + "event", + expect.stringMatching(/^web-event:/), + null, + "essential", ); }); diff --git a/functions/api/events.test.ts b/functions/api/events.test.ts index 525af15..f66cf32 100644 --- a/functions/api/events.test.ts +++ b/functions/api/events.test.ts @@ -6,12 +6,13 @@ function createDatabase() { const run = vi.fn().mockResolvedValue({ success: true }); const bind = vi.fn(() => ({ run })); const prepare = vi.fn(() => ({ bind })); - return { database: { prepare } as unknown as D1Database, bind, prepare, run }; + const batch = vi.fn().mockResolvedValue([]); + return { database: { prepare, batch } as unknown as D1Database, bind, prepare, run, batch }; } function createContext( body: unknown, - options?: { readonly origin?: string; readonly url?: string }, + options?: { readonly origin?: string; readonly url?: string; readonly cookie?: string }, ) { const database = createDatabase(); const url = options?.url ?? "https://scientfactory.com/api/events"; @@ -20,6 +21,7 @@ function createContext( headers: { "Content-Type": "application/json", Origin: options?.origin ?? new URL(url).origin, + ...(options?.cookie ? { Cookie: options.cookie } : {}), }, body: JSON.stringify(body), }); @@ -55,6 +57,10 @@ describe("browser event endpoint", () => { expect.any(String), expect.stringMatching(/^web-event:/), JSON.stringify({ page_path: "/docs" }), + "event", + expect.stringMatching(/^web-event:/), + null, + "essential", ); }); @@ -80,6 +86,42 @@ describe("browser event endpoint", () => { destination_host: "github.com", destination_path: "/ScientFactory/scient-desktop", }), + "event", + expect.stringMatching(/^web-event:/), + null, + "essential", + ); + }); + + it("connects consented events to a stable visitor and session", async () => { + const visitorId = "visitor:16ace444-e7c3-4b26-893f-98713188ae52"; + const sessionId = "session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665"; + const { context, database } = createContext( + { event_name: "page_viewed", page_path: "/", session_id: sessionId }, + { cookie: `sf_analytics=product; sf_visitor=${visitorId}` }, + ); + + const response = await onRequestPost(context); + + expect(response.status).toBe(204); + await context.waitUntil.mock.calls[0]?.[0]; + expect(database.bind).toHaveBeenCalledWith( + visitorId, + visitorId, + expect.any(String), + expect.any(String), + ); + expect(database.bind).toHaveBeenCalledWith( + expect.any(String), + "page_viewed", + "product", + expect.any(String), + visitorId, + JSON.stringify({ page_path: "/" }), + "web_visitor", + visitorId, + sessionId, + "product", ); }); diff --git a/functions/api/events.ts b/functions/api/events.ts index 0bafca5..159f083 100644 --- a/functions/api/events.ts +++ b/functions/api/events.ts @@ -30,7 +30,11 @@ function parseClientEvent(body: unknown, request: Request): SiteEvent | null { if (!pagePath) return null; if (body.event_name === "page_viewed") { - return { eventName: "page_viewed", pagePath }; + return { + eventName: "page_viewed", + pagePath, + sessionId: typeof body.session_id === "string" ? body.session_id : null, + }; } if (body.event_name === "outbound_link_clicked") { @@ -40,6 +44,7 @@ function parseClientEvent(body: unknown, request: Request): SiteEvent | null { return { eventName: "outbound_link_clicked", pagePath, + sessionId: typeof body.session_id === "string" ? body.session_id : null, ...destination, }; } diff --git a/migrations/0003_identity_tracking.sql b/migrations/0003_identity_tracking.sql new file mode 100644 index 0000000..a6014ff --- /dev/null +++ b/migrations/0003_identity_tracking.sql @@ -0,0 +1,109 @@ +CREATE TABLE analytics_identities ( + identity_id TEXT PRIMARY KEY, + identity_type TEXT NOT NULL CHECK ( + identity_type IN ('web_visitor', 'desktop_installation', 'account') + ), + canonical_id TEXT NOT NULL, + consent_level TEXT NOT NULL CHECK ( + consent_level IN ('essential', 'product', 'diagnostic', 'contribution') + ), + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + linked_at TEXT +); + +CREATE INDEX analytics_identities_canonical_id + ON analytics_identities (canonical_id); + +CREATE TABLE analytics_identity_links ( + link_id TEXT PRIMARY KEY, + source_identity_id TEXT NOT NULL UNIQUE, + canonical_id TEXT NOT NULL, + linked_at TEXT NOT NULL, + posthog_state TEXT NOT NULL DEFAULT 'pending' CHECK (posthog_state IN ('pending', 'sent')), + posthog_attempts INTEGER NOT NULL DEFAULT 0, + posthog_last_error TEXT, + posthog_sent_at TEXT, + FOREIGN KEY (source_identity_id) REFERENCES analytics_identities(identity_id), + FOREIGN KEY (canonical_id) REFERENCES analytics_identities(identity_id) +); + +CREATE INDEX analytics_identity_links_posthog_queue + ON analytics_identity_links (posthog_state, linked_at); + +CREATE TABLE analytics_consents ( + consent_id TEXT PRIMARY KEY, + identity_id TEXT, + source TEXT NOT NULL CHECK (source IN ('website', 'desktop')), + consent_level TEXT NOT NULL CHECK ( + consent_level IN ('essential', 'product', 'diagnostic', 'contribution') + ), + notice_version TEXT NOT NULL, + recorded_at TEXT NOT NULL, + FOREIGN KEY (identity_id) REFERENCES analytics_identities(identity_id) +); + +CREATE INDEX analytics_consents_identity_recorded_at + ON analytics_consents (identity_id, recorded_at); + +CREATE TRIGGER analytics_desktop_identity_consent_created +AFTER INSERT ON analytics_identities +WHEN NEW.identity_type = 'desktop_installation' +BEGIN + INSERT INTO analytics_consents ( + consent_id, identity_id, source, consent_level, notice_version, recorded_at + ) VALUES ( + lower(hex(randomblob(16))), + NEW.identity_id, + 'desktop', + NEW.consent_level, + '2026-07-identity-v1', + NEW.first_seen_at + ); +END; + +CREATE TRIGGER analytics_desktop_identity_consent_changed +AFTER UPDATE OF consent_level ON analytics_identities +WHEN NEW.identity_type = 'desktop_installation' AND OLD.consent_level <> NEW.consent_level +BEGIN + INSERT INTO analytics_consents ( + consent_id, identity_id, source, consent_level, notice_version, recorded_at + ) VALUES ( + lower(hex(randomblob(16))), + NEW.identity_id, + 'desktop', + NEW.consent_level, + '2026-07-identity-v1', + NEW.last_seen_at + ); +END; + +ALTER TABLE analytics_events + ADD COLUMN identity_type TEXT NOT NULL DEFAULT 'event'; + +ALTER TABLE analytics_events + ADD COLUMN canonical_id TEXT NOT NULL DEFAULT ''; + +ALTER TABLE analytics_events + ADD COLUMN session_id TEXT; + +ALTER TABLE analytics_events + ADD COLUMN consent_level TEXT NOT NULL DEFAULT 'essential'; + +UPDATE analytics_events +SET + canonical_id = distinct_id, + identity_type = CASE + WHEN distinct_id LIKE 'installation:%' THEN 'desktop_installation' + ELSE 'event' + END, + consent_level = CASE + WHEN source = 'desktop' THEN privacy_level + ELSE 'essential' + END; + +CREATE INDEX analytics_events_canonical_occurred_at + ON analytics_events (canonical_id, occurred_at); + +CREATE INDEX analytics_events_session_occurred_at + ON analytics_events (session_id, occurred_at); diff --git a/package.json b/package.json index 9a6be55..4c89e8b 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "test": "vitest run", "db:migrate": "wrangler d1 migrations apply scientfactory-downloads --remote", "analytics:report": "node scripts/analytics-report.mjs", + "identity:link": "node scripts/link-analytics-identity.mjs", "events:types": "wrangler types workers/events/worker-configuration.d.ts --config workers/events/wrangler.jsonc --env-interface AnalyticsWorkerBindings --include-runtime false", "events:types:check": "wrangler types workers/events/worker-configuration.d.ts --config workers/events/wrangler.jsonc --env-interface AnalyticsWorkerBindings --include-runtime false --check", "events:typecheck": "tsc --noEmit --project workers/events/tsconfig.json", diff --git a/scripts/analytics-report.mjs b/scripts/analytics-report.mjs index 918098d..86932b1 100644 --- a/scripts/analytics-report.mjs +++ b/scripts/analytics-report.mjs @@ -1,6 +1,40 @@ import { spawnSync } from "node:child_process"; const query = ` + SELECT + '30_day_identity' AS report_section, + identity_type AS item, + COUNT(*) AS event_count, + MAX(last_seen_at) AS latest_event + FROM analytics_identities + WHERE last_seen_at >= datetime('now', '-30 days') + GROUP BY identity_type + + UNION ALL + + SELECT + '30_day_session' AS report_section, + source AS item, + COUNT(DISTINCT session_id) AS event_count, + MAX(occurred_at) AS latest_event + FROM analytics_events + WHERE session_id IS NOT NULL + AND occurred_at >= datetime('now', '-30 days') + GROUP BY source + + UNION ALL + + SELECT + '30_day_consent' AS report_section, + source || ':' || consent_level AS item, + COUNT(*) AS event_count, + MAX(recorded_at) AS latest_event + FROM analytics_consents + WHERE recorded_at >= datetime('now', '-30 days') + GROUP BY source, consent_level + + UNION ALL + SELECT 'legacy_all_time_event' AS report_section, event_name AS item, diff --git a/scripts/link-analytics-identity.mjs b/scripts/link-analytics-identity.mjs new file mode 100644 index 0000000..409e465 --- /dev/null +++ b/scripts/link-analytics-identity.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node + +const args = process.argv.slice(2); + +function argument(name) { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; +} + +function argumentsFor(name) { + return args.flatMap((value, index) => + value === name && args[index + 1] ? [args[index + 1]] : [], + ); +} + +function fail(message) { + console.error(message); + process.exitCode = 1; +} + +const accountId = argument("--account"); +const identityIds = argumentsFor("--identity"); +const endpoint = + process.env.SCIENT_IDENTITY_LINK_ENDPOINT ?? "https://events.scientfactory.com/v1/identity/link"; +const token = process.env.SCIENT_IDENTITY_LINK_TOKEN; + +if (!accountId || identityIds.length === 0) { + fail( + "Usage: bun run identity:link --account account: --identity installation: [--identity visitor:]", + ); +} else if (!token) { + fail( + "Set SCIENT_IDENTITY_LINK_TOKEN to the Worker identity-link secret before running this command.", + ); +} else { + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + schema_version: 1, + account_id: accountId, + identity_ids: identityIds, + }), + }); + const body = await response.text(); + if (!response.ok) { + fail(`Identity link failed (${response.status}): ${body}`); + } else { + console.log(body); + } +} diff --git a/src/layouts/Layout.astro b/src/layouts/Layout.astro index d955cc8..15f0208 100644 --- a/src/layouts/Layout.astro +++ b/src/layouts/Layout.astro @@ -87,6 +87,9 @@ const primaryLinks = [
© {new Date().getFullYear()} ScientFactory
+ Privacy GitHub Issues @@ -94,16 +97,65 @@ const primaryLinks = [
+ + @@ -399,14 +505,107 @@ const primaryLinks = [ gap: 1.25rem; } - .footer-links a { + .footer-link-button { + appearance: none; + padding: 0; + border: 0; + background: transparent; + color: inherit; + cursor: pointer; + } + + .footer-links a, + .footer-link-button { transition: color 220ms var(--ease-out); } - .footer-links a:hover { + .footer-links a:hover, + .footer-link-button:hover { + color: var(--ink); + } + + .footer-link-button:focus-visible { + outline: 2px solid var(--focus-ring); + outline-offset: 0.25rem; + } + + .analytics-consent { + position: fixed; + z-index: 50; + right: clamp(1rem, 3vw, 2rem); + bottom: clamp(1rem, 3vw, 2rem); + width: min(35rem, calc(100vw - 2rem)); + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + gap: 1.25rem; + padding: 1rem 1.1rem; + border: 1px solid var(--line); + border-radius: 0.75rem; + background: color-mix(in srgb, var(--surface) 96%, transparent); + box-shadow: 0 1rem 3rem rgba(0, 0, 0, 0.12); + backdrop-filter: blur(18px); + } + + .analytics-consent[hidden] { + display: none; + } + + .analytics-consent-copy strong { + display: block; + margin-bottom: 0.2rem; + font-size: 0.92rem; + font-weight: 600; + } + + .analytics-consent-copy p { + margin: 0; + color: var(--ink-muted); + font-size: 0.78rem; + line-height: 1.45; + } + + .analytics-consent-copy a { + text-decoration: underline; + text-underline-offset: 0.16em; + } + + .analytics-consent-actions { + display: flex; + gap: 0.5rem; + } + + .analytics-consent-actions button { + min-height: 2.25rem; + padding: 0.5rem 0.75rem; + border: 1px solid var(--line); + border-radius: 0.45rem; + cursor: pointer; + font-size: 0.78rem; + white-space: nowrap; + transition: transform 180ms var(--ease-out), background-color 180ms var(--ease-out); + } + + .analytics-consent-actions button:hover:not(:disabled) { + transform: translateY(-1px); + } + + .analytics-consent-secondary { + background: var(--surface); color: var(--ink); } + .analytics-consent-primary { + border-color: var(--burgundy) !important; + background: var(--burgundy); + color: #fff; + } + + .analytics-consent-actions button:disabled { + cursor: wait; + opacity: 0.6; + } + @media (max-width: 640px) { .nav { min-height: 5.5rem; @@ -449,5 +648,18 @@ const primaryLinks = [ flex-wrap: wrap; gap: 0.75rem 1.1rem; } + + .analytics-consent { + grid-template-columns: 1fr; + gap: 0.85rem; + } + + .analytics-consent-actions { + width: 100%; + } + + .analytics-consent-actions button { + flex: 1; + } } diff --git a/src/pages/privacy.astro b/src/pages/privacy.astro index 09b5c3f..1e3c403 100644 --- a/src/pages/privacy.astro +++ b/src/pages/privacy.astro @@ -17,18 +17,21 @@ import { REPO_URL } from "../lib/releases";

Analytics

-

Limited, first-party measurement without visitor profiles.

+

First-party measurement with a clear choice.

The website records four anonymous events: a page was viewed, a download was clicked, a ScientFactory download redirect failed, or an external link was clicked. These events help us understand which pages and installers are useful and whether the download service is working.

- The event database stores the time, page path, and only the details needed for the event—for example, the installer type or the destination website and path. Query strings are discarded. We do not add an IP address, cookie, browser fingerprint, advertising identifier, referrer, or persistent visitor ID to these event records. This means the counts describe events, not unique people or a person's journey through the site. + With “Essential only,” each event receives a new event-specific identifier. The database stores the time, page path, and only the details needed for that event—for example, the installer type or destination website and path. Query strings are discarded. These counts describe events, not unique people or a person's journey through the site.

- ScientFactory stores these events first in its Cloudflare database. Anonymous copies may also be processed in ScientFactory's EU-hosted PostHog project so we can analyze product usage. PostHog receives the same limited event details and a new event-specific identifier—not a persistent website visitor profile. + If you choose “Allow analytics,” we set a random first-party visitor identifier for up to 180 days and create a temporary identifier for the current browser session. This lets us measure returning visits, sessions, and the path from page view to installer download. The identifiers are random: they are not derived from your name, email address, device fingerprint, IP address, or account with another service.

- ScientFactory does not use advertising trackers, behavioral session recording, or cross-site tracking on this website. + ScientFactory stores events and consent records first in its Cloudflare database. Pseudonymous copies may also be processed in ScientFactory's EU-hosted PostHog project so we can analyze usage and retention. You can change your choice at any time through “Analytics settings” in the footer; choosing “Essential only” removes the persistent visitor cookie. +

+

+ ScientFactory does not use advertising trackers, browser fingerprinting, behavioral session recording, or cross-site tracking on this website. Research content is never included in website analytics events.

@@ -62,12 +65,12 @@ import { REPO_URL } from "../lib/releases";

Scient desktop

-

The desktop application and connected AI providers are separate from this website.

+

Desktop measurement is controlled separately inside Scient.

- Scient runs as a workspace layer on your computer. When you use a connected AI provider, that provider receives the prompts and supporting context needed for the session under its own terms and privacy policy. + Scient runs as a workspace layer on your computer. Its Data & Privacy setting controls whether the app sends no telemetry, essential reliability events, product-usage events, diagnostic events, or voluntary contribution data. Desktop events use a random installation identifier and never derive identity from a connected AI-provider account.

- This website notice does not replace the privacy terms of any AI provider you choose to use. + Normal product analytics exclude prompts, research documents, filenames, file paths, source text, and generated scientific content. When you use a connected AI provider, that provider receives the prompts and supporting context needed for the session under its own terms and privacy policy. This website notice does not replace those terms.

@@ -77,7 +80,7 @@ import { REPO_URL } from "../lib/releases";

We will update this page when the website changes.

- If we add accounts, forms, more detailed measurement, or other ways of collecting visitor information, this notice will be updated to describe them. + If we add Scient accounts, forms, or additional identity links, this notice will be updated before those features are enabled for visitors.

For non-sensitive questions, use GitHub Issues. Security concerns should be reported privately through GitHub's security reporting flow rather than in a public issue. diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index da66734..0da87ac 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -1,31 +1,41 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import worker, { flushPendingEvents, validateIngestionPayload } from "./index"; +import worker, { + flushPendingEvents, + flushPendingIdentityLinks, + validateIdentityLinkPayload, + validateIngestionPayload, +} from "./index"; function validPayload() { return { - schema_version: 1, + schema_version: 2, source: "desktop", events: [ { id: "8e0ee7d5-2c4b-48b6-8209-08f1e536f665", name: "provider.turn.sent", distinct_id: "installation:16ace444-e7c3-4b26-893f-98713188ae52", + session_id: "session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", occurred_at: new Date().toISOString(), privacy_level: "product", + consent_level: "product", properties: { provider: "codex", clientType: "desktop-app" }, }, ], }; } -function createDatabase() { +function createDatabase(existingCanonicalId?: string) { const run = vi.fn().mockResolvedValue({ success: true }); const all = vi.fn().mockResolvedValue({ results: [] }); - const bind = vi.fn(() => ({ run, all })); + const first = vi + .fn() + .mockResolvedValue(existingCanonicalId ? { canonical_id: existingCanonicalId } : null); + const bind = vi.fn(() => ({ run, all, first })); const prepare = vi.fn(() => ({ bind })); const batch = vi.fn().mockResolvedValue([]); - return { database: { prepare, batch } as unknown as D1Database, prepare, bind, batch }; + return { database: { prepare, batch } as unknown as D1Database, prepare, bind, batch, first }; } function incoming(request: Request): Request { @@ -57,6 +67,18 @@ describe("event gateway validation", () => { }), ).toThrow("At most 50 events are accepted"); }); + + it("rejects a desktop event that impersonates an account identity", () => { + const payload = validPayload(); + expect(() => + validateIngestionPayload({ + ...payload, + events: [ + { ...payload.events[0], distinct_id: "account:16ace444-e7c3-4b26-893f-98713188ae52" }, + ], + }), + ).toThrow("Desktop events require an installation identity"); + }); }); describe("event gateway routes", () => { @@ -78,6 +100,13 @@ describe("event gateway routes", () => { expect(response.status).toBe(202); expect(await response.json()).toEqual({ accepted: 1 }); expect(database.batch).toHaveBeenCalledOnce(); + expect(database.bind).toHaveBeenCalledWith( + "installation:16ace444-e7c3-4b26-893f-98713188ae52", + "installation:16ace444-e7c3-4b26-893f-98713188ae52", + "product", + expect.any(String), + expect.any(String), + ); expect(database.bind).toHaveBeenCalledWith( "8e0ee7d5-2c4b-48b6-8209-08f1e536f665", "provider.turn.sent", @@ -85,6 +114,10 @@ describe("event gateway routes", () => { expect.any(String), "installation:16ace444-e7c3-4b26-893f-98713188ae52", JSON.stringify({ provider: "codex", clientType: "desktop-app" }), + "installation:16ace444-e7c3-4b26-893f-98713188ae52", + "installation:16ace444-e7c3-4b26-893f-98713188ae52", + "session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", + "product", ); expect(waitUntil).toHaveBeenCalledOnce(); }); @@ -119,7 +152,104 @@ describe("event gateway routes", () => { expect(await response.json()).toMatchObject({ status: "ready", posthog_forwarding: "pending_configuration", + identity_linking: "pending_configuration", + }); + }); + + it("requires service authentication before linking account identity", async () => { + const database = createDatabase(); + const response = await worker.fetch!( + incoming( + new Request("https://events.scientfactory.com/v1/identity/link", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + schema_version: 1, + account_id: "account:16ace444-e7c3-4b26-893f-98713188ae52", + identity_ids: ["installation:8e0ee7d5-2c4b-48b6-8209-08f1e536f665"], + }), + }), + ), + { ANALYTICS_DB: database.database, IDENTITY_LINK_TOKEN: "secret" } as Cloudflare.Env, + { waitUntil: vi.fn() } as unknown as ExecutionContext, + ); + + expect(response.status).toBe(401); + expect(database.batch).not.toHaveBeenCalled(); + }); + + it("links installation history to an authenticated account id", async () => { + const database = createDatabase(); + const waitUntil = vi.fn(); + const accountId = "account:16ace444-e7c3-4b26-893f-98713188ae52"; + const installationId = "installation:8e0ee7d5-2c4b-48b6-8209-08f1e536f665"; + const response = await worker.fetch!( + incoming( + new Request("https://events.scientfactory.com/v1/identity/link", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer secret", + }, + body: JSON.stringify({ + schema_version: 1, + account_id: accountId, + identity_ids: [installationId], + }), + }), + ), + { ANALYTICS_DB: database.database, IDENTITY_LINK_TOKEN: "secret" } as Cloudflare.Env, + { waitUntil } as unknown as ExecutionContext, + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ linked: 1, canonical_id: accountId }); + expect(database.batch).toHaveBeenCalledOnce(); + expect(database.bind).toHaveBeenCalledWith(accountId, installationId); + expect(waitUntil).toHaveBeenCalledOnce(); + }); + + it("refuses to reassign an identity to a different account", async () => { + const database = createDatabase("account:11111111-1111-4111-8111-111111111111"); + const response = await worker.fetch!( + incoming( + new Request("https://events.scientfactory.com/v1/identity/link", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer secret", + }, + body: JSON.stringify({ + schema_version: 1, + account_id: "account:22222222-2222-4222-8222-222222222222", + identity_ids: ["installation:8e0ee7d5-2c4b-48b6-8209-08f1e536f665"], + }), + }), + ), + { ANALYTICS_DB: database.database, IDENTITY_LINK_TOKEN: "secret" } as Cloudflare.Env, + { waitUntil: vi.fn() } as unknown as ExecutionContext, + ); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "Source identity is already linked to another account", }); + expect(database.batch).not.toHaveBeenCalled(); + }); +}); + +describe("identity link validation", () => { + it("accepts visitor and installation ids only", () => { + expect( + validateIdentityLinkPayload({ + schema_version: 1, + account_id: "account:16ace444-e7c3-4b26-893f-98713188ae52", + identity_ids: [ + "visitor:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", + "installation:f48176b0-03e0-4f2b-8f4b-e1c9ebf4fb7e", + ], + }).identityIds, + ).toHaveLength(2); }); }); @@ -132,12 +262,16 @@ describe("PostHog forwarding", () => { privacy_level: "product", occurred_at: new Date().toISOString(), distinct_id: "installation:16ace444-e7c3-4b26-893f-98713188ae52", + canonical_id: "installation:16ace444-e7c3-4b26-893f-98713188ae52", + identity_type: "desktop_installation", + session_id: "session:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", + consent_level: "product", properties_json: JSON.stringify({ provider: "codex" }), }; const run = vi.fn().mockResolvedValue({ success: true }); const all = vi.fn().mockResolvedValue({ results: [row] }); const bind = vi.fn(() => ({ run, all })); - const prepare = vi.fn(() => ({ bind })); + const prepare = vi.fn((_query: string) => ({ bind })); const batch = vi.fn().mockResolvedValue([]); const database = { prepare, batch } as unknown as D1Database; const fetchMock = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); @@ -162,8 +296,53 @@ describe("PostHog forwarding", () => { event_id: row.event_id, source: "desktop", privacy_level: "product", + consent_level: "product", + identity_type: "desktop_installation", + $session_id: row.session_id, $process_person_profile: false, }); expect(batch).toHaveBeenCalledOnce(); }); + + it("forwards anonymous-to-account identity events from the first-party link queue", async () => { + const row = { + link_id: "link-1", + source_identity_id: "installation:16ace444-e7c3-4b26-893f-98713188ae52", + canonical_id: "account:8e0ee7d5-2c4b-48b6-8209-08f1e536f665", + linked_at: new Date().toISOString(), + }; + const run = vi.fn().mockResolvedValue({ success: true }); + const all = vi.fn().mockResolvedValue({ results: [row] }); + const bind = vi.fn(() => ({ run, all })); + const prepare = vi.fn((_query: string) => ({ bind })); + const batch = vi.fn().mockResolvedValue([]); + const fetchMock = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + + const forwarded = await flushPendingIdentityLinks({ + ANALYTICS_DB: { prepare, batch } as unknown as D1Database, + POSTHOG_PROJECT_TOKEN: "phc_scientfactory_test", + }); + + expect(forwarded).toBe(1); + expect(prepare.mock.calls[0]?.[0]).toContain( + "identities.consent_level IN ('product', 'diagnostic', 'contribution')", + ); + const [, request] = fetchMock.mock.calls[0] ?? []; + const payload = JSON.parse(String(request?.body)) as { + batch: ReadonlyArray<{ + event: string; + distinct_id: string; + properties: Record; + }>; + }; + expect(payload.batch[0]).toMatchObject({ + event: "$identify", + distinct_id: row.canonical_id, + properties: { + $anon_distinct_id: row.source_identity_id, + $process_person_profile: true, + }, + }); + }); }); diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 5d56681..0ca8b46 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -1,6 +1,10 @@ const ALLOWED_WEB_ORIGINS = new Set(["https://scientfactory.com", "https://www.scientfactory.com"]); const EVENT_NAME_PATTERN = /^[a-z][a-z0-9_.-]{0,79}$/; const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const INSTALLATION_ID_PATTERN = /^installation:[0-9a-f-]{36}$/i; +const VISITOR_ID_PATTERN = /^visitor:[0-9a-f-]{36}$/i; +const ACCOUNT_ID_PATTERN = /^account:[0-9a-f-]{36}$/i; +const SESSION_ID_PATTERN = /^session:[0-9a-f-]{36}$/i; const PRIVACY_LEVELS = new Set(["essential", "product", "diagnostic", "contribution"]); const MAX_REQUEST_BYTES = 128 * 1024; const MAX_BATCH_SIZE = 50; @@ -10,14 +14,18 @@ const POSTHOG_HOST = "https://eu.i.posthog.com"; type AnalyticsEnv = AnalyticsWorkerBindings & { readonly POSTHOG_PROJECT_TOKEN?: string; + readonly IDENTITY_LINK_TOKEN?: string; }; interface AcceptedEvent { readonly id: string; readonly name: string; readonly distinctId: string; + readonly identityType: "desktop_installation"; + readonly sessionId: string | null; readonly occurredAt: string; readonly privacyLevel: string; + readonly consentLevel: string; readonly properties: Record; } @@ -28,9 +36,20 @@ interface PendingEventRow { readonly privacy_level: string; readonly occurred_at: string; readonly distinct_id: string; + readonly canonical_id: string; + readonly identity_type: string; + readonly session_id: string | null; + readonly consent_level: string; readonly properties_json: string; } +interface PendingIdentityLinkRow { + readonly link_id: string; + readonly source_identity_id: string; + readonly canonical_id: string; + readonly linked_at: string; +} + class RequestValidationError extends Error {} function jsonResponse(body: unknown, status = 200, origin?: string | null): Response { @@ -86,18 +105,41 @@ function parseEvent(value: unknown): AcceptedEvent { throw new RequestValidationError("occurred_at is too old"); } + const distinctId = requireString(value.distinct_id, "distinct_id", IDENTIFIER_PATTERN); + if (!INSTALLATION_ID_PATTERN.test(distinctId)) { + throw new RequestValidationError("Desktop events require an installation identity"); + } + const sessionId = + value.session_id === undefined || value.session_id === null + ? null + : requireString(value.session_id, "session_id", SESSION_ID_PATTERN); + const consentLevel = + value.consent_level === undefined + ? privacyLevel + : requireString(value.consent_level, "consent_level", /^[a-z]+$/); + if (!PRIVACY_LEVELS.has(consentLevel)) { + throw new RequestValidationError("Invalid consent_level"); + } + return { id: requireString(value.id, "event id", IDENTIFIER_PATTERN), name: requireString(value.name, "event name", EVENT_NAME_PATTERN), - distinctId: requireString(value.distinct_id, "distinct_id", IDENTIFIER_PATTERN), + distinctId, + identityType: "desktop_installation", + sessionId, occurredAt: occurredAtDate.toISOString(), privacyLevel, + consentLevel, properties, }; } export function validateIngestionPayload(value: unknown): ReadonlyArray { - if (!isRecord(value) || value.schema_version !== 1 || value.source !== "desktop") { + if ( + !isRecord(value) || + (value.schema_version !== 1 && value.schema_version !== 2) || + value.source !== "desktop" + ) { throw new RequestValidationError("Unsupported event payload"); } if (!Array.isArray(value.events) || value.events.length < 1) { @@ -129,6 +171,19 @@ async function persistEvents( database: D1Database, events: ReadonlyArray, ): Promise { + const upsertIdentity = ` + INSERT INTO analytics_identities ( + identity_id, + identity_type, + canonical_id, + consent_level, + first_seen_at, + last_seen_at + ) VALUES (?, 'desktop_installation', ?, ?, ?, ?) + ON CONFLICT(identity_id) DO UPDATE SET + consent_level = excluded.consent_level, + last_seen_at = excluded.last_seen_at + `; const insert = ` INSERT OR IGNORE INTO analytics_events ( event_id, @@ -137,11 +192,28 @@ async function persistEvents( privacy_level, occurred_at, distinct_id, - properties_json - ) VALUES (?, ?, 'desktop', ?, ?, ?, ?) + properties_json, + identity_type, + canonical_id, + session_id, + consent_level + ) VALUES ( + ?, ?, 'desktop', ?, ?, ?, ?, 'desktop_installation', + COALESCE((SELECT canonical_id FROM analytics_identities WHERE identity_id = ?), ?), + ?, ? + ) `; await database.batch( - events.map((event) => + events.flatMap((event) => [ + database + .prepare(upsertIdentity) + .bind( + event.distinctId, + event.distinctId, + event.consentLevel, + event.occurredAt, + event.occurredAt, + ), database .prepare(insert) .bind( @@ -151,8 +223,12 @@ async function persistEvents( event.occurredAt, event.distinctId, JSON.stringify(event.properties), + event.distinctId, + event.distinctId, + event.sessionId, + event.consentLevel, ), - ), + ]), ); } @@ -166,14 +242,17 @@ function posthogEvent(row: PendingEventRow): Record { } return { event: row.event_name, - distinct_id: row.distinct_id, + distinct_id: row.canonical_id, timestamp: row.occurred_at, properties: { ...properties, event_id: row.event_id, source: row.source, privacy_level: row.privacy_level, - $process_person_profile: false, + consent_level: row.consent_level, + identity_type: row.identity_type, + ...(row.session_id ? { $session_id: row.session_id } : {}), + $process_person_profile: row.canonical_id.startsWith("account:"), }, }; } @@ -209,6 +288,10 @@ export async function flushPendingEvents(env: AnalyticsEnv): Promise { privacy_level, occurred_at, distinct_id, + canonical_id, + identity_type, + session_id, + consent_level, properties_json FROM analytics_events WHERE posthog_state = 'pending' @@ -257,6 +340,235 @@ export async function flushPendingEvents(env: AnalyticsEnv): Promise { return rows.length; } +function identityIdentifyEvent(row: PendingIdentityLinkRow): Record { + return { + event: "$identify", + distinct_id: row.canonical_id, + timestamp: row.linked_at, + properties: { + $anon_distinct_id: row.source_identity_id, + link_id: row.link_id, + source: "identity_gateway", + $process_person_profile: true, + }, + }; +} + +export async function flushPendingIdentityLinks(env: AnalyticsEnv): Promise { + if (!env.POSTHOG_PROJECT_TOKEN) return 0; + const result = await env.ANALYTICS_DB.prepare( + `SELECT links.link_id, links.source_identity_id, links.canonical_id, links.linked_at + FROM analytics_identity_links AS links + JOIN analytics_identities AS identities + ON identities.identity_id = links.source_identity_id + WHERE links.posthog_state = 'pending' + AND identities.consent_level IN ('product', 'diagnostic', 'contribution') + ORDER BY links.linked_at, links.link_id + LIMIT ?`, + ) + .bind(POSTHOG_BATCH_SIZE) + .all(); + const rows = result.results; + if (rows.length === 0) return 0; + + let response: Response; + try { + response = await fetch(`${POSTHOG_HOST}/batch`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + api_key: env.POSTHOG_PROJECT_TOKEN, + batch: rows.map(identityIdentifyEvent), + }), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await env.ANALYTICS_DB.batch( + rows.map((row) => + env.ANALYTICS_DB.prepare( + `UPDATE analytics_identity_links + SET posthog_attempts = posthog_attempts + 1, + posthog_last_error = ? + WHERE link_id = ? AND posthog_state = 'pending'`, + ).bind(message.slice(0, 500), row.link_id), + ), + ); + throw error; + } + + if (!response.ok) { + const message = `PostHog returned ${response.status}`; + await env.ANALYTICS_DB.batch( + rows.map((row) => + env.ANALYTICS_DB.prepare( + `UPDATE analytics_identity_links + SET posthog_attempts = posthog_attempts + 1, + posthog_last_error = ? + WHERE link_id = ? AND posthog_state = 'pending'`, + ).bind(message, row.link_id), + ), + ); + throw new Error(message); + } + + await env.ANALYTICS_DB.batch( + rows.map((row) => + env.ANALYTICS_DB.prepare( + `UPDATE analytics_identity_links + SET posthog_state = 'sent', + posthog_attempts = posthog_attempts + 1, + posthog_last_error = NULL, + posthog_sent_at = CURRENT_TIMESTAMP + WHERE link_id = ? AND posthog_state = 'pending'`, + ).bind(row.link_id), + ), + ); + return rows.length; +} + +function identityType(identityId: string): "web_visitor" | "desktop_installation" { + if (VISITOR_ID_PATTERN.test(identityId)) return "web_visitor"; + if (INSTALLATION_ID_PATTERN.test(identityId)) return "desktop_installation"; + throw new RequestValidationError("Invalid source identity"); +} + +export function validateIdentityLinkPayload(value: unknown): { + readonly accountId: string; + readonly identityIds: ReadonlyArray; +} { + if (!isRecord(value) || value.schema_version !== 1) { + throw new RequestValidationError("Unsupported identity link payload"); + } + const accountId = requireString(value.account_id, "account_id", ACCOUNT_ID_PATTERN); + if (!Array.isArray(value.identity_ids) || value.identity_ids.length < 1) { + throw new RequestValidationError("At least one source identity is required"); + } + if (value.identity_ids.length > 20) { + throw new RequestValidationError("At most 20 source identities are accepted"); + } + const identityIds = [ + ...new Set( + value.identity_ids.map((id) => requireString(id, "identity_id", IDENTIFIER_PATTERN)), + ), + ]; + for (const identityId of identityIds) identityType(identityId); + return { accountId, identityIds }; +} + +async function persistIdentityLinks( + database: D1Database, + accountId: string, + identityIds: ReadonlyArray, +): Promise { + for (const identityId of identityIds) { + const existing = await database + .prepare( + `SELECT canonical_id + FROM analytics_identity_links + WHERE source_identity_id = ?`, + ) + .bind(identityId) + .first<{ readonly canonical_id: string }>(); + if (existing && existing.canonical_id !== accountId) { + throw new RequestValidationError("Source identity is already linked to another account"); + } + } + + const linkedAt = new Date().toISOString(); + const statements: D1PreparedStatement[] = [ + database + .prepare( + `INSERT INTO analytics_identities ( + identity_id, identity_type, canonical_id, consent_level, first_seen_at, last_seen_at + ) VALUES (?, 'account', ?, 'essential', ?, ?) + ON CONFLICT(identity_id) DO UPDATE SET last_seen_at = excluded.last_seen_at`, + ) + .bind(accountId, accountId, linkedAt, linkedAt), + ]; + + for (const identityId of identityIds) { + const type = identityType(identityId); + statements.push( + database + .prepare( + `INSERT INTO analytics_identities ( + identity_id, identity_type, canonical_id, consent_level, first_seen_at, last_seen_at, linked_at + ) VALUES (?, ?, ?, 'essential', ?, ?, ?) + ON CONFLICT(identity_id) DO UPDATE SET + canonical_id = excluded.canonical_id, + last_seen_at = excluded.last_seen_at, + linked_at = excluded.linked_at`, + ) + .bind(identityId, type, accountId, linkedAt, linkedAt, linkedAt), + database + .prepare( + `INSERT INTO analytics_identity_links ( + link_id, source_identity_id, canonical_id, linked_at + ) VALUES (?, ?, ?, ?) + ON CONFLICT(source_identity_id) DO UPDATE SET + canonical_id = excluded.canonical_id, + linked_at = excluded.linked_at, + posthog_state = CASE + WHEN analytics_identity_links.canonical_id = excluded.canonical_id + THEN analytics_identity_links.posthog_state + ELSE 'pending' + END`, + ) + .bind(crypto.randomUUID(), identityId, accountId, linkedAt), + database + .prepare( + `UPDATE analytics_events + SET canonical_id = ? + WHERE distinct_id = ?`, + ) + .bind(accountId, identityId), + ); + } + await database.batch(statements); +} + +async function handleIdentityLink( + request: Request, + env: AnalyticsEnv, + context: ExecutionContext, +): Promise { + if (!env.IDENTITY_LINK_TOKEN) { + return jsonResponse({ error: "Identity linking is not configured" }, 503); + } + if (request.headers.get("Authorization") !== `Bearer ${env.IDENTITY_LINK_TOKEN}`) { + return jsonResponse({ error: "Unauthorized" }, 401); + } + if (!request.headers.get("Content-Type")?.toLowerCase().startsWith("application/json")) { + return jsonResponse({ error: "Content-Type must be application/json" }, 415); + } + try { + const link = validateIdentityLinkPayload(await readJsonBody(request)); + await persistIdentityLinks(env.ANALYTICS_DB, link.accountId, link.identityIds); + context.waitUntil( + flushPendingIdentityLinks(env).catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "PostHog identity linking failed", + error: error instanceof Error ? error.message : String(error), + }), + ); + }), + ); + return jsonResponse({ linked: link.identityIds.length, canonical_id: link.accountId }, 200); + } catch (error) { + if (error instanceof RequestValidationError) { + return jsonResponse({ error: error.message }, 400); + } + console.error( + JSON.stringify({ + message: "Identity linking failed", + error: error instanceof Error ? error.message : String(error), + }), + ); + return jsonResponse({ error: "Identity linking failed" }, 500); + } +} + async function handleIngestion( request: Request, env: AnalyticsEnv, @@ -306,6 +618,7 @@ const worker: ExportedHandler = { status: "ready", storage: "configured", posthog_forwarding: env.POSTHOG_PROJECT_TOKEN ? "configured" : "pending_configuration", + identity_linking: env.IDENTITY_LINK_TOKEN ? "configured" : "pending_configuration", }); } if (request.method === "OPTIONS" && url.pathname === "/v1/events") { @@ -324,19 +637,24 @@ const worker: ExportedHandler = { if (request.method === "POST" && url.pathname === "/v1/events") { return handleIngestion(request, env, context); } + if (request.method === "POST" && url.pathname === "/v1/identity/link") { + return handleIdentityLink(request, env, context); + } return jsonResponse({ error: "Not found" }, 404); }, async scheduled(_controller, env, context) { context.waitUntil( - flushPendingEvents(env).catch((error: unknown) => { - console.error( - JSON.stringify({ - message: "Scheduled PostHog forwarding failed", - error: error instanceof Error ? error.message : String(error), - }), - ); - }), + flushPendingIdentityLinks(env) + .then(() => flushPendingEvents(env)) + .catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "Scheduled PostHog forwarding failed", + error: error instanceof Error ? error.message : String(error), + }), + ); + }), ); }, };