Add client-domain backup hosting

SETUP-INFRASTRUCTURE.md

@@ -14,7 +14,7 @@ All code changes are complete. This document covers the manual infrastructure pr
                  |  Upstash Redis HTTP   |
                  +----------+------------+
                             |
-              HTTP enqueue  |  SSE polling
+               HTTP enqueue  |  SSE polling
                             v
                  +-----------------------+
                  |  Railway (Worker)     |
@@ -25,10 +25,17 @@ All code changes are complete. This document covers the manual infrastructure pr
                             |
               ioredis TCP   |  S3-compat API
                             v
-         +-------------+  +----------------+
-         | Upstash     |  | Cloudflare R2  |
-         | Redis       |  | (storage)      |
-         +-------------+  +----------------+
+          +-------------+  +----------------+
+          | Upstash     |  | Cloudflare R2  |
+          | Redis       |  | (storage)      |
+          +-------------+  +----------------+
+                            ^
+                            |
+                 +----------+------------+
+                 | Cloudflare Worker     |
+                 | Backup Hosting        |
+                 | client CNAME domains  |
+                 +-----------------------+
                             |
               Hyperdrive    |
                             v
@@ -213,6 +220,9 @@ Go to your domain's Workers Routes and add:
 - [ ] Test: Create a crawl, verify it runs and SSE events arrive
 - [ ] Test: Preview an archived site
 - [ ] Test: Download a zip archive
+- [ ] Test: Publish a completed crawl and verify `published/<site>/<crawl>/index.html` exists in R2
+- [ ] Test: Add a client-owned CNAME hostname and wait for Cloudflare SSL status to become active
+- [ ] Test: Visit the client hostname and verify the hosted backup loads from R2
 - [ ] Keep old Railway API running for 48-72h as hot standby
 - [ ] Tear down old Railway API after stabilization
 
@@ -235,6 +245,7 @@ Set via `wrangler.toml` [vars]:
 - `NODE_ENV` — `production`
 - `FRONTEND_URL` — Frontend app URL
 - `CORS_ALLOWED_ORIGINS` — Comma-separated extra origins
+- `HOSTING_CNAME_TARGET` — hostname clients CNAME to for backup hosting
 
 Set via `wrangler secret put`:
 - `AUTH_SECRET` — Auth.js secret
@@ -245,17 +256,29 @@ Set via `wrangler secret put`:
 - `UPSTASH_REDIS_REST_TOKEN` — Upstash HTTP token
 - `WORKER_SERVICE_URL` — Railway worker HTTP URL
 - `WORKER_API_SECRET` — Shared auth secret for worker HTTP API
+- `CLOUDFLARE_ZONE_ID` — zone configured for Cloudflare for SaaS custom hostnames
+- `CLOUDFLARE_API_TOKEN` — token with permission to manage custom hostnames
 
 Bindings (in wrangler.toml):
 - `STORAGE_BUCKET` — R2 bucket binding
 - `HYPERDRIVE` — Hyperdrive binding to Railway Postgres
 
+### Backup Hosting Worker (Cloudflare)
+
+- Deploy `apps/hosting-worker` with the same `STORAGE_BUCKET` and `HYPERDRIVE` bindings.
+- Configure Cloudflare for SaaS/custom hostnames on the zone that owns your fallback hostname.
+- Set the API/Railway `HOSTING_CNAME_TARGET` to that fallback hostname. Clients create `CNAME backup.client.com -> HOSTING_CNAME_TARGET`.
+- The hosting Worker only serves domains present in `site_domains` with `status = active` and an active published backup.
+
 ### Worker Service (Railway)
 
 - `DATABASE_URL` — PostgreSQL connection string (unchanged)
 - `REDIS_URL` — Upstash Redis TCP URL (was local/Railway Redis)
 - `WORKER_HTTP_PORT` — Port for HTTP API server (default: 3002)
 - `WORKER_API_SECRET` — Shared auth secret
+- `HOSTING_CNAME_TARGET` — same client CNAME target shown in the dashboard
+- `CLOUDFLARE_ZONE_ID` — optional; enables automated custom hostname provisioning
+- `CLOUDFLARE_API_TOKEN` — optional; enables automated custom hostname provisioning
 - `S3_ENDPOINT` — R2 S3-compatible endpoint (was AWS S3)
 - `S3_ACCESS_KEY_ID` — R2 API token access key
 - `S3_SECRET_ACCESS_KEY` — R2 API token secret key

apps/api/src/app.ts

@@ -11,6 +11,7 @@ import { crawlsRoutes } from "./routes/crawls.js";
 import { settingsRoutes } from "./routes/settings.js";
 import { sseRoutes } from "./routes/sse.js";
 import { previewRoutes } from "./routes/preview.js";
+import { hostingRoutes } from "./routes/hosting.js";
 
 export interface AppConfig {
   deps: AppDeps;
@@ -119,11 +120,15 @@ export function createApp(config: AppConfig) {
   app.use("/api/crawls/*", requireAuth);
   app.use("/api/settings/*", requireAuth);
   app.use("/api/sse/*", requireAuth);
+  app.use("/api/sites/*/hosting", requireAuth);
+  app.use("/api/sites/*/publications", requireAuth);
+  app.use("/api/sites/*/domains/*", requireAuth);
 
   app.route("/api/sites", sitesRoutes);
   app.route("/api/crawls", crawlsRoutes);
   app.route("/api/settings", settingsRoutes);
   app.route("/api/sse", sseRoutes);
+  app.route("/api/sites", hostingRoutes);
 
   // Defense-in-depth: keep preview responses out of search indexes.
   app.use("/preview/*", async (c, next) => {

apps/api/src/db/migrations/0003_add_static_hosting.sql

@@ -0,0 +1,33 @@
+CREATE TABLE IF NOT EXISTS "site_publications" (
+  "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+  "site_id" uuid NOT NULL REFERENCES "sites"("id") ON DELETE cascade,
+  "crawl_id" uuid NOT NULL REFERENCES "crawls"("id") ON DELETE cascade,
+  "status" varchar(50) DEFAULT 'pending' NOT NULL,
+  "r2_prefix" varchar(500) NOT NULL,
+  "file_count" integer,
+  "total_bytes" bigint,
+  "error_message" text,
+  "published_at" timestamp with time zone,
+  "created_at" timestamp with time zone DEFAULT now(),
+  "updated_at" timestamp with time zone DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS "site_domains" (
+  "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+  "site_id" uuid NOT NULL REFERENCES "sites"("id") ON DELETE cascade,
+  "hostname" varchar(255) NOT NULL,
+  "status" varchar(50) DEFAULT 'pending_dns' NOT NULL,
+  "cname_target" varchar(255) NOT NULL,
+  "active_publication_id" uuid REFERENCES "site_publications"("id") ON DELETE set null,
+  "cloudflare_hostname_id" varchar(255),
+  "ownership_verification_name" varchar(255),
+  "ownership_verification_value" text,
+  "ssl_status" varchar(100),
+  "error_message" text,
+  "created_at" timestamp with time zone DEFAULT now(),
+  "updated_at" timestamp with time zone DEFAULT now()
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS "site_domains_hostname_idx" ON "site_domains" ("hostname");
+CREATE INDEX IF NOT EXISTS "site_publications_site_id_idx" ON "site_publications" ("site_id");
+CREATE INDEX IF NOT EXISTS "site_domains_site_id_idx" ON "site_domains" ("site_id");

apps/api/src/db/migrations/0004_add_hosting_controls_and_billing.sql

@@ -0,0 +1,4 @@
+ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_auto_publish" boolean DEFAULT true;
+ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_billing_email" varchar(255);
+ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_payment_link_url" varchar(1000);
+ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_billing_status" varchar(50) DEFAULT 'not_sent';

apps/api/src/db/migrations/0005_add_hosting_redirect_mode.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "site_domains" ADD COLUMN IF NOT EXISTS "redirect_enabled" boolean DEFAULT false;
+ALTER TABLE "site_domains" ADD COLUMN IF NOT EXISTS "redirect_target_origin" varchar(500);

apps/api/src/db/migrations/meta/_journal.json

@@ -22,6 +22,27 @@
       "when": 1738968780000,
       "tag": "0002_add_upload_progress",
       "breakpoints": true
+    },
+    {
+      "idx": 3,
+      "version": "7",
+      "when": 1777334400000,
+      "tag": "0003_add_static_hosting",
+      "breakpoints": true
+    },
+    {
+      "idx": 4,
+      "version": "7",
+      "when": 1777338000000,
+      "tag": "0004_add_hosting_controls_and_billing",
+      "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1777341600000,
+      "tag": "0005_add_hosting_redirect_mode",
+      "breakpoints": true
     }
   ]
 }

apps/api/src/db/schema.ts

@@ -9,6 +9,7 @@ import {
   timestamp,
   jsonb,
   primaryKey,
+  uniqueIndex,
 } from "drizzle-orm/pg-core";
 import { relations } from "drizzle-orm";
 import type { AdapterAccountType } from "@auth/core/adapters";
@@ -110,6 +111,12 @@ export const sites = pgTable("sites", {
   storageType: varchar("storage_type", { length: 50 }).default("local"),
   storagePath: varchar("storage_path", { length: 500 }),
 
+  // Hosting/billing
+  hostingAutoPublish: boolean("hosting_auto_publish").default(true),
+  hostingBillingEmail: varchar("hosting_billing_email", { length: 255 }),
+  hostingPaymentLinkUrl: varchar("hosting_payment_link_url", { length: 1000 }),
+  hostingBillingStatus: varchar("hosting_billing_status", { length: 50 }).default("not_sent"),
+
   // Metadata
   createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(),
   updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
@@ -163,9 +170,47 @@ export const settings = pgTable("settings", {
   updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
 });
 
+export const sitePublications = pgTable("site_publications", {
+  id: uuid("id").primaryKey().defaultRandom(),
+  siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }),
+  crawlId: uuid("crawl_id").notNull().references(() => crawls.id, { onDelete: "cascade" }),
+  status: varchar("status", { length: 50 }).notNull().default("pending"),
+  r2Prefix: varchar("r2_prefix", { length: 500 }).notNull(),
+  fileCount: integer("file_count"),
+  totalBytes: bigint("total_bytes", { mode: "number" }),
+  errorMessage: text("error_message"),
+  publishedAt: timestamp("published_at", { withTimezone: true }),
+  createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(),
+  updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
+});
+
+export const siteDomains = pgTable(
+  "site_domains",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }),
+    hostname: varchar("hostname", { length: 255 }).notNull(),
+    status: varchar("status", { length: 50 }).notNull().default("pending_dns"),
+    cnameTarget: varchar("cname_target", { length: 255 }).notNull(),
+    activePublicationId: uuid("active_publication_id").references(() => sitePublications.id, { onDelete: "set null" }),
+    redirectEnabled: boolean("redirect_enabled").default(false),
+    redirectTargetOrigin: varchar("redirect_target_origin", { length: 500 }),
+    cloudflareHostnameId: varchar("cloudflare_hostname_id", { length: 255 }),
+    ownershipVerificationName: varchar("ownership_verification_name", { length: 255 }),
+    ownershipVerificationValue: text("ownership_verification_value"),
+    sslStatus: varchar("ssl_status", { length: 100 }),
+    errorMessage: text("error_message"),
+    createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(),
+    updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(),
+  },
+  (domain) => [uniqueIndex("site_domains_hostname_idx").on(domain.hostname)]
+);
+
 // Relations
 export const sitesRelations = relations(sites, ({ many }) => ({
   crawls: many(crawls),
+  publications: many(sitePublications),
+  domains: many(siteDomains),
 }));
 
 export const crawlsRelations = relations(crawls, ({ one, many }) => ({
@@ -174,6 +219,30 @@ export const crawlsRelations = relations(crawls, ({ one, many }) => ({
     references: [sites.id],
   }),
   logs: many(crawlLogs),
+  publications: many(sitePublications),
+}));
+
+export const sitePublicationsRelations = relations(sitePublications, ({ one, many }) => ({
+  site: one(sites, {
+    fields: [sitePublications.siteId],
+    references: [sites.id],
+  }),
+  crawl: one(crawls, {
+    fields: [sitePublications.crawlId],
+    references: [crawls.id],
+  }),
+  domains: many(siteDomains),
+}));
+
+export const siteDomainsRelations = relations(siteDomains, ({ one }) => ({
+  site: one(sites, {
+    fields: [siteDomains.siteId],
+    references: [sites.id],
+  }),
+  activePublication: one(sitePublications, {
+    fields: [siteDomains.activePublicationId],
+    references: [sitePublications.id],
+  }),
 }));
 
 export const crawlLogsRelations = relations(crawlLogs, ({ one }) => ({
@@ -214,3 +283,7 @@ export type CrawlLog = typeof crawlLogs.$inferSelect;
 export type NewCrawlLog = typeof crawlLogs.$inferInsert;
 export type Setting = typeof settings.$inferSelect;
 export type AllowedEmail = typeof allowedEmails.$inferSelect;
+export type SitePublication = typeof sitePublications.$inferSelect;
+export type NewSitePublication = typeof sitePublications.$inferInsert;
+export type SiteDomain = typeof siteDomains.$inferSelect;
+export type NewSiteDomain = typeof siteDomains.$inferInsert;

apps/api/src/env.ts

@@ -30,6 +30,11 @@ export type Bindings = {
   WORKER_SERVICE_URL: string;
   WORKER_API_SECRET: string;
 
+  // Static backup hosting
+  HOSTING_CNAME_TARGET?: string;
+  CLOUDFLARE_ZONE_ID?: string;
+  CLOUDFLARE_API_TOKEN?: string;
+
   // Storage env vars (used by Node entry only, for backward compat)
   STORAGE_TYPE?: string;
   S3_ENDPOINT?: string;

apps/api/src/queue/client.ts

@@ -7,6 +7,7 @@ import Redis from "ioredis";
 
 export interface QueueClient {
   addCrawlJob(siteId: string, crawlId: string): Promise<void>;
+  addPublicationJob(siteId: string, crawlId: string, publicationId: string, activate: boolean, autoPublish?: boolean): Promise<void>;
   removeCrawlJob(crawlId: string): Promise<void>;
 }
 
@@ -34,6 +35,17 @@ export function createNodeQueueClient(redisUrl: string): QueueClient {
     async addCrawlJob(siteId, crawlId) {
       await queue.add("crawl", { siteId, crawlId }, { jobId: crawlId });
     },
+    async addPublicationJob(siteId, crawlId, publicationId, activate, autoPublish) {
+      const publishQueue = new Queue("publication-jobs", {
+        connection: redis,
+        defaultJobOptions: {
+          removeOnComplete: 100,
+          removeOnFail: 100,
+          attempts: 1,
+        },
+      });
+      await publishQueue.add("publish", { siteId, crawlId, publicationId, activate, autoPublish: autoPublish ?? false }, { jobId: publicationId });
+    },
     async removeCrawlJob(crawlId) {
       try {
         await queue.remove(crawlId);
@@ -87,6 +99,20 @@ export function createHttpQueueClient(workerServiceUrl: string, apiSecret: strin
         throw new Error(`Failed to enqueue crawl job: ${res.status} ${text}`);
       }
     },
+    async addPublicationJob(siteId, crawlId, publicationId, activate, autoPublish) {
+      const res = await fetch(`${baseUrl}/publish`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiSecret}`,
+        },
+        body: JSON.stringify({ siteId, crawlId, publicationId, activate, autoPublish: autoPublish ?? false }),
+      });
+      if (!res.ok) {
+        const text = await res.text();
+        throw new Error(`Failed to enqueue publication job: ${res.status} ${text}`);
+      }
+    },
     async removeCrawlJob(crawlId) {
       const res = await fetch(`${baseUrl}/cancel`, {
         method: "POST",