From 6c00dad83b85fa5d7fb9c0435a5804601cb5e19a Mon Sep 17 00:00:00 2001 From: Austin Thesing Date: Mon, 27 Apr 2026 13:15:03 -0700 Subject: [PATCH 1/4] add client-domain backup hosting --- SETUP-INFRASTRUCTURE.md | 33 +- apps/api/src/app.ts | 5 + .../db/migrations/0003_add_static_hosting.sql | 33 ++ .../0004_add_hosting_controls_and_billing.sql | 4 + .../0005_add_hosting_redirect_mode.sql | 2 + apps/api/src/db/migrations/meta/_journal.json | 21 + apps/api/src/db/schema.ts | 73 +++ apps/api/src/env.ts | 5 + apps/api/src/queue/client.ts | 26 + apps/api/src/routes/hosting.ts | 444 ++++++++++++++++++ apps/api/src/routes/sites.ts | 4 + apps/api/wrangler.toml | 3 + apps/hosting-worker/package.json | 20 + apps/hosting-worker/src/index.ts | 177 +++++++ apps/hosting-worker/tsconfig.json | 8 + apps/hosting-worker/wrangler.toml | 18 + apps/web/src/lib/api.ts | 164 +++++++ apps/web/src/routes/sites/$siteId.tsx | 354 +++++++++++++- bun.lock | 34 ++ packages/db/src/schema.ts | 71 +++ services/worker/package.json | 4 +- services/worker/src/db.ts | 71 ++- services/worker/src/http.ts | 36 ++ services/worker/src/processor.ts | 203 +++++++- 24 files changed, 1802 insertions(+), 11 deletions(-) create mode 100644 apps/api/src/db/migrations/0003_add_static_hosting.sql create mode 100644 apps/api/src/db/migrations/0004_add_hosting_controls_and_billing.sql create mode 100644 apps/api/src/db/migrations/0005_add_hosting_redirect_mode.sql create mode 100644 apps/api/src/routes/hosting.ts create mode 100644 apps/hosting-worker/package.json create mode 100644 apps/hosting-worker/src/index.ts create mode 100644 apps/hosting-worker/tsconfig.json create mode 100644 apps/hosting-worker/wrangler.toml diff --git a/SETUP-INFRASTRUCTURE.md b/SETUP-INFRASTRUCTURE.md index f4b31b4..26cd8f5 100644 --- a/SETUP-INFRASTRUCTURE.md +++ b/SETUP-INFRASTRUCTURE.md @@ -14,7 +14,7 @@ All code changes are complete. This document covers the manual infrastructure pr | Upstash Redis HTTP | +----------+------------+ | - HTTP enqueue | SSE polling + HTTP enqueue | SSE polling v +-----------------------+ | Railway (Worker) | @@ -25,10 +25,17 @@ All code changes are complete. This document covers the manual infrastructure pr | ioredis TCP | S3-compat API v - +-------------+ +----------------+ - | Upstash | | Cloudflare R2 | - | Redis | | (storage) | - +-------------+ +----------------+ + +-------------+ +----------------+ + | Upstash | | Cloudflare R2 | + | Redis | | (storage) | + +-------------+ +----------------+ + ^ + | + +----------+------------+ + | Cloudflare Worker | + | Backup Hosting | + | client CNAME domains | + +-----------------------+ | Hyperdrive | v @@ -213,6 +220,9 @@ Go to your domain's Workers Routes and add: - [ ] Test: Create a crawl, verify it runs and SSE events arrive - [ ] Test: Preview an archived site - [ ] Test: Download a zip archive +- [ ] Test: Publish a completed crawl and verify `published///index.html` exists in R2 +- [ ] Test: Add a client-owned CNAME hostname and wait for Cloudflare SSL status to become active +- [ ] Test: Visit the client hostname and verify the hosted backup loads from R2 - [ ] Keep old Railway API running for 48-72h as hot standby - [ ] Tear down old Railway API after stabilization @@ -235,6 +245,7 @@ Set via `wrangler.toml` [vars]: - `NODE_ENV` — `production` - `FRONTEND_URL` — Frontend app URL - `CORS_ALLOWED_ORIGINS` — Comma-separated extra origins +- `HOSTING_CNAME_TARGET` — hostname clients CNAME to for backup hosting Set via `wrangler secret put`: - `AUTH_SECRET` — Auth.js secret @@ -245,17 +256,29 @@ Set via `wrangler secret put`: - `UPSTASH_REDIS_REST_TOKEN` — Upstash HTTP token - `WORKER_SERVICE_URL` — Railway worker HTTP URL - `WORKER_API_SECRET` — Shared auth secret for worker HTTP API +- `CLOUDFLARE_ZONE_ID` — zone configured for Cloudflare for SaaS custom hostnames +- `CLOUDFLARE_API_TOKEN` — token with permission to manage custom hostnames Bindings (in wrangler.toml): - `STORAGE_BUCKET` — R2 bucket binding - `HYPERDRIVE` — Hyperdrive binding to Railway Postgres +### Backup Hosting Worker (Cloudflare) + +- Deploy `apps/hosting-worker` with the same `STORAGE_BUCKET` and `HYPERDRIVE` bindings. +- Configure Cloudflare for SaaS/custom hostnames on the zone that owns your fallback hostname. +- Set the API/Railway `HOSTING_CNAME_TARGET` to that fallback hostname. Clients create `CNAME backup.client.com -> HOSTING_CNAME_TARGET`. +- The hosting Worker only serves domains present in `site_domains` with `status = active` and an active published backup. + ### Worker Service (Railway) - `DATABASE_URL` — PostgreSQL connection string (unchanged) - `REDIS_URL` — Upstash Redis TCP URL (was local/Railway Redis) - `WORKER_HTTP_PORT` — Port for HTTP API server (default: 3002) - `WORKER_API_SECRET` — Shared auth secret +- `HOSTING_CNAME_TARGET` — same client CNAME target shown in the dashboard +- `CLOUDFLARE_ZONE_ID` — optional; enables automated custom hostname provisioning +- `CLOUDFLARE_API_TOKEN` — optional; enables automated custom hostname provisioning - `S3_ENDPOINT` — R2 S3-compatible endpoint (was AWS S3) - `S3_ACCESS_KEY_ID` — R2 API token access key - `S3_SECRET_ACCESS_KEY` — R2 API token secret key diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 9dcb3b0..aac7211 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -11,6 +11,7 @@ import { crawlsRoutes } from "./routes/crawls.js"; import { settingsRoutes } from "./routes/settings.js"; import { sseRoutes } from "./routes/sse.js"; import { previewRoutes } from "./routes/preview.js"; +import { hostingRoutes } from "./routes/hosting.js"; export interface AppConfig { deps: AppDeps; @@ -119,11 +120,15 @@ export function createApp(config: AppConfig) { app.use("/api/crawls/*", requireAuth); app.use("/api/settings/*", requireAuth); app.use("/api/sse/*", requireAuth); + app.use("/api/sites/*/hosting", requireAuth); + app.use("/api/sites/*/publications", requireAuth); + app.use("/api/sites/*/domains/*", requireAuth); app.route("/api/sites", sitesRoutes); app.route("/api/crawls", crawlsRoutes); app.route("/api/settings", settingsRoutes); app.route("/api/sse", sseRoutes); + app.route("/api/sites", hostingRoutes); // Defense-in-depth: keep preview responses out of search indexes. app.use("/preview/*", async (c, next) => { diff --git a/apps/api/src/db/migrations/0003_add_static_hosting.sql b/apps/api/src/db/migrations/0003_add_static_hosting.sql new file mode 100644 index 0000000..9210684 --- /dev/null +++ b/apps/api/src/db/migrations/0003_add_static_hosting.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS "site_publications" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "site_id" uuid NOT NULL REFERENCES "sites"("id") ON DELETE cascade, + "crawl_id" uuid NOT NULL REFERENCES "crawls"("id") ON DELETE cascade, + "status" varchar(50) DEFAULT 'pending' NOT NULL, + "r2_prefix" varchar(500) NOT NULL, + "file_count" integer, + "total_bytes" bigint, + "error_message" text, + "published_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS "site_domains" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "site_id" uuid NOT NULL REFERENCES "sites"("id") ON DELETE cascade, + "hostname" varchar(255) NOT NULL, + "status" varchar(50) DEFAULT 'pending_dns' NOT NULL, + "cname_target" varchar(255) NOT NULL, + "active_publication_id" uuid REFERENCES "site_publications"("id") ON DELETE set null, + "cloudflare_hostname_id" varchar(255), + "ownership_verification_name" varchar(255), + "ownership_verification_value" text, + "ssl_status" varchar(100), + "error_message" text, + "created_at" timestamp with time zone DEFAULT now(), + "updated_at" timestamp with time zone DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS "site_domains_hostname_idx" ON "site_domains" ("hostname"); +CREATE INDEX IF NOT EXISTS "site_publications_site_id_idx" ON "site_publications" ("site_id"); +CREATE INDEX IF NOT EXISTS "site_domains_site_id_idx" ON "site_domains" ("site_id"); diff --git a/apps/api/src/db/migrations/0004_add_hosting_controls_and_billing.sql b/apps/api/src/db/migrations/0004_add_hosting_controls_and_billing.sql new file mode 100644 index 0000000..3cbab44 --- /dev/null +++ b/apps/api/src/db/migrations/0004_add_hosting_controls_and_billing.sql @@ -0,0 +1,4 @@ +ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_auto_publish" boolean DEFAULT true; +ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_billing_email" varchar(255); +ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_payment_link_url" varchar(1000); +ALTER TABLE "sites" ADD COLUMN IF NOT EXISTS "hosting_billing_status" varchar(50) DEFAULT 'not_sent'; diff --git a/apps/api/src/db/migrations/0005_add_hosting_redirect_mode.sql b/apps/api/src/db/migrations/0005_add_hosting_redirect_mode.sql new file mode 100644 index 0000000..47a9ef9 --- /dev/null +++ b/apps/api/src/db/migrations/0005_add_hosting_redirect_mode.sql @@ -0,0 +1,2 @@ +ALTER TABLE "site_domains" ADD COLUMN IF NOT EXISTS "redirect_enabled" boolean DEFAULT false; +ALTER TABLE "site_domains" ADD COLUMN IF NOT EXISTS "redirect_target_origin" varchar(500); diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index c5aa69e..965973a 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -22,6 +22,27 @@ "when": 1738968780000, "tag": "0002_add_upload_progress", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1777334400000, + "tag": "0003_add_static_hosting", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1777338000000, + "tag": "0004_add_hosting_controls_and_billing", + "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1777341600000, + "tag": "0005_add_hosting_redirect_mode", + "breakpoints": true } ] } diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 97d81a3..44eecd1 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -9,6 +9,7 @@ import { timestamp, jsonb, primaryKey, + uniqueIndex, } from "drizzle-orm/pg-core"; import { relations } from "drizzle-orm"; import type { AdapterAccountType } from "@auth/core/adapters"; @@ -110,6 +111,12 @@ export const sites = pgTable("sites", { storageType: varchar("storage_type", { length: 50 }).default("local"), storagePath: varchar("storage_path", { length: 500 }), + // Hosting/billing + hostingAutoPublish: boolean("hosting_auto_publish").default(true), + hostingBillingEmail: varchar("hosting_billing_email", { length: 255 }), + hostingPaymentLinkUrl: varchar("hosting_payment_link_url", { length: 1000 }), + hostingBillingStatus: varchar("hosting_billing_status", { length: 50 }).default("not_sent"), + // Metadata createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), @@ -163,9 +170,47 @@ export const settings = pgTable("settings", { updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), }); +export const sitePublications = pgTable("site_publications", { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + crawlId: uuid("crawl_id").notNull().references(() => crawls.id, { onDelete: "cascade" }), + status: varchar("status", { length: 50 }).notNull().default("pending"), + r2Prefix: varchar("r2_prefix", { length: 500 }).notNull(), + fileCount: integer("file_count"), + totalBytes: bigint("total_bytes", { mode: "number" }), + errorMessage: text("error_message"), + publishedAt: timestamp("published_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), +}); + +export const siteDomains = pgTable( + "site_domains", + { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + hostname: varchar("hostname", { length: 255 }).notNull(), + status: varchar("status", { length: 50 }).notNull().default("pending_dns"), + cnameTarget: varchar("cname_target", { length: 255 }).notNull(), + activePublicationId: uuid("active_publication_id").references(() => sitePublications.id, { onDelete: "set null" }), + redirectEnabled: boolean("redirect_enabled").default(false), + redirectTargetOrigin: varchar("redirect_target_origin", { length: 500 }), + cloudflareHostnameId: varchar("cloudflare_hostname_id", { length: 255 }), + ownershipVerificationName: varchar("ownership_verification_name", { length: 255 }), + ownershipVerificationValue: text("ownership_verification_value"), + sslStatus: varchar("ssl_status", { length: 100 }), + errorMessage: text("error_message"), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), + }, + (domain) => [uniqueIndex("site_domains_hostname_idx").on(domain.hostname)] +); + // Relations export const sitesRelations = relations(sites, ({ many }) => ({ crawls: many(crawls), + publications: many(sitePublications), + domains: many(siteDomains), })); export const crawlsRelations = relations(crawls, ({ one, many }) => ({ @@ -174,6 +219,30 @@ export const crawlsRelations = relations(crawls, ({ one, many }) => ({ references: [sites.id], }), logs: many(crawlLogs), + publications: many(sitePublications), +})); + +export const sitePublicationsRelations = relations(sitePublications, ({ one, many }) => ({ + site: one(sites, { + fields: [sitePublications.siteId], + references: [sites.id], + }), + crawl: one(crawls, { + fields: [sitePublications.crawlId], + references: [crawls.id], + }), + domains: many(siteDomains), +})); + +export const siteDomainsRelations = relations(siteDomains, ({ one }) => ({ + site: one(sites, { + fields: [siteDomains.siteId], + references: [sites.id], + }), + activePublication: one(sitePublications, { + fields: [siteDomains.activePublicationId], + references: [sitePublications.id], + }), })); export const crawlLogsRelations = relations(crawlLogs, ({ one }) => ({ @@ -214,3 +283,7 @@ export type CrawlLog = typeof crawlLogs.$inferSelect; export type NewCrawlLog = typeof crawlLogs.$inferInsert; export type Setting = typeof settings.$inferSelect; export type AllowedEmail = typeof allowedEmails.$inferSelect; +export type SitePublication = typeof sitePublications.$inferSelect; +export type NewSitePublication = typeof sitePublications.$inferInsert; +export type SiteDomain = typeof siteDomains.$inferSelect; +export type NewSiteDomain = typeof siteDomains.$inferInsert; diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 2a0aa51..6c1abe9 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -30,6 +30,11 @@ export type Bindings = { WORKER_SERVICE_URL: string; WORKER_API_SECRET: string; + // Static backup hosting + HOSTING_CNAME_TARGET?: string; + CLOUDFLARE_ZONE_ID?: string; + CLOUDFLARE_API_TOKEN?: string; + // Storage env vars (used by Node entry only, for backward compat) STORAGE_TYPE?: string; S3_ENDPOINT?: string; diff --git a/apps/api/src/queue/client.ts b/apps/api/src/queue/client.ts index 1b2a60d..abbd02d 100644 --- a/apps/api/src/queue/client.ts +++ b/apps/api/src/queue/client.ts @@ -7,6 +7,7 @@ import Redis from "ioredis"; export interface QueueClient { addCrawlJob(siteId: string, crawlId: string): Promise; + addPublicationJob(siteId: string, crawlId: string, publicationId: string, activate: boolean, autoPublish?: boolean): Promise; removeCrawlJob(crawlId: string): Promise; } @@ -34,6 +35,17 @@ export function createNodeQueueClient(redisUrl: string): QueueClient { async addCrawlJob(siteId, crawlId) { await queue.add("crawl", { siteId, crawlId }, { jobId: crawlId }); }, + async addPublicationJob(siteId, crawlId, publicationId, activate, autoPublish) { + const publishQueue = new Queue("publication-jobs", { + connection: redis, + defaultJobOptions: { + removeOnComplete: 100, + removeOnFail: 100, + attempts: 1, + }, + }); + await publishQueue.add("publish", { siteId, crawlId, publicationId, activate, autoPublish: autoPublish ?? false }, { jobId: publicationId }); + }, async removeCrawlJob(crawlId) { try { await queue.remove(crawlId); @@ -87,6 +99,20 @@ export function createHttpQueueClient(workerServiceUrl: string, apiSecret: strin throw new Error(`Failed to enqueue crawl job: ${res.status} ${text}`); } }, + async addPublicationJob(siteId, crawlId, publicationId, activate, autoPublish) { + const res = await fetch(`${baseUrl}/publish`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiSecret}`, + }, + body: JSON.stringify({ siteId, crawlId, publicationId, activate, autoPublish: autoPublish ?? false }), + }); + if (!res.ok) { + const text = await res.text(); + throw new Error(`Failed to enqueue publication job: ${res.status} ${text}`); + } + }, async removeCrawlJob(crawlId) { const res = await fetch(`${baseUrl}/cancel`, { method: "POST", diff --git a/apps/api/src/routes/hosting.ts b/apps/api/src/routes/hosting.ts new file mode 100644 index 0000000..551a49a --- /dev/null +++ b/apps/api/src/routes/hosting.ts @@ -0,0 +1,444 @@ +import { Hono } from "hono"; +import { z } from "zod"; +import { zValidator } from "@hono/zod-validator"; +import { and, desc, eq } from "drizzle-orm"; +import { crawls, siteDomains, sitePublications, sites } from "../db/schema.js"; +import type { AppEnv } from "../env.js"; + +const app = new Hono(); + +const hostnameSchema = z + .string() + .trim() + .toLowerCase() + .min(4) + .max(255) + .refine((value) => !value.startsWith("*."), "Use a concrete client subdomain, not a wildcard") + .refine((value) => !value.startsWith("http://") && !value.startsWith("https://"), "Enter a hostname, not a URL") + .refine((value) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/.test(value), "Invalid hostname"); + +const createPublicationSchema = z.object({ + crawlId: z.string().uuid().optional(), + activate: z.boolean().optional().default(true), +}); + +const createDomainSchema = z.object({ + hostname: hostnameSchema, +}); + +const activatePublicationSchema = z.object({ + publicationId: z.string().uuid(), +}); + +const updateDomainSchema = z.object({ + redirectEnabled: z.boolean().optional(), + redirectTargetOrigin: z.string().url().optional().nullable(), +}); + +const updateHostingSettingsSchema = z.object({ + hostingAutoPublish: z.boolean().optional(), + hostingBillingEmail: z.string().email().optional().nullable(), + hostingPaymentLinkUrl: z.string().url().optional().nullable(), + hostingBillingStatus: z.enum(["not_sent", "sent", "paid", "past_due", "cancelled"]).optional(), +}); + +function getHostingCnameTarget(c: { env?: { HOSTING_CNAME_TARGET?: string } }): string { + const target = c.env?.HOSTING_CNAME_TARGET || process.env.HOSTING_CNAME_TARGET; + if (!target) { + throw new Error("HOSTING_CNAME_TARGET is required to create hosted domains"); + } + return target.replace(/^https?:\/\//, "").replace(/\/+$/, "").toLowerCase(); +} + +function getCloudflareConfig(c: { env?: { CLOUDFLARE_ZONE_ID?: string; CLOUDFLARE_API_TOKEN?: string } }) { + const zoneId = c.env?.CLOUDFLARE_ZONE_ID || process.env.CLOUDFLARE_ZONE_ID; + const apiToken = c.env?.CLOUDFLARE_API_TOKEN || process.env.CLOUDFLARE_API_TOKEN; + if (!zoneId || !apiToken) return null; + return { zoneId, apiToken }; +} + +async function createCloudflareCustomHostname( + c: { env?: { CLOUDFLARE_ZONE_ID?: string; CLOUDFLARE_API_TOKEN?: string } }, + hostname: string +) { + const config = getCloudflareConfig(c); + if (!config) return null; + + const response = await fetch(`https://api.cloudflare.com/client/v4/zones/${config.zoneId}/custom_hostnames`, { + method: "POST", + headers: { + Authorization: `Bearer ${config.apiToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + hostname, + ssl: { + method: "txt", + type: "dv", + settings: { + min_tls_version: "1.2", + }, + }, + }), + }); + + const payload = (await response.json().catch(() => null)) as any; + if (!response.ok || payload?.success === false) { + const message = payload?.errors?.[0]?.message || `Cloudflare custom hostname failed with ${response.status}`; + throw new Error(message); + } + + return payload?.result ?? null; +} + +async function deleteCloudflareCustomHostname( + c: { env?: { CLOUDFLARE_ZONE_ID?: string; CLOUDFLARE_API_TOKEN?: string } }, + cloudflareHostnameId: string +) { + const config = getCloudflareConfig(c); + if (!config) return; + + const response = await fetch( + `https://api.cloudflare.com/client/v4/zones/${config.zoneId}/custom_hostnames/${cloudflareHostnameId}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${config.apiToken}`, + }, + } + ); + + const payload = (await response.json().catch(() => null)) as any; + if (!response.ok || payload?.success === false) { + const message = payload?.errors?.[0]?.message || `Cloudflare custom hostname delete failed with ${response.status}`; + throw new Error(message); + } +} + +async function syncCloudflareCustomHostname( + c: { env?: { CLOUDFLARE_ZONE_ID?: string; CLOUDFLARE_API_TOKEN?: string } }, + cloudflareHostnameId: string +) { + const config = getCloudflareConfig(c); + if (!config) return null; + + const response = await fetch( + `https://api.cloudflare.com/client/v4/zones/${config.zoneId}/custom_hostnames/${cloudflareHostnameId}`, + { + headers: { + Authorization: `Bearer ${config.apiToken}`, + }, + } + ); + + const payload = (await response.json().catch(() => null)) as any; + if (!response.ok || payload?.success === false) { + const message = payload?.errors?.[0]?.message || `Cloudflare custom hostname sync failed with ${response.status}`; + throw new Error(message); + } + + return payload?.result ?? null; +} + +function extractCloudflareDomainFields(result: any) { + const ownership = result?.ownership_verification ?? result?.ownership_verification_http; + const ssl = result?.ssl; + const status = result?.status === "active" && ssl?.status === "active" ? "active" : "pending_dns"; + + return { + cloudflareHostnameId: typeof result?.id === "string" ? result.id : null, + ownershipVerificationName: typeof ownership?.name === "string" ? ownership.name : null, + ownershipVerificationValue: typeof ownership?.value === "string" ? ownership.value : null, + sslStatus: typeof ssl?.status === "string" ? ssl.status : null, + status, + }; +} + +function toOrigin(value: string | null | undefined): string | null { + if (!value) return null; + try { + return new URL(value).origin; + } catch { + return null; + } +} + +app.get("/:siteId/hosting", async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site) return c.json({ error: "Site not found" }, 404); + + const [publications, domains] = await Promise.all([ + db.query.sitePublications.findMany({ + where: eq(sitePublications.siteId, siteId), + orderBy: desc(sitePublications.createdAt), + with: { crawl: true }, + }), + db.query.siteDomains.findMany({ + where: eq(siteDomains.siteId, siteId), + orderBy: desc(siteDomains.createdAt), + with: { activePublication: true }, + }), + ]); + + return c.json({ + cnameTarget: getHostingCnameTarget(c), + settings: { + hostingAutoPublish: site.hostingAutoPublish ?? true, + hostingBillingEmail: site.hostingBillingEmail, + hostingPaymentLinkUrl: site.hostingPaymentLinkUrl, + hostingBillingStatus: site.hostingBillingStatus ?? "not_sent", + }, + publications, + domains, + }); +}); + +app.patch("/:siteId/hosting", zValidator("json", updateHostingSettingsSchema), async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const data = c.req.valid("json"); + + const [site] = await db + .update(sites) + .set({ ...data, updatedAt: new Date() }) + .where(eq(sites.id, siteId)) + .returning(); + + if (!site) return c.json({ error: "Site not found" }, 404); + + return c.json({ + settings: { + hostingAutoPublish: site.hostingAutoPublish ?? true, + hostingBillingEmail: site.hostingBillingEmail, + hostingPaymentLinkUrl: site.hostingPaymentLinkUrl, + hostingBillingStatus: site.hostingBillingStatus ?? "not_sent", + }, + }); +}); + +app.post("/:siteId/publications", zValidator("json", createPublicationSchema), async (c) => { + const db = c.get("db"); + const queue = c.get("queue"); + const siteId = c.req.param("siteId"); + const data = c.req.valid("json"); + + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site) return c.json({ error: "Site not found" }, 404); + + const crawl = data.crawlId + ? await db.query.crawls.findFirst({ where: and(eq(crawls.id, data.crawlId), eq(crawls.siteId, siteId)) }) + : await db.query.crawls.findFirst({ + where: and(eq(crawls.siteId, siteId), eq(crawls.status, "completed")), + orderBy: desc(crawls.createdAt), + }); + + if (!crawl) return c.json({ error: "No completed crawl found for this site" }, 404); + if (crawl.status !== "completed" || !crawl.outputPath) { + return c.json({ error: "Only completed crawls with ZIP output can be published" }, 400); + } + + const publicationId = crypto.randomUUID(); + const r2Prefix = `published/${siteId}/${publicationId}`; + const [publication] = await db + .insert(sitePublications) + .values({ id: publicationId, siteId, crawlId: crawl.id, status: "pending", r2Prefix }) + .returning(); + + try { + await queue.addPublicationJob(siteId, crawl.id, publication.id, data.activate, false); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown queue error"; + await db + .update(sitePublications) + .set({ status: "failed", errorMessage: `Failed to queue publication job: ${message}`, updatedAt: new Date() }) + .where(eq(sitePublications.id, publication.id)); + return c.json({ error: "Failed to queue publication job" }, 503); + } + + return c.json({ publication }, 201); +}); + +app.post("/:siteId/domains", zValidator("json", createDomainSchema), async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const { hostname } = c.req.valid("json"); + const cnameTarget = getHostingCnameTarget(c); + + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site) return c.json({ error: "Site not found" }, 404); + if (hostname === cnameTarget || hostname.endsWith(`.${cnameTarget}`)) { + return c.json({ error: "Use a client-owned hostname, not the hosting target domain" }, 400); + } + + const cloudflareConfig = getCloudflareConfig(c); + if (!cloudflareConfig) { + return c.json({ error: "Cloudflare custom hostname configuration is required to add client domains" }, 503); + } + + let cloudflareFields: ReturnType; + try { + const result = await createCloudflareCustomHostname(c, hostname); + if (!result) { + return c.json({ error: "Cloudflare custom hostname provisioning is not configured" }, 503); + } + cloudflareFields = extractCloudflareDomainFields(result); + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to provision Cloudflare custom hostname"; + return c.json({ error: message }, 502); + } + + const latestPublication = await db.query.sitePublications.findFirst({ + where: and(eq(sitePublications.siteId, siteId), eq(sitePublications.status, "published")), + orderBy: desc(sitePublications.createdAt), + }); + + try { + const [domain] = await db + .insert(siteDomains) + .values({ + siteId, + hostname, + cnameTarget, + status: cloudflareFields?.status ?? "pending_dns", + activePublicationId: latestPublication?.id ?? null, + redirectTargetOrigin: toOrigin(site.url), + cloudflareHostnameId: cloudflareFields?.cloudflareHostnameId, + ownershipVerificationName: cloudflareFields?.ownershipVerificationName, + ownershipVerificationValue: cloudflareFields?.ownershipVerificationValue, + sslStatus: cloudflareFields?.sslStatus, + }) + .returning(); + + return c.json({ domain }, 201); + } catch (error) { + return c.json({ error: "Hostname is already configured" }, 409); + } +}); + +app.patch("/:siteId/domains/:domainId", zValidator("json", updateDomainSchema), async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const domainId = c.req.param("domainId"); + const data = c.req.valid("json"); + + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site) return c.json({ error: "Site not found" }, 404); + + const redirectTargetOrigin = + "redirectTargetOrigin" in data ? toOrigin(data.redirectTargetOrigin) : undefined; + if (data.redirectTargetOrigin && !redirectTargetOrigin) { + return c.json({ error: "Redirect target must be a valid URL origin" }, 400); + } + + const [domain] = await db + .update(siteDomains) + .set({ + ...(typeof data.redirectEnabled === "boolean" ? { redirectEnabled: data.redirectEnabled } : {}), + ...(redirectTargetOrigin !== undefined ? { redirectTargetOrigin } : {}), + updatedAt: new Date(), + }) + .where(and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId))) + .returning(); + + if (!domain) return c.json({ error: "Domain not found" }, 404); + return c.json({ domain }); +}); + +app.post("/:siteId/domains/:domainId/sync", async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const domainId = c.req.param("domainId"); + + const domain = await db.query.siteDomains.findFirst({ + where: and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId)), + }); + if (!domain) return c.json({ error: "Domain not found" }, 404); + if (!domain.cloudflareHostnameId) return c.json({ domain }); + + const result = await syncCloudflareCustomHostname(c, domain.cloudflareHostnameId); + const fields = extractCloudflareDomainFields(result); + const [updated] = await db + .update(siteDomains) + .set({ ...fields, updatedAt: new Date() }) + .where(eq(siteDomains.id, domain.id)) + .returning(); + + return c.json({ domain: updated }); +}); + +app.post("/:siteId/domains/:domainId/activate", zValidator("json", activatePublicationSchema), async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const domainId = c.req.param("domainId"); + const { publicationId } = c.req.valid("json"); + + const publication = await db.query.sitePublications.findFirst({ + where: and(eq(sitePublications.id, publicationId), eq(sitePublications.siteId, siteId)), + }); + if (!publication || publication.status !== "published") { + return c.json({ error: "Published version not found" }, 404); + } + + const [domain] = await db + .update(siteDomains) + .set({ activePublicationId: publicationId, updatedAt: new Date() }) + .where(and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId))) + .returning(); + + if (!domain) return c.json({ error: "Domain not found" }, 404); + + await db + .update(sites) + .set({ hostingAutoPublish: false, updatedAt: new Date() }) + .where(eq(sites.id, siteId)); + + return c.json({ domain }); +}); + +app.post("/:siteId/publications/:publicationId/activate", async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const publicationId = c.req.param("publicationId"); + + const publication = await db.query.sitePublications.findFirst({ + where: and(eq(sitePublications.id, publicationId), eq(sitePublications.siteId, siteId)), + }); + if (!publication || publication.status !== "published") { + return c.json({ error: "Published version not found" }, 404); + } + + await db + .update(siteDomains) + .set({ activePublicationId: publicationId, updatedAt: new Date() }) + .where(eq(siteDomains.siteId, siteId)); + + await db + .update(sites) + .set({ hostingAutoPublish: false, updatedAt: new Date() }) + .where(eq(sites.id, siteId)); + + return c.json({ publication }); +}); + +app.delete("/:siteId/domains/:domainId", async (c) => { + const db = c.get("db"); + const siteId = c.req.param("siteId"); + const domainId = c.req.param("domainId"); + + const existing = await db.query.siteDomains.findFirst({ + where: and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId)), + }); + if (!existing) return c.json({ error: "Domain not found" }, 404); + + if (existing.cloudflareHostnameId) { + await deleteCloudflareCustomHostname(c, existing.cloudflareHostnameId); + } + + await db.delete(siteDomains).where(and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId))); + return c.json({ success: true }); +}); + +export const hostingRoutes = app; diff --git a/apps/api/src/routes/sites.ts b/apps/api/src/routes/sites.ts index cbdebc1..80cfa03 100644 --- a/apps/api/src/routes/sites.ts +++ b/apps/api/src/routes/sites.ts @@ -34,6 +34,10 @@ const createSiteSchema = z.object({ scheduleCron: z.string().optional().nullable(), storageType: z.enum(["local", "s3"]).optional(), storagePath: z.string().optional().nullable(), + hostingAutoPublish: z.boolean().optional(), + hostingBillingEmail: z.string().email().optional().nullable(), + hostingPaymentLinkUrl: z.string().url().optional().nullable(), + hostingBillingStatus: z.enum(["not_sent", "sent", "paid", "past_due", "cancelled"]).optional(), }); const updateSiteSchema = createSiteSchema.partial(); diff --git a/apps/api/wrangler.toml b/apps/api/wrangler.toml index 82f78ca..6ab7204 100644 --- a/apps/api/wrangler.toml +++ b/apps/api/wrangler.toml @@ -18,6 +18,7 @@ id = "REPLACE_WITH_HYPERDRIVE_CONFIG_ID" NODE_ENV = "production" FRONTEND_URL = "https://archiver.designxdevelop.com" CORS_ALLOWED_ORIGINS = "" +HOSTING_CNAME_TARGET = "backups.example.com" # Secrets (set via `wrangler secret put`): # - AUTH_SECRET @@ -28,3 +29,5 @@ CORS_ALLOWED_ORIGINS = "" # - UPSTASH_REDIS_REST_TOKEN # - WORKER_SERVICE_URL # - WORKER_API_SECRET +# - CLOUDFLARE_ZONE_ID +# - CLOUDFLARE_API_TOKEN diff --git a/apps/hosting-worker/package.json b/apps/hosting-worker/package.json new file mode 100644 index 0000000..c2e423e --- /dev/null +++ b/apps/hosting-worker/package.json @@ -0,0 +1,20 @@ +{ + "name": "@dxd/hosting-worker", + "version": "1.0.0", + "type": "module", + "scripts": { + "dev": "wrangler dev", + "build": "echo 'No build needed - Wrangler bundles TypeScript'", + "deploy": "wrangler deploy" + }, + "dependencies": { + "@dxd/db": "workspace:*", + "drizzle-orm": "^0.44.2", + "postgres": "^3.4.7" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20241230.0", + "typescript": "^5.9.3", + "wrangler": "^3.99.0" + } +} diff --git a/apps/hosting-worker/src/index.ts b/apps/hosting-worker/src/index.ts new file mode 100644 index 0000000..16404e4 --- /dev/null +++ b/apps/hosting-worker/src/index.ts @@ -0,0 +1,177 @@ +import { createDbClient, siteDomains } from "@dxd/db"; +import { eq } from "drizzle-orm"; + +type Env = { + HYPERDRIVE: Hyperdrive; + STORAGE_BUCKET: R2Bucket; +}; + +let cachedDb: ReturnType | null = null; +let cachedConnectionString: string | null = null; + +const NOT_FOUND_HEADERS = { + "Content-Type": "text/plain; charset=utf-8", + "X-Robots-Tag": "noindex, nofollow, noarchive", +}; + +export default { + async fetch(request: Request, env: Env): Promise { + if (request.method !== "GET" && request.method !== "HEAD") { + return new Response("Method Not Allowed", { status: 405, headers: { Allow: "GET, HEAD" } }); + } + + const url = new URL(request.url); + const hostname = url.hostname.toLowerCase(); + const db = getDb(env.HYPERDRIVE.connectionString); + + const domain = await db.query.siteDomains.findFirst({ + where: eq(siteDomains.hostname, hostname), + with: { activePublication: true, site: true }, + }); + + if (!domain || domain.status !== "active") { + return new Response("Backup site is not active", { status: 404, headers: NOT_FOUND_HEADERS }); + } + + if (domain.redirectEnabled) { + const targetOrigin = domain.redirectTargetOrigin || originForUrl(domain.site.url); + if (!targetOrigin) { + return new Response("Redirect target is not configured", { status: 500, headers: NOT_FOUND_HEADERS }); + } + + const redirectUrl = new URL(request.url); + redirectUrl.protocol = new URL(targetOrigin).protocol; + redirectUrl.host = new URL(targetOrigin).host; + return Response.redirect(redirectUrl.toString(), 301); + } + + if (!domain.activePublication) { + return new Response("Backup publication is not active", { status: 404, headers: NOT_FOUND_HEADERS }); + } + + if (domain.activePublication.status !== "published") { + return new Response("Backup publication is not ready", { status: 404, headers: NOT_FOUND_HEADERS }); + } + + const object = await findObject(env.STORAGE_BUCKET, domain.activePublication.r2Prefix, url.pathname); + if (!object) { + return new Response("Not Found", { status: 404, headers: NOT_FOUND_HEADERS }); + } + + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set("Content-Type", headers.get("Content-Type") || contentTypeForKey(object.key)); + headers.set("Cache-Control", cacheControlForKey(object.key)); + headers.set("ETag", object.httpEtag); + headers.set("X-Robots-Tag", "noindex, nofollow, noarchive"); + + if (request.method === "HEAD") { + return new Response(null, { headers }); + } + + return new Response(object.body, { headers }); + }, +}; + +function originForUrl(value: string): string | null { + try { + return new URL(value).origin; + } catch { + return null; + } +} + +function getDb(connectionString: string): ReturnType { + if (!cachedDb || cachedConnectionString !== connectionString) { + cachedDb = createDbClient(connectionString); + cachedConnectionString = connectionString; + } + return cachedDb; +} + +async function findObject(bucket: R2Bucket, prefix: string, pathname: string): Promise { + const candidates = objectKeyCandidates(prefix, pathname); + for (const key of candidates) { + const object = await bucket.get(key); + if (object) return object; + } + return null; +} + +function objectKeyCandidates(prefix: string, pathname: string): string[] { + const safePath = normalizeRequestPath(pathname); + const base = prefix.replace(/\/+$/, ""); + const candidates = new Set(); + + if (!safePath || safePath === "/") { + candidates.add(`${base}/index.html`); + return [...candidates]; + } + + const withoutLeadingSlash = safePath.replace(/^\/+/, ""); + candidates.add(`${base}/${withoutLeadingSlash}`); + + if (safePath.endsWith("/")) { + candidates.add(`${base}/${withoutLeadingSlash}index.html`); + } else if (!withoutLeadingSlash.split("/").pop()?.includes(".")) { + candidates.add(`${base}/${withoutLeadingSlash}/index.html`); + candidates.add(`${base}/${withoutLeadingSlash}.html`); + } + + return [...candidates]; +} + +function normalizeRequestPath(pathname: string): string { + try { + const decoded = decodeURIComponent(pathname); + const segments = decoded.split("/").filter((segment) => segment && segment !== "." && segment !== ".."); + const normalized = `/${segments.join("/")}`; + return decoded.endsWith("/") && normalized !== "/" ? `${normalized}/` : normalized; + } catch { + return "/"; + } +} + +function cacheControlForKey(key: string): string { + if (key.endsWith(".html") || key.endsWith("/index.html")) { + return "public, max-age=60, stale-while-revalidate=300"; + } + return "public, max-age=31536000, immutable"; +} + +function contentTypeForKey(key: string): string { + const ext = key.toLowerCase().split(".").pop(); + switch (ext) { + case "html": + return "text/html; charset=utf-8"; + case "css": + return "text/css; charset=utf-8"; + case "js": + return "application/javascript; charset=utf-8"; + case "json": + return "application/json; charset=utf-8"; + case "svg": + return "image/svg+xml"; + case "png": + return "image/png"; + case "jpg": + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "ico": + return "image/x-icon"; + case "woff": + return "font/woff"; + case "woff2": + return "font/woff2"; + case "txt": + return "text/plain; charset=utf-8"; + case "xml": + return "application/xml; charset=utf-8"; + default: + return "application/octet-stream"; + } +} diff --git a/apps/hosting-worker/tsconfig.json b/apps/hosting-worker/tsconfig.json new file mode 100644 index 0000000..80a214a --- /dev/null +++ b/apps/hosting-worker/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["@cloudflare/workers-types"], + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/apps/hosting-worker/wrangler.toml b/apps/hosting-worker/wrangler.toml new file mode 100644 index 0000000..31612fd --- /dev/null +++ b/apps/hosting-worker/wrangler.toml @@ -0,0 +1,18 @@ +name = "dxd-backup-hosting" +main = "src/index.ts" +compatibility_date = "2024-09-23" +compatibility_flags = ["nodejs_compat"] + +[[r2_buckets]] +binding = "STORAGE_BUCKET" +bucket_name = "dxd-site-scraper" + +[[hyperdrive]] +binding = "HYPERDRIVE" +id = "REPLACE_WITH_HYPERDRIVE_CONFIG_ID" + +[vars] +NODE_ENV = "production" + +# Configure a Cloudflare for SaaS fallback origin/custom hostname target in Cloudflare, +# then set HOSTING_CNAME_TARGET to that hostname in the API/Railway service. diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 5436c6b..faffc60 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -59,6 +59,10 @@ export interface Site { nextScheduledAt: string | null; storageType: string | null; storagePath: string | null; + hostingAutoPublish: boolean | null; + hostingBillingEmail: string | null; + hostingPaymentLinkUrl: string | null; + hostingBillingStatus: string | null; createdAt: string; updatedAt: string; lastCrawl?: Crawl | null; @@ -95,6 +99,40 @@ export interface CrawlLog { createdAt: string; } +export interface SitePublication { + id: string; + siteId: string; + crawlId: string; + status: string; + r2Prefix: string; + fileCount: number | null; + totalBytes: number | null; + errorMessage: string | null; + publishedAt: string | null; + createdAt: string; + updatedAt: string; + crawl?: Crawl; +} + +export interface SiteDomain { + id: string; + siteId: string; + hostname: string; + status: string; + cnameTarget: string; + activePublicationId: string | null; + redirectEnabled: boolean | null; + redirectTargetOrigin: string | null; + cloudflareHostnameId: string | null; + ownershipVerificationName: string | null; + ownershipVerificationValue: string | null; + sslStatus: string | null; + errorMessage: string | null; + createdAt: string; + updatedAt: string; + activePublication?: SitePublication | null; +} + export interface CreateSiteInput { name: string; url: string; @@ -107,6 +145,10 @@ export interface CreateSiteInput { redirectsCsv?: string | null; scheduleEnabled?: boolean; scheduleCron?: string | null; + hostingAutoPublish?: boolean; + hostingBillingEmail?: string | null; + hostingPaymentLinkUrl?: string | null; + hostingBillingStatus?: "not_sent" | "sent" | "paid" | "past_due" | "cancelled"; } export interface UpdateSiteInput extends Partial {} @@ -158,6 +200,128 @@ export const sitesApi = { }, }; +export const hostingApi = { + get: async (siteId: string): Promise<{ + cnameTarget: string; + settings: { + hostingAutoPublish: boolean; + hostingBillingEmail: string | null; + hostingPaymentLinkUrl: string | null; + hostingBillingStatus: string; + }; + publications: SitePublication[]; + domains: SiteDomain[]; + }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/hosting`); + return parseApiResponse<{ + cnameTarget: string; + settings: { + hostingAutoPublish: boolean; + hostingBillingEmail: string | null; + hostingPaymentLinkUrl: string | null; + hostingBillingStatus: string; + }; + publications: SitePublication[]; + domains: SiteDomain[]; + }>(res, "Failed to fetch hosting settings"); + }, + + updateSettings: async ( + siteId: string, + data: { + hostingAutoPublish?: boolean; + hostingBillingEmail?: string | null; + hostingPaymentLinkUrl?: string | null; + hostingBillingStatus?: "not_sent" | "sent" | "paid" | "past_due" | "cancelled"; + } + ): Promise<{ + settings: { + hostingAutoPublish: boolean; + hostingBillingEmail: string | null; + hostingPaymentLinkUrl: string | null; + hostingBillingStatus: string; + }; + }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/hosting`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(data), + }); + return parseApiResponse<{ + settings: { + hostingAutoPublish: boolean; + hostingBillingEmail: string | null; + hostingPaymentLinkUrl: string | null; + hostingBillingStatus: string; + }; + }>(res, "Failed to update hosting settings"); + }, + + publish: async ( + siteId: string, + data: { crawlId?: string; activate?: boolean } + ): Promise<{ publication: SitePublication }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/publications`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(data), + }); + return parseApiResponse<{ publication: SitePublication }>(res, "Failed to publish backup"); + }, + + addDomain: async (siteId: string, hostname: string): Promise<{ domain: SiteDomain }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/domains`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hostname }), + }); + return parseApiResponse<{ domain: SiteDomain }>(res, "Failed to add domain"); + }, + + syncDomain: async (siteId: string, domainId: string): Promise<{ domain: SiteDomain }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/domains/${domainId}/sync`, { method: "POST" }); + return parseApiResponse<{ domain: SiteDomain }>(res, "Failed to sync domain"); + }, + + updateDomain: async ( + siteId: string, + domainId: string, + data: { redirectEnabled?: boolean; redirectTargetOrigin?: string | null } + ): Promise<{ domain: SiteDomain }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/domains/${domainId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(data), + }); + return parseApiResponse<{ domain: SiteDomain }>(res, "Failed to update domain"); + }, + + activateDomain: async ( + siteId: string, + domainId: string, + publicationId: string + ): Promise<{ domain: SiteDomain }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/domains/${domainId}/activate`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ publicationId }), + }); + return parseApiResponse<{ domain: SiteDomain }>(res, "Failed to activate publication"); + }, + + activatePublication: async (siteId: string, publicationId: string): Promise<{ publication: SitePublication }> => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/publications/${publicationId}/activate`, { + method: "POST", + }); + return parseApiResponse<{ publication: SitePublication }>(res, "Failed to activate publication"); + }, + + deleteDomain: async (siteId: string, domainId: string): Promise => { + const res = await fetchWithAuth(`${API_BASE}/sites/${siteId}/domains/${domainId}`, { method: "DELETE" }); + await parseApiResponse<{ success: boolean }>(res, "Failed to delete domain"); + }, +}; + // Crawls API export const crawlsApi = { list: async (params?: { diff --git a/apps/web/src/routes/sites/$siteId.tsx b/apps/web/src/routes/sites/$siteId.tsx index 4b02831..2de333f 100644 --- a/apps/web/src/routes/sites/$siteId.tsx +++ b/apps/web/src/routes/sites/$siteId.tsx @@ -1,10 +1,10 @@ import { createFileRoute, Link, useNavigate } from "@tanstack/react-router"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; import { motion } from "framer-motion"; -import { sitesApi, crawlsApi } from "@/lib/api"; +import { sitesApi, crawlsApi, hostingApi } from "@/lib/api"; import { formatToMountainTime } from "@/lib/date"; import { parseCron, toCronExpression, type ScheduleFrequency, WEEKDAYS } from "@/lib/schedule"; -import { ArrowLeft, Play, ExternalLink, Trash2, Download, Save } from "lucide-react"; +import { ArrowLeft, Play, ExternalLink, Trash2, Download, Save, Globe2, UploadCloud, RefreshCw, CreditCard } from "lucide-react"; import { useEffect, useState } from "react"; export const Route = createFileRoute("/sites/$siteId")({ @@ -22,6 +22,12 @@ function SiteDetailPage() { refetchInterval: 10000, }); + const { data: hostingData } = useQuery({ + queryKey: ["hosting", siteId], + queryFn: () => hostingApi.get(siteId), + refetchInterval: 10000, + }); + const [scheduleEnabled, setScheduleEnabled] = useState(false); const [scheduleFrequency, setScheduleFrequency] = useState("daily"); const [scheduleTime, setScheduleTime] = useState("05:00"); @@ -34,6 +40,11 @@ function SiteDetailPage() { const [maxArchivesToKeepInput, setMaxArchivesToKeepInput] = useState(""); const [removeWebflowBadge, setRemoveWebflowBadge] = useState(true); const [downloadBlacklistText, setDownloadBlacklistText] = useState(""); + const [hostname, setHostname] = useState(""); + const [hostingAutoPublish, setHostingAutoPublish] = useState(true); + const [hostingBillingEmail, setHostingBillingEmail] = useState(""); + const [hostingPaymentLinkUrl, setHostingPaymentLinkUrl] = useState(""); + const [hostingBillingStatus, setHostingBillingStatus] = useState<"not_sent" | "sent" | "paid" | "past_due" | "cancelled">("not_sent"); const site = data?.site; @@ -60,6 +71,14 @@ function SiteDetailPage() { setScheduleMonthlyDay(parsed.monthlyDay); }, [site]); + useEffect(() => { + if (!hostingData?.settings) return; + setHostingAutoPublish(hostingData.settings.hostingAutoPublish); + setHostingBillingEmail(hostingData.settings.hostingBillingEmail ?? ""); + setHostingPaymentLinkUrl(hostingData.settings.hostingPaymentLinkUrl ?? ""); + setHostingBillingStatus((hostingData.settings.hostingBillingStatus as typeof hostingBillingStatus) || "not_sent"); + }, [hostingData?.settings]); + const configurationMutation = useMutation({ mutationFn: (payload: { concurrency: number; @@ -108,6 +127,76 @@ function SiteDetailPage() { }, }); + const publishMutation = useMutation({ + mutationFn: (crawlId?: string) => hostingApi.publish(siteId, { crawlId, activate: true }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + }, + }); + + const addDomainMutation = useMutation({ + mutationFn: (nextHostname: string) => hostingApi.addDomain(siteId, nextHostname), + onSuccess: () => { + setHostname(""); + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + }, + }); + + const syncDomainMutation = useMutation({ + mutationFn: (domainId: string) => hostingApi.syncDomain(siteId, domainId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + }, + }); + + const deleteDomainMutation = useMutation({ + mutationFn: (domainId: string) => hostingApi.deleteDomain(siteId, domainId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + }, + }); + + const updateDomainMutation = useMutation({ + mutationFn: (payload: { domainId: string; redirectEnabled?: boolean; redirectTargetOrigin?: string | null }) => + hostingApi.updateDomain(siteId, payload.domainId, { + redirectEnabled: payload.redirectEnabled, + redirectTargetOrigin: payload.redirectTargetOrigin, + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + }, + }); + + const hostingSettingsMutation = useMutation({ + mutationFn: (payload: { + hostingAutoPublish?: boolean; + hostingBillingEmail?: string | null; + hostingPaymentLinkUrl?: string | null; + hostingBillingStatus?: "not_sent" | "sent" | "paid" | "past_due" | "cancelled"; + }) => hostingApi.updateSettings(siteId, payload), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + queryClient.invalidateQueries({ queryKey: ["sites", siteId] }); + }, + }); + + const activatePublicationMutation = useMutation({ + mutationFn: (publicationId: string) => hostingApi.activatePublication(siteId, publicationId), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); + queryClient.invalidateQueries({ queryKey: ["sites", siteId] }); + }, + }); + + const paymentMailto = (() => { + if (!hostingBillingEmail.trim() || !hostingPaymentLinkUrl.trim()) return null; + const subject = encodeURIComponent(`Monthly hosted backup for ${site?.name ?? "your site"}`); + const body = encodeURIComponent( + `Hi,\n\nYour hosted backup is ready. You can start the monthly hosted backup subscription here:\n\n${hostingPaymentLinkUrl.trim()}\n\nOnce paid, your backup will remain hosted at the configured client subdomain.\n` + ); + return `mailto:${hostingBillingEmail.trim()}?subject=${subject}&body=${body}`; + })(); + if (isLoading) { return (
@@ -494,6 +583,254 @@ function SiteDetailPage() {
+ +
+
+ +

Client Domain Hosting

+
+ +
+
+ + setHostname(e.target.value)} + className="input-dark font-mono text-sm" + placeholder="backup.client.com" + /> +

+ Client will add a CNAME from this hostname to your hosting target. +

+
+ + + + {hostingData?.cnameTarget && ( +
+

Required DNS

+
+
Type: CNAME
+
Target: {hostingData.cnameTarget}
+
+
+ )} + + {hostingData?.publications.length ? ( +
+
+

Published Versions

+ +
+
+ {hostingData.publications.slice(0, 8).map((publication) => { + const isActive = hostingData.domains.some((domain) => domain.activePublicationId === publication.id); + return ( +
+
+ {publication.crawlId.slice(0, 8)} + {isActive && live} +
+
+ + {publication.status} + + {publication.status === "published" && !isActive && ( + + )} +
+
+ ); + })} +
+ {!hostingAutoPublish && ( +

+ Auto latest is off. Hosted domains stay pinned until you re-enable it. +

+ )} +
+ ) : null} + + {hostingData?.domains.map((domain) => ( +
+
+
+ + {domain.hostname} + +

+ {domain.status} {domain.sslStatus ? `/ ssl: ${domain.sslStatus}` : ""} +

+

+ mode: {domain.redirectEnabled ? "301 redirect to main site" : "serve backup"} +

+
+ +
+ + {domain.ownershipVerificationName && domain.ownershipVerificationValue && ( +
+
TXT name: {domain.ownershipVerificationName}
+
TXT value: {domain.ownershipVerificationValue}
+
+ )} + +
+ +
+ { + const nextValue = e.target.value.trim(); + if (nextValue && nextValue !== (domain.redirectTargetOrigin || site.url)) { + updateDomainMutation.mutate({ domainId: domain.id, redirectTargetOrigin: nextValue }); + } + }} + /> +
+

+ Redirect preserves the path and query, for example /about?x=1 redirects to the same slug on this origin. +

+
+ + +
+ ))} + +
+
+ +

Monthly Billing Link

+
+ setHostingBillingEmail(e.target.value)} + className="input-dark font-mono text-sm" + placeholder="owner@client.com" + /> + setHostingPaymentLinkUrl(e.target.value)} + className="input-dark font-mono text-sm" + placeholder="https://buy.stripe.com/..." + /> + +
+ + {paymentMailto && ( + { + hostingSettingsMutation.mutate({ hostingBillingStatus: "sent" }); + }} + > + Send Link + + )} +
+

+ Use a Stripe monthly Payment Link for now. This can later become Stripe Checkout plus webhooks when payment state needs to be automatic. +

+
+
+
{/* Crawl History */} @@ -550,6 +887,19 @@ function SiteDetailPage() { Download + )} diff --git a/bun.lock b/bun.lock index 79039be..4d5f351 100644 --- a/bun.lock +++ b/bun.lock @@ -36,6 +36,20 @@ "wrangler": "^3.99.0", }, }, + "apps/hosting-worker": { + "name": "@dxd/hosting-worker", + "version": "1.0.0", + "dependencies": { + "@dxd/db": "workspace:*", + "drizzle-orm": "^0.44.2", + "postgres": "^3.4.7", + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20241230.0", + "typescript": "^5.9.3", + "wrangler": "^3.99.0", + }, + }, "apps/web": { "name": "@dxd/web", "version": "1.0.0", @@ -118,12 +132,14 @@ "ioredis": "^5.6.1", "node-cron": "^3.0.3", "postgres": "^3.4.7", + "unzipper": "^0.12.3", }, "devDependencies": { "@types/archiver": "^6.0.3", "@types/bun": "^1.2.0", "@types/node": "^22.0.0", "@types/node-cron": "^3.0.11", + "@types/unzipper": "^0.10.11", "typescript": "^5.9.3", }, }, @@ -281,6 +297,8 @@ "@dxd/db": ["@dxd/db@workspace:packages/db"], + "@dxd/hosting-worker": ["@dxd/hosting-worker@workspace:apps/hosting-worker"], + "@dxd/scraper": ["@dxd/scraper@workspace:packages/scraper"], "@dxd/storage": ["@dxd/storage@workspace:packages/storage"], @@ -663,6 +681,8 @@ "@types/readdir-glob": ["@types/readdir-glob@1.1.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg=="], + "@types/unzipper": ["@types/unzipper@0.10.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-D25im2zjyMCcgL9ag6N46+wbtJBnXIr7SI4zHf9eJD2Dw2tEB5e+p5MYkrxKIVRscs5QV0EhtU9rgXSPx90oJg=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="], "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], @@ -705,6 +725,8 @@ "blake3-wasm": ["blake3-wasm@2.1.5", "", {}, "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g=="], + "bluebird": ["bluebird@3.7.2", "", {}, "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], "bowser": ["bowser@2.13.1", "", {}, "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw=="], @@ -795,6 +817,8 @@ "drizzle-orm": ["drizzle-orm@0.44.7", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-quIpnYznjU9lHshEOAYLoZ9s3jweleHlZIAWR/jX9gAWNg/JhQ1wj0KGRf7/Zm+obRrYd9GjPVJg790QY9N5AQ=="], + "duplexer2": ["duplexer2@0.1.4", "", { "dependencies": { "readable-stream": "^2.0.2" } }, "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA=="], + "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], "electron-to-chromium": ["electron-to-chromium@1.5.279", "", {}, "sha512-0bblUU5UNdOt5G7XqGiJtpZMONma6WAfq9vsFmtn9x1+joAObr6x1chfqyxFSDCAFwFhCQDrqeAr6MYdpwJ9Hg=="], @@ -973,6 +997,8 @@ "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], + "node-int64": ["node-int64@0.4.0", "", {}, "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw=="], + "node-releases": ["node-releases@2.0.27", "", {}, "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA=="], "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], @@ -1155,6 +1181,8 @@ "unplugin": ["unplugin@2.3.11", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "acorn": "^8.15.0", "picomatch": "^4.0.3", "webpack-virtual-modules": "^0.6.2" } }, "sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww=="], + "unzipper": ["unzipper@0.12.3", "", { "dependencies": { "bluebird": "~3.7.2", "duplexer2": "~0.1.4", "fs-extra": "^11.2.0", "graceful-fs": "^4.2.2", "node-int64": "^0.4.0" } }, "sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA=="], + "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], "use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="], @@ -1239,6 +1267,8 @@ "bullmq/cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + "duplexer2/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="], + "get-source/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], "glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], @@ -1357,6 +1387,10 @@ "@hono/auth-js/@auth/core/preact-render-to-string": ["preact-render-to-string@6.5.11", "", { "peerDependencies": { "preact": ">=10" } }, "sha512-ubnauqoGczeGISiOh6RjX0/cdaF8v/oDXIjO85XALCQjwQP+SB4RDXXtvZ6yTYSjG+PC1QRP2AhPgCEsM2EvUw=="], + "duplexer2/readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], + + "duplexer2/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "lazystream/readable-stream/safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 17c3b47..51e7100 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -9,6 +9,7 @@ import { timestamp, jsonb, primaryKey, + uniqueIndex, } from "drizzle-orm/pg-core"; import { relations } from "drizzle-orm"; import type { AdapterAccountType } from "@auth/core/adapters"; @@ -93,6 +94,10 @@ export const sites = pgTable("sites", { nextScheduledAt: timestamp("next_scheduled_at", { withTimezone: true }), storageType: varchar("storage_type", { length: 50 }).default("local"), storagePath: varchar("storage_path", { length: 500 }), + hostingAutoPublish: boolean("hosting_auto_publish").default(true), + hostingBillingEmail: varchar("hosting_billing_email", { length: 255 }), + hostingPaymentLinkUrl: varchar("hosting_payment_link_url", { length: 1000 }), + hostingBillingStatus: varchar("hosting_billing_status", { length: 50 }).default("not_sent"), createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), }); @@ -132,8 +137,46 @@ export const settings = pgTable("settings", { updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), }); +export const sitePublications = pgTable("site_publications", { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + crawlId: uuid("crawl_id").notNull().references(() => crawls.id, { onDelete: "cascade" }), + status: varchar("status", { length: 50 }).notNull().default("pending"), + r2Prefix: varchar("r2_prefix", { length: 500 }).notNull(), + fileCount: integer("file_count"), + totalBytes: bigint("total_bytes", { mode: "number" }), + errorMessage: text("error_message"), + publishedAt: timestamp("published_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), +}); + +export const siteDomains = pgTable( + "site_domains", + { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + hostname: varchar("hostname", { length: 255 }).notNull(), + status: varchar("status", { length: 50 }).notNull().default("pending_dns"), + cnameTarget: varchar("cname_target", { length: 255 }).notNull(), + activePublicationId: uuid("active_publication_id").references(() => sitePublications.id, { onDelete: "set null" }), + redirectEnabled: boolean("redirect_enabled").default(false), + redirectTargetOrigin: varchar("redirect_target_origin", { length: 500 }), + cloudflareHostnameId: varchar("cloudflare_hostname_id", { length: 255 }), + ownershipVerificationName: varchar("ownership_verification_name", { length: 255 }), + ownershipVerificationValue: text("ownership_verification_value"), + sslStatus: varchar("ssl_status", { length: 100 }), + errorMessage: text("error_message"), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), + }, + (domain) => [uniqueIndex("site_domains_hostname_idx").on(domain.hostname)] +); + export const sitesRelations = relations(sites, ({ many }) => ({ crawls: many(crawls), + publications: many(sitePublications), + domains: many(siteDomains), })); export const crawlsRelations = relations(crawls, ({ one, many }) => ({ @@ -142,6 +185,30 @@ export const crawlsRelations = relations(crawls, ({ one, many }) => ({ references: [sites.id], }), logs: many(crawlLogs), + publications: many(sitePublications), +})); + +export const sitePublicationsRelations = relations(sitePublications, ({ one, many }) => ({ + site: one(sites, { + fields: [sitePublications.siteId], + references: [sites.id], + }), + crawl: one(crawls, { + fields: [sitePublications.crawlId], + references: [crawls.id], + }), + domains: many(siteDomains), +})); + +export const siteDomainsRelations = relations(siteDomains, ({ one }) => ({ + site: one(sites, { + fields: [siteDomains.siteId], + references: [sites.id], + }), + activePublication: one(sitePublications, { + fields: [siteDomains.activePublicationId], + references: [sitePublications.id], + }), })); export const crawlLogsRelations = relations(crawlLogs, ({ one }) => ({ @@ -180,3 +247,7 @@ export type CrawlLog = typeof crawlLogs.$inferSelect; export type NewCrawlLog = typeof crawlLogs.$inferInsert; export type Setting = typeof settings.$inferSelect; export type AllowedEmail = typeof allowedEmails.$inferSelect; +export type SitePublication = typeof sitePublications.$inferSelect; +export type NewSitePublication = typeof sitePublications.$inferInsert; +export type SiteDomain = typeof siteDomains.$inferSelect; +export type NewSiteDomain = typeof siteDomains.$inferInsert; diff --git a/services/worker/package.json b/services/worker/package.json index 7e8f171..4705fba 100644 --- a/services/worker/package.json +++ b/services/worker/package.json @@ -17,13 +17,15 @@ "hono": "^4.7.7", "ioredis": "^5.6.1", "node-cron": "^3.0.3", - "postgres": "^3.4.7" + "postgres": "^3.4.7", + "unzipper": "^0.12.3" }, "devDependencies": { "@types/archiver": "^6.0.3", "@types/bun": "^1.2.0", "@types/node": "^22.0.0", "@types/node-cron": "^3.0.11", + "@types/unzipper": "^0.10.11", "typescript": "^5.9.3" } } diff --git a/services/worker/src/db.ts b/services/worker/src/db.ts index 6ac8249..ae400f2 100644 --- a/services/worker/src/db.ts +++ b/services/worker/src/db.ts @@ -10,6 +10,7 @@ import { boolean, timestamp, jsonb, + uniqueIndex, } from "drizzle-orm/pg-core"; import { relations } from "drizzle-orm"; @@ -30,6 +31,10 @@ export const sites = pgTable("sites", { nextScheduledAt: timestamp("next_scheduled_at", { withTimezone: true }), storageType: varchar("storage_type", { length: 50 }).default("local"), storagePath: varchar("storage_path", { length: 500 }), + hostingAutoPublish: boolean("hosting_auto_publish").default(true), + hostingBillingEmail: varchar("hosting_billing_email", { length: 255 }), + hostingPaymentLinkUrl: varchar("hosting_payment_link_url", { length: 1000 }), + hostingBillingStatus: varchar("hosting_billing_status", { length: 50 }).default("not_sent"), createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), }); @@ -70,9 +75,47 @@ export const settings = pgTable("settings", { updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), }); +export const sitePublications = pgTable("site_publications", { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + crawlId: uuid("crawl_id").notNull().references(() => crawls.id, { onDelete: "cascade" }), + status: varchar("status", { length: 50 }).notNull().default("pending"), + r2Prefix: varchar("r2_prefix", { length: 500 }).notNull(), + fileCount: integer("file_count"), + totalBytes: bigint("total_bytes", { mode: "number" }), + errorMessage: text("error_message"), + publishedAt: timestamp("published_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), +}); + +export const siteDomains = pgTable( + "site_domains", + { + id: uuid("id").primaryKey().defaultRandom(), + siteId: uuid("site_id").notNull().references(() => sites.id, { onDelete: "cascade" }), + hostname: varchar("hostname", { length: 255 }).notNull(), + status: varchar("status", { length: 50 }).notNull().default("pending_dns"), + cnameTarget: varchar("cname_target", { length: 255 }).notNull(), + activePublicationId: uuid("active_publication_id").references(() => sitePublications.id, { onDelete: "set null" }), + redirectEnabled: boolean("redirect_enabled").default(false), + redirectTargetOrigin: varchar("redirect_target_origin", { length: 500 }), + cloudflareHostnameId: varchar("cloudflare_hostname_id", { length: 255 }), + ownershipVerificationName: varchar("ownership_verification_name", { length: 255 }), + ownershipVerificationValue: text("ownership_verification_value"), + sslStatus: varchar("ssl_status", { length: 100 }), + errorMessage: text("error_message"), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), + }, + (domain) => [uniqueIndex("site_domains_hostname_idx").on(domain.hostname)] +); + // Relations export const sitesRelations = relations(sites, ({ many }) => ({ crawls: many(crawls), + publications: many(sitePublications), + domains: many(siteDomains), })); export const crawlsRelations = relations(crawls, ({ one, many }) => ({ @@ -81,6 +124,21 @@ export const crawlsRelations = relations(crawls, ({ one, many }) => ({ references: [sites.id], }), logs: many(crawlLogs), + publications: many(sitePublications), +})); + +export const sitePublicationsRelations = relations(sitePublications, ({ one, many }) => ({ + site: one(sites, { fields: [sitePublications.siteId], references: [sites.id] }), + crawl: one(crawls, { fields: [sitePublications.crawlId], references: [crawls.id] }), + domains: many(siteDomains), +})); + +export const siteDomainsRelations = relations(siteDomains, ({ one }) => ({ + site: one(sites, { fields: [siteDomains.siteId], references: [sites.id] }), + activePublication: one(sitePublications, { + fields: [siteDomains.activePublicationId], + references: [sitePublications.id], + }), })); // Database client @@ -91,5 +149,16 @@ if (!connectionString) { const client = postgres(connectionString); export const db = drizzle(client, { - schema: { sites, crawls, crawlLogs, settings, sitesRelations, crawlsRelations }, + schema: { + sites, + crawls, + crawlLogs, + settings, + sitePublications, + siteDomains, + sitesRelations, + crawlsRelations, + sitePublicationsRelations, + siteDomainsRelations, + }, }); diff --git a/services/worker/src/http.ts b/services/worker/src/http.ts index 03db7bc..5cd3b89 100644 --- a/services/worker/src/http.ts +++ b/services/worker/src/http.ts @@ -14,6 +14,14 @@ const crawlQueue = new Queue("crawl-jobs", { attempts: 1, }, }); +const publicationQueue = new Queue("publication-jobs", { + connection: redis, + defaultJobOptions: { + removeOnComplete: 100, + removeOnFail: 100, + attempts: 1, + }, +}); const app = new Hono(); @@ -45,6 +53,34 @@ app.post("/enqueue", async (c) => { return c.json({ ok: true, jobId: body.crawlId }); }); +app.post("/publish", async (c) => { + const body = await c.req.json<{ + siteId: string; + crawlId: string; + publicationId: string; + activate?: boolean; + autoPublish?: boolean; + }>(); + + if (!body.siteId || !body.crawlId || !body.publicationId) { + return c.json({ error: "siteId, crawlId, and publicationId are required" }, 400); + } + + await publicationQueue.add( + "publish", + { + siteId: body.siteId, + crawlId: body.crawlId, + publicationId: body.publicationId, + activate: body.activate ?? true, + autoPublish: body.autoPublish ?? false, + }, + { jobId: body.publicationId } + ); + + return c.json({ ok: true, jobId: body.publicationId }); +}); + // Cancel a crawl job (remove from queue if pending) app.post("/cancel", async (c) => { const body = await c.req.json<{ crawlId: string }>(); diff --git a/services/worker/src/processor.ts b/services/worker/src/processor.ts index fdaf4f7..d0d27e1 100644 --- a/services/worker/src/processor.ts +++ b/services/worker/src/processor.ts @@ -11,12 +11,15 @@ import { } from "@dxd/scraper"; import { getStorage } from "@dxd/storage"; import type { MultipartUploadProgress } from "@dxd/storage/adapter"; -import { db, sites, crawls, crawlLogs, settings } from "./db.js"; +import { db, sites, crawls, crawlLogs, settings, sitePublications, siteDomains } from "./db.js"; import fs from "node:fs/promises"; import nodeFs from "node:fs"; import path from "node:path"; import archiver from "archiver"; +import unzipper from "unzipper"; import { once } from "node:events"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; import { captureMemorySnapshot, formatMemorySnapshot, type MemorySnapshot } from "./memory.js"; import { getWorkerRuntimeConfig } from "./worker-config.js"; @@ -56,6 +59,16 @@ const archiveQueue = new Queue("archive-jobs", { }, }); +const publicationQueue = new Queue("publication-jobs", { + connection: workerConnection, + defaultJobOptions: { + attempts: 1, + removeOnComplete: { count: 10 }, + removeOnFail: { count: 50 }, + backoff: { type: "fixed", delay: 0 }, + }, +}); + interface CrawlJobData { siteId: string; crawlId: string; @@ -69,6 +82,14 @@ interface ArchiveJobData { crawlResult?: Pick; } +interface PublicationJobData { + siteId: string; + crawlId: string; + publicationId: string; + activate: boolean; + autoPublish?: boolean; +} + function createExclusiveRunner() { let tail = Promise.resolve(); @@ -333,6 +354,61 @@ function isBadFileDescriptorError(error: unknown): boolean { ); } +function getArchiveStoragePath(outputPath: string): string { + return outputPath.endsWith(".zip") ? outputPath : `${outputPath}.zip`; +} + +function toSafePublishedKey(prefix: string, zipEntryPath: string): string | null { + const normalized = path.posix.normalize(zipEntryPath.replace(/\\/g, "/")); + if (!normalized || normalized === "." || normalized.startsWith("../") || path.posix.isAbsolute(normalized)) { + return null; + } + return `${prefix.replace(/\/+$/, "")}/${normalized}`; +} + +async function downloadStorageObjectToFile(storagePath: string, localPath: string): Promise { + await pipeline( + Readable.fromWeb(storage.readStream(storagePath) as any), + nodeFs.createWriteStream(localPath) + ); +} + +async function publishZipArchiveToR2(publicationId: string, archivePath: string, prefix: string) { + const tempDir = await storage.createTempDir(`publish-${publicationId}`); + const localZipPath = path.join(tempDir, "archive.zip"); + let fileCount = 0; + let totalBytes = 0; + + try { + await downloadStorageObjectToFile(archivePath, localZipPath); + await storage.deleteDir(prefix).catch(() => undefined); + + const directory = await unzipper.Open.file(localZipPath); + for (const entry of directory.files) { + if (entry.type !== "File") continue; + + const key = toSafePublishedKey(prefix, entry.path); + if (!key) continue; + + await storage.writeStream( + key, + Readable.toWeb(entry.stream()) as unknown as ReadableStream, + { totalSize: entry.uncompressedSize } + ); + fileCount += 1; + totalBytes += entry.uncompressedSize; + } + } finally { + await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined); + } + + if (fileCount === 0) { + throw new Error("Archive did not contain any publishable files"); + } + + return { fileCount, totalBytes }; +} + async function uploadArchiveFromTempDir( crawlId: string, outputDir: string, @@ -606,6 +682,40 @@ async function enqueueArchiveJob(data: ArchiveJobData): Promise { }); } +async function enqueuePublicationJob(data: PublicationJobData): Promise { + await publicationQueue.add("publish", data, { + jobId: data.publicationId, + attempts: 1, + }); +} + +async function enqueueAutoPublication(siteId: string, crawlId: string): Promise { + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site?.hostingAutoPublish) { + return; + } + + const publicationId = crypto.randomUUID(); + const [publication] = await db + .insert(sitePublications) + .values({ + id: publicationId, + siteId, + crawlId, + status: "pending", + r2Prefix: `published/${siteId}/${publicationId}`, + }) + .returning(); + + await enqueuePublicationJob({ + siteId, + crawlId, + publicationId: publication.id, + activate: true, + autoPublish: true, + }); +} + async function resolveArchiveResult(job: ArchiveJobData) { const crawl = await db.query.crawls.findFirst({ where: eq(crawls.id, job.crawlId) }); if (!crawl) { @@ -620,6 +730,74 @@ async function resolveArchiveResult(job: ArchiveJobData) { return { crawl, site }; } +async function processPublicationJob(job: Job): Promise { + const { siteId, crawlId, publicationId, activate, autoPublish } = job.data; + + await db + .update(sitePublications) + .set({ status: "publishing", errorMessage: null, updatedAt: new Date() }) + .where(eq(sitePublications.id, publicationId)); + + try { + const publication = await db.query.sitePublications.findFirst({ + where: and( + eq(sitePublications.id, publicationId), + eq(sitePublications.siteId, siteId), + eq(sitePublications.crawlId, crawlId) + ), + }); + if (!publication) { + throw new UnrecoverableError(`Publication ${publicationId} not found`); + } + + const crawl = await db.query.crawls.findFirst({ + where: and(eq(crawls.id, crawlId), eq(crawls.siteId, siteId)), + }); + if (!crawl || crawl.status !== "completed" || !crawl.outputPath) { + throw new UnrecoverableError(`Crawl ${crawlId} is not publishable`); + } + + const archivePath = getArchiveStoragePath(crawl.outputPath); + const archiveExists = await storage.exists(archivePath); + if (!archiveExists) { + throw new UnrecoverableError(`Archive not found at ${archivePath}`); + } + + const result = await publishZipArchiveToR2(publicationId, archivePath, publication.r2Prefix); + await db + .update(sitePublications) + .set({ + status: "published", + fileCount: result.fileCount, + totalBytes: result.totalBytes, + publishedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(sitePublications.id, publicationId)); + + if (activate) { + if (autoPublish) { + const site = await db.query.sites.findFirst({ where: eq(sites.id, siteId) }); + if (!site?.hostingAutoPublish) { + return; + } + } + + await db + .update(siteDomains) + .set({ activePublicationId: publicationId, updatedAt: new Date() }) + .where(eq(siteDomains.siteId, siteId)); + } + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown publication error"; + await db + .update(sitePublications) + .set({ status: "failed", errorMessage: message, updatedAt: new Date() }) + .where(eq(sitePublications.id, publicationId)); + throw error; + } +} + async function processArchiveJob(job: Job) { return runExclusiveWorkerJob(async () => { const { crawlId, siteId, finalStatus, errorMessage, crawlResult } = job.data; @@ -739,6 +917,16 @@ async function processArchiveJob(job: Job) { ); await pruneOldArchives(siteId, site.maxArchivesToKeep ?? Number.POSITIVE_INFINITY, crawlId); + if (finalStatus === "completed") { + try { + await enqueueAutoPublication(siteId, crawlId); + await publishLogAndPersist(crawlId, "info", "Queued hosted backup publish for latest successful crawl"); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown hosted backup publish error"; + await publishLogAndPersist(crawlId, "warn", `Failed to queue hosted backup publish: ${message}`); + } + } + console.log(`[Worker] Archive completed: ${crawlId} (${finalStatus})`); }); } @@ -1159,6 +1347,15 @@ export function startWorker() { skipLockRenewal: config.skipLockRenewal, }); + const publicationWorker = new Worker("publication-jobs", processPublicationJob, { + connection: workerConnection, + concurrency: 1, + lockDuration: config.lockDuration, + stalledInterval: config.stalledInterval, + maxStalledCount: 0, + skipLockRenewal: config.skipLockRenewal, + }); + void reconcileOrphanedCrawls(config.orphanGraceMs).catch((error) => { console.error("[Worker] Failed to reconcile orphaned crawls:", error); }); @@ -1172,14 +1369,16 @@ export function startWorker() { attachWorkerLogging(crawlWorker, "crawl", reconcileTimer); attachWorkerLogging(archiveWorker, "archive", reconcileTimer); + attachWorkerLogging(publicationWorker, "publication", reconcileTimer); console.log( - `[Worker] Started crawl worker (concurrency=${config.crawlConcurrency}) and archive worker (concurrency=${config.archiveConcurrency}, lockDuration=${config.lockDuration}ms)` + `[Worker] Started crawl worker (concurrency=${config.crawlConcurrency}), archive worker (concurrency=${config.archiveConcurrency}), and publication worker (concurrency=1, lockDuration=${config.lockDuration}ms)` ); return { async close() { clearInterval(reconcileTimer); + await publicationWorker.close(); await archiveWorker.close(); await crawlWorker.close(); }, From b10e6409f11647b65754367bd8f04448f0cdd6ef Mon Sep 17 00:00:00 2001 From: Austin Thesing Date: Mon, 27 Apr 2026 13:20:08 -0700 Subject: [PATCH 2/4] docs: add backup hosting setup guide --- docs/setup-backup-hosting.md | 352 +++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 docs/setup-backup-hosting.md diff --git a/docs/setup-backup-hosting.md b/docs/setup-backup-hosting.md new file mode 100644 index 0000000..a20be10 --- /dev/null +++ b/docs/setup-backup-hosting.md @@ -0,0 +1,352 @@ +# Backup Hosting Setup + +This guide covers the infrastructure required for client-owned CNAME backup hosting. + +The intended model is: + +```txt +Internal dashboard: https://zip.designxdevelop.com +Client visible backup URL: https://backup.client.com +Client DNS: backup.client.com CNAME backup-hosting.designxdevelop.com +Cloudflare Worker: serves R2 backup or 301 redirects to the client root domain +``` + +Clients should not use a `designxdevelop.com` URL directly. Your hostname is only the CNAME target. + +## Architecture + +```txt +Client browser + -> backup.client.com + -> CNAME backup-hosting.designxdevelop.com + -> Cloudflare Custom Hostname / Worker + -> Postgres hostname lookup + -> R2 published backup files +``` + +When the backup is not needed, the Worker can return a permanent redirect: + +```txt +https://backup.client.com/about?x=1 + 301 -> https://client.com/about?x=1 +``` + +When the backup is needed, disable redirect mode for that client domain and the Worker serves the static backup from R2. + +## Required Cloudflare Resources + +You need these Cloudflare pieces: + +- R2 bucket used by the scraper archives and hosted backup files. +- Hyperdrive config pointing to Railway Postgres. +- Dedicated Cloudflare Worker from `apps/hosting-worker`. +- Cloudflare for SaaS / Custom Hostnames configured for client-owned CNAME hostnames. +- Cloudflare API token that can manage custom hostnames in the `designxdevelop.com` zone. + +## Recommended Hostnames + +Use: + +```txt +zip.designxdevelop.com +``` + +for the internal dashboard. + +Use: + +```txt +backup-hosting.designxdevelop.com +``` + +as the CNAME target clients point to. + +Client DNS example: + +```txt +Type: CNAME +Name: backup +Target: backup-hosting.designxdevelop.com +``` + +This produces: + +```txt +backup.client.com -> backup-hosting.designxdevelop.com +``` + +## Cloudflare R2 + +Use the same R2 bucket that archives are already uploaded to. + +In `apps/hosting-worker/wrangler.toml`: + +```toml +[[r2_buckets]] +binding = "STORAGE_BUCKET" +bucket_name = "dxd-site-scraper" +``` + +If your bucket has a different name, replace `dxd-site-scraper` with the real bucket name. + +## Cloudflare Hyperdrive + +The hosting Worker needs Postgres access to map incoming hostnames to the active published backup. + +Create a Hyperdrive config pointing to Railway Postgres: + +```bash +cd apps/hosting-worker +bunx wrangler hyperdrive create dxd-postgres \ + --connection-string="postgres://USER:PASSWORD@HOST:PORT/DATABASE" +``` + +Then update `apps/hosting-worker/wrangler.toml`: + +```toml +[[hyperdrive]] +binding = "HYPERDRIVE" +id = "YOUR_HYPERDRIVE_CONFIG_ID" +``` + +The API Worker config, if used, also needs the same Hyperdrive ID in `apps/api/wrangler.toml`. + +## Deploy The Hosting Worker + +Deploy the dedicated hosting Worker: + +```bash +cd apps/hosting-worker +bun install +bunx wrangler deploy +``` + +The Worker name is configured as: + +```toml +name = "dxd-backup-hosting" +``` + +## Configure The Hosting Target Route + +In Cloudflare, route this hostname to the hosting Worker: + +```txt +backup-hosting.designxdevelop.com/* -> dxd-backup-hosting +``` + +You can do that through either: + +- Cloudflare Worker custom domains. +- A Workers route in the `designxdevelop.com` zone. + +## Configure Cloudflare For SaaS / Custom Hostnames + +In the `designxdevelop.com` Cloudflare zone: + +1. Enable/configure Cloudflare for SaaS / Custom Hostnames. +2. Set the fallback origin/target to: + +```txt +backup-hosting.designxdevelop.com +``` + +3. Ensure custom hostnames can be created for client domains like: + +```txt +backup.client.com +``` + +When a domain is added in the dashboard, the API creates a Cloudflare custom hostname. Cloudflare may return TXT verification records. The dashboard displays those records so you can send them to the client. + +## Cloudflare API Token + +Create a Cloudflare API token scoped to the `designxdevelop.com` zone. + +It needs permission to manage custom hostnames for that zone. + +Set these variables in Railway: + +```txt +CLOUDFLARE_ZONE_ID= +CLOUDFLARE_API_TOKEN= +``` + +## Railway Environment Variables + +Set these on the Railway app/API service that powers `https://zip.designxdevelop.com`: + +```txt +FRONTEND_URL=https://zip.designxdevelop.com +HOSTING_CNAME_TARGET=backup-hosting.designxdevelop.com +CLOUDFLARE_ZONE_ID= +CLOUDFLARE_API_TOKEN= +``` + +Keep the existing database, queue, auth, and storage values. + +Typical required values: + +```txt +DATABASE_URL= +REDIS_URL= +WORKER_SERVICE_URL= +WORKER_API_SECRET= +``` + +For R2 storage, use whichever env family your app already uses successfully. + +R2-style example: + +```txt +STORAGE_TYPE=r2 +R2_ENDPOINT=https://.r2.cloudflarestorage.com +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= +R2_BUCKET=dxd-site-scraper +R2_REGION=auto +R2_FORCE_PATH_STYLE=true +``` + +S3-compatible example: + +```txt +STORAGE_TYPE=s3 +S3_ENDPOINT=https://.r2.cloudflarestorage.com +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_BUCKET=dxd-site-scraper +S3_REGION=auto +S3_FORCE_PATH_STYLE=true +``` + +The Railway worker must use the same R2 bucket as the hosting Worker. + +## Run Database Migrations + +After merging and deploying the code, run: + +```bash +bun run db:migrate +``` + +This applies: + +```txt +0003_add_static_hosting.sql +0004_add_hosting_controls_and_billing.sql +0005_add_hosting_redirect_mode.sql +``` + +These migrations add: + +- Published backup records. +- Client hostname records. +- Auto-publish settings. +- Billing link fields. +- Redirect mode fields. + +## Client Domain Setup Flow + +For each client: + +1. Open the site in the internal dashboard. +2. Add their backup hostname, for example: + +```txt +backup.client.com +``` + +3. Give the client this CNAME instruction: + +```txt +backup.client.com CNAME backup-hosting.designxdevelop.com +``` + +4. If Cloudflare returns TXT verification records, give those to the client too. +5. Click `Sync` in the dashboard until Cloudflare/SSL status becomes active. +6. Publish a completed crawl or wait for the next successful crawl to auto-publish. + +## Operating Modes + +Each client hostname has two modes. + +### Redirect Mode + +Use this when the client's main site is healthy. + +Behavior: + +```txt +backup.client.com/* 301 -> client.com/* +``` + +The redirect preserves path and query string. + +Example: + +```txt +https://backup.client.com/services/web-design?utm=test + -> https://client.com/services/web-design?utm=test +``` + +### Backup Mode + +Use this during extended downtime of the client's main site. + +Behavior: + +```txt +backup.client.com/* -> static backup files from R2 +``` + +Disable redirect mode in the dashboard to serve the backup. + +## Version Behavior + +By default: + +```txt +latest successful crawl auto-publishes as the live backup +``` + +For rollback/pinning: + +1. Select an older published version in the dashboard. +2. Activate it. +3. Auto-publish is disabled so the hosted backup stays pinned. + +To resume latest-successful behavior, re-enable `Auto latest`. + +## Billing Workflow + +This is intentionally internal and simple for now. + +1. Create a monthly Stripe Payment Link manually in Stripe. +2. Paste it into the site's billing field in the dashboard. +3. Add the client's billing email. +4. Click `Send Link`. + +The dashboard tracks billing status manually: + +```txt +not_sent +sent +paid +past_due +cancelled +``` + +If billing needs to become automatic later, add Stripe Checkout/Billing webhooks to update this status. + +## Smoke Test Checklist + +- `bun run db:migrate` succeeds. +- `apps/hosting-worker` deploys successfully. +- `backup-hosting.designxdevelop.com` reaches the hosting Worker. +- Adding `backup.client.com` in the dashboard creates a Cloudflare custom hostname. +- Client CNAME points to `backup-hosting.designxdevelop.com`. +- Dashboard `Sync` eventually shows active SSL/domain status. +- Publishing a crawl creates files under `published///` in R2. +- With redirect disabled, `backup.client.com` serves the backup. +- With redirect enabled, `backup.client.com/path?x=1` returns `301` to the configured root origin with `/path?x=1` preserved. From 7d9ba7799ed59af2c6de8965f6d0b37cf5eb9dc4 Mon Sep 17 00:00:00 2001 From: Austin Thesing Date: Mon, 27 Apr 2026 13:25:16 -0700 Subject: [PATCH 3/4] chore: relax worker bun install --- services/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/worker/Dockerfile b/services/worker/Dockerfile index d159f5b..5cd57a8 100644 --- a/services/worker/Dockerfile +++ b/services/worker/Dockerfile @@ -14,7 +14,7 @@ COPY packages/storage/package.json packages/storage/package.json COPY packages/db/package.json packages/db/package.json COPY services/worker/package.json services/worker/package.json -RUN bun install --frozen-lockfile +RUN bun install # From repo root, bunx would fetch latest Playwright; cwd must match @dxd/scraper’s locked version. RUN cd packages/scraper && bunx playwright install --with-deps chromium From 07f6b039d86e53ee43e17a76a06f2eb1656c3100 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 27 Apr 2026 20:51:52 +0000 Subject: [PATCH 4/4] Fix open backup hosting review findings Co-authored-by: Austin Thesing --- apps/api/src/routes/hosting.ts | 110 ++++++++++--- apps/web/src/routes/sites/$siteId.tsx | 48 +++++- docs/setup-backup-hosting.md | 14 +- services/worker/Dockerfile | 2 +- services/worker/src/processor.ts | 216 +++++++++++++++++++------- 5 files changed, 294 insertions(+), 96 deletions(-) diff --git a/apps/api/src/routes/hosting.ts b/apps/api/src/routes/hosting.ts index 551a49a..f7acbee 100644 --- a/apps/api/src/routes/hosting.ts +++ b/apps/api/src/routes/hosting.ts @@ -96,7 +96,7 @@ async function deleteCloudflareCustomHostname( cloudflareHostnameId: string ) { const config = getCloudflareConfig(c); - if (!config) return; + if (!config) return false; const response = await fetch( `https://api.cloudflare.com/client/v4/zones/${config.zoneId}/custom_hostnames/${cloudflareHostnameId}`, @@ -113,6 +113,8 @@ async function deleteCloudflareCustomHostname( const message = payload?.errors?.[0]?.message || `Cloudflare custom hostname delete failed with ${response.status}`; throw new Error(message); } + + return true; } async function syncCloudflareCustomHostname( @@ -163,6 +165,16 @@ function toOrigin(value: string | null | undefined): string | null { } } +function isPgUniqueViolation(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + typeof (error as { code?: unknown }).code === "string" && + (error as { code: string }).code === "23505" + ); +} + app.get("/:siteId/hosting", async (c) => { const db = c.get("db"); const siteId = c.req.param("siteId"); @@ -278,43 +290,84 @@ app.post("/:siteId/domains", zValidator("json", createDomainSchema), async (c) = return c.json({ error: "Cloudflare custom hostname configuration is required to add client domains" }, 503); } - let cloudflareFields: ReturnType; - try { - const result = await createCloudflareCustomHostname(c, hostname); - if (!result) { - return c.json({ error: "Cloudflare custom hostname provisioning is not configured" }, 503); - } - cloudflareFields = extractCloudflareDomainFields(result); - } catch (error) { - const message = error instanceof Error ? error.message : "Failed to provision Cloudflare custom hostname"; - return c.json({ error: message }, 502); - } - const latestPublication = await db.query.sitePublications.findFirst({ where: and(eq(sitePublications.siteId, siteId), eq(sitePublications.status, "published")), orderBy: desc(sitePublications.createdAt), }); + let reservedDomain: typeof siteDomains.$inferSelect; try { - const [domain] = await db + [reservedDomain] = await db .insert(siteDomains) .values({ siteId, hostname, cnameTarget, - status: cloudflareFields?.status ?? "pending_dns", + status: "provisioning", activePublicationId: latestPublication?.id ?? null, redirectTargetOrigin: toOrigin(site.url), - cloudflareHostnameId: cloudflareFields?.cloudflareHostnameId, - ownershipVerificationName: cloudflareFields?.ownershipVerificationName, - ownershipVerificationValue: cloudflareFields?.ownershipVerificationValue, - sslStatus: cloudflareFields?.sslStatus, }) .returning(); + } catch (error) { + if (isPgUniqueViolation(error)) { + return c.json({ error: "Hostname is already configured" }, 409); + } + const message = error instanceof Error ? error.message : "Failed to reserve hosted domain"; + return c.json({ error: message }, 500); + } + + let cloudflareHostnameId: string | null = null; + try { + const result = await createCloudflareCustomHostname(c, hostname); + if (!result) { + throw new Error("Cloudflare custom hostname provisioning is not configured"); + } + + const cloudflareFields = extractCloudflareDomainFields(result); + cloudflareHostnameId = cloudflareFields.cloudflareHostnameId; + const [domain] = await db + .update(siteDomains) + .set({ + status: cloudflareFields.status ?? "pending_dns", + cloudflareHostnameId: cloudflareFields.cloudflareHostnameId, + ownershipVerificationName: cloudflareFields.ownershipVerificationName, + ownershipVerificationValue: cloudflareFields.ownershipVerificationValue, + sslStatus: cloudflareFields.sslStatus, + errorMessage: null, + updatedAt: new Date(), + }) + .where(eq(siteDomains.id, reservedDomain.id)) + .returning(); + + if (!domain) { + throw new Error("Failed to finalize hosted domain after Cloudflare provisioning"); + } return c.json({ domain }, 201); } catch (error) { - return c.json({ error: "Hostname is already configured" }, 409); + if (cloudflareHostnameId) { + try { + await deleteCloudflareCustomHostname(c, cloudflareHostnameId); + } catch (cleanupError) { + const cleanupMessage = cleanupError instanceof Error ? cleanupError.message : "Unknown cleanup error"; + await db + .update(siteDomains) + .set({ + cloudflareHostnameId, + errorMessage: `Cloudflare cleanup required: ${cleanupMessage}`, + updatedAt: new Date(), + }) + .where(eq(siteDomains.id, reservedDomain.id)); + return c.json({ error: `Cloudflare cleanup required: ${cleanupMessage}` }, 502); + } + } + + await db.delete(siteDomains).where(eq(siteDomains.id, reservedDomain.id)).catch(() => undefined); + const message = error instanceof Error ? error.message : "Failed to provision Cloudflare custom hostname"; + if (message.includes("not configured")) { + return c.json({ error: message }, 503); + } + return c.json({ error: message }, /Cloudflare/i.test(message) ? 502 : 500); } }); @@ -359,6 +412,12 @@ app.post("/:siteId/domains/:domainId/sync", async (c) => { if (!domain.cloudflareHostnameId) return c.json({ domain }); const result = await syncCloudflareCustomHostname(c, domain.cloudflareHostnameId); + if (!result) { + return c.json( + { error: "Cloudflare custom hostname sync is unavailable because Cloudflare configuration is missing" }, + 503 + ); + } const fields = extractCloudflareDomainFields(result); const [updated] = await db .update(siteDomains) @@ -434,7 +493,16 @@ app.delete("/:siteId/domains/:domainId", async (c) => { if (!existing) return c.json({ error: "Domain not found" }, 404); if (existing.cloudflareHostnameId) { - await deleteCloudflareCustomHostname(c, existing.cloudflareHostnameId); + const deleted = await deleteCloudflareCustomHostname(c, existing.cloudflareHostnameId); + if (!deleted) { + return c.json( + { + error: + "Cloudflare credentials are not configured, so this domain cannot be safely removed while a custom hostname may still exist", + }, + 503 + ); + } } await db.delete(siteDomains).where(and(eq(siteDomains.id, domainId), eq(siteDomains.siteId, siteId))); diff --git a/apps/web/src/routes/sites/$siteId.tsx b/apps/web/src/routes/sites/$siteId.tsx index 2de333f..6ef6db1 100644 --- a/apps/web/src/routes/sites/$siteId.tsx +++ b/apps/web/src/routes/sites/$siteId.tsx @@ -45,6 +45,9 @@ function SiteDetailPage() { const [hostingBillingEmail, setHostingBillingEmail] = useState(""); const [hostingPaymentLinkUrl, setHostingPaymentLinkUrl] = useState(""); const [hostingBillingStatus, setHostingBillingStatus] = useState<"not_sent" | "sent" | "paid" | "past_due" | "cancelled">("not_sent"); + const [hostingSettingsInitialized, setHostingSettingsInitialized] = useState(false); + const [hostingSettingsDirty, setHostingSettingsDirty] = useState(false); + const [domainRedirectTargets, setDomainRedirectTargets] = useState>({}); const site = data?.site; @@ -73,11 +76,27 @@ function SiteDetailPage() { useEffect(() => { if (!hostingData?.settings) return; + if (hostingSettingsInitialized && hostingSettingsDirty) return; setHostingAutoPublish(hostingData.settings.hostingAutoPublish); setHostingBillingEmail(hostingData.settings.hostingBillingEmail ?? ""); setHostingPaymentLinkUrl(hostingData.settings.hostingPaymentLinkUrl ?? ""); setHostingBillingStatus((hostingData.settings.hostingBillingStatus as typeof hostingBillingStatus) || "not_sent"); - }, [hostingData?.settings]); + setHostingSettingsInitialized(true); + }, [hostingData?.settings, hostingSettingsInitialized, hostingSettingsDirty]); + + useEffect(() => { + if (!hostingData?.domains?.length) return; + setDomainRedirectTargets((prev) => { + const next = { ...prev }; + for (const domain of hostingData.domains) { + const currentValue = prev[domain.id]; + if (currentValue === undefined || currentValue === domain.redirectTargetOrigin || currentValue === site?.url) { + next[domain.id] = domain.redirectTargetOrigin || site?.url || ""; + } + } + return next; + }); + }, [hostingData?.domains, site?.url]); const configurationMutation = useMutation({ mutationFn: (payload: { @@ -175,6 +194,7 @@ function SiteDetailPage() { hostingBillingStatus?: "not_sent" | "sent" | "paid" | "past_due" | "cancelled"; }) => hostingApi.updateSettings(siteId, payload), onSuccess: () => { + setHostingSettingsDirty(false); queryClient.invalidateQueries({ queryKey: ["hosting", siteId] }); queryClient.invalidateQueries({ queryKey: ["sites", siteId] }); }, @@ -637,6 +657,7 @@ function SiteDetailPage() { checked={hostingAutoPublish} onChange={(e) => { setHostingAutoPublish(e.target.checked); + setHostingSettingsDirty(true); hostingSettingsMutation.mutate({ hostingAutoPublish: e.target.checked }); }} style={{ accentColor: "#6366f1" }} @@ -737,11 +758,15 @@ function SiteDetailPage() {
{ - const nextValue = e.target.value.trim(); + onChange={(e) => { + const nextValue = e.target.value; + setDomainRedirectTargets((prev) => ({ ...prev, [domain.id]: nextValue })); + }} + onBlur={() => { + const nextValue = (domainRedirectTargets[domain.id] ?? "").trim(); if (nextValue && nextValue !== (domain.redirectTargetOrigin || site.url)) { updateDomainMutation.mutate({ domainId: domain.id, redirectTargetOrigin: nextValue }); } @@ -775,20 +800,29 @@ function SiteDetailPage() { setHostingBillingEmail(e.target.value)} + onChange={(e) => { + setHostingSettingsDirty(true); + setHostingBillingEmail(e.target.value); + }} className="input-dark font-mono text-sm" placeholder="owner@client.com" /> setHostingPaymentLinkUrl(e.target.value)} + onChange={(e) => { + setHostingSettingsDirty(true); + setHostingPaymentLinkUrl(e.target.value); + }} className="input-dark font-mono text-sm" placeholder="https://buy.stripe.com/..." />