From fec626c1d32f1b91b31187ec0f2b29034fd5c8f1 Mon Sep 17 00:00:00 2001 From: us Date: Mon, 15 Jun 2026 02:47:54 +0300 Subject: [PATCH 1/2] feat: add fastCRW support --- .env.example | 5 +++++ environment.d.ts | 2 ++ src/lib/crw.ts | 9 +++++++++ src/server/jstack.ts | 2 ++ 4 files changed, 18 insertions(+) create mode 100644 src/lib/crw.ts diff --git a/.env.example b/.env.example index 77cbc45a..cf5a970f 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,11 @@ DATABASE_URL= # WEB CRAWLING (FIRECRAWL) FIRECRAWL_API_KEY= +# WEB CRAWLING (FASTCRW) — Firecrawl-compatible web scraper (self-host or cloud) +CRW_API_KEY= +# Optional self-host base URL override (defaults to https://fastcrw.com/api) +CRW_API_URL= + # AWS AWS_GENERAL_ACCESS_KEY= AWS_GENERAL_SECRET_KEY= diff --git a/environment.d.ts b/environment.d.ts index 584c1703..107bbc95 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -19,6 +19,8 @@ declare global { GOOGLE_CLIENT_SECRET: string RESEND_API_KEY: string FIRECRAWL_API_KEY: string + CRW_API_KEY: string + CRW_API_URL: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string diff --git a/src/lib/crw.ts b/src/lib/crw.ts new file mode 100644 index 00000000..a9e0f54d --- /dev/null +++ b/src/lib/crw.ts @@ -0,0 +1,9 @@ +import FirecrawlApp from '@mendable/firecrawl-js' + +// fastCRW: Firecrawl-compatible web scraper (single binary; self-host or cloud). +// Reuses the Firecrawl client with a custom base URL. Defaults to the managed +// cloud; set CRW_API_URL to point at a self-hosted server. +export const crw = new FirecrawlApp({ + apiKey: process.env.CRW_API_KEY, + apiUrl: process.env.CRW_API_URL ?? 'https://fastcrw.com/api', +}) diff --git a/src/server/jstack.ts b/src/server/jstack.ts index e2024783..ea711e20 100644 --- a/src/server/jstack.ts +++ b/src/server/jstack.ts @@ -22,6 +22,8 @@ interface Env { GOOGLE_CLIENT_SECRET: string RESEND_API_KEY: string FIRECRAWL_API_KEY: string + CRW_API_KEY: string + CRW_API_URL: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string From 9dc8f82812f303f58fa6c03961ea51aaecfbe8df Mon Sep 17 00:00:00 2001 From: us Date: Mon, 15 Jun 2026 03:09:17 +0300 Subject: [PATCH 2/2] fix: make CRW_API_URL optional to match runtime default --- environment.d.ts | 2 +- src/server/jstack.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.d.ts b/environment.d.ts index 107bbc95..93e85a04 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -20,7 +20,7 @@ declare global { RESEND_API_KEY: string FIRECRAWL_API_KEY: string CRW_API_KEY: string - CRW_API_URL: string + CRW_API_URL?: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string diff --git a/src/server/jstack.ts b/src/server/jstack.ts index ea711e20..23436ba3 100644 --- a/src/server/jstack.ts +++ b/src/server/jstack.ts @@ -23,7 +23,7 @@ interface Env { RESEND_API_KEY: string FIRECRAWL_API_KEY: string CRW_API_KEY: string - CRW_API_URL: string + CRW_API_URL?: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string