diff --git a/.env.example b/.env.example index 77cbc45..cf5a970 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,11 @@ DATABASE_URL= # WEB CRAWLING (FIRECRAWL) FIRECRAWL_API_KEY= +# WEB CRAWLING (FASTCRW) — Firecrawl-compatible web scraper (self-host or cloud) +CRW_API_KEY= +# Optional self-host base URL override (defaults to https://fastcrw.com/api) +CRW_API_URL= + # AWS AWS_GENERAL_ACCESS_KEY= AWS_GENERAL_SECRET_KEY= diff --git a/environment.d.ts b/environment.d.ts index 584c170..93e85a0 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -19,6 +19,8 @@ declare global { GOOGLE_CLIENT_SECRET: string RESEND_API_KEY: string FIRECRAWL_API_KEY: string + CRW_API_KEY: string + CRW_API_URL?: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string diff --git a/src/lib/crw.ts b/src/lib/crw.ts new file mode 100644 index 0000000..a9e0f54 --- /dev/null +++ b/src/lib/crw.ts @@ -0,0 +1,9 @@ +import FirecrawlApp from '@mendable/firecrawl-js' + +// fastCRW: Firecrawl-compatible web scraper (single binary; self-host or cloud). +// Reuses the Firecrawl client with a custom base URL. Defaults to the managed +// cloud; set CRW_API_URL to point at a self-hosted server. +export const crw = new FirecrawlApp({ + apiKey: process.env.CRW_API_KEY, + apiUrl: process.env.CRW_API_URL ?? 'https://fastcrw.com/api', +}) diff --git a/src/server/jstack.ts b/src/server/jstack.ts index e202478..23436ba 100644 --- a/src/server/jstack.ts +++ b/src/server/jstack.ts @@ -22,6 +22,8 @@ interface Env { GOOGLE_CLIENT_SECRET: string RESEND_API_KEY: string FIRECRAWL_API_KEY: string + CRW_API_KEY: string + CRW_API_URL?: string AWS_GENERAL_ACCESS_KEY: string AWS_GENERAL_SECRET_KEY: string AWS_REGION: string