From 70da095d73bf61810a09175f93dc850d55b49d3b Mon Sep 17 00:00:00 2001 From: Kellen Busby Date: Mon, 6 Jul 2026 22:53:25 -0700 Subject: [PATCH] ci: seed preview database locally then bulk-import to Turso MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preview deploy seeded each PR's fresh Turso DB by running seed:standalone with DATABASE_URI pointed at remote Turso. Payload's seed does thousands of per-row create() calls, so against a remote libSQL database every row is a network round-trip (~289s on ubuntu-latest, and long enough on higher-latency runners to lose the connection mid-seed). Seed into a local SQLite file instead, then bulk-import that file into the fresh Turso preview DB in a single operation: - Seed step overrides NODE_ENV off `production` so the SQLite adapter runs in push mode and builds the current schema directly from the Payload config — the same fast path the CI build job uses. Running migrations from zero isn't viable here: the backfill_document_references data migration reads collections whose block tables a later migration adds, so the query hits `no such table` on a fresh DB. - Import with `turso db create --from-dump` (a sqlite3 .dump) rather than `--from-file`, which requires WAL mode and mis-parses valid WAL databases on sqlite3 >= 3.50 (tursodatabase/turso-cli#1030). The end state is identical — a fully seeded Turso preview DB — but the seed/import drops from ~5 min to seconds and no longer depends on sustained low-latency round-trips, so it works on any runner. Refs #1143 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/preview.yaml | 76 +++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/.github/workflows/preview.yaml b/.github/workflows/preview.yaml index f0476c2f1..9abea4507 100644 --- a/.github/workflows/preview.yaml +++ b/.github/workflows/preview.yaml @@ -13,6 +13,11 @@ concurrency: jobs: deploy: if: ${{ github.event_name != 'merge_group' }} + # Seeding now writes to a local SQLite file and is bulk-imported into the + # fresh Turso preview DB in one operation, so it no longer streams thousands + # of latency-bound round-trips that killed Blacksmith runners mid-seed. + # Moving this job onto a Blacksmith runner is now a trivial follow-up — see + # NWACus/web#1143. runs-on: ubuntu-latest environment: Preview env: @@ -22,7 +27,7 @@ jobs: run: curl -sSfL https://get.tur.so/install.sh | bash env: TURSO_INSTALL_SKIP_SIGNUP: 'true' - - name: Create a new database for the preview + - name: Compute the preview database name id: database run: | set -o errexit @@ -42,13 +47,53 @@ jobs: echo "ref=${ref}" >> "${GITHUB_OUTPUT}" name="payloadcms-preview-${ref}" + echo "name=${name}" >> "${GITHUB_OUTPUT}" + - name: 🏗 Setup repo + uses: actions/checkout@v4 + - name: 🏗 Setup pnpm + uses: pnpm/action-setup@v4 + - name: 🏗 Setup Node + uses: actions/setup-node@v4 + with: + node-version: 24.x + cache: pnpm + - name: 📦 Install dependencies + run: pnpm ii + shell: bash + - name: Seed a local SQLite database + # Seed against a local file instead of streaming thousands of per-row + # writes to remote Turso. This is the same fast path the CI build job + # uses: NODE_ENV is overridden off `production` so the SQLite adapter + # runs in push mode and builds the current schema directly from the + # Payload config (running migrations from zero isn't viable — a data + # migration reads collections whose block tables a later migration + # adds). The resulting schema matches what the deployed preview expects. + run: pnpm seed:standalone + env: + NODE_ENV: development + DATABASE_URI: 'file:./preview-seed.db' + PAYLOAD_SEED_PASSWORD: '${{ secrets.PAYLOAD_SEED_PASSWORD }}' + PAYLOAD_SECRET: '${{ secrets.PAYLOAD_SECRET }}' + - name: Create the preview database from the seeded file + # Bulk-import the seeded local DB into a fresh Turso DB in one operation. + # We use `--from-dump` (plain SQL) rather than `--from-file`: the latter + # requires WAL mode and mis-parses valid WAL databases on sqlite3 >= 3.50 + # (tursodatabase/turso-cli#1030), whereas a `.dump` is runner-agnostic. + run: | + set -o errexit + set -o nounset + set -o pipefail + + name="${{ steps.database.outputs.name }}" + + sqlite3 ./preview-seed.db .dump > ./preview-seed.sql + /home/runner/.turso/turso org switch nwac if /home/runner/.turso/turso db show "${name}"; then /home/runner/.turso/turso db destroy "${name}" --yes fi - /home/runner/.turso/turso db create "${name}" --wait + /home/runner/.turso/turso db create "${name}" --from-dump ./preview-seed.sql --wait /home/runner/.turso/turso db shell "${name}" .tables - echo "name=${name}" >> "${GITHUB_OUTPUT}" env: TURSO_API_TOKEN: ${{ secrets.TURSO_API_TOKEN }} - name: Record database connection URI and token @@ -62,18 +107,6 @@ jobs: echo "token=$(/home/runner/.turso/turso db tokens create "${{ steps.database.outputs.name }}")" >> "${GITHUB_OUTPUT}" env: TURSO_API_TOKEN: ${{ secrets.TURSO_API_TOKEN }} - - name: 🏗 Setup repo - uses: actions/checkout@v4 - - name: 🏗 Setup pnpm - uses: pnpm/action-setup@v4 - - name: 🏗 Setup Node - uses: actions/setup-node@v4 - with: - node-version: 24.x - cache: pnpm - - name: 📦 Install dependencies - run: pnpm ii - shell: bash - name: Install Vercel CLI run: npm install --global vercel@latest - name: Generate preview domain names @@ -101,19 +134,6 @@ jobs: run: echo -n "${{ steps.domains.outputs.preview_alias }}" | vercel env add NEXT_PUBLIC_ROOT_DOMAIN preview ${{ github.head_ref }} --no-sensitive --force --token=${{ secrets.VERCEL_TOKEN }} - name: Pull Vercel Environment Variables into .env file run: vercel env pull --yes --environment=preview --git-branch="${{ github.head_ref }}" --token=${{ secrets.VERCEL_TOKEN }} .env - - name: Run database migrations - run: pnpm migrate - env: - DATABASE_URI: '${{ steps.connection.outputs.uri }}' - DATABASE_AUTH_TOKEN: '${{ steps.connection.outputs.token }}' - PAYLOAD_SECRET: '${{ secrets.PAYLOAD_SECRET }}' - - name: Seed the database - run: pnpm seed:standalone - env: - DATABASE_URI: '${{ steps.connection.outputs.uri }}' - DATABASE_AUTH_TOKEN: '${{ steps.connection.outputs.token }}' - PAYLOAD_SEED_PASSWORD: '${{ secrets.PAYLOAD_SEED_PASSWORD }}' - PAYLOAD_SECRET: '${{ secrets.PAYLOAD_SECRET }}' - name: Pull Vercel Environment Information run: vercel pull --yes --environment=preview --git-branch="${{ github.head_ref }}" --token=${{ secrets.VERCEL_TOKEN }} - name: Build Project Artifacts