From 2bf71064d2ea0a923f9eabeb7d78012d173a1ae9 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 05:23:28 -0400 Subject: [PATCH 01/13] fix(ci): honor shared-cache consumer contract --- .github/workflows/ci.yml | 67 +++++++++++++----------------- AGENTS.md | 2 +- BUILD.bazel | 11 +++++ README.md | 12 +++--- package.json | 2 +- scripts/bazel-cache-backed.sh | 18 ++++++++ scripts/test-bazel-cache-backed.sh | 48 +++++++++++++++++++++ 7 files changed, 113 insertions(+), 47 deletions(-) create mode 100755 scripts/test-bazel-cache-backed.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39d0dbf..b5afcef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,12 +33,7 @@ jobs: bazel-remote-gates: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository # Per-ref dedupe only: a newer push to the same ref supersedes its own - # stale gates run. Cross-PR quota pressure on the gf-reapi-cell tenant - # (maxConcurrentExecutions=4; runs 28732755094, 28847693875) is handled - # by the exit-34 retry loop in scripts/bazel-cache-backed.sh — a shared - # repo-wide group was tried first, but GitHub evicts (cancels) middle - # pending jobs in a group, which breaks required checks (run - # 28849318814 + main 09c1dd0 were cancelled, not queued). + # stale gates run without canceling unrelated pull requests. concurrency: group: gf-reapi-tenant-gates-${{ github.ref }} cancel-in-progress: true @@ -47,6 +42,13 @@ jobs: id-token: write runs-on: tinyland-dind timeout-minutes: 60 + env: + GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed + GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} + # The GF consumer registry records this repo as shared-cache-backed. + # Clear any executor endpoint inherited from the generic ARC pod. + BAZEL_REMOTE_EXECUTOR: '' + BAZEL_REMOTE_EXECUTOR_CACHE: '' steps: - uses: actions/checkout@v6 with: @@ -55,6 +57,7 @@ jobs: - uses: actions/setup-node@v6 with: node-version-file: '.nvmrc' + cache: npm - name: Cache Playwright browsers uses: actions/cache@v5 @@ -62,26 +65,22 @@ jobs: path: ~/.cache/ms-playwright key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - - name: Provision Chromium for browser gates + - name: Provision package-lock-pinned Chromium run: | set -euo pipefail - # Executor images ship /bin/chromium; the dind runner image ships no - # browser, so the puppeteer smoke and playwright e2e need one - # provisioned here. Playwright's installer also pulls the shared - # system libraries the binary links against. - for c in /bin/chromium /usr/bin/chromium /usr/bin/chromium-browser /usr/bin/google-chrome; do - if [ -x "$c" ]; then - echo "System chromium present at $c" - exit 0 - fi - done - PLAYWRIGHT_VERSION="$(node -p "require('./package.json').devDependencies['@playwright/test'].replace(/^[^0-9]*/, '')")" - npx --yes "playwright@${PLAYWRIGHT_VERSION}" install --with-deps chromium - BIN="$(ls -d "${HOME}/.cache/ms-playwright"/chromium-*/chrome-linux*/chrome 2>/dev/null | head -1)" - test -x "${BIN}" - echo "GF_RBE_CHROMIUM_EXECUTABLE=${BIN}" >> "$GITHUB_ENV" - - - name: Resolve GloriousFlywheel substrate lane + # Install the revision pinned by @playwright/test outside Bazel, + # before token minting and SSH setup, then pass its absolute path + # into browser tests. + npm ci --ignore-scripts + npx --no-install playwright install --with-deps chromium + chromium_path="$(node --input-type=module -e "import { chromium } from '@playwright/test'; process.stdout.write(chromium.executablePath())")" + if [[ "${chromium_path}" != /* || ! -x "${chromium_path}" ]]; then + echo "::error::Playwright Chromium resolution produced an unusable path" + exit 1 + fi + echo "GF_RBE_CHROMIUM_EXECUTABLE=${chromium_path}" >> "${GITHUB_ENV}" + + - name: Require shared-cache-backed GloriousFlywheel lane run: | set -euo pipefail if [ -z "${BAZEL_REMOTE_CACHE:-}" ]; then @@ -89,28 +88,18 @@ jobs: exit 1 fi - # This repo's GF consumer-registry entry is substrate_mode= - # shared-cache-backed (latch: remote_processes=0), so executor - # tokens are not ours to mint — the exchange grants cache-write - # only to trusted default-branch pushes and cache-read to any - # enrolled ref. Strip the executor attachment so the wrapper and - # cache-attachment-contract agree on shared-cache mode. - echo "GF_BAZEL_SUBSTRATE_MODE=shared-cache-backed" >> "$GITHUB_ENV" - echo "BAZEL_REMOTE_EXECUTOR=" >> "$GITHUB_ENV" - echo "BAZEL_REMOTE_EXECUTOR_CACHE=" >> "$GITHUB_ENV" - if [ "${GITHUB_EVENT_NAME}" = "push" ] && [ "${GITHUB_REF}" = "refs/heads/main" ]; then - echo "GF_REAPI_TOKEN_EXCHANGE_REQUEST=cache-write" >> "$GITHUB_ENV" - else - echo "GF_REAPI_TOKEN_EXCHANGE_REQUEST=cache-read" >> "$GITHUB_ENV" - echo "GF_BAZEL_CACHE_ROLE=read-only" >> "$GITHUB_ENV" + if [ -n "${BAZEL_REMOTE_EXECUTOR:-}" ]; then + echo "::error::BAZEL_REMOTE_EXECUTOR must be empty for this shared-cache-backed consumer." + exit 1 fi - name: Mint GloriousFlywheel REAPI token env: GF_REAPI_TOKEN_EXCHANGE_TTL: 45m + GF_REAPI_TOKEN_EXCHANGE_REQUEST: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'cache-write' || 'cache-read' }} run: bash scripts/mint-gf-reapi-token-from-exchange.sh - - name: Validate GloriousFlywheel RBE attachment + - name: Validate GloriousFlywheel cache attachment run: bash scripts/cache-attachment-contract.sh --strict - name: Set up SSH for spear-resumes diff --git a/AGENTS.md b/AGENTS.md index 61f98ac..252dd09 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ These rules apply before the writing-style guidance below. ## Bazel And GloriousFlywheel -- GloriousFlywheel-backed validation runs through `scripts/bazel-cache-backed.sh`. The remote scripts must fail closed when `BAZEL_REMOTE_CACHE`, `BAZEL_REMOTE_EXECUTOR`, or the expected executor-backed substrate is missing. +- GloriousFlywheel-backed validation runs through `scripts/bazel-cache-backed.sh`. This repository is registered as `shared-cache-backed`; the remote scripts must fail closed when `BAZEL_REMOTE_CACHE` or that expected substrate mode is missing. CI must clear generic-runner executor hints. Executor-backed mode remains an explicit opt-in only after a reviewed GloriousFlywheel consumer-registry promotion. - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` are the authoritative Bazel entrypoints. They cover SvelteKit check/build smoke, package checks, CV PDF sync, Vitest, blog-agent tests, browser smoke, Playwright e2e, and graph hygiene. - `package-lock.json` is the npm authority. `pnpm-lock.yaml` is the generated `rules_js` lock consumed by Bazel; do not casually hand-edit it. - Browser-backed Bazel tests use the pinned worker Chromium path. Do not add lifecycle downloads or host-local browser assumptions to the RBE path. diff --git a/BUILD.bazel b/BUILD.bazel index 9d7bd54..d9e6135 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -293,6 +293,16 @@ sh_test( visibility = ["//visibility:public"], ) +sh_test( + name = "bazel_cache_backed_contract", + srcs = ["scripts/test-bazel-cache-backed.sh"], + data = [ + "scripts/bazel-cache-backed.sh", + "scripts/cache-attachment-contract.sh", + ], + visibility = ["//visibility:public"], +) + exports_files( [ "package-lock.json", @@ -312,6 +322,7 @@ exports_files( "scripts/run-sveltekit-check-bazel.mjs", "scripts/run-sveltekit-vite-build-bazel.mjs", "scripts/serve-static-build.mjs", + "scripts/test-bazel-cache-backed.sh", "scripts/run-vitest-bazel.mjs", "vitest.bazel.config.ts", ], diff --git a/README.md b/README.md index b0c700b..85a2bf1 100644 --- a/README.md +++ b/README.md @@ -100,29 +100,29 @@ flowchart LR PrivateMirror --> Tailnet["jesssullivan-blog-shadow.taila4c78d.ts.net"] ``` -## GloriousFlywheel Bazel/RBE Pilot Surface +## GloriousFlywheel Bazel Substrate Surface This repo still uses the npm/SvelteKit workflow for normal local development and deployment. CI also carries a blocking GloriousFlywheel Bazel lane on the `tinyland-dind` ARC runner so check, test, and Chromium e2e coverage are not proven only by local `npm`/`npx` commands. - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. -- GitHub CI runs the Bazel lane on `tinyland-dind` and refuses to run the RBE gate unless `BAZEL_REMOTE_EXECUTOR` and `BAZEL_REMOTE_CACHE` are attached by the ARC runner environment. Before the strict gate, CI exchanges GitHub Actions OIDC with the GF token exchange and writes the short-lived cell JWT to `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`. -- In executor-backed mode, the wrapper uses the executor endpoint as Bazel's effective REAPI CAS/cache by default so remote Execute can read uploaded action/input digests. This profile disables remote cache compression for the current GF proof cell and gives remote actions writable GF worker cache paths. `BAZEL_REMOTE_EXECUTOR_CACHE` can override the cache endpoint only when a separate executor-readable CAS is wired. +- GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. +- Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. -- The CI lane sets `BAZEL_REMOTE_INSTANCE_NAME=default` for the current GF tenant. Literal shell placeholders are rejected before Bazel starts. +- The token exchange supplies the repository-scoped `BAZEL_REMOTE_INSTANCE_NAME`. Literal shell placeholders are rejected before Bazel starts. - `//:sveltekit_check` runs the SvelteKit check path under Bazel. - `//static/cv:pdfs_synced_test` byte-compares the checked-in resume/CV PDFs against the Bazel-built `spear_resumes` outputs; `.bazelrc` pins `SOURCE_DATE_EPOCH` and `TZ` so Tectonic output stays reproducible across local sync, CI, and GF RBE. - `//:vitest_unit_tests` wraps the root and Pulse Vitest suites through `vitest.bazel.config.ts`. - `//:blog_agent_node_tests` wraps the blog-agent `node:test` suite through `tsx --test`. - `//:sveltekit_vite_build_smoke` runs a copied-workdir SvelteKit/Vite production build smoke. It proves the build target class, not the full npm prebuild/postbuild publication chain. -- `//:playwright_chromium_e2e` runs the Chromium Playwright e2e suite through Bazel against the pinned GloriousFlywheel Chromium runtime path. +- `//:playwright_chromium_e2e` runs the Chromium Playwright e2e suite through Bazel against the pinned GloriousFlywheel runner Chromium path. - `//:playwright_chromium_smoke` remains a narrow diagnostic target for browser runtime authority, not the remote e2e gate. - `//:puppeteer_chromium_smoke` launches Puppeteer against the same pinned Chromium runtime path. It proves Puppeteer can consume browser runtime authority without lifecycle downloads. - `package-lock.json` remains the npm dependency authority for the app. `pnpm-workspace.yaml` makes the package importers explicit for Bazel, and `pnpm-lock.yaml` is the generated `rules_js` lock consumed by Bazel. - Bazel npm lifecycle hooks skip Playwright and Puppeteer browser downloads. Browser-backed RBE must use the pinned worker Chromium path rather than downloading browsers during proof actions. - GloriousFlywheel proof runs use the external GF REAPI proof harness or the repository `remote:*` scripts against this public repo checkout; remote cache hits, hosted runners, and shared-cache-only execution do not count as RBE. -Current boundary: this gates Bazel check/test/Chromium-e2e and CV PDF sync drift under GloriousFlywheel. Deployment still publishes the SvelteKit static artifact through the existing Pages workflows. +Current boundary: this gates Bazel check/test/Chromium-e2e and CV PDF sync drift with GloriousFlywheel shared-cache attachment. It does not claim remote action execution. Deployment still publishes the SvelteKit static artifact through the existing Pages workflows. ## Production DNS And Health diff --git a/package.json b/package.json index b7db507..7c2f79f 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "check": "npm run remote:check", "icons:generate": "tsx scripts/generate-favicon-kit.mts", "remote:check": "bash scripts/bazel-cache-backed.sh test //:sveltekit_check //:sveltekit_vite_build_smoke //:workspace_package_checks //static/cv:pdfs_synced_test", - "remote:test": "bash scripts/bazel-cache-backed.sh test //:bazel_graph_hygiene //:vitest_unit_tests //:blog_agent_node_tests //:puppeteer_chromium_smoke", + "remote:test": "bash scripts/bazel-cache-backed.sh test //:bazel_graph_hygiene //:bazel_cache_backed_contract //:vitest_unit_tests //:blog_agent_node_tests //:puppeteer_chromium_smoke", "remote:e2e": "bash scripts/bazel-cache-backed.sh test //:playwright_chromium_e2e", "remote:all": "npm run remote:check && npm run remote:test && npm run remote:e2e", "ingest:tinyland-posts": "tsx scripts/ingest-tinyland-posts.mts --adopt-reviewed", diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index 864d0b2..a65acfb 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -29,6 +29,8 @@ Environment: action/input digests from the same REAPI service. GF_BAZEL_REMOTE_EXECUTION_PLATFORM optionally overrides the executor platform property; defaults to gloriousflywheel-rbe-linux-x86_64. + GF_BAZEL_REMOTE_UPLOAD controls remote result uploads. Set it to false for + cache-read-only refs and true for trusted cache-write refs; defaults to true. GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE or GF_REAPI_CREDENTIAL_HELPER_TOKEN must be set when a gf-reapi-cell endpoint requires JWT auth. The helper also uses /var/run/secrets/tokens/gf-reapi-cell-token when present. @@ -82,6 +84,8 @@ external_fetch_args=() executor_args=() credential_args=() routing_args=() +upload_args=() +remote_upload="${GF_BAZEL_REMOTE_UPLOAD:-true}" endpoint_host() { local endpoint="$1" @@ -147,6 +151,19 @@ if [[ -n ${BAZEL_DISTDIR:-} ]]; then done fi +case "${remote_upload}" in +true | 1) + upload_args+=(--remote_upload_local_results=true) + ;; +false | 0) + upload_args+=(--remote_upload_local_results=false) + ;; +*) + echo "ERROR: GF_BAZEL_REMOTE_UPLOAD must be true, false, 1, or 0; got ${remote_upload}." >&2 + exit 2 + ;; +esac + bash ./scripts/cache-attachment-contract.sh --strict if ! command -v "${bazel_bin}" >/dev/null 2>&1; then @@ -193,6 +210,7 @@ info) "${bazel_bin}" "${command}" \ --config="${bazel_config}" \ --remote_cache="${effective_remote_cache}" \ + "${upload_args[@]}" \ "${executor_args[@]}" \ "${credential_args[@]}" \ "${routing_args[@]}" \ diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh new file mode 100755 index 0000000..6f61076 --- /dev/null +++ b/scripts/test-bazel-cache-backed.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -n ${TEST_SRCDIR:-} && -n ${TEST_WORKSPACE:-} ]]; then + repo_root="${TEST_SRCDIR}/${TEST_WORKSPACE}" +else + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fi +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/blog-bazel-cache-contract.XXXXXX")" +trap 'rm -rf "${tmp_dir}"' EXIT + +fake_bazel="${tmp_dir}/bazel" +cat >"${fake_bazel}" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +: "${FAKE_BAZEL_ARGS:?}" +printf '%s\n' "$@" >"${FAKE_BAZEL_ARGS}" +EOF +chmod +x "${fake_bazel}" + +run_upload_case() { + local value="$1" + local expected="$2" + local args_file="${tmp_dir}/${value}.args" + + BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ + GF_BAZEL_REMOTE_UPLOAD="${value}" \ + FAKE_BAZEL_ARGS="${args_file}" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null + + grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null +} + +run_upload_case false false +run_upload_case true true + +if BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ + GF_BAZEL_REMOTE_UPLOAD="sometimes" \ + FAKE_BAZEL_ARGS="${tmp_dir}/invalid.args" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>&1; then + echo "ERROR: invalid GF_BAZEL_REMOTE_UPLOAD unexpectedly succeeded" >&2 + exit 1 +fi From 16fafbd920d6f7e65cf08a0991c59d6ec3a6fb01 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 05:30:50 -0400 Subject: [PATCH 02/13] fix(ci): give local cache actions writable paths --- .bazelrc | 8 ++++++++ README.md | 1 + 2 files changed, 9 insertions(+) diff --git a/.bazelrc b/.bazelrc index c6d275a..431eeaf 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,6 +39,14 @@ build:ci-remote-base --remote_timeout=60 build:ci-cached --config=ci-remote-base build:ci-cached --remote_cache_compression +# Shared-cache actions execute on the runner. Give tools such as Tectonic +# stable writable cache/config paths instead of inheriting the ARC image's +# read-only /root home. +build:ci-cached --action_env=HOME=/tmp +build:ci-cached --action_env=TMPDIR=/tmp +build:ci-cached --action_env=XDG_CACHE_HOME=/tmp/xdg-cache +build:ci-cached --action_env=XDG_CONFIG_HOME=/tmp/xdg-config +build:ci-cached --action_env=XDG_DATA_HOME=/tmp/xdg-data build:executor-backed --config=ci-remote-base build:executor-backed --action_env=HOME=/var/lib/gf-reapi-cell/home diff --git a/README.md b/README.md index 85a2bf1..db7b217 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. +- Shared-cache actions execute locally on the runner with stable writable `/tmp` HOME/XDG paths. This keeps Tectonic and other cache-using tools off the ARC image's read-only `/root` without introducing runner-specific paths into Bazel action keys. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. - The token exchange supplies the repository-scoped `BAZEL_REMOTE_INSTANCE_NAME`. Literal shell placeholders are rejected before Bazel starts. From 07e7c802918e87bfcb49d1a6217eeb1cf6f2a691 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 05:53:15 -0400 Subject: [PATCH 03/13] fix(ci): provision Chromium for cache-backed tests --- .github/workflows/ci.yml | 16 ++++++++++++++++ AGENTS.md | 2 +- README.md | 6 +++--- scripts/bazel-cache-backed.sh | 6 ++++++ scripts/test-bazel-cache-backed.sh | 2 ++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5afcef..d610335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,6 +142,22 @@ jobs: sudo mv "${tmp}" /usr/local/bin/bazel fi + - name: Provision package-lock-pinned Chromium + run: | + set -euo pipefail + # Shared-cache actions execute on this ARC runner, not in the GF + # executor image where /bin/chromium is part of the worker contract. + # Install the browser revision pinned by @playwright/test outside the + # Bazel action, then pass its absolute path into browser tests. + npm ci --ignore-scripts + npx --no-install playwright install --with-deps chromium + chromium_path="$(node --input-type=module -e "import { chromium } from '@playwright/test'; process.stdout.write(chromium.executablePath())")" + if [[ "${chromium_path}" != /* || ! -x "${chromium_path}" ]]; then + echo "::error::Playwright Chromium resolution produced an unusable path" + exit 1 + fi + echo "GF_RBE_CHROMIUM_EXECUTABLE=${chromium_path}" >> "${GITHUB_ENV}" + - name: Remote check run: npm run remote:check diff --git a/AGENTS.md b/AGENTS.md index 252dd09..13ee45b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,7 @@ These rules apply before the writing-style guidance below. - GloriousFlywheel-backed validation runs through `scripts/bazel-cache-backed.sh`. This repository is registered as `shared-cache-backed`; the remote scripts must fail closed when `BAZEL_REMOTE_CACHE` or that expected substrate mode is missing. CI must clear generic-runner executor hints. Executor-backed mode remains an explicit opt-in only after a reviewed GloriousFlywheel consumer-registry promotion. - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` are the authoritative Bazel entrypoints. They cover SvelteKit check/build smoke, package checks, CV PDF sync, Vitest, blog-agent tests, browser smoke, Playwright e2e, and graph hygiene. - `package-lock.json` is the npm authority. `pnpm-lock.yaml` is the generated `rules_js` lock consumed by Bazel; do not casually hand-edit it. -- Browser-backed Bazel tests use the pinned worker Chromium path. Do not add lifecycle downloads or host-local browser assumptions to the RBE path. +- Shared-cache CI provisions the package-lock-pinned Playwright Chromium outside Bazel actions and passes its absolute path explicitly. Browser-backed RBE uses the pinned worker Chromium path; do not add lifecycle downloads or host-local browser assumptions inside the RBE action path. - Broad `bazel query //...` and `bazel test //...` must not traverse agent scratch trees. `.bazelignore` intentionally excludes `.claude`, `.gstack`, `.worktrees`, build outputs, caches, and `workers/dns-guard/.wrangler`. - If you touch Bazel structure, scratch-tree ignores, or broad query behavior, run `npm run test:bazel-graph-hygiene` and, when possible, `bazelisk --output_user_root=/tmp/jess-ghio-bazel-codex test //:bazel_graph_hygiene`. diff --git a/README.md b/README.md index db7b217..a6d01b3 100644 --- a/README.md +++ b/README.md @@ -116,11 +116,11 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `//:vitest_unit_tests` wraps the root and Pulse Vitest suites through `vitest.bazel.config.ts`. - `//:blog_agent_node_tests` wraps the blog-agent `node:test` suite through `tsx --test`. - `//:sveltekit_vite_build_smoke` runs a copied-workdir SvelteKit/Vite production build smoke. It proves the build target class, not the full npm prebuild/postbuild publication chain. -- `//:playwright_chromium_e2e` runs the Chromium Playwright e2e suite through Bazel against the pinned GloriousFlywheel runner Chromium path. +- `//:playwright_chromium_e2e` runs the Chromium Playwright e2e suite through Bazel. Shared-cache CI provisions the package-lock-pinned Playwright browser before Bazel starts and passes its absolute executable path into the local test action. - `//:playwright_chromium_smoke` remains a narrow diagnostic target for browser runtime authority, not the remote e2e gate. -- `//:puppeteer_chromium_smoke` launches Puppeteer against the same pinned Chromium runtime path. It proves Puppeteer can consume browser runtime authority without lifecycle downloads. +- `//:puppeteer_chromium_smoke` launches Puppeteer against the same pinned Chromium runtime path. It proves Puppeteer and Playwright consume one explicit browser authority rather than relying on an undeclared host path. - `package-lock.json` remains the npm dependency authority for the app. `pnpm-workspace.yaml` makes the package importers explicit for Bazel, and `pnpm-lock.yaml` is the generated `rules_js` lock consumed by Bazel. -- Bazel npm lifecycle hooks skip Playwright and Puppeteer browser downloads. Browser-backed RBE must use the pinned worker Chromium path rather than downloading browsers during proof actions. +- Bazel npm lifecycle hooks skip Playwright and Puppeteer browser downloads. Shared-cache CI provisions the lockfile-pinned Playwright browser outside Bazel actions; browser-backed RBE continues to use the pinned worker Chromium path and never downloads a browser inside proof actions. - GloriousFlywheel proof runs use the external GF REAPI proof harness or the repository `remote:*` scripts against this public repo checkout; remote cache hits, hosted runners, and shared-cache-only execution do not count as RBE. Current boundary: this gates Bazel check/test/Chromium-e2e and CV PDF sync drift with GloriousFlywheel shared-cache attachment. It does not claim remote action execution. Deployment still publishes the SvelteKit static artifact through the existing Pages workflows. diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index a65acfb..803b34c 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -85,6 +85,7 @@ executor_args=() credential_args=() routing_args=() upload_args=() +browser_args=() remote_upload="${GF_BAZEL_REMOTE_UPLOAD:-true}" endpoint_host() { @@ -196,6 +197,10 @@ info) validate_runtime_value "BAZEL_REMOTE_INSTANCE_NAME" "${BAZEL_REMOTE_INSTANCE_NAME}" routing_args+=(--remote_instance_name="${BAZEL_REMOTE_INSTANCE_NAME}") fi + if [[ ${command} == test || ${command} == coverage ]] && [[ -n ${GF_RBE_CHROMIUM_EXECUTABLE:-} ]]; then + validate_runtime_value "GF_RBE_CHROMIUM_EXECUTABLE" "${GF_RBE_CHROMIUM_EXECUTABLE}" + browser_args+=(--test_env="GF_RBE_CHROMIUM_EXECUTABLE=${GF_RBE_CHROMIUM_EXECUTABLE}") + fi # Bounded retry on the gf-reapi-cell per-tenant quota (Bazel exit 34, # RESOURCE_EXHAUSTED: concurrent-execution limit). Mirrors the @@ -215,6 +220,7 @@ info) "${credential_args[@]}" \ "${routing_args[@]}" \ "${external_fetch_args[@]}" \ + "${browser_args[@]}" \ "$@" || rc=$? if [[ ${rc} -ne 34 || ${attempt} -ge ${quota_attempts} ]]; then exit "${rc}" diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index 6f61076..959f6f1 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -28,10 +28,12 @@ run_upload_case() { BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="${value}" \ + GF_RBE_CHROMIUM_EXECUTABLE="/opt/pinned-chromium" \ FAKE_BAZEL_ARGS="${args_file}" \ bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null + grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null } run_upload_case false false From 35124c688dabe65934710df2f8d361904a22bd12 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 06:09:12 -0400 Subject: [PATCH 04/13] fix(ci): provision browser before credentials --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d610335..b5afcef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,22 +142,6 @@ jobs: sudo mv "${tmp}" /usr/local/bin/bazel fi - - name: Provision package-lock-pinned Chromium - run: | - set -euo pipefail - # Shared-cache actions execute on this ARC runner, not in the GF - # executor image where /bin/chromium is part of the worker contract. - # Install the browser revision pinned by @playwright/test outside the - # Bazel action, then pass its absolute path into browser tests. - npm ci --ignore-scripts - npx --no-install playwright install --with-deps chromium - chromium_path="$(node --input-type=module -e "import { chromium } from '@playwright/test'; process.stdout.write(chromium.executablePath())")" - if [[ "${chromium_path}" != /* || ! -x "${chromium_path}" ]]; then - echo "::error::Playwright Chromium resolution produced an unusable path" - exit 1 - fi - echo "GF_RBE_CHROMIUM_EXECUTABLE=${chromium_path}" >> "${GITHUB_ENV}" - - name: Remote check run: npm run remote:check From 3aa2a382e0993bcdaf7744034323b1d6f7f3abfa Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 06:19:56 -0400 Subject: [PATCH 05/13] docs(ci): retire stale RBE labels --- .github/workflows/ci.yml | 2 +- README.md | 2 +- scripts/run-playwright-chromium-bazel.mjs | 6 +++--- scripts/run-puppeteer-chromium-bazel.mjs | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5afcef..cc348ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,7 @@ jobs: - name: Test authority run: | echo "check/test/e2e are intentionally owned by bazel-remote-gates." - echo "The GloriousFlywheel RBE lane runs remote:check, remote:test, and remote:e2e." + echo "The GloriousFlywheel shared-cache lane runs remote:check, remote:test, and remote:e2e." - name: Blog agent production dependency audit (report-only) run: | diff --git a/README.md b/README.md index a6d01b3..b2ac7a7 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. - The token exchange supplies the repository-scoped `BAZEL_REMOTE_INSTANCE_NAME`. Literal shell placeholders are rejected before Bazel starts. - `//:sveltekit_check` runs the SvelteKit check path under Bazel. -- `//static/cv:pdfs_synced_test` byte-compares the checked-in resume/CV PDFs against the Bazel-built `spear_resumes` outputs; `.bazelrc` pins `SOURCE_DATE_EPOCH` and `TZ` so Tectonic output stays reproducible across local sync, CI, and GF RBE. +- `//static/cv:pdfs_synced_test` byte-compares the checked-in resume/CV PDFs against the Bazel-built `spear_resumes` outputs; `.bazelrc` pins `SOURCE_DATE_EPOCH` and `TZ` so Tectonic output stays reproducible across local sync, shared-cache CI, and explicit executor proofs. - `//:vitest_unit_tests` wraps the root and Pulse Vitest suites through `vitest.bazel.config.ts`. - `//:blog_agent_node_tests` wraps the blog-agent `node:test` suite through `tsx --test`. - `//:sveltekit_vite_build_smoke` runs a copied-workdir SvelteKit/Vite production build smoke. It proves the build target class, not the full npm prebuild/postbuild publication chain. diff --git a/scripts/run-playwright-chromium-bazel.mjs b/scripts/run-playwright-chromium-bazel.mjs index 4cea91a..01d2a58 100644 --- a/scripts/run-playwright-chromium-bazel.mjs +++ b/scripts/run-playwright-chromium-bazel.mjs @@ -38,7 +38,7 @@ try { const title = await page.locator('main h1').textContent(); const target = await page.locator('[data-rbe-target]').textContent(); const headingBox = await page.locator('main h1').boundingBox(); - if (title !== 'GloriousFlywheel browser RBE smoke') { + if (title !== 'GloriousFlywheel browser smoke') { throw new Error(`unexpected smoke title: ${title}`); } if (target !== packageJson.name) { @@ -66,11 +66,11 @@ function renderSmokePage(packageName) { - Browser RBE smoke + Browser smoke
-

GloriousFlywheel browser RBE smoke

+

GloriousFlywheel browser smoke

${escapeHtml(packageName)}

diff --git a/scripts/run-puppeteer-chromium-bazel.mjs b/scripts/run-puppeteer-chromium-bazel.mjs index 5555888..f2c0491 100644 --- a/scripts/run-puppeteer-chromium-bazel.mjs +++ b/scripts/run-puppeteer-chromium-bazel.mjs @@ -39,7 +39,7 @@ try { const rect = element.getBoundingClientRect(); return { width: rect.width, height: rect.height }; }); - if (title !== 'GloriousFlywheel Puppeteer browser RBE smoke') { + if (title !== 'GloriousFlywheel Puppeteer browser smoke') { throw new Error(`unexpected smoke title: ${title}`); } if (target !== packageJson.name) { @@ -67,11 +67,11 @@ function renderSmokePage(packageName) { - Puppeteer Browser RBE smoke + Puppeteer Browser smoke
-

GloriousFlywheel Puppeteer browser RBE smoke

+

GloriousFlywheel Puppeteer browser smoke

${escapeHtml(packageName)}

From 48cf2cfbf99680cc8ae93acbf7e773019dba771c Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 07:07:01 -0400 Subject: [PATCH 06/13] fix(ci): keep shared-cache actions hermetic --- .bazelrc | 8 -------- README.md | 1 - 2 files changed, 9 deletions(-) diff --git a/.bazelrc b/.bazelrc index 431eeaf..c6d275a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,14 +39,6 @@ build:ci-remote-base --remote_timeout=60 build:ci-cached --config=ci-remote-base build:ci-cached --remote_cache_compression -# Shared-cache actions execute on the runner. Give tools such as Tectonic -# stable writable cache/config paths instead of inheriting the ARC image's -# read-only /root home. -build:ci-cached --action_env=HOME=/tmp -build:ci-cached --action_env=TMPDIR=/tmp -build:ci-cached --action_env=XDG_CACHE_HOME=/tmp/xdg-cache -build:ci-cached --action_env=XDG_CONFIG_HOME=/tmp/xdg-config -build:ci-cached --action_env=XDG_DATA_HOME=/tmp/xdg-data build:executor-backed --config=ci-remote-base build:executor-backed --action_env=HOME=/var/lib/gf-reapi-cell/home diff --git a/README.md b/README.md index b2ac7a7..6418830 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,6 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache actions execute locally on the runner with stable writable `/tmp` HOME/XDG paths. This keeps Tectonic and other cache-using tools off the ARC image's read-only `/root` without introducing runner-specific paths into Bazel action keys. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. - The token exchange supplies the repository-scoped `BAZEL_REMOTE_INSTANCE_NAME`. Literal shell placeholders are rejected before Bazel starts. From 23272812ea4d6a598b10fa05be583ef369d24006 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 08:51:00 -0400 Subject: [PATCH 07/13] fix(ci): bound cold shared-cache builds --- .bazelrc | 4 ++++ .github/workflows/ci.yml | 15 +++++++++++++++ README.md | 2 ++ scripts/bazel-cache-backed.sh | 22 ++++++++++++++++++++++ scripts/test-bazel-cache-backed.sh | 24 ++++++++++++++++++++++++ 5 files changed, 67 insertions(+) diff --git a/.bazelrc b/.bazelrc index c6d275a..517df3a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,6 +39,10 @@ build:ci-remote-base --remote_timeout=60 build:ci-cached --config=ci-remote-base build:ci-cached --remote_cache_compression +# Shared-cache actions execute inside the ARC pod. Bound local parallelism so +# a cold graph cannot launch dozens of compilers and starve its own test +# processes; executor-backed proofs retain their separately managed capacity. +build:ci-cached --jobs=4 build:executor-backed --config=ci-remote-base build:executor-backed --action_env=HOME=/var/lib/gf-reapi-cell/home diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc348ba..0b91fac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: env: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} + TECTONIC_CACHE_DIR: /tmp/tectonic-cache-v33 # The GF consumer registry records this repo as shared-cache-backed. # Clear any executor endpoint inherited from the generic ARC pod. BAZEL_REMOTE_EXECUTOR: '' @@ -65,6 +66,13 @@ jobs: path: ~/.cache/ms-playwright key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - name: Cache Tectonic resources + uses: actions/cache@v5 + with: + path: /tmp/tectonic-cache-v33 + key: tectonic-v33-${{ runner.os }}-${{ hashFiles('MODULE.bazel') }} + restore-keys: tectonic-v33-${{ runner.os }}- + - name: Provision package-lock-pinned Chromium run: | set -euo pipefail @@ -142,6 +150,13 @@ jobs: sudo mv "${tmp}" /usr/local/bin/bazel fi + - name: Warm Tectonic resources serially + run: | + bash scripts/bazel-cache-backed.sh build --jobs=1 \ + @spear_resumes//generic:resume \ + @spear_resumes//generic:precis \ + @spear_resumes//generic:cv + - name: Remote check run: npm run remote:check diff --git a/README.md b/README.md index 6418830..33359fb 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. +- Shared-cache CI bounds local Bazel parallelism to four jobs. This protects the ARC pod from cold-graph process storms without changing the independently managed capacity of explicit executor-backed proofs. +- The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. - The token exchange supplies the repository-scoped `BAZEL_REMOTE_INSTANCE_NAME`. Literal shell placeholders are rejected before Bazel starts. diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index 803b34c..4055bcd 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -31,6 +31,9 @@ Environment: property; defaults to gloriousflywheel-rbe-linux-x86_64. GF_BAZEL_REMOTE_UPLOAD controls remote result uploads. Set it to false for cache-read-only refs and true for trusted cache-write refs; defaults to true. + TECTONIC_CACHE_DIR optionally names an absolute, runner-local cache for + shared-cache builds. The wrapper makes it writable inside Bazel sandboxes; + executor-backed mode rejects this host path. GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE or GF_REAPI_CREDENTIAL_HELPER_TOKEN must be set when a gf-reapi-cell endpoint requires JWT auth. The helper also uses /var/run/secrets/tokens/gf-reapi-cell-token when present. @@ -86,7 +89,9 @@ credential_args=() routing_args=() upload_args=() browser_args=() +local_action_args=() remote_upload="${GF_BAZEL_REMOTE_UPLOAD:-true}" +tectonic_cache_dir="${TECTONIC_CACHE_DIR:-}" endpoint_host() { local endpoint="$1" @@ -201,6 +206,22 @@ info) validate_runtime_value "GF_RBE_CHROMIUM_EXECUTABLE" "${GF_RBE_CHROMIUM_EXECUTABLE}" browser_args+=(--test_env="GF_RBE_CHROMIUM_EXECUTABLE=${GF_RBE_CHROMIUM_EXECUTABLE}") fi + if [[ -n ${tectonic_cache_dir} ]]; then + validate_runtime_value "TECTONIC_CACHE_DIR" "${tectonic_cache_dir}" + if [[ ${tectonic_cache_dir} != /* ]]; then + echo "ERROR: TECTONIC_CACHE_DIR must be an absolute path; got ${tectonic_cache_dir}." >&2 + exit 2 + fi + if [[ -n ${remote_executor} ]]; then + echo "ERROR: TECTONIC_CACHE_DIR is runner-local and cannot be used in executor-backed mode." >&2 + exit 2 + fi + mkdir -p "${tectonic_cache_dir}" + local_action_args+=( + --action_env="TECTONIC_CACHE_DIR=${tectonic_cache_dir}" + --sandbox_writable_path="${tectonic_cache_dir}" + ) + fi # Bounded retry on the gf-reapi-cell per-tenant quota (Bazel exit 34, # RESOURCE_EXHAUSTED: concurrent-execution limit). Mirrors the @@ -221,6 +242,7 @@ info) "${routing_args[@]}" \ "${external_fetch_args[@]}" \ "${browser_args[@]}" \ + "${local_action_args[@]}" \ "$@" || rc=$? if [[ ${rc} -ne 34 || ${attempt} -ge ${quota_attempts} ]]; then exit "${rc}" diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index 959f6f1..2d824db 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -29,11 +29,14 @@ run_upload_case() { GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="${value}" \ GF_RBE_CHROMIUM_EXECUTABLE="/opt/pinned-chromium" \ + TECTONIC_CACHE_DIR="${tmp_dir}/tectonic-cache" \ FAKE_BAZEL_ARGS="${args_file}" \ bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null + grep -Fx -- "--action_env=TECTONIC_CACHE_DIR=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null + grep -Fx -- "--sandbox_writable_path=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null } run_upload_case false false @@ -48,3 +51,24 @@ if BAZEL_BIN="${fake_bazel}" \ echo "ERROR: invalid GF_BAZEL_REMOTE_UPLOAD unexpectedly succeeded" >&2 exit 1 fi + +if BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ + TECTONIC_CACHE_DIR="relative/cache" \ + FAKE_BAZEL_ARGS="${tmp_dir}/relative.args" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>&1; then + echo "ERROR: relative TECTONIC_CACHE_DIR unexpectedly succeeded" >&2 + exit 1 +fi + +if BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + BAZEL_REMOTE_EXECUTOR="grpc://executor.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="executor-backed" \ + TECTONIC_CACHE_DIR="${tmp_dir}/executor-cache" \ + FAKE_BAZEL_ARGS="${tmp_dir}/executor.args" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>&1; then + echo "ERROR: runner-local TECTONIC_CACHE_DIR unexpectedly reached executor-backed mode" >&2 + exit 1 +fi From a58489c2d7dabbc59cca0351b82be99cec7438b5 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 12:02:55 -0400 Subject: [PATCH 08/13] fix(ci): serialize local test actions --- .bazelrc | 3 +++ BUILD.bazel | 1 + README.md | 2 +- scripts/test-bazel-cache-backed.sh | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 517df3a..d83965a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -43,6 +43,9 @@ build:ci-cached --remote_cache_compression # a cold graph cannot launch dozens of compilers and starve its own test # processes; executor-backed proofs retain their separately managed capacity. build:ci-cached --jobs=4 +# Browser-backed test actions execute inside the 4 GiB ARC pod in this mode. +# Keep them serial so concurrent Chromium processes cannot OOM the runner. +test:ci-cached --local_test_jobs=1 build:executor-backed --config=ci-remote-base build:executor-backed --action_env=HOME=/var/lib/gf-reapi-cell/home diff --git a/BUILD.bazel b/BUILD.bazel index d9e6135..bd969f8 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -297,6 +297,7 @@ sh_test( name = "bazel_cache_backed_contract", srcs = ["scripts/test-bazel-cache-backed.sh"], data = [ + ".bazelrc", "scripts/bazel-cache-backed.sh", "scripts/cache-attachment-contract.sh", ], diff --git a/README.md b/README.md index 33359fb..d88acc7 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache CI bounds local Bazel parallelism to four jobs. This protects the ARC pod from cold-graph process storms without changing the independently managed capacity of explicit executor-backed proofs. +- Shared-cache CI bounds local Bazel parallelism to four jobs and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. - The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index 2d824db..fef2c88 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -10,6 +10,9 @@ fi tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/blog-bazel-cache-contract.XXXXXX")" trap 'rm -rf "${tmp_dir}"' EXIT +grep -Fx -- "build:ci-cached --jobs=4" "${repo_root}/.bazelrc" >/dev/null +grep -Fx -- "test:ci-cached --local_test_jobs=1" "${repo_root}/.bazelrc" >/dev/null + fake_bazel="${tmp_dir}/bazel" cat >"${fake_bazel}" <<'EOF' #!/usr/bin/env bash From 576f2acfb238cb8640a038f00095b8b77bab66bf Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 12:45:00 -0400 Subject: [PATCH 09/13] fix(ci): fit shared-cache work inside runner memory --- .bazelrc | 12 +++++++----- .github/workflows/ci.yml | 4 +++- README.md | 2 +- scripts/test-bazel-cache-backed.sh | 3 ++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.bazelrc b/.bazelrc index d83965a..91a4aa4 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,11 +39,13 @@ build:ci-remote-base --remote_timeout=60 build:ci-cached --config=ci-remote-base build:ci-cached --remote_cache_compression -# Shared-cache actions execute inside the ARC pod. Bound local parallelism so -# a cold graph cannot launch dozens of compilers and starve its own test -# processes; executor-backed proofs retain their separately managed capacity. -build:ci-cached --jobs=4 -# Browser-backed test actions execute inside the 4 GiB ARC pod in this mode. +# Shared-cache actions execute inside a 2-CPU, 4 GiB ARC runner container. +# Bound both action parallelism and per-process Node heap so one local action +# plus Bazel overhead remains inside that hard memory limit. Executor-backed +# proofs retain their separately managed capacity. +build:ci-cached --jobs=1 +build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1536 +# Browser-backed test actions execute inside the same 4 GiB ARC pod in this mode. # Keep them serial so concurrent Chromium processes cannot OOM the runner. test:ci-cached --local_test_jobs=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b91fac..1c31c77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,9 @@ jobs: contents: read id-token: write runs-on: tinyland-dind - timeout-minutes: 60 + # Shared-cache actions execute locally inside a 2-CPU, 4 GiB ARC pod. + # The serial memory-safe lane trades wall-clock speed for runner survival. + timeout-minutes: 90 env: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} diff --git a/README.md b/README.md index d88acc7..1153ed2 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache CI bounds local Bazel parallelism to four jobs and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. +- Shared-cache CI runs one local Bazel action at a time, caps each Node action at a 1536 MiB heap, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. - The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index fef2c88..e2b7c04 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -10,7 +10,8 @@ fi tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/blog-bazel-cache-contract.XXXXXX")" trap 'rm -rf "${tmp_dir}"' EXIT -grep -Fx -- "build:ci-cached --jobs=4" "${repo_root}/.bazelrc" >/dev/null +grep -Fx -- "build:ci-cached --jobs=1" "${repo_root}/.bazelrc" >/dev/null +grep -Fx -- "build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1536" "${repo_root}/.bazelrc" >/dev/null grep -Fx -- "test:ci-cached --local_test_jobs=1" "${repo_root}/.bazelrc" >/dev/null fake_bazel="${tmp_dir}/bazel" From 56a045942721cad2863690cedff851fe70804f3d Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 13:10:49 -0400 Subject: [PATCH 10/13] fix(ci): cap Bazel server memory --- .github/workflows/ci.yml | 1 + README.md | 2 +- scripts/bazel-cache-backed.sh | 17 +++++++++++++++-- scripts/test-bazel-cache-backed.sh | 12 ++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c31c77..e546f87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: env: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} + GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '1024' TECTONIC_CACHE_DIR: /tmp/tectonic-cache-v33 # The GF consumer registry records this repo as shared-cache-backed. # Clear any executor endpoint inherited from the generic ARC pod. diff --git a/README.md b/README.md index 1153ed2..07caed1 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache CI runs one local Bazel action at a time, caps each Node action at a 1536 MiB heap, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. +- Shared-cache CI runs one local Bazel action at a time, caps the Bazel host JVM at 1024 MiB and each Node action at 1536 MiB, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. - The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index 4055bcd..751f908 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -31,6 +31,8 @@ Environment: property; defaults to gloriousflywheel-rbe-linux-x86_64. GF_BAZEL_REMOTE_UPLOAD controls remote result uploads. Set it to false for cache-read-only refs and true for trusted cache-write refs; defaults to true. + GF_BAZEL_HOST_JVM_MAX_HEAP_MB optionally caps the Bazel server JVM heap in + MiB. CI sets this explicitly to fit the shared-cache runner memory limit. TECTONIC_CACHE_DIR optionally names an absolute, runner-local cache for shared-cache builds. The wrapper makes it writable inside Bazel sandboxes; executor-backed mode rejects this host path. @@ -90,8 +92,10 @@ routing_args=() upload_args=() browser_args=() local_action_args=() +startup_args=() remote_upload="${GF_BAZEL_REMOTE_UPLOAD:-true}" tectonic_cache_dir="${TECTONIC_CACHE_DIR:-}" +host_jvm_max_heap_mb="${GF_BAZEL_HOST_JVM_MAX_HEAP_MB:-}" endpoint_host() { local endpoint="$1" @@ -144,6 +148,15 @@ validate_runtime_value() { fi } +if [[ -n ${host_jvm_max_heap_mb} ]]; then + if [[ ! ${host_jvm_max_heap_mb} =~ ^[0-9]+$ ]] || + ((host_jvm_max_heap_mb < 256 || host_jvm_max_heap_mb > 4096)); then + echo "ERROR: GF_BAZEL_HOST_JVM_MAX_HEAP_MB must be an integer from 256 through 4096; got ${host_jvm_max_heap_mb}." >&2 + exit 2 + fi + startup_args+=(--host_jvm_args="-Xmx${host_jvm_max_heap_mb}m") +fi + if [[ -n ${BAZEL_REPOSITORY_CACHE:-} ]]; then external_fetch_args+=(--repository_cache="${BAZEL_REPOSITORY_CACHE}") fi @@ -179,7 +192,7 @@ fi case "${command}" in info) - exec "${bazel_bin}" info "${external_fetch_args[@]}" "$@" + exec "${bazel_bin}" "${startup_args[@]}" info "${external_fetch_args[@]}" "$@" ;; *) bazel_config="ci-cached" @@ -233,7 +246,7 @@ info) attempt=1 while :; do rc=0 - "${bazel_bin}" "${command}" \ + "${bazel_bin}" "${startup_args[@]}" "${command}" \ --config="${bazel_config}" \ --remote_cache="${effective_remote_cache}" \ "${upload_args[@]}" \ diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index e2b7c04..5d7e856 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -32,12 +32,14 @@ run_upload_case() { BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="${value}" \ + GF_BAZEL_HOST_JVM_MAX_HEAP_MB="1024" \ GF_RBE_CHROMIUM_EXECUTABLE="/opt/pinned-chromium" \ TECTONIC_CACHE_DIR="${tmp_dir}/tectonic-cache" \ FAKE_BAZEL_ARGS="${args_file}" \ bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null + grep -Fx -- "--host_jvm_args=-Xmx1024m" "${args_file}" >/dev/null grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null grep -Fx -- "--action_env=TECTONIC_CACHE_DIR=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null grep -Fx -- "--sandbox_writable_path=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null @@ -46,6 +48,16 @@ run_upload_case() { run_upload_case false false run_upload_case true true +if BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ + GF_BAZEL_HOST_JVM_MAX_HEAP_MB="unbounded" \ + FAKE_BAZEL_ARGS="${tmp_dir}/invalid-heap.args" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>&1; then + echo "ERROR: invalid GF_BAZEL_HOST_JVM_MAX_HEAP_MB unexpectedly succeeded" >&2 + exit 1 +fi + if BAZEL_BIN="${fake_bazel}" \ BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ From de941d885d60f714b6326b502d5772f69faa4d47 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 14:28:13 -0400 Subject: [PATCH 11/13] fix(ci): rebalance shared-cache memory --- .bazelrc | 2 +- .github/workflows/ci.yml | 2 +- README.md | 2 +- scripts/test-bazel-cache-backed.sh | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bazelrc b/.bazelrc index 91a4aa4..1872832 100644 --- a/.bazelrc +++ b/.bazelrc @@ -44,7 +44,7 @@ build:ci-cached --remote_cache_compression # plus Bazel overhead remains inside that hard memory limit. Executor-backed # proofs retain their separately managed capacity. build:ci-cached --jobs=1 -build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1536 +build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1024 # Browser-backed test actions execute inside the same 4 GiB ARC pod in this mode. # Keep them serial so concurrent Chromium processes cannot OOM the runner. test:ci-cached --local_test_jobs=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e546f87..aadb96a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: env: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} - GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '1024' + GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '1536' TECTONIC_CACHE_DIR: /tmp/tectonic-cache-v33 # The GF consumer registry records this repo as shared-cache-backed. # Clear any executor endpoint inherited from the generic ARC pod. diff --git a/README.md b/README.md index 07caed1..27124b5 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache CI runs one local Bazel action at a time, caps the Bazel host JVM at 1024 MiB and each Node action at 1536 MiB, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. +- Shared-cache CI runs one local Bazel action at a time, caps the Bazel host JVM at 1536 MiB and each Node action at 1024 MiB, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure while leaving enough JVM heap to analyze the full workspace graph, without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. - The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index 5d7e856..487fed0 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -11,7 +11,7 @@ tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/blog-bazel-cache-contract.XXXXXX")" trap 'rm -rf "${tmp_dir}"' EXIT grep -Fx -- "build:ci-cached --jobs=1" "${repo_root}/.bazelrc" >/dev/null -grep -Fx -- "build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1536" "${repo_root}/.bazelrc" >/dev/null +grep -Fx -- "build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1024" "${repo_root}/.bazelrc" >/dev/null grep -Fx -- "test:ci-cached --local_test_jobs=1" "${repo_root}/.bazelrc" >/dev/null fake_bazel="${tmp_dir}/bazel" @@ -32,14 +32,14 @@ run_upload_case() { BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="${value}" \ - GF_BAZEL_HOST_JVM_MAX_HEAP_MB="1024" \ + GF_BAZEL_HOST_JVM_MAX_HEAP_MB="1536" \ GF_RBE_CHROMIUM_EXECUTABLE="/opt/pinned-chromium" \ TECTONIC_CACHE_DIR="${tmp_dir}/tectonic-cache" \ FAKE_BAZEL_ARGS="${args_file}" \ bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null - grep -Fx -- "--host_jvm_args=-Xmx1024m" "${args_file}" >/dev/null + grep -Fx -- "--host_jvm_args=-Xmx1536m" "${args_file}" >/dev/null grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null grep -Fx -- "--action_env=TECTONIC_CACHE_DIR=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null grep -Fx -- "--sandbox_writable_path=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null From e51d8279870311881caa08cc2cc8d1b870dd1fb0 Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 14:52:37 -0400 Subject: [PATCH 12/13] fix(ci): bound Bazel phase memory --- .bazelrc | 4 ++-- .github/workflows/ci.yml | 19 +++++++++++++------ README.md | 2 +- scripts/bazel-cache-backed.sh | 6 +++++- scripts/test-bazel-cache-backed.sh | 30 ++++++++++++++++++++++++------ 5 files changed, 45 insertions(+), 16 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1872832..1173480 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,13 +39,13 @@ build:ci-remote-base --remote_timeout=60 build:ci-cached --config=ci-remote-base build:ci-cached --remote_cache_compression -# Shared-cache actions execute inside a 2-CPU, 4 GiB ARC runner container. +# Shared-cache actions execute inside a bounded ARC runner container. # Bound both action parallelism and per-process Node heap so one local action # plus Bazel overhead remains inside that hard memory limit. Executor-backed # proofs retain their separately managed capacity. build:ci-cached --jobs=1 build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1024 -# Browser-backed test actions execute inside the same 4 GiB ARC pod in this mode. +# Browser-backed test actions execute inside the same ARC pod in this mode. # Keep them serial so concurrent Chromium processes cannot OOM the runner. test:ci-cached --local_test_jobs=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aadb96a..b6f2cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,13 +41,13 @@ jobs: contents: read id-token: write runs-on: tinyland-dind - # Shared-cache actions execute locally inside a 2-CPU, 4 GiB ARC pod. - # The serial memory-safe lane trades wall-clock speed for runner survival. + # Shared-cache actions execute locally inside a fleet-managed ARC pod. + # Keep the repo process envelope bounded independently of fleet sizing. timeout-minutes: 90 env: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} - GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '1536' + GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '2560' TECTONIC_CACHE_DIR: /tmp/tectonic-cache-v33 # The GF consumer registry records this repo as shared-cache-backed. # Clear any executor endpoint inherited from the generic ARC pod. @@ -159,15 +159,22 @@ jobs: @spear_resumes//generic:resume \ @spear_resumes//generic:precis \ @spear_resumes//generic:cv + bash scripts/bazel-cache-backed.sh shutdown - name: Remote check - run: npm run remote:check + run: | + npm run remote:check + bash scripts/bazel-cache-backed.sh shutdown - name: Remote test - run: npm run remote:test + run: | + npm run remote:test + bash scripts/bazel-cache-backed.sh shutdown - name: Remote e2e - run: npm run remote:e2e + run: | + npm run remote:e2e + bash scripts/bazel-cache-backed.sh shutdown build-and-test: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 27124b5..54a61ba 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ This repo still uses the npm/SvelteKit workflow for normal local development and - `npm run remote:check`, `npm run remote:test`, and `npm run remote:e2e` route through `scripts/bazel-cache-backed.sh`, which refuses to run without a valid `BAZEL_REMOTE_CACHE` and the expected GloriousFlywheel substrate mode. - Local developer shells attach through the endpoint-free GloriousFlywheel front-door kit (`justfile.flywheel` plus `.bazelrc.flywheel`). The managed Nix/Home Manager profile is the preferred source of `BAZEL_REMOTE_CACHE` and auth metadata; a gitignored `.env.flywheel.local` generated by `just flywheel-enroll ...` is only the fallback fixture, and is sourced by both `.envrc` and `scripts/bazel-cache-backed.sh`. - GitHub CI runs the Bazel lane on `tinyland-dind` as the `shared-cache-backed` consumer recorded in the GloriousFlywheel registry. Pull requests mint cache-read tokens and disable result uploads; trusted `main` pushes mint cache-write tokens. Generic-runner executor hints are cleared so the workflow cannot silently claim executor-backed behavior. -- Shared-cache CI runs one local Bazel action at a time, caps the Bazel host JVM at 1536 MiB and each Node action at 1024 MiB, and serializes local test actions. This protects the 4 GiB ARC pod from cold-graph and concurrent-browser memory pressure while leaving enough JVM heap to analyze the full workspace graph, without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. +- Shared-cache CI runs one local Bazel action at a time, caps the Bazel host JVM at 2560 MiB and each Node action at 1024 MiB, and serializes local test actions. It shuts the Bazel server down between the Tectonic warm, check, test, and e2e phases so retained analysis state cannot grow across the entire job. This keeps the repo process envelope below fleet-managed ARC capacity while leaving enough JVM heap to analyze the full workspace graph, without changing the independently managed capacity of explicit executor-backed proofs. The hosted gate allows 90 minutes because this reliability-first envelope deliberately trades wall-clock speed for runner survival. - The lane restores one v33 Tectonic resource cache, warms the three public CV targets serially, and explicitly makes that cache writable inside local Bazel sandboxes. This avoids concurrent bundle fetches; the runner-local path is rejected in executor-backed mode. - Executor-backed mode remains available in the wrapper only as a separate opt-in contract. It requires a reviewed GloriousFlywheel registry promotion plus an explicit executor endpoint; this repository's normal CI does not use it. - `gf-reapi-cell` endpoints also require scoped Bazel credential-helper auth. `scripts/bazel-cache-backed.sh` attaches `scripts/gf-reapi-bazel-credential-helper.mjs` only for the GF REAPI host, and the helper reads a short-lived JWT from `GF_REAPI_CREDENTIAL_HELPER_TOKEN_FILE`, `GF_REAPI_CREDENTIAL_HELPER_TOKEN`, or the projected-token file at `/var/run/secrets/tokens/gf-reapi-cell-token`. diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index 751f908..a1438f1 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -18,6 +18,7 @@ Commands: run Run a cache-backed Bazel target coverage Run cache-backed Bazel coverage info Validate cache attachment, then run bazel info + shutdown Validate cache attachment, then stop the matching Bazel server Environment: BAZEL_REMOTE_CACHE must be a real grpc://, grpcs://, http://, or https:// endpoint. @@ -57,7 +58,7 @@ command="$1" shift case "${command}" in -build | test | run | coverage | info) ;; +build | test | run | coverage | info | shutdown) ;; -h | --help) usage exit 0 @@ -194,6 +195,9 @@ case "${command}" in info) exec "${bazel_bin}" "${startup_args[@]}" info "${external_fetch_args[@]}" "$@" ;; +shutdown) + exec "${bazel_bin}" "${startup_args[@]}" shutdown "$@" + ;; *) bazel_config="ci-cached" effective_remote_cache="${remote_cache}" diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index 487fed0..b4de868 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -32,14 +32,14 @@ run_upload_case() { BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="${value}" \ - GF_BAZEL_HOST_JVM_MAX_HEAP_MB="1536" \ + GF_BAZEL_HOST_JVM_MAX_HEAP_MB="2560" \ GF_RBE_CHROMIUM_EXECUTABLE="/opt/pinned-chromium" \ TECTONIC_CACHE_DIR="${tmp_dir}/tectonic-cache" \ FAKE_BAZEL_ARGS="${args_file}" \ bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null grep -Fx -- "--remote_upload_local_results=${expected}" "${args_file}" >/dev/null - grep -Fx -- "--host_jvm_args=-Xmx1536m" "${args_file}" >/dev/null + grep -Fx -- "--host_jvm_args=-Xmx2560m" "${args_file}" >/dev/null grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null grep -Fx -- "--action_env=TECTONIC_CACHE_DIR=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null grep -Fx -- "--sandbox_writable_path=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null @@ -48,43 +48,61 @@ run_upload_case() { run_upload_case false false run_upload_case true true +shutdown_args="${tmp_dir}/shutdown.args" +BAZEL_BIN="${fake_bazel}" \ + BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ + GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ + GF_BAZEL_HOST_JVM_MAX_HEAP_MB="2560" \ + FAKE_BAZEL_ARGS="${shutdown_args}" \ + bash "${repo_root}/scripts/bazel-cache-backed.sh" shutdown >/dev/null +grep -Fx -- "--host_jvm_args=-Xmx2560m" "${shutdown_args}" >/dev/null +grep -Fx -- "shutdown" "${shutdown_args}" >/dev/null + +invalid_heap_stderr="${tmp_dir}/invalid-heap.stderr" if BAZEL_BIN="${fake_bazel}" \ BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_HOST_JVM_MAX_HEAP_MB="unbounded" \ FAKE_BAZEL_ARGS="${tmp_dir}/invalid-heap.args" \ - bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>&1; then + bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>"${invalid_heap_stderr}"; then echo "ERROR: invalid GF_BAZEL_HOST_JVM_MAX_HEAP_MB unexpectedly succeeded" >&2 exit 1 fi +grep -F -- "GF_BAZEL_HOST_JVM_MAX_HEAP_MB must be an integer from 256 through 4096" "${invalid_heap_stderr}" >/dev/null +invalid_upload_stderr="${tmp_dir}/invalid-upload.stderr" if BAZEL_BIN="${fake_bazel}" \ BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ GF_BAZEL_REMOTE_UPLOAD="sometimes" \ FAKE_BAZEL_ARGS="${tmp_dir}/invalid.args" \ - bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>&1; then + bash "${repo_root}/scripts/bazel-cache-backed.sh" test //:fake_target >/dev/null 2>"${invalid_upload_stderr}"; then echo "ERROR: invalid GF_BAZEL_REMOTE_UPLOAD unexpectedly succeeded" >&2 exit 1 fi +grep -F -- "GF_BAZEL_REMOTE_UPLOAD must be true, false, 1, or 0" "${invalid_upload_stderr}" >/dev/null +relative_cache_stderr="${tmp_dir}/relative-cache.stderr" if BAZEL_BIN="${fake_bazel}" \ BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="shared-cache-backed" \ TECTONIC_CACHE_DIR="relative/cache" \ FAKE_BAZEL_ARGS="${tmp_dir}/relative.args" \ - bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>&1; then + bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>"${relative_cache_stderr}"; then echo "ERROR: relative TECTONIC_CACHE_DIR unexpectedly succeeded" >&2 exit 1 fi +grep -F -- "TECTONIC_CACHE_DIR must be an absolute path" "${relative_cache_stderr}" >/dev/null +executor_cache_stderr="${tmp_dir}/executor-cache.stderr" if BAZEL_BIN="${fake_bazel}" \ BAZEL_REMOTE_CACHE="grpc://cache.example:9092" \ BAZEL_REMOTE_EXECUTOR="grpc://executor.example:9092" \ GF_BAZEL_SUBSTRATE_MODE="executor-backed" \ TECTONIC_CACHE_DIR="${tmp_dir}/executor-cache" \ FAKE_BAZEL_ARGS="${tmp_dir}/executor.args" \ - bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>&1; then + bash "${repo_root}/scripts/bazel-cache-backed.sh" build //:fake_target >/dev/null 2>"${executor_cache_stderr}"; then echo "ERROR: runner-local TECTONIC_CACHE_DIR unexpectedly reached executor-backed mode" >&2 exit 1 fi +grep -F -- "TECTONIC_CACHE_DIR is runner-local and cannot be used in executor-backed mode" "${executor_cache_stderr}" >/dev/null From 7a4cfcd39fd0b632bb6538664540b272077cf76a Mon Sep 17 00:00:00 2001 From: Jess Sullivan Date: Tue, 14 Jul 2026 15:19:36 -0400 Subject: [PATCH 13/13] fix(ci): refresh shared-cache credentials per phase --- .github/workflows/ci.yml | 17 ++++++++++++++--- BUILD.bazel | 1 + scripts/test-bazel-cache-backed.sh | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f2cd9..2c31398 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,8 @@ jobs: GF_BAZEL_SUBSTRATE_MODE: shared-cache-backed GF_BAZEL_REMOTE_UPLOAD: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} GF_BAZEL_HOST_JVM_MAX_HEAP_MB: '2560' + GF_REAPI_TOKEN_EXCHANGE_REQUEST: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'cache-write' || 'cache-read' }} + GF_REAPI_TOKEN_EXCHANGE_TTL: 45m TECTONIC_CACHE_DIR: /tmp/tectonic-cache-v33 # The GF consumer registry records this repo as shared-cache-backed. # Clear any executor endpoint inherited from the generic ARC pod. @@ -105,9 +107,6 @@ jobs: fi - name: Mint GloriousFlywheel REAPI token - env: - GF_REAPI_TOKEN_EXCHANGE_TTL: 45m - GF_REAPI_TOKEN_EXCHANGE_REQUEST: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'cache-write' || 'cache-read' }} run: bash scripts/mint-gf-reapi-token-from-exchange.sh - name: Validate GloriousFlywheel cache attachment @@ -153,6 +152,9 @@ jobs: sudo mv "${tmp}" /usr/local/bin/bazel fi + - name: Refresh cache token for Tectonic warm + run: bash scripts/mint-gf-reapi-token-from-exchange.sh + - name: Warm Tectonic resources serially run: | bash scripts/bazel-cache-backed.sh build --jobs=1 \ @@ -161,16 +163,25 @@ jobs: @spear_resumes//generic:cv bash scripts/bazel-cache-backed.sh shutdown + - name: Refresh cache token for remote check + run: bash scripts/mint-gf-reapi-token-from-exchange.sh + - name: Remote check run: | npm run remote:check bash scripts/bazel-cache-backed.sh shutdown + - name: Refresh cache token for remote test + run: bash scripts/mint-gf-reapi-token-from-exchange.sh + - name: Remote test run: | npm run remote:test bash scripts/bazel-cache-backed.sh shutdown + - name: Refresh cache token for remote e2e + run: bash scripts/mint-gf-reapi-token-from-exchange.sh + - name: Remote e2e run: | npm run remote:e2e diff --git a/BUILD.bazel b/BUILD.bazel index bd969f8..3b9a673 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -298,6 +298,7 @@ sh_test( srcs = ["scripts/test-bazel-cache-backed.sh"], data = [ ".bazelrc", + ".github/workflows/ci.yml", "scripts/bazel-cache-backed.sh", "scripts/cache-attachment-contract.sh", ], diff --git a/scripts/test-bazel-cache-backed.sh b/scripts/test-bazel-cache-backed.sh index b4de868..bba3e52 100755 --- a/scripts/test-bazel-cache-backed.sh +++ b/scripts/test-bazel-cache-backed.sh @@ -13,6 +13,12 @@ trap 'rm -rf "${tmp_dir}"' EXIT grep -Fx -- "build:ci-cached --jobs=1" "${repo_root}/.bazelrc" >/dev/null grep -Fx -- "build:ci-cached --action_env=NODE_OPTIONS=--max-old-space-size=1024" "${repo_root}/.bazelrc" >/dev/null grep -Fx -- "test:ci-cached --local_test_jobs=1" "${repo_root}/.bazelrc" >/dev/null +grep -Fx -- " GF_REAPI_TOKEN_EXCHANGE_TTL: 45m" "${repo_root}/.github/workflows/ci.yml" >/dev/null +mint_count="$(grep -c -F -- "run: bash scripts/mint-gf-reapi-token-from-exchange.sh" "${repo_root}/.github/workflows/ci.yml")" +if [[ ${mint_count} -ne 5 ]]; then + echo "ERROR: CI must mint once for attachment validation and refresh before all four Bazel phases; found ${mint_count}." >&2 + exit 1 +fi fake_bazel="${tmp_dir}/bazel" cat >"${fake_bazel}" <<'EOF' @@ -43,6 +49,19 @@ run_upload_case() { grep -Fx -- "--test_env=GF_RBE_CHROMIUM_EXECUTABLE=/opt/pinned-chromium" "${args_file}" >/dev/null grep -Fx -- "--action_env=TECTONIC_CACHE_DIR=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null grep -Fx -- "--sandbox_writable_path=${tmp_dir}/tectonic-cache" "${args_file}" >/dev/null + + host_jvm_line="$(awk '$0 == "--host_jvm_args=-Xmx2560m" { print NR; exit }' "${args_file}")" + command_line="$(awk '$0 == "test" { print NR; exit }' "${args_file}")" + config_line="$(awk '$0 == "--config=ci-cached" { print NR; exit }' "${args_file}")" + upload_line="$(awk -v expected="--remote_upload_local_results=${expected}" '$0 == expected { print NR; exit }' "${args_file}")" + if [[ -z ${host_jvm_line} || -z ${command_line} || ${host_jvm_line} -ge ${command_line} ]]; then + echo "ERROR: Bazel host JVM startup option must precede the command." >&2 + exit 1 + fi + if [[ -z ${config_line} || -z ${upload_line} || ${config_line} -ge ${upload_line} ]]; then + echo "ERROR: explicit remote-upload policy must follow the selected Bazel config." >&2 + exit 1 + fi } run_upload_case false false