diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a1e6b6..1603b77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,16 @@ jobs: bazel-remote-gates: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - # The gf-reapi-cell tenant (consumer-jesssullivan-jesssullivan-github-io) - # has maxConcurrentExecutions=4; parallel gates jobs across PRs trip - # RESOURCE_EXHAUSTED (observed runs 28732755094, 28847693875). Serialize - # the tenant's bazel gates repo-wide; queue instead of cancel. + # Per-ref dedupe only: a newer push to the same ref supersedes its own + # stale gates run. Cross-PR quota pressure on the gf-reapi-cell tenant + # (maxConcurrentExecutions=4; runs 28732755094, 28847693875) is handled + # by the exit-34 retry loop in scripts/bazel-cache-backed.sh — a shared + # repo-wide group was tried first, but GitHub evicts (cancels) middle + # pending jobs in a group, which breaks required checks (run + # 28849318814 + main 09c1dd0 were cancelled, not queued). concurrency: - group: gf-reapi-tenant-gates - cancel-in-progress: false + group: gf-reapi-tenant-gates-${{ github.ref }} + cancel-in-progress: true permissions: contents: read id-token: write diff --git a/scripts/bazel-cache-backed.sh b/scripts/bazel-cache-backed.sh index d9a5eca..6de50f5 100755 --- a/scripts/bazel-cache-backed.sh +++ b/scripts/bazel-cache-backed.sh @@ -175,13 +175,31 @@ info) routing_args+=(--remote_instance_name="${BAZEL_REMOTE_INSTANCE_NAME}") fi - exec "${bazel_bin}" "${command}" \ - --config="${bazel_config}" \ - --remote_cache="${effective_remote_cache}" \ - "${executor_args[@]}" \ - "${credential_args[@]}" \ - "${routing_args[@]}" \ - "${external_fetch_args[@]}" \ - "$@" + # Bounded retry on the gf-reapi-cell per-tenant quota (Bazel exit 34, + # RESOURCE_EXHAUSTED: concurrent-execution limit). Mirrors the + # tinyland.dev lane's 3-attempt pattern. Any other exit code propagates + # immediately. Concurrency-group serialization was tried first but GitHub + # evicts (cancels) middle pending jobs in a shared group, which breaks + # required checks. + quota_attempts="${GF_BAZEL_QUOTA_RETRIES:-3}" + attempt=1 + while :; do + rc=0 + "${bazel_bin}" "${command}" \ + --config="${bazel_config}" \ + --remote_cache="${effective_remote_cache}" \ + "${executor_args[@]}" \ + "${credential_args[@]}" \ + "${routing_args[@]}" \ + "${external_fetch_args[@]}" \ + "$@" || rc=$? + if [[ ${rc} -ne 34 || ${attempt} -ge ${quota_attempts} ]]; then + exit "${rc}" + fi + backoff=$((75 * attempt + RANDOM % 30)) + echo "bazel-cache-backed: tenant quota exhausted (exit 34); retry ${attempt}/${quota_attempts} in ${backoff}s" >&2 + sleep "${backoff}" + attempt=$((attempt + 1)) + done ;; esac