fix(ci): retry gates on tenant quota; per-ref concurrency dedupe

.github/workflows/ci.yml

@@ -32,13 +32,16 @@ jobs:
 
   bazel-remote-gates:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    # The gf-reapi-cell tenant (consumer-jesssullivan-jesssullivan-github-io)
-    # has maxConcurrentExecutions=4; parallel gates jobs across PRs trip
-    # RESOURCE_EXHAUSTED (observed runs 28732755094, 28847693875). Serialize
-    # the tenant's bazel gates repo-wide; queue instead of cancel.
+    # Per-ref dedupe only: a newer push to the same ref supersedes its own
+    # stale gates run. Cross-PR quota pressure on the gf-reapi-cell tenant
+    # (maxConcurrentExecutions=4; runs 28732755094, 28847693875) is handled
+    # by the exit-34 retry loop in scripts/bazel-cache-backed.sh — a shared
+    # repo-wide group was tried first, but GitHub evicts (cancels) middle
+    # pending jobs in a group, which breaks required checks (run
+    # 28849318814 + main 09c1dd0 were cancelled, not queued).
     concurrency:
-      group: gf-reapi-tenant-gates
-      cancel-in-progress: false
+      group: gf-reapi-tenant-gates-${{ github.ref }}
+      cancel-in-progress: true
     permissions:
       contents: read
       id-token: write

scripts/bazel-cache-backed.sh

@@ -175,13 +175,31 @@ info)
     routing_args+=(--remote_instance_name="${BAZEL_REMOTE_INSTANCE_NAME}")
   fi
 
-  exec "${bazel_bin}" "${command}" \
-    --config="${bazel_config}" \
-    --remote_cache="${effective_remote_cache}" \
-    "${executor_args[@]}" \
-    "${credential_args[@]}" \
-    "${routing_args[@]}" \
-    "${external_fetch_args[@]}" \
-    "$@"
+  # Bounded retry on the gf-reapi-cell per-tenant quota (Bazel exit 34,
+  # RESOURCE_EXHAUSTED: concurrent-execution limit). Mirrors the
+  # tinyland.dev lane's 3-attempt pattern. Any other exit code propagates
+  # immediately. Concurrency-group serialization was tried first but GitHub
+  # evicts (cancels) middle pending jobs in a shared group, which breaks
+  # required checks.
+  quota_attempts="${GF_BAZEL_QUOTA_RETRIES:-3}"
+  attempt=1
+  while :; do
+    rc=0
+    "${bazel_bin}" "${command}" \
+      --config="${bazel_config}" \
+      --remote_cache="${effective_remote_cache}" \
+      "${executor_args[@]}" \
+      "${credential_args[@]}" \
+      "${routing_args[@]}" \
+      "${external_fetch_args[@]}" \
+      "$@" || rc=$?
+    if [[ ${rc} -ne 34 || ${attempt} -ge ${quota_attempts} ]]; then
+      exit "${rc}"
+    fi
+    backoff=$((75 * attempt + RANDOM % 30))
+    echo "bazel-cache-backed: tenant quota exhausted (exit 34); retry ${attempt}/${quota_attempts} in ${backoff}s" >&2
+    sleep "${backoff}"
+    attempt=$((attempt + 1))
+  done
   ;;
 esac