.github: tell a CI-Gate timeout apart from a cancellation

.github/workflows/ci-gate.yml

@@ -28,9 +28,12 @@ concurrency:
 # pull-requests: write lets the gate call the `dequeuePullRequest` GraphQL
 # mutation on its own merge-group PR when a required check actually fails,
 # since GitHub does not auto-remove UNMERGEABLE entries from the queue.
+# checks: read lets the gate read a cancelled leaf's annotations, the only
+# signal that separates a `timeout-minutes` kill from an external cancel.
 # Reusable leaves inherit these from the caller.
 permissions:
   actions: write
+  checks: read
   contents: read
   pull-requests: write
 

.github/workflows/scripts/ci-gate-check.sh

@@ -8,7 +8,8 @@
 # GITHUB_EVENT_NAME (reshuffle fast-path is merge_group-only),
 # GH_TOKEN/GITHUB_REPOSITORY/GITHUB_RUN_ID to fetch the jobs list.
 # Test seam: CI_GATE_NO_FETCH=1 uses CI_GATE_JOBS_JSON verbatim instead of the
-# API (an empty value simulates a failed fetch).
+# API (an empty value simulates a failed fetch) and CI_GATE_ANNOTATIONS_JSON
+# ({"<job id>": ["<message>"]}) instead of the per-job annotations endpoint.
 set -eo pipefail
 
 needs="${NEEDS:-}"
@@ -38,25 +39,61 @@ fi
 # so a failed step still signals a real failure.
 failed_steps=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | .name' <<<"$jobs" 2>/dev/null || true)
 
+annotations_of() {
+  if [ -n "${CI_GATE_NO_FETCH:-}" ]; then
+    jq -r --arg id "$1" '.[$id] // [] | .[]' <<<"${CI_GATE_ANNOTATIONS_JSON:-{\}}" 2>/dev/null || true
+    return 0
+  fi
+  gh api "repos/${GITHUB_REPOSITORY}/check-runs/$1/annotations" --jq '.[].message' 2>/dev/null || true
+}
+
+# `timeout-minutes` kills a job with conclusion "cancelled" — the same value an
+# external cancel or a reshuffle produces — and lets its reporting step run on,
+# so the failed step points at the report rather than the overrun. The runner's
+# annotation is the only reliable discriminator.
+timed_out_names=""
+timeouts=""
+while IFS=$'\t' read -r job_id job_name; do
+  [ -n "$job_id" ] && [ "$job_id" != "null" ] || continue
+  case "$(annotations_of "$job_id")" in
+    *"exceeded the maximum execution time"*) ;;
+    *) continue ;;
+  esac
+  overran=$(jq -r --arg n "$job_name" \
+    '[.jobs[] | select(.name == $n) | .steps[]? | select(.conclusion == "cancelled") | .name] | join("; ")' \
+    <<<"$jobs" 2>/dev/null || true)
+  timed_out_names="${timed_out_names}${job_name}"$'\n'
+  timeouts="${timeouts}::error title=CI timeout::${job_name} — exceeded its timeout-minutes budget${overran:+ while running: $overran}"$'\n'
+done < <(jq -r '.jobs[] | select(.name != "ci-gate") | select(.conclusion == "cancelled") | "\(.id)\t\(.name)"' <<<"$jobs" 2>/dev/null || true)
+
 # Run cancelled with no failure = GitHub tore down a superseded merge group
 # (reshuffle); failing here would spuriously evict the PR. Scope to merge_group
 # (reshuffles only happen in the queue) and require a successful jobs fetch so an
-# empty failed_steps can be trusted.
-if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then
+# empty failed_steps can be trusted. A timed-out leaf looks identical from the
+# needs rollup, so it is excluded explicitly or it would merge unchecked.
+if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ -z "$timeouts" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then
   echo "::notice::Merge-queue reshuffle cancelled this run (no failed jobs or steps); passing the gate so the PR stays queued."
   echo "Cancelled jobs: $(tr '\n' ' ' <<<"$cancelled")"
   exit 0
 fi
 
+printf '%s' "$timeouts"
 echo "The following gate jobs failed or were cancelled:"
 echo "$failed"
 echo "$cancelled"
 # The leaf that fast-cancelled the run is the true root cause; other failed
-# steps may be collateral.
-root=$(jq -r '.jobs[] | select(any(.steps[]?; .name == "Cancel workflow run on failure" and .conclusion == "success")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true)
+# steps may be collateral. Timed-out leaves are already reported above, and
+# their reporting step would otherwise be blamed for the overrun.
+root_cause='"::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))'
+# $n and $to are jq variables, not shell.
+# shellcheck disable=SC2016
+not_timed_out='select(.name as $n | ($to | split("\n") | map(select(length > 0))) | index($n) | not)'
+root=$(jq -r --arg to "$timed_out_names" ".jobs[] | $not_timed_out | select(any(.steps[]?; .name == \"Cancel workflow run on failure\" and .conclusion == \"success\")) | $root_cause" <<<"$jobs" 2>/dev/null || true)
 if [ -z "$root" ]; then
-  root=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true)
+  root=$(jq -r --arg to "$timed_out_names" ".jobs[] | select(.name != \"ci-gate\") | $not_timed_out | select(any(.steps[]?; .conclusion == \"failure\")) | $root_cause" <<<"$jobs" 2>/dev/null || true)
+fi
+if [ -n "$root" ]; then
+  echo "Root-cause job(s):"
+  echo "$root"
 fi
-echo "Root-cause job(s):"
-echo "$root"
 exit 1

.github/workflows/scripts/ci-gate-check.test.sh

@@ -26,6 +26,30 @@ run_case() {
   fi
 }
 
+# run_output <name> <want_exit> <want_regex|-> <reject_regex|-> [VAR=VAL ...]
+run_output() {
+  local name="$1" want="$2" want_re="$3" reject_re="$4"
+  shift 4
+  local out rc why=""
+  out=$(env -i PATH="$PATH" CI_GATE_NO_FETCH=1 "$@" bash "$script" 2>&1)
+  rc=$?
+  [ "$rc" -eq "$want" ] || why="want exit $want, got $rc"
+  if [ -z "$why" ] && [ "$want_re" != "-" ] && ! grep -qE "$want_re" <<<"$out"; then
+    why="output missing /$want_re/"
+  fi
+  if [ -z "$why" ] && [ "$reject_re" != "-" ] && grep -qE "$reject_re" <<<"$out"; then
+    why="output unexpectedly matched /$reject_re/"
+  fi
+  if [ -z "$why" ]; then
+    printf 'ok   - %s (exit %d)\n' "$name" "$rc"
+    pass=$((pass + 1))
+  else
+    printf 'FAIL - %s: %s\n' "$name" "$why"
+    printf '%s\n' "$out" | sed 's/^/       | /'
+    fail=$((fail + 1))
+  fi
+}
+
 # Everything green -> pass.
 run_case "all success" 0 \
   NEEDS='{"lint":{"result":"success"},"tests":{"result":"success"}}'
@@ -93,6 +117,34 @@ run_case "error page mid-pagination -> fail closed" 1 \
   CI_GATE_JOBS_JSON='{"jobs":[{"name":"hive","steps":[{"name":"run","conclusion":"cancelled"}]}]}
 {"message":"Server Error"}'
 
+# A job killed by timeout-minutes reports conclusion "cancelled" and, because
+# its reporting step still runs, a failed step that is not the real cause.
+# Name the timeout and the step that overran; don't blame the reporting step.
+run_output "leaf timeout is named as a timeout" 1 \
+  'CI timeout::hive-eest .*glamsterdam-devnet.*Run hive tests and parse output' \
+  'root cause' \
+  NEEDS='{"hive-eest":{"result":"cancelled"},"lint":{"result":"success"}}' \
+  CI_GATE_JOBS_JSON='{"jobs":[{"id":1,"name":"hive-eest / test-hive-eest (glamsterdam-devnet, serial)","conclusion":"cancelled","steps":[{"name":"Run hive tests and parse output","conclusion":"cancelled"},{"name":"Test Results","conclusion":"failure"}]}]}' \
+  CI_GATE_ANNOTATIONS_JSON='{"1":["The job has exceeded the maximum execution time of 1h0m0s"]}'
+
+# A cancelled leaf with no timeout annotation (runner dropped the job, external
+# cancel) must not be mislabelled as a timeout.
+run_output "external cancel is not a timeout" 1 \
+  '-' 'CI timeout' \
+  NEEDS='{"hive":{"result":"cancelled"},"lint":{"result":"success"}}' \
+  CI_GATE_JOBS_JSON='{"jobs":[{"id":2,"name":"hive / test-hive (engine, api, parallel)","conclusion":"cancelled","steps":[{"name":"Set up job","conclusion":"cancelled"}]}]}' \
+  CI_GATE_ANNOTATIONS_JSON='{"2":["The operation was canceled."]}'
+
+# A timeout leaves no failed step at all when the reporting step is skipped, so
+# the reshuffle fast-path would otherwise swallow it -> must still fail.
+run_output "timeout is not absorbed by the reshuffle fast-path" 1 \
+  'CI timeout::tests / tests-mac-linux' '-' \
+  NEEDS='{"tests":{"result":"cancelled"},"lint":{"result":"success"}}' \
+  RUN_CANCELLED=true \
+  GITHUB_EVENT_NAME=merge_group \
+  CI_GATE_JOBS_JSON='{"jobs":[{"id":3,"name":"tests / tests-mac-linux (windows-2025, parallel)","conclusion":"cancelled","steps":[{"name":"Run tests","conclusion":"cancelled"}]}]}' \
+  CI_GATE_ANNOTATIONS_JSON='{"3":["The job has exceeded the maximum execution time of 1h0m0s"]}'
+
 echo "----"
 printf '%d passed, %d failed\n' "$pass" "$fail"
 [ "$fail" -eq 0 ]