From bc19e11ca0e6eeae9e851a658a4f8b4c6a639b7b Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Wed, 12 Aug 2026 10:22:59 +0700 Subject: [PATCH 1/2] .github: tell a CI-Gate timeout apart from a cancellation A job killed by `timeout-minutes` reports conclusion `cancelled`, the same value an external cancel or a merge-queue reshuffle produces, and its reporting step still runs. The gate therefore blamed the reporting step for the overrun, and a timeout whose reporting step was skipped was absorbed by the reshuffle fast-path and passed the gate. Read the runner's annotation for every cancelled leaf: the message "The job has exceeded the maximum execution time of ..." is the only reliable discriminator. Report those jobs as timeouts with the step that overran, exclude them from the reshuffle fast-path, and keep them out of the root-cause list. --- .github/workflows/ci-gate.yml | 3 ++ .github/workflows/scripts/ci-gate-check.sh | 53 ++++++++++++++++--- .../workflows/scripts/ci-gate-check.test.sh | 52 ++++++++++++++++++ 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-gate.yml b/.github/workflows/ci-gate.yml index 3a474bc0927..3cf6e51b4b9 100644 --- a/.github/workflows/ci-gate.yml +++ b/.github/workflows/ci-gate.yml @@ -28,9 +28,12 @@ concurrency: # pull-requests: write lets the gate call the `dequeuePullRequest` GraphQL # mutation on its own merge-group PR when a required check actually fails, # since GitHub does not auto-remove UNMERGEABLE entries from the queue. +# checks: read lets the gate read a cancelled leaf's annotations, the only +# signal that separates a `timeout-minutes` kill from an external cancel. # Reusable leaves inherit these from the caller. permissions: actions: write + checks: read contents: read pull-requests: write diff --git a/.github/workflows/scripts/ci-gate-check.sh b/.github/workflows/scripts/ci-gate-check.sh index 40ef287b715..52a16218528 100755 --- a/.github/workflows/scripts/ci-gate-check.sh +++ b/.github/workflows/scripts/ci-gate-check.sh @@ -8,7 +8,8 @@ # GITHUB_EVENT_NAME (reshuffle fast-path is merge_group-only), # GH_TOKEN/GITHUB_REPOSITORY/GITHUB_RUN_ID to fetch the jobs list. # Test seam: CI_GATE_NO_FETCH=1 uses CI_GATE_JOBS_JSON verbatim instead of the -# API (an empty value simulates a failed fetch). +# API (an empty value simulates a failed fetch) and CI_GATE_ANNOTATIONS_JSON +# ({"": [""]}) instead of the per-job annotations endpoint. set -eo pipefail needs="${NEEDS:-}" @@ -38,25 +39,61 @@ fi # so a failed step still signals a real failure. failed_steps=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | .name' <<<"$jobs" 2>/dev/null || true) +annotations_of() { + if [ -n "${CI_GATE_NO_FETCH:-}" ]; then + jq -r --arg id "$1" '.[$id] // [] | .[]' <<<"${CI_GATE_ANNOTATIONS_JSON:-{\}}" 2>/dev/null || true + return 0 + fi + gh api "repos/${GITHUB_REPOSITORY}/check-runs/$1/annotations" --jq '.[].message' 2>/dev/null || true +} + +# `timeout-minutes` kills a job with conclusion "cancelled" — the same value an +# external cancel or a reshuffle produces — and lets its reporting step run on, +# so the failed step points at the report rather than the overrun. The runner's +# annotation is the only reliable discriminator. +timed_out_names="" +timeouts="" +while IFS=$'\t' read -r job_id job_name; do + [ -n "$job_id" ] && [ "$job_id" != "null" ] || continue + case "$(annotations_of "$job_id")" in + *"exceeded the maximum execution time"*) ;; + *) continue ;; + esac + overran=$(jq -r --arg n "$job_name" \ + '[.jobs[] | select(.name == $n) | .steps[]? | select(.conclusion == "cancelled") | .name] | join("; ")' \ + <<<"$jobs" 2>/dev/null || true) + timed_out_names="${timed_out_names}${job_name}"$'\n' + timeouts="${timeouts}::error title=CI timeout::${job_name} — exceeded its timeout-minutes budget${overran:+ while running: $overran}"$'\n' +done < <(jq -r '.jobs[] | select(.name != "ci-gate") | select(.conclusion == "cancelled") | "\(.id)\t\(.name)"' <<<"$jobs" 2>/dev/null || true) + # Run cancelled with no failure = GitHub tore down a superseded merge group # (reshuffle); failing here would spuriously evict the PR. Scope to merge_group # (reshuffles only happen in the queue) and require a successful jobs fetch so an -# empty failed_steps can be trusted. -if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then +# empty failed_steps can be trusted. A timed-out leaf looks identical from the +# needs rollup, so it is excluded explicitly or it would merge unchecked. +if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ -z "$timeouts" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then echo "::notice::Merge-queue reshuffle cancelled this run (no failed jobs or steps); passing the gate so the PR stays queued." echo "Cancelled jobs: $(tr '\n' ' ' <<<"$cancelled")" exit 0 fi +printf '%s' "$timeouts" echo "The following gate jobs failed or were cancelled:" echo "$failed" echo "$cancelled" # The leaf that fast-cancelled the run is the true root cause; other failed -# steps may be collateral. -root=$(jq -r '.jobs[] | select(any(.steps[]?; .name == "Cancel workflow run on failure" and .conclusion == "success")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true) +# steps may be collateral. Timed-out leaves are already reported above, and +# their reporting step would otherwise be blamed for the overrun. +root_cause='"::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' +# $n and $to are jq variables, not shell. +# shellcheck disable=SC2016 +not_timed_out='select(.name as $n | ($to | split("\n") | map(select(length > 0))) | index($n) | not)' +root=$(jq -r --arg to "$timed_out_names" ".jobs[] | $not_timed_out | select(any(.steps[]?; .name == \"Cancel workflow run on failure\" and .conclusion == \"success\")) | $root_cause" <<<"$jobs" 2>/dev/null || true) if [ -z "$root" ]; then - root=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true) + root=$(jq -r --arg to "$timed_out_names" ".jobs[] | select(.name != \"ci-gate\") | $not_timed_out | select(any(.steps[]?; .conclusion == \"failure\")) | $root_cause" <<<"$jobs" 2>/dev/null || true) +fi +if [ -n "$root" ]; then + echo "Root-cause job(s):" + echo "$root" fi -echo "Root-cause job(s):" -echo "$root" exit 1 diff --git a/.github/workflows/scripts/ci-gate-check.test.sh b/.github/workflows/scripts/ci-gate-check.test.sh index c24c4b033bd..24aa5c97d79 100755 --- a/.github/workflows/scripts/ci-gate-check.test.sh +++ b/.github/workflows/scripts/ci-gate-check.test.sh @@ -26,6 +26,30 @@ run_case() { fi } +# run_output [VAR=VAL ...] +run_output() { + local name="$1" want="$2" want_re="$3" reject_re="$4" + shift 4 + local out rc why="" + out=$(env -i PATH="$PATH" CI_GATE_NO_FETCH=1 "$@" bash "$script" 2>&1) + rc=$? + [ "$rc" -eq "$want" ] || why="want exit $want, got $rc" + if [ -z "$why" ] && [ "$want_re" != "-" ] && ! grep -qE "$want_re" <<<"$out"; then + why="output missing /$want_re/" + fi + if [ -z "$why" ] && [ "$reject_re" != "-" ] && grep -qE "$reject_re" <<<"$out"; then + why="output unexpectedly matched /$reject_re/" + fi + if [ -z "$why" ]; then + printf 'ok - %s (exit %d)\n' "$name" "$rc" + pass=$((pass + 1)) + else + printf 'FAIL - %s: %s\n' "$name" "$why" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + fi +} + # Everything green -> pass. run_case "all success" 0 \ NEEDS='{"lint":{"result":"success"},"tests":{"result":"success"}}' @@ -93,6 +117,34 @@ run_case "error page mid-pagination -> fail closed" 1 \ CI_GATE_JOBS_JSON='{"jobs":[{"name":"hive","steps":[{"name":"run","conclusion":"cancelled"}]}]} {"message":"Server Error"}' +# A job killed by timeout-minutes reports conclusion "cancelled" and, because +# its reporting step still runs, a failed step that is not the real cause. +# Name the timeout and the step that overran; don't blame the reporting step. +run_output "leaf timeout is named as a timeout" 1 \ + 'CI timeout::hive-eest .*glamsterdam-devnet.*Run hive tests and parse output' \ + 'root cause' \ + NEEDS='{"hive-eest":{"result":"cancelled"},"lint":{"result":"success"}}' \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":1,"name":"hive-eest / test-hive-eest (glamsterdam-devnet, serial)","conclusion":"cancelled","steps":[{"name":"Run hive tests and parse output","conclusion":"cancelled"},{"name":"Test Results","conclusion":"failure"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"1":["The job has exceeded the maximum execution time of 1h0m0s"]}' + +# A cancelled leaf with no timeout annotation (runner dropped the job, external +# cancel) must not be mislabelled as a timeout. +run_output "external cancel is not a timeout" 1 \ + '-' 'CI timeout' \ + NEEDS='{"hive":{"result":"cancelled"},"lint":{"result":"success"}}' \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":2,"name":"hive / test-hive (engine, api, parallel)","conclusion":"cancelled","steps":[{"name":"Set up job","conclusion":"cancelled"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"2":["The operation was canceled."]}' + +# A timeout leaves no failed step at all when the reporting step is skipped, so +# the reshuffle fast-path would otherwise swallow it -> must still fail. +run_output "timeout is not absorbed by the reshuffle fast-path" 1 \ + 'CI timeout::tests / tests-mac-linux' '-' \ + NEEDS='{"tests":{"result":"cancelled"},"lint":{"result":"success"}}' \ + RUN_CANCELLED=true \ + GITHUB_EVENT_NAME=merge_group \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":3,"name":"tests / tests-mac-linux (windows-2025, parallel)","conclusion":"cancelled","steps":[{"name":"Run tests","conclusion":"cancelled"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"3":["The job has exceeded the maximum execution time of 1h0m0s"]}' + echo "----" printf '%d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ] From 0764d8f2dcb4195e2d8e757cb846a87872fd76be Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Wed, 12 Aug 2026 12:01:38 +0700 Subject: [PATCH 2/2] .github: re-trigger CI