diff --git a/.github/workflows/ci-gate.yml b/.github/workflows/ci-gate.yml index 3a474bc0927..3cf6e51b4b9 100644 --- a/.github/workflows/ci-gate.yml +++ b/.github/workflows/ci-gate.yml @@ -28,9 +28,12 @@ concurrency: # pull-requests: write lets the gate call the `dequeuePullRequest` GraphQL # mutation on its own merge-group PR when a required check actually fails, # since GitHub does not auto-remove UNMERGEABLE entries from the queue. +# checks: read lets the gate read a cancelled leaf's annotations, the only +# signal that separates a `timeout-minutes` kill from an external cancel. # Reusable leaves inherit these from the caller. permissions: actions: write + checks: read contents: read pull-requests: write diff --git a/.github/workflows/scripts/ci-gate-check.sh b/.github/workflows/scripts/ci-gate-check.sh index 40ef287b715..52a16218528 100755 --- a/.github/workflows/scripts/ci-gate-check.sh +++ b/.github/workflows/scripts/ci-gate-check.sh @@ -8,7 +8,8 @@ # GITHUB_EVENT_NAME (reshuffle fast-path is merge_group-only), # GH_TOKEN/GITHUB_REPOSITORY/GITHUB_RUN_ID to fetch the jobs list. # Test seam: CI_GATE_NO_FETCH=1 uses CI_GATE_JOBS_JSON verbatim instead of the -# API (an empty value simulates a failed fetch). +# API (an empty value simulates a failed fetch) and CI_GATE_ANNOTATIONS_JSON +# ({"": [""]}) instead of the per-job annotations endpoint. set -eo pipefail needs="${NEEDS:-}" @@ -38,25 +39,61 @@ fi # so a failed step still signals a real failure. failed_steps=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | .name' <<<"$jobs" 2>/dev/null || true) +annotations_of() { + if [ -n "${CI_GATE_NO_FETCH:-}" ]; then + jq -r --arg id "$1" '.[$id] // [] | .[]' <<<"${CI_GATE_ANNOTATIONS_JSON:-{\}}" 2>/dev/null || true + return 0 + fi + gh api "repos/${GITHUB_REPOSITORY}/check-runs/$1/annotations" --jq '.[].message' 2>/dev/null || true +} + +# `timeout-minutes` kills a job with conclusion "cancelled" — the same value an +# external cancel or a reshuffle produces — and lets its reporting step run on, +# so the failed step points at the report rather than the overrun. The runner's +# annotation is the only reliable discriminator. +timed_out_names="" +timeouts="" +while IFS=$'\t' read -r job_id job_name; do + [ -n "$job_id" ] && [ "$job_id" != "null" ] || continue + case "$(annotations_of "$job_id")" in + *"exceeded the maximum execution time"*) ;; + *) continue ;; + esac + overran=$(jq -r --arg n "$job_name" \ + '[.jobs[] | select(.name == $n) | .steps[]? | select(.conclusion == "cancelled") | .name] | join("; ")' \ + <<<"$jobs" 2>/dev/null || true) + timed_out_names="${timed_out_names}${job_name}"$'\n' + timeouts="${timeouts}::error title=CI timeout::${job_name} — exceeded its timeout-minutes budget${overran:+ while running: $overran}"$'\n' +done < <(jq -r '.jobs[] | select(.name != "ci-gate") | select(.conclusion == "cancelled") | "\(.id)\t\(.name)"' <<<"$jobs" 2>/dev/null || true) + # Run cancelled with no failure = GitHub tore down a superseded merge group # (reshuffle); failing here would spuriously evict the PR. Scope to merge_group # (reshuffles only happen in the queue) and require a successful jobs fetch so an -# empty failed_steps can be trusted. -if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then +# empty failed_steps can be trusted. A timed-out leaf looks identical from the +# needs rollup, so it is excluded explicitly or it would merge unchecked. +if [ -z "$failed" ] && [ -n "$jobs" ] && [ -z "$failed_steps" ] && [ -z "$timeouts" ] && [ "${RUN_CANCELLED:-}" = "true" ] && [ "${GITHUB_EVENT_NAME:-}" = "merge_group" ]; then echo "::notice::Merge-queue reshuffle cancelled this run (no failed jobs or steps); passing the gate so the PR stays queued." echo "Cancelled jobs: $(tr '\n' ' ' <<<"$cancelled")" exit 0 fi +printf '%s' "$timeouts" echo "The following gate jobs failed or were cancelled:" echo "$failed" echo "$cancelled" # The leaf that fast-cancelled the run is the true root cause; other failed -# steps may be collateral. -root=$(jq -r '.jobs[] | select(any(.steps[]?; .name == "Cancel workflow run on failure" and .conclusion == "success")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true) +# steps may be collateral. Timed-out leaves are already reported above, and +# their reporting step would otherwise be blamed for the overrun. +root_cause='"::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' +# $n and $to are jq variables, not shell. +# shellcheck disable=SC2016 +not_timed_out='select(.name as $n | ($to | split("\n") | map(select(length > 0))) | index($n) | not)' +root=$(jq -r --arg to "$timed_out_names" ".jobs[] | $not_timed_out | select(any(.steps[]?; .name == \"Cancel workflow run on failure\" and .conclusion == \"success\")) | $root_cause" <<<"$jobs" 2>/dev/null || true) if [ -z "$root" ]; then - root=$(jq -r '.jobs[] | select(.name != "ci-gate") | select(any(.steps[]?; .conclusion == "failure")) | "::error title=CI root cause::" + .name + " — failed step: " + ([.steps[] | select(.conclusion == "failure") | .name] | join("; "))' <<<"$jobs" 2>/dev/null || true) + root=$(jq -r --arg to "$timed_out_names" ".jobs[] | select(.name != \"ci-gate\") | $not_timed_out | select(any(.steps[]?; .conclusion == \"failure\")) | $root_cause" <<<"$jobs" 2>/dev/null || true) +fi +if [ -n "$root" ]; then + echo "Root-cause job(s):" + echo "$root" fi -echo "Root-cause job(s):" -echo "$root" exit 1 diff --git a/.github/workflows/scripts/ci-gate-check.test.sh b/.github/workflows/scripts/ci-gate-check.test.sh index c24c4b033bd..24aa5c97d79 100755 --- a/.github/workflows/scripts/ci-gate-check.test.sh +++ b/.github/workflows/scripts/ci-gate-check.test.sh @@ -26,6 +26,30 @@ run_case() { fi } +# run_output [VAR=VAL ...] +run_output() { + local name="$1" want="$2" want_re="$3" reject_re="$4" + shift 4 + local out rc why="" + out=$(env -i PATH="$PATH" CI_GATE_NO_FETCH=1 "$@" bash "$script" 2>&1) + rc=$? + [ "$rc" -eq "$want" ] || why="want exit $want, got $rc" + if [ -z "$why" ] && [ "$want_re" != "-" ] && ! grep -qE "$want_re" <<<"$out"; then + why="output missing /$want_re/" + fi + if [ -z "$why" ] && [ "$reject_re" != "-" ] && grep -qE "$reject_re" <<<"$out"; then + why="output unexpectedly matched /$reject_re/" + fi + if [ -z "$why" ]; then + printf 'ok - %s (exit %d)\n' "$name" "$rc" + pass=$((pass + 1)) + else + printf 'FAIL - %s: %s\n' "$name" "$why" + printf '%s\n' "$out" | sed 's/^/ | /' + fail=$((fail + 1)) + fi +} + # Everything green -> pass. run_case "all success" 0 \ NEEDS='{"lint":{"result":"success"},"tests":{"result":"success"}}' @@ -93,6 +117,34 @@ run_case "error page mid-pagination -> fail closed" 1 \ CI_GATE_JOBS_JSON='{"jobs":[{"name":"hive","steps":[{"name":"run","conclusion":"cancelled"}]}]} {"message":"Server Error"}' +# A job killed by timeout-minutes reports conclusion "cancelled" and, because +# its reporting step still runs, a failed step that is not the real cause. +# Name the timeout and the step that overran; don't blame the reporting step. +run_output "leaf timeout is named as a timeout" 1 \ + 'CI timeout::hive-eest .*glamsterdam-devnet.*Run hive tests and parse output' \ + 'root cause' \ + NEEDS='{"hive-eest":{"result":"cancelled"},"lint":{"result":"success"}}' \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":1,"name":"hive-eest / test-hive-eest (glamsterdam-devnet, serial)","conclusion":"cancelled","steps":[{"name":"Run hive tests and parse output","conclusion":"cancelled"},{"name":"Test Results","conclusion":"failure"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"1":["The job has exceeded the maximum execution time of 1h0m0s"]}' + +# A cancelled leaf with no timeout annotation (runner dropped the job, external +# cancel) must not be mislabelled as a timeout. +run_output "external cancel is not a timeout" 1 \ + '-' 'CI timeout' \ + NEEDS='{"hive":{"result":"cancelled"},"lint":{"result":"success"}}' \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":2,"name":"hive / test-hive (engine, api, parallel)","conclusion":"cancelled","steps":[{"name":"Set up job","conclusion":"cancelled"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"2":["The operation was canceled."]}' + +# A timeout leaves no failed step at all when the reporting step is skipped, so +# the reshuffle fast-path would otherwise swallow it -> must still fail. +run_output "timeout is not absorbed by the reshuffle fast-path" 1 \ + 'CI timeout::tests / tests-mac-linux' '-' \ + NEEDS='{"tests":{"result":"cancelled"},"lint":{"result":"success"}}' \ + RUN_CANCELLED=true \ + GITHUB_EVENT_NAME=merge_group \ + CI_GATE_JOBS_JSON='{"jobs":[{"id":3,"name":"tests / tests-mac-linux (windows-2025, parallel)","conclusion":"cancelled","steps":[{"name":"Run tests","conclusion":"cancelled"}]}]}' \ + CI_GATE_ANNOTATIONS_JSON='{"3":["The job has exceeded the maximum execution time of 1h0m0s"]}' + echo "----" printf '%d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ]