feat(guard): require a bounded timeout-minutes on every required-context job (#398)

.github/workflows/ci.yml

@@ -35,6 +35,10 @@ jobs:
   changes:
     name: Detect changed paths
     runs-on: [self-hosted, linux, x64, light]
+    # Trivial classifier (observed <=15s), but it gates 10 of the other jobs, so
+    # a wedge here stalls the whole suite. 5m is a generous ceiling vs the 360m
+    # default. Enforced by tools/check_job_timeouts.py (#398).
+    timeout-minutes: 5
     outputs:
       code: ${{ steps.filter.outputs.code }}
     steps:
@@ -84,6 +88,8 @@ jobs:
   fmt:
     name: Format
     runs-on: [self-hosted, linux, x64, light]
+    # fmt across every workspace (observed <=40s). Cap far under the 360m default.
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -113,6 +119,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # Workspace clippy (observed ~21m on a cold cache). 45m is ~2x that.
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -136,6 +144,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # nextest across the workspace (observed ~7.5m). 30m absorbs a cold cache.
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -169,6 +179,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # Generated-crate + wasmtime/kiln runtime oracles (observed ~27m cold). 60m ~2x.
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -196,6 +208,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # Compile-only bench check (observed ~9m). 30m absorbs a cold cache.
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -217,6 +231,9 @@ jobs:
     # Follow-up: drop sccache from ralf's profile before bumping the
     # toolchains-role version, then move this back to [self-hosted, light].
     runs-on: ubuntu-latest
+    # rustsec/audit-check hits the network advisory DB (observed ~3m). 15m for
+    # network variance.
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
       - uses: rustsec/audit-check@v2
@@ -231,6 +248,8 @@ jobs:
     # newuidmap is setuid but NoNewPrivileges=true blocks the escalation
     # (newuidmap: write to uid_map failed: Operation not permitted).
     runs-on: [self-hosted, linux, x64, light]
+    # cargo-deny bans/licenses/sources (observed <=20s).
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -249,6 +268,8 @@ jobs:
     needs: [changes, test]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # llvm-cov across the workspace (observed ~8.5m). 30m absorbs a cold cache.
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -285,6 +306,8 @@ jobs:
     # lean-mem class — Miri allocates aggressively and benefits from the 24G
     # MemoryHigh ceiling on smithy lean-mem runners over the 12G rust-cpu cap.
     runs-on: [self-hosted, linux, x64, lean-mem]
+    # Miri instance-model tests (observed ~2m; lean-mem contention can stretch it).
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -310,6 +333,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # proptest at 10x cases (observed ~8m). 30m absorbs a cold cache.
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
@@ -410,6 +435,9 @@ jobs:
   fuzz-smoke:
     name: Fuzz smoke (60s/target)
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # 3 targets x 60s + cold ASan builds (observed ~5m on PRs; this job is
+    # PR-only). 30m ceiling.
+    timeout-minutes: 30
     # Only run on PRs — pushes to main hit the nightly workflow instead.
     needs: [changes]
     if: github.event_name == 'pull_request' && needs.changes.outputs.code == 'true'
@@ -439,6 +467,8 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.code == 'true'
     runs-on: [self-hosted, linux, x64, light]
+    # cargo-vet supply-chain check (observed <=20s).
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -456,6 +486,9 @@ jobs:
   rivet-validate:
     name: Rivet validate (artifacts)
     runs-on: [self-hosted, linux, x64, rust-cpu]
+    # rivet validate + all artifact guardrail self-tests (observed ~21m cold,
+    # dominated by the verification-filter guardrail). 45m is ~2x.
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -580,6 +613,15 @@ jobs:
         run: tools/check_required_contexts.py --self-test
       - name: Required-context guardrail
         run: tools/check_required_contexts.py
+      # Required-context timeouts (#398). Sibling of the guardrail above, reusing
+      # its restricted workflow reader: every job that backs a required context
+      # must declare a static timeout-minutes in (0, 360), so a wedged merge gate
+      # cancels instead of holding a runner (and, with strict=true, the queue) for
+      # the 6-hour default. Same steps-of-this-job rationale as above.
+      - name: Job-timeout guardrail self-test
+        run: tools/check_job_timeouts.py --self-test
+      - name: Job-timeout guardrail
+        run: tools/check_job_timeouts.py
       # Closure MSRV drift (#369). Here rather than in a code-gated job for the
       # same reason as above, and it matters more here than anywhere: the PR
       # that raises the real floor is a `cargo update`, which touches only

.github/workflows/proofs.yml

@@ -36,6 +36,9 @@ jobs:
   changes:
     name: Detect changed paths (proofs)
     runs-on: ubuntu-latest
+    # Trivial classifier (observed <10s); gates the required Lean typecheck.
+    # Enforced by tools/check_job_timeouts.py (#398).
+    timeout-minutes: 5
     outputs:
       code: ${{ steps.filter.outputs.code }}
     steps:

artifacts/requirements.yaml

@@ -5093,3 +5093,68 @@ artifacts:
     status: verified
     release: v0.36.0
     tags: [process, guardrail, ci, tooling, fuzzing]
+
+  - id: REQ-GUARD-CONTEXT-TIMEOUT-001
+    type: requirement
+    title: Every job backing a required context shall declare a bounded timeout-minutes
+    description: >
+      A required status check that hangs is not a wrong-green; it is an
+      indefinite block with no signal (#398). A wedged job leaves its context at
+      "Expected — waiting for status to be reported", which is byte-identical to
+      "still running", and a job with no `timeout-minutes` inherits GitHub's
+      360-minute default before anything cancels it. For up to six hours the PR is
+      unmergeable for a reason nobody can see, the job holds a self-hosted runner,
+      and — branch protection is `strict = true` — every other open PR queues
+      behind it. `Detect changed paths` is the sharpest case: it gates ten of the
+      other jobs, so wedging it stalls the entire suite.
+
+      Measured against the live required set when this landed, 15 of the 18
+      required contexts declared no `timeout-minutes`. That is not hypothetical —
+      it is the blast radius of the first hang, whenever it comes. This is the
+      same shape REQ-GUARD-GATE-EVIDENCE-002 names ("nothing happened" must render
+      differently from "it worked"), applied to the merge gate's own liveness.
+
+      OBLIGATION. Every GATEABLE job — PR-triggered against main, no path filter,
+      no continue-on-error, static name, i.e. exactly the set
+      `check_required_contexts.py` proves must be required — shall declare a
+      `timeout-minutes` that is a static integer strictly inside (0, 360). Missing
+      is the defect; a `${{ }}` template is not a static bound a reader can
+      evaluate; <= 0 is not a number of minutes; >= 360 is no tighter than the
+      default it replaces and buys nothing. The check does NOT assert the value is
+      well-sized ("a few multiples of p95") — that is a human judgement a gate
+      would only be guessing at; it draws the one line that is not a judgement
+      call: an unbounded (or uselessly-bounded) required job must not ship.
+
+      ORACLE (executed, non-vacuous). `tools/check_job_timeouts.py --self-test`
+      carries a 9-row decision table plus two production-corpus assertions. The
+      rows flip a verdict each way on the same tool: a gateable job with a valid
+      timeout exits 0, the same job with no timeout exits 1, and templated /
+      zero / non-integer / >=360 each exit 1 while `359` exits 0 (the upper bound
+      is satisfiable, not merely restrictive). Two rows prove SCOPE — an advisory
+      (continue-on-error) job and a path-scoped (undeliverable) job with no
+      timeout both exit 0, so the gate flags only jobs that can actually block a
+      merge. The corpus assertions fail closed: a scan that finds zero gateable
+      jobs, or a glob that misses `.yaml`, is a blind read, not a PASS.
+      Demonstrated live: against the tree before this change the check exited 1
+      naming all 15 unbounded contexts; against the tree after, it exits 0 with
+      "18 gateable job(s)". The reader is the SAME restricted parser as
+      `check_required_contexts.py` (extended only to capture `timeout-minutes`)
+      and stays `--cross-check`ed against PyYAML on the real corpus.
+
+      PROVEN vs ASSUMED, stated precisely. PROVEN: every job that backs a required
+      context now declares a static bound in (0, 360), and the gate fails closed
+      on a missing, templated, or degenerate bound and on a blind scan. ASSUMED,
+      not proven here: (1) that the numbers chosen are correctly sized — they are
+      a human judgement from the run history, cited per-job in the workflow
+      comments, not a machine-checked property; (2) that GitHub actually cancels a
+      job at its declared `timeout-minutes` — runtime behaviour this gate reads
+      the YAML for but does not exercise; (3) that the GATEABLE set equals the
+      enforced branch-protection set — `check_required_contexts.py` reconciles the
+      committed list to the workflows but cannot read branch protection without an
+      admin token, a gap it states and this REQ inherits. NOT CLAIMED: that
+      advisory jobs are bounded — a hung `continue-on-error` job still wastes a
+      runner, but it cannot wedge the merge gate, so it is out of scope here and
+      left to a successor.
+    status: implemented
+    release: v0.37.0
+    tags: [process, guardrail, ci, tooling, v0370]

artifacts/verification.yaml

@@ -4725,3 +4725,65 @@ artifacts:
 
       - type: verifies
         target: REQ-NC-BUS-PAYLOAD-001
+
+  - id: TEST-GUARD-CONTEXT-TIMEOUT
+    type: feature
+    title: The job-timeout gate fails on an unbounded required job and is blind to advisory ones
+    description: >
+      Verifies REQ-GUARD-CONTEXT-TIMEOUT-001 through the decision table in
+      tools/check_job_timeouts.py --self-test, which runs as its own CI step
+      BEFORE the gate it protects, in the same rivet-validate job as the other
+      artifact guardrails.
+
+      THE LOAD-BEARING CASE is "gateable job with no timeout-minutes" -> exit 1.
+      That was the live state of 15 of the 18 required contexts before this
+      landed; on that same tree GitHub's PR UI shows every one of them as a green,
+      well-behaved check. The case is a fixture because the defect is invisible
+      until a job hangs, and only executing the gate against a job that lacks the
+      key demonstrates it.
+
+      NON-VACUITY IS TESTED BOTH WAYS AND ACROSS THE SCOPE BOUNDARY. A valid
+      timeout exits 0 and a missing one exits 1 on the same tool (distinct inputs,
+      distinct verdicts). Templated (`${{ }}`), zero, non-integer and `>= 360`
+      each exit 1 — a bound a reader cannot evaluate, or one no tighter than the
+      default it replaces, is not a bound — while `359` exits 0, so the upper
+      guard is satisfiable rather than merely restrictive. Two cases pin the
+      SCOPE: a continue-on-error (advisory) job and a path-scoped (undeliverable)
+      job, each with NO timeout, both exit 0, because neither can block a merge;
+      treating either as gateable would report a violation this gate does not own.
+
+      FAIL-CLOSED ON A BLIND SCAN, the recurring hazard in this family. Two
+      production-corpus assertions run against the real WORKFLOW_GLOB: a scan that
+      finds zero gateable jobs fails (on this repo there are 18, so zero means the
+      reader went blind, which must not render as PASS), and the glob must match
+      `.yaml` as well as `.yml` (Actions loads both; a glob that sees one reports
+      PASS on a directory it never fully read).
+
+      MEASURED AGAINST THE REAL TREE, not only fixtures. Before this change the
+      production run exited 1 naming all 15 unbounded required contexts; after
+      adding a `timeout-minutes` to each of the 15 it exits 0 with "18 gateable
+      job(s)". The reader is check_required_contexts.py's restricted parser
+      (extended only to capture `timeout-minutes`), still --cross-checked against
+      PyYAML on the real corpus, so the extension did not silently diverge.
+
+      HONEST CEILING. The table verifies the gate's own logic and its read of the
+      committed workflows; it does NOT verify that GitHub cancels a job at the
+      declared minute (runtime behaviour, not asserted), nor that the chosen
+      values are well-sized (a human judgement cited in the workflow comments),
+      nor that branch protection enforces exactly this set (the LIST-to-branch
+      -protection gap check_required_contexts.py states and this inherits). Like
+      its siblings it assumes python3 on the runner class; the failure mode is
+      fail-closed (a missing interpreter 127s the required step red), so the PR
+      that lands this is the measurement.
+    fields:
+      method: automated-test
+      steps:
+        - run: tools/check_job_timeouts.py --self-test
+        - run: tools/check_job_timeouts.py
+    status: implemented
+    tags: [process, guardrail, ci, tooling, v0370]
+    links:
+      - type: satisfies
+        target: REQ-GUARD-CONTEXT-TIMEOUT-001
+      - type: verifies
+        target: REQ-GUARD-CONTEXT-TIMEOUT-001