diff --git a/.claude/hooks/agent-bash-gate.sh b/.claude/hooks/agent-bash-gate.sh index 8a45627c..8ddac89e 100755 --- a/.claude/hooks/agent-bash-gate.sh +++ b/.claude/hooks/agent-bash-gate.sh @@ -57,7 +57,7 @@ if printf '%s\n' "$stripped" | grep -qE '(^|[[:space:];|&]+)gh[[:space:]]+pr[[:s fi # gh pr create / gh pr edit --title: validate the title against the SAME -# Conventional-Commits rule the required `PR housekeeping` check enforces +# Conventional-Commits rule the required `CI` check's `PR title` job enforces # (scripts/lint-pr-title.sh is the shared rule) — so a too-long or wrong-format # title is caught locally BEFORE the PR exists, not after the required check # fails. Extract the quoted --title/-t value from the ORIGINAL command (the diff --git a/.githooks/pre-push b/.githooks/pre-push index 779cf633..6050848b 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -1,21 +1,74 @@ #!/usr/bin/env bash -# Blocks `git push` unless `make ci` validated the tree of each ref tip. -# Tree-keyed. Bypass: `git push --no-verify` (humans only). +# Blocks `git push` unless the tree of each pushed ref tip was validated +# locally. The bar scales with what changed — classified by the SAME +# allowlist CI uses (scripts/classify-paths.sh), so local and CI agree on +# what a change "is": +# - any code change → full `make ci` must have passed (tree-keyed marker +# tmp/ci-passed-tree-), exactly as before; +# - docs/prose-only → `make verify` is enough (CI skips the Go/SDK suites +# for these too), so only the verify marker is required — no spinning +# up the local Docker suites for a typo fix. pre-commit already runs +# `make verify`, so this is usually a zero-extra-work push. +# Fail-closed: if the change set can't be classified (no resolvable base, +# classifier error), fall back to requiring the full `make ci` marker. +# Bypass: `git push --no-verify` (humans only). set -uo pipefail cd "$(git rev-parse --show-toplevel)" || exit 1 -while read -r _ local_sha _ _; do - [ -z "${local_sha:-}" ] && continue - [ "$local_sha" = "0000000000000000000000000000000000000000" ] && continue - marker=$(scripts/ci-marker.sh path-for-commit "$local_sha" 2>/dev/null) || { - echo "🛑 pre-push: can't resolve tree for ${local_sha:0:8}." >&2; exit 1 +zero=0000000000000000000000000000000000000000 + +# Net changed files for a pushed ref (base..tip), one per line. New branch +# (no remote tip) → diff against the merge-base with origin/main. Returns +# non-zero when no base resolves, so the caller fails closed. +changed_files() { + local tip="$1" remote="$2" base + if [ "$remote" != "$zero" ]; then + base="$remote" + else + base="$(git merge-base origin/main "$tip" 2>/dev/null)" || return 1 + fi + git diff --name-only "$base" "$tip" 2>/dev/null +} + +block_ci() { # — require the full make-ci marker + local sha="$1" marker tree + marker=$(scripts/ci-marker.sh path-for-commit "$sha" 2>/dev/null) || { + echo "🛑 pre-push: can't resolve tree for ${sha:0:8}." >&2; exit 1 } if [ ! -f "$marker" ]; then tree="${marker#tmp/ci-passed-tree-}" - echo "🛑 pre-push: 'make ci' has not been run for ${local_sha:0:8} (tree ${tree:0:8}). Run 'make ci' or 'git push --no-verify'." >&2 + echo "🛑 pre-push: 'make ci' has not been run for ${sha:0:8} (tree ${tree:0:8}). Run 'make ci' or 'git push --no-verify'." >&2 exit 1 fi +} + +while read -r _ local_sha _ remote_sha; do + [ -z "${local_sha:-}" ] && continue + [ "$local_sha" = "$zero" ] && continue # branch deletion — nothing to validate + + # Classify the push, failing closed to a code change on any hiccup. + code=true + if files="$(changed_files "$local_sha" "${remote_sha:-$zero}")"; then + verdict="$(printf '%s\n' "$files" | scripts/classify-paths.sh 2>/dev/null || true)" + case "$verdict" in + *code=false*) code=false ;; + esac + fi + + if [ "$code" = false ]; then + marker=$(scripts/ci-marker.sh verify-path-for-commit "$local_sha" 2>/dev/null) || { + echo "🛑 pre-push: can't resolve tree for ${local_sha:0:8}." >&2; exit 1 + } + if [ ! -f "$marker" ]; then + echo "🛑 pre-push: docs-only push, but 'make verify' hasn't passed for ${local_sha:0:8}. Run 'make verify' (no Docker) or 'git push --no-verify'." >&2 + exit 1 + fi + echo "✔ pre-push: docs/prose-only change — 'make verify' marker present for ${local_sha:0:8} (CI skips the suites too)." >&2 + continue + fi + + block_ci "$local_sha" done exit 0 diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 983cc0cc..f16fc5dd 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -1,69 +1,107 @@ -# Restores caches + sets up Go/pnpm/Node for `make ci`. Composite -# actions can't self-checkout, so ci.yml must checkout first. Saves -# live in ci.yml gated on the `*-cache-hit` outputs below. +# Caches + toolchains for the CI jobs, in one place. Composite actions +# can't self-checkout, so the consuming job must checkout first. Each CI +# job opts into exactly the toolchains it needs via the inputs below. # -# 1. Go module + build cache keyed on every go.sum. `restore-keys` -# keeps the build cache warm across go.sum bumps — avoids -# setup-go's exact-key-only cache, see actions/setup-go#357. +# Caching is OWNED here end-to-end: each cache is a nested `actions/cache` +# step, whose automatic post-job step saves on an exact-key miss — no save +# steps in ci.yml, so restore and save can never drift. Two consequences +# to know about (both accepted for the boilerplate this kills): +# - saves happen at the END of the job (post steps), and only when the +# job SUCCEEDED — a failed run rebuilds from restore-keys next time; +# - when several jobs miss the same key in one run (e.g. a lockfile +# bump), each saves; the backend keeps the first and the rest log a +# benign "already exists" warning. # +# The cache inventory (full architecture: .github/workflows/README.md): +# +# 1. Go module + build cache keyed on every go.sum, `gobuild-v2-` key +# family. `restore-keys` keeps the build cache warm across go.sum +# bumps — avoids setup-go's exact-key-only cache (actions/setup-go#357). +# `go-cache-suffix` partitions the cache per job: the unit, +# integration, and e2e jobs compile with different flags +# (-race/-cover/-coverpkg/-tags), so a shared entry would only ever +# be warm for whichever job saved it. Cross-suffix restore-keys +# still share the (identical) module cache on a cold start. # 2. golangci-lint binary + analysis cache keyed on Makefile + # .golangci.yml. Analysis cache is the win (~10s warm vs ~90s). -# +# Only the lint job needs it. # 3. pnpm store (path resolved at runtime) keyed on the root -# lockfile — the three pnpm projects (clients/ts, tests/e2e/sdk, -# docs) share one workspace lockfile + store. Path is dynamic because -# pnpm's documented default ~/.local/share/pnpm/store only -# applies when $HOME and the project tree share a mount; on some -# runners it falls back to a workspace-relative path. Hard-coding -# the default silently fails the save with a Path Validation -# Error. -# +# lockfile — the pnpm workspace projects share one lockfile + +# store. Path is dynamic because pnpm's documented default +# ~/.local/share/pnpm/store only applies when $HOME and the project +# tree share a mount; on some runners it falls back to a +# workspace-relative path. Hard-coding the default silently fails +# the save with a Path Validation Error. # 4. Playwright browser cache (~/.cache/ms-playwright) keyed on the # root lockfile. ~130 MB Chromium download otherwise re-fetched -# every run. See #132. -# +# every docs build (rehype-mermaid renders via headless Chrome). +# Only the docs-build job needs it. See #132. # 5. Astro content-collection cache (docs/.astro/) keyed on the root -# lockfile + astro.config.mjs. Speeds up warm `astro build` — -# unchanged content skips the parse + transform pipeline. See -# #132. +# lockfile + astro.config.mjs. Speeds up warm `astro check` / +# `astro build` — unchanged content skips the parse + transform +# pipeline. See #132. name: Setup CI environment -description: Restores caches and sets up Go + pnpm + Node for WaveHouse CI +description: Caches (restore + automatic post-job save) and toolchains for WaveHouse CI -# Cache-hit flags + resolved pnpm store path surfaced to ci.yml. +inputs: + go: + description: "Set up the Go toolchain + the Go module/build cache" + default: "true" + go-cache-suffix: + description: "Per-job Go build-cache partition, e.g. '-unit' (different jobs compile with different flags). Empty = the shared default key." + default: "" + golangci: + description: "Cache the golangci-lint binary + analysis cache (lint job only)" + default: "false" + node: + description: "Set up pnpm + Node and cache the pnpm store" + default: "true" + playwright: + description: "Cache the Playwright Chromium install (docs build only)" + default: "false" + astro: + description: "Cache the Astro content collections (docs check/build)" + default: "false" + +# Exact-key cache-hit flags, for consumers that want to skip work on a +# warm cache (e.g. docs-build's pnpm-store prune). Saves are NOT gated on +# these from the outside — the nested actions/cache steps handle their +# own post-job saves. outputs: - gobuild-cache-hit: - description: "true if the exact-key Go module + build cache was restored" - value: ${{ steps.gobuild-cache.outputs.cache-hit }} - golangci-cache-hit: - description: "true if the exact-key golangci-lint cache was restored" - value: ${{ steps.golangci-cache.outputs.cache-hit }} pnpm-cache-hit: description: "true if the exact-key pnpm store cache was restored" value: ${{ steps.pnpm-cache.outputs.cache-hit }} - pnpm-store-path: - description: "Absolute path to the pnpm content-addressable store on this runner — pass to actions/cache/save in the consuming workflow." - value: ${{ steps.pnpm-store.outputs.path }} - playwright-cache-hit: - description: "true if the exact-key Playwright browser cache was restored" - value: ${{ steps.playwright-cache.outputs.cache-hit }} - astro-cache-hit: - description: "true if the exact-key Astro content cache was restored" - value: ${{ steps.astro-cache.outputs.cache-hit }} runs: using: composite steps: - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + # Key version (v2): bumped when the cache's expected CONTENTS change + # shape — v2 added the go toolchain itself (~/go/pkg/mod/golang.org/ + # toolchain, see the GOTOOLCHAIN note below). Saves only happen on an + # exact-key miss, so without a bump the pre-change entry would + # exact-hit forever and the new content would never be saved. + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + if: ${{ inputs.go == 'true' }} id: gobuild-cache with: path: | ~/go/pkg/mod ~/.cache/go-build - key: gobuild-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + key: gobuild-v2-${{ runner.os }}-go${{ inputs.go-cache-suffix }}-${{ hashFiles('**/go.sum') }} + # Same-suffix prefix first (this job's flavor across go.sum bumps), + # then the bare prefix as a cold-start fallback — it matches any + # other job's suffixed entry, which still carries the shared module + # cache even if its build objects don't apply. The unversioned (v1) + # prefixes are transitional warm-start fallbacks; drop them once a + # v2 entry has been saved on main. restore-keys: | + gobuild-v2-${{ runner.os }}-go${{ inputs.go-cache-suffix }}- + gobuild-v2-${{ runner.os }}-go- + gobuild-${{ runner.os }}-go${{ inputs.go-cache-suffix }}- gobuild-${{ runner.os }}-go- - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + if: ${{ inputs.golangci == 'true' }} id: golangci-cache with: path: | @@ -73,25 +111,38 @@ runs: restore-keys: | golangci-${{ runner.os }}- - # `cache: false` — Go cache is managed explicitly above. - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - cache: false + # No actions/setup-go: it spends ~9s/job downloading + extracting a + # toolchain the runner can already provide. The image's preinstalled + # `go` + GOTOOLCHAIN=auto (the Go ≥1.21 default) resolve go.mod's + # pinned version exactly — and the auto-fetched toolchain lands in + # ~/go/pkg/mod/golang.org/toolchain, i.e. inside the module cache + # restored above, so warm runs download nothing. A cold cache (first + # run after a key rotation, when the image go is older than go.mod's + # pin) pays a one-time toolchain fetch from proxy.golang.org here — + # this step makes that cost visible and the post-job save captures it. + # Trade-off: setup-go's inline problem matchers (PR-file annotations + # on compile errors) are gone; the log output is unchanged. + - name: Verify Go resolves go.mod's toolchain + if: ${{ inputs.go == 'true' }} + shell: bash + run: go version - # pnpm must install before its cache restore so the next step can - # ask pnpm for the actual store path. + # pnpm must install before its cache step so the store path can be + # resolved from pnpm itself. - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + if: ${{ inputs.node == 'true' }} with: version: "11.1.3" run_install: false - name: Resolve pnpm store directory + if: ${{ inputs.node == 'true' }} id: pnpm-store shell: bash run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + if: ${{ inputs.node == 'true' }} id: pnpm-cache with: path: ${{ steps.pnpm-store.outputs.path }} @@ -100,11 +151,12 @@ runs: pnpm-${{ runner.os }}- # Playwright Chromium binary (~130 MB) lives outside the pnpm store at - # ~/.cache/ms-playwright. Without this cache, every CI run cold-pulls + # ~/.cache/ms-playwright. Without this cache, every docs build cold-pulls # Chromium during install-playwright-docs — adds 30-60s per run. Keyed # on the root pnpm-lock.yaml since that's where the Playwright version # is locked. Closes the Playwright half of #132. - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + if: ${{ inputs.playwright == 'true' }} id: playwright-cache with: path: ~/.cache/ms-playwright @@ -113,13 +165,14 @@ runs: playwright-${{ runner.os }}- # Astro's content-collection cache (docs/.astro/data-store.json plus - # generated content types). Restored across runs so unchanged Markdown + # generated content types). Kept across runs so unchanged Markdown # / MDX content skips the parse + transform pipeline. Keyed on # lockfile + astro.config.mjs since plugin-chain changes invalidate # the cache shape; intra-key content changes are caught by Astro's # own file-hash check inside data-store.json. Closes the build-cache # half of #132. - - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + if: ${{ inputs.astro == 'true' }} id: astro-cache with: path: docs/.astro @@ -128,5 +181,6 @@ runs: astro-${{ runner.os }}- - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + if: ${{ inputs.node == 'true' }} with: node-version-file: ".nvmrc" diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000..8a1ea527 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,261 @@ +# CI architecture + +How `ci.yml` is shaped and why. This is the canonical reference — the +workflow file's comments only explain what's local to a step, and +[development.md](../../docs/src/content/docs/development.md) carries the +contributor-facing summary. Wall-clock for a full PR run: **~3m15s push → +all green** (e2e ~170s is the long pole; the `coverage` job overlaps its +setup with the suites and merges within ~10s of e2e finishing, then the +~4s aggregator. Reliable now that the variable Cloudflare `docs-preview` +deploy is non-gating — only `docs-build` gates. Was 5m54s before the +2026-06 reshape, and ~4m when the +coverage job still serialized its setup via `needs`). + +## The graph + +```mermaid +graph TB + changes["changes (classify)"] --> unit["unit"] + changes --> integration["integration"] + changes --> e2e["e2e — build SDK+cover → suite"] + changes --> docsbuild["docs-build"] + changes --> coverage["coverage — poll fragments → merge → gate"] + docsbuild --> preview["docs-preview (PRs) — non-gating"] + unit -. "coverage-unit (poll)" .-> coverage + integration -. "coverage-integration (poll)" .-> coverage + e2e -. "coverage-e2e (poll)" .-> coverage + title["title (PRs)"] --> ci["CI (aggregator — sole required check)"] + lint["lint"] --> ci + coverage --> ci + e2e --> ci + unit --> ci + integration --> ci + docsbuild --> ci + deploy["docs-deploy (main)"] --> ci + changes --> deploy + lint --> deploy + docsbuild --> deploy + coverage --> deploy +``` + +Solid arrows are `needs` edges. Dotted arrows are **artifact polls** +(invariant 2): the `coverage` job starts off `changes` alone and polls +for the suites' fragments rather than `needs`-ing the suites, so its setup +overlaps them. + +## Design invariants + +Break one of these knowingly or not at all. + +1. **The aggregator job named `CI` is the only required status check.** + The `main branch protection` ruleset requires `CI` and nothing else. + The aggregator fails on any `failure`/`cancelled` need and treats + `skipped` as passing — so path-filtered jobs (docs-only PRs skip the + Go suites) and event-filtered jobs (title on pushes, deploys on PRs) + never orphan the required check, and adding/renaming jobs never + requires a ruleset edit. Consequence: every job that must gate merges + **must be in the aggregator's `needs` list**. Two jobs are deliberately + non-gating and excluded: `timing` (advisory wall-clock table) and + `docs-preview` (the convenience Cloudflare preview deploy — `docs-build` + already validates the build and *is* a need, so only the build gates; + the preview deploy reports its own "Docs preview" check but, slow or + failed, never delays or reds `CI`). + +2. **A dedicated `coverage` job applies the consolidated gate, polling — + not `needs`-ing — the suites.** Each suite (`unit`, `integration`, + `e2e`) runs with `COV_DEFER=1` and uploads a `coverage-` + fragment; the `coverage` job runs `make cov` (merge + every threshold + gate) over all three — exactly like local `make ci`'s final step. + Keeping it a separate job (not folded into e2e's tail) decouples the + gate result from the e2e suite's pass/fail. Crucially it is + `needs: changes` **only, not the suites**: a `needs` edge is a + *scheduling* barrier — GitHub won't pick up a runner, check out, restore + caches, or `pnpm install` until the needed jobs finish — so needing the + suites would serialize this job's ~50s of setup onto the critical path + after the last suite, for nothing (the setup doesn't depend on their + results). Instead it starts at run creation, runs its setup in parallel + with the suites, and blocks only at the merge by polling for the three + fragments with [`scripts/ci/wait-artifact.sh`](../../scripts/ci/wait-artifact.sh) + (fails fast if a producer concluded without producing). Tail on the + critical path: ~10s, not ~50s. **The aggregator and `docs-deploy` must + keep `coverage` *and* every suite in their `needs`** — the suites + directly (a suite failure must red the gate even though `coverage` + no longer needs them), and `coverage` (else a coverage-gate failure + wouldn't block merge or a prod deploy). + +3. **e2e builds its own inputs and mirrors local `make test-e2e`.** It + compiles the SDK dist + cover binary itself (`make -j test-e2e`, warm + per-suffix cache) rather than waiting on a builder job, and runs the + suite exactly as a developer does — one orchestrator, one ClickHouse + testcontainer, sequential files. The ClickHouse image pulls in the + background while caches restore (also in the integration job). + +4. **One change classifier, split into a pure core + a CI wrapper.** The + pure allowlist — file list on stdin ⇒ `code`/`docs` — lives in + [`scripts/classify-paths.sh`](../../scripts/classify-paths.sh), + dependency-free and unit-tested by + [`scripts/classify-paths.test.sh`](../../scripts/classify-paths.test.sh) + (`make test-classify-paths`, a `verify` leaf) so the allowlists can't + silently regress. The `changes` job runs the thin wrapper + [`scripts/ci/classify-changes.sh`](../../scripts/ci/classify-changes.sh), + which adds the CI-only policy (API file-list fetch + fail-closed: + pushes, dispatches, API hiccups ⇒ `code=true`) on top. Keeping the core + pure means the local git hooks can share it (`git diff --name-only | + scripts/classify-paths.sh`). The `code`/`docs` outputs gate the suites + and docs jobs — gate on these, never on workflow-level `paths:` + filters, which would orphan the required check (invariant 1). + +5. **Trust domains.** Jobs that can reach deploy secrets (`docs-preview`, + `docs-deploy`) check out **trusted `main`** and execute only files + resolved from it — wrangler, the worker source, `wrangler.jsonc`, and + any `scripts/ci/*.sh` they call ([#305](https://github.com/Wave-RF/WaveHouse/issues/305)). + The only PR-derived input they touch is the static `docs-dist` + artifact, consumed as data. Inline `run:` blocks in those jobs are + acceptable (the workflow file itself is the reviewed surface); + PR-tree *files* are not. Everything else (suites, lint, docs-build) + runs the PR tree with no secrets beyond a read-mostly `GITHUB_TOKEN`. + Fork PRs: secrets are absent and `docs-preview` skips itself. + +6. **Caches are owned end-to-end by `setup-env`** + ([.github/actions/setup-env](../actions/setup-env/action.yml)): each + cache is a nested `actions/cache` step that restores inline and saves + automatically at job end on an exact-key miss. No save steps in + `ci.yml`. Trade-offs accepted: failed jobs don't save (restore-keys + cushion the next run), and concurrent same-key misses produce benign + "already exists" warnings. + +## Merge queue + +PRs land through a **merge queue**: "Merge when ready" enqueues the PR, +GitHub builds a merge-group ref (current main ⊕ the PRs ahead ⊕ this +PR), runs the required `CI` check against **that**, and fast-forwards +main only on green. This is the integration gate — it catches semantic +conflicts with a main that advanced after the PR's own run, replaces +the old "require branches to be up to date" rule (PRs no longer show +"out of date", and nobody clicks Update-branch), and never touches the +PR branch itself (so `require_last_push_approval` is never reset by it). + +How a `merge_group` run flows through the DAG: the classifier treats it +like a push (full suite — the queue never skips code checks; `docs` +still gates docs-build), while `title` (already validated on the PR), +`docs-preview` (PR-scoped), and `docs-deploy` (push-scoped) sit out and +the aggregator counts their skips as passes, exactly like any other +event-filtered run. **Removing the `merge_group:` trigger from ci.yml +would hang every queued PR** — no trigger means the required `CI` check +never reports on the merge group, and entries bounce out only after the +60-minute check timeout. + +Queue settings live in the `main branch protection` ruleset's +`merge_queue` rule: squash merges, land-as-ready (`min_entries_to_merge: +1`, no batching wait), up to 5 speculative builds. + +## Cache inventory + +| Cache | Key | Saved by | Notes | +|---|---|---|---| +| Go modules + build | `gobuild-v2--go-` | every Go job (own suffix) | Suffix partitions by compile flavor (`-lint`, `-unit`, `-integration`, `-e2e-cov`, `-cov`). v2 = the GOTOOLCHAIN=auto toolchain rides in `~/go/pkg/mod` (no setup-go). **TODO:** drop the unversioned v1 restore-keys fallbacks once a v2 entry exists on main. | +| golangci binary + analysis | `golangci--` | lint | Analysis cache: ~10s warm vs ~90s. `.bin` also carries shellcheck + actionlint. | +| pnpm store | `pnpm--` | any node job on miss | Store path resolved from pnpm at runtime. docs-build prunes before its save on a key rotation. | +| Playwright Chromium | `playwright--` | docs-build | rehype-mermaid renders via headless Chrome at docs build. | +| Astro content collections | `astro--` | lint / docs-build | Warm `astro check`/`build` skip unchanged content. | + +Key-versioning policy: bump the `v` prefix whenever the cache's +expected *contents* change shape — saves only fire on an exact-key miss, +so without a bump the old entry exact-hits forever and the new content +is never captured. Keep the old prefixes as transitional restore-keys, +then delete them once main has saved the new version. + +## Timing (steady state, full pipeline) + +The non-gating **Timing summary** job writes a per-job wall-clock table +to every run's Summary page. Reference shape: + +| Job | Starts | Duration | +|---|---:|---:| +| e2e (long pole) | +8s | ~170s — ~25 setup, ~120 suite (15 builds ∥ image prefetch, ~100 vitest, ~10 cover-binary OTel exit [#288](https://github.com/Wave-RF/WaveHouse/issues/288)), ~5 fragment upload | +| coverage | +18s | ~50s setup overlaps the suites, then idle-polls; ~10s critical-path tail (poll-detect + download + ~3s merge) after e2e | +| integration | +8s | ~85s | +| docs-build (gates) → docs-preview (non-gating) | +8s | build ~80s; the Cloudflare preview deploy is ~1.5–3min and varies, but doesn't gate — only `docs-build` does | +| lint / unit | +2s / +8s | ~65s / ~50s | +| CI aggregator | after coverage / docs-build | ~4s | + +## Deferred optimizations + +Designed and measured during the 2026-06 reshape, then backed out to +keep CI simple and in parity with local `make test-e2e`. If the e2e +suite's wall-clock becomes a problem again, start here: + +- **e2e sharding** (the big one, ~60s): N concurrent orchestrators in + one runner, each with its own ClickHouse + server — file-level + parallelism is impossible *within* one server (shared global policy + state, [#214](https://github.com/Wave-RF/WaveHouse/issues/214)), but + isolated stacks dissolve the constraint with zero test changes. + Measured green at 3 shards: suite wall 100s → ~40s (floor = the + slowest file, `ingest.test.ts` at 36s), CPU contention negligible on + the 4-core runners. Needs: per-shard scratch paths + vitest file + filters in the orchestrator, a shard-map driver script, per-shard TS + coverage dirs nyc-merged back into `ts-e2e/` (Go covdata can share + one GOCOVERDIR — covcounters are pid-stamped). See PR #312's history + (commit `ed1db1c`) for a working implementation. +- **No-`needs` e2e** (~5s): e2e classifies the change set itself and + starts at run creation; requires a second classifier run plus an + aggregator cross-check so drift fails closed. Also in `ed1db1c`. +- **docs-preview artifact-poll** (~0s on today's critical path): preview + does its trusted-main setup in parallel with docs-build and polls for + `docs-dist`. Only worth it if e2e drops under ~150s again. + +## Adding a job + +1. Pick the gate: must it block merges? Add it to **both** the + aggregator's and `timing`'s `needs` lists. Advisory-only? Mirror + `timing` (`continue-on-error: true`, not in the aggregator's needs). +2. Gate on the change set via `needs: changes` + `if:` on its outputs — + never with workflow-level `paths` filters (they'd orphan the required + check, invariant 1). +3. Use `setup-env` with a fresh `go-cache-suffix` if it compiles Go with + new flags; never add cache save steps (invariant 6). +4. Need a build product / data from another job? Upload it as an artifact + there, then either `needs` the producer + `download-artifact` (simple, + but serializes this job's setup behind the producer), or — when this + job has its own setup to overlap and sits on the critical path — start + it off `changes` and poll with + [`scripts/ci/wait-artifact.sh`](../../scripts/ci/wait-artifact.sh) (see + the `coverage` job). The poll holds a runner idle while waiting; worth + it to keep setup off the critical path. +5. Declare least-privilege `permissions:` on the job; the workflow + default is `contents: read`. +6. Nontrivial logic goes in `scripts/ci/*.sh` (shellcheck-gated via + `make lint-sh`), not inline YAML — except in the trusted-main deploy + jobs (invariant 5) where inline is the point. +7. Run `make lint-gha` (actionlint) before pushing; it's part of + `make verify`. + +## Debugging a slow or red run + +- Start at the run's **Summary** page: the Timing table says where the + wall-clock went; the coverage table comes from the `coverage` job. +- `coverage` red in "Wait for coverage fragments" means a suite failed or + was cancelled before uploading its fragment — the aggregator is already + red from that suite. Fix that job first; it's not a `coverage` bug. +- Re-run failed jobs is safe everywhere: fragments/dist artifacts + persist per-run, `download-artifact` finds them instantly, and the + sticky preview comment updates in place. + +## Transition notes (delete when done) + +- **v1 gobuild restore-keys** in setup-env: drop after the first main + run saves `gobuild-v2-*` entries. +- **docs-preview comment guard**: the comment step skips with a notice + until `scripts/ci/docs-preview-comment.sh` exists on main (it executes + from the trusted-main checkout). Remove the guard after merge. +- **PR-title script guard**: the `PR title` job's step skips with a notice + until `scripts/ci/check-pr-title.sh` exists on main (same trusted-main + checkout). Title enforcement is advisory via `PR housekeeping` until + then. Remove the guard after merge. +- **Merge-queue ruleset flip**: AFTER the PR adding the `merge_group:` + trigger lands on main (ordering matters — flipping first strands every + other queued PR with no CI run), update the `main branch protection` + ruleset: add the `merge_queue` rule and set + `strict_required_status_checks_policy: false` (prepared payload: + `gh api -X PUT repos/Wave-RF/WaveHouse/rulesets/15353356 --input + tmp/ruleset-merge-queue.json`). Then delete this note. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bae2dc22..d02a58e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,53 @@ name: CI +# Job DAG over the same Makefile targets `make ci` runs locally — local +# `make ci` stays the dev mirror; CI spreads the phases across free +# 4-core public-repo runners and shapes the graph for wall-clock. +# +# READ FIRST: .github/workflows/README.md — the architecture doc. It has +# the DAG diagram, the design invariants (sole required check, the +# coverage job, cache key policy, trust domains), and the +# how-to-add-a-job recipe. Comments in this file explain only what's +# local to a step. +# +# The short version: +# - `changes` classifies the change set once; the test/docs jobs hang +# off it and skip when their inputs didn't change. +# - e2e (the long pole) builds its own inputs and runs the suite +# exactly like local `make test-e2e`; each suite uploads a raw +# coverage fragment. +# - the `coverage` job merges those fragments and applies every +# threshold gate once (`make cov`) after all suites finish — exactly +# like local `make ci`'s final step. +# - the aggregator job NAMED "CI" is the ruleset's sole required check +# (skipped jobs pass). + on: push: branches: [main] pull_request: # `ready_for_review` isn't in the default trigger set — without it, - # draft → ready transitions don't re-trigger CI. + # draft → ready transitions don't re-trigger CI. `edited` is left out + # deliberately: a title/description edit would cancel + restart a full + # in-flight run via the concurrency group. Title edits are instead + # nudged along by housekeeping.yml re-running the failed title job. types: [opened, synchronize, reopened, ready_for_review] branches: [main] + # Merge queue: every queued PR gets a CI run against its merge-group ref + # (main ⊕ the PRs ahead of it ⊕ this PR) — the integration gate that + # replaces the old "require branches to be up to date" rule. Event- + # filtered jobs sit out (title was validated on the PR; the deploys are + # push/PR-scoped); everything else runs. A workflow WITHOUT this trigger + # never reports the required `CI` check on merge groups, so queued PRs + # would hang — keep it even if the queue is temporarily disabled. + merge_group: # Manual re-run escape hatch — `gh workflow run CI --ref `. workflow_dispatch: +# Default-deny; each job below redeclares exactly what it needs. Only the +# docs-preview job (sticky PR comment) holds a write scope. permissions: - contents: write - # The docs-deploy tail steps post a sticky preview-URL comment on PRs. - pull-requests: write + contents: read # PR pushes: cancel in-flight, only the latest commit matters. Main pushes: # queue serially so every commit gets its own run. @@ -23,84 +56,317 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - ci: - # Single job: `make ci` runs every gate in one process so caches and - # build artifacts stay hot across phases. Parallelism lives in the - # Makefile. - name: CI + # ── Change detection ─────────────────────────────────────────────── + # Classifies the changed files (scripts/ci/classify-changes.sh — the + # single source of truth for the code/docs outputs and their fail-closed + # rules) so the test/docs jobs can skip work. + changes: + name: Detect changes + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + code: ${{ steps.detect.outputs.code }} + docs: ${{ steps.detect.outputs.docs }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Classify changed files + id: detect + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + # On merge_group events the diff base is the merge group's own + # base (current main), not an `event.before` — `||` picks + # whichever the event provides. + PUSH_BEFORE: ${{ github.event.merge_group.base_sha || github.event.before }} + PUSH_SHA: ${{ github.sha }} + run: scripts/ci/classify-changes.sh >> "$GITHUB_OUTPUT" + + # ── PR title (Conventional Commits) ──────────────────────────────── + # The blocking half of the title gate (housekeeping.yml keeps the sticky + # explainer comment, which needs pull_request_target write perms on fork + # PRs). Lives under the CI aggregator so "CI" stays the only required + # check. The script runs from a checkout of the DEFAULT branch — never + # the PR tree — so a PR can't tamper with its own validator. + title: + name: PR title + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + persist-credentials: false + - name: Check Conventional Commits format + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + # Logic lives in scripts/ci/check-pr-title.sh (shellcheck-gated), + # resolved from this trusted-main checkout like scripts/lint-pr-title.sh. + # Transition guard: until this PR merges, main doesn't have the + # script yet — skip with a notice (housekeeping.yml's advisory + # mirror still reds a bad title, and this PR's own title is valid). + # Drop the guard once it's on main. See README "Transition notes". + run: | + if [ ! -f scripts/ci/check-pr-title.sh ]; then + echo "::notice::scripts/ci/check-pr-title.sh is not on the default branch yet — title checked by PR housekeeping during transition." + exit 0 + fi + scripts/ci/check-pr-title.sh + + # ── Static checks ────────────────────────────────────────────────── + lint: + name: Lint runs-on: ubuntu-latest - # Public-repo ubuntu-latest (4-core). Warm: ~5-10 min. Cold: ~15. - # 45 leaves headroom + a hung-job cap. - timeout-minutes: 45 + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - id: setup + uses: ./.github/actions/setup-env + with: + go-cache-suffix: "-lint" + golangci: "true" + astro: "true" # check-docs (astro check) reuses the content cache + # tidy + gofumpt + golangci + vulncheck + shellcheck + actionlint + + # Biome + markdownlint + misspell + astro check + tsc. No Docker, + # no browser. (Cache saves are automatic — setup-env's nested + # actions/cache steps save at job end on an exact-key miss.) + - name: Run static checks + run: make verify + + # ── Docs site build ──────────────────────────────────────────────── + # Astro site → docs-dist artifact, feeding the two deploy jobs. Pure + # pnpm (build-docs → check-docs → build-ts needs no Go toolchain), and + # only runs when a docs-affecting file changed — Go binaries are no + # longer built here: compile breakage is already caught by lint + # (golangci type-checks every package), the test suites, and the e2e + # job linking the cover binary; release builds by goreleaser-validate / + # publish-dev. + docs-build: + name: Docs build + needs: changes + if: needs.changes.outputs.docs == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 steps: - # Composite actions can't self-checkout — checkout first. # fetch-depth: 0 (full history) so the docs build can read each page's # real last-commit date from git: Starlight's `lastUpdated` footer - # silently falls back to the build time on the default shallow (depth-1) - # clone, making every page look freshly edited. + # silently falls back to the build time on the default shallow + # (depth-1) clone, making every page look freshly edited. - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 - + persist-credentials: false - id: setup uses: ./.github/actions/setup-env - - - name: Run CI pipeline - # Verify + builds + unit/SDK tests in parallel, then integration → - # e2e → cov sequentially. Mirrors local `make ci` exactly. - run: make ci - - # Saves run on every branch, gated on `*-cache-hit != 'true'` so a - # same-key re-push is a no-op. PRs inherit main's cache scope, so a - # main-push save warms every PR. Saves mirror restores in setup-env - # — keep paths and keys in sync. - - name: Save Go module + build cache - if: steps.setup.outputs.gobuild-cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: gobuild-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - - - name: Save golangci-lint cache - if: steps.setup.outputs.golangci-cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + go: "false" + playwright: "true" # rehype-mermaid renders diagrams via Chromium + astro: "true" + - name: Build docs site + run: make build-docs + - name: Upload docs site + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - path: | - .bin - ~/.cache/golangci-lint - key: golangci-${{ runner.os }}-${{ hashFiles('Makefile', '.golangci.yml') }} - - # Drop store entries unreferenced by any workspace project before - # save, so the cache doesn't grow unboundedly as deps churn. + name: docs-dist + path: docs/dist + if-no-files-found: error + retention-days: 7 + overwrite: true # re-runs of this job re-upload + # On a lockfile change (= pnpm cache-key rotation; lockfile changes + # always flip changes.docs=true, so this job runs), drop store + # entries unreferenced by any workspace project before the post-job + # save captures the store — else the cache grows unboundedly as + # deps churn. Runs after the artifact upload, so docs-preview's + # poll never waits on it. - name: Prune pnpm store before save if: steps.setup.outputs.pnpm-cache-hit != 'true' - shell: bash run: pnpm store prune - # Path resolved dynamically — see setup-env header. - - name: Save pnpm store cache - if: steps.setup.outputs.pnpm-cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + # ── Test suites ──────────────────────────────────────────────────── + # COV_DEFER=1: collect coverage, skip the per-suite render + gate — + # each suite just uploads its raw fragment and the `coverage` job + # applies every gate once, exactly like local `make ci`. + unit: + name: Unit tests + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - id: setup + uses: ./.github/actions/setup-env with: - path: ${{ steps.setup.outputs.pnpm-store-path }} - key: pnpm-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + go-cache-suffix: "-unit" + - name: Run Go unit tests + SDK vitest tests + run: make test-unit test-ts COV_DEFER=1 + - name: Upload coverage fragment + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-unit + path: tmp/coverage + if-no-files-found: error + retention-days: 3 + overwrite: true - - name: Save Playwright browser cache - if: steps.setup.outputs.playwright-cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + integration: + name: Integration tests + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + persist-credentials: false + # Background pull overlaps the ~25s of cache restores below, so + # testcontainers finds the image already local (a concurrent pull of + # the same tag is safe — dockerd dedupes layer downloads). Image tag + # must match tests/integration/setup_test.go. + - name: Prefetch ClickHouse image (background) + run: docker pull -q clickhouse/clickhouse-server:latest >/tmp/clickhouse-pull.log 2>&1 & + - id: setup + uses: ./.github/actions/setup-env + with: + go-cache-suffix: "-integration" + node: "false" # pure Go + testcontainers ClickHouse + - name: Run Go integration tests + run: make test-integration COV_DEFER=1 + - name: Upload coverage fragment + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-integration + path: tmp/coverage + if-no-files-found: error + retention-days: 3 + overwrite: true - - name: Save Astro content cache - if: steps.setup.outputs.astro-cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + # The run's long pole. Builds its own inputs (SDK dist + cover binary, + # in parallel, on a warm cache) instead of waiting on a builder job, + # runs the suite exactly like local `make test-e2e` (one orchestrator, + # sequential files), and uploads its coverage fragment for the + # `coverage` job to merge + gate — symmetric with unit/integration. + e2e: + name: E2E tests + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + # Background pull overlaps the ~35s of setup + build below, so the + # orchestrator finds the image already local (a concurrent pull of + # the same tag is safe — dockerd dedupes layer downloads). Image tag + # must match scripts/orchestrator/main.go. + - name: Prefetch ClickHouse image (background) + run: docker pull -q clickhouse/clickhouse-server:latest >/tmp/clickhouse-pull.log 2>&1 & + # Suffix is -e2e-cov, not -e2e: this cache must carry the + # cover-instrumented server objects build-cover compiles below, but + # saves are gated on an exact-key miss — the pre-existing -e2e entry + # (orchestrator objects only) would exact-hit forever and never + # absorb them, leaving build-cover cold on every run. + - id: setup + uses: ./.github/actions/setup-env + with: + go-cache-suffix: "-e2e-cov" + # `-j` builds the prereqs (build-ts ∥ build-cover) concurrently, + # then runs the orchestrator: ClickHouse testcontainer + the cover + # binary + the SDK vitest suite. + - name: Build SDK dist + cover binary, run E2E suite + run: make -j "$(nproc)" test-e2e COV_DEFER=1 + - name: Upload coverage fragment + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - path: docs/.astro - key: astro-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml', 'docs/astro.config.mjs') }} + name: coverage-e2e + path: tmp/coverage + if-no-files-found: error + retention-days: 3 + overwrite: true + # ── Consolidated coverage report + gates ─────────────────────────── + # Merges the three suites' coverage fragments and applies every + # threshold gate once (`make cov`) — exactly like local `make ci`'s + # final step. A dedicated job (rather than folded into e2e's tail) so + # the gate is decoupled from the e2e suite's own pass/fail. + # + # `needs: changes` only — NOT the suites: a `needs` edge is a scheduling + # barrier (GitHub won't pick up a runner / checkout / restore caches / + # install deps until the needed jobs finish), which would serialize this + # job's ~50s of setup onto the critical path after the last suite, for + # nothing — the setup doesn't depend on the suites' results. Instead it + # starts at run creation, does its setup IN PARALLEL with the suites, and + # blocks only at the merge, polling for the three fragments + # (scripts/ci/wait-artifact.sh) so `make cov` fires within seconds of the + # last suite finishing. Tail on the critical path: ~10s, not ~50s. + coverage: + name: Coverage + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + # Generous: this job starts early and idle-polls until e2e (the long + # pole, itself 25-min capped) uploads. Must comfortably outwait a + # slow-but-valid e2e that was ALSO scheduled later than this job on a + # busy runner pool (both hang off `changes`, no inter-dependency); a + # genuinely failed/timed-out e2e is caught fast by the poll's + # producer-check, not this cap. + timeout-minutes: 35 + permissions: + contents: read + actions: read # poll for + download the suites' coverage fragments from this run + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - id: setup + uses: ./.github/actions/setup-env + with: + go-cache-suffix: "-cov" + # `go run ./scripts/cov report` (make cov) shells `pnpm exec nyc` + # for the TS coverage merge, so node_modules must be present. This + # (and the checkout + cache restores above) overlaps the suites. + - name: Install workspace deps + run: make pnpm-install + # Block here — and only here — until the data the merge needs exists. + # Fails fast if a producer concluded without producing (its own + # failure already reds the aggregator). Producer names must match the + # suites' job `name:` fields. + - name: Wait for coverage fragments + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # --timeout (30 min) comfortably outwaits e2e's own 25-min job cap + # plus any queue delay between the two independently-scheduled jobs + # (default is 10 min); a failed/timed-out e2e is caught fast by the + # producer-check, so the large cap costs nothing on the failure path. + run: >- + scripts/ci/wait-artifact.sh + --artifacts coverage-unit,coverage-integration,coverage-e2e + --producers "Unit tests,Integration tests,E2E tests" + --timeout 1800 + - name: Download coverage fragments + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: coverage-* + merge-multiple: true + path: tmp/coverage + - name: Render consolidated report + gate thresholds + run: make cov # Coverage in the job-summary panel. See #133 for post-launch # reporting decisions. - name: Coverage summary @@ -113,62 +379,72 @@ jobs: go tool cover -func=tmp/coverage/total/coverage.txt | tail -n 30 echo '```' } >> "$GITHUB_STEP_SUMMARY" - elif [ -f tmp/coverage/unit/coverage.txt ]; then - { - echo '## Test Coverage (unit only — full pipeline did not complete)' - echo '```' - go tool cover -func=tmp/coverage/unit/coverage.txt | tail -n 30 - echo '```' - } >> "$GITHUB_STEP_SUMMARY" fi - # ── Docs deploy ────────────────────────────────────────────────── - # `make ci` already built the site (build-all → build-docs renders - # Mermaid via Chromium + validates links → docs/dist/). These tail - # steps reuse that artifact instead of rebuilding, and run only when - # the WHOLE pipeline above is green: this head step is gated on - # success() explicitly, and every step below chains off its outputs, - # so any earlier failure short-circuits the deploy. Cloudflare's own - # Workers Builds can't build this site (no headless browser at build - # time), so we ship from here. See docs/wrangler.jsonc. + # ── Docs deploys ───────────────────────────────────────────────────── + # Both deploy jobs check out the DEFAULT branch (or, on push, the pushed + # main commit itself) and install wrangler from that trusted lockfile — + # PR-authored code never executes in a job that can read the Cloudflare + # token (#305). The only PR-derived input is the static docs-dist + # artifact, uploaded as data. Consequence: a PR changing docs/worker/ or + # docs/wrangler.jsonc previews through MAIN's worker + config; those + # changes take effect when the PR merges. + # + # Cloudflare's own Workers Builds can't build this site (no headless + # browser at build time for rehype-mermaid), so we ship from here. See + # docs/wrangler.jsonc. Both jobs soft-gate on the secrets so a + # not-yet-configured repo warns and skips instead of failing CI. - # Only deploy when docs-affecting files actually changed, so unrelated - # Go-only PRs/merges don't upload previews or redeploy prod. Computed - # from the GitHub API (the checkout is shallow, so `git diff` against - # the base can't see it). Skipped on fork PRs — no secrets there. - - name: Detect docs changes - id: docs_changed - if: >- - success() && - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository) - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "result=true" >> "$GITHUB_OUTPUT"; exit 0 # manual → always - fi - if [ "${{ github.event_name }}" = "pull_request" ]; then - files="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${{ github.event.pull_request.number }}/files" \ - --paginate --jq '.[].filename' || true)" - else - files="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${{ github.event.before }}...${{ github.sha }}" \ - --jq '.files[].filename' || true)" - fi - # Empty (API hiccup / new branch) → deploy to be safe. - if [ -z "$files" ] || printf '%s\n' "$files" | grep -qE '^(docs/|pnpm-lock\.yaml|pnpm-workspace\.yaml|\.github/workflows/ci\.yml|\.github/actions/setup-env/)'; then - echo "result=true" >> "$GITHUB_OUTPUT" - else - echo "result=false" >> "$GITHUB_OUTPUT" - fi - - # Soft-gate on the secrets so a not-yet-configured repo (or a run - # before they're added) warns and skips instead of failing CI. + # PRs: upload a non-active version and post its preview URL as a sticky + # comment. Needs only the build artifact, so it publishes right after the + # build. NON-GATING: deliberately NOT in the `ci` aggregator's `needs` + # (it is in `timing`'s, for the table) — the preview is a convenience, + # not a merge gate. `docs-build` already validates the build (and is + # required); this only deploys MAIN's worker + the static dist to a + # preview URL, so a slow or failed Cloudflare deploy must not delay or + # red the required `CI` check. It still reports its own "Docs preview" + # check status (red on failure, with the URL on success). Fork PRs skip: + # no secrets there, and the job would have nothing to do. + docs-preview: + name: Docs preview + needs: [changes, docs-build] + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + needs.changes.outputs.docs == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + # The sticky-comment endpoint (/repos/.../issues/{num}/comments) + # requires `issues: write`; `pull-requests: write` alone doesn't + # grant it. + issues: write + pull-requests: write + steps: + # Trusted tree: wrangler + worker source + config resolve from main, + # not the PR. No persisted credentials — the comment step's fallback + # `git fetch origin ` works anonymously on a public repo (PR + # heads are advertised under refs/pull/*). Note this also pins + # setup-env itself to main's copy in this job. + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + persist-credentials: false + - id: setup + uses: ./.github/actions/setup-env + with: + go: "false" + - name: Install workspace deps (trusted lockfile) + run: make pnpm-install + - name: Download docs site artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: docs-dist + path: docs/dist - name: Check deploy secrets id: cfsecrets - if: steps.docs_changed.outputs.result == 'true' - shell: bash env: TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} ACCT: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -177,35 +453,16 @@ jobs: echo "ok=true" >> "$GITHUB_OUTPUT" else echo "ok=false" >> "$GITHUB_OUTPUT" - echo "::warning::CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID not set — skipping docs deploy." + echo "::warning::CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID not set — skipping docs preview." fi - - # Production: push (or manual dispatch) on main → publish the active - # version served on wavehouse.dev. Not best-effort: a failure here - # should surface as a red CI run. - - name: Deploy docs (production) - if: >- - steps.cfsecrets.outputs.ok == 'true' - && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - && github.ref == 'refs/heads/main' - working-directory: docs - shell: bash - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - run: pnpm exec wrangler deploy - - # PRs: upload a non-active version and capture its preview URL. A real - # upload failure (e.g. a misscoped CLOUDFLARE_API_TOKEN → "Authentication - # error [code: 10000]") reds the PR instead of hiding — the retry loop - # first absorbs transient Cloudflare API blips so a one-off hiccup - # doesn't. (Still gated behind the whole `make ci` above, so an unrelated - # test flake there can also block it.) + # A real upload failure (e.g. a misscoped CLOUDFLARE_API_TOKEN → + # "Authentication error [code: 10000]") reds the PR instead of hiding + # — the retry loop first absorbs transient Cloudflare API blips so a + # one-off hiccup doesn't. - name: Upload docs preview id: preview - if: steps.cfsecrets.outputs.ok == 'true' && github.event_name == 'pull_request' + if: steps.cfsecrets.outputs.ok == 'true' working-directory: docs - shell: bash env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -233,16 +490,20 @@ jobs: fi # One sticky comment, updated in place (matched by the marker) so - # re-pushes don't spam the PR. Runs whether the upload succeeded or - # failed — `!cancelled()` + an outcome check, since a bare `if:` is - # implicitly `success()` and would skip after the upload step's exit 1 — - # so a failed deploy is noted rather than leaving a stale "live" comment. - # Skipped only when no upload was attempted (outcome == 'skipped'). gh + - # GITHUB_TOKEN, same pattern as the triage/housekeeping workflows. + # re-pushes don't spam the PR — built by scripts/ci/docs-preview-comment.sh, + # which resolves from the trusted MAIN checkout like everything else in + # this job. Runs whether the upload succeeded or failed — `!cancelled()` + # + an outcome check, since a bare `if:` is implicitly `success()` and + # would skip after the upload step's exit 1 — so a failed deploy is + # noted rather than leaving a stale "live" comment. Skipped only when + # no upload was attempted (outcome == 'skipped'). + # + # Transition guard: until this PR merges, main doesn't have the script + # yet — skip with a notice instead of failing (the preview URL is still + # in the step summary). Drop the guard once it's on main. - name: Comment docs preview status if: ${{ !cancelled() && (steps.preview.outcome == 'success' || steps.preview.outcome == 'failure') }} continue-on-error: true - shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR: ${{ github.event.pull_request.number }} @@ -250,100 +511,143 @@ jobs: SHA: ${{ github.event.pull_request.head.sha }} PREVIEW_OUTCOME: ${{ steps.preview.outcome }} run: | - marker='' - # Build a skimmable status comment (one labelled bullet per fact). - # Commit metadata — subject, author names/emails, and the commit's - # own-timezone timestamp (%cI) — is read from git, not the REST API, - # which normalizes commit dates to UTC and drops the zone. GitHub - # @-logins aren't in git, so author/committer logins come from one - # commits-API call (GitHub matches them by verified email), used only - # when the call succeeds, with a fallback that reads the login out of - # a github.com noreply address. fetch-depth: 0 guarantees the head - # commit is present. All user-controlled text (subject, names) is only - # ever a printf %s arg, never the format string, so a `%` can't act as - # a format directive. - git cat-file -e "${SHA}^{commit}" 2>/dev/null || git fetch --quiet origin "$SHA" 2>/dev/null || true - server="${GITHUB_SERVER_URL:-https://github.com}" - # @-link a person when a login is known, else show their plain name; - # login_from_email reads it out of [ID+]login@users.noreply.github.com. - gh_link() { if [ -n "$2" ]; then printf '[@%s](%s/%s)' "$2" "$server" "$2"; else printf '%s' "$1"; fi; } - login_from_email() { case "$1" in *@users.noreply.github.com) e="${1%@users.noreply.github.com}"; printf '%s' "${e#*+}";; esac; } - - subject="$(git show -s --format=%s "$SHA" 2>/dev/null || true)" - [ -z "$subject" ] && subject='(commit subject unavailable)' - author_name="$(git show -s --format=%an "$SHA" 2>/dev/null || true)" - author_email="$(git show -s --format=%ae "$SHA" 2>/dev/null || true)" - committer_name="$(git show -s --format=%cn "$SHA" 2>/dev/null || true)" - committer_email="$(git show -s --format=%ce "$SHA" 2>/dev/null || true)" - - # Logins, best-effort: one commits-API call (TSV: author, committer), - # used only on success so a transient API error can't leak its body - # into the comment; then a noreply-email fallback. - author_login=''; committer_login='' - if logins="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${SHA}" --jq '[.author.login // "", .committer.login // ""] | @tsv' 2>/dev/null)"; then - author_login="$(printf '%s' "$logins" | cut -f1)" - committer_login="$(printf '%s' "$logins" | cut -f2)" + if [ ! -f scripts/ci/docs-preview-comment.sh ]; then + echo "::notice::scripts/ci/docs-preview-comment.sh is not on the default branch yet — skipping the sticky comment." + exit 0 fi - [ -z "$author_login" ] && author_login="$(login_from_email "$author_email")" - [ -z "$committer_login" ] && committer_login="$(login_from_email "$committer_email")" + scripts/ci/docs-preview-comment.sh - # Author(s): commit author, then Co-authored-by names, each @-linked - # when possible, de-duplicated by name. - authors="$(gh_link "$author_name" "$author_login")" - seen="|$author_name|" - while IFS= read -r line; do - [ -z "$line" ] && continue - cname="$(printf '%s' "$line" | sed -E 's/ *<[^>]*> *$//; s/[[:space:]]+$//')" - cmail="$(printf '%s' "$line" | sed -nE 's/.*<([^>]*)>.*/\1/p')" - [ -z "$cname" ] && continue - case "$seen" in *"|$cname|"*) continue ;; esac - authors="$authors, $(gh_link "$cname" "$(login_from_email "$cmail")")" - seen="$seen$cname|" - done < <(git show -s --format=%B "$SHA" 2>/dev/null \ - | grep -iE '^[[:space:]]*Co-authored-by:' | sed -E 's/^[^:]*:[[:space:]]*//') - [ -z "$author_name" ] && authors='(author unavailable)' - - # A distinct human committer (someone else applied the patch); skip - # GitHub's web-flow bot and anyone already credited above. - if [ -n "$committer_name" ] && [ "$committer_name" != "$author_name" ] && [ "$committer_name" != "GitHub" ]; then - case "$seen" in - *"|$committer_name|"*) ;; - *) authors="$authors (committed by $(gh_link "$committer_name" "$committer_login"))" ;; - esac + # Production: push (or manual dispatch) on main → publish the active + # version served on wavehouse.dev. Gated on the full pipeline (lint + + # docs build + every suite + the merged coverage gate) — not + # best-effort: a failure here surfaces as a red CI run. `title`/ + # `docs-preview` are deliberately NOT in `needs`: they skip on push + # events, and a failed need would skip this job; a skipped need + # (coverage/suites on a docs-only push) is treated as success, so the + # deploy still runs. + docs-deploy: + name: Docs deploy + needs: [changes, lint, docs-build, unit, integration, e2e, coverage] + if: >- + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + github.ref == 'refs/heads/main' && + needs.changes.outputs.docs == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + # Push-to-main checkout IS the trusted tree at the pushed commit. + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - id: setup + uses: ./.github/actions/setup-env + with: + go: "false" + - name: Install workspace deps + run: make pnpm-install + - name: Download docs site artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: docs-dist + path: docs/dist + - name: Check deploy secrets + id: cfsecrets + env: + TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + ACCT: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if [ -n "$TOKEN" ] && [ -n "$ACCT" ]; then + echo "ok=true" >> "$GITHUB_OUTPUT" + else + echo "ok=false" >> "$GITHUB_OUTPUT" + echo "::warning::CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID not set — skipping docs deploy." fi + - name: Deploy docs (production) + if: steps.cfsecrets.outputs.ok == 'true' + working-directory: docs + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: pnpm exec wrangler deploy - # Commit time in its own timezone: 2026-06-09T10:04:27-04:00 -> 2026-06-09 10:04 (UTC-04:00) - ciso="$(git show -s --format=%cI "$SHA" 2>/dev/null || true)" - committed='(unknown)' - if [ -n "$ciso" ]; then - day="${ciso%%T*}"; hm="${ciso#*T}"; hm="${hm:0:5}" - case "$ciso" in *Z|*+00:00) off='UTC' ;; *) off="UTC${ciso: -6}" ;; esac - committed="$day $hm ($off)" + # ── Aggregator: the ruleset's sole required check ────────────────── + # Fails when any upstream job failed or was cancelled; treats `skipped` + # as a pass (path-filtered suites, event-filtered title/deploy jobs are + # intentional no-ops). `!cancelled()` rather than `always()` so a + # cancel-in-progress superseded run doesn't burn a runner just to report + # a failure nobody reads — the replacement run produces the fresh "CI". + # + # `docs-preview` is deliberately NOT a need: it's the convenience preview + # deploy (see its job), non-gating by design, so it neither delays nor + # reds the required check — its own "Docs preview" status reports it. The + # `docs-build` it depends on IS here, so a broken docs BUILD still gates. + # `timing` (non-gating) is the only other job excluded. + ci: + name: CI + needs: + [ + changes, + title, + lint, + docs-build, + unit, + integration, + e2e, + coverage, + docs-deploy, + ] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Gate on all jobs + env: + NEEDS: ${{ toJSON(needs) }} + run: | + jq -r 'to_entries[] | "\(.value.result)\t\(.key)"' <<<"$NEEDS" + bad="$(jq -r '[to_entries[] | select(.value.result == "failure" or .value.result == "cancelled") | .key] | join(", ")' <<<"$NEEDS")" + if [ -n "$bad" ]; then + echo "::error::Jobs failed or were cancelled: $bad" + exit 1 fi - # Deploy time in the maintainer team's zone, DST-aware (EST/EDT); UTC - # if the runner lacks the zone. Distinct from the commit time — a - # re-run, or a push made long after authoring, makes them differ. - deployed="$(TZ='America/Detroit' date '+%Y-%m-%d %H:%M %Z')" - commit_url="$server/${GITHUB_REPOSITORY}/commit/${SHA}" + echo "All jobs succeeded (or were intentionally skipped)." - # Common bullets, built once (one labelled fact per line). `printf --` - # because the format starts with a list dash; user-controlled values - # are %s args here, and again in the final body printf below, so a `%` - # never reaches a format string. - bullets="$(printf -- '- **Commit** — [`%s`](%s): %s\n- **Author** — %s\n- **Committed** — %s' \ - "${SHA:0:7}" "$commit_url" "$subject" "$authors" "$committed")" - # Headline + final time-label reflect how the upload actually went. - case "$PREVIEW_OUTCOME/${URL:+url}" in - success/url) headline="📚 **Docs preview is live** → $URL"; last="- **Deployed** — $deployed" ;; - success/) headline="📚 **Docs preview** uploaded, but its URL could not be parsed from the wrangler output — see the run log."; last="- **Deployed** — $deployed" ;; - *) headline="⚠️ **Docs preview failed to upload** (3 attempts) — any earlier preview is now stale. See the run log."; last="- **Upload failed** — $deployed" ;; - esac - body="$(printf '%s\n%s\n\n%s\n%s' "$marker" "$headline" "$bullets" "$last")" - cid="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate \ - --jq 'map(select(.body | startswith(""))) | .[0].id // empty' | head -1 || true)" - if [ -n "$cid" ]; then - gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${cid}" -f body="$body" >/dev/null - else - gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -f body="$body" >/dev/null - fi - echo "Updated docs-preview comment on PR #${PR} (outcome=${PREVIEW_OUTCOME}, url=${URL:-none})" + # ── Wall-clock summary (non-gating) ──────────────────────────────── + # Writes a per-job timing table to the run's Summary page so "where did + # the time go" never needs per-job archaeology. Deliberately NOT in the + # aggregator: it needs a checkout + an API call, and the aggregator is + # the critical path's last hop. continue-on-error so a hiccup here can + # never red a run the aggregator passed. + timing: + name: Timing summary + needs: + [ + changes, + title, + lint, + docs-build, + unit, + integration, + e2e, + coverage, + docs-preview, + docs-deploy, + ] + if: ${{ !cancelled() }} + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read # reads the run's own job timings + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Write wall-clock table + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: scripts/ci/timing-summary.sh >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/housekeeping.yml b/.github/workflows/housekeeping.yml index d8a34742..c0c64928 100644 --- a/.github/workflows/housekeeping.yml +++ b/.github/workflows/housekeeping.yml @@ -1,19 +1,24 @@ name: PR housekeeping -# Runs once per PR open/push to: +# Runs once per PR open/edit/push to: # 1. Apply path-based labels via actions/labeler. -# 2. Validate PR title against Conventional Commits. +# 2. Mirror the PR-title (Conventional Commits) check as a sticky +# explainer comment — and, when a title edit fixes it, re-run the +# failed `PR title` job inside the required `CI` check. +# +# The BLOCKING title gate lives in ci.yml's `PR title` job (under the +# `CI` aggregator — the ruleset's sole required check). This workflow +# keeps the parts that need base-repo write permissions on fork PRs, +# which only `pull_request_target` grants: labels and the sticky +# comment. It still fails on a bad title so the mirror is honest, but +# it is NOT a required check. # # Reviewer assignment is NOT here — GitHub does it natively: the ruleset's # `required_reviewers` rule requests the @Wave-RF/wavehouse-admins team, and # the team's code-review assignment auto-assigns + load-balances a member. # -# Uses `pull_request_target` so fork PRs get labels + sticky comments -# with base-repo permissions (a `pull_request` trigger lacks the -# write scopes on forks). -# -# Required by branch protection. Fails only on a bad PR title; labeling -# failures are non-fatal so a flaky API call can't block merge. +# Labeling failures are non-fatal so a flaky API call can't block the +# title mirror. on: pull_request_target: @@ -23,9 +28,11 @@ permissions: # `issues: write` is required for the sticky-comment endpoint # (/repos/.../issues/{num}/comments) — `pull-requests: write` alone # does not grant it, and fork PRs fail to comment without it. + # `actions: write` lets the title-fix step re-run CI's failed jobs. contents: read issues: write pull-requests: write + actions: write concurrency: group: pr-housekeeping-${{ github.event.pull_request.number }} @@ -83,7 +90,7 @@ jobs: - name: Post / update sticky comment on title fail # continue-on-error: GITHUB_TOKEN is read-only for fork PRs so # the comment fails there; that must not mask the final exit 1 - # that drives the required-check status. + # that keeps this check an honest mirror of the title state. if: steps.check.outputs.passed == 'false' continue-on-error: true env: @@ -113,7 +120,7 @@ jobs: --- - This check is required by the \`main branch protection\` ruleset — merge is blocked until the title matches. Edit the PR title to trigger a re-check automatically. I'll clean up this comment once the title is valid. + The required \`CI\` check enforces this via its \`PR title\` job — merge is blocked until the title matches. Edit the PR title and the failed job is re-run automatically. I'll clean up this comment once the title is valid. EOF ) @@ -147,6 +154,39 @@ jobs: echo "Removed stale pr-title-lint comment ($EXISTING)" fi + # Title edits don't re-trigger the CI workflow (its `pull_request` + # trigger deliberately excludes `edited`: a description edit would + # cancel + restart a full in-flight run via the concurrency group). + # So when an edit makes the title valid, nudge the required `CI` + # check along: if the latest completed CI run for this head SHA + # failed its `PR title` job, re-run the run's failed jobs. That job + # re-reads the title from the API — not the stale event payload a + # re-run replays — so it turns green without a new push. Best-effort + # (continue-on-error): a manual re-run or the next push covers any + # miss, e.g. a title fixed while the CI run was still in flight. + - name: Re-run CI title job after a title fix + if: github.event.action == 'edited' && steps.check.outputs.passed == 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + run_id="$(gh api "repos/$REPO/actions/runs?head_sha=$HEAD_SHA&event=pull_request" \ + --jq '[.workflow_runs[] | select(.name == "CI" and .status == "completed" and .conclusion == "failure")][0].id // empty')" + if [[ -z "$run_id" ]]; then + echo "No failed completed CI run for $HEAD_SHA — nothing to re-run." + exit 0 + fi + title_failed="$(gh api "repos/$REPO/actions/runs/$run_id/jobs?per_page=100" \ + --jq '[.jobs[] | select(.name == "PR title" and .conclusion == "failure")] | length')" + if [[ "$title_failed" -gt 0 ]]; then + gh api -X POST "repos/$REPO/actions/runs/$run_id/rerun-failed-jobs" >/dev/null + echo "Re-ran failed jobs of CI run $run_id (its PR title job had failed)." + else + echo "CI run $run_id failed for reasons other than the title — leaving it alone." + fi + - name: Fail the check if title was invalid # Final step owns the exit code so the check status reflects # the regex result, independent of comment/review-assign I/O. diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index 444cc119..432c7149 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: Load board config - run: grep -E '^[A-Z_][A-Z0-9_]*=' .github/board-config.env >> $GITHUB_ENV + run: grep -E '^[A-Z_][A-Z0-9_]*=' .github/board-config.env >> "$GITHUB_ENV" - name: Fetch current area labels id: areas @@ -112,6 +112,7 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} REPO: ${{ github.repository }} run: | + # shellcheck disable=SC2016 # the sed program legitimately contains backticks in single quotes # Strip any code fences the model may have added despite instructions. json=$(printf '%s' "$RESPONSE" | sed -E 's/^\s*```(json)?\s*$//g; s/^\s*```\s*$//g') echo "Parsed model output:" @@ -151,6 +152,7 @@ jobs: ISSUE_NUMBER: ${{ github.event.issue.number }} REPO: ${{ github.repository }} run: | + # shellcheck disable=SC2016 # the sed program legitimately contains backticks in single quotes json=$(printf '%s' "$RESPONSE" | sed -E 's/^\s*```(json)?\s*$//g; s/^\s*```\s*$//g') priority=$(echo "$json" | jq -r '.priority // empty' 2>/dev/null || echo "") diff --git a/AGENTS.md b/AGENTS.md index 92bc788f..ae86f52d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,7 +123,7 @@ Tooling notes (the non-obvious bits `make help` won't tell you): - **Policy helpers**: Use `policy.NewMemoryStore(p)` for in-memory policy testing without NATS. - **Pipes helpers**: Use `pipes.NewMemoryStore(queries...)` for in-memory pipes testing without NATS. - **Response assertions**: Use `testutil.AssertJSONResponse(t, rec, status, expected)` and `testutil.AssertJSONContains(t, rec, status, substring)`. -- **Coverage target**: 80% project-wide (CI enforces `threshold.total` in `.testcoverage.yml` against the merged unit + integration + e2e profile). Per-suite minima also enforced: unit 70%, integration 12%, e2e 50%, sdk 50%. Aim for 80%+ on new code. +- **Coverage target**: 80% project-wide (CI enforces `threshold.total` in `.testcoverage.yml` against the merged unit + integration + e2e profile). Per-suite minima also enforced: unit 80%, integration 20%, e2e 60%, sdk 50%. Aim for 80%+ on new code. - **Every new function should have corresponding test cases.** Run `make lint` and `make test` before considering work complete. - **E2E tests via SDK**: The TypeScript SDK is the primary E2E test harness. Tests in `tests/e2e/sdk/` exercise the full pipeline (ingest → ClickHouse → query) and simultaneously validate backend behavior and SDK correctness. Use `make test-e2e` to run. Add new E2E scenarios as `tests/e2e/sdk/*.test.ts` files using helpers from `tests/e2e/sdk/helpers.ts`. - **Per-suite table isolation**: Each e2e test file owns its own ClickHouse tables — `clicks_` / `events_` / `users_`, generated from `tests/e2e/sdk/tables.ts` and created by `setup.ts`. A new test file must (1) add its suite name to `SUITES` in `tables.ts` and (2) get its names via `const T = suiteTables("")`, then reference `T.clicks` etc. — never a bare `clicks`. This makes cross-file *data* contamination structurally impossible. Files still run **sequentially** (`vitest.config.ts` `maxWorkers: 1`): running them in parallel is blocked by shared *global policy* state (several files read-modify-write the single policy document; `streaming.test.ts` flips the global `default_role`), so policy-mutating tests snapshot the full policy and restore it. Dropping `maxWorkers: 1` is a deferred follow-up tracked in #214 (per-table policy storage; see `docs/src/content/docs/ingest-pipeline.md` § Deferred). @@ -138,7 +138,7 @@ Tooling notes (the non-obvious bits `make help` won't tell you): make ci # Full parity with CI: parallel verify + builds + unit/SDK tests, then integration + E2E + cov ``` -If `make ci` passes locally, your commit has crossed the same gates CI will run. For workflow-only changes, read the YAML diff carefully and run `actionlint` if you have it installed. +If `make ci` passes locally, your commit has crossed the same gates CI will run — the CI workflow (`.github/workflows/ci.yml`) is a job DAG over the *same Makefile targets* (`verify`, `build-docs`, `test-unit`/`test-ts`, `test-integration`, `test-e2e`, `cov`), just spread across parallel runners. For workflow-only changes, read the YAML diff carefully and run `actionlint` if you have it installed. ### Running `make ci` (for agents) @@ -162,7 +162,7 @@ On success `make ci` writes the tree-keyed `tmp/ci-passed-tree-` marker (s `make tools` installs team-wide git hooks via `git config core.hooksPath .githooks`. They apply to humans and Claude Code alike: - **`.githooks/pre-commit`** runs `make verify` (~30s) on every commit; blocks on failure. Skipped if `make ci` or `make verify` already ran for the current tree state (cached marker). See §Documentation Sync / §SDK Sync for what to update by hand. -- **`.githooks/pre-push`** checks for `tmp/ci-passed-tree-` written by `make ci`. Tree-keyed (not commit-keyed) so `make ci → commit → push` works without a re-run when the tree is unchanged. Editing the tree (or staging a different subset than CI saw) requires a re-run. `make ci` skips the marker write entirely when `$CI` is set (CI runners don't push). +- **`.githooks/pre-push`** scales the bar to what the push changes, classified by the same allowlist CI uses (`scripts/classify-paths.sh`): a code change requires the full `make ci` marker (`tmp/ci-passed-tree-`), while a docs/prose-only push requires only the `make verify` marker (`tmp/verify-passed-tree-`) — CI skips the Go/SDK suites for those too, so there's no reason to run them locally (and pre-commit's `make verify` usually already wrote it). Fail-closed: an unclassifiable push (no resolvable base, classifier error) falls back to requiring `make ci`. Both markers are tree-keyed (not commit-keyed) so `make {ci,verify} → commit → push` works without a re-run when the tree is unchanged; editing the tree (or staging a different subset than CI saw) requires a re-run. The marker write is skipped entirely when `$CI` is set (CI runners don't push). Bypass with `git commit --no-verify` / `git push --no-verify` only when explicitly intentional (WIP / draft pushes where you accept the consequences). Don't disable the hooks globally; that defeats the gate. @@ -230,7 +230,7 @@ Agents follow the same universal git hooks as humans (pre-commit + pre-push in ` Agents must create PRs with `gh pr create --draft`. Only humans transition draft → ready-for-review (`gh pr ready` is blocked for agents). Only humans approve or request changes (`gh pr review --approve` / `--request-changes` are blocked). -**PR title format.** The title becomes the squash-merge subject on `main` and is gated by the required `PR housekeeping` check — a bad title blocks merge, so don't discover it from CI. It must be Conventional Commits — `(optional-scope)(optional-!): ` — **≤ 72 chars**, subject **lowercase-first** with **no trailing period**. Types: `feat fix docs refactor test chore ci deps build perf revert style`. Validate before creating: `scripts/lint-pr-title.sh ""` (exit 0 = valid; it prints the reason on failure). `.claude/hooks/agent-bash-gate.sh` runs the same check on `gh pr create` / `gh pr edit --title`, so a malformed title is caught locally before the PR exists. The rule has a single source of truth — `scripts/lint-pr-title.sh` — used by **both** this local gate and the required `PR housekeeping` check (`.github/workflows/housekeeping.yml` calls the same script), so local and CI never drift. +**PR title format.** The title becomes the squash-merge subject on `main` and is gated by the required `CI` check's `PR title` job — a bad title blocks merge, so don't discover it from CI. It must be Conventional Commits — `<type>(optional-scope)(optional-!): <subject>` — **≤ 72 chars**, subject **lowercase-first** with **no trailing period**. Types: `feat fix docs refactor test chore ci deps build perf revert style`. Validate before creating: `scripts/lint-pr-title.sh "<title>"` (exit 0 = valid; it prints the reason on failure). `.claude/hooks/agent-bash-gate.sh` runs the same check on `gh pr create` / `gh pr edit --title`, so a malformed title is caught locally before the PR exists. The rule has a single source of truth — `scripts/lint-pr-title.sh` — called by this local gate, by the `PR title` job in `ci.yml` (from a trusted `main` checkout), and by `housekeeping.yml`'s advisory sticky-comment mirror, so local and CI never drift. Fixing a title after the fact needs no new push: editing it triggers housekeeping, which re-runs the failed `PR title` job (the job re-reads the title from the API). ### Human reviewer assignment is humans-only @@ -380,7 +380,7 @@ Internal-only backend changes (middleware refactors, observability internals, de 4. Use `testutil.MakeJWT(t, claims)` for auth tests, `discovery.NewSchemaRegistryFromMap(...)` for schema-aware tests, `policy.NewMemoryStore(p)` for policy tests, `pipes.NewMemoryStore(queries...)` for pipes tests. 5. Use `testutil.AssertJSONResponse` and `testutil.AssertJSONContains` for HTTP handler assertions. 6. Run `make test` — it gates the unit-test coverage threshold from `.testcoverage.yml`, so a passing run already confirms coverage. -7. Aim for 80%+ coverage on new code. The project-wide CI-enforced minimum is 80% (merged unit + integration + e2e via `.testcoverage.yml`'s `threshold.total`); per-suite minima are unit 70%, integration 12%, e2e 50%, sdk 50%. +7. Aim for 80%+ coverage on new code. The project-wide CI-enforced minimum is 80% (merged unit + integration + e2e via `.testcoverage.yml`'s `threshold.total`); per-suite minima are unit 80%, integration 20%, e2e 60%, sdk 50%. ## File Structure @@ -419,19 +419,20 @@ docs/ → Project documentation - **Column-level access control is a hard cap on every read path.** A role's `allow_columns`/`deny_columns` is enforced against *every* column a structured query references (projection, aggregations, `filters`, `group_by`, `order_by`, `time_range`) inside `query.Build`, and a `select_all` request expands to the role's allowed columns rather than `SELECT *` (an omitted projection selects nothing; `["*"]` is a literal column, not a wildcard — see Key Design Decision #12). The structured-query and live-stream paths share one decision function (`policy.IsColumnAllowed`). Don't move column checks out of the builder or special-case `SELECT *` — that reintroduces the #223 fail-open. - ClickHouse queries are passed through directly — use appropriate access controls on ClickHouse itself - **Dependency vulnerability scanning**: `govulncheck ./...` runs in CI on every push/PR. Dependabot (`.github/dependabot.yml`) opens weekly grouped PRs for outdated Go modules and GitHub Actions. -- **GitHub Actions supply chain**: Third-party actions are pinned to full commit SHAs with version comments (see `.github/workflows/ci.yml`, `release.yml`). New workflows must follow the same pattern — never `@main` or floating tags on third-party actions. Prefer inline bash or official `actions/*` / `github/*` actions when feasible (e.g. the PR-title check in `housekeeping.yml` is inline bash calling `scripts/lint-pr-title.sh`, not a third-party action). +- **GitHub Actions supply chain**: Third-party actions are pinned to full commit SHAs with version comments (see `.github/workflows/ci.yml`, `release.yml`). New workflows must follow the same pattern — never `@main` or floating tags on third-party actions. Prefer inline bash or official `actions/*` / `github/*` actions when feasible (e.g. the PR-title check in `ci.yml` / `housekeeping.yml` is inline bash calling `scripts/lint-pr-title.sh`, and the change-detection job in `ci.yml` is inline `gh api`, not a third-party paths-filter action). ## Repository Automation +- **CI** (`ci.yml`): a job DAG over the same Makefile targets as local `make ci` (~3m15s push → green). **The architecture doc is [`.github/workflows/README.md`](.github/workflows/README.md)** — DAG diagram, design invariants, cache policy, and the add-a-job recipe; read it before editing `ci.yml`. The load-bearing facts: the `changes` job (`scripts/ci/classify-changes.sh`, fail-closed) gates the test/docs jobs; the long-pole `e2e` job builds its own SDK dist + cover binary and runs the suite exactly like local `make test-e2e`; each suite uploads a `coverage-<suite>` fragment and a dedicated `coverage` job merges them + applies every threshold gate via `make cov` (like local `make ci`'s final step — kept separate so the gate is decoupled from the e2e suite; it's `needs: changes` only and *polls* for the fragments via `scripts/ci/wait-artifact.sh` rather than `needs`-ing the suites, so its ~50s setup overlaps them and the merge fires ~10s after the last suite instead of serializing setup onto the critical path); an **aggregator job named `CI`** is the ruleset's sole required status check (fails on failed/cancelled needs, treats skipped as passing) — the Cloudflare `docs-preview` deploy is deliberately NOT a need (non-gating, like `timing`: only `docs-build` gates, so a slow/failed preview never delays or reds the required check); caches are owned end-to-end by `.github/actions/setup-env` (nested `actions/cache`, automatic post-job saves — never add save steps to `ci.yml`). Plain `make build` binaries are not linked in CI — compile breakage is caught by lint/tests/the cover-binary link, release builds by `goreleaser-validate.yml` / `publish-dev.yml`. Fork PRs run the full secretless pipeline; merge-queue `merge_group` runs re-test the full suite against current main (the queue replaces the old require-up-to-date rule — never remove the `merge_group:` trigger, or queued PRs hang). CI logic lives in `scripts/ci/*.sh`, gated by `make lint-sh` (shellcheck) and `make lint-gha` (actionlint) inside `make verify` — not in inline YAML, except in the trusted-main deploy jobs where inline is the trust boundary. - **Issue triage** (`triage.yml`): GitHub Models classifies new/edited issues and applies `area/*` + `security` + `breaking-change` labels. -- **Code review** (advisory; the `main branch protection` ruleset is the actual merge gate — its `required_reviewers` rule requires an approval from the `@Wave-RF/wavehouse-admins` team, alongside the `CI` / `PR housekeeping` required checks): handled by external marketplace apps (CodeRabbit, Copilot) configured at the org/repo level, not by in-repo workflows. Inline findings post as review threads that `required_review_thread_resolution: true` blocks merge on until resolved. +- **Code review** (advisory; the `main branch protection` ruleset is the actual merge gate — its `required_reviewers` rule requires an approval from the `@Wave-RF/wavehouse-admins` team, alongside the required `CI` status check): handled by external marketplace apps (CodeRabbit, Copilot) configured at the org/repo level, not by in-repo workflows. Inline findings post as review threads that `required_review_thread_resolution: true` blocks merge on until resolved. - **Dependabot** (`.github/dependabot.yml`): opens weekly grouped PRs for Go modules, GitHub Actions, and the npm workspaces. They go through the same gate as any PR — an `@Wave-RF/wavehouse-admins` approval (`required_reviewers`) + the required checks — with **no auto-merge** (the former `dependabot-automerge.yml` was removed; auto-approve-and-merge is intentionally off, so every bump gets a human admin review). -- **Docs site deploy** (`wavehouse.dev`): a tail step of the CI job (`.github/workflows/ci.yml`), **not** Cloudflare's Workers Builds. Workers Builds can't build this site — `rehype-mermaid` renders diagrams to themed SVG at build time via headless Chromium, and the Workers Builds image has no browser (and no root to apt-install one). `make ci` already builds `docs/dist/` on a runner with a cached Chromium, so the deploy reuses that artifact and runs only once the whole pipeline is green: push to `main` runs `wrangler deploy` (production → `wavehouse.dev`); PR branches run `wrangler versions upload`, publishing a per-version preview at `<version-prefix>-wavehouse-docs.wave-rf.workers.dev` posted as a sticky PR comment. Deploys are skipped when no docs-affecting files changed and on fork PRs. **Requires `CLOUDFLARE_API_TOKEN` + `CLOUDFLARE_ACCOUNT_ID` repo secrets**, and Cloudflare Workers Builds must stay **disconnected** from the `wavehouse-docs` Worker (else it double-deploys and fails the browser-dependent build on every push). Wrangler config (custom domain, observability, source maps, preview URLs) lives in `docs/wrangler.jsonc`. The worker (`docs/worker/index.ts`, delegating to `cloudflare-md-router`) deploys alongside the static assets so `Accept: text/markdown` content negotiation works in production. +- **Docs site deploy** (`wavehouse.dev`): the `docs-preview` / `docs-deploy` jobs of the CI workflow (`.github/workflows/ci.yml`), **not** Cloudflare's Workers Builds. Workers Builds can't build this site — `rehype-mermaid` renders diagrams to themed SVG at build time via headless Chromium, and the Workers Builds image has no browser (and no root to apt-install one). CI's `docs-build` job builds `docs/dist/` on a runner with a cached Chromium and uploads it as an artifact; the deploy jobs consume that artifact from a checkout of **trusted `main`** — wrangler, the worker source, and `docs/wrangler.jsonc` never resolve from a PR tree, so PR-authored code can't reach the Cloudflare token (#305); a PR's `docs/worker/` or wrangler-config changes take effect on merge, not in its preview. Push to `main` runs `wrangler deploy` once the whole pipeline is green (production → `wavehouse.dev`); same-repo PR branches run `wrangler versions upload` right after the build (an unrelated test flake no longer blocks the preview), publishing a per-version preview at `<version-prefix>-wavehouse-docs.wave-rf.workers.dev` posted as a sticky PR comment. Deploys are skipped when no docs-affecting files changed and on fork PRs. **Requires `CLOUDFLARE_API_TOKEN` + `CLOUDFLARE_ACCOUNT_ID` repo secrets** (referenced only in the two deploy jobs), and Cloudflare Workers Builds must stay **disconnected** from the `wavehouse-docs` Worker (else it double-deploys and fails the browser-dependent build on every push). Wrangler config (custom domain, observability, source maps, preview URLs) lives in `docs/wrangler.jsonc`. The worker (`docs/worker/index.ts`, delegating to `cloudflare-md-router`) deploys alongside the static assets so `Accept: text/markdown` content negotiation works in production. ## Governance Files - **No `CODEOWNERS`**: admin approval is enforced natively by the `main branch protection` ruleset's `required_reviewers` rule — an approval from the `@Wave-RF/wavehouse-admins` team — rather than a CODEOWNERS file or a custom status-check workflow. - - Reviewer *assignment* is native too: the `required_reviewers` rule requests the `@Wave-RF/wavehouse-admins` team and the team's code-review assignment auto-assigns + load-balances the member (no workflow). `housekeeping.yml` now only applies path-labels + validates the PR title. Task Board placement is handled by native Projects v2 workflows configured in the project UI. + - Reviewer *assignment* is native too: the `required_reviewers` rule requests the `@Wave-RF/wavehouse-admins` team and the team's code-review assignment auto-assigns + load-balances the member (no workflow). `housekeeping.yml` (non-required, `pull_request_target` so it can write on fork PRs) applies path-labels, mirrors the PR-title check as a sticky explainer comment, and re-runs CI's failed `PR title` job when a title edit fixes it — the *blocking* title gate is the `PR title` job under `ci.yml`'s required `CI` aggregator. Task Board placement is handled by native Projects v2 workflows configured in the project UI. - **`CLAUDE.md`**: a thin pointer file to AGENTS.md. Keep the pointer short; never duplicate content. -- **`CONTRIBUTING.md`**: the Conventional Commits type list must stay in sync with the regex in `scripts/lint-pr-title.sh` (the single source of truth used by both the local gate and the required `PR housekeeping` check). The title linter validates squash-merge commit messages. +- **`CONTRIBUTING.md`**: the Conventional Commits type list must stay in sync with the regex in `scripts/lint-pr-title.sh` (the single source of truth used by the local gate, the required `CI` check's `PR title` job, and housekeeping's sticky-comment mirror). The title linter validates squash-merge commit messages. - **`SUPPORT.md`** (alpha-stage public triage policy): the externally-promised cadence is **best-effort, 1–2 business days for an initial response** on bugs / features / usage questions; **security reports are prioritized** with the 48-hour acknowledge / 5-business-day initial-assessment targets in `SECURITY.md`. Usage questions ("how do I…") are routed to [GitHub Discussions → Q&A](https://github.com/Wave-RF/WaveHouse/discussions/categories/q-a) — do not file them as bug-report Issues; bug-reporters who use the wrong template get redirected. There is no Discord/Slack. Don't quietly let threads slip — if one sits longer than a week, that's a miss. **Out-of-scope items publicly stated in `SUPPORT.md` are only "Older releases" and "Non-ClickHouse backends"**. When tweaking the policy, update `SUPPORT.md` first and keep this paragraph in sync. The docs footer (`docs/src/components/Footer.astro`) and sidebar (`docs/src/config/sidebar.ts`) cross-link Discussions, `SUPPORT.md`, and `SECURITY.md` so they're one click from anywhere on `wavehouse.dev`; `README.md`, `CONTRIBUTING.md`, and both issue templates (`.github/ISSUE_TEMPLATE/bug_report.md`, `feature_request.md`) also link out — change those together if the policy moves. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e388942..e2818855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed + +- **CI is now a job DAG instead of one monolithic job, and the docs deploys no longer expose the Cloudflare token to PR-authored code** (`.github/workflows/ci.yml`, `.github/workflows/housekeeping.yml`, `.github/actions/setup-env/action.yml`, `Makefile`, `docs/wrangler.jsonc`, `AGENTS.md`, `docs/src/content/docs/development.md`, `docs/src/content/docs/claude-code.md`, `CONTRIBUTING.md`, `scripts/lint-pr-title.sh`, `.claude/hooks/agent-bash-gate.sh`): closes #305. The single `make ci` job becomes parallel jobs over the *same Makefile targets* (local `make ci` stays the dev mirror): `lint`, `unit`, `integration`, `e2e` (builds its own SDK dist + cover binary via `make -j test-e2e` on a warm per-suite cache and runs the suite exactly like a local run), `coverage` (a dedicated job that merges every suite's `coverage-<suite>` fragment and applies every threshold gate via `make cov` — like local `make ci`'s final step, so the gate is decoupled from the e2e suite; it's `needs: changes` only and *polls* for the fragments rather than `needs`-ing the suites, so its setup overlaps them and the merge fires ~10s after the last suite instead of serializing ~50s of setup onto the critical path), and `docs-build` (`make build-docs`, docs-affecting changes only, uploads the docs dist artifact the preview/deploy jobs consume) — public-repo runners are free and 4-core, so the pipeline spreads horizontally instead of queueing in one process. The architecture is documented once, in `.github/workflows/README.md` (DAG diagram, design invariants, cache key policy, add-a-job recipe, and the measured-but-deferred optimizations — e2e sharding among them), and the workflow's logic lives in shellcheck-gated scripts (`scripts/ci/` — `classify-changes.sh`, `check-pr-title.sh`, `docs-preview-comment.sh`, `timing-summary.sh`, `wait-artifact.sh`; over the shared, dependency-free path classifier `scripts/classify-paths.sh`, unit-tested by `scripts/classify-paths.test.sh` via `make test-classify-paths` and reused by the `pre-push` git hook so a docs/prose-only push requires only `make verify`, not a full `make ci` — the same suites CI skips for those changes) rather than inline YAML; caches are owned end-to-end by `setup-env` via nested `actions/cache` (automatic post-job saves — the per-job save-step boilerplate is gone); a non-gating `Timing summary` job writes a per-job wall-clock table to every run's Summary page; and `make verify` gains two leaves that gate the new surface area — `lint-sh` (shellcheck `v0.11.0`, checksum-verified install via `scripts/install-shellcheck.sh`) and `lint-gha` (actionlint `v1.7.12`) — so the CI plumbing is linted like any other source. The workflow also handles `merge_group` events (full suite against the merge-group ref), enabling a **merge queue** on `main`: the queue re-tests each PR against current main at landing time, which replaces the ruleset's "require branches to be up to date" rule — no more manual branch updates after every sibling merge. A new aggregator job named `CI` is the ruleset's **sole required status check** (it fails on any failed/cancelled job and counts skipped jobs as passing), so docs-only PRs skip the Go suites without orphaning the gate and future job changes never require ruleset edits. The PR-title (Conventional Commits) gate moves into the `PR title` job under that aggregator, validated by the same `scripts/lint-pr-title.sh` from a trusted `main` checkout; `PR housekeeping` (`pull_request_target`) drops to non-required and keeps what needs fork-PR write access — path labels, the sticky title-explainer comment, and a new nudge that re-runs the failed `PR title` job when a title edit fixes it (the job re-reads the title from the API, so no new push is needed). The **#305 fix**: docs previews/production deploys run in dedicated `docs-preview`/`docs-deploy` jobs that check out trusted `main` (wrangler, worker source, and config never resolve from the PR tree), consume only the static `docs/dist` artifact, and are the only jobs that reference `CLOUDFLARE_*` secrets; previews now publish right after `docs-build` instead of waiting on the full test pipeline, and the `docs-preview` deploy is **non-gating** — it's not in the `CI` aggregator's `needs` (only `docs-build` gates), so a slow or failed Cloudflare preview reports its own "Docs preview" check but never delays or reds the required check; production (`docs-deploy`, on the post-merge main push) still requires everything green. Per-job least-privilege permissions replace the old workflow-wide `contents: write`, and the Go build cache is partitioned per job (unit/integration/e2e compile with different flags) so each suite stays warm. + ### Security - **The structured-query column allowlist is now a hard cap on every column a query references — closing a fail-open data-exposure family on the primary read path** (`internal/query/builder.go`, `internal/query/errors.go` (new), `internal/policy/policy.go`, `internal/discovery/discovery.go`, `internal/api/structured_query.go`, `docs/src/content/docs/access-control.md`, `docs/src/content/docs/api.md`, plus tests in `internal/query/builder_test.go`, `internal/api/structured_query_test.go`, `internal/policy/policy_test.go`, `internal/discovery/discovery_test.go`, `tests/e2e/sdk/query.test.ts`): closes #223. `POST /v1/query` previously authorized only the columns a caller *explicitly* listed, so several inputs bypassed a role's `allow_columns`/`deny_columns` entirely: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a29bbea1..82372627 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ Open a [feature request issue](https://github.com/Wave-RF/WaveHouse/issues/new?t make ci # full local pipeline: verify + builds + all test suites (Docker required) ``` - The pre-push hook (installed by `make tools`) blocks any push until `make ci` has passed for the current tree. `make lint` / `make test` / `make build` are fast inner-loop subsets. + The pre-push hook (installed by `make tools`) blocks a push until the tree has been validated locally: a code change needs `make ci`, a docs/prose-only change needs only `make verify` (the same split CI makes). `make lint` / `make test` / `make build` are fast inner-loop subsets. 2. Write tests for new functionality. Unit tests go alongside the code in `internal/`. Integration tests go in `tests/` with the `//go:build integration` tag. @@ -76,7 +76,7 @@ Types: - `revert` — Reverting a previous commit - `style` — Formatting changes with no functional effect -The PR title is enforced by the `PR housekeeping` required check (`.github/workflows/housekeeping.yml`): Conventional Commits format, ≤ 72 characters, lowercase-first subject with no trailing period — it becomes the squash-merge commit message on `main`, so keep it parseable. Check a title locally with `scripts/lint-pr-title.sh "<title>"` before opening the PR. +The PR title is enforced by the required `CI` check's `PR title` job (`.github/workflows/ci.yml`): Conventional Commits format, ≤ 72 characters, lowercase-first subject with no trailing period — it becomes the squash-merge commit message on `main`, so keep it parseable. Check a title locally with `scripts/lint-pr-title.sh "<title>"` before opening the PR. Examples: diff --git a/Makefile b/Makefile index 128d27a7..de1ebea2 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,19 @@ AIR := $(LOCAL_BIN)/air-$(AIR_VERSION) MISSPELL_VERSION := v0.8.0 MISSPELL := $(LOCAL_BIN)/misspell-$(MISSPELL_VERSION) +# shellcheck: shell-script linter — the CI workflow logic lives in scripts/ +# now, so it gates like any other source. Haskell binary (no `go install` +# path): official release tarball, checksum-verified by +# scripts/install-shellcheck.sh. The version is pinned THERE (with the +# per-platform checksums); this variable only names the installed file. +SHELLCHECK_VERSION := v0.11.0 +SHELLCHECK := $(LOCAL_BIN)/shellcheck-$(SHELLCHECK_VERSION) + +# actionlint: GitHub Actions workflow linter (also shellchecks the inline +# `run:` blocks via $(SHELLCHECK)). Pure Go — go-install pattern like air. +ACTIONLINT_VERSION := v1.7.12 +ACTIONLINT := $(LOCAL_BIN)/actionlint-$(ACTIONLINT_VERSION) + # --- Coverage Directories ----------------------------------------------------- # One path per suite. Internal layout (managed by scripts/coverage.sh): # $(COV_X)/data/ binary covdata (covmeta.* / covcounters.*) @@ -371,6 +384,30 @@ lint-md: pnpm-install lint-prose: $(MISSPELL) $(call run,misspell (US spelling),$(MISSPELL) -locale US -source text -error $(DOCS_PROSE),run make fix to auto-correct) +# lint-sh: shellcheck over every tracked shell script (scripts/, hooks, +# docs tooling). -x follows `source`d files; -P SCRIPTDIR resolves +# `# shellcheck source=` directives relative to the sourcing script, not +# the cwd. Lazily expanded so the ls-files only runs when the target does. +SHELL_SOURCES = $(shell git ls-files '*.sh') +.PHONY: lint-sh +lint-sh: $(SHELLCHECK) + $(call run,shellcheck,$(SHELLCHECK) -x -P SCRIPTDIR $(SHELL_SOURCES),) + +# lint-gha: actionlint over .github/workflows/*.yml — expression/type +# errors, action-input mismatches, SHA-pin syntax, and shellcheck (via the +# pinned $(SHELLCHECK)) on inline run: blocks. +.PHONY: lint-gha +lint-gha: $(ACTIONLINT) $(SHELLCHECK) + $(call run,actionlint (workflows),$(ACTIONLINT) -shellcheck $(SHELLCHECK),) + +# test-classify-paths: assert scripts/classify-paths.sh (the shared change +# classifier behind CI's `changes` job and the local git hooks) against the +# canonical change shapes — fast, dependency-free, so the allowlists can't +# silently regress. A verify leaf so CI's lint job runs it. +.PHONY: test-classify-paths +test-classify-paths: + $(call run,classify-paths test,scripts/classify-paths.test.sh,) + .PHONY: vulncheck vulncheck: go-mod-download ## Run govulncheck (V=1 for full call stacks) ifdef V @@ -446,7 +483,7 @@ verify: ## Run all static checks across the repo (Go + TS + docs, parallelized) @printf "$(GREEN)$(BOLD)✔ All static checks passed$(RESET)\n" .PHONY: verify-parallel -verify-parallel: tidy fmt-go lint-go lint-ts lint-md lint-prose vulncheck check-docs typecheck-ts +verify-parallel: tidy fmt-go lint-go lint-ts lint-md lint-prose lint-sh lint-gha test-classify-paths vulncheck check-docs typecheck-ts # typecheck-ts: tsc --noEmit on the SDK. Its own target (was inline in verify's # recipe) so it can run as a parallel leaf of verify-parallel. @@ -651,6 +688,7 @@ test-integration: go-mod-download ## Run Go integration tests + render coverage # from the cover binary → tmp/coverage/e2e/data/; vitest v8 coverage of # the SDK source → tmp/coverage/ts-e2e/) — same "always coverage" pattern # as the Go test targets. `make cov` merges ts-unit + ts-e2e after. +# .PHONY: test-e2e test-e2e: build-ts build-cover ## Run E2E SDK suite against cover binary + render coverage + gate @printf "$(CYAN)==> Running E2E Tests...$(RESET)\n" @@ -837,10 +875,11 @@ clean-all: clean clean-test clean-tools ## Full reset — clean + clean-test + c # them pre-compiled (offline CI image baking), run them once with --help. .PHONY: tools tools: ## Install pinned tools, Go modules, pnpm deps, and git hooks - @# The five installs are independent — fan them out under -j (golangci-lint - @# download ∥ air ∥ misspell ∥ go-mod-download ∥ pnpm-install). Go's module - @# cache is concurrency-safe, so this is just faster on a cold clone. - @$(MAKE) -j $(JOBS) $(GOLANGCI_LINT) $(AIR) $(MISSPELL) go-mod-download pnpm-install + @# The installs are independent — fan them out under -j (golangci-lint + @# download ∥ air ∥ misspell ∥ shellcheck ∥ actionlint ∥ go-mod-download + @# ∥ pnpm-install). Go's module cache is concurrency-safe, so this is + @# just faster on a cold clone. + @$(MAKE) -j $(JOBS) $(GOLANGCI_LINT) $(AIR) $(MISSPELL) $(SHELLCHECK) $(ACTIONLINT) go-mod-download pnpm-install @# Install team-wide git hooks via core.hooksPath. Idempotent — running @# `make tools` repeatedly just re-asserts the config. The .githooks/ @# directory is committed; this line plumbs git to it. Users can opt out @@ -882,3 +921,15 @@ $(MISSPELL): @GOBIN=$(LOCAL_BIN) go install github.com/golangci/misspell/cmd/misspell@$(MISSPELL_VERSION) @mv $(LOCAL_BIN)/misspell $@ @echo "$(GREEN)==> Installed: $@$(RESET)" + +# shellcheck is a Haskell binary — fetched as the official release tarball +# and checksum-verified; version + per-platform sha256 live in the script. +$(SHELLCHECK): + @scripts/install-shellcheck.sh $@ + +$(ACTIONLINT): + @echo "$(YELLOW)==> Installing actionlint $(ACTIONLINT_VERSION) for $(OS)_$(ARCH)...$(RESET)" + @mkdir -p $(LOCAL_BIN) + @GOBIN=$(LOCAL_BIN) go install github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION) + @mv $(LOCAL_BIN)/actionlint $@ + @echo "$(GREEN)==> Installed: $@$(RESET)" diff --git a/docs/src/content/docs/claude-code.md b/docs/src/content/docs/claude-code.md index 8d9b71b8..c418517f 100644 --- a/docs/src/content/docs/claude-code.md +++ b/docs/src/content/docs/claude-code.md @@ -23,7 +23,7 @@ If you're new to Claude Code itself, the [official docs](https://code.claude.com | Layer | Lives in | Applies to | Purpose | | ----- | -------- | ---------- | ------- | -| **Git hooks** | `.githooks/` (installed by `make tools`) | Humans + Claude uniformly | Hard enforcement: `make verify` on commit, `make ci` passed before push | +| **Git hooks** | `.githooks/` (installed by `make tools`) | Humans + Claude uniformly | Hard enforcement: `make verify` on commit; before push, `make ci` for a code change or `make verify` for a docs-only one (classifier-gated) | | **Claude Code agent gate** | `.claude/hooks/agent-bash-gate.sh` (PreToolUse Bash) + `.claude/settings.json` deny rules | Agents only | Catches accidental violations of [Agent PR Discipline](#agent-pr-discipline): drafts only, no human reviewer adds, a marker (from a review or a logged skip) from every reviewer in `scripts/pre-push-reviewers.sh` required on any push to a non-main branch with commits ahead of `main`; PR title linted via `scripts/lint-pr-title.sh` on `gh pr create` / `gh pr edit --title` | | **Claude Code ergonomic hooks** | `.claude/hooks/gofumpt-on-save.sh` (PostToolUse Edit/Write/MultiEdit), `.claude/hooks/review-marker.sh` (SubagentStop) | Claude only | gofumpt: auto-format on file edits (humans get this from their IDE). review-marker: on a reviewer's `VERDICT: ship_it`, writes that reviewer's `tmp/<name>-passed-<HEAD-sha>` marker | | **Claude Code skills / agents / commands** | `.claude/skills/`, `.claude/agents/`, `.claude/commands/` | Claude only (when relevant) | Workflow guidance and on-demand helpers — not gates | @@ -37,11 +37,11 @@ Two scripts, both committed to the repo: | Hook | Behavior | | ---- | -------- | | `pre-commit` | Runs `make verify` (tidy + fmt + vulncheck + lint, ~30s) — **blocks on failure**. Skipped if `make ci` or `make verify` already ran for the current tree state (cached via `scripts/ci-marker.sh`). | -| `pre-push` | Checks for `tmp/ci-passed-tree-<TREE-sha>` written by `make ci`. **Blocks** if absent — run `make ci`, fix failures, retry push. | +| `pre-push` | Scales the bar to the change set (same classifier CI uses, `scripts/classify-paths.sh`): a **code** change requires the `make ci` marker (`tmp/ci-passed-tree-<TREE-sha>`); a **docs/prose-only** push requires only the `make verify` marker (`tmp/verify-passed-tree-<TREE-sha>`) — CI skips the Go/SDK suites for those too. **Blocks** if the required marker is absent. Fail-closed: an unclassifiable push falls back to requiring `make ci`. | `--no-verify` is for intentional WIP / draft pushes. Agents should not use it — policy in AGENTS.md §"Agent PR Discipline", not regex-enforced. -Tree-keyed so commit-then-push works without a re-run when the tree is unchanged. `make ci` skips the marker write when `$CI` is set (CI runners don't push). Shared logic lives in `scripts/ci-marker.sh`. +Both markers are tree-keyed so commit-then-push works without a re-run when the tree is unchanged. `make ci` / `make verify` skip the marker write when `$CI` is set (CI runners don't push). Shared logic lives in `scripts/ci-marker.sh`. ## What's in `.claude/` and `.config/` @@ -103,7 +103,7 @@ To add a skill: create `.claude/skills/<name>/SKILL.md` with frontmatter `name` Agents (Claude Code etc.) have additional gating beyond what humans face — enforced by `.claude/hooks/agent-bash-gate.sh` (PreToolUse Bash) + deny rules in `.claude/settings.json`. Humans keep full git/gh affordances; agents have these extra constraints: -- **Drafts only.** `gh pr create` must include `--draft`. Only humans transition draft → ready (`gh pr ready` is blocked), approve (`gh pr review --approve` is blocked), or request changes (`gh pr review --request-changes` is blocked). The gate also lints the PR title on `gh pr create` / `gh pr edit --title` via `scripts/lint-pr-title.sh` — the same rule as the required `PR housekeeping` check, so a malformed title is caught before the PR exists (fail-open when no quoted title is parseable). +- **Drafts only.** `gh pr create` must include `--draft`. Only humans transition draft → ready (`gh pr ready` is blocked), approve (`gh pr review --approve` is blocked), or request changes (`gh pr review --request-changes` is blocked). The gate also lints the PR title on `gh pr create` / `gh pr edit --title` via `scripts/lint-pr-title.sh` — the same rule as the required `CI` check's `PR title` job, so a malformed title is caught before the PR exists (fail-open when no quoted title is parseable). - **No human reviewer assignment.** `gh pr edit --add-reviewer / --add-assignee` and `POST /requested_reviewers` are blocked. GitHub assigns reviewers natively — the `required_reviewers` ruleset rule requests the `@Wave-RF/wavehouse-admins` team and the team's code-review assignment picks the member; humans handle the rest. - **Bot re-triggers via comments.** Agents CAN mention bots in PR comments to re-trigger reviews — `@coderabbitai review`, etc. This goes through `gh pr comment` (allowed), not the reviewer API. - **Pre-push review required on PR branches.** Before `git push` to a branch with commits ahead of `main`, the agent runs the reviewers in `scripts/pre-push-reviewers.sh` (today: `pre-push-reviewer` for code, `docs-reviewer` for docs prose + code↔docs sync) that the change needs — in fresh context, in parallel — and **skips** any with nothing to do via `scripts/skip-pre-push-review.sh <name> "<reason>"` (a logged skip that satisfies the marker, so a docs typo doesn't pay for a full code review). `/prepush` does all of this. `ship_it` requires zero findings at any severity — any `[MUST]` / `[SHOULD]` / `[MAY]` forces iterate; fix and re-invoke (always fresh context) until clean. The push gate requires a marker — from a `ship_it` or a logged skip — from every listed reviewer, and echoes the skips at push time. @@ -210,7 +210,7 @@ Not committed at project level. Personal preference — put in `.claude/settings 1. Write code (gofumpt-on-save formats Go files as you go). 2. `git commit` → pre-commit hook runs `make verify` (or skips if `make ci` already validated this tree). Fix anything that fails. -3. `git push` → the pre-push hook blocks until `make ci` passed for the current tree (run `make ci`, fix, retry). For agents, the agent-bash-gate hook also requires a marker for HEAD from every reviewer in `scripts/pre-push-reviewers.sh` on any push of a branch with commits ahead of `main` — including the first push, before the PR exists. Run `/prepush` → it judges which reviewers the change needs, runs those in parallel (on Ship it each marker auto-writes), and skips the rest on the record (`skip-pre-push-review.sh` writes their markers + logs why) → push succeeds. On Iterate/Block, fix, re-invoke (fresh context each time), repeat. +3. `git push` → the pre-push hook blocks until the tree is validated for what changed — `make ci` for a code change, `make verify` for a docs/prose-only one (run it, fix, retry). For agents, the agent-bash-gate hook also requires a marker for HEAD from every reviewer in `scripts/pre-push-reviewers.sh` on any push of a branch with commits ahead of `main` — including the first push, before the PR exists. Run `/prepush` → it judges which reviewers the change needs, runs those in parallel (on Ship it each marker auto-writes), and skips the rest on the record (`skip-pre-push-review.sh` writes their markers + logs why) → push succeeds. On Iterate/Block, fix, re-invoke (fresh context each time), repeat. 4. Open the PR with `gh pr create --draft` (agents required to use `--draft`; humans flip to ready when ready). 5. CI workflows fire on the new HEAD. Address review comments per AGENTS.md §Review Response. diff --git a/docs/src/content/docs/development.md b/docs/src/content/docs/development.md index c3961a0d..212e0930 100644 --- a/docs/src/content/docs/development.md +++ b/docs/src/content/docs/development.md @@ -349,7 +349,7 @@ The primary E2E integration test suite lives in `tests/e2e/sdk/`. It uses the Ty **Architecture**: -- `scripts/orchestrator` — the E2E entrypoint behind `make test-e2e`: it starts a clean ClickHouse **testcontainer** per run, launches the `wavehouse-cov` binary on a random free port, runs the SDK suite against it, then SIGINTs the binary to flush coverage. No Compose file is involved. +- `scripts/orchestrator` — the E2E entrypoint behind `make test-e2e`: it starts a clean ClickHouse **testcontainer** per run, launches the `wavehouse-cov` binary on a random free port, runs the SDK suite against it, then SIGINTs the binary to flush coverage. No Compose file is involved. CI runs the exact same path. - `tests/e2e/sdk/setup.ts` — Smart `globalSetup` that probes ports before starting Docker services, so tests work seamlessly whether you started services manually or let the setup do it. - `tests/e2e/sdk/helpers.ts` — JWT factories, typed client constructors, async wait helpers, direct ClickHouse query helper. @@ -557,7 +557,7 @@ This repo has three tiers of AI automation sitting alongside the normal CI check ### PR title and Conventional Commits -PR titles must match Conventional Commits format and stay ≤ 72 characters — the title becomes the squash-merge commit subject. Both rules are enforced by the required `PR housekeeping` check (`.github/workflows/housekeeping.yml`); validate locally with `scripts/lint-pr-title.sh "<title>"`: +PR titles must match Conventional Commits format and stay ≤ 72 characters — the title becomes the squash-merge commit subject. Both rules are enforced by the `PR title` job under the required `CI` check (`.github/workflows/ci.yml`); validate locally with `scripts/lint-pr-title.sh "<title>"`: ```text <type>(optional-scope)(optional-!): <lowercase subject, no trailing period> @@ -567,17 +567,20 @@ Allowed types: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `ci`, `deps`, The `!` before `:` marks a breaking change per Conventional Commits 1.0.0 (e.g., `feat!: remove deprecated endpoint`, `refactor(api)!: rename handlers`). Titles are also capped at **72 characters** — they become squash-merge commit subjects (Dependabot PRs are exempt from the cap). -If the title doesn't match, a sticky comment posts on the PR explaining the format; it auto-removes once the title is fixed. +If the title doesn't match, a sticky comment posts on the PR explaining the format (from the `PR housekeeping` workflow, which mirrors the same script); it auto-removes once the title is fixed. Fixing the title needs no new push — the edit triggers housekeeping, which re-runs the failed `PR title` job (the job re-reads the title from the API, not the stale event payload). ### Required status checks -The `main branch protection` ruleset requires two status checks to pass before any PR can merge: +The `main branch protection` ruleset requires one status check to pass before any PR can merge: -- `CI` — the full `make ci` pipeline (verify + builds + unit/SDK tests, then integration + E2E + coverage gates), run as a single job in `.github/workflows/ci.yml` -- `PR housekeeping` — PR title is Conventional Commits (`.github/workflows/housekeeping.yml`) +- `CI` — the aggregator job of `.github/workflows/ci.yml`. The workflow is a job DAG over the same Makefile targets local `make ci` runs: `lint` (`make verify`), `unit` (`make test-unit test-ts`), `integration` (`make test-integration`), `e2e` (`make -j test-e2e` — builds its own SDK dist + cover binary on a warm cache, runs the suite exactly like a local run), `coverage` (`make cov` over every suite's uploaded coverage fragment + threshold gates, like local `make ci`'s final step), `docs-build` (`make build-docs` when docs-affecting files changed, uploading the docs dist artifact), `PR title` (Conventional Commits), and the docs preview/deploy jobs. The aggregator fails if any job failed or was canceled and treats skipped jobs as passing — docs-only PRs skip the Go test suites by design, and fork PRs run everything except the (secret-bearing) docs deploys. Every run's Summary page gets a per-job wall-clock table from the non-gating `Timing summary` job. The full architecture — DAG diagram, design invariants, cache policy, how to add a job — lives in [`.github/workflows/README.md`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/workflows/README.md). + +The `PR housekeeping` workflow still runs on every PR (labels + the title explainer comment) but is no longer a required check. The ruleset also requires an approval from the `@Wave-RF/wavehouse-admins` team (the `required_reviewers` rule — this is what mandates an admin sign-off, replacing the old `Admin approval` status-check workflow), plus 1 approving review, approval of the most recent push by someone other than its author, resolution of all review threads, linear history, no branch deletion, no force-push, and squash-merge only. Repository admins may bypass these requirements when merging their own PR (e.g. a trivial `.github` change) but still cannot push directly to `main`. +Approved, green PRs land through a **merge queue** ("Merge when ready"): the queue re-runs the required `CI` check against the PR merged with *current* main (a `merge_group` event — the CI workflow runs the full test suite for these) and fast-forwards only on green. That integration re-test replaces the old "branch is out-of-date with the base branch" requirement — queued PRs don't need manual branch updates, and the queue never pushes to the PR branch. + Dependabot PRs go through the same admin review as any other PR — there is no auto-merge (see the Dependabot section above). ### Merge behavior diff --git a/docs/wrangler.jsonc b/docs/wrangler.jsonc index 9d40b020..60d0d53c 100644 --- a/docs/wrangler.jsonc +++ b/docs/wrangler.jsonc @@ -3,14 +3,18 @@ // and the worker in worker/index.ts (using cloudflare-md-router) handles // Accept-header / user-agent routing before falling through to ASSETS. // - // Deploy is a tail step of the CI job (.github/workflows/ci.yml), NOT - // Cloudflare's Workers Builds: the build renders Mermaid diagrams via - // headless Chromium (rehype-mermaid) at build time, which the Workers - // Builds image can't provide (no browser, and no root to apt-install one). - // `make ci` already builds the site (where Chromium is available), so the - // deploy reuses that docs/dist/ and only runs once the whole pipeline is - // green — `wrangler deploy` on pushes to `main` (production → wavehouse.dev), - // `wrangler versions upload` on PRs (a per-version preview URL of the form + // Deploys run as dedicated jobs of the CI workflow (.github/workflows/ + // ci.yml), NOT Cloudflare's Workers Builds: the build renders Mermaid + // diagrams via headless Chromium (rehype-mermaid) at build time, which the + // Workers Builds image can't provide (no browser, and no root to + // apt-install one). CI's build job uploads docs/dist/ as an artifact; the + // deploy jobs consume it from a checkout of trusted `main` — wrangler, the + // worker source, and this config never come from a PR tree (#305), so PR + // changes to worker/ or this file take effect on merge, not in previews. + // `docs-deploy` runs `wrangler deploy` on pushes to `main` once the whole + // pipeline is green (production → wavehouse.dev); `docs-preview` runs + // `wrangler versions upload` on PRs right after the build (a per-version + // preview URL of the form // `<version-prefix>-wavehouse-docs.wave-rf.workers.dev`, posted on the PR). // Needs CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID repo secrets, and // Workers Builds must stay disconnected from this Worker (else it diff --git a/scripts/ci-marker.sh b/scripts/ci-marker.sh index eebf9503..8b59e8c0 100755 --- a/scripts/ci-marker.sh +++ b/scripts/ci-marker.sh @@ -10,7 +10,7 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" 2>/dev/null || exit 1 usage() { - echo "usage: $0 {write|write-verify|has-verify-marker|path-for-commit <sha>}" >&2 + echo "usage: $0 {write|write-verify|has-verify-marker|path-for-commit <sha>|verify-path-for-commit <sha>}" >&2 exit 2 } @@ -19,6 +19,7 @@ usage() { tree_of_working_dir() { local tmp_idx tmp_idx=$(mktemp); rm -f "$tmp_idx" + # shellcheck disable=SC2064 # expand NOW on purpose: $tmp_idx is function-local, gone at EXIT-trap time trap "rm -f '$tmp_idx'" EXIT export GIT_INDEX_FILE="$tmp_idx" git read-tree HEAD @@ -47,5 +48,11 @@ case "${1:-}" in [ -n "${2:-}" ] || usage echo "tmp/ci-passed-tree-$(git rev-parse "$2^{tree}")" ;; + verify-path-for-commit) + # The verify (not full-ci) marker for a commit's tree — the pre-push + # hook requires only this when the push is docs/prose-only. + [ -n "${2:-}" ] || usage + echo "tmp/verify-passed-tree-$(git rev-parse "$2^{tree}")" + ;; *) usage ;; esac diff --git a/scripts/ci/check-pr-title.sh b/scripts/ci/check-pr-title.sh new file mode 100755 index 00000000..be40c6e5 --- /dev/null +++ b/scripts/ci/check-pr-title.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Validate a PR title against Conventional Commits — the blocking half of +# the title gate (ci.yml's `PR title` job). Delegates the rule to the +# shared scripts/lint-pr-title.sh; this wrapper handles the CI specifics: +# +# - Reads the CURRENT title + author from the API, NOT the event payload. +# Re-runs replay the original payload, so the "fix the title, re-run +# the failed job" flow (automated by housekeeping.yml) would otherwise +# keep judging the stale title. +# - Exempts Dependabot from the length cap only (its grouped-update +# titles routinely exceed 72 chars and the format isn't configurable; +# the format check still applies). +# - Emits a `::error::` annotation on failure. +# +# Runs from a checkout of the DEFAULT branch (the job's trust model) so a +# PR can't tamper with its own validator. Env: GH_TOKEN, GITHUB_REPOSITORY, +# PR_NUMBER. + +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +pr_json="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" +title="$(jq -r '.title' <<<"$pr_json")" +author="$(jq -r '.user.login' <<<"$pr_json")" + +if [[ "$author" == "dependabot[bot]" || "$author" == "app/dependabot" ]]; then + export PR_TITLE_SKIP_LENGTH=1 +fi + +if reason="$(bash "$here/../lint-pr-title.sh" "$title" 2>&1)"; then + echo "PR title OK: $title" +else + printf '%s\n' "$reason" + echo "::error::$(printf '%s' "$reason" | head -1)" + exit 1 +fi diff --git a/scripts/ci/classify-changes.sh b/scripts/ci/classify-changes.sh new file mode 100755 index 00000000..02d9ecc9 --- /dev/null +++ b/scripts/ci/classify-changes.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# CI change-set classifier for ci.yml's `changes` job. Fetches the run's +# changed-file list from the GitHub API (CI checkouts are shallow, so the +# list can't come from `git diff`) and pipes it through the shared pure +# classifier, scripts/classify-paths.sh — the single source of truth for +# the code/docs allowlists, also usable from the local git hooks. +# +# This wrapper owns only the CI-specific policy on top of that: +# - manual dispatch ⇒ run + deploy everything; +# - an empty / errored file list (API hiccup, brand-new branch) ⇒ fail +# closed and run everything; +# - pushes to main and merge-group runs never skip code work — they're +# the last gate before main and they warm the caches every PR inherits. +# +# Prints `code=…` / `docs=…` to stdout (the step redirects to +# $GITHUB_OUTPUT) and a one-line summary to stderr. Env in: +# GITHUB_EVENT_NAME, GITHUB_REPOSITORY, GH_TOKEN, PR_NUMBER (pull_request), +# PUSH_BEFORE + PUSH_SHA (push: before/after; merge_group: group base/head). + +# -e/-pipefail: an unexpected failure aborts (and the job reds) instead of +# misclassifying; the two `gh api … || true` calls stay soft because their +# empty-output case already fails closed below. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +emit() { echo "code=$1"; echo "docs=$2"; echo "classified: code=$1 docs=$2" >&2; } + +if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then + emit true true # manual → run + deploy everything + exit 0 +fi + +if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + files="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename' || true)" +else + files="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${PUSH_BEFORE}...${PUSH_SHA}" \ + --jq '.files[].filename' || true)" +fi + +# Empty (API hiccup / new branch) → fail closed: run everything. +if [ -z "$files" ]; then + emit true true + exit 0 +fi + +# Pure classification of the file list, then the push/merge-group override. +code=""; docs="" +while IFS='=' read -r key value; do + case "$key" in + code) code="$value" ;; + docs) docs="$value" ;; + esac +done < <(printf '%s\n' "$files" | "$here/../classify-paths.sh") + +if [ "${GITHUB_EVENT_NAME}" = "push" ] || [ "${GITHUB_EVENT_NAME}" = "merge_group" ]; then + code=true +fi +emit "$code" "$docs" diff --git a/scripts/ci/docs-preview-comment.sh b/scripts/ci/docs-preview-comment.sh new file mode 100755 index 00000000..a1635072 --- /dev/null +++ b/scripts/ci/docs-preview-comment.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Post/refresh the sticky docs-preview status comment on a PR — one +# comment, updated in place (matched by the marker) so re-pushes don't +# spam the thread. Called by ci.yml's docs-preview job, which checks out +# trusted MAIN — so this file executes from the default branch, never a +# PR tree (the job's trust model; see .github/workflows/README.md). +# +# Builds a skimmable status comment (one labelled bullet per fact). +# Commit metadata — subject, author names/emails, and the commit's +# own-timezone timestamp (%cI) — is read from git, not the REST API, +# which normalizes commit dates to UTC and drops the zone. GitHub +# @-logins aren't in git, so author/committer logins come from one +# commits-API call (GitHub matches them by verified email), used only +# when the call succeeds, with a fallback that reads the login out of +# a github.com noreply address. The working tree is main, so the PR +# head commit is fetched by SHA first. All user-controlled text +# (subject, names) is only ever a printf %s arg, never the format +# string, so a `%` can't act as a format directive. +# +# Env: GH_TOKEN, GITHUB_REPOSITORY, PR (number), SHA (PR head), +# PREVIEW_OUTCOME (success|failure), URL (preview URL, may be empty). + +set -uo pipefail + +marker='<!-- docs-preview-comment -->' +git cat-file -e "${SHA}^{commit}" 2>/dev/null || git fetch --quiet origin "$SHA" 2>/dev/null || true +server="${GITHUB_SERVER_URL:-https://github.com}" +# @-link a person when a login is known, else show their plain name; +# login_from_email reads it out of [ID+]login@users.noreply.github.com. +gh_link() { if [ -n "$2" ]; then printf '[@%s](%s/%s)' "$2" "$server" "$2"; else printf '%s' "$1"; fi; } +login_from_email() { case "$1" in *@users.noreply.github.com) e="${1%@users.noreply.github.com}"; printf '%s' "${e#*+}";; esac; } + +subject="$(git show -s --format=%s "$SHA" 2>/dev/null || true)" +[ -z "$subject" ] && subject='(commit subject unavailable)' +author_name="$(git show -s --format=%an "$SHA" 2>/dev/null || true)" +author_email="$(git show -s --format=%ae "$SHA" 2>/dev/null || true)" +committer_name="$(git show -s --format=%cn "$SHA" 2>/dev/null || true)" +committer_email="$(git show -s --format=%ce "$SHA" 2>/dev/null || true)" + +# Logins, best-effort: one commits-API call (TSV: author, committer), +# used only on success so a transient API error can't leak its body +# into the comment; then a noreply-email fallback. +author_login=''; committer_login='' +if logins="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${SHA}" --jq '[.author.login // "", .committer.login // ""] | @tsv' 2>/dev/null)"; then + author_login="$(printf '%s' "$logins" | cut -f1)" + committer_login="$(printf '%s' "$logins" | cut -f2)" +fi +[ -z "$author_login" ] && author_login="$(login_from_email "$author_email")" +[ -z "$committer_login" ] && committer_login="$(login_from_email "$committer_email")" + +# Author(s): commit author, then Co-authored-by names, each @-linked +# when possible, de-duplicated by name. +authors="$(gh_link "$author_name" "$author_login")" +seen="|$author_name|" +while IFS= read -r line; do + [ -z "$line" ] && continue + cname="$(printf '%s' "$line" | sed -E 's/ *<[^>]*> *$//; s/[[:space:]]+$//')" + cmail="$(printf '%s' "$line" | sed -nE 's/.*<([^>]*)>.*/\1/p')" + [ -z "$cname" ] && continue + case "$seen" in *"|$cname|"*) continue ;; esac + authors="$authors, $(gh_link "$cname" "$(login_from_email "$cmail")")" + seen="$seen$cname|" +done < <(git show -s --format=%B "$SHA" 2>/dev/null \ + | grep -iE '^[[:space:]]*Co-authored-by:' | sed -E 's/^[^:]*:[[:space:]]*//') +[ -z "$author_name" ] && authors='(author unavailable)' + +# A distinct human committer (someone else applied the patch); skip +# GitHub's web-flow bot and anyone already credited above. +if [ -n "$committer_name" ] && [ "$committer_name" != "$author_name" ] && [ "$committer_name" != "GitHub" ]; then + case "$seen" in + *"|$committer_name|"*) ;; + *) authors="$authors (committed by $(gh_link "$committer_name" "$committer_login"))" ;; + esac +fi + +# Commit time in its own timezone: 2026-06-09T10:04:27-04:00 -> 2026-06-09 10:04 (UTC-04:00) +ciso="$(git show -s --format=%cI "$SHA" 2>/dev/null || true)" +committed='(unknown)' +if [ -n "$ciso" ]; then + day="${ciso%%T*}"; hm="${ciso#*T}"; hm="${hm:0:5}" + case "$ciso" in *Z|*+00:00) off='UTC' ;; *) off="UTC${ciso: -6}" ;; esac + committed="$day $hm ($off)" +fi +# Deploy time in the maintainer team's zone, DST-aware (EST/EDT); UTC +# if the runner lacks the zone. Distinct from the commit time — a +# re-run, or a push made long after authoring, makes them differ. +deployed="$(TZ='America/Detroit' date '+%Y-%m-%d %H:%M %Z')" +commit_url="$server/${GITHUB_REPOSITORY}/commit/${SHA}" + +# Common bullets, built once (one labelled fact per line). `printf --` +# because the format starts with a list dash; user-controlled values +# are %s args here, and again in the final body printf below, so a `%` +# never reaches a format string. +# shellcheck disable=SC2016 # the markdown template legitimately holds %s placeholders in single quotes +bullets="$(printf -- '- **Commit** — [`%s`](%s): %s\n- **Author** — %s\n- **Committed** — %s' \ + "${SHA:0:7}" "$commit_url" "$subject" "$authors" "$committed")" +# Headline + final time-label reflect how the upload actually went. +case "$PREVIEW_OUTCOME/${URL:+url}" in + success/url) headline="📚 **Docs preview is live** → $URL"; last="- **Deployed** — $deployed" ;; + success/) headline="📚 **Docs preview** uploaded, but its URL could not be parsed from the wrangler output — see the run log."; last="- **Deployed** — $deployed" ;; + *) headline="⚠️ **Docs preview failed to upload** (3 attempts) — any earlier preview is now stale. See the run log."; last="- **Upload failed** — $deployed" ;; +esac +body="$(printf '%s\n%s\n\n%s\n%s' "$marker" "$headline" "$bullets" "$last")" +# $marker interpolated into the jq program (gh api --jq takes no --arg); +# it's a constant HTML comment, so the double-quoting is safe. +cid="$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" --paginate \ + --jq "map(select(.body | startswith(\"$marker\"))) | .[0].id // empty" | head -1 || true)" +if [ -n "$cid" ]; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${cid}" -f body="$body" >/dev/null \ + || { echo "::error::Failed to update docs-preview comment ${cid} on PR #${PR}." >&2; exit 1; } +else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR}/comments" -f body="$body" >/dev/null \ + || { echo "::error::Failed to create the docs-preview comment on PR #${PR}." >&2; exit 1; } +fi +echo "Updated docs-preview comment on PR #${PR} (outcome=${PREVIEW_OUTCOME}, url=${URL:-none})" diff --git a/scripts/ci/timing-summary.sh b/scripts/ci/timing-summary.sh new file mode 100755 index 00000000..98ce8cc2 --- /dev/null +++ b/scripts/ci/timing-summary.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Emit a markdown wall-clock table for the current workflow run — which +# job started when (relative to run creation), how long it took, and how +# it concluded — so "where did the time go" is answerable from the run's +# Summary page without spelunking per-job logs. Written for +# $GITHUB_STEP_SUMMARY by ci.yml's non-gating "Timing summary" job. +# +# Env: GH_TOKEN, GITHUB_REPOSITORY, GITHUB_RUN_ID. + +set -euo pipefail + +run="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}")" +jobs="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100")" + +created="$(jq -r '.run_started_at' <<<"$run")" + +echo "## ⏱ Wall-clock by job" +echo +echo "| Job | Started | Duration | Result |" +echo "|---|---:|---:|---|" +jq -r --arg t0 "$created" ' + ($t0 | fromdateiso8601) as $start | + [.jobs[] + | select(.name != "Timing summary") + | . + {s: ((.started_at // empty | fromdateiso8601) // null), + e: ((.completed_at // empty | fromdateiso8601) // null)}] + | sort_by(.s // 0)[] + | [ .name, + (if .s then "+\(.s - $start)s" else "—" end), + (if .s and .e then "\(.e - .s)s" else "running" end), + (.conclusion // .status) ] + | "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) |" +' <<<"$jobs" +echo +echo "_Durations include runner pickup-to-completion per job; the run's" +echo "wall-clock is the latest end above. Job graph + design rationale:" +echo "[.github/workflows/README.md](https://github.com/${GITHUB_REPOSITORY}/blob/main/.github/workflows/README.md)._" diff --git a/scripts/ci/wait-artifact.sh b/scripts/ci/wait-artifact.sh new file mode 100755 index 00000000..36583202 --- /dev/null +++ b/scripts/ci/wait-artifact.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Poll the current workflow run until the named artifacts exist — the +# poll-not-`needs` pattern (.github/workflows/README.md): the consumer +# job does its own setup in parallel with the producer and only blocks +# right before the artifact is actually needed, instead of serializing +# whole jobs with a `needs` edge. +# +# Fails fast when a producer job already concluded without producing +# (failure / cancelled / skipped) — the aggregator is red from the +# producer itself in those cases; this just stops a pointless wait. +# +# Usage: +# scripts/ci/wait-artifact.sh --artifacts <name>[,<name>...] \ +# --producers <job name>[,<job name>...] [--timeout <seconds>] +# +# Producer names must match the jobs' `name:` fields in ci.yml. +# Env: GH_TOKEN, GITHUB_REPOSITORY, GITHUB_RUN_ID. + +set -euo pipefail + +artifacts="" producers="" timeout=600 +while [ $# -gt 0 ]; do + case "$1" in + --artifacts) artifacts="$2"; shift 2 ;; + --producers) producers="$2"; shift 2 ;; + --timeout) timeout="$2"; shift 2 ;; + *) echo "wait-artifact: unknown arg $1" >&2; exit 2 ;; + esac +done +[ -n "$artifacts" ] || { echo "wait-artifact: --artifacts is required" >&2; exit 2; } + +# Note: `gh api --jq` accepts a bare expression only (no --arg), so the +# parameterized filters below pipe through standalone jq instead. +# +# Resilience: this polls for the WHOLE producer duration (e2e ≈ minutes), +# so it makes many API calls — a single transient 5xx / rate-limit must +# NOT red an otherwise-green run. A failed call just skips that iteration; +# only a PERSISTENT outage lasting past --timeout fails. So each `gh api` +# is guarded (its non-zero exit is caught, never aborts under `set -e`), +# and the producer fail-fast fires only on a SUCCESSFUL jobs query. +deadline=$(( $(date +%s) + timeout )) +while :; do + if artifacts_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" 2>/dev/null)"; then + names="$(jq '[.artifacts[].name]' <<<"$artifacts_json" 2>/dev/null || echo '[]')" + if jq -e --arg want "$artifacts" \ + '($want | split(",")) - . == []' >/dev/null <<<"$names"; then + echo "All artifacts available: $artifacts" + exit 0 + fi + # Fail fast only on a successful jobs query that shows a producer + # already concluded without producing; a transient jobs-API failure + # just skips this check for the iteration. + if [ -n "$producers" ] && jobs_json="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" 2>/dev/null)"; then + bad="$(jq -r --arg jobs "$producers" \ + '[.jobs[] | select(.name as $n | ($jobs | split(",")) | index($n)) + | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "skipped") + | "\(.name) (\(.conclusion))"] | join(", ")' <<<"$jobs_json" 2>/dev/null || true)" + if [ -n "$bad" ]; then + echo "::error::Producer job(s) concluded without producing: $bad — cannot wait for: $artifacts" + exit 1 + fi + fi + echo "Waiting for $artifacts (have: $names) — retrying in 5s." + else + echo "Artifacts API call failed (transient?) — retrying in 5s." >&2 + fi + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "::error::Timed out (${timeout}s) waiting for artifacts: $artifacts" + exit 1 + fi + sleep 5 +done diff --git a/scripts/classify-paths.sh b/scripts/classify-paths.sh new file mode 100755 index 00000000..6f8d6e98 --- /dev/null +++ b/scripts/classify-paths.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Pure path classifier — the single source of truth for which file changes +# count as `code` (Go/SDK test work) vs `docs` (docs-site build inputs). +# +# Reads a newline-delimited file list on stdin, writes two key=value lines: +# +# code=true|false false only when EVERY path is prose/repo-meta +# (docs/, *.md, license/attribution, labels, issue + +# PR templates, editor + agent config) — then the +# Go/SDK suites can skip. +# docs=true|false true when any path feeds the docs-site build: docs/, +# clients/ts/ (the landing page bundles @wavehouse/sdk), +# the workspace lockfile, or the CI build plumbing +# itself (ci.yml / setup-env). +# +# Dependency-free and side-effect-free on purpose, so every caller can +# trust it with just a file list: +# - CI — scripts/ci/classify-changes.sh pipes the GitHub API file +# list through here (CI checkouts are shallow, so the list +# can't come from `git diff`). +# - hooks — the local git hooks pipe `git diff --name-only`. +# The event-level fail-closed policy (pushes / dispatches / merge-group +# runs always run everything) is a CI decision and lives in the wrapper, +# NOT here. Empty stdin ⇒ code=false docs=false; callers decide what an +# empty change set means (CI's wrapper fails closed to true). + +set -euo pipefail + +files="$(cat)" + +# Empty guard: without it the `grep -qvE` below sees a lone blank line, +# which matches no allowlist entry and would wrongly read as code=true. +if [ -z "$files" ]; then + echo "code=false" + echo "docs=false" + exit 0 +fi + +# code=true unless EVERY changed path matches the prose/meta allowlist. +# `grep` here is case-sensitive: GitHub meta paths use their real casing +# (`.github/ISSUE_TEMPLATE/`, `.github/PULL_REQUEST_TEMPLATE.md`). +if printf '%s\n' "$files" | grep -qvE '^(docs/|.*\.md$|LICENSE|NOTICE|\.gitignore$|\.gitattributes$|\.github/labeler\.yml$|\.github/ISSUE_TEMPLATE/|\.github/PULL_REQUEST_TEMPLATE|\.claude/|\.vscode/)'; then + echo "code=true" +else + echo "code=false" +fi + +# docs=true when ANY changed path is a docs-site build input. +if printf '%s\n' "$files" | grep -qE '^(docs/|clients/ts/|pnpm-lock\.yaml|pnpm-workspace\.yaml|\.github/workflows/ci\.yml|\.github/actions/setup-env/)'; then + echo "docs=true" +else + echo "docs=false" +fi diff --git a/scripts/classify-paths.test.sh b/scripts/classify-paths.test.sh new file mode 100755 index 00000000..4c517ef9 --- /dev/null +++ b/scripts/classify-paths.test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Behavioral test for scripts/classify-paths.sh — the shared change +# classifier behind CI's `changes` job and (potentially) the local git +# hooks. Pins the canonical change shapes so the allowlists can't silently +# regress; run by `make verify` (target: test-classify-paths), so it gates +# in CI exactly like a unit test. Dependency-free — no network, no gh. + +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1 # repo root (scripts/..) + +classify=scripts/classify-paths.sh +fails=0 + +# check <name> <want_code> <want_docs> <file>... +check() { + local name="$1" want_code="$2" want_docs="$3"; shift 3 + local got code docs + got="$(printf '%s\n' "$@" | "$classify")" + code="$(printf '%s\n' "$got" | sed -n 's/^code=//p')" + docs="$(printf '%s\n' "$got" | sed -n 's/^docs=//p')" + if [ "$code" = "$want_code" ] && [ "$docs" = "$want_docs" ]; then + printf ' ok %-18s code=%s docs=%s\n' "$name" "$code" "$docs" + else + printf ' FAIL %-18s want code=%s docs=%s, got code=%s docs=%s\n' \ + "$name" "$want_code" "$want_docs" "$code" "$docs" >&2 + fails=$((fails + 1)) + fi +} + +# name code docs paths... +check docs-only false true docs/src/content/docs/intro.md +check prose-only false false README.md CHANGELOG.md +check go-only true false internal/api/handler.go +check sdk true true clients/ts/src/client.ts +check workflow-ci true true .github/workflows/ci.yml +check workflow-other true false .github/workflows/housekeeping.yml +check setup-env true true .github/actions/setup-env/action.yml +check worker-under-docs false true docs/worker/index.ts +check agent-config false false .claude/hooks/agent-bash-gate.sh +check editor-config false false .vscode/settings.json +# GitHub meta paths, real casing. labeler.yml + ISSUE_TEMPLATE/config.yml +# are NOT .md, so they exercise their own allowlist entries (not `.md$`). +check labeler false false .github/labeler.yml +check issue-template false false .github/ISSUE_TEMPLATE/config.yml +check pr-template false false .github/PULL_REQUEST_TEMPLATE.md +check dep-bump-pnpm true true pnpm-lock.yaml +check dep-bump-go true false go.mod go.sum +check mixed-docs-go true true docs/x.md internal/a.go +# Empty change set (no paths) — the empty guard, distinct from "a blank line". +check empty false false + +if [ "$fails" -gt 0 ]; then + printf '\n%d case(s) failed\n' "$fails" >&2 + exit 1 +fi +echo "classify-paths: all cases passed" diff --git a/scripts/install-shellcheck.sh b/scripts/install-shellcheck.sh new file mode 100755 index 00000000..c3bd779d --- /dev/null +++ b/scripts/install-shellcheck.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Fetch the pinned shellcheck release binary, checksum-verified, into the +# path given as $1 (the Makefile's $(SHELLCHECK) — version encoded in the +# path so a version bump triggers reinstall). shellcheck is a Haskell +# binary, so the repo's `go install`-to-.bin pattern doesn't apply; +# official static release tarballs + pinned sha256 per platform instead. +# +# Usage: scripts/install-shellcheck.sh <target-path> + +set -euo pipefail + +VERSION="v0.11.0" +target="${1:?usage: install-shellcheck.sh <target-path>}" + +os="$(uname -s | tr '[:upper:]' '[:lower:]')" +arch="$(uname -m)" +case "$os.$arch" in + linux.x86_64) plat="linux.x86_64"; sha="8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198" ;; + linux.aarch64 | linux.arm64) plat="linux.aarch64"; sha="12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588" ;; + darwin.arm64) plat="darwin.aarch64"; sha="56affdd8de5527894dca6dc3d7e0a99a873b0f004d7aabc30ae407d3f48b0a79" ;; + darwin.x86_64) plat="darwin.x86_64"; sha="3c89db4edcab7cf1c27bff178882e0f6f27f7afdf54e859fa041fca10febe4c6" ;; + *) echo "install-shellcheck: unsupported platform $os/$arch" >&2; exit 1 ;; +esac + +url="https://github.com/koalaman/shellcheck/releases/download/${VERSION}/shellcheck-${VERSION}.${plat}.tar.xz" +tmp="$(mktemp -d)" +# shellcheck disable=SC2064 # expand NOW on purpose: $tmp must be the value at trap-set time +trap "rm -rf '$tmp'" EXIT + +echo "==> Downloading shellcheck ${VERSION} (${plat})..." +curl -sSfL -o "$tmp/sc.tar.xz" "$url" +echo "${sha} $tmp/sc.tar.xz" | shasum -a 256 -c - >/dev/null +tar -xJf "$tmp/sc.tar.xz" -C "$tmp" +mkdir -p "$(dirname "$target")" +install -m 0755 "$tmp/shellcheck-${VERSION}/shellcheck" "$target" +echo "==> Installed: $target" diff --git a/scripts/lint-pr-title.sh b/scripts/lint-pr-title.sh index 12c09902..abb5f36d 100755 --- a/scripts/lint-pr-title.sh +++ b/scripts/lint-pr-title.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # Canonical PR-title (Conventional Commits) validator. The SINGLE rule the local # pre-create gate (.claude/hooks/agent-bash-gate.sh) checks against, kept identical -# to the required `PR housekeeping` check (.github/workflows/housekeeping.yml) so a -# bad title is caught BEFORE `gh pr create` instead of after the required check fails -# — the recurring "title too long / wrong format" round-trip. +# to the required `CI` check's `PR title` job (.github/workflows/ci.yml) and the +# advisory mirror in housekeeping.yml, so a bad title is caught BEFORE +# `gh pr create` instead of after the required check fails — the recurring +# "title too long / wrong format" round-trip. # # Usage: scripts/lint-pr-title.sh "<title>" (or pipe the title on stdin) # Exit: 0 = valid; 1 = invalid (human-readable reason on stderr). # Env: PR_TITLE_SKIP_LENGTH=1 exempt the length cap (Dependabot grouped updates). # PR_TITLE_MAX_LEN=N override the 72-char cap. # -# Rule (KEEP IN SYNC with .github/workflows/housekeeping.yml's `pattern`): +# Rule (this script IS the single source — ci.yml and housekeeping.yml call it): # <type>(optional-scope)(optional-!): <subject> # - type in {feat,fix,docs,refactor,test,chore,ci,deps,build,perf,revert,style} # - subject starts lowercase (house style) and has no trailing period @@ -18,7 +19,7 @@ set -uo pipefail MAX_LEN="${PR_TITLE_MAX_LEN:-72}" -# Type list — keep in sync with CONTRIBUTING.md and housekeeping.yml. +# Type list — keep in sync with CONTRIBUTING.md. PATTERN='^(feat|fix|docs|refactor|test|chore|ci|deps|build|perf|revert|style)(\([^)]+\))?!?: [^A-Z].+[^.]$' # Use the argument if one was passed (even an empty one — never block on stdin diff --git a/scripts/otel/aspire.sh b/scripts/otel/aspire.sh index 3bf79632..27b4cfa7 100755 --- a/scripts/otel/aspire.sh +++ b/scripts/otel/aspire.sh @@ -26,7 +26,7 @@ echo "==> Dashboard running at: http://localhost:$PORT" if [ -z "${CI:-}" ]; then # Open browser once the dashboard is actually accepting connections ( - for i in $(seq 1 30); do + for _ in $(seq 1 30); do if curl -sf "http://localhost:${PORT}/" >/dev/null 2>&1; then if command -v open >/dev/null 2>&1; then open "http://localhost:${PORT}" diff --git a/scripts/otel/grafana.sh b/scripts/otel/grafana.sh index e1e9ef26..62ca3d29 100755 --- a/scripts/otel/grafana.sh +++ b/scripts/otel/grafana.sh @@ -26,7 +26,7 @@ echo "==> Dashboard running at: http://localhost:$PORT" if [ -z "${CI:-}" ]; then # Open browser once the dashboard is actually accepting connections ( - for i in $(seq 1 30); do + for _ in $(seq 1 30); do if curl -sf "http://localhost:${PORT}/" >/dev/null 2>&1; then if command -v open >/dev/null 2>&1; then open "http://localhost:${PORT}" diff --git a/scripts/otel/otel-front.sh b/scripts/otel/otel-front.sh index ff9ba024..3c09ff83 100755 --- a/scripts/otel/otel-front.sh +++ b/scripts/otel/otel-front.sh @@ -24,7 +24,7 @@ echo "==> Dashboard running at: http://localhost:$PORT" if [ -z "${CI:-}" ]; then # Open browser once the dashboard is actually accepting connections ( - for i in $(seq 1 30); do + for _ in $(seq 1 30); do if curl -sf "http://localhost:${PORT}/" >/dev/null 2>&1; then if command -v open >/dev/null 2>&1; then open "http://localhost:${PORT}"