diff --git a/.github/scripts/post-commit-status.sh b/.github/scripts/post-commit-status.sh new file mode 100755 index 00000000..55a1213f --- /dev/null +++ b/.github/scripts/post-commit-status.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Post a commit status to a PR head SHA. +# +# Usage: post-commit-status.sh [description] [target-url] +# state ∈ {pending, success, failure, error} +# +# The status context is fixed to "bench-ok" — the name branch protection +# requires. Splitting benchmark execution across two workflows (a fork-safe +# `pull_request` planner and a trusted `workflow_run` executor) means the real +# pass/fail verdict is posted here from the trusted run, while the check keeps +# the single stable name the required-checks list already knows. +# +# Requires GH_TOKEN with `statuses: write`. In the trusted workflow_run context +# the default GITHUB_TOKEN has it; in the fork `pull_request` run it can still +# write a status to its own head SHA (pending / short-circuit success). +# +# The SHA is fork-controlled (it arrives via a handoff artifact), so validate it +# is a full 40-hex commit id before using it in the API path — same defensive +# posture as the numeric PR-number guard in the workflow_run handoff. + +set -euo pipefail + +SHA="$1" +STATE="$2" +DESCRIPTION="${3:-}" +TARGET_URL="${4:-}" + +if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: refusing to post status — '$SHA' is not a 40-hex commit SHA." >&2 + exit 1 +fi + +case "$STATE" in + pending | success | failure | error) ;; + *) + echo "ERROR: invalid state '$STATE' (want pending|success|failure|error)." >&2 + exit 1 + ;; +esac + +# GitHub caps status descriptions at 140 chars; trim defensively. +DESCRIPTION="${DESCRIPTION:0:140}" + +ARGS=( + --method POST + "repos/${GITHUB_REPOSITORY}/statuses/${SHA}" + -f "state=${STATE}" + -f "context=bench-ok" +) +[[ -n "$DESCRIPTION" ]] && ARGS+=(-f "description=${DESCRIPTION}") +[[ -n "$TARGET_URL" ]] && ARGS+=(-f "target_url=${TARGET_URL}") + +echo "Posting bench-ok=${STATE} to ${SHA}" +gh api "${ARGS[@]}" diff --git a/.github/workflows/benchmark-execute.yml b/.github/workflows/benchmark-execute.yml new file mode 100644 index 00000000..204ea98d --- /dev/null +++ b/.github/workflows/benchmark-execute.yml @@ -0,0 +1,431 @@ +name: Benchmark execute + +# Trusted half of the fork-safe benchmark flow. The Benchmark workflow runs on +# `pull_request` — on fork PRs it gets a read-only token and no secrets, so it +# can only plan and upload a metadata handoff. This workflow fills the gap. +# +# `workflow_run` always runs in the BASE repo's context, using the workflow +# definition from the default branch — trusted code with `packages: write` and +# full secret access — regardless of whether the triggering run came from a +# fork. It therefore does everything the planner couldn't: +# +# • build the PR's changed solver images and push them to (private) GHCR, +# • run the benchmarks (pulling the private images it just pushed, plus the +# immutable base-SHA images for unchanged solvers), +# • render the docs preview, and +# • post the `bench-ok` commit status + the PR comment / RTD preview. +# +# SECURITY: this run holds write scope and secrets, so it must never execute +# PR-authored *workflow/script* code. It checks out the base repo's default +# branch for its own scripts; the only PR-authored code it touches is the +# solver source it builds into a container and runs in the same sandboxed +# runners same-repo PRs already use. Every fork-controlled value (SHAs, PR +# number, label, matrices) arrives via the handoff artifact and is re-validated +# before use — same posture the retired docs-publish.yml used. + +on: + workflow_run: + workflows: ["Benchmark"] + types: [completed] + +permissions: + contents: read + packages: write # push/pull private solver images + statuses: write # post the bench-ok commit status + actions: read # download the handoff artifact from the triggering run + +concurrency: + # Serialize per triggering PR head; a newer push supersedes an in-flight run. + group: benchmark-execute-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: true + +jobs: + # ── Recover + validate the handoff from the triggering PR run ────────── + recover-meta: + name: Recover handoff + # Only PR-triggered Benchmark runs that succeeded (a failed planner already + # posted bench-ok=failure and there's nothing to execute). + if: >- + github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + found: ${{ steps.parse.outputs.found }} + should_run: ${{ steps.parse.outputs.should_run }} + pr_number: ${{ steps.parse.outputs.pr_number }} + head_sha: ${{ steps.parse.outputs.head_sha }} + base_sha: ${{ steps.parse.outputs.base_sha }} + label: ${{ steps.parse.outputs.label }} + is_release_pr: ${{ steps.parse.outputs.is_release_pr }} + problems: ${{ steps.parse.outputs.problems }} + suites: ${{ steps.parse.outputs.suites }} + solvers: ${{ steps.parse.outputs.solvers }} + solver_matrix: ${{ steps.parse.outputs.solver_matrix }} + run_matrix: ${{ steps.parse.outputs.run_matrix }} + steps: + - name: Download handoff + id: dl + uses: actions/download-artifact@v8 + continue-on-error: true + with: + pattern: bench-handoff-* + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: handoff/ + merge-multiple: true + + - name: Parse and validate handoff + id: parse + run: | + set -euo pipefail + META="handoff/handoff-meta.txt" + if [[ ! -f "$META" ]]; then + echo "No handoff metadata — nothing to execute." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + get() { grep -oP "^$1=\K.*" "$META" || true; } + PR_NUMBER="$(get pr_number)" + HEAD_SHA="$(get head_sha)" + BASE_SHA="$(get base_sha)" + SHOULD_RUN="$(get should_run)" + LABEL="$(get label)" + IS_RELEASE_PR="$(get is_release_pr)" + PROBLEMS="$(get problems)" + SUITES="$(get suites)" + SOLVERS="$(get solvers)" + + # ── Validate every fork-controlled value before use ────────────── + fail() { echo "::warning::$1 — skipping."; echo "found=false" >> "$GITHUB_OUTPUT"; exit 0; } + [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || fail "PR number '$PR_NUMBER' not numeric" + [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "head_sha '$HEAD_SHA' not a 40-hex SHA" + [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "base_sha '$BASE_SHA' not a 40-hex SHA" + case "$SHOULD_RUN" in true | false) ;; *) fail "should_run '$SHOULD_RUN' invalid" ;; esac + case "$LABEL" in none | solver | all | "") ;; *) fail "label '$LABEL' invalid" ;; esac + case "$IS_RELEASE_PR" in true | false) ;; *) fail "is_release_pr '$IS_RELEASE_PR' invalid" ;; esac + + # Matrices are JSON; validate they parse and re-emit compactly. + SOLVER_MATRIX="$(python3 -c 'import json,sys; print(json.dumps(json.load(open("handoff/solver-matrix.json"))))' 2>/dev/null || echo "INVALID")" + [[ "$SOLVER_MATRIX" != "INVALID" ]] || fail "solver-matrix.json not valid JSON" + # run-matrix is only present/needed when benchmarks run. + if [[ "$SHOULD_RUN" == "true" ]]; then + RUN_MATRIX="$(python3 -c 'import json,sys; print(json.dumps(json.load(open("handoff/run-matrix.json"))))' 2>/dev/null || echo "INVALID")" + [[ "$RUN_MATRIX" != "INVALID" ]] || fail "run-matrix.json not valid JSON" + else + RUN_MATRIX='{"include":[]}' + fi + + { + echo "found=true" + echo "pr_number=$PR_NUMBER" + echo "head_sha=$HEAD_SHA" + echo "base_sha=$BASE_SHA" + echo "should_run=$SHOULD_RUN" + echo "label=$LABEL" + echo "is_release_pr=$IS_RELEASE_PR" + echo "problems=$PROBLEMS" + echo "suites=$SUITES" + echo "solvers=$SOLVERS" + echo "solver_matrix=$SOLVER_MATRIX" + echo "run_matrix=$RUN_MATRIX" + } >> "$GITHUB_OUTPUT" + echo "Recovered handoff for PR #$PR_NUMBER (head $HEAD_SHA, should_run=$SHOULD_RUN)" + + # ── Build changed solver images → private GHCR : ───────────── + build: + name: Build solver images + needs: [recover-meta] + if: >- + needs.recover-meta.outputs.found == 'true' + && needs.recover-meta.outputs.should_run == 'true' + && needs.recover-meta.outputs.solver_matrix != '[]' + uses: ./.github/workflows/build-tesseracts.yml + with: + matrix: ${{ needs.recover-meta.outputs.solver_matrix }} + ref: ${{ needs.recover-meta.outputs.head_sha }} + image_sha: ${{ needs.recover-meta.outputs.head_sha }} + permissions: + contents: read + packages: write + # Deliberately NOT `secrets: inherit`. This job checks out and executes + # PR-authored code (the solver's build_base.sh and Dockerfile) to build the + # image, so it must see the smallest possible secret surface. build-tesseracts + # needs only the auto-provided GITHUB_TOKEN for GHCR; RTD_TOKEN / the bot PAT + # stay out of reach of fork build scripts. (The images are then run in the + # same sandboxed runners same-repo PRs already use.) + + # ── Run benchmarks (ics + solver suites) ─────────────────────────────── + benchmarks: + name: Run benchmarks + needs: [recover-meta, build] + if: >- + always() && !cancelled() + && needs.recover-meta.outputs.found == 'true' + && needs.recover-meta.outputs.should_run == 'true' + && (needs.build.result == 'success' || needs.build.result == 'skipped') + uses: ./.github/workflows/benchmark-run.yml + with: + matrix: ${{ needs.recover-meta.outputs.run_matrix }} + problems: ${{ needs.recover-meta.outputs.problems }} + suites: ${{ needs.recover-meta.outputs.suites }} + solvers: ${{ needs.recover-meta.outputs.solvers }} + # Primary tag: this PR's HEAD (built by the build job above). Fallback: + # the PR's base SHA — always built by the push-to-main run and immutable, + # so unchanged solvers pull a stable image rather than the mutable :latest. + image_tag: ${{ needs.recover-meta.outputs.head_sha }} + base_image_tag: ${{ needs.recover-meta.outputs.base_sha }} + timeout: 600 + permissions: + contents: read + packages: read + # No `secrets: inherit`: this job runs the fork-built solver container, so + # it sees the smallest secret surface. benchmark-run.yml needs only the + # auto-provided GITHUB_TOKEN (GHCR pull); RTD_TOKEN / the bot PAT stay out. + + # ── Merge results + render docs preview ───────────────────────────────── + # Runs for every recovered PR (even benchmark:none / docs-only), so the RTD + # preview always reflects this PR. Benchmark-result steps are guarded on + # should_run; the render always runs. Mirrors the old benchmark.yml `report` + # job, but here in the trusted context it also consumes the results produced + # by the `benchmarks` job in *this* run. + report: + name: Report & render + needs: [recover-meta, benchmarks] + if: >- + always() && !cancelled() + && needs.recover-meta.outputs.found == 'true' + && (needs.benchmarks.result == 'success' || needs.benchmarks.result == 'skipped') + runs-on: ubuntu-latest + outputs: + report_ok: ${{ steps.done.outputs.ok }} + steps: + - uses: actions/checkout@v6 + # Base-repo default branch: our own trusted scripts, never PR code. + + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Install Mosaic + run: cp production.uv.lock uv.lock && uv sync --frozen + + - name: Download all results + # Only runs that ran benchmarks produce results-* artifacts. They live + # in *this* run (benchmarks job above), so a same-run download works. + if: needs.recover-meta.outputs.should_run == 'true' + uses: actions/download-artifact@v8 + with: + pattern: results-* + path: staging-results/ + + - name: Merge results from CPU/GPU artifacts + if: needs.recover-meta.outputs.should_run == 'true' + run: | + mkdir -p staging-results + uv run python .github/scripts/merge-results.py staging-results/ mosaic-results/ + + - name: Download baseline artifact + if: needs.recover-meta.outputs.is_release_pr != 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + run: | + # Exit 2 = baseline not found (non-fatal: first run has no baseline). + bash .github/scripts/fetch-artifact.sh baseline baseline/mosaic-results/ \ + || [ "$?" -eq 2 ] + + - name: Overlay PR results on baseline + if: needs.recover-meta.outputs.is_release_pr != 'true' + run: | + if [ -d baseline/mosaic-results ] && [ "$(ls baseline/mosaic-results/ 2>/dev/null)" ]; then + if [ -d mosaic-results ] && [ "$(ls mosaic-results/ 2>/dev/null)" ]; then + MERGED=$(mktemp -d) + uv run python .github/scripts/overlay-results.py \ + baseline/mosaic-results mosaic-results "$MERGED" + rm -rf mosaic-results + mv "$MERGED" mosaic-results + else + rm -rf mosaic-results + cp -r baseline/mosaic-results mosaic-results + fi + fi + + - name: Extract baseline snapshot + id: baseline + if: needs.recover-meta.outputs.should_run == 'true' + run: | + BASELINE_PATH="" + if [ -f baseline/mosaic-results/snapshot.json ]; then + cp baseline/mosaic-results/snapshot.json baseline-snapshot.json + BASELINE_PATH="baseline-snapshot.json" + echo "Fetched baseline snapshot" + fi + echo "path=$BASELINE_PATH" >> "$GITHUB_OUTPUT" + + - name: Generate status snapshot (JSON) + if: needs.recover-meta.outputs.should_run == 'true' + run: uv run mosaic status --format json > snapshot.json + + - name: Generate status report (markdown) + if: needs.recover-meta.outputs.should_run == 'true' + run: | + DIFF_FLAG="" + if [[ -n "${{ steps.baseline.outputs.path }}" ]]; then + DIFF_FLAG="--diff-against ${{ steps.baseline.outputs.path }}" + fi + uv run mosaic status --format md $DIFF_FLAG > status-report.md + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: "1.7.31" + + - name: Regenerate plots from merged results + run: uv run mosaic run --plots-only + + - name: Generate results pages + run: uv run python docs/generate_results.py + + - name: Render docs site + run: uv run quarto render --output-dir _site + + - name: Upload rendered docs site + uses: actions/upload-artifact@v7 + with: + name: docs-site-pr-${{ needs.recover-meta.outputs.pr_number }} + path: _site/ + overwrite: true + retention-days: 90 + + - name: Stash status report for finalize + if: needs.recover-meta.outputs.should_run == 'true' + uses: actions/upload-artifact@v7 + with: + name: status-report-${{ needs.recover-meta.outputs.pr_number }} + path: status-report.md + if-no-files-found: ignore + overwrite: true + retention-days: 7 + + - id: done + run: echo "ok=true" >> "$GITHUB_OUTPUT" + + # ── Post the bench-ok status + RTD preview + PR comment ──────────────── + # Runs for EVERY triggering PR run (not gated on found/should_run), because + # its first job is the terminal-state guarantee: whenever the planner seeded + # bench-ok=pending, something here must resolve it, or the PR is stuck + # unmergeable forever. So this job always posts a definitive bench-ok: + # + # • handoff missing/invalid (found=false) → error (couldn't verify) + # • benchmarks ran (should_run=true) → success | failure by results + # • nothing ran (should_run=false) → skip; plan-gate seeded success + # + # The head SHA comes from the validated handoff when present, else from + # github.event.workflow_run.head_sha (the PR head of the triggering run) — + # both identify the same commit; post-commit-status.sh re-validates it. + finalize: + name: Finalize (status, RTD, comment) + needs: [recover-meta, build, benchmarks, report] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + # ── bench-ok commit status: the merge gate + deadlock backstop ──── + # Only act when the PLANNER SUCCEEDED. If it concluded failure/cancelled + # (policy block, or a handoff-upload failure), plan-gate already owns the + # terminal bench-ok — either it posted `failure` for the policy violation, + # or nothing was seeded. Posting here in that case would clobber a correct + # status (e.g. flip a docs-only PR's green to error). So this step runs + # only for a successful planner run, which is exactly when a pending may + # have been seeded and we owe the terminal verdict. + - name: Post bench-ok status + if: github.event.workflow_run.conclusion == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FOUND: ${{ needs.recover-meta.outputs.found }} + SHOULD_RUN: ${{ needs.recover-meta.outputs.should_run }} + META_SHA: ${{ needs.recover-meta.outputs.head_sha }} + EVENT_SHA: ${{ github.event.workflow_run.head_sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + HEAD_SHA="${META_SHA:-$EVENT_SHA}" + + if [[ "$FOUND" != "true" ]]; then + # Planner succeeded (so a pending was seeded) but we couldn't recover + # the handoff — resolve the pending as error rather than orphan it. + if [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + bash .github/scripts/post-commit-status.sh "$HEAD_SHA" error \ + "Could not recover benchmark handoff — re-run required" "$RUN_URL" + else + echo "No valid head SHA available — cannot post status." + fi + exit 0 + fi + + # benchmark:none / docs-only: no benchmarks ran, so bench-ok is + # success. plan-gate already seeded this, but re-assert it as the + # authoritative terminal post in case the fork seed couldn't write. + if [[ "$SHOULD_RUN" != "true" ]]; then + bash .github/scripts/post-commit-status.sh "$HEAD_SHA" success \ + "No benchmarks to run for this PR" "$RUN_URL" + exit 0 + fi + + BUILD="${{ needs.build.result }}" + BENCH="${{ needs.benchmarks.result }}" + REPORT="${{ needs.report.result }}" + echo "build=$BUILD benchmarks=$BENCH report=$REPORT" + + STATE="success"; DESC="Benchmarks passed" + for r in "$BUILD" "$BENCH" "$REPORT"; do + if [[ "$r" != "success" && "$r" != "skipped" ]]; then + STATE="failure"; DESC="A benchmark job failed" + fi + done + bash .github/scripts/post-commit-status.sh "$HEAD_SHA" "$STATE" "$DESC" "$RUN_URL" + + # The remaining steps (RTD, PR comment) only make sense with a recovered + # handoff; skip them cleanly when found=false. + - name: Download status report + if: needs.recover-meta.outputs.found == 'true' && needs.recover-meta.outputs.should_run == 'true' + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: status-report-${{ needs.recover-meta.outputs.pr_number }} + path: report/ + + # ── Trigger RTD preview rebuild + resolve URL (best-effort) ──────── + - name: Trigger RTD PR preview rebuild + if: needs.recover-meta.outputs.found == 'true' + continue-on-error: true + env: + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_PROJECT: ${{ vars.RTD_PROJECT }} + run: bash .github/scripts/rtd-trigger.sh "${{ needs.recover-meta.outputs.pr_number }}" + + - name: Resolve RTD preview URL + id: rtd_url + if: needs.recover-meta.outputs.found == 'true' + continue-on-error: true + env: + RTD_TOKEN: ${{ secrets.RTD_TOKEN }} + RTD_PROJECT: ${{ vars.RTD_PROJECT }} + run: bash .github/scripts/rtd-preview-url.sh "${{ needs.recover-meta.outputs.pr_number }}" + + # ── Post / update the PR status comment ─────────────────────────── + - name: Post status comment on PR + if: needs.recover-meta.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.PL_PASTEURBOT_PAT_PUBLIC }} + GITHUB_REPOSITORY: ${{ github.repository }} + DOCS_PREVIEW_URL: ${{ steps.rtd_url.outputs.preview_url }} + run: | + REPORT="" + if [[ -f report/status-report.md ]]; then + REPORT="report/status-report.md" + fi + .github/scripts/post-status-comment.sh \ + "${{ needs.recover-meta.outputs.pr_number }}" \ + "$REPORT" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9099662c..b4bcdc61 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -12,10 +12,19 @@ on: description: "Comma-separated suites to run (or 'all')" default: "all" +# This workflow runs on `pull_request`, so on fork PRs it gets a read-only +# token and no repo secrets. It therefore does only the secret-free work: +# planning and a metadata handoff. The privileged half — building solver +# images, pushing to (private) GHCR, running benchmarks, rendering docs, and +# commenting — lives in benchmark-execute.yml, which fires on `workflow_run` +# in the trusted base-repo context. See that file's header for the trust model. +# +# `statuses: write` lets the plan-gate job post the `bench-ok` commit status +# (the name branch protection requires) to this PR's head SHA — pending when a +# trusted run will follow, or an immediate success/failure when it won't. permissions: contents: read - pull-requests: write - packages: read + statuses: write actions: read concurrency: @@ -176,286 +185,150 @@ jobs: echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix: $MATRIX" - # ── Build changed solver images → GHCR : ───────────────────────── - build: - name: Build solver images - needs: plan - if: needs.plan.outputs.should_run == 'true' && needs.plan.outputs.solver_matrix != '[]' - uses: ./.github/workflows/build-tesseracts.yml - with: - matrix: ${{ needs.plan.outputs.solver_matrix }} - permissions: - contents: read - packages: write - secrets: inherit - - # ── Run benchmarks (ics + solver suites) ─────────────────────────────── - benchmarks: - name: Run benchmarks - needs: [plan, build] - if: always() && needs.plan.outputs.should_run == 'true' && !cancelled() && needs.plan.result == 'success' && (needs.build.result == 'success' || needs.build.result == 'skipped') - uses: ./.github/workflows/benchmark-run.yml - with: - matrix: ${{ needs.plan.outputs.matrix }} - problems: ${{ needs.plan.outputs.problems }} - suites: ${{ needs.plan.outputs.suites }} - solvers: ${{ needs.plan.outputs.solvers }} - # Primary tag: this PR's HEAD (built by the build job for changed solvers). - # Fallback tag: the PR's base, i.e. the main commit this branch sits on — - # always built by the push-to-main "Build Tesseracts" run and immutable. - # Solvers not rebuilt by this PR (unchanged solvers, or any solver on a - # release PR that builds nothing) are pulled at the base SHA rather than - # the mutable :latest, so a stale/racy :latest can never be served. - image_tag: ${{ github.event.pull_request.head.sha || github.sha }} - base_image_tag: ${{ github.event.pull_request.base.sha || github.sha }} - timeout: 600 - permissions: - contents: read - packages: read - secrets: inherit + # Guard: should_run=true but an empty include matrix means the plan + # decided to run yet generated no jobs (e.g. the should_run filter and + # the matrix generator disagreed). That would yield a green bench-ok + # with nothing benchmarked — fail loudly instead of passing silently. + N=$(echo "$MATRIX" | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('include', [])))") + if [[ "$N" -eq 0 ]]; then + echo "::error::should_run=true but the benchmark matrix is empty — planning inconsistency." + exit 1 + fi - # ── Merge results, report, publish ───────────────────────────────────── - report: - name: Report & publish - needs: [plan, benchmarks] - # Runs on every PR (and every dispatch where benchmarks ran), even for - # benchmark:none PRs that ran no benchmarks. The benchmark-result steps - # below are guarded on should_run; the docs render + upload steps run - # regardless so doc-only PRs still get a Read the Docs preview. A - # benchmark:none PR still fetches the published baseline, so its preview - # shows the full site populated with the last released results. - # - # This job runs in the PR head's context and uses NO repo secrets, so it - # works identically for fork PRs. The secret-requiring follow-up (trigger - # the RTD rebuild, resolve the preview URL, post the PR comment) lives in - # docs-publish.yml, which fires on workflow_run in the base-repo context. - if: >- - always() && !cancelled() - && needs.plan.result == 'success' - && (github.event_name == 'pull_request' || needs.plan.outputs.should_run == 'true') + # ── Plan-time merge gate + bench-ok status seeding ───────────────────── + # Renamed from `bench-ok` (that name is now the *commit status* the trusted + # benchmark-execute run posts — see below). This job enforces only what is + # decidable at plan time (the label policy) and seeds the `bench-ok` status: + # + # • policy violation (mosaic/ touched, no label, non-release) → failure + # • nothing will run (benchmark:none, no solver change, docs-only) → the + # trusted run still renders docs but posts no gate, so seed success here + # • a trusted run will run benchmarks → seed pending; benchmark-execute + # flips it to success/failure when it finishes + # + # Branch protection keeps requiring the `bench-ok` context, so the merge + # can't go green until either this job short-circuits it or the trusted run + # reports real results. `github.actor` from a fork can still write a status + # to its own head SHA with the (read-only-elsewhere) default token. + plan-gate: + name: Plan gate + if: always() + needs: [plan] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Install Mosaic - run: cp production.uv.lock uv.lock && uv sync --frozen - - - name: Download all results - # Only PR runs that actually ran benchmarks produce results-* artifacts. - # benchmark:none PRs skip this and rely on the baseline below. - if: needs.plan.outputs.should_run == 'true' - uses: actions/download-artifact@v8 - with: - pattern: results-* - path: staging-results/ - - - name: Merge results from CPU/GPU artifacts - if: needs.plan.outputs.should_run == 'true' - run: | - # download-artifact doesn't create the path dir when nothing matches. - mkdir -p staging-results - uv run python .github/scripts/merge-results.py staging-results/ mosaic-results/ - - - name: Download baseline artifact - # The baseline artifact is produced by a *different* workflow run - # (publish-results.yml on main), so actions/download-artifact (which - # only sees the current run) can't fetch it. fetch-artifact.sh looks up - # the most recent baseline across the whole repo via the API. - if: >- - github.event_name == 'pull_request' - && needs.plan.outputs.is_release_pr != 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPO: ${{ github.repository }} - run: | - # Exit 2 = baseline not found (non-fatal: first run has no baseline). - bash .github/scripts/fetch-artifact.sh baseline baseline/mosaic-results/ \ - || [ "$?" -eq 2 ] - - - name: Overlay PR results on baseline - if: >- - github.event_name == 'pull_request' - && needs.plan.outputs.is_release_pr != 'true' - run: | - if [ -d baseline/mosaic-results ] && [ "$(ls baseline/mosaic-results/ 2>/dev/null)" ]; then - if [ -d mosaic-results ] && [ "$(ls mosaic-results/ 2>/dev/null)" ]; then - # PR ran benchmarks: overlay its (possibly partial) results on the - # baseline, PR winning per solver. - MERGED=$(mktemp -d) - uv run python .github/scripts/overlay-results.py \ - baseline/mosaic-results mosaic-results "$MERGED" - rm -rf mosaic-results - mv "$MERGED" mosaic-results - else - # benchmark:none (or a PR that ran nothing): no PR results to - # overlay, so the preview just shows the published baseline. - rm -rf mosaic-results - cp -r baseline/mosaic-results mosaic-results - fi - fi - - # The status snapshot/report exist only to summarise *this run's* results - # against the baseline. When no benchmarks ran (e.g. benchmark:none or a - # docs-only PR), the result tree is just a copy of the baseline, so the - # report would be an all-zero diff — pure noise. Skip it; the PR comment - # then carries only the docs-preview link. - - name: Extract baseline snapshot - id: baseline - if: needs.plan.outputs.should_run == 'true' - run: | - BASELINE_PATH="" - if [ -f baseline/mosaic-results/snapshot.json ]; then - cp baseline/mosaic-results/snapshot.json baseline-snapshot.json - BASELINE_PATH="baseline-snapshot.json" - echo "Fetched baseline snapshot" - fi - echo "path=$BASELINE_PATH" >> "$GITHUB_OUTPUT" - - - name: Generate status snapshot (JSON) - if: needs.plan.outputs.should_run == 'true' - run: uv run mosaic status --format json > snapshot.json - - - name: Generate status report (markdown) - if: needs.plan.outputs.should_run == 'true' - run: | - DIFF_FLAG="" - if [[ -n "${{ steps.baseline.outputs.path }}" ]]; then - DIFF_FLAG="--diff-against ${{ steps.baseline.outputs.path }}" - fi - uv run mosaic status --format md $DIFF_FLAG > status-report.md - - # ── Render docs site on GHA ──────────────────────────────────────── - # The docs are rendered here (where the full mosaic/jax stack is already - # installed and memory is ample) rather than on Read the Docs, whose - # constrained builders OOM-kill the heavy `import mosaic`. RTD only - # downloads and serves the pre-rendered _site/ (see .readthedocs.yaml). - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: "1.7.31" - - # Regenerate plots for ALL solvers from the merged result.json files. - # The baseline artifact stores data only (plots are stripped before - # upload), and only the changed solver re-ran here, so without this step - # generate_results.py would glob just the changed solver's PNGs and the - # docs would omit every baseline solver. --plots-only is no-Docker/no-GPU. - - name: Regenerate plots from merged results - run: uv run mosaic run --plots-only - - - name: Generate results pages - run: uv run python docs/generate_results.py - - - name: Render docs site - # Always render the full site, so doc-only (benchmark:none) PRs preview - # their prose changes, not just the results pages. Results pages are - # populated from the merged PR+baseline result tree above. - run: uv run quarto render --output-dir _site - - # ── Metadata for the downstream publish workflow ─────────────────── - # The RTD trigger, preview-URL lookup, and PR comment all need repo - # secrets, which GitHub withholds from pull_request runs on fork PRs. - # So this job (which runs in the *fork's* context) does the secret-free - # work — render + upload — and hands off to docs-publish.yml, which runs - # on workflow_run in the *base* repo's context (trusted main code, full - # secret access). The handoff carries everything that workflow needs but - # can't otherwise recover: the PR number and whether benchmarks ran. - - name: Write publish metadata + - name: Evaluate policy and seed bench-ok status if: github.event_name == 'pull_request' - run: | - { - echo "pr_number=${{ github.event.pull_request.number }}" - echo "should_run=${{ needs.plan.outputs.should_run }}" - } > publish-meta.txt - # Ship the status report alongside the site so the downstream comment - # can include it. Absent (benchmark:none) → downstream posts only the - # preview banner. - if [[ "${{ needs.plan.outputs.should_run }}" == "true" ]]; then - cp status-report.md _site-status-report.md - fi - - - name: Upload rendered docs site - uses: actions/upload-artifact@v7 - with: - # One artifact per PR (RTD serves it keyed off the PR number). For - # non-PR runs (e.g. workflow_dispatch) fall back to the commit SHA. - name: docs-site-pr-${{ github.event.pull_request.number || github.sha }} - path: _site/ - overwrite: true - retention-days: 90 - - - name: Upload publish handoff - # Consumed by docs-publish.yml (workflow_run). Kept separate from the - # _site artifact so the downstream can grab just the small metadata + - # report without unpacking the whole site. - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v7 - with: - name: docs-publish-meta-${{ github.event.pull_request.number }} - path: | - publish-meta.txt - _site-status-report.md - if-no-files-found: ignore - overwrite: true - retention-days: 7 - - # ── Merge gate ───────────────────────────────────────────────────────── - bench-ok: - name: Benchmark checks passed - if: always() - needs: [plan, build, benchmarks, report] - runs-on: ubuntu-latest - steps: - - name: Decide status + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | echo "plan: ${{ needs.plan.result }}" - echo "build: ${{ needs.build.result }}" - echo "benchmarks: ${{ needs.benchmarks.result }}" - echo "report: ${{ needs.report.result }}" echo "label: ${{ needs.plan.outputs.label }}" echo "mosaic_touched: ${{ needs.plan.outputs.mosaic_touched }}" + echo "should_run: ${{ needs.plan.outputs.should_run }}" + + # The seed posts are BEST-EFFORT: a fork token that can't write a + # status must not fail this step (that would flip the run conclusion + # to failure and gate out the trusted executor, orphaning the gate). + # The executor's finalize job is the authoritative terminal post for + # every found=true PR — these seeds only surface state sooner. The + # policy `exit 1` paths below stay fatal (they intentionally block). + post() { bash .github/scripts/post-commit-status.sh "$HEAD_SHA" "$1" "$2" "$RUN_URL" || echo "::warning::bench-ok seed ($1) could not be posted; finalize will set it."; } # plan must always succeed if [[ "${{ needs.plan.result }}" != "success" ]]; then + post failure "Plan job failed" echo "::error::Plan failed." exit 1 fi # PR that touches mosaic/ without a benchmark label → block merge - # (release PRs are exempt — they always run full benchmarks) - if [[ "${{ github.event_name }}" == "pull_request" \ - && "${{ needs.plan.outputs.mosaic_touched }}" == "true" \ + # (release PRs are exempt — they always run full benchmarks). The + # planner run then concludes failure, so the executor never runs and + # this failure status is the final word until a label is applied. + if [[ "${{ needs.plan.outputs.mosaic_touched }}" == "true" \ && "${{ needs.plan.outputs.is_release_pr }}" != "true" \ && -z "${{ needs.plan.outputs.label }}" ]]; then + post failure "Add a benchmark:{none,solver,all} label" echo "::error::This PR modifies mosaic/ code but has no benchmark label." echo "::error::A maintainer must add exactly one of: benchmark:none, benchmark:solver, benchmark:all" exit 1 fi - # benchmark:none → no benchmarks to check, but the report job still - # runs to render the docs preview. A failed render (e.g. a broken - # .qmd) should block merge. RTD publishing / PR-commenting happen in - # docs-publish.yml (workflow_run) and never gate the merge. - if [[ "${{ needs.plan.outputs.label }}" == "none" ]]; then - if [[ "${{ github.event_name }}" == "pull_request" \ - && "${{ needs.report.result }}" != "success" \ - && "${{ needs.report.result }}" != "skipped" ]]; then - echo "::error::Docs preview (report job) failed (${{ needs.report.result }})." - exit 1 - fi - echo "benchmark:none — skipping benchmark checks (docs preview OK)" + # Nothing will run (benchmark:none, no solver change, docs-only): + # seed success now so the check shows green promptly. finalize + # re-asserts success for this case too, so a failed seed still + # resolves. + if [[ "${{ needs.plan.outputs.should_run }}" != "true" ]]; then + post success "No benchmarks to run for this PR" + echo "No benchmarks will run — bench-ok seeded success." exit 0 fi - # When benchmarks ran, all downstream jobs must succeed (or be skipped) - if [[ "${{ needs.plan.outputs.should_run }}" == "true" ]]; then - for result in "${{ needs.build.result }}" "${{ needs.benchmarks.result }}" "${{ needs.report.result }}"; do - if [[ "$result" != "success" && "$result" != "skipped" ]]; then - echo "::error::Benchmark job failed ($result)." - exit 1 - fi - done - fi + # Benchmarks will run in the trusted benchmark-execute workflow. + # Seed pending; finalize flips bench-ok to success/failure. + post pending "Benchmarks running in benchmark-execute" + echo "Seeded bench-ok=pending; trusted run will report results." + + # ── Handoff to the trusted benchmark-execute workflow ────────────────── + # The privileged work (build + push to private GHCR, run benchmarks, render + # docs, comment) can't happen here on fork PRs. Ship everything the trusted + # workflow_run needs — but can't recover from its own context — as a small + # artifact. Every value is a plan output (numeric / enum / SHA / JSON) and is + # re-validated on the far side before use. + handoff: + name: Handoff to benchmark-execute + needs: [plan] + if: always() && needs.plan.result == 'success' && github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Write handoff metadata + # All values go through env rather than inline ${{ }} interpolation: + # PR-controlled strings (labels, and especially any free-form field) + # must never be spliced into a shell script directly. The executor + # re-validates each one before use. + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + SHOULD_RUN: ${{ needs.plan.outputs.should_run }} + LABEL: ${{ needs.plan.outputs.label }} + IS_RELEASE_PR: ${{ needs.plan.outputs.is_release_pr }} + PROBLEMS: ${{ needs.plan.outputs.problems }} + SUITES: ${{ needs.plan.outputs.suites }} + SOLVERS: ${{ needs.plan.outputs.solvers }} + SOLVER_MATRIX: ${{ needs.plan.outputs.solver_matrix }} + RUN_MATRIX: ${{ needs.plan.outputs.matrix }} + run: | + { + echo "pr_number=${PR_NUMBER}" + echo "head_sha=${HEAD_SHA}" + echo "base_sha=${BASE_SHA}" + echo "should_run=${SHOULD_RUN}" + echo "label=${LABEL}" + echo "is_release_pr=${IS_RELEASE_PR}" + echo "problems=${PROBLEMS}" + echo "suites=${SUITES}" + echo "solvers=${SOLVERS}" + } > handoff-meta.txt + # Matrices are JSON — keep them in their own files to avoid quoting + # hazards in the key=value metadata. + printf '%s\n' "$SOLVER_MATRIX" > solver-matrix.json + printf '%s\n' "$RUN_MATRIX" > run-matrix.json + echo "--- handoff-meta.txt ---" + cat handoff-meta.txt + + - name: Upload handoff + uses: actions/upload-artifact@v7 + with: + name: bench-handoff-${{ github.event.pull_request.number }} + path: | + handoff-meta.txt + solver-matrix.json + run-matrix.json + if-no-files-found: error + overwrite: true + retention-days: 7 diff --git a/.github/workflows/build-tesseracts.yml b/.github/workflows/build-tesseracts.yml index 1fe851fd..a88256c2 100644 --- a/.github/workflows/build-tesseracts.yml +++ b/.github/workflows/build-tesseracts.yml @@ -7,6 +7,20 @@ on: description: 'JSON array of {"domain", "solver"} objects to build' type: string required: true + ref: + description: >- + Commit SHA to check out and build. The trusted benchmark-execute + workflow passes a PR's validated HEAD so it builds the PR's solver + code from the base-repo context. Empty → the workflow's own ref. + type: string + default: "" + image_sha: + description: >- + SHA to tag the pushed image with. benchmark-execute passes the same + validated PR HEAD so the tag matches what the benchmark run pulls. + Empty → the event's PR-head/commit SHA. + type: string + default: "" push: branches: [main] workflow_dispatch: @@ -75,6 +89,11 @@ jobs: include: ${{ fromJSON(inputs.matrix || needs.discover.outputs.matrix) }} steps: - uses: actions/checkout@v6 + with: + # When invoked by benchmark-execute (workflow_run), check out the PR's + # HEAD commit explicitly rather than the default-branch ref this + # trusted run starts on, so we build the PR's solver code. + ref: ${{ inputs.ref || github.sha }} - name: Free disk space run: | @@ -91,10 +110,10 @@ jobs: run: pip install tesseract-core - name: Log in to GHCR - # PR runs need to push too so the benchmark workflow can pull - # PR-specific images by SHA. Fork PRs lack package:write on the - # default token and will skip push naturally (the push step - # tolerates auth failure; the benchmark falls back to :latest). + # Runs either on push-to-main or from the trusted benchmark-execute + # (workflow_run) context — both hold a token with package:write, so the + # push below always succeeds. (Fork `pull_request` runs no longer reach + # this workflow; they hand off to benchmark-execute instead.) uses: docker/login-action@v3 with: registry: ghcr.io @@ -130,17 +149,16 @@ jobs: tesseract --loglevel debug build --tag latest "${SOLVER_DIR}" - name: Push to GHCR - # Always push : (PR or push) so the benchmark workflow can pull - # the exact image built for this commit. Only push :latest on - # push-to-main so PR builds don't bump main's latest pointer. - # Tolerate auth failure (fork PRs lack package:write); the benchmark - # then falls back to :latest. - continue-on-error: ${{ github.event_name == 'pull_request' }} + # Always push : so the benchmark run can pull the exact image + # built for this commit. Only push :latest on push-to-main so PR + # builds don't bump main's latest pointer. No auth-failure tolerance: + # both callers (push-to-main, trusted benchmark-execute) hold + # package:write, so a push failure is a real error worth failing on. env: - # On `pull_request`, ``github.sha`` is the GH-generated merge - # commit, not the PR HEAD. The benchmark workflow polls for the - # PR HEAD's SHA, so push under that to keep the two in sync. - IMAGE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + # benchmark-execute passes the validated PR HEAD via image_sha so the + # tag matches what the benchmark run pulls; push-to-main falls back to + # the commit SHA. + IMAGE_SHA: ${{ inputs.image_sha || github.event.pull_request.head.sha || github.sha }} run: | IMAGE_NAME="${{ steps.image.outputs.image_name }}" GHCR_BASE="${{ steps.image.outputs.ghcr_base }}" diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml deleted file mode 100644 index 1e80dd04..00000000 --- a/.github/workflows/docs-publish.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Publish docs preview - -# Second half of the fork-safe docs-preview flow. The Benchmark workflow's -# `report` job runs in the PR head's context (fork PRs included) and does the -# secret-free work: render the site and upload it as an artifact. It cannot -# trigger the RTD rebuild, resolve the preview URL, or post the PR comment, -# because GitHub withholds repo secrets from pull_request runs on fork PRs. -# -# This workflow fills that gap. `workflow_run` always runs in the BASE repo's -# context using the workflow definition from the default branch — trusted code -# with full secret access — regardless of whether the triggering run came from -# a fork. It downloads the artifacts the report job produced and performs the -# privileged steps. Because it never checks out or executes PR-authored code, -# exposing secrets here is safe. - -on: - workflow_run: - workflows: ["Benchmark"] - types: [completed] - -permissions: - contents: read - actions: read # download artifacts from the triggering run - pull-requests: write # post the status comment - -jobs: - publish: - name: Publish RTD preview & comment - runs-on: ubuntu-latest - # Only for PR-triggered Benchmark runs that succeeded. A failed report job - # means there's nothing worth publishing (and the merge gate already - # surfaces that failure on the PR). - if: >- - github.event.workflow_run.event == 'pull_request' - && github.event.workflow_run.conclusion == 'success' - steps: - - uses: actions/checkout@v6 - - # ── Recover the handoff from the triggering run ──────────────────── - # download-artifact with an explicit run-id reaches into the upstream - # run (a different workflow), which the default same-run lookup can't do. - - name: Download publish handoff - id: dl_meta - uses: actions/download-artifact@v8 - continue-on-error: true - with: - pattern: docs-publish-meta-* - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - path: handoff/ - merge-multiple: true - - # No handoff artifact → the report job didn't reach the upload (e.g. a - # non-PR dispatch, or a render failure). Nothing to publish; exit clean. - - name: Parse handoff - id: meta - run: | - META="handoff/publish-meta.txt" - if [[ ! -f "$META" ]]; then - echo "No handoff metadata — nothing to publish." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - PR_NUMBER=$(grep -oP '^pr_number=\K.*' "$META" || true) - SHOULD_RUN=$(grep -oP '^should_run=\K.*' "$META" || true) - # Guard: the PR number originates from fork-controlled workflow - # output, so validate it's purely numeric before using it in API - # calls / URLs. Anything else → bail rather than act on it. - if [[ ! "$PR_NUMBER" =~ ^[0-9]+$ ]]; then - echo "::warning::Handoff PR number '${PR_NUMBER}' is not numeric — skipping." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "found=true" >> "$GITHUB_OUTPUT" - echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" - echo "should_run=${SHOULD_RUN}" >> "$GITHUB_OUTPUT" - echo "Publishing for PR #${PR_NUMBER} (should_run=${SHOULD_RUN})" - - # ── Trigger RTD PR preview rebuild ───────────────────────────────── - # RTD's webhook already built a preview on PR push, but that build ran - # before this run rendered the results pages. Trigger a fresh build so - # the preview reflects the merged PR+baseline result tree. - - name: Trigger RTD PR preview rebuild - if: steps.meta.outputs.found == 'true' - continue-on-error: true - env: - RTD_TOKEN: ${{ secrets.RTD_TOKEN }} - RTD_PROJECT: ${{ vars.RTD_PROJECT }} - run: bash .github/scripts/rtd-trigger.sh "${{ steps.meta.outputs.pr_number }}" - - # ── Resolve RTD preview URL ──────────────────────────────────────── - - name: Resolve RTD preview URL - id: rtd_url - if: steps.meta.outputs.found == 'true' - continue-on-error: true - env: - RTD_TOKEN: ${{ secrets.RTD_TOKEN }} - RTD_PROJECT: ${{ vars.RTD_PROJECT }} - run: bash .github/scripts/rtd-preview-url.sh "${{ steps.meta.outputs.pr_number }}" - - # ── Post PR comment ──────────────────────────────────────────────── - # Include the status report if the report job shipped one (benchmarks - # ran); otherwise the comment is just the preview banner. - - name: Post status comment on PR - if: steps.meta.outputs.found == 'true' - env: - GH_TOKEN: ${{ secrets.PL_PASTEURBOT_PAT_PUBLIC }} - DOCS_PREVIEW_URL: ${{ steps.rtd_url.outputs.preview_url }} - run: | - REPORT="" - if [[ -f handoff/_site-status-report.md ]]; then - REPORT="handoff/_site-status-report.md" - fi - .github/scripts/post-status-comment.sh \ - "${{ steps.meta.outputs.pr_number }}" \ - "$REPORT"