diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 00000000000..cdedd4ffa8a --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,138 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Detect Changes' +description: >- + Decide whether a caller's gated jobs should run for the triggering event by + matching the pull request's changed files against caller-supplied ERE + patterns. Shared by build.yaml and arm-ci.yml. Non-PR events (and a file + listing failure) fail open with should_run=true. + Callers gate their jobs with `if: steps/needs ... should_run == 'true'` rather + than a workflow-level `paths:` filter, so a job that is (or may become) a + required status check still gets created and reports green when skipped — a + not-triggered required check would otherwise stay pending forever. + +inputs: + patterns: + description: >- + Newline-delimited entries of the form ` :: ` + (description optional). If any regex matches a changed file path, + should_run is true. + required: true + triggered-jobs-push: + description: "Human-readable list of jobs triggered on push events (step-summary text only)." + required: false + default: "(post-merge integration jobs)" + triggered-jobs-pr: + description: "Human-readable list of jobs triggered on pull_request events (step-summary text only)." + required: false + default: "(gated jobs)" + +outputs: + should_run: + description: "Whether the caller's gated jobs should run for this event." + value: ${{ steps.detect.outputs.should_run }} + +runs: + using: composite + steps: + - id: detect + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_NAME: ${{ github.event_name }} + REPO: ${{ github.repository }} + PATTERNS: ${{ inputs.patterns }} + TRIGGERED_JOBS_PUSH: ${{ inputs.triggered-jobs-push }} + TRIGGERED_JOBS_PR: ${{ inputs.triggered-jobs-pr }} + run: | + set -euo pipefail + + # Parse " :: " lines into parallel arrays (blank + # lines skipped; the description is optional). + regexes=(); descs=() + while IFS= read -r line; do + [ -z "${line//[[:space:]]/}" ] && continue + regexes+=("${line%% :: *}") + if [[ "$line" == *" :: "* ]]; then descs+=("${line#* :: }"); else descs+=(""); fi + done < <(printf '%s\n' "$PATTERNS") + + if [ "$EVENT_NAME" = "push" ]; then + triggered_jobs="$TRIGGERED_JOBS_PUSH" + else + triggered_jobs="$TRIGGERED_JOBS_PR" + fi + + any_match() { + local files="$1" regex + for regex in "${regexes[@]}"; do + if grep -qE -- "$regex" <<< "$files"; then + return 0 + fi + done + return 1 + } + + render_table() { + local files="$1" i regex desc count sample shown + echo "| Pattern | What it covers | Matched files |" + echo "|---|---|---|" + for i in "${!regexes[@]}"; do + regex="${regexes[$i]}"; desc="${descs[$i]:-}" + # escape | so it doesn't end the markdown table cell mid-regex + shown="${regex//|/\\|}" + count=$(grep -cE -- "$regex" <<< "$files" || true) + if [ "$count" -gt 0 ]; then + # paste -sd cycles delimiter chars, so join on ',' then space it out. + sample=$(grep -m 3 -E -- "$regex" <<< "$files" | paste -sd , - | sed 's/,/, /g') + [ "$count" -gt 3 ] && sample="$sample (and $((count - 3)) more)" + echo "| \`$shown\` | ${desc:--} | $sample |" + else + echo "| \`$shown\` | ${desc:--} | - |" + fi + done + } + + decide() { + local decision="$1" reason="$2" files="${3:-}" + echo "Decision: should_run=$decision ($reason)" + echo "should_run=$decision" >> "$GITHUB_OUTPUT" + { + echo "## Change detection" + echo "" + if [ "$decision" = "true" ]; then + echo "Gated jobs will **run**: $reason." + else + echo "Gated jobs will be **skipped**: $reason." + fi + echo "" + echo "Triggered jobs: $triggered_jobs." + if [ -n "$files" ]; then + echo "" + render_table "$files" + fi + } >> "$GITHUB_STEP_SUMMARY" + } + + if [ "$EVENT_NAME" != "pull_request" ]; then + decide true "non-PR event ($EVENT_NAME)" + exit 0 + fi + + if ! changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then + # Fail-safe: a transient API error must not block merge. Default to running. + echo "::warning::Could not list changed files; defaulting to running tests" + decide true "fail-safe (could not list changed files)" + exit 0 + fi + + printf '%s\n' "$changed_files" + + if any_match "$changed_files"; then + decide true "relevant paths changed" "$changed_files" + else + decide false "no relevant paths changed" "$changed_files" + fi diff --git a/.github/workflows/arm-ci.yml b/.github/workflows/arm-ci.yml new file mode 100644 index 00000000000..38728f254f1 --- /dev/null +++ b/.github/workflows/arm-ci.yml @@ -0,0 +1,211 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# region help +# aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners. +# +# Split out of build.yaml (Docker + Tests) so this slow arm64 build+render owns +# its own workflow run. In build.yaml it shared the run with every test-* job, +# so the whole run stayed open until arm-ci finished (up to its 60-minute +# timeout) — and GitHub only exposes "Re-run failed jobs" once the entire run +# completes, forcing a fast test job that failed early to wait on arm-ci before +# it could be retried. As its own workflow, arm-ci no longer gates the main +# workflow's lifecycle, and it can be re-run independently. +# +# arm-ci is currently informational (continue-on-error), but it is wired to be +# promotable to a REQUIRED status check: it triggers on every PR and gates the +# expensive arm64 build behind the shared `changes` detector + `if:`. An +# irrelevant PR SKIPS the job (which branch protection reads as green) instead +# of never creating the check — a workflow-level `paths:` filter is deliberately +# avoided because a not-triggered required check stays pending forever and +# blocks the PR. This mirrors the `changes`+`if:` pattern build.yaml already +# uses for its required test jobs. +# +# Trigger parity with the former build.yaml job: +# - push to protected branches always runs (post-merge integration); +# - pull_request runs only when arm-relevant paths change (via `changes`+`if:`). +# endregion + +name: ARM CI + +on: + push: + branches: + - main + - develop + - 'release/**' + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - develop + - 'release/**' + workflow_dispatch: + +# Concurrency control to prevent parallel runs on the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + pull-requests: read + +env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + +jobs: + # Shared change-detection gate. The trigger is unfiltered (see header) so the + # arm-ci check is always creatable; this decides whether the expensive jobs + # actually run. Lists arm-ci's real inputs — including the root files + # Dockerfile.base copies (pyproject.toml, environment.yml, isaaclab.*) that a + # path filter would otherwise miss. + changes: + name: Detect Changes + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.detect.outputs.should_run }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: .github/actions/detect-changes + sparse-checkout-cone-mode: false + - id: detect + uses: ./.github/actions/detect-changes + with: + triggered-jobs-push: arm-ci + triggered-jobs-pr: arm-ci + patterns: | + ^source/ :: Library source code + ^docker/ :: Container build inputs + ^tools/ :: Build tooling + ^apps/ :: Standalone apps + ^scripts/ :: Standalone scripts + ^\.dockerignore$ :: Docker build context filter + ^pyproject\.toml$ :: Root Python project (OV pins + Docker copy) + ^environment\.yml$ :: Conda env (Docker copy) + ^isaaclab\. :: isaaclab.sh/.bat entrypoints (Docker copy) + ^\.github/workflows/arm-ci\.yml$ :: This workflow file + ^\.github/workflows/config\.yaml$ :: Base image config + ^\.github/actions/ :: CI actions + + # Loads the Isaac Sim base image name/tag from config.yaml and computes the + # per-run CI image tag. Inlined here (rather than shared with build.yaml) + # because arm-ci builds its own independent -arm64 image and only needs three + # of the values build.yaml's config job produces. + config: + name: Load Config + needs: [changes] + if: needs.changes.outputs.should_run == 'true' + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + ci_image_tag: ${{ steps.image_tag.outputs.ci_image_tag }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + # NOTE: this ci_image_tag formula is intentionally duplicated in build.yaml's + # config job — the two images are independent (-arm64 here vs x86 there), so + # they are not shared. Keep the two formulas in sync if either changes. + - id: image_tag + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REF_NAME: ${{ github.ref_name }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "pull_request" ]; then + ref_component="pr-${PR_NUMBER}" + else + ref_component="$REF_NAME" + fi + + # Sanitize the ref name for use as a Docker tag suffix. + sanitized_ref=$(echo "$ref_component" | sed 's/[^a-zA-Z0-9._-]/-/g') + echo "ci_image_tag=isaac-lab-ci:${sanitized_ref}-${SHA}" >> "$GITHUB_OUTPUT" + echo "CI image tag: isaac-lab-ci:${sanitized_ref}-${SHA}" + + # aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners. + # Build and test must share one runner because ECR is not wired for arm64 — + # the locally-built image cannot be handed off across machines. + arm-ci: + name: Build & Test + runs-on: [self-hosted, arm64] + needs: [changes, config] + if: needs.changes.outputs.should_run == 'true' + timeout-minutes: 60 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + - name: Build base image (linux/arm64) + uses: ./.github/actions/docker-build + with: + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + platform: linux/arm64 + # The Spark runner is long-lived self-hosted with no ECR, so its local + # deps-cache tags accumulate; evict ones older than 14 days each run. + evict-stale-cache: "true" + + - name: Resolve OV runtime pins from pyproject + uses: ./.github/actions/resolve-ov-pins + id: ov_pins + + - name: Run arm_ci marker tests + uses: ./.github/actions/run-tests + with: + test-path: tools + result-file: arm-ci-report.xml + container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + # ovphysx stays pinned to 0.4.13 here (see the aarch64 note below). + extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ovphysx==0.4.13" + # ovphysx-backed test params require ovphysx >= 0.5.1, which has no aarch64 + # wheel yet; keep the newton/ovrtx rendering coverage, mirroring the public + # pip-index fallback of rendering-correctness-kitless. Running them anyway + # exhausts ovrtx SyncScopeIds (>15 renderer creations in one process). + test-k-expr: not ovphysx + ci-marker: arm_ci + volume-mount-source: ${{ github.workspace }} + + - name: Run shared Cartpole smoke + uses: ./.github/actions/run-tests + with: + test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py + result-file: arm-ci-cartpole-smoke-report.xml + container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + volume-mount-source: ${{ github.workspace }} + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: arm-ci-reports + path: reports/ + retention-days: 7 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 01df1aec196..9fc053248e6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -78,111 +78,37 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read outputs: - run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} + should_run: ${{ steps.detect.outputs.should_run }} steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + sparse-checkout: .github/actions/detect-changes + sparse-checkout-cone-mode: false - id: detect - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - EVENT_NAME: ${{ github.event_name }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - - # Docker test jobs run only when paths in the patterns table change. - # Otherwise they skip via `if:` and report green to branch protection, - # which is why we don't use a workflow-level `paths:` filter (a - # not-triggered required check would block the PR forever). - # config.yaml is included because it controls the base image names and - # tags consumed by the Docker build jobs. - patterns=( - $'^source/\tLibrary source code' - $'^docker/\tContainer build inputs' - $'^tools/\tBuild tooling' - $'^apps/\tStandalone apps' - $'^scripts/\tStandalone scripts' - $'^\\.github/workflows/build\\.yaml$\tThis workflow file' - $'^\\.github/workflows/config\\.yaml$\tBase image config' - $'^\\.github/actions/\tCI actions' - $'^\\.github/test-subsets/\tCI test subset config' - ) - if [ "$EVENT_NAME" = "push" ]; then - triggered_jobs="Docker base build job + rendering-correctness + rendering-correctness-kitless" - else - triggered_jobs="Docker build jobs + all test-* matrix jobs" - fi - - render_table() { - local files="$1" entry regex desc count sample shown - echo "| Pattern | What it covers | Matched files |" - echo "|---|---|---|" - for entry in "${patterns[@]}"; do - IFS=$'\t' read -r regex desc <<< "$entry" - # escape | so it doesn't end the markdown table cell mid-regex - shown="${regex//|/\\|}" - count=$(grep -cE "$regex" <<< "$files" || true) - if [ "$count" -gt 0 ]; then - sample=$(grep -m 3 -E "$regex" <<< "$files" | paste -sd ', ' -) - [ "$count" -gt 3 ] && sample="$sample (and $((count - 3)) more)" - echo "| \`$shown\` | $desc | $sample |" - else - echo "| \`$shown\` | $desc | - |" - fi - done - } - - any_match() { - local files="$1" entry regex - for entry in "${patterns[@]}"; do - IFS=$'\t' read -r regex _ <<< "$entry" - if grep -qE "$regex" <<< "$files"; then - return 0 - fi - done - return 1 - } - - decide() { - local decision="$1" reason="$2" files="${3:-}" - echo "Decision: run_docker_tests=$decision ($reason)" - echo "run_docker_tests=$decision" >> "$GITHUB_OUTPUT" - { - echo "## Docker test gating" - echo "" - if [ "$decision" = "true" ]; then - echo "Docker tests will **run**: $reason." - else - echo "Docker tests will be **skipped**: $reason." - fi - echo "" - echo "Triggered jobs: $triggered_jobs." - if [ -n "$files" ]; then - echo "" - render_table "$files" - fi - } >> "$GITHUB_STEP_SUMMARY" - } - - if [ "$EVENT_NAME" != "pull_request" ]; then - decide true "non-PR event ($EVENT_NAME)" - exit 0 - fi - - if ! changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then - # Fail-safe: a transient API error must not block merge. Default to running. - echo "::warning::Could not list changed files; defaulting to running tests" - decide true "fail-safe (could not list changed files)" - exit 0 - fi - - printf '%s\n' "$changed_files" - - if any_match "$changed_files"; then - decide true "relevant paths changed" "$changed_files" - else - decide false "no relevant paths changed" "$changed_files" - fi + uses: ./.github/actions/detect-changes + with: + triggered-jobs-push: "Docker base build job + rendering-correctness + rendering-correctness-kitless" + triggered-jobs-pr: "Docker build jobs + all test-* matrix jobs" + # config.yaml is included because it controls the base image names and tags + # consumed by the Docker build jobs. + patterns: | + ^source/ :: Library source code + ^docker/ :: Container build inputs + ^tools/ :: Build tooling + ^apps/ :: Standalone apps + ^scripts/ :: Standalone scripts + ^\.dockerignore$ :: Docker build context filter + ^pyproject\.toml$ :: Root Python project (Docker copy) + ^environment\.yml$ :: Conda env (Docker copy) + ^isaaclab\. :: isaaclab.sh/.bat entrypoints (Docker copy) + ^\.github/workflows/build\.yaml$ :: This workflow file + ^\.github/actions/ :: CI actions + ^\.github/test-subsets/ :: CI test subset config config: name: Load Config @@ -207,6 +133,9 @@ jobs: echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" echo "isaaclab_image_name=$(yq -r .isaaclab_image_name "$f")" >> "$GITHUB_OUTPUT" echo "ovphysx_wheelhouse_resource=$(yq -r .ovphysx_wheelhouse_resource "$f")" >> "$GITHUB_OUTPUT" + # NOTE: this ci_image_tag formula is intentionally duplicated in arm-ci.yml's + # config job — the two images are independent (x86 here vs -arm64 there), so + # they are not shared. Keep the two formulas in sync if either changes. - id: image_tag shell: bash env: @@ -233,7 +162,7 @@ jobs: name: Build Base Docker Image runs-on: [self-hosted, gpu] needs: [changes, config] - if: needs.changes.outputs.run_docker_tests == 'true' + if: needs.changes.outputs.should_run == 'true' steps: - name: Checkout Code uses: actions/checkout@v6 @@ -256,7 +185,7 @@ jobs: needs: [changes, config] if: >- github.event_name != 'push' && - needs.changes.outputs.run_docker_tests == 'true' + needs.changes.outputs.should_run == 'true' steps: - name: Checkout Code uses: actions/checkout@v6 @@ -273,72 +202,6 @@ jobs: dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo - # aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners. - # Build and test must share one runner because ECR is not wired for arm64 — - # the locally-built image cannot be handed off across machines. - arm-ci: - name: arm-ci - runs-on: [self-hosted, arm64] - needs: [changes, config] - if: needs.changes.outputs.run_docker_tests == 'true' - timeout-minutes: 60 - continue-on-error: true - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - lfs: true - - - name: Build base image (linux/arm64) - uses: ./.github/actions/docker-build - with: - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} - isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} - dockerfile-path: docker/Dockerfile.base - platform: linux/arm64 - # The Spark runner is long-lived self-hosted with no ECR, so its local - # deps-cache tags accumulate; evict ones older than 14 days each run. - evict-stale-cache: "true" - - - name: Resolve OV runtime pins from pyproject - uses: ./.github/actions/resolve-ov-pins - id: ov_pins - - - name: Run arm_ci marker tests - uses: ./.github/actions/run-tests - with: - test-path: tools - result-file: arm-ci-report.xml - container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - # ovphysx stays pinned to 0.4.13 here (see the aarch64 note below). - extra-pip-packages: "${{ steps.ov_pins.outputs.ovrtx }} ovphysx==0.4.13" - # ovphysx-backed test params require ovphysx >= 0.5.1, which has no aarch64 - # wheel yet; keep the newton/ovrtx rendering coverage, mirroring the public - # pip-index fallback of rendering-correctness-kitless. Running them anyway - # exhausts ovrtx SyncScopeIds (>15 renderer creations in one process). - test-k-expr: not ovphysx - ci-marker: arm_ci - volume-mount-source: ${{ github.workspace }} - - - name: Run shared Cartpole smoke - uses: ./.github/actions/run-tests - with: - test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py - result-file: arm-ci-cartpole-smoke-report.xml - container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }} - image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 - volume-mount-source: ${{ github.workspace }} - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v7 - with: - name: arm-ci-reports - path: reports/ - retention-days: 7 - #endregion #region test jobs