[CI] Cross-platform — Part 2: ARM/Spark workflow

.github/actions/_lib/compute-deps-hash/action.yml

@@ -0,0 +1,75 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'Compute deps hash'
+description: >
+  Compute the deps-cache hash for the Isaac Lab Docker build. Shared by the
+  docker-build (local-store check) and ecr-build-push-pull (registry check)
+  actions so a local hit and a registry hit always agree on the same
+  `deps-<hash>` tag. Hashes the install-relevant files, resolved base image
+  digest, and target platform.
+
+inputs:
+  dockerfile-path:
+    description: 'Path to Dockerfile'
+    required: true
+  isaacsim-base-image:
+    description: 'IsaacSim base image'
+    required: true
+  isaacsim-version:
+    description: 'IsaacSim version'
+    required: true
+  platform:
+    description: 'Target platform included in the dependency-cache identity'
+    default: 'linux/amd64'
+    required: false
+
+outputs:
+  hash:
+    description: '16-char deps-cache hash'
+    value: ${{ steps.compute.outputs.hash }}
+
+runs:
+  using: composite
+  steps:
+    - id: compute
+      shell: bash
+      env:
+        DOCKERFILE_PATH: ${{ inputs.dockerfile-path }}
+        ISAACSIM_BASE_IMAGE: ${{ inputs.isaacsim-base-image }}
+        ISAACSIM_VERSION: ${{ inputs.isaacsim-version }}
+        TARGET_PLATFORM: ${{ inputs.platform }}
+      run: |
+        set -euo pipefail
+
+        # Exact files/dirs whose full content is hashed. The Dockerfile is first.
+        deps_files=(
+          "${DOCKERFILE_PATH}"
+          isaaclab.sh
+          environment.yml
+          source/isaaclab/isaaclab/cli
+        )
+        deps_manifest_pattern='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$'
+
+        # Resolve the actual base image digest so a new push of a mutable tag
+        # (e.g. latest-develop) invalidates the deps cache automatically.
+        base_image_digest=$(docker buildx imagetools inspect \
+          "${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}" \
+          --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true)
+        if [ -n "${base_image_digest}" ]; then
+          base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}:${base_image_digest}"
+        else
+          echo "🟠 Could not resolve base image digest, falling back to tag string"
+          base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}"
+        fi
+
+        mapfile -t manifest_files < <(git ls-files | grep -E "${deps_manifest_pattern}" || true)
+        file_hash=$(git ls-files -s "${deps_files[@]}" "${manifest_files[@]}" 2>/dev/null \
+          | sha256sum | cut -c1-16)
+        deps_hash=$(printf '%s %s %s' "${file_hash}" "${base_image_uniq_id}" "${TARGET_PLATFORM}" \
+          | sha256sum | cut -c1-16)
+
+        echo "🔵 Deps hash: ${deps_hash}"
+        echo "hash=${deps_hash}" >> "$GITHUB_OUTPUT"

.github/actions/_lib/setup-docker-config/action.yml

@@ -0,0 +1,43 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'Setup docker config'
+description: >
+  Point DOCKER_CONFIG at a temp config with the credential helper disabled and
+  log into nvcr.io. Shared by the docker-build and ecr-build-push-pull actions.
+  Idempotent: re-invoking it in the same job is a no-op, so callers (e.g.
+  ecr-build-push-pull delegating to docker-build) don't need to coordinate.
+  Reads NGC_API_KEY from the environment (optional; warns when missing).
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        # The runner's credential helper backend is broken ("not implemented")
+        # and causes docker login calls to fail unless we point DOCKER_CONFIG at
+        # a temp config with credsStore disabled. The value is written to
+        # $GITHUB_ENV so subsequent steps in the job inherit it; a second
+        # invocation sees it already set and short-circuits.
+        if [ -n "${DOCKER_CONFIG:-}" ] && [ -f "${DOCKER_CONFIG}/config.json" ]; then
+          echo "🟢 Docker config already set up at ${DOCKER_CONFIG}, skipping"
+          exit 0
+        fi
+
+        DOCKER_CONFIG_DIR=$(mktemp -d)
+        if [ -f "${HOME}/.docker/config.json" ]; then
+          python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))"
+        else
+          echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
+        fi
+        export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
+        echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV"
+
+        if [ -n "${NGC_API_KEY:-}" ]; then
+          echo "🔵 Logging into nvcr.io..."
+          docker login -u '$oauthtoken' -p "${NGC_API_KEY}" nvcr.io
+        else
+          echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)"
+        fi

.github/actions/docker-build/action.yml

@@ -24,61 +24,222 @@ inputs:
     description: 'Build context path'
     default: '.'
     required: false
+  platform:
+    description: 'Target platform for `docker buildx build --platform`.'
+    default: 'linux/amd64'
+    required: false
+  cache-from:
+    description: >
+      Optional value for `docker buildx build --cache-from`. Typically a
+      `type=registry,ref=<image>` for cross-host layer cache. Leave empty for
+      pure local-only builds.
+    default: ''
+    required: false
+  cache-to:
+    description: >
+      Optional value for `docker buildx build --cache-to`. Pairs with
+      `cache-from` for registry-backed layer cache writes.
+    default: ''
+    required: false
+  deps-hash:
+    description: >
+      Pre-computed deps-hash to use for the local deps-tag check. When empty,
+      this action computes the hash itself via the `_lib/compute-deps-hash`
+      action. Set by callers (e.g. `ecr-build-push-pull`) that already compute
+      the hash for a registry-side check, to avoid recomputing here.
+    default: ''
+    required: false
+  evict-stale-cache:
+    description: >
+      When 'true', evict `isaac-lab*:deps-*` tags older than 14 days at the
+      end of the build to bound disk growth on long-lived self-hosted
+      runners. Default 'false' — no implicit cleanup.
+    default: 'false'
+    required: false
 
 runs:
   using: composite
   steps:
-    - name: NGC Login
-      shell: sh
+
+    ##### 1: Setup docker config + login to nvcr.io (optional) #####
+
+    - name: Setup docker config and login to nvcr.io
+      uses: ./.github/actions/_lib/setup-docker-config
+
+    ##### 2: Host disk snapshot (pre) #####
+
+    - name: Host disk snapshot (pre)
+      shell: bash
+      run: |
+        set +e
+        docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker")
+        deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l)
+        commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l)
+        {
+          echo "## Disk snapshot (pre)"
+          echo '```'
+          echo "Filesystem:"
+          df -h / "${docker_root}" 2>/dev/null | sort -u
+          echo
+          echo "docker system df:"
+          docker system df
+          echo
+          echo "Tag counts:"
+          echo "  isaac-lab* (commit + deps tags): ${commit_count}"
+          echo "  isaac-lab*:deps-* (deps cache) :  ${deps_count}"
+          echo
+          echo "Deps tags (newest first):"
+          docker images --filter 'reference=isaac-lab*:deps-*' \
+            --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \
+            | head -20
+          echo '```'
+        } | tee -a "$GITHUB_STEP_SUMMARY"
+
+    ##### 3: Local exact-tag short-circuit #####
+
+    - name: Check image locally
+      id: local
+      shell: bash
+      run: |
+        if docker image inspect "${{ inputs.image-tag }}" >/dev/null 2>&1; then
+          echo "🟢 Image already in local docker store: ${{ inputs.image-tag }}"
+          echo "hit=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "🔵 Image not present locally, will check deps-cache / build"
+        fi
+
+    ##### 4: Local deps-tag short-circuit #####
+
+    - name: Compute deps hash
+      id: deps-hash
+      if: steps.local.outputs.hit != 'true' && inputs.deps-hash == ''
+      uses: ./.github/actions/_lib/compute-deps-hash
+      with:
+        dockerfile-path: ${{ inputs.dockerfile-path }}
+        isaacsim-base-image: ${{ inputs.isaacsim-base-image }}
+        isaacsim-version: ${{ inputs.isaacsim-version }}
+        platform: ${{ inputs.platform }}
+
+    - name: Check deps-tag locally
+      id: local-deps
+      if: steps.local.outputs.hit != 'true'
+      shell: bash
       run: |
-        # Only attempt NGC login if API key is available
-        if [ -n "${{ env.NGC_API_KEY }}" ]; then
-          echo "Logging into NGC registry..."
-          docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io
-          echo "✅ Successfully logged into NGC registry"
+        DEPS_HASH="${{ inputs.deps-hash || steps.deps-hash.outputs.hash }}"
+        LOCAL_DEPS_TAG="$(echo "${{ inputs.image-tag }}" | cut -d: -f1):deps-${DEPS_HASH}"
+
+        echo "🔵 Local deps tag: ${LOCAL_DEPS_TAG}"
+        echo "LOCAL_DEPS_TAG=${LOCAL_DEPS_TAG}" >> "$GITHUB_ENV"
+
+        if docker image inspect "${LOCAL_DEPS_TAG}" >/dev/null 2>&1; then
+          echo "🟢 Local deps-cache HIT! Retagging as ${{ inputs.image-tag }}"
+          docker tag "${LOCAL_DEPS_TAG}" "${{ inputs.image-tag }}"
+          echo "hit=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "🟠 Local deps-cache MISS (will build then tag for future hits)"
+        fi
+
+    ##### 5: Full build #####
+
+    - name: Build image
+      id: build
+      if: >
+        steps.local.outputs.hit != 'true' &&
+        steps.local-deps.outputs.hit != 'true'
+      shell: bash
+      run: |
+        BUILD_ARGS=(
+          --progress=plain
+          --platform "${{ inputs.platform }}"
+          -f "${{ inputs.dockerfile-path }}"
+          --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}"
+          --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}"
+          --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim"
+          --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab"
+          --build-arg "DOCKER_USER_HOME_ARG=/root"
+          -t "${{ inputs.image-tag }}"
+        )
+        if [ -n "${{ inputs.cache-from }}" ]; then
+          BUILD_ARGS+=( --cache-from "${{ inputs.cache-from }}" )
+        fi
+        if [ -n "${{ inputs.cache-to }}" ]; then
+          BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" )
+        fi
+
+        BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}"
+        docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \
+          || docker buildx use "${BUILDER_NAME}"
+        trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT
+
+        echo "🔵 Building ${{ inputs.image-tag }}..."
+        docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}"
+        echo "was-built=true" >> "$GITHUB_OUTPUT"
+
+    ##### 6: Tag built image with local deps-tag #####
+
+    # Runs only when a real build happened (not on cache hits). Populates the
+    # deps-tag so the next build with identical deps short-circuits at step 4.
+
+    - name: Tag built image with local deps-tag
+      if: steps.build.outputs.was-built == 'true'
+      shell: bash
+      run: |
+        if [ -n "${LOCAL_DEPS_TAG:-}" ]; then
+          docker tag "${{ inputs.image-tag }}" "${LOCAL_DEPS_TAG}"
+          echo "🟢 Tagged local deps-cache: ${LOCAL_DEPS_TAG}"
         else
-          echo "⚠️ NGC_API_KEY not available - skipping NGC login"
-          echo "This is normal for PRs from forks or when secrets are not configured"
+          echo "🟠 LOCAL_DEPS_TAG not set, skipping local deps-cache tag"
         fi
 
-    - name: Build Docker Image
-      shell: sh
+    ##### 7: Evict stale local deps-cache tags (>14d) — opt-in #####
+
+    - name: Evict stale local deps-cache tags (>14d)
+      if: always() && inputs.evict-stale-cache == 'true'
+      shell: bash
       run: |
-        # Function to build Docker image
-        build_docker_image() {
-          local image_tag="$1"
-          local isaacsim_base_image="$2"
-          local isaacsim_version="$3"
-          local dockerfile_path="$4"
-          local context_path="$5"
-
-          # Skip build if image already exists locally (e.g. built by a prior job on the same runner)
-          if docker image inspect "$image_tag" > /dev/null 2>&1; then
-            echo "Image $image_tag already exists locally, skipping build."
-            return 0
+        set +e
+        TTL_DAYS=14
+        cutoff=$(date -u -d "${TTL_DAYS} days ago" +%s)
+        evicted=0
+        while IFS='|' read -r created tag; do
+          [ -z "$tag" ] && continue
+          created_epoch=$(date -d "$created" +%s 2>/dev/null) || continue
+          if [ "$created_epoch" -lt "$cutoff" ]; then
+            days_old=$(( (cutoff - created_epoch) / 86400 + TTL_DAYS ))
+            echo "🟠 Evicting deps tag (~${days_old}d old): ${tag}"
+            docker rmi -f "$tag" >/dev/null 2>&1 || true
+            evicted=$(( evicted + 1 ))
           fi
+        done < <(docker images --filter 'reference=isaac-lab*:deps-*' \
+          --format '{{.CreatedAt}}|{{.Repository}}:{{.Tag}}' 2>/dev/null)
+        echo "🔵 Evicted ${evicted} deps tag(s) older than ${TTL_DAYS}d"
 
-          echo "Building Docker image: $image_tag"
-          echo "Using Dockerfile: $dockerfile_path"
-          echo "Build context: $context_path"
-
-          # Build Docker image
-          docker buildx build --progress=plain --platform linux/amd64 \
-            -t $image_tag \
-            --build-arg ISAACSIM_BASE_IMAGE_ARG="$isaacsim_base_image" \
-            --build-arg ISAACSIM_VERSION_ARG="$isaacsim_version" \
-            --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \
-            --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \
-            --build-arg DOCKER_USER_HOME_ARG=/root \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
-            -f $dockerfile_path \
-            --load $context_path
-
-          echo "✅ Docker image built successfully: $image_tag"
-          echo "Current local Docker images:"
-          docker images
-        }
-
-        # Call the function with provided parameters
-        build_docker_image "${{ inputs.image-tag }}" "${{ inputs.isaacsim-base-image }}" "${{ inputs.isaacsim-version }}" "${{ inputs.dockerfile-path }}" "${{ inputs.context-path }}"
+    ##### 8: Host disk snapshot (post) #####
+
+    - name: Host disk snapshot (post)
+      if: always()
+      shell: bash
+      run: |
+        set +e
+        docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker")
+        deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l)
+        commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l)
+        {
+          echo "## Disk snapshot (post)"
+          echo '```'
+          echo "Filesystem:"
+          df -h / "${docker_root}" 2>/dev/null | sort -u
+          echo
+          echo "docker system df:"
+          docker system df
+          echo
+          echo "Tag counts:"
+          echo "  isaac-lab* (commit + deps tags): ${commit_count}"
+          echo "  isaac-lab*:deps-* (deps cache) :  ${deps_count}"
+          echo
+          echo "Deps tags (newest first):"
+          docker images --filter 'reference=isaac-lab*:deps-*' \
+            --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \
+            | head -20
+          echo '```'
+        } | tee -a "$GITHUB_STEP_SUMMARY"