diff --git a/.github/actions/_lib/compute-deps-hash/action.yml b/.github/actions/_lib/compute-deps-hash/action.yml new file mode 100644 index 000000000000..218339cbac96 --- /dev/null +++ b/.github/actions/_lib/compute-deps-hash/action.yml @@ -0,0 +1,75 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Compute deps hash' +description: > + Compute the deps-cache hash for the Isaac Lab Docker build. Shared by the + docker-build (local-store check) and ecr-build-push-pull (registry check) + actions so a local hit and a registry hit always agree on the same + `deps-` tag. Hashes the install-relevant files, resolved base image + digest, and target platform. + +inputs: + dockerfile-path: + description: 'Path to Dockerfile' + required: true + isaacsim-base-image: + description: 'IsaacSim base image' + required: true + isaacsim-version: + description: 'IsaacSim version' + required: true + platform: + description: 'Target platform included in the dependency-cache identity' + default: 'linux/amd64' + required: false + +outputs: + hash: + description: '16-char deps-cache hash' + value: ${{ steps.compute.outputs.hash }} + +runs: + using: composite + steps: + - id: compute + shell: bash + env: + DOCKERFILE_PATH: ${{ inputs.dockerfile-path }} + ISAACSIM_BASE_IMAGE: ${{ inputs.isaacsim-base-image }} + ISAACSIM_VERSION: ${{ inputs.isaacsim-version }} + TARGET_PLATFORM: ${{ inputs.platform }} + run: | + set -euo pipefail + + # Exact files/dirs whose full content is hashed. The Dockerfile is first. + deps_files=( + "${DOCKERFILE_PATH}" + isaaclab.sh + environment.yml + source/isaaclab/isaaclab/cli + ) + deps_manifest_pattern='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$' + + # Resolve the actual base image digest so a new push of a mutable tag + # (e.g. latest-develop) invalidates the deps cache automatically. + base_image_digest=$(docker buildx imagetools inspect \ + "${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}" \ + --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true) + if [ -n "${base_image_digest}" ]; then + base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}:${base_image_digest}" + else + echo "🟠 Could not resolve base image digest, falling back to tag string" + base_image_uniq_id="${ISAACSIM_BASE_IMAGE}:${ISAACSIM_VERSION}" + fi + + mapfile -t manifest_files < <(git ls-files | grep -E "${deps_manifest_pattern}" || true) + file_hash=$(git ls-files -s "${deps_files[@]}" "${manifest_files[@]}" 2>/dev/null \ + | sha256sum | cut -c1-16) + deps_hash=$(printf '%s %s %s' "${file_hash}" "${base_image_uniq_id}" "${TARGET_PLATFORM}" \ + | sha256sum | cut -c1-16) + + echo "🔵 Deps hash: ${deps_hash}" + echo "hash=${deps_hash}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/_lib/setup-docker-config/action.yml b/.github/actions/_lib/setup-docker-config/action.yml new file mode 100644 index 000000000000..09effa56eaa3 --- /dev/null +++ b/.github/actions/_lib/setup-docker-config/action.yml @@ -0,0 +1,43 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Setup docker config' +description: > + Point DOCKER_CONFIG at a temp config with the credential helper disabled and + log into nvcr.io. Shared by the docker-build and ecr-build-push-pull actions. + Idempotent: re-invoking it in the same job is a no-op, so callers (e.g. + ecr-build-push-pull delegating to docker-build) don't need to coordinate. + Reads NGC_API_KEY from the environment (optional; warns when missing). + +runs: + using: composite + steps: + - shell: bash + run: | + # The runner's credential helper backend is broken ("not implemented") + # and causes docker login calls to fail unless we point DOCKER_CONFIG at + # a temp config with credsStore disabled. The value is written to + # $GITHUB_ENV so subsequent steps in the job inherit it; a second + # invocation sees it already set and short-circuits. + if [ -n "${DOCKER_CONFIG:-}" ] && [ -f "${DOCKER_CONFIG}/config.json" ]; then + echo "🟢 Docker config already set up at ${DOCKER_CONFIG}, skipping" + exit 0 + fi + + DOCKER_CONFIG_DIR=$(mktemp -d) + if [ -f "${HOME}/.docker/config.json" ]; then + python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))" + else + echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" + fi + export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" + echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV" + + if [ -n "${NGC_API_KEY:-}" ]; then + echo "🔵 Logging into nvcr.io..." + docker login -u '$oauthtoken' -p "${NGC_API_KEY}" nvcr.io + else + echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)" + fi diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 7f88241cfb8c..8ae7110c78bb 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -24,61 +24,222 @@ inputs: description: 'Build context path' default: '.' required: false + platform: + description: 'Target platform for `docker buildx build --platform`.' + default: 'linux/amd64' + required: false + cache-from: + description: > + Optional value for `docker buildx build --cache-from`. Typically a + `type=registry,ref=` for cross-host layer cache. Leave empty for + pure local-only builds. + default: '' + required: false + cache-to: + description: > + Optional value for `docker buildx build --cache-to`. Pairs with + `cache-from` for registry-backed layer cache writes. + default: '' + required: false + deps-hash: + description: > + Pre-computed deps-hash to use for the local deps-tag check. When empty, + this action computes the hash itself via the `_lib/compute-deps-hash` + action. Set by callers (e.g. `ecr-build-push-pull`) that already compute + the hash for a registry-side check, to avoid recomputing here. + default: '' + required: false + evict-stale-cache: + description: > + When 'true', evict `isaac-lab*:deps-*` tags older than 14 days at the + end of the build to bound disk growth on long-lived self-hosted + runners. Default 'false' — no implicit cleanup. + default: 'false' + required: false runs: using: composite steps: - - name: NGC Login - shell: sh + + ##### 1: Setup docker config + login to nvcr.io (optional) ##### + + - name: Setup docker config and login to nvcr.io + uses: ./.github/actions/_lib/setup-docker-config + + ##### 2: Host disk snapshot (pre) ##### + + - name: Host disk snapshot (pre) + shell: bash + run: | + set +e + docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker") + deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l) + commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l) + { + echo "## Disk snapshot (pre)" + echo '```' + echo "Filesystem:" + df -h / "${docker_root}" 2>/dev/null | sort -u + echo + echo "docker system df:" + docker system df + echo + echo "Tag counts:" + echo " isaac-lab* (commit + deps tags): ${commit_count}" + echo " isaac-lab*:deps-* (deps cache) : ${deps_count}" + echo + echo "Deps tags (newest first):" + docker images --filter 'reference=isaac-lab*:deps-*' \ + --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \ + | head -20 + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + ##### 3: Local exact-tag short-circuit ##### + + - name: Check image locally + id: local + shell: bash + run: | + if docker image inspect "${{ inputs.image-tag }}" >/dev/null 2>&1; then + echo "🟢 Image already in local docker store: ${{ inputs.image-tag }}" + echo "hit=true" >> "$GITHUB_OUTPUT" + else + echo "🔵 Image not present locally, will check deps-cache / build" + fi + + ##### 4: Local deps-tag short-circuit ##### + + - name: Compute deps hash + id: deps-hash + if: steps.local.outputs.hit != 'true' && inputs.deps-hash == '' + uses: ./.github/actions/_lib/compute-deps-hash + with: + dockerfile-path: ${{ inputs.dockerfile-path }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + platform: ${{ inputs.platform }} + + - name: Check deps-tag locally + id: local-deps + if: steps.local.outputs.hit != 'true' + shell: bash run: | - # Only attempt NGC login if API key is available - if [ -n "${{ env.NGC_API_KEY }}" ]; then - echo "Logging into NGC registry..." - docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io - echo "✅ Successfully logged into NGC registry" + DEPS_HASH="${{ inputs.deps-hash || steps.deps-hash.outputs.hash }}" + LOCAL_DEPS_TAG="$(echo "${{ inputs.image-tag }}" | cut -d: -f1):deps-${DEPS_HASH}" + + echo "🔵 Local deps tag: ${LOCAL_DEPS_TAG}" + echo "LOCAL_DEPS_TAG=${LOCAL_DEPS_TAG}" >> "$GITHUB_ENV" + + if docker image inspect "${LOCAL_DEPS_TAG}" >/dev/null 2>&1; then + echo "🟢 Local deps-cache HIT! Retagging as ${{ inputs.image-tag }}" + docker tag "${LOCAL_DEPS_TAG}" "${{ inputs.image-tag }}" + echo "hit=true" >> "$GITHUB_OUTPUT" + else + echo "🟠 Local deps-cache MISS (will build then tag for future hits)" + fi + + ##### 5: Full build ##### + + - name: Build image + id: build + if: > + steps.local.outputs.hit != 'true' && + steps.local-deps.outputs.hit != 'true' + shell: bash + run: | + BUILD_ARGS=( + --progress=plain + --platform "${{ inputs.platform }}" + -f "${{ inputs.dockerfile-path }}" + --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" + --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" + --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim" + --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab" + --build-arg "DOCKER_USER_HOME_ARG=/root" + -t "${{ inputs.image-tag }}" + ) + if [ -n "${{ inputs.cache-from }}" ]; then + BUILD_ARGS+=( --cache-from "${{ inputs.cache-from }}" ) + fi + if [ -n "${{ inputs.cache-to }}" ]; then + BUILD_ARGS+=( --cache-to "${{ inputs.cache-to }}" ) + fi + + BUILDER_NAME="docker-build-${{ github.run_id }}-${{ github.job }}" + docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ + || docker buildx use "${BUILDER_NAME}" + trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT + + echo "🔵 Building ${{ inputs.image-tag }}..." + docker buildx build --load "${BUILD_ARGS[@]}" "${{ inputs.context-path }}" + echo "was-built=true" >> "$GITHUB_OUTPUT" + + ##### 6: Tag built image with local deps-tag ##### + + # Runs only when a real build happened (not on cache hits). Populates the + # deps-tag so the next build with identical deps short-circuits at step 4. + + - name: Tag built image with local deps-tag + if: steps.build.outputs.was-built == 'true' + shell: bash + run: | + if [ -n "${LOCAL_DEPS_TAG:-}" ]; then + docker tag "${{ inputs.image-tag }}" "${LOCAL_DEPS_TAG}" + echo "🟢 Tagged local deps-cache: ${LOCAL_DEPS_TAG}" else - echo "⚠️ NGC_API_KEY not available - skipping NGC login" - echo "This is normal for PRs from forks or when secrets are not configured" + echo "🟠 LOCAL_DEPS_TAG not set, skipping local deps-cache tag" fi - - name: Build Docker Image - shell: sh + ##### 7: Evict stale local deps-cache tags (>14d) — opt-in ##### + + - name: Evict stale local deps-cache tags (>14d) + if: always() && inputs.evict-stale-cache == 'true' + shell: bash run: | - # Function to build Docker image - build_docker_image() { - local image_tag="$1" - local isaacsim_base_image="$2" - local isaacsim_version="$3" - local dockerfile_path="$4" - local context_path="$5" - - # Skip build if image already exists locally (e.g. built by a prior job on the same runner) - if docker image inspect "$image_tag" > /dev/null 2>&1; then - echo "Image $image_tag already exists locally, skipping build." - return 0 + set +e + TTL_DAYS=14 + cutoff=$(date -u -d "${TTL_DAYS} days ago" +%s) + evicted=0 + while IFS='|' read -r created tag; do + [ -z "$tag" ] && continue + created_epoch=$(date -d "$created" +%s 2>/dev/null) || continue + if [ "$created_epoch" -lt "$cutoff" ]; then + days_old=$(( (cutoff - created_epoch) / 86400 + TTL_DAYS )) + echo "🟠 Evicting deps tag (~${days_old}d old): ${tag}" + docker rmi -f "$tag" >/dev/null 2>&1 || true + evicted=$(( evicted + 1 )) fi + done < <(docker images --filter 'reference=isaac-lab*:deps-*' \ + --format '{{.CreatedAt}}|{{.Repository}}:{{.Tag}}' 2>/dev/null) + echo "🔵 Evicted ${evicted} deps tag(s) older than ${TTL_DAYS}d" - echo "Building Docker image: $image_tag" - echo "Using Dockerfile: $dockerfile_path" - echo "Build context: $context_path" - - # Build Docker image - docker buildx build --progress=plain --platform linux/amd64 \ - -t $image_tag \ - --build-arg ISAACSIM_BASE_IMAGE_ARG="$isaacsim_base_image" \ - --build-arg ISAACSIM_VERSION_ARG="$isaacsim_version" \ - --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \ - --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \ - --build-arg DOCKER_USER_HOME_ARG=/root \ - --cache-from type=gha \ - --cache-to type=gha,mode=max \ - -f $dockerfile_path \ - --load $context_path - - echo "✅ Docker image built successfully: $image_tag" - echo "Current local Docker images:" - docker images - } - - # Call the function with provided parameters - build_docker_image "${{ inputs.image-tag }}" "${{ inputs.isaacsim-base-image }}" "${{ inputs.isaacsim-version }}" "${{ inputs.dockerfile-path }}" "${{ inputs.context-path }}" + ##### 8: Host disk snapshot (post) ##### + + - name: Host disk snapshot (post) + if: always() + shell: bash + run: | + set +e + docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "/var/lib/docker") + deps_count=$(docker images --filter 'reference=isaac-lab*:deps-*' -q 2>/dev/null | wc -l) + commit_count=$(docker images --filter 'reference=isaac-lab*' -q 2>/dev/null | wc -l) + { + echo "## Disk snapshot (post)" + echo '```' + echo "Filesystem:" + df -h / "${docker_root}" 2>/dev/null | sort -u + echo + echo "docker system df:" + docker system df + echo + echo "Tag counts:" + echo " isaac-lab* (commit + deps tags): ${commit_count}" + echo " isaac-lab*:deps-* (deps cache) : ${deps_count}" + echo + echo "Deps tags (newest first):" + docker images --filter 'reference=isaac-lab*:deps-*' \ + --format 'table {{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}' 2>/dev/null \ + | head -20 + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml index b661d4b9fd62..d6fe0cb27987 100644 --- a/.github/actions/ecr-build-push-pull/action.yml +++ b/.github/actions/ecr-build-push-pull/action.yml @@ -50,23 +50,7 @@ runs: # (including ECR login in step 3) inherit it automatically. - name: Setup docker config and login to nvcr.io - shell: bash - run: | - DOCKER_CONFIG_DIR=$(mktemp -d) - if [ -f "${HOME}/.docker/config.json" ]; then - python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))" - else - echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json" - fi - echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV" - export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}" - - if [ -n "${{ env.NGC_API_KEY }}" ]; then - echo "🔵 Logging into nvcr.io..." - docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io - else - echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)" - fi + uses: ./.github/actions/_lib/setup-docker-config ##### 2: Resolve ECR URL ##### @@ -191,40 +175,21 @@ runs: # Edit DEPS_FILES or DEPS_MANIFEST_PATTERN when install # inputs change (new packages, new manifests, etc.). + - name: Compute deps hash + id: deps-hash + if: steps.resolve-ecr.outputs.available == 'true' && steps.pull-exact.outputs.hit != 'true' + uses: ./.github/actions/_lib/compute-deps-hash + with: + dockerfile-path: ${{ inputs.dockerfile-path }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + - name: Check deps cache id: deps-cache if: steps.resolve-ecr.outputs.available == 'true' && steps.pull-exact.outputs.hit != 'true' shell: bash run: | - ##### Deps-hash configuration ##### - # Exact files/dirs whose full content is hashed. The Dockerfile is first. - DEPS_FILES=( - "${{ inputs.dockerfile-path }}" - isaaclab.sh - environment.yml - source/isaaclab/isaaclab/cli - ) - # Manifest files matched repo-wide via git ls-files. - DEPS_MANIFEST_PATTERN='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$' - - # Resolve the actual base image digest so a new push of a mutable tag - # (e.g. latest-develop) invalidates the deps cache automatically. - BASE_IMAGE_DIGEST=$(docker buildx imagetools inspect \ - "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \ - --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true) - if [ -n "${BASE_IMAGE_DIGEST}" ]; then - BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}:${BASE_IMAGE_DIGEST}" - else - echo "🟠 Could not resolve base image digest, falling back to tag string" - BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" - fi - - echo "🔵 Base image ID: ${BASE_IMAGE_UNIQ_ID}" - - MANIFEST_FILES=$(git ls-files | grep -E "${DEPS_MANIFEST_PATTERN}" || true) - FILE_HASH=$(git ls-files -s "${DEPS_FILES[@]}" ${MANIFEST_FILES} 2>/dev/null \ - | sha256sum | cut -c1-16) - DEPS_HASH=$(printf '%s %s' "${FILE_HASH}" "${BASE_IMAGE_UNIQ_ID}" | sha256sum | cut -c1-16) + DEPS_HASH="${{ steps.deps-hash.outputs.hash }}" DEPS_ECR_IMAGE="${ECR_URL}:deps-${DEPS_HASH}" echo "🔵 Deps hash: ${DEPS_HASH}" echo "🔵 Checking if deps image ${DEPS_ECR_IMAGE} exists in ECR..." @@ -245,41 +210,34 @@ runs: echo "PUSH_DEPS_IMAGE=true" >> "$GITHUB_ENV" fi - ##### 6: Full build ##### + ##### 6: Full build (delegated to docker-build) ##### # Runs when neither the exact image nor the deps cache was available. - # Uses ECR layer cache (--cache-from/--cache-to) when ECR is available. + # docker-build does the actual buildx invocation; we pass ECR layer-cache + # refs and the ECR-prefixed tag so the push steps below have something to + # push. - name: Full build if: steps.pull-exact.outputs.hit != 'true' && steps.deps-cache.outputs.deps-cache-hit != 'true' + uses: ./.github/actions/docker-build + with: + image-tag: ${{ inputs.image-tag }} + isaacsim-base-image: ${{ inputs.isaacsim-base-image }} + isaacsim-version: ${{ inputs.isaacsim-version }} + dockerfile-path: ${{ inputs.dockerfile-path }} + cache-from: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0}', env.CACHE_IMAGE) || '' }} + cache-to: ${{ steps.resolve-ecr.outputs.available == 'true' && format('type=registry,ref={0},mode=max', env.CACHE_IMAGE) || '' }} + deps-hash: ${{ steps.deps-hash.outputs.hash }} + + - name: Tag built image with ECR-prefixed name + if: > + steps.resolve-ecr.outputs.available == 'true' && + steps.pull-exact.outputs.hit != 'true' && + steps.deps-cache.outputs.deps-cache-hit != 'true' shell: bash run: | - BUILD_ARGS=( - --progress=plain - --platform linux/amd64 - -f "${{ inputs.dockerfile-path }}" - --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}" - --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}" - --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim" - --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab" - --build-arg "DOCKER_USER_HOME_ARG=/root" - -t "${{ inputs.image-tag }}" - ) - if [ -n "${ECR_URL:-}" ]; then - BUILD_ARGS+=( - --cache-from "type=registry,ref=${CACHE_IMAGE}" - --cache-to "type=registry,ref=${CACHE_IMAGE},mode=max" - -t "${ECR_IMAGE}" - ) - fi - - BUILDER_NAME="ci-builder-${{ github.run_id }}-${{ github.job }}" - docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \ - || docker buildx use "${BUILDER_NAME}" - trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT - - echo "🔵 Building ${{ inputs.image-tag }}..." - docker buildx build --load "${BUILD_ARGS[@]}" . + docker tag "${{ inputs.image-tag }}" "${ECR_IMAGE}" + echo "🟢 Tagged ${ECR_IMAGE}" ##### 7: Push to ECR ##### diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index fbc7043cdf8f..d3829cc8300e 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -92,6 +92,10 @@ inputs: wheelhouse-packages: description: 'Space-separated packages to install from /tmp/ovphysx-wheelhouse with pip --no-index' default: '' + ci-marker: + description: 'CI_MARKER value forwarded to the container (read by tools/conftest.py to select test files by pytest marker)' + default: '' + required: false omni-github-test-type: description: 'Test type stored on each uploaded omni-github test row' default: 'pytest' @@ -108,6 +112,7 @@ runs: # the run_tests positional arguments if substituted textually. PYTEST_OPTIONS: ${{ inputs.pytest-options }} TEST_K_EXPR_INPUT: ${{ inputs.test-k-expr }} + CI_MARKER_INPUT: ${{ inputs.ci-marker }} run: | # Function to run tests in Docker container run_tests() { @@ -131,6 +136,7 @@ runs: local wheelhouse_host_dir="${18}" local wheelhouse_packages="${19}" local test_k_expr="${20}" + local ci_marker="${21}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -279,6 +285,11 @@ runs: echo "Setting per-file pytest -k expression: $test_k_expr" fi + if [ -n "$ci_marker" ]; then + docker_env_vars="$docker_env_vars -e CI_MARKER=$ci_marker" + echo "Setting CI_MARKER=$ci_marker" + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -512,7 +523,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" - name: Kill container on cancellation if: cancelled() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 39f9ffab1b2c..7e704d6a6ff5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -273,6 +273,67 @@ jobs: dockerfile-path: docker/Dockerfile.curobo cache-tag: cache-curobo + # aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners. + # Build and test must share one runner because ECR is not wired for arm64 — + # the locally-built image cannot be handed off across machines. + arm-ci: + name: arm-ci + runs-on: [self-hosted, arm64] + needs: [changes, config] + if: needs.changes.outputs.run_docker_tests == 'true' + timeout-minutes: 60 + continue-on-error: true + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + lfs: true + + - name: Build base image (linux/arm64) + uses: ./.github/actions/docker-build + with: + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + platform: linux/arm64 + # The Spark runner is long-lived self-hosted with no ECR, so its local + # deps-cache tags accumulate; evict ones older than 14 days each run. + evict-stale-cache: "true" + + - name: Run arm_ci marker tests + uses: ./.github/actions/run-tests + with: + test-path: tools + result-file: arm-ci-report.xml + container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + extra-pip-packages: "ovrtx ovphysx==0.4.13" + # ovphysx-backed test params require ovphysx >= 0.5.1, which has no aarch64 + # wheel yet; keep the newton/ovrtx rendering coverage, mirroring the public + # pip-index fallback of rendering-correctness-kitless. Running them anyway + # exhausts ovrtx SyncScopeIds (>15 renderer creations in one process). + test-k-expr: not ovphysx + ci-marker: arm_ci + volume-mount-source: ${{ github.workspace }} + + - name: Run shared Cartpole smoke + uses: ./.github/actions/run-tests + with: + test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py + result-file: arm-ci-cartpole-smoke-report.xml + container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }} + image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64 + volume-mount-source: ${{ github.workspace }} + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: arm-ci-reports + path: reports/ + retention-days: 7 + #endregion #region test jobs diff --git a/.github/workflows/daily-compatibility.yml b/.github/workflows/daily-compatibility.yml index 26693761c27a..5606a02b8a9e 100644 --- a/.github/workflows/daily-compatibility.yml +++ b/.github/workflows/daily-compatibility.yml @@ -3,6 +3,23 @@ # # SPDX-License-Identifier: BSD-3-Clause +# Nightly canary: runs the IsaacLab test suite against multiple pinned +# IsaacSim versions to catch backwards-compatibility regressions. +# +# Caveats for editors of this file or its action dependencies +# (`.github/actions/docker-build`, `run-tests`, `combine-results`): +# +# * No `pull_request:` trigger — changes are not validated automatically +# at PR time. Manually trigger against the PR branch before merge: +# gh workflow run "Backwards Compatibility Tests" --ref +# Otherwise breakage surfaces only on the next nightly cron after merge. +# +# * The build steps below pass `cache-from: type=gha` / `cache-to: +# type=gha,mode=max` to docker-build explicitly. Future migration to +# `./.github/actions/ecr-build-push-pull` would share build.yaml's +# cross-runner ECR layer cache, but that's deferred until daily-compat +# is ready to wire ECR auth. + name: Backwards Compatibility Tests on: @@ -114,6 +131,8 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Run IsaacLab Tasks Tests uses: ./.github/actions/run-tests @@ -172,6 +191,8 @@ jobs: image-tag: ${{ env.DOCKER_IMAGE_TAG }} isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ matrix.isaacsim_version }} + cache-from: type=gha + cache-to: type=gha,mode=max - name: Run General Tests uses: ./.github/actions/run-tests diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index f46bc3f89b21..5058341f39a8 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -47,6 +47,7 @@ RUN apt-get update && \ ncurses-term \ cmake \ git \ + git-lfs \ wget # Install the Carbonite env shim (libcarb.env.shim.so) that serializes glibc's diff --git a/source/isaaclab/changelog.d/jichuanh-arm-ci.rst b/source/isaaclab/changelog.d/jichuanh-arm-ci.rst new file mode 100644 index 000000000000..376f7f589170 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-arm-ci.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Added a defensive fallback in :class:`isaaclab.app.AppLauncher` so it derives + ``EXP_PATH`` from the installed ``isaacsim`` package when the env var is not + set. ``isaacsim.bootstrap_kernel`` normally sets ``EXP_PATH`` on first import, + but the early-return path in its bootstrap (triggered under some pip install + layouts on aarch64) skips the env-var setup. Previously this caused + ``KeyError: 'EXP_PATH'`` deep inside ``_resolve_experience_file``; now + AppLauncher resolves the path from ``isaacsim.__file__`` and stores it back + into the environment so subsequent code can rely on it. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 59f3abf0c961..a75f44cdbd38 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1138,8 +1138,22 @@ def _resolve_experience_file(self, launcher_args: dict): launcher_args.get("deterministic", AppLauncher._APPLAUNCHER_CFG_INFO["deterministic"][1]) ) - # If nothing is provided resolve the experience file based on the headless flag - kit_app_exp_path = os.environ["EXP_PATH"] + # If nothing is provided resolve the experience file based on the headless flag. + # EXP_PATH is normally set by ``isaacsim.bootstrap_kernel()`` on first import. + # If it is not set (e.g. on aarch64 where the bootstrap early-return triggered + # under certain install layouts), derive it from the installed isaacsim package. + kit_app_exp_path = os.environ.get("EXP_PATH") + if not kit_app_exp_path: + try: + import isaacsim as _isaacsim_for_paths + except ImportError as e: + raise RuntimeError( + "EXP_PATH is not set and the 'isaacsim' package is not importable." + " Install Isaac Sim (`pip install isaacsim` or the binary distribution)" + " before launching AppLauncher." + ) from e + kit_app_exp_path = os.path.join(os.path.dirname(_isaacsim_for_paths.__file__), "apps") + os.environ["EXP_PATH"] = kit_app_exp_path isaaclab_app_exp_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), *[".."] * 4, "apps") # For Isaac Sim 4.5 compatibility, we use the 4.5 app files in a different folder # if launcher_args.get("use_isaacsim_45", False): diff --git a/source/isaaclab/test/controllers/test_operational_space.py b/source/isaaclab/test/controllers/test_operational_space.py index 75e3810fab16..94f84899401d 100644 --- a/source/isaaclab/test/controllers/test_operational_space.py +++ b/source/isaaclab/test/controllers/test_operational_space.py @@ -16,6 +16,8 @@ import torch from flaky import flaky +pytestmark = pytest.mark.arm_ci + import isaaclab.envs.mdp as mdp import isaaclab.sim as sim_utils from isaaclab import cloner diff --git a/source/isaaclab/test/deps/test_scipy.py b/source/isaaclab/test/deps/test_scipy.py index 91d47a5c9e62..6ddf0441da6d 100644 --- a/source/isaaclab/test/deps/test_scipy.py +++ b/source/isaaclab/test/deps/test_scipy.py @@ -13,7 +13,7 @@ import numpy as np import scipy.interpolate as interpolate -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.arm_ci] @pytest.mark.isaacsim_ci diff --git a/source/isaaclab/test/deps/test_torch.py b/source/isaaclab/test/deps/test_torch.py index 2d4881346f72..342e3731078f 100644 --- a/source/isaaclab/test/deps/test_torch.py +++ b/source/isaaclab/test/deps/test_torch.py @@ -7,7 +7,7 @@ import torch import torch.utils.benchmark as benchmark -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.arm_ci] @pytest.mark.isaacsim_ci diff --git a/source/isaaclab/test/install_ci/cli/test_cli_install_all_in_uvenv_training.py b/source/isaaclab/test/install_ci/cli/test_cli_install_all_in_uvenv_training.py index 79e0ac180ea8..8cb3d8e85cce 100644 --- a/source/isaaclab/test/install_ci/cli/test_cli_install_all_in_uvenv_training.py +++ b/source/isaaclab/test/install_ci/cli/test_cli_install_all_in_uvenv_training.py @@ -7,8 +7,17 @@ Setup: - ./isaaclab.sh -u Tests: - - ./isaaclab.sh -i core -> verify core submodules importable - - ./isaaclab.sh -i all -> verify cartpole training works + - ./isaaclab.sh -i core + -> verify core submodules are importable + - ./isaaclab.sh -i all + -> verify the full installation succeeds + - ./isaaclab.sh train --rl_library rsl_rl --task Isaac-Cartpole-Direct + --num_envs 16 presets=newton_mjwarp --max_iterations 5 + -> verify state training completes + - ./isaaclab.sh train --rl_library rsl_rl --task Isaac-Cartpole-Camera-Direct + --num_envs 16 presets=newton_mjwarp,newton_renderer --max_iterations 2 + --headless --enable_cameras + -> verify camera rendering is valid and camera training completes """ from __future__ import annotations @@ -18,36 +27,6 @@ import pytest from utils import UV_Mixin -# --------------------------------------------------------------------------- -# Shared training helper -# --------------------------------------------------------------------------- - -_TRAIN_CMD = [ - "train", - "--rl_library", - "rsl_rl", - "--task", - "Isaac-Cartpole-Direct", - "--num_envs", - "16", - "presets=newton_mjwarp", - "--max_iterations", - "5", - "--headless", -] - - -def _assert_training_passed(result) -> None: - output = result.stdout + (result.stderr or "") - assert result.returncode == 0, f"Training failed (rc={result.returncode}):\n{output}" - assert "Traceback (most recent call last):" not in output, f"Training produced a traceback:\n{output}" - assert "Training time:" in output, f"Training did not report completion:\n{output}" - - -# --------------------------------------------------------------------------- -# uv-based tests -# --------------------------------------------------------------------------- - class Test_Cli_Install_All_In_Uvenv_Training(UV_Mixin): """Installation and training smoke tests using uv environments.""" @@ -93,9 +72,9 @@ def test_install_core_makes_core_submodules_importable(self, isaaclab_root): @pytest.mark.uv @pytest.mark.slow @pytest.mark.gpu - @pytest.mark.timeout(1800) - def test_install_all_trains_cartpole(self, isaaclab_root): - """``./isaaclab.sh -i all`` (full install) + training completes successfully.""" + @pytest.mark.timeout(3600) + def test_install_all_trains_cartpole(self, isaaclab_root, cartpole_smoke_script): + """``-i all`` supports state training, camera rendering, and camera training.""" try: self.create_uv_env(isaaclab_root) result = self.run_in_uv_env( @@ -105,10 +84,10 @@ def test_install_all_trains_cartpole(self, isaaclab_root): ) assert result.returncode == 0, f"isaaclab -i all failed:\n{result.stdout}\n{result.stderr}" result = self.run_in_uv_env( - [str(self.cli_script)] + _TRAIN_CMD, + [str(self.python), str(cartpole_smoke_script)], cwd=isaaclab_root, - timeout=600, + timeout=3000, ) - _assert_training_passed(result) + assert result.returncode == 0, f"Cartpole smoke failed:\n{result.stdout}\n{result.stderr}" finally: self.destroy_uv_env() diff --git a/source/isaaclab/test/install_ci/conftest.py b/source/isaaclab/test/install_ci/conftest.py index d1dc50ab486d..d7bc590c8285 100644 --- a/source/isaaclab/test/install_ci/conftest.py +++ b/source/isaaclab/test/install_ci/conftest.py @@ -41,6 +41,12 @@ def isaaclab_root() -> Path: return find_isaaclab_root() +@pytest.fixture(scope="session") +def cartpole_smoke_script() -> Path: + """Path to the shared Cartpole smoke probe executed inside installed environments.""" + return Path(__file__).resolve().parent / "misc" / "cartpole_training_smoke.py" + + @pytest.fixture def tmp_venv(tmp_path: Path): """Create a temporary Python virtual-environment and tear it down after the test. diff --git a/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py new file mode 100644 index 000000000000..58fd7d1fc0cb --- /dev/null +++ b/source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py @@ -0,0 +1,124 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared state-training, camera-rendering, and camera-training Cartpole smoke probes. + +The caller owns environment preparation. Installation CI executes this file +inside the environment it just installed, while architecture CI executes the +same probes inside its prepared image. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +_STATE_TRAIN_CMD = [ + "train", + "--rl_library", + "rsl_rl", + "--task", + "Isaac-Cartpole-Direct", + "--num_envs", + "16", + "presets=newton_mjwarp", + "--max_iterations", + "5", +] + +_CAMERA_TRAIN_CMD = [ + "train", + "--rl_library", + "rsl_rl", + "--task", + "Isaac-Cartpole-Camera-Direct", + "--num_envs", + "16", + "presets=newton_mjwarp,newton_renderer", + "--max_iterations", + "2", + "--headless", + "--enable_cameras", +] + + +def _find_isaaclab_root() -> Path: + """Return the repository root containing the Isaac Lab launcher.""" + for parent in Path(__file__).resolve().parents: + if (parent / "isaaclab.sh").exists() or (parent / "isaaclab.bat").exists(): + return parent + raise FileNotFoundError("Could not locate the Isaac Lab repository root") + + +def _assert_training_passed(result: subprocess.CompletedProcess[str]) -> None: + """Assert that a short training run completed without a traceback.""" + output = result.stdout + (result.stderr or "") + assert result.returncode == 0, f"Training failed (rc={result.returncode}):\n{output}" + assert "Traceback (most recent call last):" not in output, f"Training produced a traceback:\n{output}" + assert "Training time:" in output, f"Training did not report completion:\n{output}" + + +def _run_training(command: list[str], timeout: int) -> None: + """Run one training command in the caller's active environment.""" + isaaclab_root = _find_isaaclab_root() + launcher = isaaclab_root / ("isaaclab.bat" if os.name == "nt" else "isaaclab.sh") + result = subprocess.run( + [str(launcher)] + command, + cwd=isaaclab_root, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + _assert_training_passed(result) + + +def test_train_cartpole_state_completes() -> None: + """Verify that state-observation Cartpole completes short rsl_rl training.""" + _run_training(_STATE_TRAIN_CMD, timeout=600) + + +def test_render_cartpole_camera_produces_valid_observation_and_reward() -> None: + """Verify that camera Cartpole renders varying pixels and produces finite rewards.""" + import torch + + from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env import CartpoleCameraEnv + from isaaclab_tasks.core.cartpole.cartpole_direct_camera_env_cfg import CartpoleCameraEnvCfg + from isaaclab_tasks.utils.hydra import resolve_presets + + env_cfg = resolve_presets(CartpoleCameraEnvCfg(), selected={"newton_mjwarp", "newton_renderer"}) + env_cfg.scene.num_envs = 2 + env_cfg.frame_stack = 1 + env = None + try: + env = CartpoleCameraEnv(cfg=env_cfg) + obs, _ = env.reset() + image = obs["policy"] + expected_shape = (2, 3, env_cfg.tiled_camera.height, env_cfg.tiled_camera.width) + assert tuple(image.shape) == expected_shape, ( + f"Camera observation shape {tuple(image.shape)} != {expected_shape}" + ) + assert torch.isfinite(image).all(), "Camera observation contains NaN or infinity" + assert image.amax() > image.amin(), "Camera observation is constant" + + action = torch.zeros(env.num_envs, 1, device=env.device) + _, reward, _, _, _ = env.step(action) + assert torch.isfinite(reward).all(), "Cartpole reward contains NaN or infinity" + finally: + if env is not None: + env.close() + + +def test_train_cartpole_camera_completes() -> None: + """Verify that camera-observation Cartpole completes short RSL-RL training.""" + # A cold camera run compiles shaders before training begins. + _run_training(_CAMERA_TRAIN_CMD, timeout=1800) + + +if __name__ == "__main__": + test_train_cartpole_state_completes() + test_render_cartpole_camera_produces_valid_observation_and_reward() + test_train_cartpole_camera_completes() diff --git a/source/isaaclab/test/install_ci/uv_pip/test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole.py b/source/isaaclab/test/install_ci/uv_pip/test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole.py index e69edb88fdae..5d201989ac43 100644 --- a/source/isaaclab/test/install_ci/uv_pip/test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole.py +++ b/source/isaaclab/test/install_ci/uv_pip/test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole.py @@ -17,7 +17,9 @@ - (aarch64 only) export LD_PRELOAD=/lib/aarch64-linux-gnu/libgomp.so.1 Tests: - ./isaaclab.sh train --rl_library rsl_rl --task Isaac-Cartpole-Direct --num_envs 16 - presets=newton_mjwarp --max_iterations 5 --headless + presets=newton_mjwarp --max_iterations 5; ./isaaclab.sh train --rl_library rsl_rl + --task Isaac-Cartpole-Camera-Direct --num_envs 16 presets=newton_mjwarp,newton_renderer --max_iterations 2 + --headless --enable_cameras -> verify state training, camera rendering, and camera training work """ from __future__ import annotations @@ -27,27 +29,6 @@ import pytest from utils import UV_Mixin, aarch64_isaacsim_env, cuda_torch_index_url, pinned_torch_specs -_TRAIN_CMD = [ - "train", - "--rl_library", - "rsl_rl", - "--task", - "Isaac-Cartpole-Direct", - "--num_envs", - "16", - "presets=newton_mjwarp", - "--max_iterations", - "5", - "--headless", -] - - -def _assert_training_passed(result) -> None: - output = result.stdout + (result.stderr or "") - assert result.returncode == 0, f"Training failed (rc={result.returncode}):\n{output}" - assert "Traceback (most recent call last):" not in output, f"Training produced a traceback:\n{output}" - assert "Training time:" in output, f"Training did not report completion:\n{output}" - @pytest.mark.install_path_uv_pip class Test_Uv_Pip_Install_Isaaclab_All_Isaacsim_Trains_Cartpole(UV_Mixin): @@ -63,8 +44,8 @@ def setup_class(cls): @pytest.mark.uv @pytest.mark.slow @pytest.mark.gpu - @pytest.mark.timeout(3600) - def test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole(self, isaaclab_root, wheel): + @pytest.mark.timeout(4800) + def test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole(self, isaaclab_root, wheel, cartpole_smoke_script): """Install the runner-supplied wheel with ``[all,isaacsim]`` via ``uv pip``, run cartpole training.""" try: # 1. Create the uv env and install the wheel with [all,isaacsim] extras. @@ -117,14 +98,13 @@ def test_uv_pip_install_isaaclab_all_isaacsim_trains_cartpole(self, isaaclab_roo ) assert result.returncode == 0, f"uv pip install CUDA torch failed:\n{result.stdout}\n{result.stderr}" - # 3. Run cartpole training via ./isaaclab.sh train (same invocation as - # test_cli_install_training_in_uvenv::test_install_all_trains_cartpole). + # 3. Run the shared state and camera Cartpole smoke in the installed environment. result = self.run_in_uv_env( - [str(self.cli_script)] + _TRAIN_CMD, + [str(self.python), str(cartpole_smoke_script)], cwd=isaaclab_root, env=aarch64_isaacsim_env(), - timeout=900, + timeout=3000, ) - _assert_training_passed(result) + assert result.returncode == 0, f"Cartpole smoke failed:\n{result.stdout}\n{result.stderr}" finally: self.destroy_uv_env() diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip new file mode 100644 index 000000000000..4e2bf0d1d9c6 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-arm-ci.skip @@ -0,0 +1,2 @@ +Test-only changes (arm_ci markers and ARM rendering coverage); no user-facing +changelog entry. diff --git a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py index 3930d0e2d998..56c269db1882 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py @@ -17,7 +17,7 @@ rendering_test_cartpole, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py index ec92b25ee773..01b855b40c61 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_dexsuite_kuka_homo_kitless.py @@ -17,7 +17,7 @@ rendering_test_dexsuite_kuka, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py index 7652b9c92fa6..bd3f6ddebf8c 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py +++ b/source/isaaclab_tasks/test/core/test_rendering_shadow_hand_kitless.py @@ -17,7 +17,7 @@ rendering_test_shadow_hand, ) -pytestmark = pytest.mark.isaacsim_ci +pytestmark = [pytest.mark.isaacsim_ci, pytest.mark.arm_ci] _COMPARISON_SCORES: list[dict] = [] diff --git a/tools/conftest.py b/tools/conftest.py index 963853194f3d..5a2421a88e2b 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -411,8 +411,9 @@ class _PassContext: test_file: Absolute path to the test file being driven. file_name: Basename of ``test_file`` (used for JUnit naming). workspace_root: Repository root; passed to pytest's ``--config-file``. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. + ci_marker: Optional pytest marker expression. When set, adds the + ``-m `` selector (e.g. ``isaacsim_ci``, ``windows_ci``, + ``arm_ci``); when falsy, no marker filter is applied. timeout: Per-pass hard timeout in seconds. startup_deadline: Per-pass startup-hang deadline in seconds. env: Environment passed to the pytest subprocess. @@ -421,7 +422,7 @@ class _PassContext: test_file: str file_name: str workspace_root: str - isaacsim_ci: bool + ci_marker: str | None timeout: int startup_deadline: int env: dict @@ -496,8 +497,8 @@ def _run_one_pass( f"--junitxml={report_file}", "--tb=short", ] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] + if ctx.ci_marker: + cmd += ["-m", ctx.ci_marker] if k_expr is not None: cmd += ["-k", k_expr] cmd += ctx.pytest_targets @@ -731,7 +732,7 @@ def _run_one_pass( ) -def run_individual_tests(test_files, workspace_root, isaacsim_ci, test_node_ids_by_file=None): +def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by_file=None): """Run each test file separately, ensuring one finishes before starting the next.""" failed_tests = [] test_status = {} @@ -774,7 +775,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci, test_node_ids_ test_file=test_file, file_name=file_name, workspace_root=workspace_root, - isaacsim_ci=isaacsim_ci, + ci_marker=ci_marker, timeout=timeout, startup_deadline=startup_deadline, env=env, @@ -959,6 +960,12 @@ def pytest_sessionstart(session): isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" + # CI_MARKER env var is a separate, parallel mechanism for cross-platform + # jobs (arm-ci, windows-ci, ...) to reuse this orchestrator with their own + # markers. Deliberately NOT aliased to ISAACSIM_CI_SHORT: the isaacsim_ci + # filter is owned by Isaac Sim's external CI pipeline; the CI_MARKER path + # leaves that contract untouched. + ci_marker = os.environ.get("CI_MARKER", "") test_node_ids_by_file = _collect_test_node_ids_by_file(workspace_root) # Parse include files list (comma-separated paths) @@ -1013,6 +1020,24 @@ def pytest_sessionstart(session): new_test_files.append(test_file) test_files = new_test_files + if ci_marker: + # Match both `@pytest.mark.` (per-function) and + # `pytestmark = pytest.mark.` / `pytestmark = [..., pytest.mark., ...]` + # (module-level) by looking for the common `pytest.mark.` substring. + marker_token = f"pytest.mark.{ci_marker}" + new_test_files = [] + for test_file in test_files: + try: + with open(test_file) as f: + if marker_token in f.read(): + new_test_files.append(test_file) + except OSError as exc: + raise RuntimeError( + f"ci_marker post-scan could not read {test_file}; refusing to" + f" silently drop a potentially marker-tagged file" + ) from exc + test_files = new_test_files + if test_node_ids_by_file: configured_files = set(test_node_ids_by_file) test_files = [test_file for test_file in test_files if os.path.normpath(test_file) in configured_files] @@ -1039,9 +1064,12 @@ def pytest_sessionstart(session): for node_id in test_node_ids_by_file[os.path.normpath(test_file)]: print(f" {node_id}") - # Run all tests individually + # Run all tests individually. CI_MARKER takes precedence when both env + # vars are set; falls back to "isaacsim_ci" when only ISAACSIM_CI_SHORT + # is set. The pytest -m flag only accepts one expression. + effective_marker = ci_marker or ("isaacsim_ci" if isaacsim_ci else "") failed_tests, test_status, xml_reports = run_individual_tests( - test_files, workspace_root, isaacsim_ci, test_node_ids_by_file + test_files, workspace_root, effective_marker, test_node_ids_by_file ) print("failed tests:", failed_tests)