diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 264e01a158b8..4bde3eb22228 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,6 +69,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: 'Extra env vars to forward into the container (one KEY=value per line)' + default: '' + required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -150,6 +154,7 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} + extra-env-vars: ${{ inputs.extra-env-vars }} - name: Check Test Results if: always() diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ceea3a1c4e65..4565f5ddcfd4 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,6 +69,14 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: >- + Extra environment variables to forward into the container, one per line in + ``KEY=value`` form. Whitespace-only lines and lines starting with ``#`` are + ignored. Used by the multi-GPU workflow to inject + ``ISAACLAB_TEST_DEVICES`` / ``ISAACLAB_SIM_DEVICE``. + default: '' + required: false runs: using: composite @@ -93,6 +101,7 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" + local extra_env_vars="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -204,6 +213,24 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + # Caller-supplied extra env vars (one KEY=value per line). Skips + # blank lines and full-line comments (line where the first + # non-whitespace char is ``#``). Mid-line ``#`` is preserved so + # values like ``IMAGE_TAG=v1.0#nightly`` survive. + if [ -n "$extra_env_vars" ]; then + while IFS= read -r line; do + # Strip leading whitespace (YAML ``|`` block can leave indent). + line="${line#"${line%%[![:space:]]*}"}" + [ -z "$line" ] && continue + [ "${line:0:1}" = "#" ] && continue + key="${line%%=*}" + value="${line#*=}" + export "$key"="$value" + docker_env_vars="$docker_env_vars -e $key" + echo "Forwarding extra env var: $key" + done <<< "$extra_env_vars" + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -392,7 +419,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.extra-env-vars }}" - name: Kill container on cancellation if: cancelled() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e09ec651fe3f..3edf0f461b4a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,12 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} + # TEMP (revert before final review / before landing): force + # run_docker_tests=false while iterating PR #5823. All gated + # test/build jobs skip via their existing if-gate; the + # single-GPU Docker + Tests matrix won't burn the gpu pool on + # every push. Per ~/.claude/skills/pr/ci-iteration-shortcut.md. + run_docker_tests: 'false' steps: - id: detect env: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 013be3a5b126..098d9ba784b0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -25,6 +25,11 @@ jobs: doc-build-type: name: Detect Doc Build Type runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip docs build entirely when the PR title is + # marked DO-NOT-MERGE (scratch / hypothesis-only branches don't need a + # full doc rebuild on every push). Push events on main/develop/release + # still run because they don't carry a PR title. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: trigger-deploy: ${{ steps.trigger-deploy.outputs.defined }} steps: @@ -42,8 +47,14 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # run on non-deploy branches to build current version docs only - if: needs.doc-build-type.outputs.trigger-deploy != 'true' + # Run on non-deploy branches to build current version docs only AND skip + # for DO-NOT-MERGE diagnostic PRs (the needed doc-build-type job already + # skips for those, but GitHub Actions evaluates `needs.X.outputs.Y` as + # empty when X is skipped, which keeps this `!= 'true'` gate true — so + # we re-check the title here to actually cascade the skip). + if: | + needs.doc-build-type.outputs.trigger-deploy != 'true' + && !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 2a2fee50c8ab..df3827bfb870 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -34,6 +34,11 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip install-ci entirely when the PR title is + # marked DO-NOT-MERGE. The downstream install-tests-x86 / install-tests-arm + # jobs gate on `needs.changes.outputs.run_install_tests == 'true'`; with + # this job skipped the output is empty and both dependents skip too. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} steps: diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml new file mode 100644 index 000000000000..e391842806fa --- /dev/null +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -0,0 +1,327 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Multi-GPU unit-test workflow +# +# Runs the non-default-GPU subset of unit tests across the multi-GPU runner +# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the +# single-GPU CI; only diffs are the runner label and the env vars that pin +# Kit and the test parametrize to the shard's non-default GPU. +# +# Adding a new test to multi-GPU coverage: give its device parametrize a +# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()`` +# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow +# auto-discovers it. + +name: Multi-GPU pytest + +on: + pull_request: + paths: + - "source/isaaclab/isaaclab/test/utils/**" + - "source/isaaclab/isaaclab/app/app_launcher.py" + - "source/**/test/**/test_*.py" + - ".github/workflows/test-multi-gpu-pytest.yaml" + - ".github/actions/run-package-tests/**" + - ".github/actions/run-tests/**" + - ".github/actions/ecr-build-push-pull/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + + build: + name: Build / cache image + detect GPUs + needs: [config] + runs-on: [self-hosted, linux, x64, multi-gpu] + outputs: + # JSON array of cuda device indices to shard the test matrix across. + # cuda:0 is omitted because single-GPU CI already covers it; we use + # the non-default GPUs (cuda:1 .. cuda:N-1) for multi-GPU coverage. + cuda_shards: ${{ steps.detect.outputs.shards }} + # Length of cuda_shards, consumed as shard-count by tools/conftest.py. + shard_count: ${{ steps.detect.outputs.count }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Detect available GPUs + # Query the actual runner instead of hard-coding ``[1, 2, 3]``. If the + # pool ever changes (4 → 2 GPUs, or 4 → 8 GPUs), the test matrix + # follows automatically. Fails fast here on a regressed pool so we + # don't allocate test runners that will then fail. + id: detect + run: | + N=$(nvidia-smi -L | wc -l) + if [ "$N" -lt 2 ]; then + echo "::error::Need at least 2 GPUs (cuda:0 + at least 1 non-default); found $N" + exit 1 + fi + shards=$(python3 -c "import json; print(json.dumps(list(range(1, $N))))") + count=$((N - 1)) + echo "shards=$shards" >> "$GITHUB_OUTPUT" + echo "count=$count" >> "$GITHUB_OUTPUT" + echo "::notice::$N GPUs → shards=$shards (shard_count=$count)" + + # Pre-populates the ECR exact-commit tag from deps-cache (registry-side + # alias). Without this prior step, run-package-tests' internal + # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves + # no local image, causing `docker run` to fail with `pull access + # denied`. Mirrors the build → test split in build.yaml. + - uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + test-multi-gpu-pytest: + name: Multi-GPU unit tests + needs: [config, build] + if: needs.build.result == 'success' + runs-on: [self-hosted, linux, x64, multi-gpu] + # DIAGNOSTIC: 45 min (target is 30). The first buffered parallel run hit + # the 30-min wall without finishing or printing anything. Raised so a + # slow-but-completing run yields real per-shard durations; tighten back + # to 30 once the split is balanced and the contention picture is clear. + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Discover opt-in test files + # Auto-discovery: any test_*.py with a non-default-capable scope — an + # argless ``test_devices()`` (defaults to "11X") or any + # ``test_devices("..X")`` mask, whose trailing ``X`` spans the + # non-default GPUs — is in scope. Adding a test to multi-GPU CI needs + # no workflow edit; opting a file out is just narrowing its scope (drop + # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. + id: discover + run: | + # File-level opt-out: a test file can exclude itself from multi-GPU CI + # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line. + # Used for files with known Kit/Isaac-Sim concurrency issues; the file + # still runs in single-GPU CI. Excluded files are reported as a notice + # below for visibility. No workflow edit needed to add/remove a file. + mapfile -t candidates < <(grep -rlE 'test_devices\(\)|test_devices\("[^"]*X"\)' source/ --include='test_*.py' | sort -u) + discovered=() + skipped=() + for f in "${candidates[@]}"; do + if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then + skipped+=("$f") + else + discovered+=("$f") + fi + done + for f in "${skipped[@]}"; do + reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//') + echo "::notice::multi-GPU skipped: $f — $reason" + done + if [ ${#discovered[@]} -eq 0 ]; then + echo "::error::No opt-in tests discovered (grep for test_devices with an X scope)" + exit 1 + fi + basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) + echo "include=$basenames" >> "$GITHUB_OUTPUT" + # Full relative paths too, to seed the shared work queue (the run step + # needs runnable paths, not just basenames). + echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT" + echo "::notice::Discovered ${#discovered[@]} opt-in test files" + printf ' %s\n' "${discovered[@]}" + + - name: Pull image from ECR + # Pulls the per-commit image the build job pushed. ecr-build-push-pull + # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which + # the build job's deps-cache-hit registry-tag created for this SHA) + # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our + # parallel ``docker run`` block below. + uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run shards in parallel on local GPUs (1-docker N-shard) + # ONE container hosts all N pytest shards as parallel subshells. Each + # shard pins ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES to its own + # non-default cuda:N and pulls files from the shared work queue. This + # collapses the previous N-container layout into one container, removing + # cross-container races on the shared workspace mount (``_isaac_sim`` + # symlink, ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker + # init per shard, at the cost of sharing the container's /isaac-sim/ + # writable subtrees. Per-shard HOME (under /tmp) keeps user-level + # caches isolated. + # + # Scaling: shard count is derived inside the container from + # ``nvidia-smi -L`` (truth — torch.cuda.device_count() under-counts MIG + # slices on the same parent GPU), so this works unchanged on any GPU + # count or topology. ``CUDA_VISIBLE_DEVICES`` is set to the + # comma-separated MIG-UUID list only on MIG-mode hosts; discrete-GPU + # runners rely on ``--gpus all``. + env: + IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} + INCLUDE_FILES: ${{ steps.discover.outputs.include }} + PATHS: ${{ steps.discover.outputs.paths }} + run: | + set +e + host_uid="$(id -u)" + host_gid="$(id -g)" + host_user="$(id -un)" + + # _isaac_sim symlink on the host — single creator now, no race. + rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + + runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" + mkdir -p "$runtime_dir" + + # Directory-based work queue (atomic os.rename pattern; conftest claims + # entries via os.rename(queue/X, inflight//X) — POSIX rename is + # atomic on the same filesystem, so two shards racing on the same + # entry are kernel-serialized: one wins, the other gets ENOENT and + # moves on). Each pending test path is materialized as an empty file + # in queue/ with '/' slugged to '__' so it's a flat filename. + # + # Subdirs: + # queue/ — pending claims + # inflight// — claimed by this shard, test in flight + # done// — test exited cleanly + # + # At job-end the reconciler step asserts queue/ + inflight/ are empty + # (anything left = silent-drop signal: never claimed OR claimed but + # crashed mid-test). + queue_root="$runtime_dir/queue" + mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" + IFS=',' read -ra _paths <<< "$PATHS" + for path in "${_paths[@]}"; do + [ -n "$path" ] || continue + slug="${path//\//__}" + : > "$queue_root/queue/$slug" + done + echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + + # Per-shard log dir mounted from the container so we can grouped-print + # the per-shard stream after the run completes. + logs_dir="$runtime_dir/logs" + mkdir -p "$logs_dir" + + # MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. + # nvidia-smi -L is authoritative — see memory rule + # ``reference_gpu_enumeration``: torch.cuda.device_count() under-counts + # MIG slices on the same parent GPU. + cvd_args=() + if nvidia-smi -L | grep -q "^ MIG "; then + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" + else + echo "::notice::discrete GPU mode — relying on --gpus all" + fi + + # --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. + docker run --rm --gpus all --network=host \ + --cap-add=SYS_PTRACE \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${{ github.run_id }}-${{ github.run_attempt }}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/tools/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh + docker_rc=$? + + # Grouped re-print of per-shard logs (preserved across the host). + for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" + done + + # Reconciler — silent-drop guard. Anything left in queue/ was never + # claimed by any shard (work-queue starvation). Anything still in + # inflight// was claimed but never moved to done/ (shard + # crashed mid-test). Either is a silent-drop signal that the docker + # exit alone can't surface. Fail the job and name the orphans. + echo "::group::work-queue reconciler" + unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) + if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' + fi + orphans=0 + for shard_dir in "$queue_root/inflight"/*/; do + [ -d "$shard_dir" ] || continue + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' + orphans=$((orphans + count)) + fi + done + done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) + echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" + echo "::endgroup::" + + # Treat reconciler signals as fatal even if all shards exited 0. + if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 + fi + + exit "$docker_rc" diff --git a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst new file mode 100644 index 000000000000..89e774bd26d7 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed :func:`isaaclab.sim.build_simulation_context` silently ignoring the + ``device`` kwarg when ``sim_cfg`` is also provided. Most test callers pass + both kwargs together; the helper now applies the explicit ``device`` over + ``sim_cfg.device`` so the caller's choice wins. Without this, warp kernel + launches in :mod:`isaaclab_newton.assets.articulation` raised device + mismatch errors on non-default GPUs (``env_ids`` allocated on the test's + device while the articulation's resolved device came from the untouched + ``sim_cfg`` default ``cuda:0``). diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..4413699daf3b --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,14 @@ +Added +^^^^^ + +* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. + When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, + ``--/renderer/multiGpu/autoEnable=False`` and ``--/renderer/multiGpu/maxGpuCount=1`` + to the Kit command line so each Kit process touches only its assigned + GPU (rather than enumerating every visible GPU at startup). Used by the + multi-GPU CI workflow to prevent the shared cubric / PhysX-fabric + GPU-interop context across sibling shards that surfaces as + ``[Error] [omni.physx.plugin] Stage X already attached`` and + ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475 + and NVBug 5687364). Off by default; single-GPU and user-facing rendering + paths are unchanged. diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst new file mode 100644 index 000000000000..00b4366e153e --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -0,0 +1,25 @@ +Added +^^^^^ + +* Added :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over + a device set resolved as ``scope ∩ runtime``: ``scope`` is the call-site mask + of devices a test is valid on (default ``"11X"`` ⇒ cpu + cuda:0 + the + non-default GPUs), and the runtime is the ``ISAACLAB_TEST_DEVICES`` env var of + devices a run may use (default ``"110"`` ⇒ cpu + cuda:0). A trailing ``X`` + includes the remaining devices. Single-GPU CI is unchanged; multi-GPU CI sets + the runtime to one non-default GPU per shard. + +* Added ``ISAACLAB_SIM_DEVICE`` env var honored by + :class:`isaaclab.app.AppLauncher` as the implicit-default device when + the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow + boot Kit on a non-default GPU without editing every test's + :class:`~isaaclab.app.AppLauncher` call site. + +* Added ``py-spy`` + ``gdb`` stack capture in ``tools/conftest.py`` on + ``shutdown_hang`` / ``startup_hang`` / ``timeout`` detection. Walks the test + subprocess's process group (cap 8 pids), captures both Python and C++ frames + before ``SIGKILL`` erases them, attaches the output to the JUnit error + report. Makes Kit binary hangs observable in CI logs; safe no-op when + ``py-spy``/``gdb`` are missing. Workflow side adds ``--cap-add=SYS_PTRACE`` + on the per-shard ``docker run`` (required to attach) and adds ``py-spy`` to + the in-container ``pip install`` list. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e8ff526496a1..a768745edc83 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1060,6 +1060,33 @@ def _resolve_device_settings(self, launcher_args: dict): launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id + # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is + # truthy. The default ``apps/isaaclab.python.headless.kit`` sets + # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable + # = true``, so each Kit process enumerates every visible GPU at + # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per + # container, one Kit per non-default cuda device), that produces a + # shared cubric / PhysX-fabric GPU-interop context across sibling + # processes -- surfacing as ``[Error] [omni.physx.plugin] Stage X + # already attached`` mid-test and ``SimulationApp.close`` hanging + # >52s in teardown (see https://github.com/isaac-sim/IsaacLab/issues/3475 + # and NVBug 5687364). Kelly Guo's documented WAR (#omni-kit thread, + # 2024-2025): set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount + # = 1`` so each Kit only touches its assigned GPU. + if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: + sys.argv.append("--/renderer/multiGpu/enabled=False") + sys.argv.append("--/renderer/multiGpu/autoEnable=False") + sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") + # Also disable the fabric GPU-interop path. The renderer multiGpu + # flags above mitigate the startup-time enumeration race; this + # mitigates the runtime GPU-interop race on top. Safe for the + # multi-GPU CI lane: it covers physics / scene / utility tests, not + # rendering. + sys.argv.append("--/physics/fabricUseGPUInterop=false") + logger.info( + "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" + ) + # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash # Kit's platform-info fork during startup. diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index c068469ec599..fae7d4d1f510 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -958,7 +958,7 @@ def _predicate(prim: Usd.Prim) -> bool: def build_simulation_context( create_new_stage: bool = True, gravity_enabled: bool = True, - device: str = "cuda:0", + device: str | None = None, dt: float = 0.01, sim_cfg: SimulationCfg | None = None, add_ground_plane: bool = False, @@ -971,7 +971,11 @@ def build_simulation_context( Args: create_new_stage: Whether to create a new stage. Defaults to True. gravity_enabled: Whether to enable gravity. Defaults to True. - device: Device to run the simulation on. Defaults to "cuda:0". + device: Device to run the simulation on. When given alongside ``sim_cfg``, + overrides ``sim_cfg.device`` so the caller's explicit choice wins + (most test callers pass both, expecting this behavior). Defaults to + ``None``, meaning ``sim_cfg.device`` is left untouched and a freshly + built ``sim_cfg`` uses :class:`SimulationCfg`'s default device. dt: Time step for the simulation. Defaults to 0.01. sim_cfg: SimulationCfg to use. Defaults to None. add_ground_plane: Whether to add a ground plane. Defaults to False. @@ -993,7 +997,17 @@ def build_simulation_context( if sim_cfg is None: gravity = (0.0, 0.0, -9.81) if gravity_enabled else (0.0, 0.0, 0.0) - sim_cfg = SimulationCfg(device=device, dt=dt, gravity=gravity) + sim_cfg = SimulationCfg(dt=dt, gravity=gravity) + if device is not None: + # Honor the explicit device kwarg in both branches: when sim_cfg is + # freshly built, this picks the device; when sim_cfg is passed in, + # this overrides its (possibly default) device. Without the override, + # callers passing both ``sim_cfg=`` and + # ``device=cuda:N`` silently got sim_cfg's device, causing warp + # kernel-launch mismatches when test fixtures allocated tensors on + # the requested device while assets resolved their device from the + # untouched sim_cfg. + sim_cfg.device = device sim = SimulationContext(sim_cfg) diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py new file mode 100644 index 000000000000..e268d22c5c16 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test-time helpers for Isaac Lab. + +Exposes :func:`test_devices` for selecting the device list to parametrize tests +over. The set is ``scope ∩ budget``: ``scope`` is the call-site argument (the +devices the test is valid on), ``budget`` is the ``ISAACLAB_TEST_DEVICES`` env +var (the devices the run may use). +""" + +from .devices import test_devices + +__all__ = ["test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py new file mode 100644 index 000000000000..c54cabfdfbd5 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Device selection for parametrizing tests over cpu / cuda devices. + +Intended use:: + + from isaaclab.test.utils import test_devices + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + def test_foo(device): ... + +Two masks, two roles +-------------------- +A test runs on ``scope ∩ runtime``: + +* **scope** — the call-site mask: the devices the *test* is valid on. The + author owns this; defaults to ``"11X"`` (device-agnostic). +* **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* + may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). + The multi-GPU workflow sets it to one device per shard, matching + ``ISAACLAB_SIM_DEVICE`` (AppLauncher's boot device — not read here). + +A test author never names the shard's GPU; the operator never inspects a +test's device support. The helper intersects the two. + +Mask grammar +------------ +Positions left to right: ``0`` = cpu, ``1`` = cuda:0 (the default GPU), +``2`` = cuda:1, ``3`` = cuda:2, ... Each position is ``0`` (exclude) or ``1`` +(include). A trailing ``X`` means "include all the remaining devices". + +cpu, cuda:0, and a non-default GPU stay distinct by position — running on +cuda:0 says nothing about cuda:1+, which is the whole reason this exists. + +Common masks +------------ +====== =================================================================== +Mask Meaning +====== =================================================================== +``11X`` cpu + cuda:0 + every non-default GPU (default scope; device-agnostic) +``110`` cpu + cuda:0 (pure-math / backend tests: cuda:0 == cuda:N) +``00X`` non-default GPUs only (validates non-default-device behavior) +``100`` cpu only (pure logic) +====== =================================================================== + +Worked example — a ``scope="11X"`` (default) test: + +* single-GPU CI (runtime unset ⇒ ``"110"``) ⇒ ``[cpu, cuda:0]``. +* a multi-GPU shard (runtime ``"0001"``, one device) ⇒ ``[cuda:2]``. + +So argless tests run on cpu + cuda:0 in single-GPU CI and on exactly one +non-default GPU per shard in multi-GPU CI (run-once is a property of the +one-device-per-shard runtime plus round-robin file assignment). + +An empty result means the test is cleanly skipped for this run (e.g. a +``"00X"`` test on a single-GPU host, or a ``"110"`` test on a non-default-GPU +shard). But if the run *explicitly* set ``ISAACLAB_TEST_DEVICES`` to devices +the host does not have, that is a misconfigured run and raises instead of +skipping everything and reporting a vacuous green. + +Local runs +---------- +Set the runtime from the shell to opt a run into non-default GPUs:: + + ISAACLAB_TEST_DEVICES=0001 ISAACLAB_SIM_DEVICE=cuda:2 \\ + ./isaaclab.sh -p -m pytest path/to/test.py +""" + +from __future__ import annotations + +import os + +import torch + +_RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" +"""Env var naming the run's devices: the devices a run may use (see module docstring).""" + +_DEFAULT_RUNTIME = "110" +"""Runtime devices when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset: cpu + cuda:0, +i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" + + +def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> list: + """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. + + ``scope`` is this argument; the runtime comes from the + ``ISAACLAB_TEST_DEVICES`` env var (see the module docstring for the grammar + and the scope / runtime split). + + Args: + scope: Device mask (e.g. ``"11X"``) the test is valid on. Defaults to + ``"11X"`` (cpu + cuda:0 + every non-default GPU): the device-agnostic + common case, so most call sites can use ``test_devices()`` with no + argument. + skip: Optional ``{device: reason}``; in-scope devices listed here are + wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with + the reason instead of being dropped. Use to gate a device variant + that is known broken while keeping it visible. + + Returns: + Ordered device entries (``"cpu"`` / ``"cuda:N"`` strings, or + :func:`pytest.param` for skipped ones) for the second argument to + :func:`pytest.mark.parametrize`. Empty means the test is skipped for + this run. + + Raises: + ValueError: When the run explicitly set ``ISAACLAB_TEST_DEVICES`` to + devices that are not available on this host (a misconfigured run that + would otherwise skip everything and pass vacuously). A scope that + merely does not intersect this run's devices skips, it does not raise. + """ + requested = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + available = _list_available_devices() + runtime_keep = _expand(requested or _DEFAULT_RUNTIME, len(available)) + # A run that explicitly named devices the host cannot provide (e.g. a shard + # pinned to cuda:2 on a 2-GPU host) is misconfigured: fail loudly rather than + # skip every test and report a vacuous green. A scope that simply does not + # intersect this run's devices (a "110" test on a non-default-GPU shard) is + # not an error — it skips. + if requested is not None and not any(runtime_keep): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" + ) + scope_keep = _expand(scope, len(available)) + devices = [ + device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime + ] + if not skip: + return devices + import pytest + + return [ + pytest.param(device, marks=pytest.mark.skip(reason=skip[device])) if device in skip else device + for device in devices + ] + + +def _expand(mask: str, count: int) -> list[bool]: + """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. + + Args: + mask: A mask string (positions plus an optional trailing ``X``). + count: The number of available devices to expand to. + + Returns: + A list of ``count`` booleans, one per device position. + """ + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return ([char == "1" for char in body] + [fill] * count)[:count] + + +def _list_available_devices() -> list[str]: + """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. + + Returns: + Ordered list of device strings as torch addresses them. + """ + devices = ["cpu"] + # torch.cuda (not warp) is deliberate: this runs at pytest collection time, + # before AppLauncher boots Kit. torch.cuda.device_count() enumerates without + # creating a CUDA context, whereas warp.get_cuda_devices() initializes the + # warp runtime — doing that at collection, ahead of Kit's device setup, + # risks the non-default-GPU init-order fragility this suite targets (#5132). + if torch.cuda.is_available(): + devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) + return devices diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index 8f13d79041e7..4ceae87b9878 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -75,7 +75,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light @pytest.mark.isaacsim_ci def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -87,8 +94,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index 8c053bc51cda..09bfe3091839 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -70,8 +70,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" - + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -83,8 +89,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt diff --git a/source/isaaclab/test/sim/test_newton_model_utils.py b/source/isaaclab/test/sim/test_newton_model_utils.py index cc157074fd42..5cfedaab3354 100644 --- a/source/isaaclab/test/sim/test_newton_model_utils.py +++ b/source/isaaclab/test/sim/test_newton_model_utils.py @@ -27,6 +27,7 @@ _scatter_shape_color_rows_kernel, replace_newton_shape_colors, ) +from isaaclab.test.utils import test_devices _WARNING_MESSAGE = "Newton shape color replacement is enabled; this workaround will be deprecated in a future release." @@ -177,7 +178,7 @@ def _run_scatter_shape_color_rows_kernel( pytest.param((-0.25, 1.75, 0.5), id="oob_clamps_pow"), ], ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_scatter_shape_color_rows_kernel(device: str, linear_rgb: tuple[float, float, float]): """Packed RGB per case; the two parametrized cases jointly cover every ``_linear_channel_to_srgb_warp`` branch.""" after = _run_scatter_shape_color_rows_kernel([linear_rgb], device=device) @@ -284,7 +285,7 @@ def test_replace_newton_shape_colors_warning(): replace_newton_shape_colors(model) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPatch, device: str): """Setting ``ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS`` to ``0`` disables the workaround.""" monkeypatch.setenv("ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS", "0") @@ -315,7 +316,7 @@ def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPa assert not any(issubclass(w.category, FutureWarning) for w in recorded) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_invalid_prim(device: str): """Invalid prim path leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -330,7 +331,7 @@ def test_replace_newton_shape_colors_invalid_prim(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_guide_purpose(device: str): """Guide-purpose mesh leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -349,7 +350,7 @@ def test_replace_newton_shape_colors_guide_purpose(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_no_material_binding(device: str): """No material: ``displayColor`` or unbound gray as linear RGB, then sRGB OETF into ``shape_color``.""" stage = Usd.Stage.CreateInMemory() @@ -381,7 +382,7 @@ def test_replace_newton_shape_colors_no_material_binding(device: str): @pytest.mark.parametrize(("diffuse_color_constant", "diffuse_tint"), _OMNIPBR_ALBEDO_INPUT_CASES) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_omnipbr_binding( device: str, diffuse_color_constant: tuple[float, float, float] | None, @@ -405,7 +406,7 @@ def test_replace_newton_shape_colors_omnipbr_binding( ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_neutral_material(device: str): """Bound ``UsdPreviewSurface`` material leaves ``shape_color`` unchanged.""" stage, mesh_path = _make_preview_surface_bound_mesh_stage() @@ -418,7 +419,7 @@ def test_replace_newton_shape_colors_neutral_material(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_respects_binding_strength(device: str): """Parent stronger-than-descendants binding overrides direct child binding.""" # Scene graph (``ComputeBoundMaterial`` on the mesh yields ParentMat / green, not ChildMat / red): @@ -470,7 +471,7 @@ def test_replace_newton_shape_colors_respects_binding_strength(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_instanced(device: str): """Instance-proxy labels deduplicate via canonical prototype paths in the per-key cache.""" stage = Usd.Stage.CreateInMemory() diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 6ea578a85e30..c96883b839a0 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -6,6 +6,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -43,7 +44,7 @@ def test_setup_teardown(): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_init(device): """Test the simulation context initialization.""" from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 64cd86a7466f..827ed20d3261 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -11,6 +11,7 @@ """ from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices simulation_app = AppLauncher(headless=True).app @@ -101,7 +102,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_toggle(device): """Test toggling visibility multiple times.""" if device == "cuda" and not torch.cuda.is_available(): @@ -129,7 +130,7 @@ def test_visibility_toggle(device): assert vis[0] and not vis[1] and vis[2] -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_parent_inheritance(device): """Making a parent invisible hides all children.""" if device == "cuda" and not torch.cuda.is_available(): @@ -155,7 +156,7 @@ def test_visibility_parent_inheritance(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_prim_ordering_follows_creation_order(device): """Prims are returned in USD creation order (DFS), not alphabetical.""" if device == "cuda" and not torch.cuda.is_available(): @@ -181,7 +182,7 @@ def test_prim_ordering_follows_creation_order(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_standardize_transform_op(device): """FrameView standardizes a prim with xformOp:transform to translate/orient/scale.""" if device == "cuda" and not torch.cuda.is_available(): @@ -209,7 +210,7 @@ def test_standardize_transform_op(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_nested_hierarchy_world_poses(device): """World pose of nested child == sum of parent + child translations.""" if device == "cuda" and not torch.cuda.is_available(): @@ -271,7 +272,7 @@ def test_compare_get_world_poses_with_isaacsim(): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_with_franka_robots(device): """Verify FrameView works with real Franka robot USD assets.""" if device == "cuda" and not torch.cuda.is_available(): diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py new file mode 100644 index 000000000000..ff09b244e20a --- /dev/null +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -0,0 +1,148 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for :func:`isaaclab.test.utils.test_devices` (the device-selection helper). + +These tests mock the host's device list, so they need no GPU and run on the +single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) +for two reasons: pytest would otherwise collect the ``test_``-prefixed function +as a test case, and the multi-GPU workflow's auto-discovery greps for +``test_devices(...)`` call sites — neither should fire on this file. +""" + +import pytest + +from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import test_devices as resolve_devices + +# Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). +SINGLE_GPU = ["cpu", "cuda:0"] +MULTI_GPU = ["cpu", "cuda:0", "cuda:1", "cuda:2"] + + +@pytest.fixture +def host(monkeypatch): + """Return a setter that pins the available device list and the runtime env var.""" + + def _set(available: list[str], runtime: str | None = None) -> None: + monkeypatch.setattr(devices_mod, "_list_available_devices", lambda: available) + if runtime is None: + monkeypatch.delenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, raising=False) + else: + monkeypatch.setenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, runtime) + + return _set + + +# --------------------------------------------------------------------------- +# scope ∩ runtime resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "available, runtime, scope, expected", + [ + # single-GPU CI lane: runtime unset -> default "110" (cpu + cuda:0). + (SINGLE_GPU, None, "11X", ["cpu", "cuda:0"]), # argless default is device-agnostic + (SINGLE_GPU, None, "110", ["cpu", "cuda:0"]), # pure-math scope + (SINGLE_GPU, None, "100", ["cpu"]), # cpu-only scope + (SINGLE_GPU, None, "00X", []), # non-default-only -> nothing here, skip + # multi-GPU shard: runtime pins exactly one non-default GPU. + (MULTI_GPU, "001", "11X", ["cuda:1"]), # cuda:1 shard + (MULTI_GPU, "0001", "11X", ["cuda:2"]), # cuda:2 shard + (MULTI_GPU, "0001", "00X", ["cuda:2"]), # non-default regression on its shard + (MULTI_GPU, "0001", "110", []), # math skips the shard (cuda:0 != this shard) + (MULTI_GPU, "0001", "100", []), # cpu test skips the shard + # a runtime that lists several GPUs hands back all in-scope ones. + (MULTI_GPU, "111", "11X", ["cpu", "cuda:0", "cuda:1"]), + ], +) +def test_resolves_scope_intersect_runtime(host, available, runtime, scope, expected): + host(available, runtime) + assert resolve_devices(scope) == expected + + +def test_argless_equals_default_scope(host): + # The common case: argless must equal the explicit default mask everywhere. + for available, runtime in [(SINGLE_GPU, None), (MULTI_GPU, "0001"), (MULTI_GPU, "001")]: + host(available, runtime) + assert resolve_devices() == resolve_devices("11X") + + +def test_argless_runs_once_on_one_non_default_gpu(host): + # Run-once property: with a one-device-per-shard runtime, an argless test + # resolves to exactly one non-default GPU. + host(MULTI_GPU, "0001") + result = resolve_devices() + assert result == ["cuda:2"] + assert all(d not in ("cpu", "cuda:0") for d in result) + + +# --------------------------------------------------------------------------- +# skip vs raise: legitimate skips never raise; only a missing runtime device does +# --------------------------------------------------------------------------- + + +def test_unset_runtime_never_raises(host): + # On a dev box / single-GPU lane (runtime unset), an out-of-scope result is a + # silent skip, not an error. + host(SINGLE_GPU, None) + assert resolve_devices("00X") == [] + + +def test_runtime_naming_absent_device_raises(host): + # A run that explicitly asked for cuda:2 on a host that only has cuda:0/cuda:1 + # is misconfigured -> fail loudly instead of a vacuous green. + host(["cpu", "cuda:0", "cuda:1"], "0001") + with pytest.raises(ValueError, match="no device available"): + resolve_devices("11X") + + +def test_in_range_runtime_with_empty_scope_does_not_raise(host): + # The runtime device exists (cuda:2 is present), the scope just doesn't include + # it -> skip, not raise. This is what keeps "110"/"100" tests green on a shard. + host(MULTI_GPU, "0001") + assert resolve_devices("110") == [] # would raise if the guard keyed off scope + + +# --------------------------------------------------------------------------- +# skip= : gate a specific device visibly +# --------------------------------------------------------------------------- + + +def test_skip_wraps_named_device_as_skipped_param(host): + host(SINGLE_GPU, None) + result = resolve_devices("11X", skip={"cuda:0": "known broken"}) + assert result[0] == "cpu" # untouched devices stay plain strings + param = result[1] + assert param.values == ("cuda:0",) + assert param.marks[0].name == "skip" + assert param.marks[0].kwargs["reason"] == "known broken" + + +def test_skip_ignores_out_of_result_devices(host): + # Skipping a device that isn't in the resolved set is a no-op (no stray params). + host(SINGLE_GPU, None) + assert resolve_devices("11X", skip={"cuda:3": "n/a"}) == ["cpu", "cuda:0"] + + +# --------------------------------------------------------------------------- +# _expand mask grammar +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mask, count, expected", + [ + ("110", 2, [True, True]), # exact length, no wildcard + ("100", 3, [True, False, False]), # short mask padded with False + ("11X", 4, [True, True, True, True]), # trailing X fills the rest True + ("00X", 4, [False, False, True, True]), # X spans the non-default GPUs + ("11X", 2, [True, True]), # X with nothing left to fill + ("0001", 3, [False, False, False]), # mask longer than host -> truncated + ], +) +def test_expand(mask, count, expected): + assert devices_mod._expand(mask, count) == expected diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index a2d570d9d6ef..6062c25d0400 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,10 +5,11 @@ import pytest import torch +from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -18,7 +19,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -55,7 +56,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -102,7 +103,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -114,7 +115,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index c2e5b3550816..e93e9af60b40 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,6 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils +from isaaclab.test.utils import test_devices DECIMAL_PRECISION = 5 """Precision of the test. @@ -22,7 +23,7 @@ """ -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_scale_unscale_transform(device, size): """Test scale_transform and unscale_transform.""" @@ -59,7 +60,7 @@ def test_scale_unscale_transform(device, size): torch.testing.assert_close(output_unscale_offset, inputs) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_saturate(device, size): "Test saturate of a tensor of differed shapes and device." @@ -81,7 +82,7 @@ def test_saturate(device, size): assert torch.all(torch.less_equal(output_per_batch, upper_per_batch)).item() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_normalize(device, size): """Test normalize of a tensor along its last dimension and check the norm of that dimension is close to 1.0.""" @@ -93,7 +94,7 @@ def test_normalize(device, size): torch.testing.assert_close(norm, torch.ones(size[0:-1], device=device)) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_copysign(device): """Test copysign by copying a sign from both a negative and positive value and verify that the new sign is the same. @@ -123,7 +124,7 @@ def test_copysign(device): torch.testing.assert_close(expected_value_neg_dim1_neg, value_neg_dim1_neg) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_is_identity_pose(device): """Test is_identity_pose method.""" # Single row identity pose (xyzw format) @@ -147,7 +148,7 @@ def test_is_identity_pose(device): assert math_utils.is_identity_pose(identity_pos, identity_rot) is False -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_axis_angle_from_quat(device): """Test axis_angle_from_quat method.""" # Quaternions of the form (2,4) and (2,2,4) in xyzw format @@ -171,7 +172,7 @@ def test_axis_angle_from_quat(device): torch.testing.assert_close(math_utils.axis_angle_from_quat(quat), angle) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_axis_angle_from_quat_approximation(device): """Test the Taylor approximation from axis_angle_from_quat method. @@ -197,7 +198,7 @@ def test_axis_angle_from_quat_approximation(device): torch.testing.assert_close(axis_angle_computed, axis_angle_expected) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_error_magnitude(device): """Test quat_error_magnitude method.""" # No rotation (xyzw format) @@ -236,7 +237,7 @@ def test_quat_error_magnitude(device): torch.testing.assert_close(q12_diff, expected_diff) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_unique(device): """Test quat_unique method.""" # Define test cases @@ -254,7 +255,7 @@ def test_quat_unique(device): torch.testing.assert_close(pos_real_quats[~non_pos_indices], quats[~non_pos_indices]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_mul_with_quat_unique(device): """Test quat_mul method with different quaternions. @@ -287,7 +288,7 @@ def test_quat_mul_with_quat_unique(device): torch.testing.assert_close(quat_result_3, quat_result_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_error_mag_with_quat_unique(device): """Test quat_error_magnitude method with positive real quaternions.""" @@ -310,7 +311,7 @@ def test_quat_error_mag_with_quat_unique(device): torch.testing.assert_close(error_4, error_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_convention_converter(device): """Test convert_camera_frame_orientation_convention to and from ros, opengl, and world conventions.""" # Quaternions in xyzw format (converted from original wxyz test values) @@ -348,7 +349,7 @@ def test_convention_converter(device): ) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("size", ((10, 4), (5, 3, 4))) def test_convert_quat(device, size): """Test convert_quat from "xyzw" to "wxyz" and back to "xyzw" and verify the correct rolling of the tensor. @@ -383,7 +384,7 @@ def test_convert_quat(device, size): math_utils.convert_quat(quat, to="xwyz") -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_conjugate(device): """Test quat_conjugate by checking the sign of the imaginary part changes but the magnitudes stay the same.""" @@ -397,7 +398,7 @@ def test_quat_conjugate(device): torch.testing.assert_close(expected_real, value[..., 3]) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("num_envs", (1, 10)) @pytest.mark.parametrize( "euler_angles", @@ -428,7 +429,7 @@ def test_quat_from_euler_xyz(device, num_envs, euler_angles): torch.testing.assert_close(expected_quat, quat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_wrap_to_pi(device): """Test wrap_to_pi method.""" # No wrapping needed @@ -464,7 +465,7 @@ def test_wrap_to_pi(device): torch.testing.assert_close(wrapped_angle, expected_angle) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("shape", ((3,), (1024, 3))) def test_skew_symmetric_matrix(device, shape): """Test skew_symmetric_matrix.""" @@ -494,7 +495,7 @@ def test_skew_symmetric_matrix(device, shape): torch.testing.assert_close(vec_rand_resized[:, 0], mat_value[:, 2, 1]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_orthogonalize_perspective_depth(device): """Test for converting perspective depth to orthogonal depth.""" # Create a sample perspective depth image (N, H, W) @@ -515,7 +516,7 @@ def test_orthogonalize_perspective_depth(device): torch.testing.assert_close(orthogonal_depth, expected_orthogonal_depth) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_combine_frame_transform(device): """Test combine_frame_transforms function.""" # create random poses @@ -540,7 +541,7 @@ def test_combine_frame_transform(device): torch.testing.assert_close(pose01, torch.cat((pos01, quat01), dim=-1)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4]) def test_interpolate_poses(device, seed): """Test interpolate_poses function. @@ -608,7 +609,7 @@ def test_pose_inv(): np.testing.assert_array_almost_equal(result, expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_to_and_from_angle_axis(device): """Test that axis_angle_from_quat against scipy and that quat_from_angle_axis are the inverse of each other.""" n = 1024 @@ -627,7 +628,7 @@ def test_quat_to_and_from_angle_axis(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_box_minus(device): """Test quat_box_minus method. @@ -645,7 +646,7 @@ def test_quat_box_minus(device): torch.testing.assert_close(expected_diff, axis_diff, atol=1e-06, rtol=1e-06) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_box_minus_and_quat_box_plus(device): """Test consistency of quat_box_plus and quat_box_minus. @@ -687,7 +688,7 @@ def test_quat_box_minus_and_quat_box_plus(device): torch.testing.assert_close(delta_result, delta_angle, atol=1e-04, rtol=1e-04) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("t12_inputs", ["True", "False"]) @pytest.mark.parametrize("q12_inputs", ["True", "False"]) def test_combine_frame_transforms(device, t12_inputs, q12_inputs): @@ -726,7 +727,7 @@ def test_combine_frame_transforms(device, t12_inputs, q12_inputs): torch.testing.assert_close(math_utils.quat_unique(expected_quat), math_utils.quat_unique(quat_value)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("t02_inputs", ["True", "False"]) @pytest.mark.parametrize("q02_inputs", ["True", "False"]) def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): @@ -766,7 +767,7 @@ def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): torch.testing.assert_close(math_utils.quat_unique(q02_expected), math_utils.quat_unique(q02_compare)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("rot_error_type", ("quat", "axis_angle")) def test_compute_pose_error(device, rot_error_type): """Test compute_pose_error for different rot_error_type.""" @@ -794,7 +795,7 @@ def test_compute_pose_error(device, rot_error_type): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_rigid_body_twist_transform(device): """Test rigid_body_twist_transform method. @@ -822,7 +823,7 @@ def test_rigid_body_twist_transform(device): torch.testing.assert_close(w_AA_, w_AA) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_yaw_quat(device): """ Test for yaw_quat methods. @@ -844,7 +845,7 @@ def test_yaw_quat(device): torch.testing.assert_close(result, expected_output) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_slerp(device): """Test quat_slerp function. @@ -874,7 +875,7 @@ def test_quat_slerp(device): np.testing.assert_array_almost_equal(result.cpu(), expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_matrix_from_quat(device): """test matrix_from_quat against scipy.""" # prepare random quaternions and vectors @@ -893,7 +894,7 @@ def test_matrix_from_quat(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize( "euler_angles", [ @@ -926,7 +927,7 @@ def test_matrix_from_euler(device, euler_angles, convention): torch.testing.assert_close(expected_mag, mat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_apply(device): """Test for quat_apply against scipy.""" # prepare random quaternions and vectors @@ -943,7 +944,7 @@ def test_quat_apply(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_apply_inverse(device): """Test for quat_apply against scipy.""" @@ -963,7 +964,7 @@ def test_quat_apply_inverse(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_inv(device): """Test for quat_inv method. @@ -1040,7 +1041,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1064,7 +1065,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) @@ -1315,7 +1316,7 @@ def test_euler_xyz_from_quat(): torch.testing.assert_close(output, wrapped) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): """Camera above target on +Z axis with Z-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, 5.0]], device=device) @@ -1327,7 +1328,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): """Camera at +Y looking at origin with Y-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 5.0, 0.0]], device=device) @@ -1339,7 +1340,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): """Camera below target looking up (-Z alignment with Z-up) should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, -5.0]], device=device) @@ -1351,7 +1352,7 @@ def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): """When eyes == targets the forward direction is undefined; all entries of the row are NaN.""" eyes = torch.tensor([[1.0, 2.0, 3.0]], device=device) @@ -1360,7 +1361,7 @@ def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_batched_partial_failure(device): """Mixed batch with one degenerate row should produce NaN in that row and a valid rotation in the other.""" eyes = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 5.0]], device=device) @@ -1371,7 +1372,7 @@ def test_create_rotation_matrix_from_view_batched_partial_failure(device): torch.testing.assert_close(torch.linalg.det(R[1]), torch.tensor(1.0, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_from_matrix_unit_norm_on_valid_input(device): """quat_from_matrix should produce unit quaternions for any valid rotation matrix.""" n = 100 @@ -1382,7 +1383,7 @@ def test_quat_from_matrix_unit_norm_on_valid_input(device): torch.testing.assert_close(norms, torch.ones(n, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_from_matrix_singular_matrix_returns_nan(device): """quat_from_matrix on a singular (non-rotation) matrix should signal NaN, not garbage.""" singular = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device) @@ -1390,7 +1391,7 @@ def test_quat_from_matrix_singular_matrix_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_standard(device): """Sanity: off-axis eye produces an orthonormal frame whose z-axis points from target back to eye.""" eyes = torch.tensor([[3.0, 0.0, 4.0]], device=device) @@ -1404,7 +1405,7 @@ def test_create_rotation_matrix_from_view_standard(device): torch.testing.assert_close(R[:, :, 2], expected_z, atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): """Non-finite input (NaN or Inf in eyes/targets) should produce NaN rows.""" eyes = torch.tensor([[float("nan"), 0.0, 0.0]], device=device) @@ -1413,7 +1414,7 @@ def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_from_matrix_reflection_returns_nan(device): """A reflection matrix (det = -1) is not a proper rotation; the safeguard should signal NaN.""" reflection = torch.diag(torch.tensor([1.0, 1.0, -1.0], device=device)).unsqueeze(0) @@ -1421,7 +1422,7 @@ def test_quat_from_matrix_reflection_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices("110")) def test_quat_from_matrix_non_orthonormal_returns_nan(device): """A non-orthonormal matrix (1% scale error on one axis) is not a valid rotation; expect NaN.""" R = torch.diag(torch.tensor([1.01, 1.0, 1.0], device=device)).unsqueeze(0) diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index 6e0e39820fdf..2be8d91015cb 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -9,6 +9,7 @@ import torch import isaaclab.utils.modifiers as modifiers +from isaaclab.test.utils import test_devices from isaaclab.utils.configclass import configclass @@ -142,7 +143,7 @@ def test_torch_relu_modifier(): assert torch.allclose(output, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_digital_filter(device): """Test digital filter modifier.""" # create test data @@ -178,7 +179,7 @@ def test_digital_filter(device): torch.testing.assert_close(processed_data, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_integral(device): """Test integral modifier.""" # create test data diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index 45f9d2d61384..c1035453b5a9 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,9 +7,10 @@ import torch import isaaclab.utils.noise as noise +from isaaclab.test.utils import test_devices -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -42,7 +43,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -77,7 +78,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab/test/utils/test_wrench_composer.py b/source/isaaclab/test/utils/test_wrench_composer.py index b711aaab44a6..2b13cf23a9f2 100644 --- a/source/isaaclab/test/utils/test_wrench_composer.py +++ b/source/isaaclab/test/utils/test_wrench_composer.py @@ -9,6 +9,7 @@ import warp as wp from isaaclab.test.mock_interfaces.assets import MockRigidObjectCollection +from isaaclab.test.utils import test_devices from isaaclab.utils.wrench_composer import WrenchComposer @@ -97,7 +98,7 @@ def random_unit_quaternion_np(rng: np.random.Generator, shape: tuple) -> np.ndar return q -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): @@ -136,7 +137,7 @@ def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_force_np, hand_calculated_composed_force_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int): @@ -175,7 +176,7 @@ def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int) assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): @@ -234,7 +235,7 @@ def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -280,7 +281,7 @@ def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -343,7 +344,7 @@ def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodi assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): @@ -392,7 +393,7 @@ def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -434,7 +435,7 @@ def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -476,7 +477,7 @@ def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: in ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies: int): @@ -540,7 +541,7 @@ def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_local_vs_global_identity_quaternion(device: str): """Test that local and global give same result with identity quaternion and zero position.""" rng = np.random.default_rng(seed=13) @@ -582,7 +583,7 @@ def test_local_vs_global_identity_quaternion(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_90_degree_rotation_global_force(device: str): """Test global force with a known 90-degree rotation for easy verification.""" num_envs, num_bodies = 1, 1 @@ -615,7 +616,7 @@ def test_90_degree_rotation_global_force(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composition_mixed_local_and_global(device: str): """Test that local and global forces can be composed together correctly.""" rng = np.random.default_rng(seed=14) @@ -660,7 +661,7 @@ def test_composition_mixed_local_and_global(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: int): @@ -705,7 +706,7 @@ def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: assert np.allclose(composed_torque_np, expected_torques, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_link_origin_no_torque(device: str): """Test that a global force applied at the link origin produces no torque.""" rng = np.random.default_rng(seed=16) @@ -753,7 +754,7 @@ def test_global_force_at_link_origin_no_torque(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): @@ -819,7 +820,7 @@ def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): ), "add_raw_buffers_from torque mismatch vs direct accumulation" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_add_raw_buffers_from_inactive_is_noop(device: str): """Test that add_raw_buffers_from is a no-op when the source composer is inactive.""" num_envs, num_bodies = 4, 2 @@ -853,7 +854,7 @@ def test_add_raw_buffers_from_inactive_is_noop(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): @@ -908,7 +909,7 @@ def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): ), f"Mask vs index torque mismatch (envs={num_envs}, bodies={num_bodies})" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): @@ -963,7 +964,7 @@ def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_overwrites_previous_add(device: str): """Test that set_forces_and_torques_index clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -992,7 +993,7 @@ def test_set_forces_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_index clears only the targeted environments.""" num_envs, num_bodies = 4, 3 @@ -1063,7 +1064,7 @@ def test_set_forces_clears_targeted_envs_only(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_partial_reset_zeros_only_specified_envs(device: str): """Test that partial reset zeros only the specified environments and leaves others intact.""" num_envs, num_bodies = 8, 3 @@ -1114,7 +1115,7 @@ def test_partial_reset_zeros_only_specified_envs(device: str): assert composer._dirty -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_full_reset_clears_active_flag(device: str): """Test that full reset (no args) clears the _active flag.""" num_envs, num_bodies = 4, 2 @@ -1138,7 +1139,7 @@ def test_full_reset_clears_active_flag(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_force_emits_deprecation_warning(device: str): """Test that accessing composed_force emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1158,7 +1159,7 @@ def test_composed_force_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_force_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_torque_emits_deprecation_warning(device: str): """Test that accessing composed_torque emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1177,7 +1178,7 @@ def test_composed_torque_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_torque_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_add_forces_and_torques_emits_warning(device: str): """Test that the deprecated add_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1202,7 +1203,7 @@ def test_deprecated_add_forces_and_torques_emits_warning(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_overwrites_previous_add(device: str): """Test that set_forces_and_torques_mask clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -1231,7 +1232,7 @@ def test_set_forces_mask_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_mask clears only the masked environments.""" num_envs, num_bodies = 4, 3 @@ -1301,7 +1302,7 @@ def test_set_forces_mask_clears_targeted_envs_only(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_matches_set_forces_index(device: str): """Test that set_forces_and_torques_mask produces the same result as the index variant.""" num_envs, num_bodies = 6, 3 @@ -1351,7 +1352,7 @@ def test_set_forces_mask_matches_set_forces_index(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_force_b_triggers_lazy_composition(device: str): """Test that accessing out_force_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1378,7 +1379,7 @@ def test_out_force_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_torque_b_triggers_lazy_composition(device: str): """Test that accessing out_torque_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1405,7 +1406,7 @@ def test_out_torque_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_lazy_composition_tracks_dirty_flag(device: str): """Test that the dirty flag is correctly managed through add/compose/add cycles.""" num_envs, num_bodies = 2, 1 @@ -1442,7 +1443,7 @@ def test_lazy_composition_tracks_dirty_flag(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), expected, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_compose_is_idempotent(device: str): """Calling compose_to_body_frame twice without intervening writes produces the same result.""" rng = np.random.default_rng(seed=456) @@ -1493,7 +1494,7 @@ def test_compose_is_idempotent(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_with_com_offset(device: str): """Test that torque correction uses CoM position, not link position, when they differ.""" num_envs, num_bodies = 2, 1 @@ -1548,7 +1549,7 @@ def test_global_force_with_com_offset(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_com_no_torque_with_com_offset(device: str): """Test that a global force at CoM position produces zero torque even with CoM offset.""" num_envs, num_bodies = 2, 1 @@ -1594,7 +1595,7 @@ def test_global_force_at_com_no_torque_with_com_offset(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_com_offset_with_rotation(device: str): """Test torque correction with both CoM offset and non-identity rotation.""" num_envs, num_bodies = 1, 1 @@ -1652,7 +1653,7 @@ def test_com_offset_with_rotation(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_emits_warning(device: str): """Test that the deprecated set_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1672,7 +1673,7 @@ def test_deprecated_set_forces_and_torques_emits_warning(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_clears_previous(device: str): """Test that deprecated set_forces_and_torques actually replaces previous values.""" num_envs, num_bodies = 4, 2 diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst new file mode 100644 index 000000000000..c53da690a8d8 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed Newton physics failing to initialize on non-default CUDA devices + (``cuda:1`` and higher). diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index d5991ed395ab..c334a77984b2 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -976,6 +976,15 @@ def start_simulation(cls) -> None: cls._cl_inject_sites_fallback() device = PhysicsManager._device + # Pin torch + Warp to the target device before any Warp/Newton + # allocations. Without this, mujoco_warp's collision pipeline later + # allocates on cuda:1 against a primary CUDA context that was never + # made current, returning null pointers (issue #5132). + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) logger.info(f"Finalizing model on device: {device}") cls._builder.up_axis = Axis.from_string(cls._up_axis) # Forward pending extended attribute requests to builder and clear them @@ -1243,6 +1252,16 @@ def initialize_solver(cls) -> None: if cfg is None: return + # Pin torch + Warp to the target device before solver build and + # collision-pipeline init. Mirrors the guard in :meth:`start_simulation` + # (issue #5132); idempotent if already pinned. + device = PhysicsManager._device + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) + with Timer(name="newton_initialize_solver", msg="Initialize solver took:"): NewtonManager._num_substeps = cfg.num_substeps # type: ignore[union-attr] NewtonManager._collision_decimation = cfg.collision_decimation # type: ignore[union-attr] @@ -1316,7 +1335,7 @@ def _capture_or_defer_graph(cls) -> None: with Timer(name="newton_cuda_graph", msg="CUDA graph took:"): if cls._usdrt_stage is None: simulate = cls._simulate_full if cls._is_all_graphable() else cls._simulate_physics_only - with wp.ScopedCapture() as capture: + with wp.ScopedCapture(device=device) as capture: simulate() NewtonManager._graph = capture.graph logger.info("Newton CUDA graph captured (standard Warp mode)") diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 8811de5b9db7..deed30bd5c53 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -121,7 +122,7 @@ def generate_cubes_scene( @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -156,7 +157,7 @@ def test_initialization(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="Newton does not support kinematic rigid bodies") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -194,7 +195,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -212,7 +213,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -229,7 +230,7 @@ def test_initialization_with_articulation_root(num_cubes, device): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -298,7 +299,7 @@ def test_external_force_buffer(device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -373,7 +374,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -462,7 +463,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -530,7 +531,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -573,7 +574,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object via view-level APIs.""" with _newton_sim_context(device, gravity_enabled=True, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -628,7 +629,7 @@ def _set_newton_material_properties(cube_object, friction_val, restitution_val, @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo contact at height=0 does not settle the same as PhysX — cube falls on z-axis") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -668,7 +669,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -681,7 +682,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo uses Coulomb friction (single mu), no static/dynamic distinction") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -761,7 +762,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo restitution model differs from PhysX — inelastic collisions still bounce") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -839,7 +840,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _newton_sim_context(device, gravity_enabled=False, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -877,7 +878,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -953,7 +954,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1066,7 +1067,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1137,7 +1138,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): @@ -1297,7 +1298,7 @@ def test_warmup_attach_stage_not_called_for_cpu(): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_link_pose_w_fresh_after_root_pose_write(device, writer): diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 2d92e3274a7e..66452b62d553 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -20,6 +20,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import math @@ -74,7 +76,7 @@ class ContactSensorTestSceneCfg(InteractiveSceneCfg): # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -193,7 +195,7 @@ def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: S assert no_contact_detected[env_idx], f"Env {env_idx}: Contact should stop after lift." -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -298,7 +300,7 @@ def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): """Test that resting object contact force equals weight and points upward. @@ -401,7 +403,7 @@ def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): assert not errs, "\n".join(errs) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contacts: bool): """Test that dropping from higher produces larger peak impact force. @@ -483,7 +485,7 @@ def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contac # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -618,7 +620,7 @@ def test_filter_enables_force_matrix(device: str, use_mujoco_contacts: bool): } -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -815,7 +817,7 @@ def _make_two_box_scene_cfg(num_envs: int) -> ContactSensorTestSceneCfg: return scene_cfg -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_sensor_metadata(device: str): """Verify sensor_names and filter_object_names match the underlying sensing and counterpart configuration across body-mode, body-mode-with-filter, and shape-mode. diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index d114a1da2a80..9ff6c7a8610a 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -13,6 +13,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) @@ -102,7 +104,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_body_path(device): """FrameView rejects prim paths that resolve to a Newton physics body.""" ctx = _sim_context(device, num_envs=2) @@ -116,7 +118,7 @@ def test_reject_body_path(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_shape_path(device): """FrameView rejects prim paths that resolve to a Newton collision shape.""" ctx = _sim_context(device, num_envs=2) @@ -183,7 +185,7 @@ def test_view_can_resolve_from_body_labels_after_reset(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_returns_initial_pose(device): """A world-rooted frame returns its configured position.""" ctx = _sim_context(device, num_envs=2) @@ -201,7 +203,7 @@ def test_world_attached_returns_initial_pose(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_set_world_roundtrip(device): """A world-attached prim can be repositioned via set_world_poses.""" ctx = _sim_context(device, num_envs=2) diff --git a/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index e4b052065127..15976d769b28 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,6 +56,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -328,7 +330,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): @@ -391,7 +393,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): @@ -455,7 +457,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -526,7 +528,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): @@ -598,7 +600,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -657,7 +659,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_OMNI_PHYSX_SCHEMAS_GAP_REASON, strict=False) @@ -725,7 +727,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): @@ -784,7 +786,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): @@ -815,7 +817,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -840,7 +842,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @@ -916,7 +918,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -949,7 +951,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -1034,7 +1036,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1092,7 +1094,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1187,7 +1189,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1247,7 +1249,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1341,7 +1343,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1403,7 +1405,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): @@ -1438,7 +1440,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1471,7 +1473,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1539,7 +1541,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.isaacsim_ci @@ -1593,7 +1595,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.isaacsim_ci @@ -1646,7 +1648,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.isaacsim_ci @@ -1708,7 +1710,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" @@ -1752,7 +1754,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @@ -1792,7 +1794,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci def test_body_root_state(sim, num_articulations, device, with_offset): @@ -1919,7 +1921,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2004,7 +2006,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, device): """Test the data.body_incoming_joint_wrench_b buffer is populated correctly and statically correct for single joint. @@ -2105,7 +2107,7 @@ def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, devic ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2124,7 +2126,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2143,7 +2145,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): @@ -2249,7 +2251,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2301,7 +2303,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2394,7 +2396,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 407cf4b41e22..2c0b5edceb5c 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,6 +32,8 @@ import warp as wp from flaky import flaky +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -175,7 +177,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -208,7 +210,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -245,7 +247,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -262,7 +264,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -278,7 +280,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -346,7 +348,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -423,7 +425,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -527,7 +529,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -592,7 +594,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -635,7 +637,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -644,7 +646,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -653,7 +655,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -662,7 +664,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -677,7 +679,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -691,7 +693,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -735,7 +737,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -774,7 +776,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -891,7 +893,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -963,7 +965,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 07ec860d6ec6..d0edfe94879c 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,6 +28,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -171,7 +173,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -200,7 +202,7 @@ def test_initialization(num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_id_conversion(device): """Test environment and object index conversion to physics view indices.""" @@ -240,7 +242,7 @@ def test_id_conversion(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -276,7 +278,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -291,7 +293,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" @@ -347,7 +349,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" @@ -409,7 +411,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -492,7 +494,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @@ -563,7 +565,7 @@ def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -663,7 +665,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravi @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -742,7 +744,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" @@ -778,7 +780,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -787,7 +789,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @@ -822,7 +824,7 @@ def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index af36a365cb66..b13e10d539bf 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -310,7 +311,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -366,7 +367,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -423,7 +424,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -487,7 +488,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -552,7 +553,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -605,7 +606,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base articulation made fixed-base using schema properties. @@ -665,7 +666,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -717,7 +718,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -747,7 +748,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -771,7 +772,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -846,7 +847,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -879,7 +880,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -963,7 +964,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1020,7 +1021,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1114,7 +1115,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1173,7 +1174,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1266,7 +1267,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1327,7 +1328,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1361,7 +1362,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1393,7 +1394,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1460,7 +1461,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1513,7 +1514,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1565,7 +1566,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1627,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1669,7 +1670,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1708,7 +1709,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1831,7 +1832,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1912,7 +1913,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1930,7 +1931,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1948,7 +1949,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2053,7 +2054,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2105,7 +2106,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2198,7 +2199,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) of articulation shapes.""" @@ -2245,7 +2246,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2262,7 +2263,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2289,7 +2290,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2314,7 +2315,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2366,7 +2367,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2402,7 +2403,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2453,7 +2454,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2487,7 +2488,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2576,7 +2577,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2634,7 +2635,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2798,7 +2799,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2832,7 +2833,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index b7f422c4f2f0..f55761b4bdad 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -98,7 +99,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -132,7 +133,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -170,7 +171,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -188,7 +189,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -205,7 +206,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -274,7 +275,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -350,7 +351,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -455,7 +456,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -521,7 +522,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -564,7 +565,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -605,7 +606,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -645,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -694,7 +695,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -705,7 +706,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -791,7 +792,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -874,7 +875,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -918,7 +919,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -958,7 +959,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -1069,7 +1070,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -1139,7 +1140,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index f42f6dfcb085..4ec56828af82 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -110,7 +111,7 @@ def sim(request): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(sim, num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -137,7 +138,7 @@ def test_initialization(sim, num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(sim, device): """Test environment and object index conversion to physics view indices.""" object_collection, _ = generate_cubes_scene(num_envs=2, num_cubes=3, device=device) @@ -175,7 +176,7 @@ def test_id_conversion(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" object_collection, origins = generate_cubes_scene( @@ -209,7 +210,7 @@ def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(sim, num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_cubes=num_cubes, has_api=False, device=device) @@ -222,7 +223,7 @@ def test_initialization_with_no_rigid_body(sim, num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -276,7 +277,7 @@ def test_external_force_buffer(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object.""" object_collection, origins = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -336,7 +337,7 @@ def test_external_force_on_single_body(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -415,7 +416,7 @@ def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, dev @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -480,7 +481,7 @@ def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -576,7 +577,7 @@ def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -656,7 +657,7 @@ def test_write_object_state(sim, num_envs, num_cubes, device, with_offset, state @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(sim, num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -689,7 +690,7 @@ def test_reset_object_collection(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(sim, num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -724,7 +725,7 @@ def test_set_material_properties(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -757,7 +758,7 @@ def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index 3cfe70095fd3..9a1c5e9e34ea 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,7 +10,6 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path @@ -30,6 +29,7 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 +from isaaclab.test.utils import test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) @@ -126,7 +126,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ------------------------------------------------------------------ -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.xfail( reason=( "Issue #5: FabricFrameView.set_world_poses writes to Fabric worldMatrix only. " @@ -155,7 +155,7 @@ def _fill_position(out: wp.array(dtype=wp.float32, ndim=2), x: float, y: float, out[i, 2] = wp.float32(z) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): """Verify that set_world_poses in Fabric mode does NOT sync back to USD. @@ -199,7 +199,7 @@ def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_rebuild_after_topology_change(device, view_factory, monkeypatch): """Forcing the topology-changed branch on a write triggers :meth:`_rebuild_fabric_arrays` and leaves the view in a state where @@ -252,11 +252,7 @@ def force_topology_changed(): # ------------------------------------------------------------------ -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -276,11 +272,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -308,11 +300,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """set_scales -> get_scales roundtrip works on cuda:1. diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..202415114478 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -5,7 +5,9 @@ import contextlib import os +import re import select +import shutil import signal import subprocess import sys @@ -145,6 +147,14 @@ def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, repo if kill_reason: pre_kill_diag = _capture_system_diagnostics() + # Capture stack traces of the hung process group before SIGKILL + # erases them. py-spy gives Python frames; gdb gives C++ frames + # inside Kit/PhysX binaries. Critical for diagnosing the + # upstream-tracked Kit shutdown hang (IsaacLab #3475 / OMPE-43816) + # since we can't read those binaries' source. + stack_diag = _capture_hang_stacks(process.pid, pgid, kill_reason) + if stack_diag: + pre_kill_diag = (pre_kill_diag + "\n\n" + stack_diag) if pre_kill_diag else stack_diag # Kill the entire process group (test + any Kit children). try: @@ -251,6 +261,80 @@ def _get_diagnostics(pre_kill_diag=""): return diag +def _capture_hang_stacks(pid: int, pgid: int, kill_reason: str) -> str: + """Capture Python and C++ stack traces of a hung process before SIGKILL. + + Used by the kill path to record where Kit was stuck when the + ``shutdown_hang`` / ``startup_hang`` / ``timeout`` deadline tripped. + Without this, SIGKILL erases the evidence and the upstream Kit hang + (IsaacLab #3475 / OMPE-43816) is opaque. + + Args: + pid: The hung test process pid. + pgid: Its process-group id; used to enumerate descendant pids + (Kit forks helper processes that may also be stuck). + kill_reason: Header to write into the captured block. + + Returns: + A string block ready to be appended to the diagnostic report. + Empty when neither py-spy nor gdb is available. + """ + sections = [f"--- hang stack capture ({kill_reason}, pid={pid}, pgid={pgid}) ---"] + + # Enumerate all pids in the process group; Kit spawns extension subprocesses + # and any of them may be the actual culprit. Cap to avoid extreme dumps. + pids = [pid] + try: + r = subprocess.run(["ps", "-o", "pid=", "-g", str(pgid)], capture_output=True, text=True, timeout=5) + if r.returncode == 0: + pids = [int(p) for p in r.stdout.split() if p.isdigit()][:8] + except Exception: + pass + + py_spy = shutil.which("py-spy") + gdb = shutil.which("gdb") + + for target_pid in pids: + sections.append(f"\n=== pid {target_pid} ===") + + # py-spy: Python frames. Needs PTRACE permissions; YAMA may block. + if py_spy: + try: + r = subprocess.run( + [py_spy, "dump", "--pid", str(target_pid)], + capture_output=True, + text=True, + timeout=10, + ) + tag = "py-spy dump" + sections.append(f"-- {tag} --\n{r.stdout or r.stderr.strip() or '(no output)'}") + except Exception as e: + sections.append(f"-- py-spy --- FAILED: {e}") + else: + sections.append("-- py-spy -- (not installed; pip install py-spy)") + + # gdb: C++ frames. Will print "thread apply all bt" for Kit/PhysX/CUDA + # threads. Needs the same PTRACE permissions. + if gdb: + try: + r = subprocess.run( + [gdb, "-batch", "-ex", "set pagination off", "-ex", "thread apply all bt", "-p", str(target_pid)], + capture_output=True, + text=True, + timeout=20, + ) + out = r.stdout.strip() + if len(out) > 8000: + out = out[:8000] + "\n... (truncated)" + sections.append(f"-- gdb thread apply all bt --\n{out or r.stderr.strip() or '(no output)'}") + except Exception as e: + sections.append(f"-- gdb --- FAILED: {e}") + else: + sections.append("-- gdb -- (not installed; apt-get install gdb)") + + return "\n".join(sections) + + def _capture_system_diagnostics(): """Capture system diagnostics (GPU, memory, processes) for crash investigation. @@ -312,6 +396,102 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) +def _slugify_test_path(test_path): + """Encode a test path as a flat queue entry name. + + The queue uses one file per pending test. Slashes are not legal inside a + filename, so we encode the relative path by replacing ``/`` with ``__``. + The decoder is :func:`_unslugify_queue_entry`. + """ + return test_path.replace("/", "__") + + +def _unslugify_queue_entry(entry_name): + """Reverse of :func:`_slugify_test_path`.""" + return entry_name.replace("__", "/") + + +def _claim_queued_file(queue_dir): + """Atomically claim one pending test from the work-queue directory. + + The queue is a directory of files (one per pending test); the shard claims + one by renaming it from ``queue/`` into its private ``inflight/cuda-N/``. + POSIX rename is atomic on the same filesystem, so two shards racing on the + same source file are serialized by the kernel: exactly one rename succeeds, + the other gets ``FileNotFoundError`` and tries the next entry. + + On success, the test's queue entry is now sitting in ``inflight/cuda-N/``; + the caller is expected to move it to ``done/cuda-N/`` after the per-test + pytest invocation exits with a clean result, leaving anything still in + ``inflight/`` at job-end as recoverable evidence of a crashed test. + + Args: + queue_dir: Path to the shared work-queue root. Must contain a + ``queue/`` subdir (pending entries) and an ``inflight//`` + subdir for this shard (claim destination). + + Returns: + The decoded test path for the claimed file, or ``None`` when the + queue is empty. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + pending_dir = os.path.join(queue_dir, "queue") + inflight_dir = os.path.join(queue_dir, "inflight", shard) + os.makedirs(inflight_dir, exist_ok=True) + + # Listdir is intentionally not cached: another shard may have just removed + # an entry we'd otherwise try. We pay one listdir per claim attempt; with + # N≤20 entries this is microseconds. + try: + entries = sorted(os.listdir(pending_dir)) + except FileNotFoundError: + return None + + for entry in entries: + src = os.path.join(pending_dir, entry) + dst = os.path.join(inflight_dir, entry) + try: + os.rename(src, dst) + except FileNotFoundError: + # Lost the race for this entry; another shard claimed it first. + # Continue to the next entry in our (potentially stale) listing. + continue + except OSError: + # Any other rename failure (e.g. permission) is a hard error. + raise + return _unslugify_queue_entry(entry) + + return None + + +def _mark_queued_file_done(queue_dir, test_path): + """Move a successfully-completed claim from ``inflight/cuda-N/`` to ``done/cuda-N/``. + + Called by the test runner after a per-file pytest invocation exits cleanly. + The inflight residual is what the post-run reconciler uses to detect + crashed shards: anything still in ``inflight/`` at job-end is an orphan. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + entry = _slugify_test_path(test_path) + src = os.path.join(queue_dir, "inflight", shard, entry) + dst_dir = os.path.join(queue_dir, "done", shard) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, entry) + # Suppress: already moved (idempotent) or the runner crashed before we + # could mark done — the reconciler catches the second case. + with contextlib.suppress(FileNotFoundError): + os.rename(src, dst) + + +def _queued_files(queue_dir): + """Yield files claimed from the shared work queue until it is empty.""" + while True: + claimed = _claim_queued_file(queue_dir) + if claimed is None: + return + yield claimed + + def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -401,13 +581,21 @@ def _retry_failed_test_in_fresh_process( def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" + """Run each test file separately, ensuring one finishes before starting the next. + + When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue file, files are claimed + from it (work-stealing across sibling shard containers) instead of iterating + ``test_files``; each file still runs once, on this container's pinned GPU. + """ failed_tests = [] test_status = {} xml_reports = [] cold_cache_applied = False - for test_file in test_files: + queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") + file_source = _queued_files(queue_path) if queue_path else test_files + + for test_file in file_source: print(f"\n\n🚀 Running {test_file} independently...\n") file_name = os.path.basename(test_file) env = os.environ.copy() @@ -433,14 +621,25 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + # Prefix the report file with a slug derived from the full test_file + # path so two concurrent shards running same-basename files (e.g. + # ``isaaclab_newton/.../test_articulation.py`` vs + # ``isaaclab_physx/.../test_articulation.py``) don't write to the same + # path inside the shared ``/workspace/isaaclab`` mount and trigger + # false shutdown_hang detections in sibling shards via the + # ``os.path.exists(report_file)`` check at line ~137. + report_slug = str(test_file).replace("/", "__").replace("\\", "__") + report_file = f"tests/test-reports-{report_slug}.xml" + cmd = [ sys.executable, "-m", "pytest", "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit "--no-header", f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", + f"--junitxml={report_file}", "--tb=short", ] @@ -450,8 +649,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): cmd.append(str(test_file)) - report_file = f"tests/test-reports-{str(file_name)}.xml" - # -- Run with retry on startup hang or hard timeout ----------------- returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" wall_time, pre_kill_diag = 0.0, "" @@ -677,6 +874,14 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "wall_time": wall_time, } + # When running under the directory-based work queue (option 2), move the + # claim entry from inflight// to done// so the post-run + # reconciler can distinguish "ran to completion" from "claimed but + # crashed mid-test". A claim that stays in inflight at job-end is a + # silent drop signal. + if queue_path: + _mark_queued_file_done(queue_path, test_file) + print("~~~~~~~~~~~~ Finished running all tests") return failed_tests, test_status, xml_reports @@ -845,6 +1050,10 @@ def pytest_sessionstart(session): # Run all tests individually failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) + # In work-queue mode this container ran only the files it claimed; report on those. + if os.environ.get("ISAACLAB_TEST_QUEUE"): + test_files = list(test_status) + print("failed tests:", failed_tests) # Collect reports @@ -861,6 +1070,10 @@ def pytest_sessionstart(session): # write content to full report result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") full_report_path = f"tests/{result_file}" + # Ensure the directory exists even when this shard claimed zero files + # from the work queue (per-test JUnit XMLs are what normally create + # ``tests/``; with no tests run there is nothing to create it). + os.makedirs("tests", exist_ok=True) print(f"Using result file: {result_file}") full_report.write(full_report_path) print("~~~~~~~~~~~~ Report written to", full_report_path) @@ -899,14 +1112,17 @@ def pytest_sessionstart(session): summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" + # GPU this run used (the shard's boot device); ``cuda:0`` when unset. + run_device = os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0" + summary_str += "\n\n=======================\n" - summary_str += "Per Test Result Summary\n" + summary_str += "Per File Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_test_result_table.align["Test Path"] = "l" - per_test_result_table.align["Test (s)"] = "r" - per_test_result_table.align["Wall (s)"] = "r" + per_file_result_table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + per_file_result_table.align["Test Path"] = "l" + per_file_result_table.align["Test (s)"] = "r" + per_file_result_table.align["Wall (s)"] = "r" for test_path in test_files: num_tests_passed = ( test_status[test_path]["tests"] @@ -914,9 +1130,10 @@ def pytest_sessionstart(session): - test_status[test_path]["errors"] - test_status[test_path]["skipped"] ) - per_test_result_table.add_row( + per_file_result_table.add_row( [ test_path, + run_device, test_status[test_path]["result"], f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", @@ -924,7 +1141,34 @@ def pytest_sessionstart(session): ] ) - summary_str += per_test_result_table.get_string() + summary_str += per_file_result_table.get_string() + + # Per-test run times, slowest first, from the merged JUnit report. The + # device is read from the test id params (e.g. ``...[size0-cuda:1]``), + # falling back to the run's boot device. + summary_str += "\n\n=================\n" + summary_str += "Per Test Run Time\n" + summary_str += "=================\n" + + per_test_time_table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + per_test_time_table.align["Test"] = "l" + per_test_time_table.align["Time (s)"] = "r" + test_times = [] + for suite in full_report: + for case in suite: + full_name = f"{case.classname}::{case.name}" if case.classname else case.name + device = run_device + bracket = re.search(r"\[(.*)\]", full_name) + if bracket: + dev_match = re.search(r"cuda:\d+|\bcpu\b", bracket.group(1)) + if dev_match: + device = dev_match.group(0) + elapsed = float(case.time) if case.time is not None else 0.0 + test_times.append((full_name, device, elapsed)) + for full_name, device, elapsed in sorted(test_times, key=lambda row: row[2], reverse=True): + per_test_time_table.add_row([full_name, device, f"{elapsed:0.3f}"]) + + summary_str += per_test_time_table.get_string() # Print summary to console and log file print(summary_str) diff --git a/tools/multi_gpu_shard_runner.sh b/tools/multi_gpu_shard_runner.sh new file mode 100755 index 000000000000..f8578b68d834 --- /dev/null +++ b/tools/multi_gpu_shard_runner.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest +# shards. Mounted by both the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) and the local probe (data/mgpu-1docker-probe/run.sh) so the +# logic lives in exactly one place and changes are version-controlled + +# shellcheck-able. +# +# Container expectations (set by the host launcher): +# --gpus all +# --user $host_uid:$host_gid +# -v :/workspace/isaaclab:rw +# -v :/mgpu:rw (subdirs queue/, inflight/, done/) +# -v :/shard-logs:rw (per-shard tee target) +# -e HOME=/tmp/mgpu-base-home (writable tmpfs for pip --user) +# -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) +# -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) +# -e TEST_INCLUDE_FILES="" +# -e ISAACLAB_PIN_KIT_GPU=1 (Kit overrides for the multi-GPU race) +# -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) +# +# Behavior: +# 1. Materializes HOME + PYTHONUSERBASE dirs (tmpfs, world-writable) +# 2. Installs pytest deps (junitparser et al.) into the shared PYTHONUSERBASE +# 3. Derives shard count from nvidia-smi -L (authoritative; torch.cuda.device_count +# under-counts MIG-on-same-parent unless CUDA_VISIBLE_DEVICES enumerates each) +# 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch +# can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) +# 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + +# ISAACLAB_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# /shard-logs/cuda-N.log for the host's grouped re-print after the run +# 6. Waits on every shard before aggregating exit codes — a fast failure doesn't +# tear down still-running siblings + +set +e +cd /workspace/isaaclab +unset HUB__ARGS__DETECT_ONLY DISPLAY + +# Container-level HOME + PYTHONUSERBASE for pip --user installs. The image +# runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so +# HOME defaults to /root which the user cannot write. /tmp/* is on tmpfs +# (1777, world-writable). +# +# PYTHONUSERBASE is the key for the 1-docker shape: pip --user writes to +# ${PYTHONUSERBASE}/lib/python3.12/site-packages, and every Python invocation +# that sees the same env var imports from there. Per-shard subshells below +# override HOME (so .cache / .nvidia-omniverse are isolated) but inherit +# PYTHONUSERBASE so junitparser et al. resolve everywhere. +mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase + +# Pytest deps (same as run-tests action). junitparser is imported at +# tools/conftest.py load time, so it must be present first. +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky py-spy "coverage>=7.6.1" + +# Shard count from nvidia-smi -L (truth; torch under-counts MIG). +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") +if [ "$MIG_COUNT" -gt 0 ]; then + DEV_COUNT=$MIG_COUNT + echo "::notice::container: MIG mode, $MIG_COUNT slices" +else + DEV_COUNT=$GPU_COUNT + echo "::notice::container: discrete mode, $GPU_COUNT GPUs" +fi +if [ "$DEV_COUNT" -lt 2 ]; then + echo "::error::Need at least 2 visible devices; found $DEV_COUNT" + exit 1 +fi + +# Cross-check with torch and cap shard count to what torch can actually +# address. Guards against a CUDA_VISIBLE_DEVICES misconfig silently fanning +# out shards that crash on device access. +TORCH_COUNT=$(/isaac-sim/python.sh -c "import torch; print(torch.cuda.device_count())") +echo "container: torch sees $TORCH_COUNT cuda devices (cross-check vs $DEV_COUNT)" +if [ "$TORCH_COUNT" -lt "$DEV_COUNT" ]; then + echo "::warning::torch sees fewer devices than nvidia-smi — capping shards to $TORCH_COUNT" + DEV_COUNT=$TORCH_COUNT +fi + +# Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME +# (per-shard isolation for .cache, .local/share, etc.) and per-shard +# ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +declare -A pids +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do + zeros="" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done + runtime_devices="${zeros}1" + + shard_home="/tmp/isaaclab-ci-home-${cuda}" + mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ + "${shard_home}/.nvidia-omniverse/config" \ + "${shard_home}/.nvidia-omniverse/logs" + + shard_log="/shard-logs/cuda-${cuda}.log" + + ( + export HOME="$shard_home" + export XDG_CACHE_HOME="${HOME}/.cache" + export XDG_DATA_HOME="${HOME}/.local/share" + export ISAACLAB_TEST_DEVICES="$runtime_devices" + export ISAACLAB_SIM_DEVICE="cuda:${cuda}" + + ./isaaclab.sh -p -m pytest \ + --ignore=tools/conftest.py \ + --ignore=source/isaaclab/test/install_ci \ + tools -v 2>&1 \ + | tee "$shard_log" \ + | stdbuf -oL sed "s/^/[cuda:${cuda}] /" + + exit "${PIPESTATUS[0]}" + ) & + pids[$cuda]=$! + echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" +done + +# Wait for every shard before aggregating exits — a fast failure must not +# tear down still-running siblings. +declare -A results +for cuda in "${!pids[@]}"; do + wait "${pids[$cuda]}" + results[$cuda]=$? +done + +fail=0 +for cuda in "${!results[@]}"; do + rc="${results[$cuda]}" + echo "shard cuda:${cuda} exited $rc" + [ "$rc" -eq 0 ] || fail=1 +done +exit $fail