diff --git a/.github/actions/multi-gpu/aggregate_test_summary.py b/.github/actions/multi-gpu/aggregate_test_summary.py new file mode 100755 index 000000000000..6e1bec25f2f7 --- /dev/null +++ b/.github/actions/multi-gpu/aggregate_test_summary.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate per-shard JUnit XML reports from the multi-GPU pytest lane. + +Reads ``RUNTIME_DIR`` (the host launcher's work-queue root, holding +``queue/done//``) and the ``tests/test-reports-*.xml`` JUnit files, +then prints a per-file + per-shard table to stdout and, when +``GITHUB_STEP_SUMMARY`` is set, writes the same content as markdown so it +renders at the top of the run page. +""" + +import glob +import os +import pathlib +import xml.etree.ElementTree as ET + +runtime_dir = os.environ.get("RUNTIME_DIR", "") +done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + +# Map slug -> shard from done//; slug uses "__" for "/". +slug_to_shard = {} +if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + +def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-") :] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + +# Per-file rows from JUnit XMLs. +rows = [] +for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, failures + errors, total_time)) + +# Per-shard aggregate. +shards = {} +for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + +# ---------- Output: plain text to stdout ---------- +def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + +out = [] +out.append("PER-FILE RESULTS") +out.append("=" * 110) +hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] +widths = [60, 7, 8, 12, 5, 5, 8] +out.append(fmt_row(hdr, widths)) +out.append("-" * 110) +for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) +out.append("") +out.append("PER-SHARD AGGREGATE") +out.append("=" * 80) +shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] +shard_widths = [10, 7, 12, 9, 8, 14] +out.append(fmt_row(shard_hdr, shard_widths)) +out.append("-" * 80) +grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} +for sh in sorted(shards): + s = shards[sh] + for k in grand: + grand[k] += s[k] + out.append( + fmt_row( + [ + sh, + str(s["files"]), + f"{s['passed']}/{s['tests']}", + str(s["skipped"]), + str(s["failed"]), + f"{s['time']:.1f}", + ], + shard_widths, + ) + ) +out.append("-" * 80) +out.append( + fmt_row( + [ + "TOTAL", + str(grand["files"]), + f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), + str(grand["failed"]), + f"{grand['time']:.1f}", + ], + shard_widths, + ) +) +print("\n".join(out)) + +# ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- +summary_path = os.environ.get("GITHUB_STEP_SUMMARY") +if summary_path: + md = [ + "## Multi-GPU CI: Aggregated Test Summary", + "", + "### Per-shard totals", + "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|", + ] + for sh in sorted(shards): + s = shards[sh] + md.append( + f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |" + ) + md.append( + f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | " + f"**{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |" + ) + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = { + "PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR", + }.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py new file mode 100644 index 000000000000..38d1ac18827c --- /dev/null +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -0,0 +1,92 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""pytest plugin: select which tests a non-default GPU shard runs. + +Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via +``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this +directory to ``PYTHONPATH`` so it is importable). It lives next to the lane +scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. + +Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME +env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's +device parametrization, so this plugin's keep/drop decision agrees with what was +parametrized. Kit-backed tests derive their AppLauncher device from the same mask. +The plugin reads the mask string directly -- no isaaclab import -- matching the +lib-free convention of every other conftest in the repo. The only shared knowledge +is the mask grammar below, mirrored from ``devices.py`` (position 0 = cpu, +position k = cuda:(k-1), trailing ``X`` = include all remaining); keep the two in sync. + +Selection rule, on a shard whose mask excludes cpu + cuda:0: + + keep a test iff it is parametrized over ``device`` AND that variant's device + is active in the mask; deselect everything else. + +That single rule is both criteria at once: candidate = device-parametrized (a +test with no ``device`` param is device-agnostic, covered by single-GPU CI on +cuda:0); scope = only this shard's device variant runs (cpu / cuda:0 / other-index +variants are dropped). Out-of-scope items are deselected (not skipped) so the +shard report shows only what ran. +""" + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +def _position(device): + """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None # not device-parametrized, or an unrecognized device value + + +def _active(position, mask): + """Whether a mask position is included. Trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + +def _shard_mask(): + """Return the runtime mask iff this is a multi-GPU shard run, else None. + + A shard targets non-default GPUs only, so cpu (position 0) and cuda:0 + (position 1) are excluded. Anything else -- unset, or single-GPU CI's default + ``"110"`` -- is not a shard and the plugin is a no-op. + """ + mask = os.environ.get(_RUNTIME_ENV, "") + return mask if mask and not _active(0, mask) and not _active(1, mask) else None + + +def pytest_collection_modifyitems(config, items): + mask = _shard_mask() + if mask is None: + return + keep, drop = [], [] + for item in items: + callspec = getattr(item, "callspec", None) + device = callspec.params.get("device") if callspec is not None else None + if _active(_position(device), mask): + keep.append(item) # device-parametrized AND in this shard's scope + else: + drop.append(item) # non-device, or cpu / cuda:0 / other-index variant + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + +def pytest_sessionfinish(session, exitstatus): + # A file whose tests are all out of scope deselects to zero, so pytest exits + # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file + # exit as a failure (tools/conftest.py), so report "nothing in scope for this + # file" as success rather than a false failure. + if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh new file mode 100755 index 000000000000..ea6bdfc719b7 --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Runs ON THE HOST runner — the counterpart to multi_gpu_shard_runner.sh, which +# runs INSIDE the container. Launches ONE container that hosts all N pytest +# shards as parallel subshells, then reconciles the shared work queue. +# +# Invoked by .github/workflows/test-multi-gpu-pytest.yaml's "Run shards in +# parallel" step. Reads from the environment (set by that step): +# IMAGE_TAG — per-commit CI image to ``docker run`` +# INCLUDE_FILES — comma-sep test basenames (conftest include filter) +# PATHS — comma-sep relative test paths (seed the work queue) +# plus the runner built-ins GITHUB_RUN_ID / GITHUB_RUN_ATTEMPT (container name) +# and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). +# +# 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its +# own non-default cuda:N via ISAACLAB_TEST_DEVICES, pulling from the shared +# queue). Collapsing the previous N-container layout into one +# removes cross-container races on the workspace mount (``_isaac_sim`` symlink, +# ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at +# the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, +# set in the shard runner) keeps user-level caches isolated. +# +# Scaling: shard count is derived inside the container from ``nvidia-smi -L`` +# (truth — torch.cuda.device_count() under-counts MIG slices on the same parent +# GPU), so this works unchanged on any GPU count/topology. CUDA_VISIBLE_DEVICES +# is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete +# runners rely on ``--gpus all``. + +set +e # keep going on errors so the reconciler below always runs and reports +host_uid="$(id -u)" +host_gid="$(id -g)" +host_user="$(id -un)" +# ``/isaac-sim`` is the Kit install root inside the container; pytest +# invocations under ``./isaaclab.sh -p`` resolve the bundled Python as +# ``./_isaac_sim/python.sh``. Mounting the workspace at ``/workspace/isaaclab`` +# brings the host's CWD into the container, so we plant a host-side symlink +# whose target resolves only inside the container. Single creator (this step) +# — no concurrent creators. +rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + +# unique per-run temp dir (RUNNER_TEMP if set, else /tmp; XXXXXX -> random suffix) +runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" +mkdir -p "$runtime_dir" + +# Directory-based work queue (atomic os.rename pattern; conftest claims entries +# via os.rename(queue/X, inflight//X) — POSIX rename is atomic on the +# same filesystem, so two shards racing on the same entry are kernel-serialized: +# one wins, the other gets ENOENT and moves on). Each pending test path is +# materialized as an empty file in queue/ with '/' slugged to '__' so it's a +# flat filename. +# +# Subdirs: +# queue/ — pending claims +# inflight// — claimed by this shard, test in flight +# done// — test exited cleanly +# +# At job-end the reconciler below asserts queue/ + inflight/ are empty — if +# either is non-empty the job FAILS (silent-drop signal: never claimed OR +# claimed but crashed mid-test). +queue_root="$runtime_dir/queue" +mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" +IFS=',' read -ra _paths <<< "$PATHS" # split the comma-separated PATHS into the _paths array +for path in "${_paths[@]}"; do + [ -n "$path" ] || continue # skip empty entries (PATHS has a trailing comma) + slug="${path//\//__}" # flatten to a filename: replace every '/' with '__' + : > "$queue_root/queue/$slug" # create an empty marker file (':' = no-op, '>' creates it) +done +echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + +# Per-shard log dir mounted from the container so we can grouped-print the +# per-shard stream after the run completes. +logs_dir="$runtime_dir/logs" +mkdir -p "$logs_dir" + +# MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. nvidia-smi -L +# is authoritative — torch.cuda.device_count() under-counts MIG slices on the +# same parent GPU. +cvd_args=() +if nvidia-smi -L | grep -q "^ MIG "; then + # read each MIG slice's UUID into the MIGS array (one element per line) + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") # join the array into a comma-separated string + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") # stays empty on discrete hosts -> no flag added + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" +else + echo "::notice::discrete GPU mode — relying on --gpus all" +fi + +# "${cvd_args[@]}" expands to the CUDA_VISIBLE_DEVICES flag on MIG hosts, nothing otherwise. +docker run --rm --gpus all --network=host \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_FABRIC_USE_GPU_INTEROP=0 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh +docker_rc=$? # $? = exit code of the docker run above + +# Grouped re-print of per-shard logs (preserved across the host). +for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) # extract N from "cuda-N.log" + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" +done + +# Reconciler — silent-drop guard. Anything left in queue/ was never claimed by +# any shard (work-queue starvation). Anything still in inflight// was +# claimed but never moved to done/ (shard crashed mid-test). Either is a +# silent-drop signal that the docker exit alone can't surface. Fail the job and +# name the orphans. +echo "::group::work-queue reconciler" +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) # files still pending (never claimed) +if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent for the log +fi +orphans=0 +for shard_dir in "$queue_root/inflight"/*/; do # each inflight// dir (the /*/ matches dirs only) + [ -d "$shard_dir" ] || continue # skip if the glob matched nothing + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent + orphans=$((orphans + count)) # $(( )) = integer arithmetic + fi +done +done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) +echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" +echo "::endgroup::" + +# Treat reconciler signals as fatal even if all shards exited 0. +if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 # clean docker exit but drops found -> force a non-zero code +fi + +# Export the runtime directory so the downstream "Aggregated test summary" step +# can locate the done// work distribution and the per-shard logs. +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" # export to later workflow steps via GitHub's env file + +exit "$docker_rc" diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh new file mode 100755 index 000000000000..6eafae80c0be --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest +# shards. Mounted by the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) so the logic lives in exactly one place and changes are +# version-controlled + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` +# so it sits next to the workflow that consumes it rather than alongside ``tools/``. +# +# Container expectations (set by the host launcher): +# --gpus all +# --user $host_uid:$host_gid +# -v :/workspace/isaaclab:rw +# -v :/mgpu:rw (subdirs queue/, inflight/, done/) +# -v :/shard-logs:rw (per-shard tee target) +# -e HOME=/tmp/mgpu-base-home (writable tmpfs for pip --user) +# -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) +# -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) +# -e TEST_INCLUDE_FILES="" +# -e ISAACLAB_FABRIC_USE_GPU_INTEROP=0 (temporary Kit/PhysX CI workaround) +# -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) +# +# Behavior: +# 1. Materializes HOME + PYTHONUSERBASE dirs (tmpfs, world-writable) +# 2. Installs pytest deps (junitparser et al.) into the shared PYTHONUSERBASE +# 3. Derives shard count from nvidia-smi -L (authoritative; torch.cuda.device_count +# under-counts MIG-on-same-parent unless CUDA_VISIBLE_DEVICES enumerates each) +# 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch +# can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) +# 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + +# ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# /shard-logs/cuda-N.log for the host's grouped re-print after the run +# 6. Waits on every shard before aggregating exit codes — a fast failure doesn't +# tear down still-running siblings + +set +e # keep going on errors; per-shard exit codes are aggregated at the end +cd /workspace/isaaclab +unset HUB__ARGS__DETECT_ONLY DISPLAY # clear vars that would force Kit into detect-only / headed (X11) mode + +# Container-level HOME + PYTHONUSERBASE for pip --user installs. The image +# runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so +# HOME defaults to /root which the user cannot write. /tmp/* is on tmpfs +# (1777, world-writable). +# +# PYTHONUSERBASE is the key for the 1-docker shape: pip --user writes to +# ${PYTHONUSERBASE}/lib/python3.12/site-packages, and every Python invocation +# that sees the same env var imports from there. Per-shard subshells below +# override HOME (so .cache / .nvidia-omniverse are isolated) but inherit +# PYTHONUSERBASE so junitparser et al. resolve everywhere. +mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase + +# Pytest deps (same as run-tests action). junitparser is imported at +# tools/conftest.py load time, so it must be present first. +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky "coverage>=7.6.1" + +# Shard count from nvidia-smi -L (truth; torch under-counts MIG). +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") # grep -c = count of matching lines (MIG slices) +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") # count of whole GPUs +if [ "$MIG_COUNT" -gt 0 ]; then + DEV_COUNT=$MIG_COUNT + echo "::notice::container: MIG mode, $MIG_COUNT slices" +else + DEV_COUNT=$GPU_COUNT + echo "::notice::container: discrete mode, $GPU_COUNT GPUs" +fi +if [ "$DEV_COUNT" -lt 2 ]; then + echo "::error::Need at least 2 visible devices; found $DEV_COUNT" + exit 1 +fi + +# Cross-check with torch and cap shard count to what torch can actually +# address. Guards against a CUDA_VISIBLE_DEVICES misconfig silently fanning +# out shards that crash on device access. +TORCH_COUNT=$(/isaac-sim/python.sh -c "import torch; print(torch.cuda.device_count())") +echo "container: torch sees $TORCH_COUNT cuda devices (cross-check vs $DEV_COUNT)" +if [ "$TORCH_COUNT" -lt "$DEV_COUNT" ]; then + echo "::warning::torch sees fewer devices than nvidia-smi — capping shards to $TORCH_COUNT" + DEV_COUNT=$TORCH_COUNT +fi + +# Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME +# (per-shard isolation for .cache, .local/share, etc.) and per-shard +# ISAACLAB_TEST_DEVICES. +declare -A pids # associative array: shard index -> background PID +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) + zeros="" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done # build the leading zeros of the device mask + runtime_devices="${zeros}1" # e.g. cuda:2 -> "0001": only this GPU active in the ISAACLAB_TEST_DEVICES mask + + shard_home="/tmp/isaaclab-ci-home-${cuda}" + mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ + "${shard_home}/.nvidia-omniverse/config" \ + "${shard_home}/.nvidia-omniverse/logs" + + shard_log="/shard-logs/cuda-${cuda}.log" + + # each shard runs in its own subshell, backgrounded with '&' below, so all shards run in parallel + ( + export HOME="$shard_home" + export XDG_CACHE_HOME="${HOME}/.cache" + export XDG_DATA_HOME="${HOME}/.local/share" + export ISAACLAB_TEST_DEVICES="$runtime_devices" + + # Full pytest output captures to $shard_log; live stdout is filtered + # down to high-signal lines (test boundaries, failures, summary stats, + # tracebacks). Kit init chatter, plugin registration, omni.usd Transfer + # logs, etc. only end up in the shard logfile, NOT on the live workflow + # log. The workflow's grouped re-print at end of run still shows the + # full $shard_log under a collapsible ``::group::shard cuda:N log``. + # (tee = full output to the log file; stdbuf -oL = flush per line so the + # filtered grep/sed stream appears live, not in delayed chunks.) + ./isaaclab.sh -p -m pytest \ + --ignore=tools/conftest.py \ + --ignore=source/isaaclab/test/install_ci \ + tools -v 2>&1 \ + | tee "$shard_log" \ + | stdbuf -oL grep -aE \ + '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ + | stdbuf -oL sed "s/^/[cuda:${cuda}] /" + + exit "${PIPESTATUS[0]}" # exit with pytest's code, not tee/grep/sed's (PIPESTATUS[0] = first pipe stage) + ) & + pids[$cuda]=$! # $! = PID of the subshell just backgrounded + echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" +done + +# Wait for every shard before aggregating exits — a fast failure must not +# tear down still-running siblings. +declare -A results # associative array: shard index -> exit code +for cuda in "${!pids[@]}"; do # "${!pids[@]}" = the array's keys (the shard indices) + wait "${pids[$cuda]}" # block until that shard's PID exits + results[$cuda]=$? # $? = the waited shard's exit code +done + +fail=0 +for cuda in "${!results[@]}"; do + rc="${results[$cuda]}" + echo "shard cuda:${cuda} exited $rc" + [ "$rc" -eq 0 ] || fail=1 # any non-zero shard makes the whole run fail +done +exit $fail diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml new file mode 100644 index 000000000000..63c03a73779a --- /dev/null +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -0,0 +1,188 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Multi-GPU unit-test workflow +# +# Runs the non-default-GPU subset of unit tests across the multi-GPU runner +# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the +# single-GPU CI; only diffs are the runner label and the env vars that pin +# Kit and the test parametrize to the shard's non-default GPU. +# +# Adding a new test to multi-GPU coverage: give its device parametrize a +# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()`` +# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow +# auto-discovers it. + +name: Multi-GPU pytest + +on: + pull_request: + paths: + - "source/isaaclab/isaaclab/test/utils/**" + - "source/isaaclab/isaaclab/app/app_launcher.py" + - "source/**/test/**/test_*.py" + - ".github/workflows/test-multi-gpu-pytest.yaml" + - ".github/actions/run-package-tests/**" + - ".github/actions/run-tests/**" + - ".github/actions/ecr-build-push-pull/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + + build: + name: Build / cache image + needs: [config] + # Must run on the SAME pool as the test job. The ECR cache repo is resolved + # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU + # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]` + # the image would land in gitci-docker-cache, which the multi-GPU test job + # cannot see, so it would rebuild from scratch on the scarce multi-GPU + # runner. Building here populates multigpu-docker-cache so the test job's + # pull hits. + runs-on: [self-hosted, linux, x64, multi-gpu] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + # Pre-populates the ECR exact-commit tag from deps-cache (registry-side + # alias). Without this prior step, run-package-tests' internal + # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves + # no local image, causing `docker run` to fail with `pull access + # denied`. Mirrors the build → test split in build.yaml. + - uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + test-multi-gpu-pytest: + name: Multi-GPU unit tests + needs: [config, build] + if: needs.build.result == 'success' + # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here: + # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the + # job onto a 1-GPU box and the shard runner aborts with "Need at least 2 + # visible devices; found 1". + runs-on: [self-hosted, linux, x64, multi-gpu] + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Discover opt-in test files + # Auto-discovery: any test_*.py with a non-default-capable scope — an + # argless ``test_devices()``, a named scope containing non-default GPUs, + # or a string mask with a trailing ``X`` — is in scope. Adding a test to + # multi-GPU CI needs no workflow edit; opting a file out is just narrowing + # its scope to ``DeviceScope.CPU_AND_DEFAULT_CUDA`` (or mask ``"110"``). + # + # Within a discovered file, tests that are NOT parametrized over the + # ``device`` argument are deselected at collection time by the + # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): + # single-GPU CI already covers them on ``cuda:0`` and re-running on every + # non-default shard adds wall-time without surfacing any new failure mode. + id: discover + run: | + # File-level opt-out: a test file can exclude itself from multi-GPU CI + # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line. + # Used for files with known Kit/Isaac-Sim concurrency issues; the file + # still runs in single-GPU CI. Excluded files are reported as a notice + # below for visibility. No workflow edit needed to add/remove a file. + device_scope_pattern='test_devices\(\)|test_devices\([^)]*DeviceScope\.(ALL|CUDA|NON_DEFAULT_CUDA)|test_devices\("[^"]*X"\)' + mapfile -t candidates < <(grep -rlE "$device_scope_pattern" source/ --include='test_*.py' | sort -u) + discovered=() + skipped=() + for f in "${candidates[@]}"; do + if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then + skipped+=("$f") + else + discovered+=("$f") + fi + done + for f in "${skipped[@]}"; do + reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//') + echo "::notice::multi-GPU skipped: $f — $reason" + done + if [ ${#discovered[@]} -eq 0 ]; then + echo "::error::No tests with a non-default-capable device scope were discovered" + exit 1 + fi + basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) + echo "include=$basenames" >> "$GITHUB_OUTPUT" + # Full relative paths too, to seed the shared work queue (the run step + # needs runnable paths, not just basenames). + echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT" + echo "::notice::Discovered ${#discovered[@]} opt-in test files" + printf ' %s\n' "${discovered[@]}" + + - name: Pull image from ECR + # Pulls the per-commit image the build job pushed. ecr-build-push-pull + # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which + # the build job's deps-cache-hit registry-tag created for this SHA) + # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our + # parallel ``docker run`` block below. + uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run shards in parallel on local GPUs (1-docker N-shard) + # ONE container hosts all N pytest shards as parallel subshells, each + # pinned to its own non-default cuda:N and pulling from a shared work + # queue; the host reconciles the queue afterward and exports + # MGPU_RUNTIME_DIR for the summary step. Full rationale + the + # 1-docker-N-shard tradeoffs live in the script header. + env: + IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} + INCLUDE_FILES: ${{ steps.discover.outputs.include }} + PATHS: ${{ steps.discover.outputs.paths }} + run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh + + - name: Aggregated test summary + # Per-shard + per-file pass/total/walltime, plus a combined table, also + # written to $GITHUB_STEP_SUMMARY. Runs on success or failure. + if: always() + env: + RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} + run: python3 .github/actions/multi-gpu/aggregate_test_summary.py diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst new file mode 100644 index 000000000000..9c3d1ac84352 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -0,0 +1,10 @@ +Added +^^^^^ + +* Added :class:`isaaclab.test.utils.DeviceScope`, + :func:`isaaclab.test.utils.test_devices`, and + :func:`isaaclab.test.utils.resolve_test_sim_device` to parametrize unit tests + and launch Kit from the same runtime device mask. Composable scopes cover + common cases, string masks support custom combinations, and multi-GPU CI + narrows the runtime to one non-default GPU per shard without changing + single-GPU CI behavior. diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py new file mode 100644 index 000000000000..c9631a080592 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test-time helpers for Isaac Lab. + +Exposes :class:`DeviceScope` and :func:`test_devices` for selecting the device +list to parametrize tests over, plus :func:`resolve_test_sim_device` for deriving +a Kit-backed test's AppLauncher device from the same runtime mask. The selected +set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the devices the +test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env var (the +devices the run may use). +""" + +from .devices import DeviceScope, resolve_test_sim_device, test_devices + +__all__ = ["DeviceScope", "resolve_test_sim_device", "test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py new file mode 100644 index 000000000000..9f5db8a609f4 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -0,0 +1,255 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Device selection for parametrizing tests over cpu / cuda devices. + +Intended use:: + + from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices + + simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app + + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) + def test_foo(device): ... + +Two inputs, two roles +--------------------- +A test runs on ``scope ∩ runtime``: + +* **scope** — a :class:`DeviceScope` or mask declaring the devices the *test* + is valid on. The author owns this; defaults to :attr:`DeviceScope.ALL`. +* **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* + may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). + The multi-GPU workflow sets it to one device per shard. Kit-backed tests use + :func:`resolve_test_sim_device` to derive their AppLauncher boot device from + the same mask. + +A test author never names the shard's GPU; the operator never inspects a +test's device support. The helper intersects the two. + +Mask grammar +------------ +Positions left to right: ``0`` = cpu, ``1`` = cuda:0 (the default GPU), +``2`` = cuda:1, ``3`` = cuda:2, ... Each position is ``0`` (exclude) or ``1`` +(include). A trailing ``X`` means "include all the remaining devices". + +cpu, cuda:0, and a non-default GPU stay distinct by position — running on +cuda:0 says nothing about cuda:1+, which is the whole reason this exists. + +Common scopes +------------- +==================================== ========= ============================================= +Named scope Mask Meaning +==================================== ========= ============================================= +``DeviceScope.ALL`` ``11X`` cpu + every GPU +``DeviceScope.CUDA`` ``01X`` every GPU +``DeviceScope.CPU_AND_DEFAULT_CUDA`` ``110`` cpu + cuda:0 +``DeviceScope.DEFAULT_CUDA`` ``010`` cuda:0 only +``DeviceScope.NON_DEFAULT_CUDA`` ``00X`` cuda:1 and above +``DeviceScope.CPU`` ``100`` cpu only +==================================== ========= ============================================= + +Named scopes cover the common cases; string masks remain supported for custom +device combinations. + +Worked example — a ``scope="11X"`` (default) test: + +* single-GPU CI (runtime unset ⇒ ``"110"``) ⇒ ``[cpu, cuda:0]``. +* a multi-GPU shard (runtime ``"0001"``, one device) ⇒ ``[cuda:2]``. + +So argless tests run on cpu + cuda:0 in single-GPU CI and on exactly one +non-default GPU per shard in multi-GPU CI (run-once is a property of the +one-device-per-shard runtime plus round-robin file assignment). + +An empty result means the test is cleanly skipped for this run (e.g. a +``"00X"`` test on a single-GPU host, or a ``"110"`` test on a non-default-GPU +shard). But if the run *explicitly* set ``ISAACLAB_TEST_DEVICES`` to devices +the host does not have, that is a misconfigured run and raises instead of +skipping everything and reporting a vacuous green. + +Local runs +---------- +Set the runtime from the shell to opt a run into non-default GPUs:: + + ISAACLAB_TEST_DEVICES=0001 ./isaaclab.sh -p -m pytest path/to/test.py +""" + +from __future__ import annotations + +import os +from enum import Flag, auto + +_RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" +"""Env var naming the run's devices: the devices a run may use (see module docstring).""" + +_DEFAULT_RUNTIME = "110" +"""Runtime devices when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset: cpu + cuda:0, +i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" + + +class DeviceScope(Flag): + """Composable device scopes for test parametrization. + + String masks remain accepted by :func:`test_devices` for custom device + combinations that are not represented here. + """ + + CPU = auto() + DEFAULT_CUDA = auto() + NON_DEFAULT_CUDA = auto() + + CPU_AND_DEFAULT_CUDA = CPU | DEFAULT_CUDA + CUDA = DEFAULT_CUDA | NON_DEFAULT_CUDA + ALL = CPU | CUDA + + @property + def mask(self) -> str: + """Return this scope in the device-mask representation.""" + return "".join( + [ + "1" if self & DeviceScope.CPU else "0", + "1" if self & DeviceScope.DEFAULT_CUDA else "0", + "X" if self & DeviceScope.NON_DEFAULT_CUDA else "0", + ] + ) + + +def resolve_test_sim_device() -> str: + """Resolve the AppLauncher device from the test runtime device mask. + + This function intentionally parses the mask without enumerating devices so + it can run before AppLauncher without importing torch or initializing Warp. + A Kit-backed test process can boot on only one GPU, so an explicitly set + runtime must select at most one CUDA device. The CPU bit may accompany that + GPU because the default single-GPU runtime covers both CPU and CUDA tests. + + Returns: + The selected ``"cuda:N"`` device, ``"cpu"`` for a CPU-only runtime, + or ``"cuda:0"`` when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset. + + Raises: + ValueError: When the runtime mask is invalid, selects no device, uses a + wildcard, or selects multiple CUDA devices. + """ + runtime = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + if runtime is None: + return "cuda:0" + if not runtime or any(char not in "01X" for char in runtime) or "X" in runtime[:-1]: + raise ValueError(f"Invalid {_RUNTIME_DEVICES_ENV_VAR} mask: {runtime!r}") + if runtime.endswith("X"): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} is ambiguous for AppLauncher; use a concrete runtime mask" + ) + + cuda_positions = [position for position, included in enumerate(runtime[1:], start=1) if included == "1"] + if len(cuda_positions) > 1: + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} selects multiple CUDA devices; " + "Kit-backed tests require one GPU per process" + ) + if cuda_positions: + return f"cuda:{cuda_positions[0] - 1}" + if runtime[0] == "1": + return "cpu" + raise ValueError(f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} selects no device") + + +def test_devices(scope: str | DeviceScope = DeviceScope.ALL, *, skip: dict[str, str] | None = None) -> list: + """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. + + ``scope`` is this argument; the runtime comes from the + ``ISAACLAB_TEST_DEVICES`` env var (see the module docstring for the grammar + and the scope / runtime split). + + Args: + scope: Named scope or device mask (e.g. ``"11X"``) the test is valid + on. Defaults to :attr:`DeviceScope.ALL`, so device-agnostic call + sites can use ``test_devices()`` with no argument. + skip: Optional ``{device: reason}``; in-scope devices listed here are + wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with + the reason instead of being dropped. Use to gate a device variant + that is known broken while keeping it visible. + + Returns: + Ordered device entries (``"cpu"`` / ``"cuda:N"`` strings, or + :func:`pytest.param` for skipped ones) for the second argument to + :func:`pytest.mark.parametrize`. Empty means the test is skipped for + this run. + + Raises: + ValueError: When the run explicitly set ``ISAACLAB_TEST_DEVICES`` to + devices that are not available on this host (a misconfigured run that + would otherwise skip everything and pass vacuously). A scope that + merely does not intersect this run's devices skips, it does not raise. + """ + requested = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + available = _list_available_devices() + runtime_keep = _expand(requested or _DEFAULT_RUNTIME, len(available)) + # A run that explicitly named devices the host cannot provide (e.g. a shard + # pinned to cuda:2 on a 2-GPU host) is misconfigured: fail loudly rather than + # skip every test and report a vacuous green. A scope that simply does not + # intersect this run's devices (a "110" test on a non-default-GPU shard) is + # not an error — it skips. + if requested is not None and not any(runtime_keep): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" + ) + scope_keep = _expand(_scope_mask(scope), len(available)) + devices = [ + device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime + ] + if not skip: + return devices + import pytest + + return [ + pytest.param(device, marks=pytest.mark.skip(reason=skip[device])) if device in skip else device + for device in devices + ] + + +# Prevent pytest from collecting this imported ``test_``-prefixed helper. +test_devices.__test__ = False + + +def _expand(mask: str, count: int) -> list[bool]: + """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. + + Args: + mask: A mask string (positions plus an optional trailing ``X``). + count: The number of available devices to expand to. + + Returns: + A list of ``count`` booleans, one per device position. + """ + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return ([char == "1" for char in body] + [fill] * count)[:count] + + +def _scope_mask(scope: str | DeviceScope) -> str: + """Convert a named device scope to its mask representation.""" + return scope if isinstance(scope, str) else scope.mask + + +def _list_available_devices() -> list[str]: + """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. + + Returns: + Ordered list of device strings as torch addresses them. + """ + # Keep the import local so importing this test helper before AppLauncher + # does not import torch (and its transitive native libraries) before Kit. + import torch + + devices = ["cpu"] + # torch.cuda (not Warp) is deliberate: warp.get_cuda_devices() initializes + # the Warp runtime, but non-default-GPU runs must select cuda:N before + # wp.init(). Enumeration here must not perturb the initialization order that + # this suite validates (#5132). + if torch.cuda.is_available(): + devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) + return devices diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 65a96259060f..34de268685d3 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -6,9 +6,10 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -45,7 +46,7 @@ def test_setup_teardown(): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_init(device): """Test the simulation context initialization.""" from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index c99593dff50a..40eec36f094c 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -11,8 +11,9 @@ """ from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app import pytest # noqa: E402 import torch # noqa: E402 @@ -101,7 +102,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_toggle(device): """Test toggling visibility multiple times.""" if device == "cuda" and not torch.cuda.is_available(): @@ -129,7 +130,7 @@ def test_visibility_toggle(device): assert vis[0] and not vis[1] and vis[2] -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_parent_inheritance(device): """Making a parent invisible hides all children.""" if device == "cuda" and not torch.cuda.is_available(): @@ -155,7 +156,7 @@ def test_visibility_parent_inheritance(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_prim_ordering_follows_creation_order(device): """Prims are returned in USD creation order (DFS), not alphabetical.""" if device == "cuda" and not torch.cuda.is_available(): @@ -181,7 +182,7 @@ def test_prim_ordering_follows_creation_order(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_standardize_transform_op(device): """FrameView standardizes a prim with xformOp:transform to translate/orient/scale.""" if device == "cuda" and not torch.cuda.is_available(): @@ -209,7 +210,7 @@ def test_standardize_transform_op(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_nested_hierarchy_world_poses(device): """World pose of nested child == sum of parent + child translations.""" if device == "cuda" and not torch.cuda.is_available(): @@ -334,7 +335,7 @@ def test_compare_get_world_poses_with_isaacsim(): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_with_franka_robots(device): """Verify FrameView works with real Franka robot USD assets.""" if device == "cuda" and not torch.cuda.is_available(): diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py new file mode 100644 index 000000000000..46d592c7f33b --- /dev/null +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -0,0 +1,205 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the test-device selection helpers. + +These tests mock the host's device list, so they need no GPU and run on the +single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) +so the multi-GPU workflow's auto-discovery does not mistake this file for an +opted-in device test. The helper itself sets ``__test__ = False`` to prevent +pytest from collecting it when imported under its public name. +""" + +import pytest + +from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import DeviceScope, resolve_test_sim_device +from isaaclab.test.utils.devices import test_devices as resolve_devices + +# Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). +SINGLE_GPU = ["cpu", "cuda:0"] +MULTI_GPU = ["cpu", "cuda:0", "cuda:1", "cuda:2"] + + +@pytest.fixture +def host(monkeypatch): + """Return a setter that pins the available device list and the runtime env var.""" + + def _set(available: list[str], runtime: str | None = None) -> None: + monkeypatch.setattr(devices_mod, "_list_available_devices", lambda: available) + if runtime is None: + monkeypatch.delenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, raising=False) + else: + monkeypatch.setenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, runtime) + + return _set + + +# --------------------------------------------------------------------------- +# scope ∩ runtime resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "available, runtime, scope, expected", + [ + # single-GPU CI lane: runtime unset -> default "110" (cpu + cuda:0). + (SINGLE_GPU, None, "11X", ["cpu", "cuda:0"]), # argless default is device-agnostic + (SINGLE_GPU, None, "110", ["cpu", "cuda:0"]), # pure-math scope + (SINGLE_GPU, None, "100", ["cpu"]), # cpu-only scope + (SINGLE_GPU, None, "00X", []), # non-default-only -> nothing here, skip + # multi-GPU shard: runtime pins exactly one non-default GPU. + (MULTI_GPU, "001", "11X", ["cuda:1"]), # cuda:1 shard + (MULTI_GPU, "0001", "11X", ["cuda:2"]), # cuda:2 shard + (MULTI_GPU, "0001", "00X", ["cuda:2"]), # non-default regression on its shard + (MULTI_GPU, "0001", "110", []), # math skips the shard (cuda:0 != this shard) + (MULTI_GPU, "0001", "100", []), # cpu test skips the shard + # a runtime that lists several GPUs hands back all in-scope ones. + (MULTI_GPU, "111", "11X", ["cpu", "cuda:0", "cuda:1"]), + ], +) +def test_resolves_scope_intersect_runtime(host, available, runtime, scope, expected): + host(available, runtime) + assert resolve_devices(scope) == expected + + +def test_argless_equals_default_scope(host): + # The common case: argless must equal the explicit default mask everywhere. + for available, runtime in [(SINGLE_GPU, None), (MULTI_GPU, "0001"), (MULTI_GPU, "001")]: + host(available, runtime) + assert resolve_devices() == resolve_devices("11X") + + +def test_argless_runs_once_on_one_non_default_gpu(host): + # Run-once property: with a one-device-per-shard runtime, an argless test + # resolves to exactly one non-default GPU. + host(MULTI_GPU, "0001") + result = resolve_devices() + assert result == ["cuda:2"] + assert all(d not in ("cpu", "cuda:0") for d in result) + + +@pytest.mark.parametrize( + "scope, mask", + [ + (DeviceScope.ALL, "11X"), + (DeviceScope.CUDA, "01X"), + (DeviceScope.CPU_AND_DEFAULT_CUDA, "110"), + (DeviceScope.NON_DEFAULT_CUDA, "00X"), + (DeviceScope.CPU, "100"), + (DeviceScope.DEFAULT_CUDA, "010"), + (DeviceScope.CPU | DeviceScope.NON_DEFAULT_CUDA, "10X"), + ], +) +def test_named_scope_matches_mask(host, scope, mask): + host(MULTI_GPU, "1111") + assert scope.mask == mask + assert resolve_devices(scope) == resolve_devices(mask) + + +def test_named_combination_matches_runtime_composition(): + assert DeviceScope.CPU_AND_DEFAULT_CUDA == DeviceScope.CPU | DeviceScope.DEFAULT_CUDA + assert DeviceScope.CUDA == DeviceScope.DEFAULT_CUDA | DeviceScope.NON_DEFAULT_CUDA + assert DeviceScope.ALL == DeviceScope.CPU | DeviceScope.DEFAULT_CUDA | DeviceScope.NON_DEFAULT_CUDA + + +def test_helper_is_not_collected_by_pytest(): + assert resolve_devices.__test__ is False + + +# --------------------------------------------------------------------------- +# AppLauncher device resolution from the runtime mask +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "runtime, expected", + [ + (None, "cuda:0"), + ("110", "cuda:0"), + ("100", "cpu"), + ("001", "cuda:1"), + ("0001", "cuda:2"), + ("101", "cuda:1"), + ], +) +def test_resolve_test_sim_device(host, runtime, expected): + host(MULTI_GPU, runtime) + assert resolve_test_sim_device() == expected + + +@pytest.mark.parametrize("runtime", ["", "000", "011", "01X", "0X1", "0A1"]) +def test_resolve_test_sim_device_rejects_invalid_or_ambiguous_runtime(host, runtime): + host(MULTI_GPU, runtime) + with pytest.raises(ValueError, match="ISAACLAB_TEST_DEVICES"): + resolve_test_sim_device() + + +# --------------------------------------------------------------------------- +# skip vs raise: legitimate skips never raise; only a missing runtime device does +# --------------------------------------------------------------------------- + + +def test_unset_runtime_never_raises(host): + # On a dev box / single-GPU lane (runtime unset), an out-of-scope result is a + # silent skip, not an error. + host(SINGLE_GPU, None) + assert resolve_devices("00X") == [] + + +def test_runtime_naming_absent_device_raises(host): + # A run that explicitly asked for cuda:2 on a host that only has cuda:0/cuda:1 + # is misconfigured -> fail loudly instead of a vacuous green. + host(["cpu", "cuda:0", "cuda:1"], "0001") + with pytest.raises(ValueError, match="no device available"): + resolve_devices("11X") + + +def test_in_range_runtime_with_empty_scope_does_not_raise(host): + # The runtime device exists (cuda:2 is present), the scope just doesn't include + # it -> skip, not raise. This is what keeps "110"/"100" tests green on a shard. + host(MULTI_GPU, "0001") + assert resolve_devices("110") == [] # would raise if the guard keyed off scope + + +# --------------------------------------------------------------------------- +# skip= : gate a specific device visibly +# --------------------------------------------------------------------------- + + +def test_skip_wraps_named_device_as_skipped_param(host): + host(SINGLE_GPU, None) + result = resolve_devices("11X", skip={"cuda:0": "known broken"}) + assert result[0] == "cpu" # untouched devices stay plain strings + param = result[1] + assert param.values == ("cuda:0",) + assert param.marks[0].name == "skip" + assert param.marks[0].kwargs["reason"] == "known broken" + + +def test_skip_ignores_out_of_result_devices(host): + # Skipping a device that isn't in the resolved set is a no-op (no stray params). + host(SINGLE_GPU, None) + assert resolve_devices("11X", skip={"cuda:3": "n/a"}) == ["cpu", "cuda:0"] + + +# --------------------------------------------------------------------------- +# _expand mask grammar +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mask, count, expected", + [ + ("110", 2, [True, True]), # exact length, no wildcard + ("100", 3, [True, False, False]), # short mask padded with False + ("11X", 4, [True, True, True, True]), # trailing X fills the rest True + ("00X", 4, [False, False, True, True]), # X spans the non-default GPUs + ("11X", 2, [True, True]), # X with nothing left to fill + ("0001", 3, [False, False, False]), # mask longer than host -> truncated + ], +) +def test_expand(mask, count, expected): + assert devices_mod._expand(mask, count) == expected diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index c767521ceb86..0a6359a0f2c2 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,12 +5,13 @@ import pytest import torch +from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils.datasets import EpisodeData pytestmark = pytest.mark.unit -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -20,7 +21,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -57,7 +58,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -104,7 +105,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -116,7 +117,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index 90bba67cbdaa..4070c6da1140 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,6 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils +from isaaclab.test.utils import DeviceScope, test_devices pytestmark = pytest.mark.unit @@ -24,7 +25,7 @@ """ -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_scale_unscale_transform(device, size): """Test scale_transform and unscale_transform.""" @@ -61,7 +62,7 @@ def test_scale_unscale_transform(device, size): torch.testing.assert_close(output_unscale_offset, inputs) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_saturate(device, size): "Test saturate of a tensor of differed shapes and device." @@ -83,7 +84,7 @@ def test_saturate(device, size): assert torch.all(torch.less_equal(output_per_batch, upper_per_batch)).item() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_normalize(device, size): """Test normalize of a tensor along its last dimension and check the norm of that dimension is close to 1.0.""" @@ -95,7 +96,7 @@ def test_normalize(device, size): torch.testing.assert_close(norm, torch.ones(size[0:-1], device=device)) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_copysign(device): """Test copysign by copying a sign from both a negative and positive value and verify that the new sign is the same. @@ -125,7 +126,7 @@ def test_copysign(device): torch.testing.assert_close(expected_value_neg_dim1_neg, value_neg_dim1_neg) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_is_identity_pose(device): """Test is_identity_pose method.""" # Single row identity pose (xyzw format) @@ -149,7 +150,7 @@ def test_is_identity_pose(device): assert math_utils.is_identity_pose(identity_pos, identity_rot) is False -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat(device): """Test axis_angle_from_quat method.""" # Quaternions of the form (2,4) and (2,2,4) in xyzw format @@ -173,7 +174,7 @@ def test_axis_angle_from_quat(device): torch.testing.assert_close(math_utils.axis_angle_from_quat(quat), angle) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat_approximation(device): """Test the Taylor approximation from axis_angle_from_quat method. @@ -199,7 +200,7 @@ def test_axis_angle_from_quat_approximation(device): torch.testing.assert_close(axis_angle_computed, axis_angle_expected) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_magnitude(device): """Test quat_error_magnitude method.""" # No rotation (xyzw format) @@ -238,7 +239,7 @@ def test_quat_error_magnitude(device): torch.testing.assert_close(q12_diff, expected_diff) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_unique(device): """Test quat_unique method.""" # Define test cases @@ -256,7 +257,7 @@ def test_quat_unique(device): torch.testing.assert_close(pos_real_quats[~non_pos_indices], quats[~non_pos_indices]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_mul_with_quat_unique(device): """Test quat_mul method with different quaternions. @@ -289,7 +290,7 @@ def test_quat_mul_with_quat_unique(device): torch.testing.assert_close(quat_result_3, quat_result_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_mag_with_quat_unique(device): """Test quat_error_magnitude method with positive real quaternions.""" @@ -312,7 +313,7 @@ def test_quat_error_mag_with_quat_unique(device): torch.testing.assert_close(error_4, error_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_convention_converter(device): """Test convert_camera_frame_orientation_convention to and from ros, opengl, and world conventions.""" # Quaternions in xyzw format (converted from original wxyz test values) @@ -350,7 +351,7 @@ def test_convention_converter(device): ) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((10, 4), (5, 3, 4))) def test_convert_quat(device, size): """Test convert_quat from "xyzw" to "wxyz" and back to "xyzw" and verify the correct rolling of the tensor. @@ -385,7 +386,7 @@ def test_convert_quat(device, size): math_utils.convert_quat(quat, to="xwyz") -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_conjugate(device): """Test quat_conjugate by checking the sign of the imaginary part changes but the magnitudes stay the same.""" @@ -399,7 +400,7 @@ def test_quat_conjugate(device): torch.testing.assert_close(expected_real, value[..., 3]) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", (1, 10)) @pytest.mark.parametrize( "euler_angles", @@ -430,7 +431,7 @@ def test_quat_from_euler_xyz(device, num_envs, euler_angles): torch.testing.assert_close(expected_quat, quat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_wrap_to_pi(device): """Test wrap_to_pi method.""" # No wrapping needed @@ -466,7 +467,7 @@ def test_wrap_to_pi(device): torch.testing.assert_close(wrapped_angle, expected_angle) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("shape", ((3,), (1024, 3))) def test_skew_symmetric_matrix(device, shape): """Test skew_symmetric_matrix.""" @@ -496,7 +497,7 @@ def test_skew_symmetric_matrix(device, shape): torch.testing.assert_close(vec_rand_resized[:, 0], mat_value[:, 2, 1]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_orthogonalize_perspective_depth(device): """Test for converting perspective depth to orthogonal depth.""" # Create a sample perspective depth image (N, H, W) @@ -517,7 +518,7 @@ def test_orthogonalize_perspective_depth(device): torch.testing.assert_close(orthogonal_depth, expected_orthogonal_depth) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_combine_frame_transform(device): """Test combine_frame_transforms function.""" # create random poses @@ -542,7 +543,7 @@ def test_combine_frame_transform(device): torch.testing.assert_close(pose01, torch.cat((pos01, quat01), dim=-1)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4]) def test_interpolate_poses(device, seed): """Test interpolate_poses function. @@ -610,7 +611,7 @@ def test_pose_inv(): np.testing.assert_array_almost_equal(result, expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_to_and_from_angle_axis(device): """Test that axis_angle_from_quat against scipy and that quat_from_angle_axis are the inverse of each other.""" n = 1024 @@ -629,7 +630,7 @@ def test_quat_to_and_from_angle_axis(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus(device): """Test quat_box_minus method. @@ -647,7 +648,7 @@ def test_quat_box_minus(device): torch.testing.assert_close(expected_diff, axis_diff, atol=1e-06, rtol=1e-06) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus_and_quat_box_plus(device): """Test consistency of quat_box_plus and quat_box_minus. @@ -689,7 +690,7 @@ def test_quat_box_minus_and_quat_box_plus(device): torch.testing.assert_close(delta_result, delta_angle, atol=1e-04, rtol=1e-04) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t12_inputs", ["True", "False"]) @pytest.mark.parametrize("q12_inputs", ["True", "False"]) def test_combine_frame_transforms(device, t12_inputs, q12_inputs): @@ -728,7 +729,7 @@ def test_combine_frame_transforms(device, t12_inputs, q12_inputs): torch.testing.assert_close(math_utils.quat_unique(expected_quat), math_utils.quat_unique(quat_value)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t02_inputs", ["True", "False"]) @pytest.mark.parametrize("q02_inputs", ["True", "False"]) def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): @@ -768,7 +769,7 @@ def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): torch.testing.assert_close(math_utils.quat_unique(q02_expected), math_utils.quat_unique(q02_compare)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("rot_error_type", ("quat", "axis_angle")) def test_compute_pose_error(device, rot_error_type): """Test compute_pose_error for different rot_error_type.""" @@ -796,7 +797,7 @@ def test_compute_pose_error(device, rot_error_type): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_twist_transform(device): """Test rigid_body_twist_transform method. @@ -824,7 +825,7 @@ def test_rigid_body_twist_transform(device): torch.testing.assert_close(w_AA_, w_AA) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_yaw_quat(device): """ Test for yaw_quat methods. @@ -846,7 +847,7 @@ def test_yaw_quat(device): torch.testing.assert_close(result, expected_output) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_slerp(device): """Test quat_slerp function. @@ -876,7 +877,7 @@ def test_quat_slerp(device): np.testing.assert_array_almost_equal(result.cpu(), expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_matrix_from_quat(device): """test matrix_from_quat against scipy.""" # prepare random quaternions and vectors @@ -895,7 +896,7 @@ def test_matrix_from_quat(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "euler_angles", [ @@ -928,7 +929,7 @@ def test_matrix_from_euler(device, euler_angles, convention): torch.testing.assert_close(expected_mag, mat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply(device): """Test for quat_apply against scipy.""" # prepare random quaternions and vectors @@ -945,7 +946,7 @@ def test_quat_apply(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply_inverse(device): """Test for quat_apply against scipy.""" @@ -965,7 +966,7 @@ def test_quat_apply_inverse(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_inv(device): """Test for quat_inv method. @@ -1042,7 +1043,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1066,7 +1067,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) @@ -1317,7 +1318,7 @@ def test_euler_xyz_from_quat(): torch.testing.assert_close(output, wrapped) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): """Camera above target on +Z axis with Z-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, 5.0]], device=device) @@ -1329,7 +1330,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): """Camera at +Y looking at origin with Y-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 5.0, 0.0]], device=device) @@ -1341,7 +1342,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): """Camera below target looking up (-Z alignment with Z-up) should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, -5.0]], device=device) @@ -1353,7 +1354,7 @@ def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): """When eyes == targets the forward direction is undefined; all entries of the row are NaN.""" eyes = torch.tensor([[1.0, 2.0, 3.0]], device=device) @@ -1362,7 +1363,7 @@ def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_batched_partial_failure(device): """Mixed batch with one degenerate row should produce NaN in that row and a valid rotation in the other.""" eyes = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 5.0]], device=device) @@ -1373,7 +1374,7 @@ def test_create_rotation_matrix_from_view_batched_partial_failure(device): torch.testing.assert_close(torch.linalg.det(R[1]), torch.tensor(1.0, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_unit_norm_on_valid_input(device): """quat_from_matrix should produce unit quaternions for any valid rotation matrix.""" n = 100 @@ -1384,7 +1385,7 @@ def test_quat_from_matrix_unit_norm_on_valid_input(device): torch.testing.assert_close(norms, torch.ones(n, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_singular_matrix_returns_nan(device): """quat_from_matrix on a singular (non-rotation) matrix should signal NaN, not garbage.""" singular = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device) @@ -1392,7 +1393,7 @@ def test_quat_from_matrix_singular_matrix_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_standard(device): """Sanity: off-axis eye produces an orthonormal frame whose z-axis points from target back to eye.""" eyes = torch.tensor([[3.0, 0.0, 4.0]], device=device) @@ -1406,7 +1407,7 @@ def test_create_rotation_matrix_from_view_standard(device): torch.testing.assert_close(R[:, :, 2], expected_z, atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): """Non-finite input (NaN or Inf in eyes/targets) should produce NaN rows.""" eyes = torch.tensor([[float("nan"), 0.0, 0.0]], device=device) @@ -1415,7 +1416,7 @@ def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_reflection_returns_nan(device): """A reflection matrix (det = -1) is not a proper rotation; the safeguard should signal NaN.""" reflection = torch.diag(torch.tensor([1.0, 1.0, -1.0], device=device)).unsqueeze(0) @@ -1423,7 +1424,7 @@ def test_quat_from_matrix_reflection_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_non_orthonormal_returns_nan(device): """A non-orthonormal matrix (1% scale error on one axis) is not a valid rotation; expect NaN.""" R = torch.diag(torch.tensor([1.01, 1.0, 1.0], device=device)).unsqueeze(0) diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index a4f5212a9fda..8624b00b8a08 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -9,6 +9,7 @@ import torch import isaaclab.utils.modifiers as modifiers +from isaaclab.test.utils import test_devices from isaaclab.utils.configclass import configclass pytestmark = pytest.mark.unit @@ -144,7 +145,7 @@ def test_torch_relu_modifier(): assert torch.allclose(output, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_digital_filter(device): """Test digital filter modifier.""" # create test data @@ -180,7 +181,7 @@ def test_digital_filter(device): torch.testing.assert_close(processed_data, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_integral(device): """Test integral modifier.""" # create test data diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index 2d9b836cd422..f80a05e8abad 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,11 +7,12 @@ import torch import isaaclab.utils.noise as noise +from isaaclab.test.utils import DeviceScope, test_devices pytestmark = pytest.mark.unit -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -44,7 +45,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -79,7 +80,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab/test/utils/test_wrench_composer.py b/source/isaaclab/test/utils/test_wrench_composer.py index 1d3da72b7c7b..4e537b1ce0eb 100644 --- a/source/isaaclab/test/utils/test_wrench_composer.py +++ b/source/isaaclab/test/utils/test_wrench_composer.py @@ -9,6 +9,7 @@ import warp as wp from isaaclab.test.mock_interfaces.assets import MockRigidObjectCollection +from isaaclab.test.utils import test_devices from isaaclab.utils.wrench_composer import WrenchComposer pytestmark = pytest.mark.unit @@ -99,7 +100,7 @@ def random_unit_quaternion_np(rng: np.random.Generator, shape: tuple) -> np.ndar return q -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): @@ -138,7 +139,7 @@ def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_force_np, hand_calculated_composed_force_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int): @@ -177,7 +178,7 @@ def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int) assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): @@ -236,7 +237,7 @@ def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -282,7 +283,7 @@ def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -345,7 +346,7 @@ def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodi assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): @@ -394,7 +395,7 @@ def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -436,7 +437,7 @@ def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -478,7 +479,7 @@ def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: in ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies: int): @@ -542,7 +543,7 @@ def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_local_vs_global_identity_quaternion(device: str): """Test that local and global give same result with identity quaternion and zero position.""" rng = np.random.default_rng(seed=13) @@ -584,7 +585,7 @@ def test_local_vs_global_identity_quaternion(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_90_degree_rotation_global_force(device: str): """Test global force with a known 90-degree rotation for easy verification.""" num_envs, num_bodies = 1, 1 @@ -617,7 +618,7 @@ def test_90_degree_rotation_global_force(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composition_mixed_local_and_global(device: str): """Test that local and global forces can be composed together correctly.""" rng = np.random.default_rng(seed=14) @@ -662,7 +663,7 @@ def test_composition_mixed_local_and_global(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: int): @@ -707,7 +708,7 @@ def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: assert np.allclose(composed_torque_np, expected_torques, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_link_origin_no_torque(device: str): """Test that a global force applied at the link origin produces no torque.""" rng = np.random.default_rng(seed=16) @@ -755,7 +756,7 @@ def test_global_force_at_link_origin_no_torque(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): @@ -821,7 +822,7 @@ def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): ), "add_raw_buffers_from torque mismatch vs direct accumulation" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_add_raw_buffers_from_inactive_is_noop(device: str): """Test that add_raw_buffers_from is a no-op when the source composer is inactive.""" num_envs, num_bodies = 4, 2 @@ -855,7 +856,7 @@ def test_add_raw_buffers_from_inactive_is_noop(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): @@ -910,7 +911,7 @@ def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): ), f"Mask vs index torque mismatch (envs={num_envs}, bodies={num_bodies})" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): @@ -965,7 +966,7 @@ def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_overwrites_previous_add(device: str): """Test that set_forces_and_torques_index clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -994,7 +995,7 @@ def test_set_forces_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_index clears only the targeted environments.""" num_envs, num_bodies = 4, 3 @@ -1065,7 +1066,7 @@ def test_set_forces_clears_targeted_envs_only(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_partial_reset_zeros_only_specified_envs(device: str): """Test that partial reset zeros only the specified environments and leaves others intact.""" num_envs, num_bodies = 8, 3 @@ -1116,7 +1117,7 @@ def test_partial_reset_zeros_only_specified_envs(device: str): assert composer._dirty -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_full_reset_clears_active_flag(device: str): """Test that full reset (no args) clears the _active flag.""" num_envs, num_bodies = 4, 2 @@ -1140,7 +1141,7 @@ def test_full_reset_clears_active_flag(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_force_emits_deprecation_warning(device: str): """Test that accessing composed_force emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1160,7 +1161,7 @@ def test_composed_force_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_force_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_torque_emits_deprecation_warning(device: str): """Test that accessing composed_torque emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1179,7 +1180,7 @@ def test_composed_torque_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_torque_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_add_forces_and_torques_emits_warning(device: str): """Test that the deprecated add_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1204,7 +1205,7 @@ def test_deprecated_add_forces_and_torques_emits_warning(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_overwrites_previous_add(device: str): """Test that set_forces_and_torques_mask clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -1233,7 +1234,7 @@ def test_set_forces_mask_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_mask clears only the masked environments.""" num_envs, num_bodies = 4, 3 @@ -1303,7 +1304,7 @@ def test_set_forces_mask_clears_targeted_envs_only(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_matches_set_forces_index(device: str): """Test that set_forces_and_torques_mask produces the same result as the index variant.""" num_envs, num_bodies = 6, 3 @@ -1353,7 +1354,7 @@ def test_set_forces_mask_matches_set_forces_index(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_force_b_triggers_lazy_composition(device: str): """Test that accessing out_force_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1380,7 +1381,7 @@ def test_out_force_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_torque_b_triggers_lazy_composition(device: str): """Test that accessing out_torque_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1407,7 +1408,7 @@ def test_out_torque_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_lazy_composition_tracks_dirty_flag(device: str): """Test that the dirty flag is correctly managed through add/compose/add cycles.""" num_envs, num_bodies = 2, 1 @@ -1444,7 +1445,7 @@ def test_lazy_composition_tracks_dirty_flag(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), expected, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_compose_is_idempotent(device: str): """Calling compose_to_body_frame twice without intervening writes produces the same result.""" rng = np.random.default_rng(seed=456) @@ -1495,7 +1496,7 @@ def test_compose_is_idempotent(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_with_com_offset(device: str): """Test that torque correction uses CoM position, not link position, when they differ.""" num_envs, num_bodies = 2, 1 @@ -1550,7 +1551,7 @@ def test_global_force_with_com_offset(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_com_no_torque_with_com_offset(device: str): """Test that a global force at CoM position produces zero torque even with CoM offset.""" num_envs, num_bodies = 2, 1 @@ -1596,7 +1597,7 @@ def test_global_force_at_com_no_torque_with_com_offset(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_com_offset_with_rotation(device: str): """Test torque correction with both CoM offset and non-identity rotation.""" num_envs, num_bodies = 1, 1 @@ -1654,7 +1655,7 @@ def test_com_offset_with_rotation(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_emits_warning(device: str): """Test that the deprecated set_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1674,7 +1675,7 @@ def test_deprecated_set_forces_and_torques_emits_warning(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_clears_previous(device: str): """Test that deprecated set_forces_and_torques actually replaces previous values.""" num_envs, num_bodies = 4, 2 diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index de757adb287e..2adbf73a3354 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -9,11 +9,12 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -481,7 +482,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -531,7 +532,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_articulations", [2, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -571,7 +572,7 @@ def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -621,7 +622,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base(sim, num_articulations, device, articulation_type): """Test initialization for fixed base. @@ -678,7 +679,7 @@ def test_initialization_fixed_base(sim, num_articulations, device, articulation_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_fixed_base_reports_body_velocities(sim, num_articulations, device, articulation_type): """Test that fixed-base articulations report live body velocities while their joints move. @@ -730,7 +731,7 @@ def test_fixed_base_reports_body_velocities(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -788,7 +789,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) def test_initialization_hand_with_tendons(sim, num_articulations, device, articulation_type): """Test initialization for fixed base articulated hand with tendons. @@ -860,7 +861,7 @@ def test_fragment_fix_root_link_uses_base_manager(sim, device, add_ground_plane, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base_made_fixed_base( @@ -914,7 +915,7 @@ def test_initialization_floating_base_made_fixed_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base_made_floating_base( @@ -960,7 +961,7 @@ def test_initialization_fixed_base_made_floating_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -991,7 +992,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_vel(sim, device, articulation_type): """Test that the default joint velocity from configuration is out of range. @@ -1016,7 +1017,7 @@ def test_out_of_range_default_joint_vel(sim, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1092,7 +1093,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1126,7 +1127,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_buffer(sim, num_articulations, device, articulation_type): """Test if external force buffer correctly updates in the force value is zero case. @@ -1211,7 +1212,7 @@ def test_external_force_buffer(sim, num_articulations, device, articulation_type @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation. @@ -1269,7 +1270,7 @@ def test_external_force_on_single_body(sim, num_articulations, device, articulat @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation at a given position. @@ -1364,7 +1365,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation. @@ -1424,7 +1425,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device, artic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation at a given position. @@ -1518,7 +1519,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_loading_gains_from_usd(sim, num_articulations, device, articulation_type): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1580,7 +1581,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device, articulation_typ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1615,7 +1616,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulation_type): """Test that gains are loaded from the configuration dictionary correctly. @@ -1648,7 +1649,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1720,7 +1721,7 @@ def test_setting_velocity_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1776,7 +1777,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) @@ -1833,7 +1834,7 @@ def test_setting_effort_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1899,7 +1900,7 @@ def test_setting_effort_limit_explicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_reset(sim, num_articulations, device, articulation_type): """Test that reset method works properly.""" @@ -1943,7 +1944,7 @@ def test_reset(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1983,7 +1984,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_body_root_state(sim, num_articulations, device, with_offset, articulation_type): @@ -2108,7 +2109,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset, articulati @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2198,7 +2199,7 @@ def test_write_root_state( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2299,7 +2300,7 @@ def test_write_root_state_functions_data_consistency( torch.testing.assert_close(root_link_vel_w[:, 3:], root_com_vel_w[:, 3:]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2318,7 +2319,7 @@ def test_setting_articulation_root_prim_path(sim, device, articulation_type): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2337,7 +2338,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_t @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled, articulation_type): @@ -2443,7 +2444,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) @pytest.mark.skip(reason="Spatial tendons are not supported in Newton yet.") def test_spatial_tendons(sim, num_articulations, device, articulation_type): @@ -2497,7 +2498,7 @@ def test_spatial_tendons(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane, articulation_type): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -2567,7 +2568,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_body_q_consistent_after_root_write(num_articulations, device, articulation_type): """Test that body_q is fresh when collide() runs after a root pose write. @@ -2649,7 +2650,7 @@ def _patched_simulate(cls): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) via view-level APIs.""" @@ -2705,7 +2706,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2729,7 +2730,7 @@ def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_pla @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2762,7 +2763,7 @@ def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail( @@ -2814,7 +2815,7 @@ def test_get_gravity_compensation_forces_not_implemented_on_newton(sim, num_arti @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2840,7 +2841,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2876,7 +2877,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2909,7 +2910,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2935,7 +2936,7 @@ def test_get_mass_matrix_shape_floating_base(sim, num_articulations, device, add assert M.dtype == torch.float32 -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -3023,7 +3024,7 @@ def test_heterogeneous_scene_per_view_shapes(sim, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3102,7 +3103,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3153,7 +3154,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3204,7 +3205,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3237,7 +3238,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3314,7 +3315,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 0e87a39c535e..4ed03df445e5 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -10,9 +10,10 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -121,7 +122,7 @@ def generate_cubes_scene( @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -156,7 +157,7 @@ def test_initialization(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="Newton does not support kinematic rigid bodies") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -194,7 +195,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -212,7 +213,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -229,7 +230,7 @@ def test_initialization_with_articulation_root(num_cubes, device): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -298,7 +299,7 @@ def test_external_force_buffer(device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -373,7 +374,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -462,7 +463,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -530,7 +531,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -573,7 +574,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object via view-level APIs.""" with _newton_sim_context(device, gravity_enabled=True, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -628,7 +629,7 @@ def _set_newton_material_properties(cube_object, friction_val, restitution_val, @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo contact at height=0 does not settle the same as PhysX — cube falls on z-axis") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -668,7 +669,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -681,7 +682,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo uses Coulomb friction (single mu), no static/dynamic distinction") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -761,7 +762,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo restitution model differs from PhysX — inelastic collisions still bounce") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -839,7 +840,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _newton_sim_context(device, gravity_enabled=False, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -877,7 +878,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -953,7 +954,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1066,7 +1067,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1153,7 +1154,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): @@ -1321,7 +1322,7 @@ def test_warmup_attach_stage_not_called_for_cpu(): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_link_pose_w_fresh_after_root_pose_write(device, writer): diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index 18928bedb5d8..8002ce57243b 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -10,9 +10,10 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -119,7 +120,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -151,7 +152,7 @@ def test_initialization(num_envs, num_cubes, device): @pytest.mark.skip(reason="Newton doesn't support kinematic rigid bodies yet") @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -187,7 +188,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -202,7 +203,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -258,7 +259,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -320,7 +321,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -403,7 +404,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_object_state(num_envs, num_cubes, device): """Test setting the state of the object. @@ -475,7 +476,7 @@ def test_set_object_state(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -510,7 +511,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object collection via view-level APIs.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -549,7 +550,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -622,7 +623,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset): """Test the object_com_state_w and object_link_state_w properties.""" @@ -717,7 +718,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_object_state(num_envs, num_cubes, device, with_offset, state_location): @@ -795,7 +796,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_object_state_functions_data_consistency(num_envs, num_cubes, device, with_offset, state_location): @@ -932,7 +933,7 @@ def test_write_object_state_functions_data_consistency(num_envs, num_cubes, devi torch.testing.assert_close(body_com_vel_w[..., 3:], link_vel_w[..., 3:]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_pose_write_marks_fk_reset_mask(device, writer): diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 2d92e3274a7e..66452b62d553 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -20,6 +20,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import math @@ -74,7 +76,7 @@ class ContactSensorTestSceneCfg(InteractiveSceneCfg): # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -193,7 +195,7 @@ def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: S assert no_contact_detected[env_idx], f"Env {env_idx}: Contact should stop after lift." -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -298,7 +300,7 @@ def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): """Test that resting object contact force equals weight and points upward. @@ -401,7 +403,7 @@ def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): assert not errs, "\n".join(errs) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contacts: bool): """Test that dropping from higher produces larger peak impact force. @@ -483,7 +485,7 @@ def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contac # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -618,7 +620,7 @@ def test_filter_enables_force_matrix(device: str, use_mujoco_contacts: bool): } -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -815,7 +817,7 @@ def _make_two_box_scene_cfg(num_envs: int) -> ContactSensorTestSceneCfg: return scene_cfg -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_sensor_metadata(device: str): """Verify sensor_names and filter_object_names match the underlying sensing and counterpart configuration across body-mode, body-mode-with-filter, and shape-mode. diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index 174f0684fb55..8ac61278d52f 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -13,6 +13,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) @@ -102,7 +104,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_body_path(device): """FrameView rejects prim paths that resolve to a Newton physics body.""" ctx = _sim_context(device, num_envs=2) @@ -116,7 +118,7 @@ def test_reject_body_path(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_shape_path(device): """FrameView rejects prim paths that resolve to a Newton collision shape.""" ctx = _sim_context(device, num_envs=2) @@ -183,7 +185,7 @@ def test_view_can_resolve_from_body_labels_after_reset(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_returns_initial_pose(device): """A world-rooted frame returns its configured position.""" ctx = _sim_context(device, num_envs=2) @@ -201,7 +203,7 @@ def test_world_attached_returns_initial_pose(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_set_world_roundtrip(device): """A world-attached prim can be repositioned via set_world_poses.""" ctx = _sim_context(device, num_envs=2) diff --git a/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index 07952dade515..483736d74c00 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -60,6 +60,8 @@ from pxr import UsdPhysics +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -324,7 +326,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -388,7 +390,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -453,7 +455,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -525,7 +527,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -598,7 +600,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -658,7 +660,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.xfail(reason=_OMNI_PHYSX_SCHEMAS_GAP_REASON, strict=False) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): @@ -768,7 +770,7 @@ def test_fragment_fix_root_reenables_existing_joint(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -828,7 +830,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -858,7 +860,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -882,7 +884,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -957,7 +959,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -990,7 +992,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -1074,7 +1076,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1131,7 +1133,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1225,7 +1227,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1284,7 +1286,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1377,7 +1379,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1438,7 +1440,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1472,7 +1474,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1504,7 +1506,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1571,7 +1573,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1624,7 +1626,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1676,7 +1678,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1737,7 +1739,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1780,7 +1782,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1819,7 +1821,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1945,7 +1947,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2073,7 +2075,7 @@ def test_body_com_pose_b_cache_and_set_coms_invalidation(sim, device): assert buffer.timestamp < articulation.data._sim_timestamp, name -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -2091,7 +2093,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -2109,7 +2111,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2214,7 +2216,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2266,7 +2268,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2359,7 +2361,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting per-shape material properties (friction/restitution). diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 229f235e902d..7e3ef125d625 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,6 +32,8 @@ import warp as wp from flaky import flaky +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -192,7 +194,7 @@ def _read_shape_material(cube_object) -> torch.Tensor: @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -224,7 +226,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -260,7 +262,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -276,7 +278,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -291,7 +293,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -358,7 +360,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -434,7 +436,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -538,7 +540,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -602,7 +604,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _ovphysx_sim_context(device=device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -643,7 +645,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting per-shape material properties of a rigid object.""" with _ovphysx_sim_context(device=device, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -671,7 +673,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties_via_view(num_cubes, device): """Test setting per-shape material via the OvPhysxView binding API.""" with _ovphysx_sim_context(device=device, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -694,7 +696,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain its velocity when sliding across a plane.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -739,7 +741,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -803,7 +805,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -867,7 +869,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _ovphysx_sim_context( @@ -910,7 +912,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -948,7 +950,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1064,7 +1066,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1135,7 +1137,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 61ebf0ef1d12..f443ff71a275 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,6 +28,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -160,7 +162,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -188,7 +190,7 @@ def test_initialization(num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(device): """Test environment and object index conversion to physics view indices.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -227,7 +229,7 @@ def test_id_conversion(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -262,7 +264,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -276,7 +278,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -331,7 +333,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -392,7 +394,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -474,7 +476,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -544,7 +546,7 @@ def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -643,7 +645,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravi @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -721,7 +723,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -755,7 +757,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting per-shape material properties of a rigid object collection. @@ -796,7 +798,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -830,7 +832,7 @@ def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index ac4755962a95..661354777d90 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -9,11 +9,12 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -310,7 +311,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -366,7 +367,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -423,7 +424,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -487,7 +488,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -552,7 +553,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -605,7 +606,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base articulation made fixed-base using schema properties. @@ -665,7 +666,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -717,7 +718,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -747,7 +748,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -771,7 +772,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -846,7 +847,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -879,7 +880,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -963,7 +964,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1020,7 +1021,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1114,7 +1115,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1173,7 +1174,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1266,7 +1267,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1327,7 +1328,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1361,7 +1362,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1393,7 +1394,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1460,7 +1461,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1513,7 +1514,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1565,7 +1566,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1627,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1669,7 +1670,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1708,7 +1709,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1832,7 +1833,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1914,7 +1915,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1932,7 +1933,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1950,7 +1951,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2055,7 +2056,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2107,7 +2108,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2200,7 +2201,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) of articulation shapes.""" @@ -2247,7 +2248,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2264,7 +2265,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2291,7 +2292,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2316,7 +2317,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2368,7 +2369,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2404,7 +2405,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2455,7 +2456,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2489,7 +2490,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2578,7 +2579,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2636,7 +2637,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2800,7 +2801,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2834,7 +2835,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index 6454c6b61c5a..072a559216eb 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -10,9 +10,10 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -98,7 +99,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -132,7 +133,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -170,7 +171,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -188,7 +189,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -205,7 +206,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -274,7 +275,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -350,7 +351,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -455,7 +456,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -521,7 +522,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -564,7 +565,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -605,7 +606,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -645,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -694,7 +695,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -705,7 +706,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -791,7 +792,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -874,7 +875,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -918,7 +919,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -958,7 +959,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -1069,7 +1070,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -1139,7 +1140,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index f42f6dfcb085..7fad208a4db6 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -10,9 +10,10 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" @@ -110,7 +111,7 @@ def sim(request): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(sim, num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -137,7 +138,7 @@ def test_initialization(sim, num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(sim, device): """Test environment and object index conversion to physics view indices.""" object_collection, _ = generate_cubes_scene(num_envs=2, num_cubes=3, device=device) @@ -175,7 +176,7 @@ def test_id_conversion(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" object_collection, origins = generate_cubes_scene( @@ -209,7 +210,7 @@ def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(sim, num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_cubes=num_cubes, has_api=False, device=device) @@ -222,7 +223,7 @@ def test_initialization_with_no_rigid_body(sim, num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -276,7 +277,7 @@ def test_external_force_buffer(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object.""" object_collection, origins = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -336,7 +337,7 @@ def test_external_force_on_single_body(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -415,7 +416,7 @@ def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, dev @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -480,7 +481,7 @@ def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -576,7 +577,7 @@ def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -656,7 +657,7 @@ def test_write_object_state(sim, num_envs, num_cubes, device, with_offset, state @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(sim, num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -689,7 +690,7 @@ def test_reset_object_collection(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(sim, num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -724,7 +725,7 @@ def test_set_material_properties(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -757,7 +758,7 @@ def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index c023ab40ba86..03a50a83d7b8 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,15 +10,15 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) from isaaclab.app import AppLauncher +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app import pytest # noqa: E402 import torch # noqa: E402 @@ -138,7 +138,7 @@ def _fill_position(out: wp.array(dtype=wp.float32, ndim=2), x: float, y: float, out[i, 2] = wp.float32(z) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): """Verify that set_world_poses in Fabric mode does NOT sync back to USD. @@ -183,7 +183,7 @@ def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_rebuild_after_topology_change(device, view_factory): """A simulated topology change rebuilds the indexed fabric arrays and leaves the view in a state where subsequent writes/reads still produce correct data. @@ -638,11 +638,7 @@ def test_multi_view_writer_isolation(device): # ------------------------------------------------------------------ -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -663,11 +659,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -696,11 +688,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """World-space scale writes roundtrip via ``worldMatrix`` on ``cuda:1``. diff --git a/tools/conftest.py b/tools/conftest.py index 5a2421a88e2b..cb78a6fdce65 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -5,6 +5,7 @@ import contextlib import os +import re import select import signal import subprocess @@ -17,6 +18,8 @@ from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable +from isaaclab.test.utils import resolve_test_sim_device + # Local imports import test_settings as test_settings # isort: skip from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip @@ -315,6 +318,102 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) +def _slugify_test_path(test_path): + """Encode a test path as a flat queue entry name. + + The queue uses one file per pending test. Slashes are not legal inside a + filename, so we encode the relative path by replacing ``/`` with ``__``. + The decoder is :func:`_unslugify_queue_entry`. + """ + return test_path.replace("/", "__") + + +def _unslugify_queue_entry(entry_name): + """Reverse of :func:`_slugify_test_path`.""" + return entry_name.replace("__", "/") + + +def _claim_queued_file(queue_dir): + """Atomically claim one pending test from the work-queue directory. + + The queue is a directory of files (one per pending test); the shard claims + one by renaming it from ``queue/`` into its private ``inflight/cuda-N/``. + POSIX rename is atomic on the same filesystem, so two shards racing on the + same source file are serialized by the kernel: exactly one rename succeeds, + the other gets ``FileNotFoundError`` and tries the next entry. + + On success, the test's queue entry is now sitting in ``inflight/cuda-N/``; + the caller is expected to move it to ``done/cuda-N/`` after the per-test + pytest invocation exits with a clean result, leaving anything still in + ``inflight/`` at job-end as recoverable evidence of a crashed test. + + Args: + queue_dir: Path to the shared work-queue root. Must contain a + ``queue/`` subdir (pending entries) and an ``inflight//`` + subdir for this shard (claim destination). + + Returns: + The decoded test path for the claimed file, or ``None`` when the + queue is empty. + """ + shard = resolve_test_sim_device().replace(":", "-") + pending_dir = os.path.join(queue_dir, "queue") + inflight_dir = os.path.join(queue_dir, "inflight", shard) + os.makedirs(inflight_dir, exist_ok=True) + + # Listdir is intentionally not cached: another shard may have just removed + # an entry we'd otherwise try. We pay one listdir per claim attempt; with + # N≤20 entries this is microseconds. + try: + entries = sorted(os.listdir(pending_dir)) + except FileNotFoundError: + return None + + for entry in entries: + src = os.path.join(pending_dir, entry) + dst = os.path.join(inflight_dir, entry) + try: + os.rename(src, dst) + except FileNotFoundError: + # Lost the race for this entry; another shard claimed it first. + # Continue to the next entry in our (potentially stale) listing. + continue + except OSError: + # Any other rename failure (e.g. permission) is a hard error. + raise + return _unslugify_queue_entry(entry) + + return None + + +def _mark_queued_file_done(queue_dir, test_path): + """Move a successfully-completed claim from ``inflight/cuda-N/`` to ``done/cuda-N/``. + + Called by the test runner after a per-file pytest invocation exits cleanly. + The inflight residual is what the post-run reconciler uses to detect + crashed shards: anything still in ``inflight/`` at job-end is an orphan. + """ + shard = resolve_test_sim_device().replace(":", "-") + entry = _slugify_test_path(test_path) + src = os.path.join(queue_dir, "inflight", shard, entry) + dst_dir = os.path.join(queue_dir, "done", shard) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, entry) + # Suppress: already moved (idempotent) or the runner crashed before we + # could mark done — the reconciler catches the second case. + with contextlib.suppress(FileNotFoundError): + os.rename(src, dst) + + +def _queued_files(queue_dir): + """Yield files claimed from the shared work queue until it is empty.""" + while True: + claimed = _claim_queued_file(queue_dir) + if claimed is None: + return + yield claimed + + def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -417,6 +516,9 @@ class _PassContext: timeout: Per-pass hard timeout in seconds. startup_deadline: Per-pass startup-hang deadline in seconds. env: Environment passed to the pytest subprocess. + inject_shard_select: Whether to load the multi-GPU ``mgpu_shard_select`` + plugin in the pytest subprocess; true only on a non-default-GPU shard. + pytest_targets: Test file or node IDs passed to the pytest subprocess. """ test_file: str @@ -426,6 +528,7 @@ class _PassContext: timeout: int startup_deadline: int env: dict + inject_shard_select: bool pytest_targets: list[str] @@ -485,18 +588,30 @@ def _run_one_pass( the ``failed_tests`` list. """ pass_file_label = f"{ctx.file_name}{suffix}" - report_file = f"tests/test-reports-{pass_file_label}.xml" + # Slug the full test path (not just the basename) into the report filename so + # two concurrent shards running same-basename files (e.g. + # ``isaaclab_newton/.../test_articulation.py`` vs + # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path + # inside the shared ``/workspace/isaaclab`` mount and trigger false + # shutdown_hang detections in sibling shards via the report-file existence check. + report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") + report_file = f"tests/test-reports-{report_slug}{suffix}.xml" cmd = [ sys.executable, "-m", "pytest", "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit "--no-header", f"--config-file={ctx.workspace_root}/pyproject.toml", f"--junitxml={report_file}", "--tb=short", ] + if ctx.inject_shard_select: + # multi-GPU lane test-selection plugin (importable via the PYTHONPATH set in + # run_individual_tests); deselects out-of-scope device variants on a shard. + cmd += ["-p", "mgpu_shard_select"] if ctx.ci_marker: cmd += ["-m", ctx.ci_marker] if k_expr is not None: @@ -733,7 +848,12 @@ def _run_one_pass( def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by_file=None): - """Run each test file separately, ensuring one finishes before starting the next.""" + """Run each test file separately, ensuring one finishes before starting the next. + + When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue file, files are claimed + from it (work-stealing across sibling shard containers) instead of iterating + ``test_files``; each file still runs once, on this container's pinned GPU. + """ failed_tests = [] test_status = {} xml_reports = [] @@ -743,12 +863,28 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by if global_k_expr is not None: print(f"Applying global pytest -k expression to every test file: '{global_k_expr}'") - for test_file in test_files: + queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") + file_source = _queued_files(queue_path) if queue_path else test_files + + for test_file in file_source: print(f"\n\n🚀 Running {test_file} independently...\n") file_name = os.path.basename(test_file) env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" + # Multi-GPU lane only: make the device-selection plugin importable in this + # per-file subprocess (injected via ``-p`` in _run_one_pass, not as a + # repo-root conftest). Detect a shard by the runtime device mask excluding cpu + # (position 0) and cuda:0 (position 1) -- the same ISAACLAB_TEST_DEVICES the + # plugin and test_devices() read. The plugin re-checks this; the cheap + # prefix test here leaves single-GPU CI's command (mask unset or "11...") + # unchanged. + _mask = os.environ.get("ISAACLAB_TEST_DEVICES", "") + _inject_shard_select = _mask[:2] == "00" + if _inject_shard_select: + _plugin_dir = os.path.join(workspace_root, ".github", "actions", "multi-gpu") + env["PYTHONPATH"] = _plugin_dir + os.pathsep + env.get("PYTHONPATH", "") + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) # Read the test file once for cold-cache and device-split detection. @@ -779,10 +915,18 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by timeout=timeout, startup_deadline=startup_deadline, env=env, + inject_shard_select=_inject_shard_select, pytest_targets=pytest_targets, ) - if is_device_split_file(test_file, source=test_content): + # On a multi-GPU shard, test_devices() already resolves to this shard's single + # GPU and mgpu_shard_select drops every other variant, so the device_split + # CPU/GPU two-pass (which exists to dodge the process-global device lock when + # CPU and GPU share one container) is unnecessary here — the CPU pass would + # collect zero tests yet still pay full Kit-startup cost. Run once on a shard. + if _inject_shard_select: + passes = [("", None)] + elif is_device_split_file(test_file, source=test_content): print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") passes = DEVICE_SPLIT_PASSES else: @@ -802,6 +946,14 @@ def run_individual_tests(test_files, workspace_root, ci_marker, test_node_ids_by assert merged_status is not None # the pass list is never empty test_status[test_file] = merged_status + # When running under the directory-based work queue (option 2), move the + # claim entry from inflight// to done// so the post-run + # reconciler can distinguish "ran to completion" from "claimed but + # crashed mid-test". A claim that stays in inflight at job-end is a + # silent drop signal. + if queue_path: + _mark_queued_file_done(queue_path, test_file) + print("~~~~~~~~~~~~ Finished running all tests") return failed_tests, test_status, xml_reports @@ -1072,6 +1224,10 @@ def pytest_sessionstart(session): test_files, workspace_root, effective_marker, test_node_ids_by_file ) + # In work-queue mode this container ran only the files it claimed; report on those. + if os.environ.get("ISAACLAB_TEST_QUEUE"): + test_files = list(test_status) + print("failed tests:", failed_tests) # Collect reports @@ -1088,6 +1244,10 @@ def pytest_sessionstart(session): # write content to full report result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") full_report_path = f"tests/{result_file}" + # Ensure the directory exists even when this shard claimed zero files + # from the work queue (per-test JUnit XMLs are what normally create + # ``tests/``; with no tests run there is nothing to create it). + os.makedirs("tests", exist_ok=True) print(f"Using result file: {result_file}") full_report.write(full_report_path) print("~~~~~~~~~~~~ Report written to", full_report_path) @@ -1126,14 +1286,18 @@ def pytest_sessionstart(session): summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" + # GPU this run used (the shard's boot device); ``cuda:0`` when the runtime + # device mask is unset. + run_device = resolve_test_sim_device() + summary_str += "\n\n=======================\n" - summary_str += "Per Test Result Summary\n" + summary_str += "Per File Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_test_result_table.align["Test Path"] = "l" - per_test_result_table.align["Test (s)"] = "r" - per_test_result_table.align["Wall (s)"] = "r" + per_file_result_table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + per_file_result_table.align["Test Path"] = "l" + per_file_result_table.align["Test (s)"] = "r" + per_file_result_table.align["Wall (s)"] = "r" for test_path in test_files: num_tests_passed = ( test_status[test_path]["tests"] @@ -1141,9 +1305,10 @@ def pytest_sessionstart(session): - test_status[test_path]["errors"] - test_status[test_path]["skipped"] ) - per_test_result_table.add_row( + per_file_result_table.add_row( [ test_path, + run_device, test_status[test_path]["result"], f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", @@ -1151,7 +1316,34 @@ def pytest_sessionstart(session): ] ) - summary_str += per_test_result_table.get_string() + summary_str += per_file_result_table.get_string() + + # Per-test run times, slowest first, from the merged JUnit report. The + # device is read from the test id params (e.g. ``...[size0-cuda:1]``), + # falling back to the run's boot device. + summary_str += "\n\n=================\n" + summary_str += "Per Test Run Time\n" + summary_str += "=================\n" + + per_test_time_table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + per_test_time_table.align["Test"] = "l" + per_test_time_table.align["Time (s)"] = "r" + test_times = [] + for suite in full_report: + for case in suite: + full_name = f"{case.classname}::{case.name}" if case.classname else case.name + device = run_device + bracket = re.search(r"\[(.*)\]", full_name) + if bracket: + dev_match = re.search(r"cuda:\d+|\bcpu\b", bracket.group(1)) + if dev_match: + device = dev_match.group(0) + elapsed = float(case.time) if case.time is not None else 0.0 + test_times.append((full_name, device, elapsed)) + for full_name, device, elapsed in sorted(test_times, key=lambda row: row[2], reverse=True): + per_test_time_table.add_row([full_name, device, f"{elapsed:0.3f}"]) + + summary_str += per_test_time_table.get_string() # Print summary to console and log file print(summary_str)