From 3ca93df0f0d3468ddb797abde1194589129d9dbb Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 30 May 2026 21:13:56 +0000 Subject: [PATCH 01/22] Honor device kwarg over sim_cfg.device in build_simulation_context Most test callers pass both ``sim_cfg=`` and ``device=`` to :func:`isaaclab.sim.build_simulation_context`, implicitly expecting the ``device`` kwarg to win. The helper previously dropped the kwarg silently when ``sim_cfg`` was provided, causing warp kernel-launch device mismatches on non-default GPUs: the test fixture allocated ``env_ids`` on the requested device while the articulation's ``self.device`` resolved from the untouched ``sim_cfg`` default (``cuda:0``), and ``wp.launch(..., device=self.device)`` failed with:: RuntimeError: Error launching kernel 'set_root_link_pose_to_sim_index', trying to launch on device='cuda:0', but input array for argument 'env_ids' is on device=cuda:2. Change ``device``'s default to ``None`` (sentinel) and apply it as an override after sim_cfg construction in both branches. The one test that asserted the old "sim_cfg overrides everything" contract is updated to cover the new override semantics. --- .../jichuanh-fix-build-sim-context-device.rst | 11 ++++++++++ .../isaaclab/sim/simulation_context.py | 20 ++++++++++++++++--- .../test_build_simulation_context_headless.py | 19 +++++++++++++++--- ...st_build_simulation_context_nonheadless.py | 20 +++++++++++++++---- 4 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst diff --git a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst new file mode 100644 index 000000000000..89e774bd26d7 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed :func:`isaaclab.sim.build_simulation_context` silently ignoring the + ``device`` kwarg when ``sim_cfg`` is also provided. Most test callers pass + both kwargs together; the helper now applies the explicit ``device`` over + ``sim_cfg.device`` so the caller's choice wins. Without this, warp kernel + launches in :mod:`isaaclab_newton.assets.articulation` raised device + mismatch errors on non-default GPUs (``env_ids`` allocated on the test's + device while the articulation's resolved device came from the untouched + ``sim_cfg`` default ``cuda:0``). diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index c068469ec599..fae7d4d1f510 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -958,7 +958,7 @@ def _predicate(prim: Usd.Prim) -> bool: def build_simulation_context( create_new_stage: bool = True, gravity_enabled: bool = True, - device: str = "cuda:0", + device: str | None = None, dt: float = 0.01, sim_cfg: SimulationCfg | None = None, add_ground_plane: bool = False, @@ -971,7 +971,11 @@ def build_simulation_context( Args: create_new_stage: Whether to create a new stage. Defaults to True. gravity_enabled: Whether to enable gravity. Defaults to True. - device: Device to run the simulation on. Defaults to "cuda:0". + device: Device to run the simulation on. When given alongside ``sim_cfg``, + overrides ``sim_cfg.device`` so the caller's explicit choice wins + (most test callers pass both, expecting this behavior). Defaults to + ``None``, meaning ``sim_cfg.device`` is left untouched and a freshly + built ``sim_cfg`` uses :class:`SimulationCfg`'s default device. dt: Time step for the simulation. Defaults to 0.01. sim_cfg: SimulationCfg to use. Defaults to None. add_ground_plane: Whether to add a ground plane. Defaults to False. @@ -993,7 +997,17 @@ def build_simulation_context( if sim_cfg is None: gravity = (0.0, 0.0, -9.81) if gravity_enabled else (0.0, 0.0, 0.0) - sim_cfg = SimulationCfg(device=device, dt=dt, gravity=gravity) + sim_cfg = SimulationCfg(dt=dt, gravity=gravity) + if device is not None: + # Honor the explicit device kwarg in both branches: when sim_cfg is + # freshly built, this picks the device; when sim_cfg is passed in, + # this overrides its (possibly default) device. Without the override, + # callers passing both ``sim_cfg=`` and + # ``device=cuda:N`` silently got sim_cfg's device, causing warp + # kernel-launch mismatches when test fixtures allocated tensors on + # the requested device while assets resolved their device from the + # untouched sim_cfg. + sim_cfg.device = device sim = SimulationContext(sim_cfg) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index 8f13d79041e7..4ceae87b9878 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -75,7 +75,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light @pytest.mark.isaacsim_ci def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -87,8 +94,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index 8c053bc51cda..09bfe3091839 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -70,8 +70,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" - + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -83,8 +89,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt From a396a07f1f299b70662dfdb671f133eb8b70e543 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:48:06 +0000 Subject: [PATCH 02/22] Add ISAACLAB_PIN_KIT_GPU to pin Kit renderer to one GPU Add an ISAACLAB_PIN_KIT_GPU env var to AppLauncher. When truthy, it appends Kit command-line overrides that pin the renderer to a single GPU (renderer.multiGpu.enabled=False, autoEnable=False, maxGpuCount=1) and disable the fabric GPU-interop path (physics.fabricUseGPUInterop= false), so each Kit process touches only its assigned GPU instead of enumerating every visible GPU at startup. Used by the multi-GPU CI workflow to avoid a shared GPU-interop context across concurrent sibling shards, which otherwise surfaces as "Stage X already attached" errors and SimulationApp.close hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). Off by default; single-GPU and user-facing rendering paths are unchanged. --- .../jichuanh-mgpu-pin-kit-resources.rst | 13 ++++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..a78313ab14d9 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,13 @@ +Added +^^^^^ + +* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. + When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, + ``--/renderer/multiGpu/autoEnable=False``, ``--/renderer/multiGpu/maxGpuCount=1`` + and ``--/physics/fabricUseGPUInterop=false`` to the Kit command line so each + Kit process touches only its assigned GPU (rather than enumerating every + visible GPU at startup). Used by the multi-GPU CI workflow to prevent the + shared GPU-interop context across sibling shards that surfaces as + ``[Error] [omni.physx.plugin] Stage X already attached`` and + ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). + Off by default; single-GPU and user-facing rendering paths are unchanged. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e8ff526496a1..53dd8065aa11 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1060,6 +1060,32 @@ def _resolve_device_settings(self, launcher_args: dict): launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id + # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is + # truthy. The default ``apps/isaaclab.python.headless.kit`` sets + # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable + # = true``, so each Kit process enumerates every visible GPU at + # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per + # container, one Kit per non-default cuda device), that produces a + # shared GPU-interop context across sibling processes -- surfacing as + # ``[Error] [omni.physx.plugin] Stage X already attached`` mid-test and + # ``SimulationApp.close`` hanging in teardown (see + # https://github.com/isaac-sim/IsaacLab/issues/3475). The mitigation is + # to set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount = 1`` so + # each Kit only touches its assigned GPU. + if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: + sys.argv.append("--/renderer/multiGpu/enabled=False") + sys.argv.append("--/renderer/multiGpu/autoEnable=False") + sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") + # Also disable the fabric GPU-interop path. The renderer multiGpu + # flags above mitigate the startup-time enumeration race; this + # mitigates the runtime GPU-interop race on top. Safe for the + # multi-GPU CI lane: it covers physics / scene / utility tests, + # not rendering. + sys.argv.append("--/physics/fabricUseGPUInterop=false") + logger.info( + "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" + ) + # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash # Kit's platform-info fork during startup. From f1c0d1844419693a51741c9539f939ff285bf9f3 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:38 +0000 Subject: [PATCH 03/22] [Tests] Add test_devices helper for device-parametrized unit tests Adds the scope-intersect-runtime device selection helper (test_devices) and its unit test, so unit tests can declare the devices they are valid on and the multi-GPU lane can narrow them via ISAACLAB_TEST_DEVICES. --- .../isaaclab/isaaclab/test/utils/__init__.py | 16 ++ .../isaaclab/isaaclab/test/utils/devices.py | 169 ++++++++++++++++++ .../test/utils/test_device_selection.py | 148 +++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 source/isaaclab/isaaclab/test/utils/__init__.py create mode 100644 source/isaaclab/isaaclab/test/utils/devices.py create mode 100644 source/isaaclab/test/utils/test_device_selection.py diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py new file mode 100644 index 000000000000..e268d22c5c16 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test-time helpers for Isaac Lab. + +Exposes :func:`test_devices` for selecting the device list to parametrize tests +over. The set is ``scope ∩ budget``: ``scope`` is the call-site argument (the +devices the test is valid on), ``budget`` is the ``ISAACLAB_TEST_DEVICES`` env +var (the devices the run may use). +""" + +from .devices import test_devices + +__all__ = ["test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py new file mode 100644 index 000000000000..c54cabfdfbd5 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Device selection for parametrizing tests over cpu / cuda devices. + +Intended use:: + + from isaaclab.test.utils import test_devices + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + def test_foo(device): ... + +Two masks, two roles +-------------------- +A test runs on ``scope ∩ runtime``: + +* **scope** — the call-site mask: the devices the *test* is valid on. The + author owns this; defaults to ``"11X"`` (device-agnostic). +* **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* + may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). + The multi-GPU workflow sets it to one device per shard, matching + ``ISAACLAB_SIM_DEVICE`` (AppLauncher's boot device — not read here). + +A test author never names the shard's GPU; the operator never inspects a +test's device support. The helper intersects the two. + +Mask grammar +------------ +Positions left to right: ``0`` = cpu, ``1`` = cuda:0 (the default GPU), +``2`` = cuda:1, ``3`` = cuda:2, ... Each position is ``0`` (exclude) or ``1`` +(include). A trailing ``X`` means "include all the remaining devices". + +cpu, cuda:0, and a non-default GPU stay distinct by position — running on +cuda:0 says nothing about cuda:1+, which is the whole reason this exists. + +Common masks +------------ +====== =================================================================== +Mask Meaning +====== =================================================================== +``11X`` cpu + cuda:0 + every non-default GPU (default scope; device-agnostic) +``110`` cpu + cuda:0 (pure-math / backend tests: cuda:0 == cuda:N) +``00X`` non-default GPUs only (validates non-default-device behavior) +``100`` cpu only (pure logic) +====== =================================================================== + +Worked example — a ``scope="11X"`` (default) test: + +* single-GPU CI (runtime unset ⇒ ``"110"``) ⇒ ``[cpu, cuda:0]``. +* a multi-GPU shard (runtime ``"0001"``, one device) ⇒ ``[cuda:2]``. + +So argless tests run on cpu + cuda:0 in single-GPU CI and on exactly one +non-default GPU per shard in multi-GPU CI (run-once is a property of the +one-device-per-shard runtime plus round-robin file assignment). + +An empty result means the test is cleanly skipped for this run (e.g. a +``"00X"`` test on a single-GPU host, or a ``"110"`` test on a non-default-GPU +shard). But if the run *explicitly* set ``ISAACLAB_TEST_DEVICES`` to devices +the host does not have, that is a misconfigured run and raises instead of +skipping everything and reporting a vacuous green. + +Local runs +---------- +Set the runtime from the shell to opt a run into non-default GPUs:: + + ISAACLAB_TEST_DEVICES=0001 ISAACLAB_SIM_DEVICE=cuda:2 \\ + ./isaaclab.sh -p -m pytest path/to/test.py +""" + +from __future__ import annotations + +import os + +import torch + +_RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" +"""Env var naming the run's devices: the devices a run may use (see module docstring).""" + +_DEFAULT_RUNTIME = "110" +"""Runtime devices when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset: cpu + cuda:0, +i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" + + +def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> list: + """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. + + ``scope`` is this argument; the runtime comes from the + ``ISAACLAB_TEST_DEVICES`` env var (see the module docstring for the grammar + and the scope / runtime split). + + Args: + scope: Device mask (e.g. ``"11X"``) the test is valid on. Defaults to + ``"11X"`` (cpu + cuda:0 + every non-default GPU): the device-agnostic + common case, so most call sites can use ``test_devices()`` with no + argument. + skip: Optional ``{device: reason}``; in-scope devices listed here are + wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with + the reason instead of being dropped. Use to gate a device variant + that is known broken while keeping it visible. + + Returns: + Ordered device entries (``"cpu"`` / ``"cuda:N"`` strings, or + :func:`pytest.param` for skipped ones) for the second argument to + :func:`pytest.mark.parametrize`. Empty means the test is skipped for + this run. + + Raises: + ValueError: When the run explicitly set ``ISAACLAB_TEST_DEVICES`` to + devices that are not available on this host (a misconfigured run that + would otherwise skip everything and pass vacuously). A scope that + merely does not intersect this run's devices skips, it does not raise. + """ + requested = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + available = _list_available_devices() + runtime_keep = _expand(requested or _DEFAULT_RUNTIME, len(available)) + # A run that explicitly named devices the host cannot provide (e.g. a shard + # pinned to cuda:2 on a 2-GPU host) is misconfigured: fail loudly rather than + # skip every test and report a vacuous green. A scope that simply does not + # intersect this run's devices (a "110" test on a non-default-GPU shard) is + # not an error — it skips. + if requested is not None and not any(runtime_keep): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" + ) + scope_keep = _expand(scope, len(available)) + devices = [ + device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime + ] + if not skip: + return devices + import pytest + + return [ + pytest.param(device, marks=pytest.mark.skip(reason=skip[device])) if device in skip else device + for device in devices + ] + + +def _expand(mask: str, count: int) -> list[bool]: + """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. + + Args: + mask: A mask string (positions plus an optional trailing ``X``). + count: The number of available devices to expand to. + + Returns: + A list of ``count`` booleans, one per device position. + """ + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return ([char == "1" for char in body] + [fill] * count)[:count] + + +def _list_available_devices() -> list[str]: + """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. + + Returns: + Ordered list of device strings as torch addresses them. + """ + devices = ["cpu"] + # torch.cuda (not warp) is deliberate: this runs at pytest collection time, + # before AppLauncher boots Kit. torch.cuda.device_count() enumerates without + # creating a CUDA context, whereas warp.get_cuda_devices() initializes the + # warp runtime — doing that at collection, ahead of Kit's device setup, + # risks the non-default-GPU init-order fragility this suite targets (#5132). + if torch.cuda.is_available(): + devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) + return devices diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py new file mode 100644 index 000000000000..ff09b244e20a --- /dev/null +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -0,0 +1,148 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for :func:`isaaclab.test.utils.test_devices` (the device-selection helper). + +These tests mock the host's device list, so they need no GPU and run on the +single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) +for two reasons: pytest would otherwise collect the ``test_``-prefixed function +as a test case, and the multi-GPU workflow's auto-discovery greps for +``test_devices(...)`` call sites — neither should fire on this file. +""" + +import pytest + +from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import test_devices as resolve_devices + +# Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). +SINGLE_GPU = ["cpu", "cuda:0"] +MULTI_GPU = ["cpu", "cuda:0", "cuda:1", "cuda:2"] + + +@pytest.fixture +def host(monkeypatch): + """Return a setter that pins the available device list and the runtime env var.""" + + def _set(available: list[str], runtime: str | None = None) -> None: + monkeypatch.setattr(devices_mod, "_list_available_devices", lambda: available) + if runtime is None: + monkeypatch.delenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, raising=False) + else: + monkeypatch.setenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, runtime) + + return _set + + +# --------------------------------------------------------------------------- +# scope ∩ runtime resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "available, runtime, scope, expected", + [ + # single-GPU CI lane: runtime unset -> default "110" (cpu + cuda:0). + (SINGLE_GPU, None, "11X", ["cpu", "cuda:0"]), # argless default is device-agnostic + (SINGLE_GPU, None, "110", ["cpu", "cuda:0"]), # pure-math scope + (SINGLE_GPU, None, "100", ["cpu"]), # cpu-only scope + (SINGLE_GPU, None, "00X", []), # non-default-only -> nothing here, skip + # multi-GPU shard: runtime pins exactly one non-default GPU. + (MULTI_GPU, "001", "11X", ["cuda:1"]), # cuda:1 shard + (MULTI_GPU, "0001", "11X", ["cuda:2"]), # cuda:2 shard + (MULTI_GPU, "0001", "00X", ["cuda:2"]), # non-default regression on its shard + (MULTI_GPU, "0001", "110", []), # math skips the shard (cuda:0 != this shard) + (MULTI_GPU, "0001", "100", []), # cpu test skips the shard + # a runtime that lists several GPUs hands back all in-scope ones. + (MULTI_GPU, "111", "11X", ["cpu", "cuda:0", "cuda:1"]), + ], +) +def test_resolves_scope_intersect_runtime(host, available, runtime, scope, expected): + host(available, runtime) + assert resolve_devices(scope) == expected + + +def test_argless_equals_default_scope(host): + # The common case: argless must equal the explicit default mask everywhere. + for available, runtime in [(SINGLE_GPU, None), (MULTI_GPU, "0001"), (MULTI_GPU, "001")]: + host(available, runtime) + assert resolve_devices() == resolve_devices("11X") + + +def test_argless_runs_once_on_one_non_default_gpu(host): + # Run-once property: with a one-device-per-shard runtime, an argless test + # resolves to exactly one non-default GPU. + host(MULTI_GPU, "0001") + result = resolve_devices() + assert result == ["cuda:2"] + assert all(d not in ("cpu", "cuda:0") for d in result) + + +# --------------------------------------------------------------------------- +# skip vs raise: legitimate skips never raise; only a missing runtime device does +# --------------------------------------------------------------------------- + + +def test_unset_runtime_never_raises(host): + # On a dev box / single-GPU lane (runtime unset), an out-of-scope result is a + # silent skip, not an error. + host(SINGLE_GPU, None) + assert resolve_devices("00X") == [] + + +def test_runtime_naming_absent_device_raises(host): + # A run that explicitly asked for cuda:2 on a host that only has cuda:0/cuda:1 + # is misconfigured -> fail loudly instead of a vacuous green. + host(["cpu", "cuda:0", "cuda:1"], "0001") + with pytest.raises(ValueError, match="no device available"): + resolve_devices("11X") + + +def test_in_range_runtime_with_empty_scope_does_not_raise(host): + # The runtime device exists (cuda:2 is present), the scope just doesn't include + # it -> skip, not raise. This is what keeps "110"/"100" tests green on a shard. + host(MULTI_GPU, "0001") + assert resolve_devices("110") == [] # would raise if the guard keyed off scope + + +# --------------------------------------------------------------------------- +# skip= : gate a specific device visibly +# --------------------------------------------------------------------------- + + +def test_skip_wraps_named_device_as_skipped_param(host): + host(SINGLE_GPU, None) + result = resolve_devices("11X", skip={"cuda:0": "known broken"}) + assert result[0] == "cpu" # untouched devices stay plain strings + param = result[1] + assert param.values == ("cuda:0",) + assert param.marks[0].name == "skip" + assert param.marks[0].kwargs["reason"] == "known broken" + + +def test_skip_ignores_out_of_result_devices(host): + # Skipping a device that isn't in the resolved set is a no-op (no stray params). + host(SINGLE_GPU, None) + assert resolve_devices("11X", skip={"cuda:3": "n/a"}) == ["cpu", "cuda:0"] + + +# --------------------------------------------------------------------------- +# _expand mask grammar +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mask, count, expected", + [ + ("110", 2, [True, True]), # exact length, no wildcard + ("100", 3, [True, False, False]), # short mask padded with False + ("11X", 4, [True, True, True, True]), # trailing X fills the rest True + ("00X", 4, [False, False, True, True]), # X spans the non-default GPUs + ("11X", 2, [True, True]), # X with nothing left to fill + ("0001", 3, [False, False, False]), # mask longer than host -> truncated + ], +) +def test_expand(mask, count, expected): + assert devices_mod._expand(mask, count) == expected From 878990a412d6a86b703b20d453bd6219d00c762c Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 04/22] [Newton] Fix Newton/Warp init on non-default CUDA devices Pins torch and Warp to the target device before allocations and scopes the CUDA graph capture to it, so Newton runs correctly on cuda:1+ (issue #5132). --- .../changelog.d/jichuanh-multi-gpu-ci.rst | 5 +++++ .../isaaclab_newton/physics/newton_manager.py | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst new file mode 100644 index 000000000000..c53da690a8d8 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed Newton physics failing to initialize on non-default CUDA devices + (``cuda:1`` and higher). diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index d5991ed395ab..c334a77984b2 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -976,6 +976,15 @@ def start_simulation(cls) -> None: cls._cl_inject_sites_fallback() device = PhysicsManager._device + # Pin torch + Warp to the target device before any Warp/Newton + # allocations. Without this, mujoco_warp's collision pipeline later + # allocates on cuda:1 against a primary CUDA context that was never + # made current, returning null pointers (issue #5132). + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) logger.info(f"Finalizing model on device: {device}") cls._builder.up_axis = Axis.from_string(cls._up_axis) # Forward pending extended attribute requests to builder and clear them @@ -1243,6 +1252,16 @@ def initialize_solver(cls) -> None: if cfg is None: return + # Pin torch + Warp to the target device before solver build and + # collision-pipeline init. Mirrors the guard in :meth:`start_simulation` + # (issue #5132); idempotent if already pinned. + device = PhysicsManager._device + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) + with Timer(name="newton_initialize_solver", msg="Initialize solver took:"): NewtonManager._num_substeps = cfg.num_substeps # type: ignore[union-attr] NewtonManager._collision_decimation = cfg.collision_decimation # type: ignore[union-attr] @@ -1316,7 +1335,7 @@ def _capture_or_defer_graph(cls) -> None: with Timer(name="newton_cuda_graph", msg="CUDA graph took:"): if cls._usdrt_stage is None: simulate = cls._simulate_full if cls._is_all_graphable() else cls._simulate_physics_only - with wp.ScopedCapture() as capture: + with wp.ScopedCapture(device=device) as capture: simulate() NewtonManager._graph = capture.graph logger.info("Newton CUDA graph captured (standard Warp mode)") From b6cf4cf5d675970cb6fdbbd2c5b71f426d9387a9 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 05/22] [CI] Add multi-GPU work-queue sharding and per-file reporting to conftest tools/conftest.py gains a directory-rename work queue (claim/inflight/done) for work-stealing across shards and a per-file report slug to avoid JUnit collisions between same-basename files. A workspace-root conftest skips non-device-parametrized tests on non-default cuda shards, since single-GPU CI already covers them on cuda:0. --- conftest.py | 33 +++++++++ tools/conftest.py | 185 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000000..30d6ae4c40dd --- /dev/null +++ b/conftest.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Workspace-root pytest hooks. + +The only hook today is the multi-GPU shard skip: when a shard pins itself to a +non-default cuda device via :envvar:`ISAACLAB_SIM_DEVICE` (e.g. ``cuda:2``), +skip any test that is not parametrized over a ``device`` argument. Such tests +exercise device-agnostic logic that single-GPU CI already covers on +``cuda:0``, so running them again on every non-default shard duplicates work +without surfacing any new failure mode. + +The hook is a no-op outside the multi-GPU lane (``ISAACLAB_SIM_DEVICE`` unset +or equal to ``cuda:0``), so this file does not change pytest behavior under +single-GPU CI. +""" + +import os + +import pytest + + +def pytest_collection_modifyitems(config, items): + sim_device = os.environ.get("ISAACLAB_SIM_DEVICE", "") + if not sim_device.startswith("cuda:") or sim_device == "cuda:0": + return + skip_marker = pytest.mark.skip(reason=f"non-device-parametrized; covered by single-GPU lane (shard={sim_device})") + for item in items: + callspec = getattr(item, "callspec", None) + if callspec is None or "device" not in callspec.params: + item.add_marker(skip_marker) diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..ad24c85830b4 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -5,6 +5,7 @@ import contextlib import os +import re import select import signal import subprocess @@ -312,6 +313,102 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) +def _slugify_test_path(test_path): + """Encode a test path as a flat queue entry name. + + The queue uses one file per pending test. Slashes are not legal inside a + filename, so we encode the relative path by replacing ``/`` with ``__``. + The decoder is :func:`_unslugify_queue_entry`. + """ + return test_path.replace("/", "__") + + +def _unslugify_queue_entry(entry_name): + """Reverse of :func:`_slugify_test_path`.""" + return entry_name.replace("__", "/") + + +def _claim_queued_file(queue_dir): + """Atomically claim one pending test from the work-queue directory. + + The queue is a directory of files (one per pending test); the shard claims + one by renaming it from ``queue/`` into its private ``inflight/cuda-N/``. + POSIX rename is atomic on the same filesystem, so two shards racing on the + same source file are serialized by the kernel: exactly one rename succeeds, + the other gets ``FileNotFoundError`` and tries the next entry. + + On success, the test's queue entry is now sitting in ``inflight/cuda-N/``; + the caller is expected to move it to ``done/cuda-N/`` after the per-test + pytest invocation exits with a clean result, leaving anything still in + ``inflight/`` at job-end as recoverable evidence of a crashed test. + + Args: + queue_dir: Path to the shared work-queue root. Must contain a + ``queue/`` subdir (pending entries) and an ``inflight//`` + subdir for this shard (claim destination). + + Returns: + The decoded test path for the claimed file, or ``None`` when the + queue is empty. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + pending_dir = os.path.join(queue_dir, "queue") + inflight_dir = os.path.join(queue_dir, "inflight", shard) + os.makedirs(inflight_dir, exist_ok=True) + + # Listdir is intentionally not cached: another shard may have just removed + # an entry we'd otherwise try. We pay one listdir per claim attempt; with + # N≤20 entries this is microseconds. + try: + entries = sorted(os.listdir(pending_dir)) + except FileNotFoundError: + return None + + for entry in entries: + src = os.path.join(pending_dir, entry) + dst = os.path.join(inflight_dir, entry) + try: + os.rename(src, dst) + except FileNotFoundError: + # Lost the race for this entry; another shard claimed it first. + # Continue to the next entry in our (potentially stale) listing. + continue + except OSError: + # Any other rename failure (e.g. permission) is a hard error. + raise + return _unslugify_queue_entry(entry) + + return None + + +def _mark_queued_file_done(queue_dir, test_path): + """Move a successfully-completed claim from ``inflight/cuda-N/`` to ``done/cuda-N/``. + + Called by the test runner after a per-file pytest invocation exits cleanly. + The inflight residual is what the post-run reconciler uses to detect + crashed shards: anything still in ``inflight/`` at job-end is an orphan. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + entry = _slugify_test_path(test_path) + src = os.path.join(queue_dir, "inflight", shard, entry) + dst_dir = os.path.join(queue_dir, "done", shard) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, entry) + # Suppress: already moved (idempotent) or the runner crashed before we + # could mark done — the reconciler catches the second case. + with contextlib.suppress(FileNotFoundError): + os.rename(src, dst) + + +def _queued_files(queue_dir): + """Yield files claimed from the shared work queue until it is empty.""" + while True: + claimed = _claim_queued_file(queue_dir) + if claimed is None: + return + yield claimed + + def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -401,13 +498,21 @@ def _retry_failed_test_in_fresh_process( def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" + """Run each test file separately, ensuring one finishes before starting the next. + + When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue file, files are claimed + from it (work-stealing across sibling shard containers) instead of iterating + ``test_files``; each file still runs once, on this container's pinned GPU. + """ failed_tests = [] test_status = {} xml_reports = [] cold_cache_applied = False - for test_file in test_files: + queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") + file_source = _queued_files(queue_path) if queue_path else test_files + + for test_file in file_source: print(f"\n\n🚀 Running {test_file} independently...\n") file_name = os.path.basename(test_file) env = os.environ.copy() @@ -433,14 +538,25 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + # Prefix the report file with a slug derived from the full test_file + # path so two concurrent shards running same-basename files (e.g. + # ``isaaclab_newton/.../test_articulation.py`` vs + # ``isaaclab_physx/.../test_articulation.py``) don't write to the same + # path inside the shared ``/workspace/isaaclab`` mount and trigger + # false shutdown_hang detections in sibling shards via the + # ``os.path.exists(report_file)`` check at line ~137. + report_slug = str(test_file).replace("/", "__").replace("\\", "__") + report_file = f"tests/test-reports-{report_slug}.xml" + cmd = [ sys.executable, "-m", "pytest", "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit "--no-header", f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", + f"--junitxml={report_file}", "--tb=short", ] @@ -450,8 +566,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): cmd.append(str(test_file)) - report_file = f"tests/test-reports-{str(file_name)}.xml" - # -- Run with retry on startup hang or hard timeout ----------------- returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" wall_time, pre_kill_diag = 0.0, "" @@ -677,6 +791,14 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "wall_time": wall_time, } + # When running under the directory-based work queue (option 2), move the + # claim entry from inflight// to done// so the post-run + # reconciler can distinguish "ran to completion" from "claimed but + # crashed mid-test". A claim that stays in inflight at job-end is a + # silent drop signal. + if queue_path: + _mark_queued_file_done(queue_path, test_file) + print("~~~~~~~~~~~~ Finished running all tests") return failed_tests, test_status, xml_reports @@ -845,6 +967,10 @@ def pytest_sessionstart(session): # Run all tests individually failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) + # In work-queue mode this container ran only the files it claimed; report on those. + if os.environ.get("ISAACLAB_TEST_QUEUE"): + test_files = list(test_status) + print("failed tests:", failed_tests) # Collect reports @@ -861,6 +987,10 @@ def pytest_sessionstart(session): # write content to full report result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") full_report_path = f"tests/{result_file}" + # Ensure the directory exists even when this shard claimed zero files + # from the work queue (per-test JUnit XMLs are what normally create + # ``tests/``; with no tests run there is nothing to create it). + os.makedirs("tests", exist_ok=True) print(f"Using result file: {result_file}") full_report.write(full_report_path) print("~~~~~~~~~~~~ Report written to", full_report_path) @@ -899,14 +1029,17 @@ def pytest_sessionstart(session): summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" + # GPU this run used (the shard's boot device); ``cuda:0`` when unset. + run_device = os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0" + summary_str += "\n\n=======================\n" - summary_str += "Per Test Result Summary\n" + summary_str += "Per File Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_test_result_table.align["Test Path"] = "l" - per_test_result_table.align["Test (s)"] = "r" - per_test_result_table.align["Wall (s)"] = "r" + per_file_result_table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + per_file_result_table.align["Test Path"] = "l" + per_file_result_table.align["Test (s)"] = "r" + per_file_result_table.align["Wall (s)"] = "r" for test_path in test_files: num_tests_passed = ( test_status[test_path]["tests"] @@ -914,9 +1047,10 @@ def pytest_sessionstart(session): - test_status[test_path]["errors"] - test_status[test_path]["skipped"] ) - per_test_result_table.add_row( + per_file_result_table.add_row( [ test_path, + run_device, test_status[test_path]["result"], f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", @@ -924,7 +1058,34 @@ def pytest_sessionstart(session): ] ) - summary_str += per_test_result_table.get_string() + summary_str += per_file_result_table.get_string() + + # Per-test run times, slowest first, from the merged JUnit report. The + # device is read from the test id params (e.g. ``...[size0-cuda:1]``), + # falling back to the run's boot device. + summary_str += "\n\n=================\n" + summary_str += "Per Test Run Time\n" + summary_str += "=================\n" + + per_test_time_table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + per_test_time_table.align["Test"] = "l" + per_test_time_table.align["Time (s)"] = "r" + test_times = [] + for suite in full_report: + for case in suite: + full_name = f"{case.classname}::{case.name}" if case.classname else case.name + device = run_device + bracket = re.search(r"\[(.*)\]", full_name) + if bracket: + dev_match = re.search(r"cuda:\d+|\bcpu\b", bracket.group(1)) + if dev_match: + device = dev_match.group(0) + elapsed = float(case.time) if case.time is not None else 0.0 + test_times.append((full_name, device, elapsed)) + for full_name, device, elapsed in sorted(test_times, key=lambda row: row[2], reverse=True): + per_test_time_table.add_row([full_name, device, f"{elapsed:0.3f}"]) + + summary_str += per_test_time_table.get_string() # Print summary to console and log file print(summary_str) From 6cef107ab0c20659bbd0d04ab86d6b44da839890 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 06/22] [CI] Forward extra env vars through run-tests / run-package-tests actions Adds an extra-env-vars input so the multi-GPU workflow can inject ISAACLAB_TEST_DEVICES / ISAACLAB_SIM_DEVICE into the test container. --- .github/actions/run-package-tests/action.yml | 5 ++++ .github/actions/run-tests/action.yml | 29 +++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 264e01a158b8..4bde3eb22228 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,6 +69,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: 'Extra env vars to forward into the container (one KEY=value per line)' + default: '' + required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -150,6 +154,7 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} + extra-env-vars: ${{ inputs.extra-env-vars }} - name: Check Test Results if: always() diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ceea3a1c4e65..4565f5ddcfd4 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,6 +69,14 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: >- + Extra environment variables to forward into the container, one per line in + ``KEY=value`` form. Whitespace-only lines and lines starting with ``#`` are + ignored. Used by the multi-GPU workflow to inject + ``ISAACLAB_TEST_DEVICES`` / ``ISAACLAB_SIM_DEVICE``. + default: '' + required: false runs: using: composite @@ -93,6 +101,7 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" + local extra_env_vars="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -204,6 +213,24 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + # Caller-supplied extra env vars (one KEY=value per line). Skips + # blank lines and full-line comments (line where the first + # non-whitespace char is ``#``). Mid-line ``#`` is preserved so + # values like ``IMAGE_TAG=v1.0#nightly`` survive. + if [ -n "$extra_env_vars" ]; then + while IFS= read -r line; do + # Strip leading whitespace (YAML ``|`` block can leave indent). + line="${line#"${line%%[![:space:]]*}"}" + [ -z "$line" ] && continue + [ "${line:0:1}" = "#" ] && continue + key="${line%%=*}" + value="${line#*=}" + export "$key"="$value" + docker_env_vars="$docker_env_vars -e $key" + echo "Forwarding extra env var: $key" + done <<< "$extra_env_vars" + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -392,7 +419,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.extra-env-vars }}" - name: Kill container on cancellation if: cancelled() From 71658df0701fe367778090b909c35eb529fefbd2 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 07/22] [CI] Add multi-GPU pytest workflow Adds a workflow that runs the unit-test suite across non-default cuda devices in one container with N parallel pytest shards pulling from a shared work queue, plus the inside-container shard runner it mounts. --- .../multi-gpu/multi_gpu_shard_runner.sh | 138 ++++++ .github/workflows/test-multi-gpu-pytest.yaml | 455 ++++++++++++++++++ .../jichuanh-multi-gpu-ci.minor.rst | 25 + 3 files changed, 618 insertions(+) create mode 100755 .github/actions/multi-gpu/multi_gpu_shard_runner.sh create mode 100644 .github/workflows/test-multi-gpu-pytest.yaml create mode 100644 source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh new file mode 100755 index 000000000000..dc5fcbfb6c1a --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest +# shards. Mounted by both the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) and the local probe under ``data/mgpu-1docker-probe/run.sh``, +# so the logic lives in exactly one place and changes are version-controlled +# + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` so it sits +# next to the workflow that consumes it rather than alongside ``tools/``. +# +# Container expectations (set by the host launcher): +# --gpus all +# --user $host_uid:$host_gid +# -v :/workspace/isaaclab:rw +# -v :/mgpu:rw (subdirs queue/, inflight/, done/) +# -v :/shard-logs:rw (per-shard tee target) +# -e HOME=/tmp/mgpu-base-home (writable tmpfs for pip --user) +# -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) +# -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) +# -e TEST_INCLUDE_FILES="" +# -e ISAACLAB_PIN_KIT_GPU=1 (Kit overrides for the multi-GPU race) +# -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) +# +# Behavior: +# 1. Materializes HOME + PYTHONUSERBASE dirs (tmpfs, world-writable) +# 2. Installs pytest deps (junitparser et al.) into the shared PYTHONUSERBASE +# 3. Derives shard count from nvidia-smi -L (authoritative; torch.cuda.device_count +# under-counts MIG-on-same-parent unless CUDA_VISIBLE_DEVICES enumerates each) +# 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch +# can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) +# 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + +# ISAACLAB_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# /shard-logs/cuda-N.log for the host's grouped re-print after the run +# 6. Waits on every shard before aggregating exit codes — a fast failure doesn't +# tear down still-running siblings + +set +e +cd /workspace/isaaclab +unset HUB__ARGS__DETECT_ONLY DISPLAY + +# Container-level HOME + PYTHONUSERBASE for pip --user installs. The image +# runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so +# HOME defaults to /root which the user cannot write. /tmp/* is on tmpfs +# (1777, world-writable). +# +# PYTHONUSERBASE is the key for the 1-docker shape: pip --user writes to +# ${PYTHONUSERBASE}/lib/python3.12/site-packages, and every Python invocation +# that sees the same env var imports from there. Per-shard subshells below +# override HOME (so .cache / .nvidia-omniverse are isolated) but inherit +# PYTHONUSERBASE so junitparser et al. resolve everywhere. +mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase + +# Pytest deps (same as run-tests action). junitparser is imported at +# tools/conftest.py load time, so it must be present first. +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky py-spy "coverage>=7.6.1" + +# Shard count from nvidia-smi -L (truth; torch under-counts MIG). +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") +if [ "$MIG_COUNT" -gt 0 ]; then + DEV_COUNT=$MIG_COUNT + echo "::notice::container: MIG mode, $MIG_COUNT slices" +else + DEV_COUNT=$GPU_COUNT + echo "::notice::container: discrete mode, $GPU_COUNT GPUs" +fi +if [ "$DEV_COUNT" -lt 2 ]; then + echo "::error::Need at least 2 visible devices; found $DEV_COUNT" + exit 1 +fi + +# Cross-check with torch and cap shard count to what torch can actually +# address. Guards against a CUDA_VISIBLE_DEVICES misconfig silently fanning +# out shards that crash on device access. +TORCH_COUNT=$(/isaac-sim/python.sh -c "import torch; print(torch.cuda.device_count())") +echo "container: torch sees $TORCH_COUNT cuda devices (cross-check vs $DEV_COUNT)" +if [ "$TORCH_COUNT" -lt "$DEV_COUNT" ]; then + echo "::warning::torch sees fewer devices than nvidia-smi — capping shards to $TORCH_COUNT" + DEV_COUNT=$TORCH_COUNT +fi + +# Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME +# (per-shard isolation for .cache, .local/share, etc.) and per-shard +# ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +declare -A pids +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do + zeros="" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done + runtime_devices="${zeros}1" + + shard_home="/tmp/isaaclab-ci-home-${cuda}" + mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ + "${shard_home}/.nvidia-omniverse/config" \ + "${shard_home}/.nvidia-omniverse/logs" + + shard_log="/shard-logs/cuda-${cuda}.log" + + ( + export HOME="$shard_home" + export XDG_CACHE_HOME="${HOME}/.cache" + export XDG_DATA_HOME="${HOME}/.local/share" + export ISAACLAB_TEST_DEVICES="$runtime_devices" + export ISAACLAB_SIM_DEVICE="cuda:${cuda}" + + # Full pytest output captures to $shard_log; live stdout is filtered + # down to high-signal lines (test boundaries, failures, summary stats, + # tracebacks). Kit init chatter, plugin registration, omni.usd Transfer + # logs, etc. only end up in the shard logfile, NOT on the live workflow + # log. The workflow's grouped re-print at end of run still shows the + # full $shard_log under a collapsible ``::group::shard cuda:N log``. + ./isaaclab.sh -p -m pytest \ + --ignore=tools/conftest.py \ + --ignore=source/isaaclab/test/install_ci \ + tools -v 2>&1 \ + | tee "$shard_log" \ + | stdbuf -oL grep -aE \ + '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ + | stdbuf -oL sed "s/^/[cuda:${cuda}] /" + + exit "${PIPESTATUS[0]}" + ) & + pids[$cuda]=$! + echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" +done + +# Wait for every shard before aggregating exits — a fast failure must not +# tear down still-running siblings. +declare -A results +for cuda in "${!pids[@]}"; do + wait "${pids[$cuda]}" + results[$cuda]=$? +done + +fail=0 +for cuda in "${!results[@]}"; do + rc="${results[$cuda]}" + echo "shard cuda:${cuda} exited $rc" + [ "$rc" -eq 0 ] || fail=1 +done +exit $fail diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml new file mode 100644 index 000000000000..55db0e9996c1 --- /dev/null +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -0,0 +1,455 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Multi-GPU unit-test workflow +# +# Runs the non-default-GPU subset of unit tests across the multi-GPU runner +# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the +# single-GPU CI; only diffs are the runner label and the env vars that pin +# Kit and the test parametrize to the shard's non-default GPU. +# +# Adding a new test to multi-GPU coverage: give its device parametrize a +# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()`` +# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow +# auto-discovers it. + +name: Multi-GPU pytest + +on: + pull_request: + paths: + - "source/isaaclab/isaaclab/test/utils/**" + - "source/isaaclab/isaaclab/app/app_launcher.py" + - "source/**/test/**/test_*.py" + - ".github/workflows/test-multi-gpu-pytest.yaml" + - ".github/actions/run-package-tests/**" + - ".github/actions/run-tests/**" + - ".github/actions/ecr-build-push-pull/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + + build: + name: Build / cache image + needs: [config] + # Build only needs Docker + ECR access, not multi-GPU. Aligns with the + # build jobs in build.yaml that also publish the per-commit image. + runs-on: [self-hosted, gpu] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + # Pre-populates the ECR exact-commit tag from deps-cache (registry-side + # alias). Without this prior step, run-package-tests' internal + # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves + # no local image, causing `docker run` to fail with `pull access + # denied`. Mirrors the build → test split in build.yaml. + - uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + test-multi-gpu-pytest: + name: Multi-GPU unit tests + needs: [config, build] + if: needs.build.result == 'success' + # Matches the runner label used by test-multi-gpu.yaml + + # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. + runs-on: [self-hosted, linux, x64, gpu, multi-gpu] + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Discover opt-in test files + # Auto-discovery: any test_*.py with a non-default-capable scope — an + # argless ``test_devices()`` (defaults to "11X") or any + # ``test_devices("..X")`` mask, whose trailing ``X`` spans the + # non-default GPUs — is in scope. Adding a test to multi-GPU CI needs + # no workflow edit; opting a file out is just narrowing its scope (drop + # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. + # + # Within a discovered file, tests that are NOT parametrized over the + # ``device`` argument are filtered out at collection time by the + # workspace-root ``conftest.py``: single-GPU CI already covers them on + # ``cuda:0`` and re-running on every non-default shard adds wall-time + # without surfacing any new failure mode. + id: discover + run: | + # File-level opt-out: a test file can exclude itself from multi-GPU CI + # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line. + # Used for files with known Kit/Isaac-Sim concurrency issues; the file + # still runs in single-GPU CI. Excluded files are reported as a notice + # below for visibility. No workflow edit needed to add/remove a file. + mapfile -t candidates < <(grep -rlE 'test_devices\(\)|test_devices\("[^"]*X"\)' source/ --include='test_*.py' | sort -u) + discovered=() + skipped=() + for f in "${candidates[@]}"; do + if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then + skipped+=("$f") + else + discovered+=("$f") + fi + done + for f in "${skipped[@]}"; do + reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//') + echo "::notice::multi-GPU skipped: $f — $reason" + done + if [ ${#discovered[@]} -eq 0 ]; then + echo "::error::No opt-in tests discovered (grep for test_devices with an X scope)" + exit 1 + fi + basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) + echo "include=$basenames" >> "$GITHUB_OUTPUT" + # Full relative paths too, to seed the shared work queue (the run step + # needs runnable paths, not just basenames). + echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT" + echo "::notice::Discovered ${#discovered[@]} opt-in test files" + printf ' %s\n' "${discovered[@]}" + + - name: Pull image from ECR + # Pulls the per-commit image the build job pushed. ecr-build-push-pull + # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which + # the build job's deps-cache-hit registry-tag created for this SHA) + # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our + # parallel ``docker run`` block below. + uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run shards in parallel on local GPUs (1-docker N-shard) + # ONE container hosts all N pytest shards as parallel subshells. Each + # shard pins ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES to its own + # non-default cuda:N and pulls files from the shared work queue. This + # collapses the previous N-container layout into one container, removing + # cross-container races on the shared workspace mount (``_isaac_sim`` + # symlink, ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker + # init per shard, at the cost of sharing the container's /isaac-sim/ + # writable subtrees. Per-shard HOME (under /tmp) keeps user-level + # caches isolated. + # + # Scaling: shard count is derived inside the container from + # ``nvidia-smi -L`` (truth — torch.cuda.device_count() under-counts MIG + # slices on the same parent GPU), so this works unchanged on any GPU + # count or topology. ``CUDA_VISIBLE_DEVICES`` is set to the + # comma-separated MIG-UUID list only on MIG-mode hosts; discrete-GPU + # runners rely on ``--gpus all``. + env: + IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} + INCLUDE_FILES: ${{ steps.discover.outputs.include }} + PATHS: ${{ steps.discover.outputs.paths }} + run: | + set +e + host_uid="$(id -u)" + host_gid="$(id -g)" + host_user="$(id -un)" + # ``/isaac-sim`` is the Kit install root inside the container; pytest + # invocations under ``./isaaclab.sh -p`` resolve the bundled Python as + # ``./_isaac_sim/python.sh``. Mounting the workspace at + # ``/workspace/isaaclab`` brings the host's CWD into the container, + # so we plant a host-side symlink whose target resolves only inside + # the container. Single creator (this step) — no concurrent creators. + rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + + runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" + mkdir -p "$runtime_dir" + + # Directory-based work queue (atomic os.rename pattern; conftest claims + # entries via os.rename(queue/X, inflight//X) — POSIX rename is + # atomic on the same filesystem, so two shards racing on the same + # entry are kernel-serialized: one wins, the other gets ENOENT and + # moves on). Each pending test path is materialized as an empty file + # in queue/ with '/' slugged to '__' so it's a flat filename. + # + # Subdirs: + # queue/ — pending claims + # inflight// — claimed by this shard, test in flight + # done// — test exited cleanly + # + # At job-end the reconciler step asserts queue/ + inflight/ are empty + # — if either is non-empty the job FAILS (silent-drop signal: never + # claimed OR claimed but crashed mid-test). See the reconciler step + # near the end of this job for the exact assertion. + queue_root="$runtime_dir/queue" + mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" + IFS=',' read -ra _paths <<< "$PATHS" + for path in "${_paths[@]}"; do + [ -n "$path" ] || continue + slug="${path//\//__}" + : > "$queue_root/queue/$slug" + done + echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + + # Per-shard log dir mounted from the container so we can grouped-print + # the per-shard stream after the run completes. + logs_dir="$runtime_dir/logs" + mkdir -p "$logs_dir" + + # MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. + # nvidia-smi -L is authoritative — see memory rule + # ``reference_gpu_enumeration``: torch.cuda.device_count() under-counts + # MIG slices on the same parent GPU. + cvd_args=() + if nvidia-smi -L | grep -q "^ MIG "; then + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" + else + echo "::notice::discrete GPU mode — relying on --gpus all" + fi + + # --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. + docker run --rm --gpus all --network=host \ + --cap-add=SYS_PTRACE \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${{ github.run_id }}-${{ github.run_attempt }}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh + docker_rc=$? + + # Grouped re-print of per-shard logs (preserved across the host). + for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" + done + + # Reconciler — silent-drop guard. Anything left in queue/ was never + # claimed by any shard (work-queue starvation). Anything still in + # inflight// was claimed but never moved to done/ (shard + # crashed mid-test). Either is a silent-drop signal that the docker + # exit alone can't surface. Fail the job and name the orphans. + echo "::group::work-queue reconciler" + unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) + if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' + fi + orphans=0 + for shard_dir in "$queue_root/inflight"/*/; do + [ -d "$shard_dir" ] || continue + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' + orphans=$((orphans + count)) + fi + done + done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) + echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" + echo "::endgroup::" + + # Treat reconciler signals as fatal even if all shards exited 0. + if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 + fi + + # Export the runtime directory so the downstream "Aggregated test + # summary" step can locate the done// work distribution + # and the per-shard logs. + echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" + + exit "$docker_rc" + + - name: Aggregated test summary + # Runs whether or not the test job exit was clean — surfaces what + # happened even on failure. Per-shard stats + per-file pass / total / + # walltime, plus a single combined table. Also writes the same content + # to $GITHUB_STEP_SUMMARY so it renders at the top of the run page. + if: always() + env: + RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} + run: | + python3 - <<'PY' + import os, glob, xml.etree.ElementTree as ET, sys, pathlib + + runtime_dir = os.environ.get("RUNTIME_DIR", "") + done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + + # Map slug -> shard from done//; slug uses "__" for "/". + slug_to_shard = {} + if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-"):] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + # Per-file rows from JUnit XMLs. + rows = [] + for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, + failures + errors, total_time)) + + # Per-shard aggregate. + shards = {} + for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, + "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + # ---------- Output: plain text to stdout ---------- + def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + out = [] + out.append("PER-FILE RESULTS") + out.append("=" * 110) + hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] + widths = [60, 7, 8, 12, 5, 5, 8] + out.append(fmt_row(hdr, widths)) + out.append("-" * 110) + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) + out.append("") + out.append("PER-SHARD AGGREGATE") + out.append("=" * 80) + shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] + shard_widths = [10, 7, 12, 9, 8, 14] + out.append(fmt_row(shard_hdr, shard_widths)) + out.append("-" * 80) + grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + for sh in sorted(shards): + s = shards[sh] + for k in grand: grand[k] += s[k] + out.append(fmt_row( + [sh, str(s["files"]), f"{s['passed']}/{s['tests']}", str(s["skipped"]), + str(s["failed"]), f"{s['time']:.1f}"], shard_widths)) + out.append("-" * 80) + out.append(fmt_row( + ["TOTAL", str(grand["files"]), f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), str(grand["failed"]), f"{grand['time']:.1f}"], + shard_widths)) + print("\n".join(out)) + + # ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + md = ["## Multi-GPU CI: Aggregated Test Summary", "", + "### Per-shard totals", "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|"] + for sh in sorted(shards): + s = shards[sh] + md.append(f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |") + md.append(f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | **{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |") + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = {"PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR"}.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") + PY diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst new file mode 100644 index 000000000000..00b4366e153e --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -0,0 +1,25 @@ +Added +^^^^^ + +* Added :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over + a device set resolved as ``scope ∩ runtime``: ``scope`` is the call-site mask + of devices a test is valid on (default ``"11X"`` ⇒ cpu + cuda:0 + the + non-default GPUs), and the runtime is the ``ISAACLAB_TEST_DEVICES`` env var of + devices a run may use (default ``"110"`` ⇒ cpu + cuda:0). A trailing ``X`` + includes the remaining devices. Single-GPU CI is unchanged; multi-GPU CI sets + the runtime to one non-default GPU per shard. + +* Added ``ISAACLAB_SIM_DEVICE`` env var honored by + :class:`isaaclab.app.AppLauncher` as the implicit-default device when + the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow + boot Kit on a non-default GPU without editing every test's + :class:`~isaaclab.app.AppLauncher` call site. + +* Added ``py-spy`` + ``gdb`` stack capture in ``tools/conftest.py`` on + ``shutdown_hang`` / ``startup_hang`` / ``timeout`` detection. Walks the test + subprocess's process group (cap 8 pids), captures both Python and C++ frames + before ``SIGKILL`` erases them, attaches the output to the JUnit error + report. Makes Kit binary hangs observable in CI logs; safe no-op when + ``py-spy``/``gdb`` are missing. Workflow side adds ``--cap-add=SYS_PTRACE`` + on the per-shard ``docker run`` (required to attach) and adds ``py-spy`` to + the in-container ``pip install`` list. From cef4569956b530ae16d8f0fcc21179b6efc12536 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 08/22] [Tests] Parametrize unit tests over multi-GPU device scope Switches device-parametrized unit tests to test_devices() so they also run on the non-default GPUs in the multi-GPU lane. Mechanical scope change only; no test logic changes. --- .../test/sim/test_newton_model_utils.py | 19 ++-- .../test/sim/test_simulation_context.py | 3 +- .../test/sim/test_views_xform_prim.py | 13 +-- .../isaaclab/test/utils/test_episode_data.py | 11 ++- source/isaaclab/test/utils/test_math.py | 95 +++++++++--------- source/isaaclab/test/utils/test_modifiers.py | 5 +- source/isaaclab/test/utils/test_noise.py | 7 +- .../test/utils/test_wrench_composer.py | 75 +++++++------- .../test/assets/test_articulation.py | 99 ++++++++++--------- .../test/assets/test_rigid_object.py | 41 ++++---- .../assets/test_rigid_object_collection.py | 29 +++--- .../test/sensors/test_contact_sensor.py | 16 +-- .../test/sim/test_views_xform_prim_newton.py | 10 +- .../changelog.d/jichuanh-multi-gpu-ci.skip | 0 .../test/assets/test_articulation.py | 70 ++++++------- .../test/assets/test_rigid_object.py | 40 ++++---- .../assets/test_rigid_object_collection.py | 30 +++--- .../changelog.d/jichuanh-multi-gpu-ci.skip | 0 .../test/assets/test_articulation.py | 91 ++++++++--------- .../test/assets/test_rigid_object.py | 41 ++++---- .../assets/test_rigid_object_collection.py | 29 +++--- .../test/sim/test_views_xform_prim_fabric.py | 26 ++--- 22 files changed, 381 insertions(+), 369 deletions(-) create mode 100644 source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip create mode 100644 source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip diff --git a/source/isaaclab/test/sim/test_newton_model_utils.py b/source/isaaclab/test/sim/test_newton_model_utils.py index cc157074fd42..5cfedaab3354 100644 --- a/source/isaaclab/test/sim/test_newton_model_utils.py +++ b/source/isaaclab/test/sim/test_newton_model_utils.py @@ -27,6 +27,7 @@ _scatter_shape_color_rows_kernel, replace_newton_shape_colors, ) +from isaaclab.test.utils import test_devices _WARNING_MESSAGE = "Newton shape color replacement is enabled; this workaround will be deprecated in a future release." @@ -177,7 +178,7 @@ def _run_scatter_shape_color_rows_kernel( pytest.param((-0.25, 1.75, 0.5), id="oob_clamps_pow"), ], ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_scatter_shape_color_rows_kernel(device: str, linear_rgb: tuple[float, float, float]): """Packed RGB per case; the two parametrized cases jointly cover every ``_linear_channel_to_srgb_warp`` branch.""" after = _run_scatter_shape_color_rows_kernel([linear_rgb], device=device) @@ -284,7 +285,7 @@ def test_replace_newton_shape_colors_warning(): replace_newton_shape_colors(model) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPatch, device: str): """Setting ``ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS`` to ``0`` disables the workaround.""" monkeypatch.setenv("ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS", "0") @@ -315,7 +316,7 @@ def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPa assert not any(issubclass(w.category, FutureWarning) for w in recorded) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_invalid_prim(device: str): """Invalid prim path leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -330,7 +331,7 @@ def test_replace_newton_shape_colors_invalid_prim(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_guide_purpose(device: str): """Guide-purpose mesh leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -349,7 +350,7 @@ def test_replace_newton_shape_colors_guide_purpose(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_no_material_binding(device: str): """No material: ``displayColor`` or unbound gray as linear RGB, then sRGB OETF into ``shape_color``.""" stage = Usd.Stage.CreateInMemory() @@ -381,7 +382,7 @@ def test_replace_newton_shape_colors_no_material_binding(device: str): @pytest.mark.parametrize(("diffuse_color_constant", "diffuse_tint"), _OMNIPBR_ALBEDO_INPUT_CASES) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_omnipbr_binding( device: str, diffuse_color_constant: tuple[float, float, float] | None, @@ -405,7 +406,7 @@ def test_replace_newton_shape_colors_omnipbr_binding( ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_neutral_material(device: str): """Bound ``UsdPreviewSurface`` material leaves ``shape_color`` unchanged.""" stage, mesh_path = _make_preview_surface_bound_mesh_stage() @@ -418,7 +419,7 @@ def test_replace_newton_shape_colors_neutral_material(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_respects_binding_strength(device: str): """Parent stronger-than-descendants binding overrides direct child binding.""" # Scene graph (``ComputeBoundMaterial`` on the mesh yields ParentMat / green, not ChildMat / red): @@ -470,7 +471,7 @@ def test_replace_newton_shape_colors_respects_binding_strength(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_instanced(device: str): """Instance-proxy labels deduplicate via canonical prototype paths in the per-key cache.""" stage = Usd.Stage.CreateInMemory() diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 6ea578a85e30..c96883b839a0 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -6,6 +6,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -43,7 +44,7 @@ def test_setup_teardown(): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_init(device): """Test the simulation context initialization.""" from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 64cd86a7466f..827ed20d3261 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -11,6 +11,7 @@ """ from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices simulation_app = AppLauncher(headless=True).app @@ -101,7 +102,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_toggle(device): """Test toggling visibility multiple times.""" if device == "cuda" and not torch.cuda.is_available(): @@ -129,7 +130,7 @@ def test_visibility_toggle(device): assert vis[0] and not vis[1] and vis[2] -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_parent_inheritance(device): """Making a parent invisible hides all children.""" if device == "cuda" and not torch.cuda.is_available(): @@ -155,7 +156,7 @@ def test_visibility_parent_inheritance(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_prim_ordering_follows_creation_order(device): """Prims are returned in USD creation order (DFS), not alphabetical.""" if device == "cuda" and not torch.cuda.is_available(): @@ -181,7 +182,7 @@ def test_prim_ordering_follows_creation_order(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_standardize_transform_op(device): """FrameView standardizes a prim with xformOp:transform to translate/orient/scale.""" if device == "cuda" and not torch.cuda.is_available(): @@ -209,7 +210,7 @@ def test_standardize_transform_op(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_nested_hierarchy_world_poses(device): """World pose of nested child == sum of parent + child translations.""" if device == "cuda" and not torch.cuda.is_available(): @@ -271,7 +272,7 @@ def test_compare_get_world_poses_with_isaacsim(): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_with_franka_robots(device): """Verify FrameView works with real Franka robot USD assets.""" if device == "cuda" and not torch.cuda.is_available(): diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index a2d570d9d6ef..6062c25d0400 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,10 +5,11 @@ import pytest import torch +from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -18,7 +19,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -55,7 +56,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -102,7 +103,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -114,7 +115,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index c2e5b3550816..4fad6881e56e 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,6 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils +from isaaclab.test.utils import test_devices DECIMAL_PRECISION = 5 """Precision of the test. @@ -22,7 +23,7 @@ """ -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_scale_unscale_transform(device, size): """Test scale_transform and unscale_transform.""" @@ -59,7 +60,7 @@ def test_scale_unscale_transform(device, size): torch.testing.assert_close(output_unscale_offset, inputs) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_saturate(device, size): "Test saturate of a tensor of differed shapes and device." @@ -81,7 +82,7 @@ def test_saturate(device, size): assert torch.all(torch.less_equal(output_per_batch, upper_per_batch)).item() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_normalize(device, size): """Test normalize of a tensor along its last dimension and check the norm of that dimension is close to 1.0.""" @@ -93,7 +94,7 @@ def test_normalize(device, size): torch.testing.assert_close(norm, torch.ones(size[0:-1], device=device)) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_copysign(device): """Test copysign by copying a sign from both a negative and positive value and verify that the new sign is the same. @@ -123,7 +124,7 @@ def test_copysign(device): torch.testing.assert_close(expected_value_neg_dim1_neg, value_neg_dim1_neg) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_is_identity_pose(device): """Test is_identity_pose method.""" # Single row identity pose (xyzw format) @@ -147,7 +148,7 @@ def test_is_identity_pose(device): assert math_utils.is_identity_pose(identity_pos, identity_rot) is False -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat(device): """Test axis_angle_from_quat method.""" # Quaternions of the form (2,4) and (2,2,4) in xyzw format @@ -171,7 +172,7 @@ def test_axis_angle_from_quat(device): torch.testing.assert_close(math_utils.axis_angle_from_quat(quat), angle) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat_approximation(device): """Test the Taylor approximation from axis_angle_from_quat method. @@ -197,7 +198,7 @@ def test_axis_angle_from_quat_approximation(device): torch.testing.assert_close(axis_angle_computed, axis_angle_expected) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_magnitude(device): """Test quat_error_magnitude method.""" # No rotation (xyzw format) @@ -236,7 +237,7 @@ def test_quat_error_magnitude(device): torch.testing.assert_close(q12_diff, expected_diff) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_unique(device): """Test quat_unique method.""" # Define test cases @@ -254,7 +255,7 @@ def test_quat_unique(device): torch.testing.assert_close(pos_real_quats[~non_pos_indices], quats[~non_pos_indices]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_mul_with_quat_unique(device): """Test quat_mul method with different quaternions. @@ -287,7 +288,7 @@ def test_quat_mul_with_quat_unique(device): torch.testing.assert_close(quat_result_3, quat_result_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_mag_with_quat_unique(device): """Test quat_error_magnitude method with positive real quaternions.""" @@ -310,7 +311,7 @@ def test_quat_error_mag_with_quat_unique(device): torch.testing.assert_close(error_4, error_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_convention_converter(device): """Test convert_camera_frame_orientation_convention to and from ros, opengl, and world conventions.""" # Quaternions in xyzw format (converted from original wxyz test values) @@ -348,7 +349,7 @@ def test_convention_converter(device): ) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((10, 4), (5, 3, 4))) def test_convert_quat(device, size): """Test convert_quat from "xyzw" to "wxyz" and back to "xyzw" and verify the correct rolling of the tensor. @@ -383,7 +384,7 @@ def test_convert_quat(device, size): math_utils.convert_quat(quat, to="xwyz") -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_conjugate(device): """Test quat_conjugate by checking the sign of the imaginary part changes but the magnitudes stay the same.""" @@ -397,7 +398,7 @@ def test_quat_conjugate(device): torch.testing.assert_close(expected_real, value[..., 3]) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", (1, 10)) @pytest.mark.parametrize( "euler_angles", @@ -428,7 +429,7 @@ def test_quat_from_euler_xyz(device, num_envs, euler_angles): torch.testing.assert_close(expected_quat, quat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_wrap_to_pi(device): """Test wrap_to_pi method.""" # No wrapping needed @@ -464,7 +465,7 @@ def test_wrap_to_pi(device): torch.testing.assert_close(wrapped_angle, expected_angle) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("shape", ((3,), (1024, 3))) def test_skew_symmetric_matrix(device, shape): """Test skew_symmetric_matrix.""" @@ -494,7 +495,7 @@ def test_skew_symmetric_matrix(device, shape): torch.testing.assert_close(vec_rand_resized[:, 0], mat_value[:, 2, 1]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_orthogonalize_perspective_depth(device): """Test for converting perspective depth to orthogonal depth.""" # Create a sample perspective depth image (N, H, W) @@ -515,7 +516,7 @@ def test_orthogonalize_perspective_depth(device): torch.testing.assert_close(orthogonal_depth, expected_orthogonal_depth) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_combine_frame_transform(device): """Test combine_frame_transforms function.""" # create random poses @@ -540,7 +541,7 @@ def test_combine_frame_transform(device): torch.testing.assert_close(pose01, torch.cat((pos01, quat01), dim=-1)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4]) def test_interpolate_poses(device, seed): """Test interpolate_poses function. @@ -608,7 +609,7 @@ def test_pose_inv(): np.testing.assert_array_almost_equal(result, expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_to_and_from_angle_axis(device): """Test that axis_angle_from_quat against scipy and that quat_from_angle_axis are the inverse of each other.""" n = 1024 @@ -627,7 +628,7 @@ def test_quat_to_and_from_angle_axis(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus(device): """Test quat_box_minus method. @@ -645,7 +646,7 @@ def test_quat_box_minus(device): torch.testing.assert_close(expected_diff, axis_diff, atol=1e-06, rtol=1e-06) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus_and_quat_box_plus(device): """Test consistency of quat_box_plus and quat_box_minus. @@ -687,7 +688,7 @@ def test_quat_box_minus_and_quat_box_plus(device): torch.testing.assert_close(delta_result, delta_angle, atol=1e-04, rtol=1e-04) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t12_inputs", ["True", "False"]) @pytest.mark.parametrize("q12_inputs", ["True", "False"]) def test_combine_frame_transforms(device, t12_inputs, q12_inputs): @@ -726,7 +727,7 @@ def test_combine_frame_transforms(device, t12_inputs, q12_inputs): torch.testing.assert_close(math_utils.quat_unique(expected_quat), math_utils.quat_unique(quat_value)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t02_inputs", ["True", "False"]) @pytest.mark.parametrize("q02_inputs", ["True", "False"]) def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): @@ -766,7 +767,7 @@ def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): torch.testing.assert_close(math_utils.quat_unique(q02_expected), math_utils.quat_unique(q02_compare)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("rot_error_type", ("quat", "axis_angle")) def test_compute_pose_error(device, rot_error_type): """Test compute_pose_error for different rot_error_type.""" @@ -794,7 +795,7 @@ def test_compute_pose_error(device, rot_error_type): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_twist_transform(device): """Test rigid_body_twist_transform method. @@ -822,7 +823,7 @@ def test_rigid_body_twist_transform(device): torch.testing.assert_close(w_AA_, w_AA) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_yaw_quat(device): """ Test for yaw_quat methods. @@ -844,7 +845,7 @@ def test_yaw_quat(device): torch.testing.assert_close(result, expected_output) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_slerp(device): """Test quat_slerp function. @@ -874,7 +875,7 @@ def test_quat_slerp(device): np.testing.assert_array_almost_equal(result.cpu(), expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_matrix_from_quat(device): """test matrix_from_quat against scipy.""" # prepare random quaternions and vectors @@ -893,7 +894,7 @@ def test_matrix_from_quat(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "euler_angles", [ @@ -926,7 +927,7 @@ def test_matrix_from_euler(device, euler_angles, convention): torch.testing.assert_close(expected_mag, mat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply(device): """Test for quat_apply against scipy.""" # prepare random quaternions and vectors @@ -943,7 +944,7 @@ def test_quat_apply(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply_inverse(device): """Test for quat_apply against scipy.""" @@ -963,7 +964,7 @@ def test_quat_apply_inverse(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_inv(device): """Test for quat_inv method. @@ -1040,7 +1041,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1064,7 +1065,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) @@ -1315,7 +1316,7 @@ def test_euler_xyz_from_quat(): torch.testing.assert_close(output, wrapped) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): """Camera above target on +Z axis with Z-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, 5.0]], device=device) @@ -1327,7 +1328,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): """Camera at +Y looking at origin with Y-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 5.0, 0.0]], device=device) @@ -1339,7 +1340,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): """Camera below target looking up (-Z alignment with Z-up) should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, -5.0]], device=device) @@ -1351,7 +1352,7 @@ def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): """When eyes == targets the forward direction is undefined; all entries of the row are NaN.""" eyes = torch.tensor([[1.0, 2.0, 3.0]], device=device) @@ -1360,7 +1361,7 @@ def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_batched_partial_failure(device): """Mixed batch with one degenerate row should produce NaN in that row and a valid rotation in the other.""" eyes = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 5.0]], device=device) @@ -1371,7 +1372,7 @@ def test_create_rotation_matrix_from_view_batched_partial_failure(device): torch.testing.assert_close(torch.linalg.det(R[1]), torch.tensor(1.0, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_unit_norm_on_valid_input(device): """quat_from_matrix should produce unit quaternions for any valid rotation matrix.""" n = 100 @@ -1382,7 +1383,7 @@ def test_quat_from_matrix_unit_norm_on_valid_input(device): torch.testing.assert_close(norms, torch.ones(n, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_singular_matrix_returns_nan(device): """quat_from_matrix on a singular (non-rotation) matrix should signal NaN, not garbage.""" singular = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device) @@ -1390,7 +1391,7 @@ def test_quat_from_matrix_singular_matrix_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_standard(device): """Sanity: off-axis eye produces an orthonormal frame whose z-axis points from target back to eye.""" eyes = torch.tensor([[3.0, 0.0, 4.0]], device=device) @@ -1404,7 +1405,7 @@ def test_create_rotation_matrix_from_view_standard(device): torch.testing.assert_close(R[:, :, 2], expected_z, atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): """Non-finite input (NaN or Inf in eyes/targets) should produce NaN rows.""" eyes = torch.tensor([[float("nan"), 0.0, 0.0]], device=device) @@ -1413,7 +1414,7 @@ def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_reflection_returns_nan(device): """A reflection matrix (det = -1) is not a proper rotation; the safeguard should signal NaN.""" reflection = torch.diag(torch.tensor([1.0, 1.0, -1.0], device=device)).unsqueeze(0) @@ -1421,7 +1422,7 @@ def test_quat_from_matrix_reflection_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_non_orthonormal_returns_nan(device): """A non-orthonormal matrix (1% scale error on one axis) is not a valid rotation; expect NaN.""" R = torch.diag(torch.tensor([1.01, 1.0, 1.0], device=device)).unsqueeze(0) diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index 6e0e39820fdf..b14622e90550 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -9,6 +9,7 @@ import torch import isaaclab.utils.modifiers as modifiers +from isaaclab.test.utils import test_devices from isaaclab.utils.configclass import configclass @@ -142,7 +143,7 @@ def test_torch_relu_modifier(): assert torch.allclose(output, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_digital_filter(device): """Test digital filter modifier.""" # create test data @@ -178,7 +179,7 @@ def test_digital_filter(device): torch.testing.assert_close(processed_data, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_integral(device): """Test integral modifier.""" # create test data diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index 45f9d2d61384..c1035453b5a9 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,9 +7,10 @@ import torch import isaaclab.utils.noise as noise +from isaaclab.test.utils import test_devices -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -42,7 +43,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -77,7 +78,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab/test/utils/test_wrench_composer.py b/source/isaaclab/test/utils/test_wrench_composer.py index b711aaab44a6..2b13cf23a9f2 100644 --- a/source/isaaclab/test/utils/test_wrench_composer.py +++ b/source/isaaclab/test/utils/test_wrench_composer.py @@ -9,6 +9,7 @@ import warp as wp from isaaclab.test.mock_interfaces.assets import MockRigidObjectCollection +from isaaclab.test.utils import test_devices from isaaclab.utils.wrench_composer import WrenchComposer @@ -97,7 +98,7 @@ def random_unit_quaternion_np(rng: np.random.Generator, shape: tuple) -> np.ndar return q -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): @@ -136,7 +137,7 @@ def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_force_np, hand_calculated_composed_force_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int): @@ -175,7 +176,7 @@ def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int) assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): @@ -234,7 +235,7 @@ def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -280,7 +281,7 @@ def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -343,7 +344,7 @@ def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodi assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): @@ -392,7 +393,7 @@ def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -434,7 +435,7 @@ def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -476,7 +477,7 @@ def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: in ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies: int): @@ -540,7 +541,7 @@ def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_local_vs_global_identity_quaternion(device: str): """Test that local and global give same result with identity quaternion and zero position.""" rng = np.random.default_rng(seed=13) @@ -582,7 +583,7 @@ def test_local_vs_global_identity_quaternion(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_90_degree_rotation_global_force(device: str): """Test global force with a known 90-degree rotation for easy verification.""" num_envs, num_bodies = 1, 1 @@ -615,7 +616,7 @@ def test_90_degree_rotation_global_force(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composition_mixed_local_and_global(device: str): """Test that local and global forces can be composed together correctly.""" rng = np.random.default_rng(seed=14) @@ -660,7 +661,7 @@ def test_composition_mixed_local_and_global(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: int): @@ -705,7 +706,7 @@ def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: assert np.allclose(composed_torque_np, expected_torques, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_link_origin_no_torque(device: str): """Test that a global force applied at the link origin produces no torque.""" rng = np.random.default_rng(seed=16) @@ -753,7 +754,7 @@ def test_global_force_at_link_origin_no_torque(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): @@ -819,7 +820,7 @@ def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): ), "add_raw_buffers_from torque mismatch vs direct accumulation" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_add_raw_buffers_from_inactive_is_noop(device: str): """Test that add_raw_buffers_from is a no-op when the source composer is inactive.""" num_envs, num_bodies = 4, 2 @@ -853,7 +854,7 @@ def test_add_raw_buffers_from_inactive_is_noop(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): @@ -908,7 +909,7 @@ def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): ), f"Mask vs index torque mismatch (envs={num_envs}, bodies={num_bodies})" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): @@ -963,7 +964,7 @@ def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_overwrites_previous_add(device: str): """Test that set_forces_and_torques_index clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -992,7 +993,7 @@ def test_set_forces_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_index clears only the targeted environments.""" num_envs, num_bodies = 4, 3 @@ -1063,7 +1064,7 @@ def test_set_forces_clears_targeted_envs_only(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_partial_reset_zeros_only_specified_envs(device: str): """Test that partial reset zeros only the specified environments and leaves others intact.""" num_envs, num_bodies = 8, 3 @@ -1114,7 +1115,7 @@ def test_partial_reset_zeros_only_specified_envs(device: str): assert composer._dirty -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_full_reset_clears_active_flag(device: str): """Test that full reset (no args) clears the _active flag.""" num_envs, num_bodies = 4, 2 @@ -1138,7 +1139,7 @@ def test_full_reset_clears_active_flag(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_force_emits_deprecation_warning(device: str): """Test that accessing composed_force emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1158,7 +1159,7 @@ def test_composed_force_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_force_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_torque_emits_deprecation_warning(device: str): """Test that accessing composed_torque emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1177,7 +1178,7 @@ def test_composed_torque_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_torque_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_add_forces_and_torques_emits_warning(device: str): """Test that the deprecated add_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1202,7 +1203,7 @@ def test_deprecated_add_forces_and_torques_emits_warning(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_overwrites_previous_add(device: str): """Test that set_forces_and_torques_mask clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -1231,7 +1232,7 @@ def test_set_forces_mask_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_mask clears only the masked environments.""" num_envs, num_bodies = 4, 3 @@ -1301,7 +1302,7 @@ def test_set_forces_mask_clears_targeted_envs_only(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_matches_set_forces_index(device: str): """Test that set_forces_and_torques_mask produces the same result as the index variant.""" num_envs, num_bodies = 6, 3 @@ -1351,7 +1352,7 @@ def test_set_forces_mask_matches_set_forces_index(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_force_b_triggers_lazy_composition(device: str): """Test that accessing out_force_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1378,7 +1379,7 @@ def test_out_force_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_torque_b_triggers_lazy_composition(device: str): """Test that accessing out_torque_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1405,7 +1406,7 @@ def test_out_torque_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_lazy_composition_tracks_dirty_flag(device: str): """Test that the dirty flag is correctly managed through add/compose/add cycles.""" num_envs, num_bodies = 2, 1 @@ -1442,7 +1443,7 @@ def test_lazy_composition_tracks_dirty_flag(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), expected, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_compose_is_idempotent(device: str): """Calling compose_to_body_frame twice without intervening writes produces the same result.""" rng = np.random.default_rng(seed=456) @@ -1493,7 +1494,7 @@ def test_compose_is_idempotent(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_with_com_offset(device: str): """Test that torque correction uses CoM position, not link position, when they differ.""" num_envs, num_bodies = 2, 1 @@ -1548,7 +1549,7 @@ def test_global_force_with_com_offset(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_com_no_torque_with_com_offset(device: str): """Test that a global force at CoM position produces zero torque even with CoM offset.""" num_envs, num_bodies = 2, 1 @@ -1594,7 +1595,7 @@ def test_global_force_at_com_no_torque_with_com_offset(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_com_offset_with_rotation(device: str): """Test torque correction with both CoM offset and non-identity rotation.""" num_envs, num_bodies = 1, 1 @@ -1652,7 +1653,7 @@ def test_com_offset_with_rotation(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_emits_warning(device: str): """Test that the deprecated set_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1672,7 +1673,7 @@ def test_deprecated_set_forces_and_torques_emits_warning(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_clears_previous(device: str): """Test that deprecated set_forces_and_torques actually replaces previous values.""" num_envs, num_bodies = 4, 2 diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index e21f520c4324..40540c0ac655 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -479,7 +480,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -529,7 +530,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_articulations", [2, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -569,7 +570,7 @@ def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -619,7 +620,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base(sim, num_articulations, device, articulation_type): """Test initialization for fixed base. @@ -676,7 +677,7 @@ def test_initialization_fixed_base(sim, num_articulations, device, articulation_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -734,7 +735,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) def test_initialization_hand_with_tendons(sim, num_articulations, device, articulation_type): """Test initialization for fixed base articulated hand with tendons. @@ -783,7 +784,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device, articu @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base_made_fixed_base( @@ -837,7 +838,7 @@ def test_initialization_floating_base_made_fixed_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base_made_floating_base( @@ -883,7 +884,7 @@ def test_initialization_fixed_base_made_floating_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -914,7 +915,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_vel(sim, device, articulation_type): """Test that the default joint velocity from configuration is out of range. @@ -939,7 +940,7 @@ def test_out_of_range_default_joint_vel(sim, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1015,7 +1016,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1049,7 +1050,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_buffer(sim, num_articulations, device, articulation_type): """Test if external force buffer correctly updates in the force value is zero case. @@ -1134,7 +1135,7 @@ def test_external_force_buffer(sim, num_articulations, device, articulation_type @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation. @@ -1192,7 +1193,7 @@ def test_external_force_on_single_body(sim, num_articulations, device, articulat @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation at a given position. @@ -1287,7 +1288,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation. @@ -1347,7 +1348,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device, artic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation at a given position. @@ -1441,7 +1442,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_loading_gains_from_usd(sim, num_articulations, device, articulation_type): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1503,7 +1504,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device, articulation_typ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1538,7 +1539,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulation_type): """Test that gains are loaded from the configuration dictionary correctly. @@ -1571,7 +1572,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1643,7 +1644,7 @@ def test_setting_velocity_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1699,7 +1700,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) @@ -1756,7 +1757,7 @@ def test_setting_effort_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1822,7 +1823,7 @@ def test_setting_effort_limit_explicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_reset(sim, num_articulations, device, articulation_type): """Test that reset method works properly.""" @@ -1866,7 +1867,7 @@ def test_reset(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1906,7 +1907,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_body_root_state(sim, num_articulations, device, with_offset, articulation_type): @@ -2031,7 +2032,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset, articulati @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2120,7 +2121,7 @@ def test_write_root_state( torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2139,7 +2140,7 @@ def test_setting_articulation_root_prim_path(sim, device, articulation_type): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2158,7 +2159,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_t @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled, articulation_type): @@ -2264,7 +2265,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) @pytest.mark.skip(reason="Spatial tendons are not supported in Newton yet.") def test_spatial_tendons(sim, num_articulations, device, articulation_type): @@ -2318,7 +2319,7 @@ def test_spatial_tendons(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane, articulation_type): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -2388,7 +2389,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_body_q_consistent_after_root_write(num_articulations, device, articulation_type): """Test that body_q is fresh when collide() runs after a root pose write. @@ -2470,7 +2471,7 @@ def _patched_simulate(cls): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) via view-level APIs.""" @@ -2526,7 +2527,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2550,7 +2551,7 @@ def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_pla @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2583,7 +2584,7 @@ def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail( @@ -2635,7 +2636,7 @@ def test_get_gravity_compensation_forces_not_implemented_on_newton(sim, num_arti @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2661,7 +2662,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2697,7 +2698,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2730,7 +2731,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2756,7 +2757,7 @@ def test_get_mass_matrix_shape_floating_base(sim, num_articulations, device, add assert M.dtype == torch.float32 -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2844,7 +2845,7 @@ def test_heterogeneous_scene_per_view_shapes(sim, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2923,7 +2924,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2974,7 +2975,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3025,7 +3026,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3058,7 +3059,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3135,7 +3136,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 8811de5b9db7..deed30bd5c53 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -121,7 +122,7 @@ def generate_cubes_scene( @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -156,7 +157,7 @@ def test_initialization(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="Newton does not support kinematic rigid bodies") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -194,7 +195,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -212,7 +213,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -229,7 +230,7 @@ def test_initialization_with_articulation_root(num_cubes, device): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -298,7 +299,7 @@ def test_external_force_buffer(device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -373,7 +374,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -462,7 +463,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -530,7 +531,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -573,7 +574,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object via view-level APIs.""" with _newton_sim_context(device, gravity_enabled=True, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -628,7 +629,7 @@ def _set_newton_material_properties(cube_object, friction_val, restitution_val, @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo contact at height=0 does not settle the same as PhysX — cube falls on z-axis") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -668,7 +669,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -681,7 +682,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo uses Coulomb friction (single mu), no static/dynamic distinction") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -761,7 +762,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo restitution model differs from PhysX — inelastic collisions still bounce") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -839,7 +840,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _newton_sim_context(device, gravity_enabled=False, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -877,7 +878,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -953,7 +954,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1066,7 +1067,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1137,7 +1138,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): @@ -1297,7 +1298,7 @@ def test_warmup_attach_stage_not_called_for_cpu(): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_link_pose_w_fresh_after_root_pose_write(device, writer): diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index 18928bedb5d8..9d5355adc31b 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -119,7 +120,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -151,7 +152,7 @@ def test_initialization(num_envs, num_cubes, device): @pytest.mark.skip(reason="Newton doesn't support kinematic rigid bodies yet") @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -187,7 +188,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -202,7 +203,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -258,7 +259,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -320,7 +321,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -403,7 +404,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_object_state(num_envs, num_cubes, device): """Test setting the state of the object. @@ -475,7 +476,7 @@ def test_set_object_state(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -510,7 +511,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object collection via view-level APIs.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -549,7 +550,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -622,7 +623,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset): """Test the object_com_state_w and object_link_state_w properties.""" @@ -717,7 +718,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_object_state(num_envs, num_cubes, device, with_offset, state_location): @@ -795,7 +796,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_object_state_functions_data_consistency(num_envs, num_cubes, device, with_offset, state_location): @@ -932,7 +933,7 @@ def test_write_object_state_functions_data_consistency(num_envs, num_cubes, devi torch.testing.assert_close(body_com_vel_w[..., 3:], link_vel_w[..., 3:]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_pose_write_marks_fk_reset_mask(device, writer): diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 2d92e3274a7e..66452b62d553 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -20,6 +20,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import math @@ -74,7 +76,7 @@ class ContactSensorTestSceneCfg(InteractiveSceneCfg): # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -193,7 +195,7 @@ def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: S assert no_contact_detected[env_idx], f"Env {env_idx}: Contact should stop after lift." -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -298,7 +300,7 @@ def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): """Test that resting object contact force equals weight and points upward. @@ -401,7 +403,7 @@ def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): assert not errs, "\n".join(errs) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contacts: bool): """Test that dropping from higher produces larger peak impact force. @@ -483,7 +485,7 @@ def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contac # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -618,7 +620,7 @@ def test_filter_enables_force_matrix(device: str, use_mujoco_contacts: bool): } -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -815,7 +817,7 @@ def _make_two_box_scene_cfg(num_envs: int) -> ContactSensorTestSceneCfg: return scene_cfg -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_sensor_metadata(device: str): """Verify sensor_names and filter_object_names match the underlying sensing and counterpart configuration across body-mode, body-mode-with-filter, and shape-mode. diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index d114a1da2a80..9ff6c7a8610a 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -13,6 +13,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) @@ -102,7 +104,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_body_path(device): """FrameView rejects prim paths that resolve to a Newton physics body.""" ctx = _sim_context(device, num_envs=2) @@ -116,7 +118,7 @@ def test_reject_body_path(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_shape_path(device): """FrameView rejects prim paths that resolve to a Newton collision shape.""" ctx = _sim_context(device, num_envs=2) @@ -183,7 +185,7 @@ def test_view_can_resolve_from_body_labels_after_reset(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_returns_initial_pose(device): """A world-rooted frame returns its configured position.""" ctx = _sim_context(device, num_envs=2) @@ -201,7 +203,7 @@ def test_world_attached_returns_initial_pose(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_set_world_roundtrip(device): """A world-attached prim can be repositioned via set_world_poses.""" ctx = _sim_context(device, num_envs=2) diff --git a/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index 02f382f3f319..fcb1e2374642 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,6 +56,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -328,7 +330,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): @@ -391,7 +393,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): @@ -455,7 +457,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -526,7 +528,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): @@ -598,7 +600,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -657,7 +659,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_OMNI_PHYSX_SCHEMAS_GAP_REASON, strict=False) @@ -725,7 +727,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): @@ -784,7 +786,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): @@ -815,7 +817,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -840,7 +842,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @@ -916,7 +918,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -949,7 +951,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -1034,7 +1036,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1092,7 +1094,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1187,7 +1189,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1247,7 +1249,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1341,7 +1343,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1403,7 +1405,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): @@ -1438,7 +1440,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1471,7 +1473,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1539,7 +1541,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.isaacsim_ci @@ -1593,7 +1595,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.isaacsim_ci @@ -1646,7 +1648,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.isaacsim_ci @@ -1708,7 +1710,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" @@ -1752,7 +1754,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @@ -1792,7 +1794,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci def test_body_root_state(sim, num_articulations, device, with_offset): @@ -1919,7 +1921,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2004,7 +2006,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, device): """Test the data.body_incoming_joint_wrench_b buffer is populated correctly and statically correct for single joint. @@ -2105,7 +2107,7 @@ def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, devic ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2124,7 +2126,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2143,7 +2145,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): @@ -2249,7 +2251,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2301,7 +2303,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2394,7 +2396,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 7c9567ade19c..d896a59975ce 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,6 +32,8 @@ import warp as wp from flaky import flaky +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -175,7 +177,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -208,7 +210,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -245,7 +247,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -262,7 +264,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -278,7 +280,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -346,7 +348,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -423,7 +425,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -527,7 +529,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -592,7 +594,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -635,7 +637,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -644,7 +646,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -653,7 +655,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -662,7 +664,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -677,7 +679,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -691,7 +693,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -735,7 +737,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -774,7 +776,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -891,7 +893,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -963,7 +965,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index f17b195e8341..470e97d3cb15 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,6 +28,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -171,7 +173,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -200,7 +202,7 @@ def test_initialization(num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_id_conversion(device): """Test environment and object index conversion to physics view indices.""" @@ -240,7 +242,7 @@ def test_id_conversion(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -276,7 +278,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -291,7 +293,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" @@ -347,7 +349,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" @@ -409,7 +411,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -492,7 +494,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @@ -563,7 +565,7 @@ def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -663,7 +665,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravi @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -742,7 +744,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" @@ -778,7 +780,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -787,7 +789,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @@ -822,7 +824,7 @@ def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index af36a365cb66..b13e10d539bf 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -310,7 +311,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -366,7 +367,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -423,7 +424,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -487,7 +488,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -552,7 +553,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -605,7 +606,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base articulation made fixed-base using schema properties. @@ -665,7 +666,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -717,7 +718,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -747,7 +748,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -771,7 +772,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -846,7 +847,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -879,7 +880,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -963,7 +964,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1020,7 +1021,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1114,7 +1115,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1173,7 +1174,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1266,7 +1267,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1327,7 +1328,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1361,7 +1362,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1393,7 +1394,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1460,7 +1461,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1513,7 +1514,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1565,7 +1566,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1627,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1669,7 +1670,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1708,7 +1709,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1831,7 +1832,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1912,7 +1913,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1930,7 +1931,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1948,7 +1949,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2053,7 +2054,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2105,7 +2106,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2198,7 +2199,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) of articulation shapes.""" @@ -2245,7 +2246,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2262,7 +2263,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2289,7 +2290,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2314,7 +2315,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2366,7 +2367,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2402,7 +2403,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2453,7 +2454,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2487,7 +2488,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2576,7 +2577,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2634,7 +2635,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2798,7 +2799,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2832,7 +2833,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index b7f422c4f2f0..f55761b4bdad 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -98,7 +99,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -132,7 +133,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -170,7 +171,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -188,7 +189,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -205,7 +206,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -274,7 +275,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -350,7 +351,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -455,7 +456,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -521,7 +522,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -564,7 +565,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -605,7 +606,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -645,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -694,7 +695,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -705,7 +706,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -791,7 +792,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -874,7 +875,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -918,7 +919,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -958,7 +959,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -1069,7 +1070,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -1139,7 +1140,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index f42f6dfcb085..4ec56828af82 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -110,7 +111,7 @@ def sim(request): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(sim, num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -137,7 +138,7 @@ def test_initialization(sim, num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(sim, device): """Test environment and object index conversion to physics view indices.""" object_collection, _ = generate_cubes_scene(num_envs=2, num_cubes=3, device=device) @@ -175,7 +176,7 @@ def test_id_conversion(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" object_collection, origins = generate_cubes_scene( @@ -209,7 +210,7 @@ def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(sim, num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_cubes=num_cubes, has_api=False, device=device) @@ -222,7 +223,7 @@ def test_initialization_with_no_rigid_body(sim, num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -276,7 +277,7 @@ def test_external_force_buffer(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object.""" object_collection, origins = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -336,7 +337,7 @@ def test_external_force_on_single_body(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -415,7 +416,7 @@ def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, dev @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -480,7 +481,7 @@ def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -576,7 +577,7 @@ def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -656,7 +657,7 @@ def test_write_object_state(sim, num_envs, num_cubes, device, with_offset, state @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(sim, num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -689,7 +690,7 @@ def test_reset_object_collection(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(sim, num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -724,7 +725,7 @@ def test_set_material_properties(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -757,7 +758,7 @@ def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index 3cfe70095fd3..9a1c5e9e34ea 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,7 +10,6 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path @@ -30,6 +29,7 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 +from isaaclab.test.utils import test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) @@ -126,7 +126,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ------------------------------------------------------------------ -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.xfail( reason=( "Issue #5: FabricFrameView.set_world_poses writes to Fabric worldMatrix only. " @@ -155,7 +155,7 @@ def _fill_position(out: wp.array(dtype=wp.float32, ndim=2), x: float, y: float, out[i, 2] = wp.float32(z) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): """Verify that set_world_poses in Fabric mode does NOT sync back to USD. @@ -199,7 +199,7 @@ def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_rebuild_after_topology_change(device, view_factory, monkeypatch): """Forcing the topology-changed branch on a write triggers :meth:`_rebuild_fabric_arrays` and leaves the view in a state where @@ -252,11 +252,7 @@ def force_topology_changed(): # ------------------------------------------------------------------ -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -276,11 +272,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -308,11 +300,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """set_scales -> get_scales roundtrip works on cuda:1. From 5715dcb2a6fa7154b699eabce21ab7d79a1802b4 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:57:18 +0000 Subject: [PATCH 09/22] [App] Honor ISAACLAB_SIM_DEVICE as implicit default device in AppLauncher When the caller does not pass an explicit device, AppLauncher reads ISAACLAB_SIM_DEVICE and uses it as the device. Lets the multi-GPU CI lane boot Kit on a non-default GPU without editing every test's AppLauncher() call site. --- source/isaaclab/isaaclab/app/app_launcher.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 53dd8065aa11..2474d0703047 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1003,6 +1003,17 @@ def _resolve_device_settings(self, launcher_args: dict): device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) + # When the caller didn't pin a device, allow ISAACLAB_SIM_DEVICE to + # override the default. Used by the multi-GPU CI workflow to boot + # Kit with active_gpu=1 without editing every test's AppLauncher() + # call site; Kit's active_gpu is process-global and locked after + # SimulationApp init, so per-test selection cannot retarget it. + if not device_explicitly_passed: + env_device = os.environ.get("ISAACLAB_SIM_DEVICE") + if env_device: + device = env_device + launcher_args["device"] = env_device + if self._xr and not device_explicitly_passed: # If no device is specified, default to the CPU device if we are running in XR device = "cpu" From c514199d41ac37de3e00295b0254cf6f2b5af9d4 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:57:18 +0000 Subject: [PATCH 10/22] TEMP: skip docker/docs/install-ci CI while iterating #5823 Speeds up iteration on the multi-GPU lane: forces run_docker_tests=false in build.yaml and gates docs.yaml / install-ci.yml behind a DO-NOT-MERGE PR-title check. Revert this commit before the PR is merged. --- .github/workflows/build.yaml | 7 ++++++- .github/workflows/docs.yaml | 15 +++++++++++++-- .github/workflows/install-ci.yml | 5 +++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1c599ebcb988..2428799e72d9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,12 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} + # TEMP (revert before final review / before landing): force + # run_docker_tests=false while iterating PR #5823. All gated + # test/build jobs skip via their existing if-gate; the + # single-GPU Docker + Tests matrix won't burn the gpu pool on + # every push. Per ~/.claude/skills/pr/ci-iteration-shortcut.md. + run_docker_tests: 'false' steps: - id: detect env: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 013be3a5b126..098d9ba784b0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -25,6 +25,11 @@ jobs: doc-build-type: name: Detect Doc Build Type runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip docs build entirely when the PR title is + # marked DO-NOT-MERGE (scratch / hypothesis-only branches don't need a + # full doc rebuild on every push). Push events on main/develop/release + # still run because they don't carry a PR title. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: trigger-deploy: ${{ steps.trigger-deploy.outputs.defined }} steps: @@ -42,8 +47,14 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # run on non-deploy branches to build current version docs only - if: needs.doc-build-type.outputs.trigger-deploy != 'true' + # Run on non-deploy branches to build current version docs only AND skip + # for DO-NOT-MERGE diagnostic PRs (the needed doc-build-type job already + # skips for those, but GitHub Actions evaluates `needs.X.outputs.Y` as + # empty when X is skipped, which keeps this `!= 'true'` gate true — so + # we re-check the title here to actually cascade the skip). + if: | + needs.doc-build-type.outputs.trigger-deploy != 'true' + && !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 2a2fee50c8ab..df3827bfb870 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -34,6 +34,11 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip install-ci entirely when the PR title is + # marked DO-NOT-MERGE. The downstream install-tests-x86 / install-tests-arm + # jobs gate on `needs.changes.outputs.run_install_tests == 'true'`; with + # this job skipped the output is empty and both dependents skip too. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} steps: From 72128e7b46f45b0ab06bdfbcff675537a76bca5f Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 05:01:54 +0000 Subject: [PATCH 11/22] Fix multi-GPU test job to target the multi-GPU runner pool The test job's runs-on carried the `gpu` label, which in the self-hosted fleet tags single-GPU runners. Requiring both `gpu` and `multi-gpu` routed the job onto a single-GPU box, so the shard runner aborted with "Need at least 2 visible devices; found 1"; the over-constrained label also left the job queued for hours. Drop `gpu` so the job targets the multi-GPU pool. --- .github/workflows/test-multi-gpu-pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 55db0e9996c1..b93237d27b25 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -89,7 +89,7 @@ jobs: if: needs.build.result == 'success' # Matches the runner label used by test-multi-gpu.yaml + # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. - runs-on: [self-hosted, linux, x64, gpu, multi-gpu] + runs-on: [self-hosted, linux, x64, multi-gpu] timeout-minutes: 60 steps: - name: Checkout repository From 9d159f39c272d02abb2084119308abf68edf32e6 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 05:05:19 +0000 Subject: [PATCH 12/22] Extract multi-GPU workflow scripts to sibling files Move the host-orchestration bash (symlink, work-queue seed, MIG detection, docker run, reconciler) and the JUnit-XML aggregation Python out of the workflow's inline run: blocks into version-controlled, lint-able scripts under .github/actions/multi-gpu/, alongside the existing multi_gpu_shard_runner.sh. The workflow drops from 456 to 181 lines and each step becomes a one-line call. Behavior-preserving: data still flows via step env, $GITHUB_OUTPUT, $GITHUB_ENV, and $GITHUB_STEP_SUMMARY; the container --name now uses the runner built-ins $GITHUB_RUN_ID/$GITHUB_RUN_ATTEMPT in place of the YAML-only ${{ github.run_id }} templating. Also documents why the test job must not carry the gpu label. --- .../multi-gpu/aggregate_test_summary.py | 176 +++++++++++ .../multi-gpu/multi_gpu_host_launcher.sh | 158 ++++++++++ .github/workflows/test-multi-gpu-pytest.yaml | 298 +----------------- 3 files changed, 347 insertions(+), 285 deletions(-) create mode 100755 .github/actions/multi-gpu/aggregate_test_summary.py create mode 100755 .github/actions/multi-gpu/multi_gpu_host_launcher.sh diff --git a/.github/actions/multi-gpu/aggregate_test_summary.py b/.github/actions/multi-gpu/aggregate_test_summary.py new file mode 100755 index 000000000000..6e1bec25f2f7 --- /dev/null +++ b/.github/actions/multi-gpu/aggregate_test_summary.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate per-shard JUnit XML reports from the multi-GPU pytest lane. + +Reads ``RUNTIME_DIR`` (the host launcher's work-queue root, holding +``queue/done//``) and the ``tests/test-reports-*.xml`` JUnit files, +then prints a per-file + per-shard table to stdout and, when +``GITHUB_STEP_SUMMARY`` is set, writes the same content as markdown so it +renders at the top of the run page. +""" + +import glob +import os +import pathlib +import xml.etree.ElementTree as ET + +runtime_dir = os.environ.get("RUNTIME_DIR", "") +done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + +# Map slug -> shard from done//; slug uses "__" for "/". +slug_to_shard = {} +if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + +def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-") :] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + +# Per-file rows from JUnit XMLs. +rows = [] +for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, failures + errors, total_time)) + +# Per-shard aggregate. +shards = {} +for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + +# ---------- Output: plain text to stdout ---------- +def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + +out = [] +out.append("PER-FILE RESULTS") +out.append("=" * 110) +hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] +widths = [60, 7, 8, 12, 5, 5, 8] +out.append(fmt_row(hdr, widths)) +out.append("-" * 110) +for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) +out.append("") +out.append("PER-SHARD AGGREGATE") +out.append("=" * 80) +shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] +shard_widths = [10, 7, 12, 9, 8, 14] +out.append(fmt_row(shard_hdr, shard_widths)) +out.append("-" * 80) +grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} +for sh in sorted(shards): + s = shards[sh] + for k in grand: + grand[k] += s[k] + out.append( + fmt_row( + [ + sh, + str(s["files"]), + f"{s['passed']}/{s['tests']}", + str(s["skipped"]), + str(s["failed"]), + f"{s['time']:.1f}", + ], + shard_widths, + ) + ) +out.append("-" * 80) +out.append( + fmt_row( + [ + "TOTAL", + str(grand["files"]), + f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), + str(grand["failed"]), + f"{grand['time']:.1f}", + ], + shard_widths, + ) +) +print("\n".join(out)) + +# ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- +summary_path = os.environ.get("GITHUB_STEP_SUMMARY") +if summary_path: + md = [ + "## Multi-GPU CI: Aggregated Test Summary", + "", + "### Per-shard totals", + "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|", + ] + for sh in sorted(shards): + s = shards[sh] + md.append( + f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |" + ) + md.append( + f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | " + f"**{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |" + ) + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = { + "PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR", + }.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh new file mode 100755 index 000000000000..332d5bd73a20 --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Runs ON THE HOST runner — the counterpart to multi_gpu_shard_runner.sh, which +# runs INSIDE the container. Launches ONE container that hosts all N pytest +# shards as parallel subshells, then reconciles the shared work queue. +# +# Invoked by .github/workflows/test-multi-gpu-pytest.yaml's "Run shards in +# parallel" step. Reads from the environment (set by that step): +# IMAGE_TAG — per-commit CI image to ``docker run`` +# INCLUDE_FILES — comma-sep test basenames (conftest include filter) +# PATHS — comma-sep relative test paths (seed the work queue) +# plus the runner built-ins GITHUB_RUN_ID / GITHUB_RUN_ATTEMPT (container name) +# and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). +# +# 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its +# own non-default cuda:N via ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES, pulling +# from the shared queue). Collapsing the previous N-container layout into one +# removes cross-container races on the workspace mount (``_isaac_sim`` symlink, +# ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at +# the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, +# set in the shard runner) keeps user-level caches isolated. +# +# Scaling: shard count is derived inside the container from ``nvidia-smi -L`` +# (truth — torch.cuda.device_count() under-counts MIG slices on the same parent +# GPU), so this works unchanged on any GPU count/topology. CUDA_VISIBLE_DEVICES +# is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete +# runners rely on ``--gpus all``. + +set +e +host_uid="$(id -u)" +host_gid="$(id -g)" +host_user="$(id -un)" +# ``/isaac-sim`` is the Kit install root inside the container; pytest +# invocations under ``./isaaclab.sh -p`` resolve the bundled Python as +# ``./_isaac_sim/python.sh``. Mounting the workspace at ``/workspace/isaaclab`` +# brings the host's CWD into the container, so we plant a host-side symlink +# whose target resolves only inside the container. Single creator (this step) +# — no concurrent creators. +rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + +runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" +mkdir -p "$runtime_dir" + +# Directory-based work queue (atomic os.rename pattern; conftest claims entries +# via os.rename(queue/X, inflight//X) — POSIX rename is atomic on the +# same filesystem, so two shards racing on the same entry are kernel-serialized: +# one wins, the other gets ENOENT and moves on). Each pending test path is +# materialized as an empty file in queue/ with '/' slugged to '__' so it's a +# flat filename. +# +# Subdirs: +# queue/ — pending claims +# inflight// — claimed by this shard, test in flight +# done// — test exited cleanly +# +# At job-end the reconciler below asserts queue/ + inflight/ are empty — if +# either is non-empty the job FAILS (silent-drop signal: never claimed OR +# claimed but crashed mid-test). +queue_root="$runtime_dir/queue" +mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" +IFS=',' read -ra _paths <<< "$PATHS" +for path in "${_paths[@]}"; do + [ -n "$path" ] || continue + slug="${path//\//__}" + : > "$queue_root/queue/$slug" +done +echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + +# Per-shard log dir mounted from the container so we can grouped-print the +# per-shard stream after the run completes. +logs_dir="$runtime_dir/logs" +mkdir -p "$logs_dir" + +# MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. nvidia-smi -L +# is authoritative — torch.cuda.device_count() under-counts MIG slices on the +# same parent GPU. +cvd_args=() +if nvidia-smi -L | grep -q "^ MIG "; then + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" +else + echo "::notice::discrete GPU mode — relying on --gpus all" +fi + +# --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. +docker run --rm --gpus all --network=host \ + --cap-add=SYS_PTRACE \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh +docker_rc=$? + +# Grouped re-print of per-shard logs (preserved across the host). +for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" +done + +# Reconciler — silent-drop guard. Anything left in queue/ was never claimed by +# any shard (work-queue starvation). Anything still in inflight// was +# claimed but never moved to done/ (shard crashed mid-test). Either is a +# silent-drop signal that the docker exit alone can't surface. Fail the job and +# name the orphans. +echo "::group::work-queue reconciler" +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) +if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' +fi +orphans=0 +for shard_dir in "$queue_root/inflight"/*/; do + [ -d "$shard_dir" ] || continue + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' + orphans=$((orphans + count)) + fi +done +done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) +echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" +echo "::endgroup::" + +# Treat reconciler signals as fatal even if all shards exited 0. +if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 +fi + +# Export the runtime directory so the downstream "Aggregated test summary" step +# can locate the done// work distribution and the per-shard logs. +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" + +exit "$docker_rc" diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index b93237d27b25..10229766c421 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -87,8 +87,10 @@ jobs: name: Multi-GPU unit tests needs: [config, build] if: needs.build.result == 'success' - # Matches the runner label used by test-multi-gpu.yaml + - # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. + # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here: + # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the + # job onto a 1-GPU box and the shard runner aborts with "Need at least 2 + # visible devices; found 1". runs-on: [self-hosted, linux, x64, multi-gpu] timeout-minutes: 60 steps: @@ -161,295 +163,21 @@ jobs: cache-tag: cache-base - name: Run shards in parallel on local GPUs (1-docker N-shard) - # ONE container hosts all N pytest shards as parallel subshells. Each - # shard pins ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES to its own - # non-default cuda:N and pulls files from the shared work queue. This - # collapses the previous N-container layout into one container, removing - # cross-container races on the shared workspace mount (``_isaac_sim`` - # symlink, ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker - # init per shard, at the cost of sharing the container's /isaac-sim/ - # writable subtrees. Per-shard HOME (under /tmp) keeps user-level - # caches isolated. - # - # Scaling: shard count is derived inside the container from - # ``nvidia-smi -L`` (truth — torch.cuda.device_count() under-counts MIG - # slices on the same parent GPU), so this works unchanged on any GPU - # count or topology. ``CUDA_VISIBLE_DEVICES`` is set to the - # comma-separated MIG-UUID list only on MIG-mode hosts; discrete-GPU - # runners rely on ``--gpus all``. + # ONE container hosts all N pytest shards as parallel subshells, each + # pinned to its own non-default cuda:N and pulling from a shared work + # queue; the host reconciles the queue afterward and exports + # MGPU_RUNTIME_DIR for the summary step. Full rationale + the + # 1-docker-N-shard tradeoffs live in the script header. env: IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} INCLUDE_FILES: ${{ steps.discover.outputs.include }} PATHS: ${{ steps.discover.outputs.paths }} - run: | - set +e - host_uid="$(id -u)" - host_gid="$(id -g)" - host_user="$(id -un)" - # ``/isaac-sim`` is the Kit install root inside the container; pytest - # invocations under ``./isaaclab.sh -p`` resolve the bundled Python as - # ``./_isaac_sim/python.sh``. Mounting the workspace at - # ``/workspace/isaaclab`` brings the host's CWD into the container, - # so we plant a host-side symlink whose target resolves only inside - # the container. Single creator (this step) — no concurrent creators. - rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim - - runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" - mkdir -p "$runtime_dir" - - # Directory-based work queue (atomic os.rename pattern; conftest claims - # entries via os.rename(queue/X, inflight//X) — POSIX rename is - # atomic on the same filesystem, so two shards racing on the same - # entry are kernel-serialized: one wins, the other gets ENOENT and - # moves on). Each pending test path is materialized as an empty file - # in queue/ with '/' slugged to '__' so it's a flat filename. - # - # Subdirs: - # queue/ — pending claims - # inflight// — claimed by this shard, test in flight - # done// — test exited cleanly - # - # At job-end the reconciler step asserts queue/ + inflight/ are empty - # — if either is non-empty the job FAILS (silent-drop signal: never - # claimed OR claimed but crashed mid-test). See the reconciler step - # near the end of this job for the exact assertion. - queue_root="$runtime_dir/queue" - mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" - IFS=',' read -ra _paths <<< "$PATHS" - for path in "${_paths[@]}"; do - [ -n "$path" ] || continue - slug="${path//\//__}" - : > "$queue_root/queue/$slug" - done - echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" - - # Per-shard log dir mounted from the container so we can grouped-print - # the per-shard stream after the run completes. - logs_dir="$runtime_dir/logs" - mkdir -p "$logs_dir" - - # MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. - # nvidia-smi -L is authoritative — see memory rule - # ``reference_gpu_enumeration``: torch.cuda.device_count() under-counts - # MIG slices on the same parent GPU. - cvd_args=() - if nvidia-smi -L | grep -q "^ MIG "; then - mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') - MIG_LIST=$(IFS=,; echo "${MIGS[*]}") - cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") - echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" - else - echo "::notice::discrete GPU mode — relying on --gpus all" - fi - - # --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. - docker run --rm --gpus all --network=host \ - --cap-add=SYS_PTRACE \ - --entrypoint bash \ - --user "${host_uid}:${host_gid}" \ - --name "isaac-lab-mgpu-${{ github.run_id }}-${{ github.run_attempt }}" \ - -v "$PWD:/workspace/isaaclab:rw" \ - -v "$queue_root:/mgpu:rw" \ - -v "$logs_dir:/shard-logs:rw" \ - -e USER="${host_user}" \ - -e LOGNAME="${host_user}" \ - -e OMNI_KIT_ACCEPT_EULA=yes \ - -e ACCEPT_EULA=Y \ - -e OMNI_KIT_DISABLE_CUP=1 \ - -e ISAAC_SIM_HEADLESS=1 \ - -e ISAAC_SIM_LOW_MEMORY=1 \ - -e PYTHONUNBUFFERED=1 \ - -e PYTHONIOENCODING=utf-8 \ - -e ISAACLAB_TEST_QUEUE=/mgpu \ - -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ - -e ISAACLAB_PIN_KIT_GPU=1 \ - -e HOME=/tmp/mgpu-base-home \ - -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ - "${cvd_args[@]}" \ - -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ - "$IMAGE_TAG" \ - /multi_gpu_shard_runner.sh - docker_rc=$? - - # Grouped re-print of per-shard logs (preserved across the host). - for log in "$logs_dir"/cuda-*.log; do - cuda=$(basename "$log" .log | sed s/cuda-//) - echo "::group::shard cuda:${cuda} log" - cat "$log" - echo "::endgroup::" - done - - # Reconciler — silent-drop guard. Anything left in queue/ was never - # claimed by any shard (work-queue starvation). Anything still in - # inflight// was claimed but never moved to done/ (shard - # crashed mid-test). Either is a silent-drop signal that the docker - # exit alone can't surface. Fail the job and name the orphans. - echo "::group::work-queue reconciler" - unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) - if [ "$unclaimed" -gt 0 ]; then - echo "::error::${unclaimed} test(s) never claimed by any shard:" - ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' - fi - orphans=0 - for shard_dir in "$queue_root/inflight"/*/; do - [ -d "$shard_dir" ] || continue - count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) - if [ "$count" -gt 0 ]; then - echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" - ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' - orphans=$((orphans + count)) - fi - done - done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) - echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" - echo "::endgroup::" - - # Treat reconciler signals as fatal even if all shards exited 0. - if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then - echo "::error::silent-drop detected — see reconciler group above" - [ "$docker_rc" -eq 0 ] && docker_rc=2 - fi - - # Export the runtime directory so the downstream "Aggregated test - # summary" step can locate the done// work distribution - # and the per-shard logs. - echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" - - exit "$docker_rc" + run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh - name: Aggregated test summary - # Runs whether or not the test job exit was clean — surfaces what - # happened even on failure. Per-shard stats + per-file pass / total / - # walltime, plus a single combined table. Also writes the same content - # to $GITHUB_STEP_SUMMARY so it renders at the top of the run page. + # Per-shard + per-file pass/total/walltime, plus a combined table, also + # written to $GITHUB_STEP_SUMMARY. Runs on success or failure. if: always() env: RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} - run: | - python3 - <<'PY' - import os, glob, xml.etree.ElementTree as ET, sys, pathlib - - runtime_dir = os.environ.get("RUNTIME_DIR", "") - done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None - - # Map slug -> shard from done//; slug uses "__" for "/". - slug_to_shard = {} - if done_root and done_root.is_dir(): - for shard_dir in done_root.iterdir(): - if not shard_dir.is_dir(): - continue - shard = shard_dir.name # e.g. "cuda-1" - for entry in shard_dir.iterdir(): - slug_to_shard[entry.name] = shard - - def unslug(name: str) -> str: - # report file pattern: test-reports-.xml - if name.startswith("test-reports-"): - name = name[len("test-reports-"):] - if name.endswith(".xml"): - name = name[:-4] - return name # this is the slug; the file path is slug.replace("__", "/") - - # Per-file rows from JUnit XMLs. - rows = [] - for path in sorted(glob.glob("tests/test-reports-*.xml")): - slug = unslug(os.path.basename(path)) - file_path = slug.replace("__", "/") - shard = slug_to_shard.get(slug, "?") - try: - tree = ET.parse(path) - root = tree.getroot() - except ET.ParseError: - rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) - continue - tests = failures = errors = skipped = 0 - total_time = 0.0 - for ts in root.iter("testsuite"): - tests += int(ts.get("tests", 0)) - failures += int(ts.get("failures", 0)) - errors += int(ts.get("errors", 0)) - skipped += int(ts.get("skipped", 0)) - total_time += float(ts.get("time", 0) or 0) - passed = tests - failures - errors - skipped - status = "PASS" - if failures + errors > 0: - status = "FAIL" - elif tests == 0: - status = "EMPTY" - rows.append((file_path, shard, status, passed, tests, skipped, - failures + errors, total_time)) - - # Per-shard aggregate. - shards = {} - for fp, sh, st, p, t, sk, f, tm in rows: - if sh not in shards: - shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, - "failed": 0, "time": 0.0} - s = shards[sh] - s["files"] += 1 - s["passed"] += p - s["tests"] += t - s["skipped"] += sk - s["failed"] += f - s["time"] += tm - - # ---------- Output: plain text to stdout ---------- - def fmt_row(cols, widths): - return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) - - out = [] - out.append("PER-FILE RESULTS") - out.append("=" * 110) - hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] - widths = [60, 7, 8, 12, 5, 5, 8] - out.append(fmt_row(hdr, widths)) - out.append("-" * 110) - for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): - out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) - out.append("") - out.append("PER-SHARD AGGREGATE") - out.append("=" * 80) - shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] - shard_widths = [10, 7, 12, 9, 8, 14] - out.append(fmt_row(shard_hdr, shard_widths)) - out.append("-" * 80) - grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} - for sh in sorted(shards): - s = shards[sh] - for k in grand: grand[k] += s[k] - out.append(fmt_row( - [sh, str(s["files"]), f"{s['passed']}/{s['tests']}", str(s["skipped"]), - str(s["failed"]), f"{s['time']:.1f}"], shard_widths)) - out.append("-" * 80) - out.append(fmt_row( - ["TOTAL", str(grand["files"]), f"{grand['passed']}/{grand['tests']}", - str(grand["skipped"]), str(grand["failed"]), f"{grand['time']:.1f}"], - shard_widths)) - print("\n".join(out)) - - # ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - md = ["## Multi-GPU CI: Aggregated Test Summary", "", - "### Per-shard totals", "", - "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", - "|---|---:|---:|---:|---:|---:|"] - for sh in sorted(shards): - s = shards[sh] - md.append(f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |") - md.append(f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | **{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |") - md.append("") - md.append("### Per-file results") - md.append("") - md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") - md.append("|---|---|---|---:|---:|---:|---:|") - for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): - status_md = {"PASS": ":white_check_mark: PASS", - "FAIL": ":x: FAIL", - "EMPTY": ":white_circle: EMPTY", - "PARSE-ERROR": ":warning: PARSE-ERROR"}.get(st, st) - md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") - md.append("") - with open(summary_path, "a") as fh: - fh.write("\n".join(md) + "\n") - PY + run: python3 .github/actions/multi-gpu/aggregate_test_summary.py From 437a6d20d410d05fec398e38b35c184010dbec17 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 06:39:07 +0000 Subject: [PATCH 13/22] Build the multi-GPU CI image on the multi-GPU pool The ECR cache repo is resolved per runner pool (single-GPU runners -> gitci-docker-cache; multi-GPU runners -> multigpu-docker-cache). Building on [self-hosted, gpu] pushed the image to gitci-docker-cache, which the multi-GPU test job cannot see, so it rebuilt from scratch (~27 min) on the multi-GPU runner inside its pull step. Build on the multi-GPU pool so the image lands in multigpu-docker-cache and the test job's pull hits. --- .github/workflows/test-multi-gpu-pytest.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 10229766c421..8771c3a69428 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -59,9 +59,14 @@ jobs: build: name: Build / cache image needs: [config] - # Build only needs Docker + ECR access, not multi-GPU. Aligns with the - # build jobs in build.yaml that also publish the per-commit image. - runs-on: [self-hosted, gpu] + # Must run on the SAME pool as the test job. The ECR cache repo is resolved + # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU + # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]` + # the image would land in gitci-docker-cache, which the multi-GPU test job + # cannot see, so it would rebuild from scratch on the scarce multi-GPU + # runner. Building here populates multigpu-docker-cache so the test job's + # pull hits. + runs-on: [self-hosted, linux, x64, multi-gpu] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: From b17679c61c66aed9e853539446c3d693a875ba86 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 14/22] Replace root device-skip conftest with a lane plugin Move non-default-shard device selection out of a repo-root conftest.py into a scoped pytest plugin (mgpu_shard_select) that tools/conftest.py injects per file only on multi-GPU shards. The plugin keys off ISAACLAB_TEST_DEVICES so keep/drop matches each test's device parametrization, deselects out-of-scope variants (cpu, cuda:0, other-index), and maps the all-deselected NO_TESTS_COLLECTED exit to OK. Confines the behavior to the lane instead of a global root conftest. --- .../actions/multi-gpu/mgpu_shard_select.py | 93 +++++++++++++++++++ .github/workflows/test-multi-gpu-pytest.yaml | 8 +- conftest.py | 33 ------- .../isaaclab/isaaclab/test/utils/__init__.py | 4 +- tools/conftest.py | 18 ++++ 5 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 .github/actions/multi-gpu/mgpu_shard_select.py delete mode 100644 conftest.py diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py new file mode 100644 index 000000000000..22f784435bb3 --- /dev/null +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -0,0 +1,93 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""pytest plugin: select which tests a non-default GPU shard runs. + +Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via +``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this +directory to ``PYTHONPATH`` so it is importable). It lives next to the lane +scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. + +Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME +env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's +device parametrization, so this plugin's keep/drop decision agrees with what was +parametrized. (``ISAACLAB_SIM_DEVICE`` is AppLauncher's Kit boot device, a +different concern, and is deliberately NOT used here.) The plugin reads the mask +string directly -- no isaaclab import -- matching the lib-free convention of every +other conftest in the repo. The only shared knowledge is the mask grammar below, +mirrored from ``devices.py`` (position 0 = cpu, position k = cuda:(k-1), trailing +``X`` = include all remaining); keep the two in sync. + +Selection rule, on a shard whose mask excludes cpu + cuda:0: + + keep a test iff it is parametrized over ``device`` AND that variant's device + is active in the mask; deselect everything else. + +That single rule is both criteria at once: candidate = device-parametrized (a +test with no ``device`` param is device-agnostic, covered by single-GPU CI on +cuda:0); scope = only this shard's device variant runs (cpu / cuda:0 / other-index +variants are dropped). Out-of-scope items are deselected (not skipped) so the +shard report shows only what ran. +""" + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +def _position(device): + """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None # not device-parametrized, or an unrecognized device value + + +def _active(position, mask): + """Whether a mask position is included. Trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + +def _shard_mask(): + """Return the runtime mask iff this is a multi-GPU shard run, else None. + + A shard targets non-default GPUs only, so cpu (position 0) and cuda:0 + (position 1) are excluded. Anything else -- unset, or single-GPU CI's default + ``"110"`` -- is not a shard and the plugin is a no-op. + """ + mask = os.environ.get(_RUNTIME_ENV, "") + return mask if mask and not _active(0, mask) and not _active(1, mask) else None + + +def pytest_collection_modifyitems(config, items): + mask = _shard_mask() + if mask is None: + return + keep, drop = [], [] + for item in items: + callspec = getattr(item, "callspec", None) + device = callspec.params.get("device") if callspec is not None else None + if _active(_position(device), mask): + keep.append(item) # device-parametrized AND in this shard's scope + else: + drop.append(item) # non-device, or cpu / cuda:0 / other-index variant + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + +def pytest_sessionfinish(session, exitstatus): + # A file whose tests are all out of scope deselects to zero, so pytest exits + # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file + # exit as a failure (tools/conftest.py), so report "nothing in scope for this + # file" as success rather than a false failure. + if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 8771c3a69428..2ea9a86abace 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -114,10 +114,10 @@ jobs: # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. # # Within a discovered file, tests that are NOT parametrized over the - # ``device`` argument are filtered out at collection time by the - # workspace-root ``conftest.py``: single-GPU CI already covers them on - # ``cuda:0`` and re-running on every non-default shard adds wall-time - # without surfacing any new failure mode. + # ``device`` argument are deselected at collection time by the + # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): + # single-GPU CI already covers them on ``cuda:0`` and re-running on every + # non-default shard adds wall-time without surfacing any new failure mode. id: discover run: | # File-level opt-out: a test file can exclude itself from multi-GPU CI diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 30d6ae4c40dd..000000000000 --- a/conftest.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Workspace-root pytest hooks. - -The only hook today is the multi-GPU shard skip: when a shard pins itself to a -non-default cuda device via :envvar:`ISAACLAB_SIM_DEVICE` (e.g. ``cuda:2``), -skip any test that is not parametrized over a ``device`` argument. Such tests -exercise device-agnostic logic that single-GPU CI already covers on -``cuda:0``, so running them again on every non-default shard duplicates work -without surfacing any new failure mode. - -The hook is a no-op outside the multi-GPU lane (``ISAACLAB_SIM_DEVICE`` unset -or equal to ``cuda:0``), so this file does not change pytest behavior under -single-GPU CI. -""" - -import os - -import pytest - - -def pytest_collection_modifyitems(config, items): - sim_device = os.environ.get("ISAACLAB_SIM_DEVICE", "") - if not sim_device.startswith("cuda:") or sim_device == "cuda:0": - return - skip_marker = pytest.mark.skip(reason=f"non-device-parametrized; covered by single-GPU lane (shard={sim_device})") - for item in items: - callspec = getattr(item, "callspec", None) - if callspec is None or "device" not in callspec.params: - item.add_marker(skip_marker) diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py index e268d22c5c16..9e4bb3d6c875 100644 --- a/source/isaaclab/isaaclab/test/utils/__init__.py +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -6,8 +6,8 @@ """Test-time helpers for Isaac Lab. Exposes :func:`test_devices` for selecting the device list to parametrize tests -over. The set is ``scope ∩ budget``: ``scope`` is the call-site argument (the -devices the test is valid on), ``budget`` is the ``ISAACLAB_TEST_DEVICES`` env +over. The set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the +devices the test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env var (the devices the run may use). """ diff --git a/tools/conftest.py b/tools/conftest.py index ad24c85830b4..7103be2aae5c 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -518,6 +518,19 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" + # Multi-GPU lane only: make the device-selection plugin importable in this + # per-file subprocess (injected via ``-p`` below, not as a repo-root + # conftest). Detect a shard by the runtime device mask excluding cpu + # (position 0) and cuda:0 (position 1) -- the same ISAACLAB_TEST_DEVICES the + # plugin and test_devices() read. The plugin re-checks this; the cheap + # prefix test here leaves single-GPU CI's command (mask unset or "11...") + # unchanged. + _mask = os.environ.get("ISAACLAB_TEST_DEVICES", "") + _inject_shard_select = _mask[:2] == "00" + if _inject_shard_select: + _plugin_dir = os.path.join(workspace_root, ".github", "actions", "multi-gpu") + env["PYTHONPATH"] = _plugin_dir + os.pathsep + env.get("PYTHONPATH", "") + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) # Read the test file once for cold-cache check. @@ -559,6 +572,11 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): f"--junitxml={report_file}", "--tb=short", ] + if _inject_shard_select: + cmd += [ + "-p", + "mgpu_shard_select", + ] # multi-GPU lane test-selection plugin (importable via PYTHONPATH set above) if isaacsim_ci: cmd.append("-m") From 49aadc572f2355687fdfbe96fd1d7b91c0c04255 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 15/22] Document multi-GPU scripts; drop py-spy remnants Annotate the uncommon bash idioms in the two multi-GPU orchestration scripts for non-bash readers. Remove the now-dead py-spy/gdb hang-capture remnants -- the SYS_PTRACE cap-add and the py-spy pip install -- along with the matching changelog bullet, since the conftest-side capture they supported was removed. --- .../multi-gpu/multi_gpu_host_launcher.sh | 39 ++++++++-------- .../multi-gpu/multi_gpu_shard_runner.sh | 44 ++++++++++--------- .../jichuanh-multi-gpu-ci.minor.rst | 9 ---- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh index 332d5bd73a20..ad623c61c649 100755 --- a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -25,7 +25,7 @@ # is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete # runners rely on ``--gpus all``. -set +e +set +e # keep going on errors so the reconciler below always runs and reports host_uid="$(id -u)" host_gid="$(id -g)" host_user="$(id -un)" @@ -37,6 +37,7 @@ host_user="$(id -un)" # — no concurrent creators. rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim +# unique per-run temp dir (RUNNER_TEMP if set, else /tmp; XXXXXX -> random suffix) runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" mkdir -p "$runtime_dir" @@ -57,11 +58,11 @@ mkdir -p "$runtime_dir" # claimed but crashed mid-test). queue_root="$runtime_dir/queue" mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" -IFS=',' read -ra _paths <<< "$PATHS" +IFS=',' read -ra _paths <<< "$PATHS" # split the comma-separated PATHS into the _paths array for path in "${_paths[@]}"; do - [ -n "$path" ] || continue - slug="${path//\//__}" - : > "$queue_root/queue/$slug" + [ -n "$path" ] || continue # skip empty entries (PATHS has a trailing comma) + slug="${path//\//__}" # flatten to a filename: replace every '/' with '__' + : > "$queue_root/queue/$slug" # create an empty marker file (':' = no-op, '>' creates it) done echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" @@ -75,17 +76,17 @@ mkdir -p "$logs_dir" # same parent GPU. cvd_args=() if nvidia-smi -L | grep -q "^ MIG "; then + # read each MIG slice's UUID into the MIGS array (one element per line) mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') - MIG_LIST=$(IFS=,; echo "${MIGS[*]}") - cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") # join the array into a comma-separated string + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") # stays empty on discrete hosts -> no flag added echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" else echo "::notice::discrete GPU mode — relying on --gpus all" fi -# --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. +# "${cvd_args[@]}" expands to the CUDA_VISIBLE_DEVICES flag on MIG hosts, nothing otherwise. docker run --rm --gpus all --network=host \ - --cap-add=SYS_PTRACE \ --entrypoint bash \ --user "${host_uid}:${host_gid}" \ --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ @@ -110,11 +111,11 @@ docker run --rm --gpus all --network=host \ -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ "$IMAGE_TAG" \ /multi_gpu_shard_runner.sh -docker_rc=$? +docker_rc=$? # $? = exit code of the docker run above # Grouped re-print of per-shard logs (preserved across the host). for log in "$logs_dir"/cuda-*.log; do - cuda=$(basename "$log" .log | sed s/cuda-//) + cuda=$(basename "$log" .log | sed s/cuda-//) # extract N from "cuda-N.log" echo "::group::shard cuda:${cuda} log" cat "$log" echo "::endgroup::" @@ -126,19 +127,19 @@ done # silent-drop signal that the docker exit alone can't surface. Fail the job and # name the orphans. echo "::group::work-queue reconciler" -unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) # files still pending (never claimed) if [ "$unclaimed" -gt 0 ]; then echo "::error::${unclaimed} test(s) never claimed by any shard:" - ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent for the log fi orphans=0 -for shard_dir in "$queue_root/inflight"/*/; do - [ -d "$shard_dir" ] || continue +for shard_dir in "$queue_root/inflight"/*/; do # each inflight// dir (the /*/ matches dirs only) + [ -d "$shard_dir" ] || continue # skip if the glob matched nothing count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) if [ "$count" -gt 0 ]; then echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" - ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' - orphans=$((orphans + count)) + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent + orphans=$((orphans + count)) # $(( )) = integer arithmetic fi done done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) @@ -148,11 +149,11 @@ echo "::endgroup::" # Treat reconciler signals as fatal even if all shards exited 0. if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then echo "::error::silent-drop detected — see reconciler group above" - [ "$docker_rc" -eq 0 ] && docker_rc=2 + [ "$docker_rc" -eq 0 ] && docker_rc=2 # clean docker exit but drops found -> force a non-zero code fi # Export the runtime directory so the downstream "Aggregated test summary" step # can locate the done// work distribution and the per-shard logs. -echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" # export to later workflow steps via GitHub's env file exit "$docker_rc" diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index dc5fcbfb6c1a..902b51f6001c 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash # Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest -# shards. Mounted by both the CI workflow (.github/workflows/test-multi-gpu- -# pytest.yaml) and the local probe under ``data/mgpu-1docker-probe/run.sh``, -# so the logic lives in exactly one place and changes are version-controlled -# + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` so it sits -# next to the workflow that consumes it rather than alongside ``tools/``. +# shards. Mounted by the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) so the logic lives in exactly one place and changes are +# version-controlled + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` +# so it sits next to the workflow that consumes it rather than alongside ``tools/``. # # Container expectations (set by the host launcher): # --gpus all @@ -32,9 +31,9 @@ # 6. Waits on every shard before aggregating exit codes — a fast failure doesn't # tear down still-running siblings -set +e +set +e # keep going on errors; per-shard exit codes are aggregated at the end cd /workspace/isaaclab -unset HUB__ARGS__DETECT_ONLY DISPLAY +unset HUB__ARGS__DETECT_ONLY DISPLAY # clear vars that would force Kit into detect-only / headed (X11) mode # Container-level HOME + PYTHONUSERBASE for pip --user installs. The image # runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so @@ -50,11 +49,11 @@ mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase # Pytest deps (same as run-tests action). junitparser is imported at # tools/conftest.py load time, so it must be present first. -./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky py-spy "coverage>=7.6.1" +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky "coverage>=7.6.1" # Shard count from nvidia-smi -L (truth; torch under-counts MIG). -MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") -GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") # grep -c = count of matching lines (MIG slices) +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") # count of whole GPUs if [ "$MIG_COUNT" -gt 0 ]; then DEV_COUNT=$MIG_COUNT echo "::notice::container: MIG mode, $MIG_COUNT slices" @@ -80,11 +79,11 @@ fi # Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME # (per-shard isolation for .cache, .local/share, etc.) and per-shard # ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. -declare -A pids -for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do +declare -A pids # associative array: shard index -> background PID +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) zeros="" - for ((i = 0; i <= cuda; i++)); do zeros+="0"; done - runtime_devices="${zeros}1" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done # build the leading zeros of the device mask + runtime_devices="${zeros}1" # e.g. cuda:2 -> "0001": only this GPU active in the ISAACLAB_TEST_DEVICES mask shard_home="/tmp/isaaclab-ci-home-${cuda}" mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ @@ -93,6 +92,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do shard_log="/shard-logs/cuda-${cuda}.log" + # each shard runs in its own subshell, backgrounded with '&' below, so all shards run in parallel ( export HOME="$shard_home" export XDG_CACHE_HOME="${HOME}/.cache" @@ -106,6 +106,8 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # logs, etc. only end up in the shard logfile, NOT on the live workflow # log. The workflow's grouped re-print at end of run still shows the # full $shard_log under a collapsible ``::group::shard cuda:N log``. + # (tee = full output to the log file; stdbuf -oL = flush per line so the + # filtered grep/sed stream appears live, not in delayed chunks.) ./isaaclab.sh -p -m pytest \ --ignore=tools/conftest.py \ --ignore=source/isaaclab/test/install_ci \ @@ -115,24 +117,24 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ | stdbuf -oL sed "s/^/[cuda:${cuda}] /" - exit "${PIPESTATUS[0]}" + exit "${PIPESTATUS[0]}" # exit with pytest's code, not tee/grep/sed's (PIPESTATUS[0] = first pipe stage) ) & - pids[$cuda]=$! + pids[$cuda]=$! # $! = PID of the subshell just backgrounded echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" done # Wait for every shard before aggregating exits — a fast failure must not # tear down still-running siblings. -declare -A results -for cuda in "${!pids[@]}"; do - wait "${pids[$cuda]}" - results[$cuda]=$? +declare -A results # associative array: shard index -> exit code +for cuda in "${!pids[@]}"; do # "${!pids[@]}" = the array's keys (the shard indices) + wait "${pids[$cuda]}" # block until that shard's PID exits + results[$cuda]=$? # $? = the waited shard's exit code done fail=0 for cuda in "${!results[@]}"; do rc="${results[$cuda]}" echo "shard cuda:${cuda} exited $rc" - [ "$rc" -eq 0 ] || fail=1 + [ "$rc" -eq 0 ] || fail=1 # any non-zero shard makes the whole run fail done exit $fail diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst index 00b4366e153e..5f66a3c858a0 100644 --- a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -14,12 +14,3 @@ Added the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow boot Kit on a non-default GPU without editing every test's :class:`~isaaclab.app.AppLauncher` call site. - -* Added ``py-spy`` + ``gdb`` stack capture in ``tools/conftest.py`` on - ``shutdown_hang`` / ``startup_hang`` / ``timeout`` detection. Walks the test - subprocess's process group (cap 8 pids), captures both Python and C++ frames - before ``SIGKILL`` erases them, attaches the output to the JUnit error - report. Makes Kit binary hangs observable in CI logs; safe no-op when - ``py-spy``/``gdb`` are missing. Workflow side adds ``--cap-add=SYS_PTRACE`` - on the per-shard ``docker run`` (required to attach) and adds ``py-spy`` to - the in-container ``pip install`` list. From 1e710b153f8847b8d1f189e62afdac6a19b50d41 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 16/22] Revert "[CI] Forward extra env vars through run-tests / run-package-tests actions" This reverts commit 6cef107ab0c20659bbd0d04ab86d6b44da839890. --- .github/actions/run-package-tests/action.yml | 5 ---- .github/actions/run-tests/action.yml | 29 +------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 4bde3eb22228..264e01a158b8 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,10 +69,6 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false - extra-env-vars: - description: 'Extra env vars to forward into the container (one KEY=value per line)' - default: '' - required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -154,7 +150,6 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} - extra-env-vars: ${{ inputs.extra-env-vars }} - name: Check Test Results if: always() diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 4565f5ddcfd4..ceea3a1c4e65 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,14 +69,6 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false - extra-env-vars: - description: >- - Extra environment variables to forward into the container, one per line in - ``KEY=value`` form. Whitespace-only lines and lines starting with ``#`` are - ignored. Used by the multi-GPU workflow to inject - ``ISAACLAB_TEST_DEVICES`` / ``ISAACLAB_SIM_DEVICE``. - default: '' - required: false runs: using: composite @@ -101,7 +93,6 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" - local extra_env_vars="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -213,24 +204,6 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi - # Caller-supplied extra env vars (one KEY=value per line). Skips - # blank lines and full-line comments (line where the first - # non-whitespace char is ``#``). Mid-line ``#`` is preserved so - # values like ``IMAGE_TAG=v1.0#nightly`` survive. - if [ -n "$extra_env_vars" ]; then - while IFS= read -r line; do - # Strip leading whitespace (YAML ``|`` block can leave indent). - line="${line#"${line%%[![:space:]]*}"}" - [ -z "$line" ] && continue - [ "${line:0:1}" = "#" ] && continue - key="${line%%=*}" - value="${line#*=}" - export "$key"="$value" - docker_env_vars="$docker_env_vars -e $key" - echo "Forwarding extra env var: $key" - done <<< "$extra_env_vars" - fi - # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -419,7 +392,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.extra-env-vars }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" - name: Kill container on cancellation if: cancelled() From 4441f68129c0040674fc9ac4d830d7c604379595 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 17/22] Revert "TEMP: skip docker/docs/install-ci CI while iterating #5823" This reverts commit c514199d41ac37de3e00295b0254cf6f2b5af9d4. --- .github/workflows/build.yaml | 7 +------ .github/workflows/docs.yaml | 15 ++------------- .github/workflows/install-ci.yml | 5 ----- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2428799e72d9..1c599ebcb988 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,12 +74,7 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - # TEMP (revert before final review / before landing): force - # run_docker_tests=false while iterating PR #5823. All gated - # test/build jobs skip via their existing if-gate; the - # single-GPU Docker + Tests matrix won't burn the gpu pool on - # every push. Per ~/.claude/skills/pr/ci-iteration-shortcut.md. - run_docker_tests: 'false' + run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} steps: - id: detect env: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 098d9ba784b0..013be3a5b126 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -25,11 +25,6 @@ jobs: doc-build-type: name: Detect Doc Build Type runs-on: ubuntu-latest - # DIAGNOSTIC PR opt-out: skip docs build entirely when the PR title is - # marked DO-NOT-MERGE (scratch / hypothesis-only branches don't need a - # full doc rebuild on every push). Push events on main/develop/release - # still run because they don't carry a PR title. - if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: trigger-deploy: ${{ steps.trigger-deploy.outputs.defined }} steps: @@ -47,14 +42,8 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # Run on non-deploy branches to build current version docs only AND skip - # for DO-NOT-MERGE diagnostic PRs (the needed doc-build-type job already - # skips for those, but GitHub Actions evaluates `needs.X.outputs.Y` as - # empty when X is skipped, which keeps this `!= 'true'` gate true — so - # we re-check the title here to actually cascade the skip). - if: | - needs.doc-build-type.outputs.trigger-deploy != 'true' - && !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) + # run on non-deploy branches to build current version docs only + if: needs.doc-build-type.outputs.trigger-deploy != 'true' steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index df3827bfb870..2a2fee50c8ab 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -34,11 +34,6 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest - # DIAGNOSTIC PR opt-out: skip install-ci entirely when the PR title is - # marked DO-NOT-MERGE. The downstream install-tests-x86 / install-tests-arm - # jobs gate on `needs.changes.outputs.run_install_tests == 'true'`; with - # this job skipped the output is empty and both dependents skip too. - if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} steps: From c7998213278dcff18579dcc0e71273a76cf1522e Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 29 Jun 2026 23:12:33 +0000 Subject: [PATCH 18/22] Refine multi-GPU test device selection --- .../actions/multi-gpu/mgpu_shard_select.py | 12 +-- .../multi-gpu/multi_gpu_host_launcher.sh | 4 +- .../multi-gpu/multi_gpu_shard_runner.sh | 6 +- .github/workflows/test-multi-gpu-pytest.yaml | 14 ++-- .../jichuanh-fix-build-sim-context-device.rst | 11 --- .../jichuanh-multi-gpu-ci.minor.rst | 18 ++-- source/isaaclab/isaaclab/app/app_launcher.py | 10 --- .../isaaclab/isaaclab/test/utils/__init__.py | 12 +-- .../isaaclab/isaaclab/test/utils/devices.py | 82 ++++++++++++------- .../test/sim/test_simulation_context.py | 4 +- .../test/sim/test_views_xform_prim.py | 4 +- .../test/utils/test_device_selection.py | 26 +++++- .../isaaclab/test/utils/test_episode_data.py | 12 +-- source/isaaclab/test/utils/test_math.py | 6 +- source/isaaclab/test/utils/test_noise.py | 8 +- .../test/assets/test_articulation.py | 36 ++++---- .../test/assets/test_rigid_object.py | 4 +- .../assets/test_rigid_object_collection.py | 4 +- .../test/assets/test_articulation.py | 30 +++---- .../test/assets/test_rigid_object.py | 4 +- .../assets/test_rigid_object_collection.py | 4 +- .../test/sim/test_views_xform_prim_fabric.py | 11 +-- tools/conftest.py | 6 +- 23 files changed, 180 insertions(+), 148 deletions(-) delete mode 100644 source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py index 22f784435bb3..a8402cb11ca6 100644 --- a/.github/actions/multi-gpu/mgpu_shard_select.py +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -13,12 +13,12 @@ Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's device parametrization, so this plugin's keep/drop decision agrees with what was -parametrized. (``ISAACLAB_SIM_DEVICE`` is AppLauncher's Kit boot device, a -different concern, and is deliberately NOT used here.) The plugin reads the mask -string directly -- no isaaclab import -- matching the lib-free convention of every -other conftest in the repo. The only shared knowledge is the mask grammar below, -mirrored from ``devices.py`` (position 0 = cpu, position k = cuda:(k-1), trailing -``X`` = include all remaining); keep the two in sync. +parametrized. (``ISAACLAB_TEST_SIM_DEVICE`` is passed explicitly to AppLauncher +by Kit-backed tests, a different concern, and is deliberately NOT used here.) The +plugin reads the mask string directly -- no isaaclab import -- matching the +lib-free convention of every other conftest in the repo. The only shared knowledge +is the mask grammar below, mirrored from ``devices.py`` (position 0 = cpu, +position k = cuda:(k-1), trailing ``X`` = include all remaining); keep the two in sync. Selection rule, on a shard whose mask excludes cpu + cuda:0: diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh index ad623c61c649..03174f427f06 100755 --- a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -12,8 +12,8 @@ # and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). # # 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its -# own non-default cuda:N via ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES, pulling -# from the shared queue). Collapsing the previous N-container layout into one +# own non-default cuda:N via ISAACLAB_TEST_SIM_DEVICE / ISAACLAB_TEST_DEVICES, +# pulling from the shared queue). Collapsing the previous N-container layout into one # removes cross-container races on the workspace mount (``_isaac_sim`` symlink, # ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at # the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index 902b51f6001c..6b835cec9e80 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -26,7 +26,7 @@ # 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch # can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) # 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + -# ISAACLAB_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# ISAACLAB_TEST_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to # /shard-logs/cuda-N.log for the host's grouped re-print after the run # 6. Waits on every shard before aggregating exit codes — a fast failure doesn't # tear down still-running siblings @@ -78,7 +78,7 @@ fi # Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME # (per-shard isolation for .cache, .local/share, etc.) and per-shard -# ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +# ISAACLAB_TEST_SIM_DEVICE / ISAACLAB_TEST_DEVICES. declare -A pids # associative array: shard index -> background PID for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) zeros="" @@ -98,7 +98,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to sk export XDG_CACHE_HOME="${HOME}/.cache" export XDG_DATA_HOME="${HOME}/.local/share" export ISAACLAB_TEST_DEVICES="$runtime_devices" - export ISAACLAB_SIM_DEVICE="cuda:${cuda}" + export ISAACLAB_TEST_SIM_DEVICE="cuda:${cuda}" # Full pytest output captures to $shard_log; live stdout is filtered # down to high-signal lines (test boundaries, failures, summary stats, diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 2ea9a86abace..e759f9f0dba0 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -107,11 +107,10 @@ jobs: - name: Discover opt-in test files # Auto-discovery: any test_*.py with a non-default-capable scope — an - # argless ``test_devices()`` (defaults to "11X") or any - # ``test_devices("..X")`` mask, whose trailing ``X`` spans the - # non-default GPUs — is in scope. Adding a test to multi-GPU CI needs - # no workflow edit; opting a file out is just narrowing its scope (drop - # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. + # argless ``test_devices()``, a named scope containing non-default GPUs, + # or a string mask with a trailing ``X`` — is in scope. Adding a test to + # multi-GPU CI needs no workflow edit; opting a file out is just narrowing + # its scope to ``DeviceScope.CPU_AND_DEFAULT_CUDA`` (or mask ``"110"``). # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the @@ -125,7 +124,8 @@ jobs: # Used for files with known Kit/Isaac-Sim concurrency issues; the file # still runs in single-GPU CI. Excluded files are reported as a notice # below for visibility. No workflow edit needed to add/remove a file. - mapfile -t candidates < <(grep -rlE 'test_devices\(\)|test_devices\("[^"]*X"\)' source/ --include='test_*.py' | sort -u) + device_scope_pattern='test_devices\(\)|test_devices\(DeviceScope\.(ALL|CUDA|NON_DEFAULT_CUDA)|test_devices\("[^"]*X"\)' + mapfile -t candidates < <(grep -rlE "$device_scope_pattern" source/ --include='test_*.py' | sort -u) discovered=() skipped=() for f in "${candidates[@]}"; do @@ -140,7 +140,7 @@ jobs: echo "::notice::multi-GPU skipped: $f — $reason" done if [ ${#discovered[@]} -eq 0 ]; then - echo "::error::No opt-in tests discovered (grep for test_devices with an X scope)" + echo "::error::No tests with a non-default-capable device scope were discovered" exit 1 fi basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) diff --git a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst deleted file mode 100644 index 89e774bd26d7..000000000000 --- a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst +++ /dev/null @@ -1,11 +0,0 @@ -Fixed -^^^^^ - -* Fixed :func:`isaaclab.sim.build_simulation_context` silently ignoring the - ``device`` kwarg when ``sim_cfg`` is also provided. Most test callers pass - both kwargs together; the helper now applies the explicit ``device`` over - ``sim_cfg.device`` so the caller's choice wins. Without this, warp kernel - launches in :mod:`isaaclab_newton.assets.articulation` raised device - mismatch errors on non-default GPUs (``env_ids`` allocated on the test's - device while the articulation's resolved device came from the untouched - ``sim_cfg`` default ``cuda:0``). diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst index 5f66a3c858a0..084c95e5f606 100644 --- a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -1,16 +1,8 @@ Added ^^^^^ -* Added :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over - a device set resolved as ``scope ∩ runtime``: ``scope`` is the call-site mask - of devices a test is valid on (default ``"11X"`` ⇒ cpu + cuda:0 + the - non-default GPUs), and the runtime is the ``ISAACLAB_TEST_DEVICES`` env var of - devices a run may use (default ``"110"`` ⇒ cpu + cuda:0). A trailing ``X`` - includes the remaining devices. Single-GPU CI is unchanged; multi-GPU CI sets - the runtime to one non-default GPU per shard. - -* Added ``ISAACLAB_SIM_DEVICE`` env var honored by - :class:`isaaclab.app.AppLauncher` as the implicit-default device when - the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow - boot Kit on a non-default GPU without editing every test's - :class:`~isaaclab.app.AppLauncher` call site. +* Added :class:`isaaclab.test.utils.DeviceScope` and + :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over a + device set resolved as ``scope ∩ runtime``. Named scopes cover common cases, + string masks support custom combinations, and multi-GPU CI narrows the runtime + to one non-default GPU per shard without changing single-GPU CI behavior. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index c8ba3660ff7e..f07fe126f3bb 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1003,16 +1003,6 @@ def _resolve_device_settings(self, launcher_args: dict): device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) - # When the caller didn't pin a device, allow ISAACLAB_SIM_DEVICE to - # override the default. Used by the multi-GPU CI workflow to boot - # Kit with active_gpu=1 without editing every test's AppLauncher() - # call site; Kit's active_gpu is process-global and locked after - # SimulationApp init, so per-test selection cannot retarget it. - if not device_explicitly_passed: - env_device = os.environ.get("ISAACLAB_SIM_DEVICE") - if env_device: - device = env_device - launcher_args["device"] = env_device if self._xr and not device_explicitly_passed: # If no device is specified, default to the CPU device if we are running in XR diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py index 9e4bb3d6c875..2c2ee0249e98 100644 --- a/source/isaaclab/isaaclab/test/utils/__init__.py +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -5,12 +5,12 @@ """Test-time helpers for Isaac Lab. -Exposes :func:`test_devices` for selecting the device list to parametrize tests -over. The set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the -devices the test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env -var (the devices the run may use). +Exposes :class:`DeviceScope` and :func:`test_devices` for selecting the device +list to parametrize tests over. The set is ``scope ∩ runtime``: ``scope`` is the +call-site argument (the devices the test is valid on), the runtime is the +``ISAACLAB_TEST_DEVICES`` env var (the devices the run may use). """ -from .devices import test_devices +from .devices import DeviceScope, test_devices -__all__ = ["test_devices"] +__all__ = ["DeviceScope", "test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py index c54cabfdfbd5..b240b701ea6b 100644 --- a/source/isaaclab/isaaclab/test/utils/devices.py +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -7,21 +7,22 @@ Intended use:: - from isaaclab.test.utils import test_devices + from isaaclab.test.utils import DeviceScope, test_devices @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) def test_foo(device): ... -Two masks, two roles --------------------- +Two inputs, two roles +--------------------- A test runs on ``scope ∩ runtime``: -* **scope** — the call-site mask: the devices the *test* is valid on. The - author owns this; defaults to ``"11X"`` (device-agnostic). +* **scope** — a :class:`DeviceScope` or mask declaring the devices the *test* + is valid on. The author owns this; defaults to :attr:`DeviceScope.ALL`. * **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). The multi-GPU workflow sets it to one device per shard, matching - ``ISAACLAB_SIM_DEVICE`` (AppLauncher's boot device — not read here). + ``ISAACLAB_TEST_SIM_DEVICE`` (the explicit AppLauncher boot device — not read here). A test author never names the shard's GPU; the operator never inspects a test's device support. The helper intersects the two. @@ -35,16 +36,20 @@ def test_foo(device): ... cpu, cuda:0, and a non-default GPU stay distinct by position — running on cuda:0 says nothing about cuda:1+, which is the whole reason this exists. -Common masks ------------- -====== =================================================================== -Mask Meaning -====== =================================================================== -``11X`` cpu + cuda:0 + every non-default GPU (default scope; device-agnostic) -``110`` cpu + cuda:0 (pure-math / backend tests: cuda:0 == cuda:N) -``00X`` non-default GPUs only (validates non-default-device behavior) -``100`` cpu only (pure logic) -====== =================================================================== +Common scopes +------------- +==================================== ========= ============================================= +Named scope Mask Meaning +==================================== ========= ============================================= +``DeviceScope.ALL`` ``11X`` cpu + every GPU +``DeviceScope.CUDA`` ``01X`` every GPU +``DeviceScope.CPU_AND_DEFAULT_CUDA`` ``110`` cpu + cuda:0 +``DeviceScope.NON_DEFAULT_CUDA`` ``00X`` cuda:1 and above +``DeviceScope.CPU`` ``100`` cpu only +==================================== ========= ============================================= + +Named scopes cover the common cases; string masks remain supported for custom +device combinations. Worked example — a ``scope="11X"`` (default) test: @@ -65,15 +70,14 @@ def test_foo(device): ... ---------- Set the runtime from the shell to opt a run into non-default GPUs:: - ISAACLAB_TEST_DEVICES=0001 ISAACLAB_SIM_DEVICE=cuda:2 \\ + ISAACLAB_TEST_DEVICES=0001 ISAACLAB_TEST_SIM_DEVICE=cuda:2 \\ ./isaaclab.sh -p -m pytest path/to/test.py """ from __future__ import annotations import os - -import torch +from enum import StrEnum _RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" """Env var naming the run's devices: the devices a run may use (see module docstring).""" @@ -83,7 +87,21 @@ def test_foo(device): ... i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" -def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> list: +class DeviceScope(StrEnum): + """Named device scopes for common test parametrization cases. + + String masks remain accepted by :func:`test_devices` for custom device + combinations that are not represented here. + """ + + ALL = "11X" + CUDA = "01X" + CPU_AND_DEFAULT_CUDA = "110" + NON_DEFAULT_CUDA = "00X" + CPU = "100" + + +def test_devices(scope: str | DeviceScope = DeviceScope.ALL, *, skip: dict[str, str] | None = None) -> list: """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. ``scope`` is this argument; the runtime comes from the @@ -91,10 +109,9 @@ def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> l and the scope / runtime split). Args: - scope: Device mask (e.g. ``"11X"``) the test is valid on. Defaults to - ``"11X"`` (cpu + cuda:0 + every non-default GPU): the device-agnostic - common case, so most call sites can use ``test_devices()`` with no - argument. + scope: Named scope or device mask (e.g. ``"11X"``) the test is valid + on. Defaults to :attr:`DeviceScope.ALL`, so device-agnostic call + sites can use ``test_devices()`` with no argument. skip: Optional ``{device: reason}``; in-scope devices listed here are wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with the reason instead of being dropped. Use to gate a device variant @@ -138,6 +155,10 @@ def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> l ] +# Prevent pytest from collecting this imported ``test_``-prefixed helper. +test_devices.__test__ = False + + def _expand(mask: str, count: int) -> list[bool]: """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. @@ -158,12 +179,15 @@ def _list_available_devices() -> list[str]: Returns: Ordered list of device strings as torch addresses them. """ + # Keep the import local so importing this test helper before AppLauncher + # does not import torch (and its transitive native libraries) before Kit. + import torch + devices = ["cpu"] - # torch.cuda (not warp) is deliberate: this runs at pytest collection time, - # before AppLauncher boots Kit. torch.cuda.device_count() enumerates without - # creating a CUDA context, whereas warp.get_cuda_devices() initializes the - # warp runtime — doing that at collection, ahead of Kit's device setup, - # risks the non-default-GPU init-order fragility this suite targets (#5132). + # torch.cuda (not Warp) is deliberate: warp.get_cuda_devices() initializes + # the Warp runtime, but non-default-GPU runs must select cuda:N before + # wp.init(). Enumeration here must not perturb the initialization order that + # this suite validates (#5132). if torch.cuda.is_available(): devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) return devices diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index c96883b839a0..81a67adc2c56 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -5,11 +5,13 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 827ed20d3261..598075d0791a 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -10,10 +10,12 @@ prim ordering, xformOp standardization, and Isaac Sim comparison. """ +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app import pytest # noqa: E402 import torch # noqa: E402 diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py index ff09b244e20a..0c7e976f825d 100644 --- a/source/isaaclab/test/utils/test_device_selection.py +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -7,14 +7,15 @@ These tests mock the host's device list, so they need no GPU and run on the single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) -for two reasons: pytest would otherwise collect the ``test_``-prefixed function -as a test case, and the multi-GPU workflow's auto-discovery greps for -``test_devices(...)`` call sites — neither should fire on this file. +so the multi-GPU workflow's auto-discovery does not mistake this file for an +opted-in device test. The helper itself sets ``__test__ = False`` to prevent +pytest from collecting it when imported under its public name. """ import pytest from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import DeviceScope from isaaclab.test.utils.devices import test_devices as resolve_devices # Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). @@ -80,6 +81,25 @@ def test_argless_runs_once_on_one_non_default_gpu(host): assert all(d not in ("cpu", "cuda:0") for d in result) +@pytest.mark.parametrize( + "scope, mask", + [ + (DeviceScope.ALL, "11X"), + (DeviceScope.CUDA, "01X"), + (DeviceScope.CPU_AND_DEFAULT_CUDA, "110"), + (DeviceScope.NON_DEFAULT_CUDA, "00X"), + (DeviceScope.CPU, "100"), + ], +) +def test_named_scope_matches_mask(host, scope, mask): + host(MULTI_GPU, "1111") + assert resolve_devices(scope) == resolve_devices(mask) + + +def test_helper_is_not_collected_by_pytest(): + assert resolve_devices.__test__ is False + + # --------------------------------------------------------------------------- # skip vs raise: legitimate skips never raise; only a missing runtime device does # --------------------------------------------------------------------------- diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index 6062c25d0400..27a3f63c519d 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,11 +5,11 @@ import pytest import torch -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import DeviceScope, test_devices from isaaclab.utils.datasets import EpisodeData -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -19,7 +19,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -56,7 +56,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -103,7 +103,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -115,7 +115,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index 4fad6881e56e..d522f8fb2e2c 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,7 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import DeviceScope, test_devices DECIMAL_PRECISION = 5 """Precision of the test. @@ -1041,7 +1041,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in test_devices("110"): + for device in test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1065,7 +1065,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in test_devices("110"): + for device in test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index c1035453b5a9..d88ef58ee5a0 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,10 +7,10 @@ import torch import isaaclab.utils.noise as noise -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import DeviceScope, test_devices -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -43,7 +43,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -78,7 +78,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", test_devices("110")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CPU_AND_DEFAULT_CUDA)) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index f72f2157a3ef..eeaa70bd6ff4 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -8,13 +8,15 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import DeviceScope, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" @@ -2491,7 +2493,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_body_q_consistent_after_root_write(num_articulations, device, articulation_type): """Test that body_q is fresh when collide() runs after a root pose write. @@ -2629,7 +2631,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2653,7 +2655,7 @@ def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_pla @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2686,7 +2688,7 @@ def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail( @@ -2738,7 +2740,7 @@ def test_get_gravity_compensation_forces_not_implemented_on_newton(sim, num_arti @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2764,7 +2766,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2800,7 +2802,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2833,7 +2835,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2859,7 +2861,7 @@ def test_get_mass_matrix_shape_floating_base(sim, num_articulations, device, add assert M.dtype == torch.float32 -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2947,7 +2949,7 @@ def test_heterogeneous_scene_per_view_shapes(sim, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3026,7 +3028,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3077,7 +3079,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3128,7 +3130,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3161,7 +3163,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( ) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3238,7 +3240,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index d175134f60f8..052198ec9133 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -9,11 +9,13 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index 9d5355adc31b..afcb0f1cafb4 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -9,11 +9,13 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index 5a6cf65c85fa..fc11edafc4d2 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -8,13 +8,15 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import DeviceScope, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" @@ -2248,7 +2250,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2265,7 +2267,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2292,7 +2294,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2317,7 +2319,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2369,7 +2371,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2405,7 +2407,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2456,7 +2458,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2490,7 +2492,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2579,7 +2581,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2637,7 +2639,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2801,7 +2803,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2835,7 +2837,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", test_devices("01X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index f7b1c280df71..01456abfa733 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -9,11 +9,13 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index 4ec56828af82..6aae0c74618a 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -9,11 +9,13 @@ """Launch Isaac Sim Simulator first.""" +import os + from isaaclab.app import AppLauncher from isaaclab.test.utils import test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index 9a1c5e9e34ea..dadbe0530af5 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,6 +10,7 @@ Camera prim type for Fabric SelectPrims compatibility). """ +import os import sys from pathlib import Path @@ -17,7 +18,7 @@ from isaaclab.app import AppLauncher -simulation_app = AppLauncher(headless=True).app +simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app import pytest # noqa: E402 import torch # noqa: E402 @@ -29,7 +30,7 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.test.utils import test_devices # noqa: E402 +from isaaclab.test.utils import DeviceScope, test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) @@ -252,7 +253,7 @@ def force_topology_changed(): # ------------------------------------------------------------------ -@pytest.mark.parametrize("device", test_devices("00X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -272,7 +273,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.parametrize("device", test_devices("00X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -300,7 +301,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.parametrize("device", test_devices("00X")) +@pytest.mark.parametrize("device", test_devices(DeviceScope.NON_DEFAULT_CUDA)) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """set_scales -> get_scales roundtrip works on cuda:1. diff --git a/tools/conftest.py b/tools/conftest.py index 3688b4d4aa77..e98562565918 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -354,7 +354,7 @@ def _claim_queued_file(queue_dir): The decoded test path for the claimed file, or ``None`` when the queue is empty. """ - shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + shard = os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda").replace(":", "-") pending_dir = os.path.join(queue_dir, "queue") inflight_dir = os.path.join(queue_dir, "inflight", shard) os.makedirs(inflight_dir, exist_ok=True) @@ -391,7 +391,7 @@ def _mark_queued_file_done(queue_dir, test_path): The inflight residual is what the post-run reconciler uses to detect crashed shards: anything still in ``inflight/`` at job-end is an orphan. """ - shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + shard = os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda").replace(":", "-") entry = _slugify_test_path(test_path) src = os.path.join(queue_dir, "inflight", shard, entry) dst_dir = os.path.join(queue_dir, "done", shard) @@ -1252,7 +1252,7 @@ def pytest_sessionstart(session): summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" # GPU this run used (the shard's boot device); ``cuda:0`` when unset. - run_device = os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0" + run_device = os.environ.get("ISAACLAB_TEST_SIM_DEVICE") or "cuda:0" summary_str += "\n\n=======================\n" summary_str += "Per File Result Summary\n" From 52a23aca73eb5999a426be04ec2ab4c674fe4f46 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 30 Jun 2026 00:02:31 +0000 Subject: [PATCH 19/22] Unify multi-GPU test device selection Make DeviceScope composable while preserving raw mask support. Derive Kit boot and shard reporting from ISAACLAB_TEST_DEVICES so test device selection has one source of truth. --- .../actions/multi-gpu/mgpu_shard_select.py | 5 +- .../multi-gpu/multi_gpu_host_launcher.sh | 4 +- .../multi-gpu/multi_gpu_shard_runner.sh | 5 +- .github/workflows/test-multi-gpu-pytest.yaml | 2 +- .../jichuanh-multi-gpu-ci.minor.rst | 12 ++- .../isaaclab/isaaclab/test/utils/__init__.py | 12 ++- .../isaaclab/isaaclab/test/utils/devices.py | 92 ++++++++++++++++--- .../test/sim/test_simulation_context.py | 6 +- .../test/sim/test_views_xform_prim.py | 6 +- .../test/utils/test_device_selection.py | 41 ++++++++- .../test/assets/test_articulation.py | 6 +- .../test/assets/test_rigid_object.py | 6 +- .../assets/test_rigid_object_collection.py | 6 +- .../test/assets/test_articulation.py | 6 +- .../test/assets/test_rigid_object.py | 6 +- .../assets/test_rigid_object_collection.py | 6 +- .../test/sim/test_views_xform_prim_fabric.py | 5 +- tools/conftest.py | 11 ++- 18 files changed, 162 insertions(+), 75 deletions(-) diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py index a8402cb11ca6..38d1ac18827c 100644 --- a/.github/actions/multi-gpu/mgpu_shard_select.py +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -13,9 +13,8 @@ Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's device parametrization, so this plugin's keep/drop decision agrees with what was -parametrized. (``ISAACLAB_TEST_SIM_DEVICE`` is passed explicitly to AppLauncher -by Kit-backed tests, a different concern, and is deliberately NOT used here.) The -plugin reads the mask string directly -- no isaaclab import -- matching the +parametrized. Kit-backed tests derive their AppLauncher device from the same mask. +The plugin reads the mask string directly -- no isaaclab import -- matching the lib-free convention of every other conftest in the repo. The only shared knowledge is the mask grammar below, mirrored from ``devices.py`` (position 0 = cpu, position k = cuda:(k-1), trailing ``X`` = include all remaining); keep the two in sync. diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh index 03174f427f06..19d6dd8bcf1e 100755 --- a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -12,8 +12,8 @@ # and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). # # 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its -# own non-default cuda:N via ISAACLAB_TEST_SIM_DEVICE / ISAACLAB_TEST_DEVICES, -# pulling from the shared queue). Collapsing the previous N-container layout into one +# own non-default cuda:N via ISAACLAB_TEST_DEVICES, pulling from the shared +# queue). Collapsing the previous N-container layout into one # removes cross-container races on the workspace mount (``_isaac_sim`` symlink, # ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at # the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index 6b835cec9e80..617226238a2b 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -26,7 +26,7 @@ # 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch # can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) # 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + -# ISAACLAB_TEST_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# ISAACLAB_TEST_DEVICES; each shard tees its stdout to # /shard-logs/cuda-N.log for the host's grouped re-print after the run # 6. Waits on every shard before aggregating exit codes — a fast failure doesn't # tear down still-running siblings @@ -78,7 +78,7 @@ fi # Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME # (per-shard isolation for .cache, .local/share, etc.) and per-shard -# ISAACLAB_TEST_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +# ISAACLAB_TEST_DEVICES. declare -A pids # associative array: shard index -> background PID for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) zeros="" @@ -98,7 +98,6 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to sk export XDG_CACHE_HOME="${HOME}/.cache" export XDG_DATA_HOME="${HOME}/.local/share" export ISAACLAB_TEST_DEVICES="$runtime_devices" - export ISAACLAB_TEST_SIM_DEVICE="cuda:${cuda}" # Full pytest output captures to $shard_log; live stdout is filtered # down to high-signal lines (test boundaries, failures, summary stats, diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index e759f9f0dba0..63c03a73779a 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -124,7 +124,7 @@ jobs: # Used for files with known Kit/Isaac-Sim concurrency issues; the file # still runs in single-GPU CI. Excluded files are reported as a notice # below for visibility. No workflow edit needed to add/remove a file. - device_scope_pattern='test_devices\(\)|test_devices\(DeviceScope\.(ALL|CUDA|NON_DEFAULT_CUDA)|test_devices\("[^"]*X"\)' + device_scope_pattern='test_devices\(\)|test_devices\([^)]*DeviceScope\.(ALL|CUDA|NON_DEFAULT_CUDA)|test_devices\("[^"]*X"\)' mapfile -t candidates < <(grep -rlE "$device_scope_pattern" source/ --include='test_*.py' | sort -u) discovered=() skipped=() diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst index 084c95e5f606..9c3d1ac84352 100644 --- a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -1,8 +1,10 @@ Added ^^^^^ -* Added :class:`isaaclab.test.utils.DeviceScope` and - :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over a - device set resolved as ``scope ∩ runtime``. Named scopes cover common cases, - string masks support custom combinations, and multi-GPU CI narrows the runtime - to one non-default GPU per shard without changing single-GPU CI behavior. +* Added :class:`isaaclab.test.utils.DeviceScope`, + :func:`isaaclab.test.utils.test_devices`, and + :func:`isaaclab.test.utils.resolve_test_sim_device` to parametrize unit tests + and launch Kit from the same runtime device mask. Composable scopes cover + common cases, string masks support custom combinations, and multi-GPU CI + narrows the runtime to one non-default GPU per shard without changing + single-GPU CI behavior. diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py index 2c2ee0249e98..c9631a080592 100644 --- a/source/isaaclab/isaaclab/test/utils/__init__.py +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -6,11 +6,13 @@ """Test-time helpers for Isaac Lab. Exposes :class:`DeviceScope` and :func:`test_devices` for selecting the device -list to parametrize tests over. The set is ``scope ∩ runtime``: ``scope`` is the -call-site argument (the devices the test is valid on), the runtime is the -``ISAACLAB_TEST_DEVICES`` env var (the devices the run may use). +list to parametrize tests over, plus :func:`resolve_test_sim_device` for deriving +a Kit-backed test's AppLauncher device from the same runtime mask. The selected +set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the devices the +test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env var (the +devices the run may use). """ -from .devices import DeviceScope, test_devices +from .devices import DeviceScope, resolve_test_sim_device, test_devices -__all__ = ["DeviceScope", "test_devices"] +__all__ = ["DeviceScope", "resolve_test_sim_device", "test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py index b240b701ea6b..9f5db8a609f4 100644 --- a/source/isaaclab/isaaclab/test/utils/devices.py +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -7,9 +7,12 @@ Intended use:: - from isaaclab.test.utils import DeviceScope, test_devices + from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices - @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app + + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU @pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) def test_foo(device): ... @@ -21,8 +24,9 @@ def test_foo(device): ... is valid on. The author owns this; defaults to :attr:`DeviceScope.ALL`. * **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). - The multi-GPU workflow sets it to one device per shard, matching - ``ISAACLAB_TEST_SIM_DEVICE`` (the explicit AppLauncher boot device — not read here). + The multi-GPU workflow sets it to one device per shard. Kit-backed tests use + :func:`resolve_test_sim_device` to derive their AppLauncher boot device from + the same mask. A test author never names the shard's GPU; the operator never inspects a test's device support. The helper intersects the two. @@ -44,6 +48,7 @@ def test_foo(device): ... ``DeviceScope.ALL`` ``11X`` cpu + every GPU ``DeviceScope.CUDA`` ``01X`` every GPU ``DeviceScope.CPU_AND_DEFAULT_CUDA`` ``110`` cpu + cuda:0 +``DeviceScope.DEFAULT_CUDA`` ``010`` cuda:0 only ``DeviceScope.NON_DEFAULT_CUDA`` ``00X`` cuda:1 and above ``DeviceScope.CPU`` ``100`` cpu only ==================================== ========= ============================================= @@ -70,14 +75,13 @@ def test_foo(device): ... ---------- Set the runtime from the shell to opt a run into non-default GPUs:: - ISAACLAB_TEST_DEVICES=0001 ISAACLAB_TEST_SIM_DEVICE=cuda:2 \\ - ./isaaclab.sh -p -m pytest path/to/test.py + ISAACLAB_TEST_DEVICES=0001 ./isaaclab.sh -p -m pytest path/to/test.py """ from __future__ import annotations import os -from enum import StrEnum +from enum import Flag, auto _RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" """Env var naming the run's devices: the devices a run may use (see module docstring).""" @@ -87,18 +91,71 @@ def test_foo(device): ... i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" -class DeviceScope(StrEnum): - """Named device scopes for common test parametrization cases. +class DeviceScope(Flag): + """Composable device scopes for test parametrization. String masks remain accepted by :func:`test_devices` for custom device combinations that are not represented here. """ - ALL = "11X" - CUDA = "01X" - CPU_AND_DEFAULT_CUDA = "110" - NON_DEFAULT_CUDA = "00X" - CPU = "100" + CPU = auto() + DEFAULT_CUDA = auto() + NON_DEFAULT_CUDA = auto() + + CPU_AND_DEFAULT_CUDA = CPU | DEFAULT_CUDA + CUDA = DEFAULT_CUDA | NON_DEFAULT_CUDA + ALL = CPU | CUDA + + @property + def mask(self) -> str: + """Return this scope in the device-mask representation.""" + return "".join( + [ + "1" if self & DeviceScope.CPU else "0", + "1" if self & DeviceScope.DEFAULT_CUDA else "0", + "X" if self & DeviceScope.NON_DEFAULT_CUDA else "0", + ] + ) + + +def resolve_test_sim_device() -> str: + """Resolve the AppLauncher device from the test runtime device mask. + + This function intentionally parses the mask without enumerating devices so + it can run before AppLauncher without importing torch or initializing Warp. + A Kit-backed test process can boot on only one GPU, so an explicitly set + runtime must select at most one CUDA device. The CPU bit may accompany that + GPU because the default single-GPU runtime covers both CPU and CUDA tests. + + Returns: + The selected ``"cuda:N"`` device, ``"cpu"`` for a CPU-only runtime, + or ``"cuda:0"`` when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset. + + Raises: + ValueError: When the runtime mask is invalid, selects no device, uses a + wildcard, or selects multiple CUDA devices. + """ + runtime = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + if runtime is None: + return "cuda:0" + if not runtime or any(char not in "01X" for char in runtime) or "X" in runtime[:-1]: + raise ValueError(f"Invalid {_RUNTIME_DEVICES_ENV_VAR} mask: {runtime!r}") + if runtime.endswith("X"): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} is ambiguous for AppLauncher; use a concrete runtime mask" + ) + + cuda_positions = [position for position, included in enumerate(runtime[1:], start=1) if included == "1"] + if len(cuda_positions) > 1: + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} selects multiple CUDA devices; " + "Kit-backed tests require one GPU per process" + ) + if cuda_positions: + return f"cuda:{cuda_positions[0] - 1}" + if runtime[0] == "1": + return "cpu" + raise ValueError(f"{_RUNTIME_DEVICES_ENV_VAR}={runtime!r} selects no device") def test_devices(scope: str | DeviceScope = DeviceScope.ALL, *, skip: dict[str, str] | None = None) -> list: @@ -141,7 +198,7 @@ def test_devices(scope: str | DeviceScope = DeviceScope.ALL, *, skip: dict[str, raise ValueError( f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" ) - scope_keep = _expand(scope, len(available)) + scope_keep = _expand(_scope_mask(scope), len(available)) devices = [ device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime ] @@ -173,6 +230,11 @@ def _expand(mask: str, count: int) -> list[bool]: return ([char == "1" for char in body] + [fill] * count)[:count] +def _scope_mask(scope: str | DeviceScope) -> str: + """Convert a named device scope to its mask representation.""" + return scope if isinstance(scope, str) else scope.mask + + def _list_available_devices() -> list[str]: """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 81a67adc2c56..018218112199 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -5,13 +5,11 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 598075d0791a..4a7f3097aae3 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -10,12 +10,10 @@ prim ordering, xformOp standardization, and Isaac Sim comparison. """ -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app import pytest # noqa: E402 import torch # noqa: E402 diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py index 0c7e976f825d..46d592c7f33b 100644 --- a/source/isaaclab/test/utils/test_device_selection.py +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Unit tests for :func:`isaaclab.test.utils.test_devices` (the device-selection helper). +"""Unit tests for the test-device selection helpers. These tests mock the host's device list, so they need no GPU and run on the single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) @@ -15,7 +15,7 @@ import pytest from isaaclab.test.utils import devices as devices_mod -from isaaclab.test.utils.devices import DeviceScope +from isaaclab.test.utils.devices import DeviceScope, resolve_test_sim_device from isaaclab.test.utils.devices import test_devices as resolve_devices # Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). @@ -89,17 +89,54 @@ def test_argless_runs_once_on_one_non_default_gpu(host): (DeviceScope.CPU_AND_DEFAULT_CUDA, "110"), (DeviceScope.NON_DEFAULT_CUDA, "00X"), (DeviceScope.CPU, "100"), + (DeviceScope.DEFAULT_CUDA, "010"), + (DeviceScope.CPU | DeviceScope.NON_DEFAULT_CUDA, "10X"), ], ) def test_named_scope_matches_mask(host, scope, mask): host(MULTI_GPU, "1111") + assert scope.mask == mask assert resolve_devices(scope) == resolve_devices(mask) +def test_named_combination_matches_runtime_composition(): + assert DeviceScope.CPU_AND_DEFAULT_CUDA == DeviceScope.CPU | DeviceScope.DEFAULT_CUDA + assert DeviceScope.CUDA == DeviceScope.DEFAULT_CUDA | DeviceScope.NON_DEFAULT_CUDA + assert DeviceScope.ALL == DeviceScope.CPU | DeviceScope.DEFAULT_CUDA | DeviceScope.NON_DEFAULT_CUDA + + def test_helper_is_not_collected_by_pytest(): assert resolve_devices.__test__ is False +# --------------------------------------------------------------------------- +# AppLauncher device resolution from the runtime mask +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "runtime, expected", + [ + (None, "cuda:0"), + ("110", "cuda:0"), + ("100", "cpu"), + ("001", "cuda:1"), + ("0001", "cuda:2"), + ("101", "cuda:1"), + ], +) +def test_resolve_test_sim_device(host, runtime, expected): + host(MULTI_GPU, runtime) + assert resolve_test_sim_device() == expected + + +@pytest.mark.parametrize("runtime", ["", "000", "011", "01X", "0X1", "0A1"]) +def test_resolve_test_sim_device_rejects_invalid_or_ambiguous_runtime(host, runtime): + host(MULTI_GPU, runtime) + with pytest.raises(ValueError, match="ISAACLAB_TEST_DEVICES"): + resolve_test_sim_device() + + # --------------------------------------------------------------------------- # skip vs raise: legitimate skips never raise; only a missing runtime device does # --------------------------------------------------------------------------- diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index eeaa70bd6ff4..1db76ff5f55e 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -8,15 +8,13 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import DeviceScope, test_devices +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 052198ec9133..4ed03df445e5 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -9,13 +9,11 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index afcb0f1cafb4..8002ce57243b 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -9,13 +9,11 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index fc11edafc4d2..661354777d90 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -8,15 +8,13 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import DeviceScope, test_devices +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices HEADLESS = True # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index 01456abfa733..072a559216eb 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -9,13 +9,11 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index 6aae0c74618a..7fad208a4db6 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -9,13 +9,11 @@ """Launch Isaac Sim Simulator first.""" -import os - from isaaclab.app import AppLauncher -from isaaclab.test.utils import test_devices +from isaaclab.test.utils import resolve_test_sim_device, test_devices # launch omniverse app -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index dadbe0530af5..8a4c277214f9 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,15 +10,15 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) from isaaclab.app import AppLauncher +from isaaclab.test.utils import DeviceScope, resolve_test_sim_device, test_devices -simulation_app = AppLauncher(headless=True, device=os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda:0")).app +simulation_app = AppLauncher(headless=True, device=resolve_test_sim_device()).app import pytest # noqa: E402 import torch # noqa: E402 @@ -30,7 +30,6 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 -from isaaclab.test.utils import DeviceScope, test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) diff --git a/tools/conftest.py b/tools/conftest.py index e98562565918..d118fdc945c4 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -18,6 +18,8 @@ from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable +from isaaclab.test.utils import resolve_test_sim_device + # Local imports import test_settings as test_settings # isort: skip from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip @@ -354,7 +356,7 @@ def _claim_queued_file(queue_dir): The decoded test path for the claimed file, or ``None`` when the queue is empty. """ - shard = os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda").replace(":", "-") + shard = resolve_test_sim_device().replace(":", "-") pending_dir = os.path.join(queue_dir, "queue") inflight_dir = os.path.join(queue_dir, "inflight", shard) os.makedirs(inflight_dir, exist_ok=True) @@ -391,7 +393,7 @@ def _mark_queued_file_done(queue_dir, test_path): The inflight residual is what the post-run reconciler uses to detect crashed shards: anything still in ``inflight/`` at job-end is an orphan. """ - shard = os.environ.get("ISAACLAB_TEST_SIM_DEVICE", "cuda").replace(":", "-") + shard = resolve_test_sim_device().replace(":", "-") entry = _slugify_test_path(test_path) src = os.path.join(queue_dir, "inflight", shard, entry) dst_dir = os.path.join(queue_dir, "done", shard) @@ -1251,8 +1253,9 @@ def pytest_sessionstart(session): summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" - # GPU this run used (the shard's boot device); ``cuda:0`` when unset. - run_device = os.environ.get("ISAACLAB_TEST_SIM_DEVICE") or "cuda:0" + # GPU this run used (the shard's boot device); ``cuda:0`` when the runtime + # device mask is unset. + run_device = resolve_test_sim_device() summary_str += "\n\n=======================\n" summary_str += "Per File Result Summary\n" From 14593eaebf322da4bd746a2d6b796bc4d34efc9e Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 30 Jun 2026 01:03:03 +0000 Subject: [PATCH 20/22] Fix OVPhysX test merge resolution --- .../test/assets/test_articulation.py | 216 ++++++------------ 1 file changed, 65 insertions(+), 151 deletions(-) diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index c463bd8acc25..412576cfb86f 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -19,11 +19,14 @@ environment. PhysX-specific ``cube_object.root_view.set_X(...)`` / ``get_X(...)`` calls are -adapted to OVPhysX by going through the backend's per-tensor-type binding -dictionary (``cube_object._bindings`` / :meth:`~isaaclab_ovphysx.assets.Articulation._get_binding`) -and the public setters (:meth:`set_masses_index`, :meth:`set_coms_index`, -:meth:`set_inertias_index`). Reads use the data-class properties -(``cube_object.data.body_mass``, ``body_inertia``, ``body_com_pose_b``). +adapted to OVPhysX by going through +:attr:`~isaaclab_ovphysx.assets.Articulation.root_view`, an +:class:`~isaaclab_ovphysx.sim.views.OvPhysxView` over the per-tensor-type bindings +(``root_view.get_attribute(tensor_type)`` / +:meth:`~isaaclab_ovphysx.assets.Articulation._get_binding`), and the public setters +(:meth:`set_masses_index`, :meth:`set_coms_index`, :meth:`set_inertias_index`). +Reads use the data-class properties (``cube_object.data.body_mass``, +``body_inertia``, ``body_com_pose_b``). Process-global device lock -------------------------- @@ -51,7 +54,6 @@ import sys -import numpy as np import pytest import torch import warp as wp @@ -96,26 +98,25 @@ _MATERIAL_GAP_REASON = ( "Requires a ``RIGID_BODY_MATERIAL`` TensorType (or a view-helper) on the " - "ovphysx wheel side. ``Articulation.root_view`` is a per-tensor-type " - "bindings dict on OVPhysX, so ``root_view.get_material_properties()`` / " + "ovphysx wheel side. ``Articulation.root_view`` is an ``OvPhysxView`` over " + "the per-tensor-type bindings on OVPhysX, so ``root_view.get_material_properties()`` / " "``set_material_properties()`` / ``max_shapes`` are not available. See " "docs/superpowers/specs/2026-04-28-ovphysx-wheel-gaps-for-marco.md." ) def _read_binding_to_torch(articulation: Articulation, tensor_type: int, device: str | torch.device) -> torch.Tensor: - """Read an OVPhysX TensorBinding into a torch tensor on *device*. + """Read an OVPhysX attribute into a torch tensor on *device*. Test-side adapter for the verbatim PhysX mirror. PhysX cross-checks the data class against the simulation via ``articulation.root_view.get_X()`` - accessors; on OVPhysX, ``root_view`` is a per-tensor-type bindings dict - (no view-level getters), so we read the binding directly into a CPU - numpy buffer (CPU-only types) and move the result to *device*. + accessors; on OVPhysX we go through the equivalent + :meth:`~isaaclab_ovphysx.sim.views.OvPhysxView.get_attribute`, which returns a + freshly allocated ``float32`` array on the attribute's native device (CPU for + CPU-only property types), then move the result to *device*. """ - binding = articulation.root_view[tensor_type] - np_buf = np.zeros(binding.shape, dtype=np.float32) - binding.read(np_buf) - return torch.from_numpy(np_buf).to(device) + arr = articulation.root_view.get_attribute(tensor_type) + return wp.to_torch(arr).to(device) # Session-locked device. Set on the first parametrized test that runs and @@ -367,15 +368,17 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -430,15 +433,17 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -492,15 +497,17 @@ def test_initialization_fixed_base(sim, num_articulations, device): # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -563,15 +570,17 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -636,11 +645,13 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): # PhysX ``root_view.max_dofs == shared_metatype.dof_count`` identity is # replaced with binding-shape checks on OVPhysX. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # -- actuator type for actuator_name, actuator in articulation.actuators.items(): is_implicit_model_cfg = isinstance(articulation_cfg.actuators[actuator_name], ImplicitActuatorCfg) @@ -692,15 +703,17 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -758,15 +771,17 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de # Cross-check binding shapes against cached counts. PhysX does this via # ``root_view.max_dofs == shared_metatype.dof_count``; on OVPhysX - # ``root_view`` is the per-tensor-type bindings dict, so the equivalent + # ``root_view`` is an ``OvPhysxView`` over the per-tensor-type bindings, so the equivalent # invariant is that each per-DOF / per-link binding's shape agrees with # the count cached on the asset. for tt in (TT.DOF_POSITION, TT.DOF_VELOCITY, TT.DOF_STIFFNESS): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_joints + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_joints for tt in (TT.BODY_MASS, TT.BODY_COM_POSE): - if tt in articulation.root_view: - assert articulation.root_view[tt].shape[1] == articulation.num_bodies + binding = articulation.root_view.try_binding_for(tt) + if binding is not None: + assert binding.shape[1] == articulation.num_bodies # Body-name ordering check is degenerate on OVPhysX: ``body_names`` is # sourced from binding metadata (``sample.body_names``), so the PhysX # ``link_paths[0]`` round-trip is a no-op here and is omitted. @@ -2025,107 +2040,6 @@ def test_body_com_pose_b_cache_and_set_coms_invalidation(sim, device): assert buffer.timestamp < articulation.data._sim_timestamp, name -@pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", test_devices()) -def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, device): - """Test the data.body_incoming_joint_wrench_b buffer is populated correctly and statically correct for single joint. - - This test verifies that: - 1. The body incoming joint wrench buffer has correct shape - 2. The wrench values are statically correct for a single joint - 3. The wrench values match expected values from gravity and external forces - - Args: - sim: The simulation fixture - num_articulations: Number of articulations to test - device: The device to run the simulation on - """ - articulation_cfg = generate_articulation_cfg(articulation_type="single_joint_implicit") - articulation, _ = generate_articulation( - articulation_cfg=articulation_cfg, num_articulations=num_articulations, device=device - ) - - # Play the simulator - sim.reset() - - # Resolve body indices by name (ordering may differ across physics backends) - arm_idx = articulation.body_names.index("Arm") - root_idx = articulation.body_names.index("CenterPivot") - # apply external force - external_force_vector_b = torch.zeros((num_articulations, articulation.num_bodies, 3), device=device) - external_force_vector_b[:, arm_idx, 1] = 10.0 # 10 N in Y direction - external_torque_vector_b = torch.zeros((num_articulations, articulation.num_bodies, 3), device=device) - external_torque_vector_b[:, arm_idx, 2] = 10.0 # 10 Nm in z direction - - # apply action to the articulation - joint_pos = torch.ones_like(articulation.data.joint_pos.torch) * 1.5708 / 2.0 - articulation.write_joint_position_to_sim_index( - position=torch.ones_like(articulation.data.joint_pos.torch), - ) - articulation.write_joint_velocity_to_sim_index( - velocity=torch.zeros_like(articulation.data.joint_vel.torch), - ) - articulation.set_joint_position_target_index(target=joint_pos) - articulation.write_data_to_sim() - for _ in range(50): - articulation.permanent_wrench_composer.set_forces_and_torques_index( - forces=external_force_vector_b, torques=external_torque_vector_b - ) - articulation.write_data_to_sim() - # perform step - sim.step() - # update buffers - articulation.update(sim.cfg.dt) - - # check shape - assert articulation.data.body_incoming_joint_wrench_b.torch.shape == ( - num_articulations, - articulation.num_bodies, - 6, - ) - - # calculate expected static - mass = articulation.data.body_mass.torch.to("cpu") - pos_w = articulation.data.body_pos_w.torch - quat_w = articulation.data.body_quat_w.torch - - mass_link2 = mass[:, arm_idx].view(num_articulations, -1) - gravity = torch.tensor(sim.cfg.gravity, device="cpu").repeat(num_articulations, 1).view((num_articulations, 3)) - - # NOTE: the com and link pose for single joint are colocated - weight_vector_w = mass_link2 * gravity - # expected wrench from link mass and external wrench - # PhysX reports the incoming joint wrench as the force FROM body0 ONTO body1 (body1's frame). - # The USD asset defines body0=CenterPivot, body1=Arm, so the wrench is the constraint/support - # force from CenterPivot onto Arm, expressed in Arm's frame. - # In static equilibrium this equals -(gravity + external forces on Arm). - total_force_w = weight_vector_w.to(device) + math_utils.quat_apply( - quat_w[:, arm_idx, :], external_force_vector_b[:, arm_idx, :] - ) - total_torque_w = torch.cross( - pos_w[:, arm_idx, :].to(device) - pos_w[:, root_idx, :].to(device), - total_force_w, - dim=-1, - ) + math_utils.quat_apply(quat_w[:, arm_idx, :], external_torque_vector_b[:, arm_idx, :]) - expected_wrench = torch.zeros((num_articulations, 6), device=device) - expected_wrench[:, :3] = math_utils.quat_apply( - math_utils.quat_conjugate(quat_w[:, arm_idx, :]), - -total_force_w, - ) - expected_wrench[:, 3:] = math_utils.quat_apply( - math_utils.quat_conjugate(quat_w[:, arm_idx, :]), - -total_torque_w, - ) - - # check value of last joint wrench - torch.testing.assert_close( - expected_wrench, - articulation.data.body_incoming_joint_wrench_b.torch[:, arm_idx, :].squeeze(1), - atol=1e-2, - rtol=1e-3, - ) - - @pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" From 6a2d55a06cafffa48dba48b82b66aba683fb50ee Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 7 Jul 2026 22:31:14 +0000 Subject: [PATCH 21/22] Switch multi-GPU CI Kit override to ISAACLAB_FABRIC_USE_GPU_INTEROP The renderer-pinning env var ISAACLAB_PIN_KIT_GPU was superseded by #5933, which makes single-GPU rendering the Kit default and adds ISAACLAB_FABRIC_USE_GPU_INTEROP for the fabric GPU-interop override. Point the multi-GPU CI launcher/runner at the new env var. --- .github/actions/multi-gpu/multi_gpu_host_launcher.sh | 2 +- .github/actions/multi-gpu/multi_gpu_shard_runner.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh index 19d6dd8bcf1e..ea6bdfc719b7 100755 --- a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -104,7 +104,7 @@ docker run --rm --gpus all --network=host \ -e PYTHONIOENCODING=utf-8 \ -e ISAACLAB_TEST_QUEUE=/mgpu \ -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ - -e ISAACLAB_PIN_KIT_GPU=1 \ + -e ISAACLAB_FABRIC_USE_GPU_INTEROP=0 \ -e HOME=/tmp/mgpu-base-home \ -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ "${cvd_args[@]}" \ diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index 617226238a2b..6eafae80c0be 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -15,7 +15,7 @@ # -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) # -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) # -e TEST_INCLUDE_FILES="" -# -e ISAACLAB_PIN_KIT_GPU=1 (Kit overrides for the multi-GPU race) +# -e ISAACLAB_FABRIC_USE_GPU_INTEROP=0 (temporary Kit/PhysX CI workaround) # -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) # # Behavior: From 998400a6ea07e2da87ac4fb9670486596a147885 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 8 Jul 2026 22:29:04 +0000 Subject: [PATCH 22/22] Scope multi-GPU CI newton changelog to test-only The Newton non-default-device init fix moved to its own PR (#6322, merged), so this PR no longer makes a user-facing isaaclab_newton change -- only device_scope test migrations. Convert the fragment to .skip. --- source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst | 5 ----- .../isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip | 0 2 files changed, 5 deletions(-) delete mode 100644 source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst create mode 100644 source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst deleted file mode 100644 index c53da690a8d8..000000000000 --- a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed -^^^^^ - -* Fixed Newton physics failing to initialize on non-default CUDA devices - (``cuda:1`` and higher). diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1