From 3ca93df0f0d3468ddb797abde1194589129d9dbb Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 30 May 2026 21:13:56 +0000 Subject: [PATCH 01/23] Honor device kwarg over sim_cfg.device in build_simulation_context Most test callers pass both ``sim_cfg=`` and ``device=`` to :func:`isaaclab.sim.build_simulation_context`, implicitly expecting the ``device`` kwarg to win. The helper previously dropped the kwarg silently when ``sim_cfg`` was provided, causing warp kernel-launch device mismatches on non-default GPUs: the test fixture allocated ``env_ids`` on the requested device while the articulation's ``self.device`` resolved from the untouched ``sim_cfg`` default (``cuda:0``), and ``wp.launch(..., device=self.device)`` failed with:: RuntimeError: Error launching kernel 'set_root_link_pose_to_sim_index', trying to launch on device='cuda:0', but input array for argument 'env_ids' is on device=cuda:2. Change ``device``'s default to ``None`` (sentinel) and apply it as an override after sim_cfg construction in both branches. The one test that asserted the old "sim_cfg overrides everything" contract is updated to cover the new override semantics. --- .../jichuanh-fix-build-sim-context-device.rst | 11 ++++++++++ .../isaaclab/sim/simulation_context.py | 20 ++++++++++++++++--- .../test_build_simulation_context_headless.py | 19 +++++++++++++++--- ...st_build_simulation_context_nonheadless.py | 20 +++++++++++++++---- 4 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst diff --git a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst new file mode 100644 index 000000000000..89e774bd26d7 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed :func:`isaaclab.sim.build_simulation_context` silently ignoring the + ``device`` kwarg when ``sim_cfg`` is also provided. Most test callers pass + both kwargs together; the helper now applies the explicit ``device`` over + ``sim_cfg.device`` so the caller's choice wins. Without this, warp kernel + launches in :mod:`isaaclab_newton.assets.articulation` raised device + mismatch errors on non-default GPUs (``env_ids`` allocated on the test's + device while the articulation's resolved device came from the untouched + ``sim_cfg`` default ``cuda:0``). diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index c068469ec599..fae7d4d1f510 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -958,7 +958,7 @@ def _predicate(prim: Usd.Prim) -> bool: def build_simulation_context( create_new_stage: bool = True, gravity_enabled: bool = True, - device: str = "cuda:0", + device: str | None = None, dt: float = 0.01, sim_cfg: SimulationCfg | None = None, add_ground_plane: bool = False, @@ -971,7 +971,11 @@ def build_simulation_context( Args: create_new_stage: Whether to create a new stage. Defaults to True. gravity_enabled: Whether to enable gravity. Defaults to True. - device: Device to run the simulation on. Defaults to "cuda:0". + device: Device to run the simulation on. When given alongside ``sim_cfg``, + overrides ``sim_cfg.device`` so the caller's explicit choice wins + (most test callers pass both, expecting this behavior). Defaults to + ``None``, meaning ``sim_cfg.device`` is left untouched and a freshly + built ``sim_cfg`` uses :class:`SimulationCfg`'s default device. dt: Time step for the simulation. Defaults to 0.01. sim_cfg: SimulationCfg to use. Defaults to None. add_ground_plane: Whether to add a ground plane. Defaults to False. @@ -993,7 +997,17 @@ def build_simulation_context( if sim_cfg is None: gravity = (0.0, 0.0, -9.81) if gravity_enabled else (0.0, 0.0, 0.0) - sim_cfg = SimulationCfg(device=device, dt=dt, gravity=gravity) + sim_cfg = SimulationCfg(dt=dt, gravity=gravity) + if device is not None: + # Honor the explicit device kwarg in both branches: when sim_cfg is + # freshly built, this picks the device; when sim_cfg is passed in, + # this overrides its (possibly default) device. Without the override, + # callers passing both ``sim_cfg=`` and + # ``device=cuda:N`` silently got sim_cfg's device, causing warp + # kernel-launch mismatches when test fixtures allocated tensors on + # the requested device while assets resolved their device from the + # untouched sim_cfg. + sim_cfg.device = device sim = SimulationContext(sim_cfg) diff --git a/source/isaaclab/test/sim/test_build_simulation_context_headless.py b/source/isaaclab/test/sim/test_build_simulation_context_headless.py index 8f13d79041e7..4ceae87b9878 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_headless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_headless.py @@ -75,7 +75,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light @pytest.mark.isaacsim_ci def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -87,8 +94,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt diff --git a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py index 8c053bc51cda..09bfe3091839 100644 --- a/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py +++ b/source/isaaclab/test/sim/test_build_simulation_context_nonheadless.py @@ -70,8 +70,14 @@ def test_build_simulation_context_auto_add_lighting(add_lighting, auto_add_light def test_build_simulation_context_cfg(): - """Test that the simulation context is built with the correct cfg and values don't get overridden.""" - + """Test that the simulation context honors sim_cfg's values, with an explicit + device override winning when both ``sim_cfg`` and ``device`` are passed. + + Most test callers pass both kwargs together expecting the device kwarg to + win; the override branch in :func:`build_simulation_context` exists for + that case. ``gravity`` and ``dt`` are not overridable by the helper's + kwargs (only sim_cfg's values are used). + """ dt = 0.001 # Non-standard gravity gravity = (0.0, 0.0, -1.81) @@ -83,8 +89,14 @@ def test_build_simulation_context_cfg(): dt=dt, ) - with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01, device="cpu") as sim: - # Values from sim_cfg should not be overridden by build_simulation_context args + # Pass only sim_cfg: gravity, device, dt all come from sim_cfg (kwargs ignored). + with build_simulation_context(sim_cfg=cfg, gravity_enabled=False, dt=0.01) as sim: assert sim.cfg.gravity == gravity assert sim.cfg.device == device assert sim.cfg.dt == dt + + # Pass sim_cfg and an explicit device override: device kwarg wins. + with build_simulation_context(sim_cfg=cfg, device="cpu") as sim: + assert sim.cfg.gravity == gravity + assert sim.cfg.device == "cpu" + assert sim.cfg.dt == dt From a396a07f1f299b70662dfdb671f133eb8b70e543 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:48:06 +0000 Subject: [PATCH 02/23] Add ISAACLAB_PIN_KIT_GPU to pin Kit renderer to one GPU Add an ISAACLAB_PIN_KIT_GPU env var to AppLauncher. When truthy, it appends Kit command-line overrides that pin the renderer to a single GPU (renderer.multiGpu.enabled=False, autoEnable=False, maxGpuCount=1) and disable the fabric GPU-interop path (physics.fabricUseGPUInterop= false), so each Kit process touches only its assigned GPU instead of enumerating every visible GPU at startup. Used by the multi-GPU CI workflow to avoid a shared GPU-interop context across concurrent sibling shards, which otherwise surfaces as "Stage X already attached" errors and SimulationApp.close hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). Off by default; single-GPU and user-facing rendering paths are unchanged. --- .../jichuanh-mgpu-pin-kit-resources.rst | 13 ++++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..a78313ab14d9 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,13 @@ +Added +^^^^^ + +* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. + When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, + ``--/renderer/multiGpu/autoEnable=False``, ``--/renderer/multiGpu/maxGpuCount=1`` + and ``--/physics/fabricUseGPUInterop=false`` to the Kit command line so each + Kit process touches only its assigned GPU (rather than enumerating every + visible GPU at startup). Used by the multi-GPU CI workflow to prevent the + shared GPU-interop context across sibling shards that surfaces as + ``[Error] [omni.physx.plugin] Stage X already attached`` and + ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). + Off by default; single-GPU and user-facing rendering paths are unchanged. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e8ff526496a1..53dd8065aa11 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1060,6 +1060,32 @@ def _resolve_device_settings(self, launcher_args: dict): launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id + # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is + # truthy. The default ``apps/isaaclab.python.headless.kit`` sets + # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable + # = true``, so each Kit process enumerates every visible GPU at + # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per + # container, one Kit per non-default cuda device), that produces a + # shared GPU-interop context across sibling processes -- surfacing as + # ``[Error] [omni.physx.plugin] Stage X already attached`` mid-test and + # ``SimulationApp.close`` hanging in teardown (see + # https://github.com/isaac-sim/IsaacLab/issues/3475). The mitigation is + # to set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount = 1`` so + # each Kit only touches its assigned GPU. + if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: + sys.argv.append("--/renderer/multiGpu/enabled=False") + sys.argv.append("--/renderer/multiGpu/autoEnable=False") + sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") + # Also disable the fabric GPU-interop path. The renderer multiGpu + # flags above mitigate the startup-time enumeration race; this + # mitigates the runtime GPU-interop race on top. Safe for the + # multi-GPU CI lane: it covers physics / scene / utility tests, + # not rendering. + sys.argv.append("--/physics/fabricUseGPUInterop=false") + logger.info( + "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" + ) + # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash # Kit's platform-info fork during startup. From f1c0d1844419693a51741c9539f939ff285bf9f3 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:38 +0000 Subject: [PATCH 03/23] [Tests] Add test_devices helper for device-parametrized unit tests Adds the scope-intersect-runtime device selection helper (test_devices) and its unit test, so unit tests can declare the devices they are valid on and the multi-GPU lane can narrow them via ISAACLAB_TEST_DEVICES. --- .../isaaclab/isaaclab/test/utils/__init__.py | 16 ++ .../isaaclab/isaaclab/test/utils/devices.py | 169 ++++++++++++++++++ .../test/utils/test_device_selection.py | 148 +++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 source/isaaclab/isaaclab/test/utils/__init__.py create mode 100644 source/isaaclab/isaaclab/test/utils/devices.py create mode 100644 source/isaaclab/test/utils/test_device_selection.py diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py new file mode 100644 index 000000000000..e268d22c5c16 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test-time helpers for Isaac Lab. + +Exposes :func:`test_devices` for selecting the device list to parametrize tests +over. The set is ``scope ∩ budget``: ``scope`` is the call-site argument (the +devices the test is valid on), ``budget`` is the ``ISAACLAB_TEST_DEVICES`` env +var (the devices the run may use). +""" + +from .devices import test_devices + +__all__ = ["test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py new file mode 100644 index 000000000000..c54cabfdfbd5 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Device selection for parametrizing tests over cpu / cuda devices. + +Intended use:: + + from isaaclab.test.utils import test_devices + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + def test_foo(device): ... + +Two masks, two roles +-------------------- +A test runs on ``scope ∩ runtime``: + +* **scope** — the call-site mask: the devices the *test* is valid on. The + author owns this; defaults to ``"11X"`` (device-agnostic). +* **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* + may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). + The multi-GPU workflow sets it to one device per shard, matching + ``ISAACLAB_SIM_DEVICE`` (AppLauncher's boot device — not read here). + +A test author never names the shard's GPU; the operator never inspects a +test's device support. The helper intersects the two. + +Mask grammar +------------ +Positions left to right: ``0`` = cpu, ``1`` = cuda:0 (the default GPU), +``2`` = cuda:1, ``3`` = cuda:2, ... Each position is ``0`` (exclude) or ``1`` +(include). A trailing ``X`` means "include all the remaining devices". + +cpu, cuda:0, and a non-default GPU stay distinct by position — running on +cuda:0 says nothing about cuda:1+, which is the whole reason this exists. + +Common masks +------------ +====== =================================================================== +Mask Meaning +====== =================================================================== +``11X`` cpu + cuda:0 + every non-default GPU (default scope; device-agnostic) +``110`` cpu + cuda:0 (pure-math / backend tests: cuda:0 == cuda:N) +``00X`` non-default GPUs only (validates non-default-device behavior) +``100`` cpu only (pure logic) +====== =================================================================== + +Worked example — a ``scope="11X"`` (default) test: + +* single-GPU CI (runtime unset ⇒ ``"110"``) ⇒ ``[cpu, cuda:0]``. +* a multi-GPU shard (runtime ``"0001"``, one device) ⇒ ``[cuda:2]``. + +So argless tests run on cpu + cuda:0 in single-GPU CI and on exactly one +non-default GPU per shard in multi-GPU CI (run-once is a property of the +one-device-per-shard runtime plus round-robin file assignment). + +An empty result means the test is cleanly skipped for this run (e.g. a +``"00X"`` test on a single-GPU host, or a ``"110"`` test on a non-default-GPU +shard). But if the run *explicitly* set ``ISAACLAB_TEST_DEVICES`` to devices +the host does not have, that is a misconfigured run and raises instead of +skipping everything and reporting a vacuous green. + +Local runs +---------- +Set the runtime from the shell to opt a run into non-default GPUs:: + + ISAACLAB_TEST_DEVICES=0001 ISAACLAB_SIM_DEVICE=cuda:2 \\ + ./isaaclab.sh -p -m pytest path/to/test.py +""" + +from __future__ import annotations + +import os + +import torch + +_RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" +"""Env var naming the run's devices: the devices a run may use (see module docstring).""" + +_DEFAULT_RUNTIME = "110" +"""Runtime devices when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset: cpu + cuda:0, +i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" + + +def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> list: + """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. + + ``scope`` is this argument; the runtime comes from the + ``ISAACLAB_TEST_DEVICES`` env var (see the module docstring for the grammar + and the scope / runtime split). + + Args: + scope: Device mask (e.g. ``"11X"``) the test is valid on. Defaults to + ``"11X"`` (cpu + cuda:0 + every non-default GPU): the device-agnostic + common case, so most call sites can use ``test_devices()`` with no + argument. + skip: Optional ``{device: reason}``; in-scope devices listed here are + wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with + the reason instead of being dropped. Use to gate a device variant + that is known broken while keeping it visible. + + Returns: + Ordered device entries (``"cpu"`` / ``"cuda:N"`` strings, or + :func:`pytest.param` for skipped ones) for the second argument to + :func:`pytest.mark.parametrize`. Empty means the test is skipped for + this run. + + Raises: + ValueError: When the run explicitly set ``ISAACLAB_TEST_DEVICES`` to + devices that are not available on this host (a misconfigured run that + would otherwise skip everything and pass vacuously). A scope that + merely does not intersect this run's devices skips, it does not raise. + """ + requested = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + available = _list_available_devices() + runtime_keep = _expand(requested or _DEFAULT_RUNTIME, len(available)) + # A run that explicitly named devices the host cannot provide (e.g. a shard + # pinned to cuda:2 on a 2-GPU host) is misconfigured: fail loudly rather than + # skip every test and report a vacuous green. A scope that simply does not + # intersect this run's devices (a "110" test on a non-default-GPU shard) is + # not an error — it skips. + if requested is not None and not any(runtime_keep): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" + ) + scope_keep = _expand(scope, len(available)) + devices = [ + device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime + ] + if not skip: + return devices + import pytest + + return [ + pytest.param(device, marks=pytest.mark.skip(reason=skip[device])) if device in skip else device + for device in devices + ] + + +def _expand(mask: str, count: int) -> list[bool]: + """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. + + Args: + mask: A mask string (positions plus an optional trailing ``X``). + count: The number of available devices to expand to. + + Returns: + A list of ``count`` booleans, one per device position. + """ + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return ([char == "1" for char in body] + [fill] * count)[:count] + + +def _list_available_devices() -> list[str]: + """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. + + Returns: + Ordered list of device strings as torch addresses them. + """ + devices = ["cpu"] + # torch.cuda (not warp) is deliberate: this runs at pytest collection time, + # before AppLauncher boots Kit. torch.cuda.device_count() enumerates without + # creating a CUDA context, whereas warp.get_cuda_devices() initializes the + # warp runtime — doing that at collection, ahead of Kit's device setup, + # risks the non-default-GPU init-order fragility this suite targets (#5132). + if torch.cuda.is_available(): + devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) + return devices diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py new file mode 100644 index 000000000000..ff09b244e20a --- /dev/null +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -0,0 +1,148 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for :func:`isaaclab.test.utils.test_devices` (the device-selection helper). + +These tests mock the host's device list, so they need no GPU and run on the +single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) +for two reasons: pytest would otherwise collect the ``test_``-prefixed function +as a test case, and the multi-GPU workflow's auto-discovery greps for +``test_devices(...)`` call sites — neither should fire on this file. +""" + +import pytest + +from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import test_devices as resolve_devices + +# Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). +SINGLE_GPU = ["cpu", "cuda:0"] +MULTI_GPU = ["cpu", "cuda:0", "cuda:1", "cuda:2"] + + +@pytest.fixture +def host(monkeypatch): + """Return a setter that pins the available device list and the runtime env var.""" + + def _set(available: list[str], runtime: str | None = None) -> None: + monkeypatch.setattr(devices_mod, "_list_available_devices", lambda: available) + if runtime is None: + monkeypatch.delenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, raising=False) + else: + monkeypatch.setenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, runtime) + + return _set + + +# --------------------------------------------------------------------------- +# scope ∩ runtime resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "available, runtime, scope, expected", + [ + # single-GPU CI lane: runtime unset -> default "110" (cpu + cuda:0). + (SINGLE_GPU, None, "11X", ["cpu", "cuda:0"]), # argless default is device-agnostic + (SINGLE_GPU, None, "110", ["cpu", "cuda:0"]), # pure-math scope + (SINGLE_GPU, None, "100", ["cpu"]), # cpu-only scope + (SINGLE_GPU, None, "00X", []), # non-default-only -> nothing here, skip + # multi-GPU shard: runtime pins exactly one non-default GPU. + (MULTI_GPU, "001", "11X", ["cuda:1"]), # cuda:1 shard + (MULTI_GPU, "0001", "11X", ["cuda:2"]), # cuda:2 shard + (MULTI_GPU, "0001", "00X", ["cuda:2"]), # non-default regression on its shard + (MULTI_GPU, "0001", "110", []), # math skips the shard (cuda:0 != this shard) + (MULTI_GPU, "0001", "100", []), # cpu test skips the shard + # a runtime that lists several GPUs hands back all in-scope ones. + (MULTI_GPU, "111", "11X", ["cpu", "cuda:0", "cuda:1"]), + ], +) +def test_resolves_scope_intersect_runtime(host, available, runtime, scope, expected): + host(available, runtime) + assert resolve_devices(scope) == expected + + +def test_argless_equals_default_scope(host): + # The common case: argless must equal the explicit default mask everywhere. + for available, runtime in [(SINGLE_GPU, None), (MULTI_GPU, "0001"), (MULTI_GPU, "001")]: + host(available, runtime) + assert resolve_devices() == resolve_devices("11X") + + +def test_argless_runs_once_on_one_non_default_gpu(host): + # Run-once property: with a one-device-per-shard runtime, an argless test + # resolves to exactly one non-default GPU. + host(MULTI_GPU, "0001") + result = resolve_devices() + assert result == ["cuda:2"] + assert all(d not in ("cpu", "cuda:0") for d in result) + + +# --------------------------------------------------------------------------- +# skip vs raise: legitimate skips never raise; only a missing runtime device does +# --------------------------------------------------------------------------- + + +def test_unset_runtime_never_raises(host): + # On a dev box / single-GPU lane (runtime unset), an out-of-scope result is a + # silent skip, not an error. + host(SINGLE_GPU, None) + assert resolve_devices("00X") == [] + + +def test_runtime_naming_absent_device_raises(host): + # A run that explicitly asked for cuda:2 on a host that only has cuda:0/cuda:1 + # is misconfigured -> fail loudly instead of a vacuous green. + host(["cpu", "cuda:0", "cuda:1"], "0001") + with pytest.raises(ValueError, match="no device available"): + resolve_devices("11X") + + +def test_in_range_runtime_with_empty_scope_does_not_raise(host): + # The runtime device exists (cuda:2 is present), the scope just doesn't include + # it -> skip, not raise. This is what keeps "110"/"100" tests green on a shard. + host(MULTI_GPU, "0001") + assert resolve_devices("110") == [] # would raise if the guard keyed off scope + + +# --------------------------------------------------------------------------- +# skip= : gate a specific device visibly +# --------------------------------------------------------------------------- + + +def test_skip_wraps_named_device_as_skipped_param(host): + host(SINGLE_GPU, None) + result = resolve_devices("11X", skip={"cuda:0": "known broken"}) + assert result[0] == "cpu" # untouched devices stay plain strings + param = result[1] + assert param.values == ("cuda:0",) + assert param.marks[0].name == "skip" + assert param.marks[0].kwargs["reason"] == "known broken" + + +def test_skip_ignores_out_of_result_devices(host): + # Skipping a device that isn't in the resolved set is a no-op (no stray params). + host(SINGLE_GPU, None) + assert resolve_devices("11X", skip={"cuda:3": "n/a"}) == ["cpu", "cuda:0"] + + +# --------------------------------------------------------------------------- +# _expand mask grammar +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mask, count, expected", + [ + ("110", 2, [True, True]), # exact length, no wildcard + ("100", 3, [True, False, False]), # short mask padded with False + ("11X", 4, [True, True, True, True]), # trailing X fills the rest True + ("00X", 4, [False, False, True, True]), # X spans the non-default GPUs + ("11X", 2, [True, True]), # X with nothing left to fill + ("0001", 3, [False, False, False]), # mask longer than host -> truncated + ], +) +def test_expand(mask, count, expected): + assert devices_mod._expand(mask, count) == expected From 878990a412d6a86b703b20d453bd6219d00c762c Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 04/23] [Newton] Fix Newton/Warp init on non-default CUDA devices Pins torch and Warp to the target device before allocations and scopes the CUDA graph capture to it, so Newton runs correctly on cuda:1+ (issue #5132). --- .../changelog.d/jichuanh-multi-gpu-ci.rst | 5 +++++ .../isaaclab_newton/physics/newton_manager.py | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst new file mode 100644 index 000000000000..c53da690a8d8 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed Newton physics failing to initialize on non-default CUDA devices + (``cuda:1`` and higher). diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index d5991ed395ab..c334a77984b2 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -976,6 +976,15 @@ def start_simulation(cls) -> None: cls._cl_inject_sites_fallback() device = PhysicsManager._device + # Pin torch + Warp to the target device before any Warp/Newton + # allocations. Without this, mujoco_warp's collision pipeline later + # allocates on cuda:1 against a primary CUDA context that was never + # made current, returning null pointers (issue #5132). + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) logger.info(f"Finalizing model on device: {device}") cls._builder.up_axis = Axis.from_string(cls._up_axis) # Forward pending extended attribute requests to builder and clear them @@ -1243,6 +1252,16 @@ def initialize_solver(cls) -> None: if cfg is None: return + # Pin torch + Warp to the target device before solver build and + # collision-pipeline init. Mirrors the guard in :meth:`start_simulation` + # (issue #5132); idempotent if already pinned. + device = PhysicsManager._device + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) + with Timer(name="newton_initialize_solver", msg="Initialize solver took:"): NewtonManager._num_substeps = cfg.num_substeps # type: ignore[union-attr] NewtonManager._collision_decimation = cfg.collision_decimation # type: ignore[union-attr] @@ -1316,7 +1335,7 @@ def _capture_or_defer_graph(cls) -> None: with Timer(name="newton_cuda_graph", msg="CUDA graph took:"): if cls._usdrt_stage is None: simulate = cls._simulate_full if cls._is_all_graphable() else cls._simulate_physics_only - with wp.ScopedCapture() as capture: + with wp.ScopedCapture(device=device) as capture: simulate() NewtonManager._graph = capture.graph logger.info("Newton CUDA graph captured (standard Warp mode)") From b6cf4cf5d675970cb6fdbbd2c5b71f426d9387a9 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 05/23] [CI] Add multi-GPU work-queue sharding and per-file reporting to conftest tools/conftest.py gains a directory-rename work queue (claim/inflight/done) for work-stealing across shards and a per-file report slug to avoid JUnit collisions between same-basename files. A workspace-root conftest skips non-device-parametrized tests on non-default cuda shards, since single-GPU CI already covers them on cuda:0. --- conftest.py | 33 +++++++++ tools/conftest.py | 185 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000000..30d6ae4c40dd --- /dev/null +++ b/conftest.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Workspace-root pytest hooks. + +The only hook today is the multi-GPU shard skip: when a shard pins itself to a +non-default cuda device via :envvar:`ISAACLAB_SIM_DEVICE` (e.g. ``cuda:2``), +skip any test that is not parametrized over a ``device`` argument. Such tests +exercise device-agnostic logic that single-GPU CI already covers on +``cuda:0``, so running them again on every non-default shard duplicates work +without surfacing any new failure mode. + +The hook is a no-op outside the multi-GPU lane (``ISAACLAB_SIM_DEVICE`` unset +or equal to ``cuda:0``), so this file does not change pytest behavior under +single-GPU CI. +""" + +import os + +import pytest + + +def pytest_collection_modifyitems(config, items): + sim_device = os.environ.get("ISAACLAB_SIM_DEVICE", "") + if not sim_device.startswith("cuda:") or sim_device == "cuda:0": + return + skip_marker = pytest.mark.skip(reason=f"non-device-parametrized; covered by single-GPU lane (shard={sim_device})") + for item in items: + callspec = getattr(item, "callspec", None) + if callspec is None or "device" not in callspec.params: + item.add_marker(skip_marker) diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..ad24c85830b4 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -5,6 +5,7 @@ import contextlib import os +import re import select import signal import subprocess @@ -312,6 +313,102 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) +def _slugify_test_path(test_path): + """Encode a test path as a flat queue entry name. + + The queue uses one file per pending test. Slashes are not legal inside a + filename, so we encode the relative path by replacing ``/`` with ``__``. + The decoder is :func:`_unslugify_queue_entry`. + """ + return test_path.replace("/", "__") + + +def _unslugify_queue_entry(entry_name): + """Reverse of :func:`_slugify_test_path`.""" + return entry_name.replace("__", "/") + + +def _claim_queued_file(queue_dir): + """Atomically claim one pending test from the work-queue directory. + + The queue is a directory of files (one per pending test); the shard claims + one by renaming it from ``queue/`` into its private ``inflight/cuda-N/``. + POSIX rename is atomic on the same filesystem, so two shards racing on the + same source file are serialized by the kernel: exactly one rename succeeds, + the other gets ``FileNotFoundError`` and tries the next entry. + + On success, the test's queue entry is now sitting in ``inflight/cuda-N/``; + the caller is expected to move it to ``done/cuda-N/`` after the per-test + pytest invocation exits with a clean result, leaving anything still in + ``inflight/`` at job-end as recoverable evidence of a crashed test. + + Args: + queue_dir: Path to the shared work-queue root. Must contain a + ``queue/`` subdir (pending entries) and an ``inflight//`` + subdir for this shard (claim destination). + + Returns: + The decoded test path for the claimed file, or ``None`` when the + queue is empty. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + pending_dir = os.path.join(queue_dir, "queue") + inflight_dir = os.path.join(queue_dir, "inflight", shard) + os.makedirs(inflight_dir, exist_ok=True) + + # Listdir is intentionally not cached: another shard may have just removed + # an entry we'd otherwise try. We pay one listdir per claim attempt; with + # N≤20 entries this is microseconds. + try: + entries = sorted(os.listdir(pending_dir)) + except FileNotFoundError: + return None + + for entry in entries: + src = os.path.join(pending_dir, entry) + dst = os.path.join(inflight_dir, entry) + try: + os.rename(src, dst) + except FileNotFoundError: + # Lost the race for this entry; another shard claimed it first. + # Continue to the next entry in our (potentially stale) listing. + continue + except OSError: + # Any other rename failure (e.g. permission) is a hard error. + raise + return _unslugify_queue_entry(entry) + + return None + + +def _mark_queued_file_done(queue_dir, test_path): + """Move a successfully-completed claim from ``inflight/cuda-N/`` to ``done/cuda-N/``. + + Called by the test runner after a per-file pytest invocation exits cleanly. + The inflight residual is what the post-run reconciler uses to detect + crashed shards: anything still in ``inflight/`` at job-end is an orphan. + """ + shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") + entry = _slugify_test_path(test_path) + src = os.path.join(queue_dir, "inflight", shard, entry) + dst_dir = os.path.join(queue_dir, "done", shard) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, entry) + # Suppress: already moved (idempotent) or the runner crashed before we + # could mark done — the reconciler catches the second case. + with contextlib.suppress(FileNotFoundError): + os.rename(src, dst) + + +def _queued_files(queue_dir): + """Yield files claimed from the shared work queue until it is empty.""" + while True: + claimed = _claim_queued_file(queue_dir) + if claimed is None: + return + yield claimed + + def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -401,13 +498,21 @@ def _retry_failed_test_in_fresh_process( def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" + """Run each test file separately, ensuring one finishes before starting the next. + + When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue file, files are claimed + from it (work-stealing across sibling shard containers) instead of iterating + ``test_files``; each file still runs once, on this container's pinned GPU. + """ failed_tests = [] test_status = {} xml_reports = [] cold_cache_applied = False - for test_file in test_files: + queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") + file_source = _queued_files(queue_path) if queue_path else test_files + + for test_file in file_source: print(f"\n\n🚀 Running {test_file} independently...\n") file_name = os.path.basename(test_file) env = os.environ.copy() @@ -433,14 +538,25 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + # Prefix the report file with a slug derived from the full test_file + # path so two concurrent shards running same-basename files (e.g. + # ``isaaclab_newton/.../test_articulation.py`` vs + # ``isaaclab_physx/.../test_articulation.py``) don't write to the same + # path inside the shared ``/workspace/isaaclab`` mount and trigger + # false shutdown_hang detections in sibling shards via the + # ``os.path.exists(report_file)`` check at line ~137. + report_slug = str(test_file).replace("/", "__").replace("\\", "__") + report_file = f"tests/test-reports-{report_slug}.xml" + cmd = [ sys.executable, "-m", "pytest", "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit "--no-header", f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", + f"--junitxml={report_file}", "--tb=short", ] @@ -450,8 +566,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): cmd.append(str(test_file)) - report_file = f"tests/test-reports-{str(file_name)}.xml" - # -- Run with retry on startup hang or hard timeout ----------------- returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" wall_time, pre_kill_diag = 0.0, "" @@ -677,6 +791,14 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "wall_time": wall_time, } + # When running under the directory-based work queue (option 2), move the + # claim entry from inflight// to done// so the post-run + # reconciler can distinguish "ran to completion" from "claimed but + # crashed mid-test". A claim that stays in inflight at job-end is a + # silent drop signal. + if queue_path: + _mark_queued_file_done(queue_path, test_file) + print("~~~~~~~~~~~~ Finished running all tests") return failed_tests, test_status, xml_reports @@ -845,6 +967,10 @@ def pytest_sessionstart(session): # Run all tests individually failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) + # In work-queue mode this container ran only the files it claimed; report on those. + if os.environ.get("ISAACLAB_TEST_QUEUE"): + test_files = list(test_status) + print("failed tests:", failed_tests) # Collect reports @@ -861,6 +987,10 @@ def pytest_sessionstart(session): # write content to full report result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") full_report_path = f"tests/{result_file}" + # Ensure the directory exists even when this shard claimed zero files + # from the work queue (per-test JUnit XMLs are what normally create + # ``tests/``; with no tests run there is nothing to create it). + os.makedirs("tests", exist_ok=True) print(f"Using result file: {result_file}") full_report.write(full_report_path) print("~~~~~~~~~~~~ Report written to", full_report_path) @@ -899,14 +1029,17 @@ def pytest_sessionstart(session): summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" + # GPU this run used (the shard's boot device); ``cuda:0`` when unset. + run_device = os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0" + summary_str += "\n\n=======================\n" - summary_str += "Per Test Result Summary\n" + summary_str += "Per File Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_test_result_table.align["Test Path"] = "l" - per_test_result_table.align["Test (s)"] = "r" - per_test_result_table.align["Wall (s)"] = "r" + per_file_result_table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + per_file_result_table.align["Test Path"] = "l" + per_file_result_table.align["Test (s)"] = "r" + per_file_result_table.align["Wall (s)"] = "r" for test_path in test_files: num_tests_passed = ( test_status[test_path]["tests"] @@ -914,9 +1047,10 @@ def pytest_sessionstart(session): - test_status[test_path]["errors"] - test_status[test_path]["skipped"] ) - per_test_result_table.add_row( + per_file_result_table.add_row( [ test_path, + run_device, test_status[test_path]["result"], f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", @@ -924,7 +1058,34 @@ def pytest_sessionstart(session): ] ) - summary_str += per_test_result_table.get_string() + summary_str += per_file_result_table.get_string() + + # Per-test run times, slowest first, from the merged JUnit report. The + # device is read from the test id params (e.g. ``...[size0-cuda:1]``), + # falling back to the run's boot device. + summary_str += "\n\n=================\n" + summary_str += "Per Test Run Time\n" + summary_str += "=================\n" + + per_test_time_table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + per_test_time_table.align["Test"] = "l" + per_test_time_table.align["Time (s)"] = "r" + test_times = [] + for suite in full_report: + for case in suite: + full_name = f"{case.classname}::{case.name}" if case.classname else case.name + device = run_device + bracket = re.search(r"\[(.*)\]", full_name) + if bracket: + dev_match = re.search(r"cuda:\d+|\bcpu\b", bracket.group(1)) + if dev_match: + device = dev_match.group(0) + elapsed = float(case.time) if case.time is not None else 0.0 + test_times.append((full_name, device, elapsed)) + for full_name, device, elapsed in sorted(test_times, key=lambda row: row[2], reverse=True): + per_test_time_table.add_row([full_name, device, f"{elapsed:0.3f}"]) + + summary_str += per_test_time_table.get_string() # Print summary to console and log file print(summary_str) From 6cef107ab0c20659bbd0d04ab86d6b44da839890 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 06/23] [CI] Forward extra env vars through run-tests / run-package-tests actions Adds an extra-env-vars input so the multi-GPU workflow can inject ISAACLAB_TEST_DEVICES / ISAACLAB_SIM_DEVICE into the test container. --- .github/actions/run-package-tests/action.yml | 5 ++++ .github/actions/run-tests/action.yml | 29 +++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 264e01a158b8..4bde3eb22228 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,6 +69,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: 'Extra env vars to forward into the container (one KEY=value per line)' + default: '' + required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -150,6 +154,7 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} + extra-env-vars: ${{ inputs.extra-env-vars }} - name: Check Test Results if: always() diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ceea3a1c4e65..4565f5ddcfd4 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,6 +69,14 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + extra-env-vars: + description: >- + Extra environment variables to forward into the container, one per line in + ``KEY=value`` form. Whitespace-only lines and lines starting with ``#`` are + ignored. Used by the multi-GPU workflow to inject + ``ISAACLAB_TEST_DEVICES`` / ``ISAACLAB_SIM_DEVICE``. + default: '' + required: false runs: using: composite @@ -93,6 +101,7 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" + local extra_env_vars="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -204,6 +213,24 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + # Caller-supplied extra env vars (one KEY=value per line). Skips + # blank lines and full-line comments (line where the first + # non-whitespace char is ``#``). Mid-line ``#`` is preserved so + # values like ``IMAGE_TAG=v1.0#nightly`` survive. + if [ -n "$extra_env_vars" ]; then + while IFS= read -r line; do + # Strip leading whitespace (YAML ``|`` block can leave indent). + line="${line#"${line%%[![:space:]]*}"}" + [ -z "$line" ] && continue + [ "${line:0:1}" = "#" ] && continue + key="${line%%=*}" + value="${line#*=}" + export "$key"="$value" + docker_env_vars="$docker_env_vars -e $key" + echo "Forwarding extra env var: $key" + done <<< "$extra_env_vars" + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -392,7 +419,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.extra-env-vars }}" - name: Kill container on cancellation if: cancelled() From 71658df0701fe367778090b909c35eb529fefbd2 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 07/23] [CI] Add multi-GPU pytest workflow Adds a workflow that runs the unit-test suite across non-default cuda devices in one container with N parallel pytest shards pulling from a shared work queue, plus the inside-container shard runner it mounts. --- .../multi-gpu/multi_gpu_shard_runner.sh | 138 ++++++ .github/workflows/test-multi-gpu-pytest.yaml | 455 ++++++++++++++++++ .../jichuanh-multi-gpu-ci.minor.rst | 25 + 3 files changed, 618 insertions(+) create mode 100755 .github/actions/multi-gpu/multi_gpu_shard_runner.sh create mode 100644 .github/workflows/test-multi-gpu-pytest.yaml create mode 100644 source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh new file mode 100755 index 000000000000..dc5fcbfb6c1a --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest +# shards. Mounted by both the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) and the local probe under ``data/mgpu-1docker-probe/run.sh``, +# so the logic lives in exactly one place and changes are version-controlled +# + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` so it sits +# next to the workflow that consumes it rather than alongside ``tools/``. +# +# Container expectations (set by the host launcher): +# --gpus all +# --user $host_uid:$host_gid +# -v :/workspace/isaaclab:rw +# -v :/mgpu:rw (subdirs queue/, inflight/, done/) +# -v :/shard-logs:rw (per-shard tee target) +# -e HOME=/tmp/mgpu-base-home (writable tmpfs for pip --user) +# -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) +# -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) +# -e TEST_INCLUDE_FILES="" +# -e ISAACLAB_PIN_KIT_GPU=1 (Kit overrides for the multi-GPU race) +# -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) +# +# Behavior: +# 1. Materializes HOME + PYTHONUSERBASE dirs (tmpfs, world-writable) +# 2. Installs pytest deps (junitparser et al.) into the shared PYTHONUSERBASE +# 3. Derives shard count from nvidia-smi -L (authoritative; torch.cuda.device_count +# under-counts MIG-on-same-parent unless CUDA_VISIBLE_DEVICES enumerates each) +# 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch +# can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) +# 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + +# ISAACLAB_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# /shard-logs/cuda-N.log for the host's grouped re-print after the run +# 6. Waits on every shard before aggregating exit codes — a fast failure doesn't +# tear down still-running siblings + +set +e +cd /workspace/isaaclab +unset HUB__ARGS__DETECT_ONLY DISPLAY + +# Container-level HOME + PYTHONUSERBASE for pip --user installs. The image +# runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so +# HOME defaults to /root which the user cannot write. /tmp/* is on tmpfs +# (1777, world-writable). +# +# PYTHONUSERBASE is the key for the 1-docker shape: pip --user writes to +# ${PYTHONUSERBASE}/lib/python3.12/site-packages, and every Python invocation +# that sees the same env var imports from there. Per-shard subshells below +# override HOME (so .cache / .nvidia-omniverse are isolated) but inherit +# PYTHONUSERBASE so junitparser et al. resolve everywhere. +mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase + +# Pytest deps (same as run-tests action). junitparser is imported at +# tools/conftest.py load time, so it must be present first. +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky py-spy "coverage>=7.6.1" + +# Shard count from nvidia-smi -L (truth; torch under-counts MIG). +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") +if [ "$MIG_COUNT" -gt 0 ]; then + DEV_COUNT=$MIG_COUNT + echo "::notice::container: MIG mode, $MIG_COUNT slices" +else + DEV_COUNT=$GPU_COUNT + echo "::notice::container: discrete mode, $GPU_COUNT GPUs" +fi +if [ "$DEV_COUNT" -lt 2 ]; then + echo "::error::Need at least 2 visible devices; found $DEV_COUNT" + exit 1 +fi + +# Cross-check with torch and cap shard count to what torch can actually +# address. Guards against a CUDA_VISIBLE_DEVICES misconfig silently fanning +# out shards that crash on device access. +TORCH_COUNT=$(/isaac-sim/python.sh -c "import torch; print(torch.cuda.device_count())") +echo "container: torch sees $TORCH_COUNT cuda devices (cross-check vs $DEV_COUNT)" +if [ "$TORCH_COUNT" -lt "$DEV_COUNT" ]; then + echo "::warning::torch sees fewer devices than nvidia-smi — capping shards to $TORCH_COUNT" + DEV_COUNT=$TORCH_COUNT +fi + +# Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME +# (per-shard isolation for .cache, .local/share, etc.) and per-shard +# ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +declare -A pids +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do + zeros="" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done + runtime_devices="${zeros}1" + + shard_home="/tmp/isaaclab-ci-home-${cuda}" + mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ + "${shard_home}/.nvidia-omniverse/config" \ + "${shard_home}/.nvidia-omniverse/logs" + + shard_log="/shard-logs/cuda-${cuda}.log" + + ( + export HOME="$shard_home" + export XDG_CACHE_HOME="${HOME}/.cache" + export XDG_DATA_HOME="${HOME}/.local/share" + export ISAACLAB_TEST_DEVICES="$runtime_devices" + export ISAACLAB_SIM_DEVICE="cuda:${cuda}" + + # Full pytest output captures to $shard_log; live stdout is filtered + # down to high-signal lines (test boundaries, failures, summary stats, + # tracebacks). Kit init chatter, plugin registration, omni.usd Transfer + # logs, etc. only end up in the shard logfile, NOT on the live workflow + # log. The workflow's grouped re-print at end of run still shows the + # full $shard_log under a collapsible ``::group::shard cuda:N log``. + ./isaaclab.sh -p -m pytest \ + --ignore=tools/conftest.py \ + --ignore=source/isaaclab/test/install_ci \ + tools -v 2>&1 \ + | tee "$shard_log" \ + | stdbuf -oL grep -aE \ + '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ + | stdbuf -oL sed "s/^/[cuda:${cuda}] /" + + exit "${PIPESTATUS[0]}" + ) & + pids[$cuda]=$! + echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" +done + +# Wait for every shard before aggregating exits — a fast failure must not +# tear down still-running siblings. +declare -A results +for cuda in "${!pids[@]}"; do + wait "${pids[$cuda]}" + results[$cuda]=$? +done + +fail=0 +for cuda in "${!results[@]}"; do + rc="${results[$cuda]}" + echo "shard cuda:${cuda} exited $rc" + [ "$rc" -eq 0 ] || fail=1 +done +exit $fail diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml new file mode 100644 index 000000000000..55db0e9996c1 --- /dev/null +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -0,0 +1,455 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Multi-GPU unit-test workflow +# +# Runs the non-default-GPU subset of unit tests across the multi-GPU runner +# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the +# single-GPU CI; only diffs are the runner label and the env vars that pin +# Kit and the test parametrize to the shard's non-default GPU. +# +# Adding a new test to multi-GPU coverage: give its device parametrize a +# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()`` +# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow +# auto-discovers it. + +name: Multi-GPU pytest + +on: + pull_request: + paths: + - "source/isaaclab/isaaclab/test/utils/**" + - "source/isaaclab/isaaclab/app/app_launcher.py" + - "source/**/test/**/test_*.py" + - ".github/workflows/test-multi-gpu-pytest.yaml" + - ".github/actions/run-package-tests/**" + - ".github/actions/run-tests/**" + - ".github/actions/ecr-build-push-pull/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + + build: + name: Build / cache image + needs: [config] + # Build only needs Docker + ECR access, not multi-GPU. Aligns with the + # build jobs in build.yaml that also publish the per-commit image. + runs-on: [self-hosted, gpu] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + # Pre-populates the ECR exact-commit tag from deps-cache (registry-side + # alias). Without this prior step, run-package-tests' internal + # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves + # no local image, causing `docker run` to fail with `pull access + # denied`. Mirrors the build → test split in build.yaml. + - uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + test-multi-gpu-pytest: + name: Multi-GPU unit tests + needs: [config, build] + if: needs.build.result == 'success' + # Matches the runner label used by test-multi-gpu.yaml + + # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. + runs-on: [self-hosted, linux, x64, gpu, multi-gpu] + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Discover opt-in test files + # Auto-discovery: any test_*.py with a non-default-capable scope — an + # argless ``test_devices()`` (defaults to "11X") or any + # ``test_devices("..X")`` mask, whose trailing ``X`` spans the + # non-default GPUs — is in scope. Adding a test to multi-GPU CI needs + # no workflow edit; opting a file out is just narrowing its scope (drop + # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. + # + # Within a discovered file, tests that are NOT parametrized over the + # ``device`` argument are filtered out at collection time by the + # workspace-root ``conftest.py``: single-GPU CI already covers them on + # ``cuda:0`` and re-running on every non-default shard adds wall-time + # without surfacing any new failure mode. + id: discover + run: | + # File-level opt-out: a test file can exclude itself from multi-GPU CI + # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line. + # Used for files with known Kit/Isaac-Sim concurrency issues; the file + # still runs in single-GPU CI. Excluded files are reported as a notice + # below for visibility. No workflow edit needed to add/remove a file. + mapfile -t candidates < <(grep -rlE 'test_devices\(\)|test_devices\("[^"]*X"\)' source/ --include='test_*.py' | sort -u) + discovered=() + skipped=() + for f in "${candidates[@]}"; do + if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then + skipped+=("$f") + else + discovered+=("$f") + fi + done + for f in "${skipped[@]}"; do + reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//') + echo "::notice::multi-GPU skipped: $f — $reason" + done + if [ ${#discovered[@]} -eq 0 ]; then + echo "::error::No opt-in tests discovered (grep for test_devices with an X scope)" + exit 1 + fi + basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) + echo "include=$basenames" >> "$GITHUB_OUTPUT" + # Full relative paths too, to seed the shared work queue (the run step + # needs runnable paths, not just basenames). + echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT" + echo "::notice::Discovered ${#discovered[@]} opt-in test files" + printf ' %s\n' "${discovered[@]}" + + - name: Pull image from ECR + # Pulls the per-commit image the build job pushed. ecr-build-push-pull + # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which + # the build job's deps-cache-hit registry-tag created for this SHA) + # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our + # parallel ``docker run`` block below. + uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run shards in parallel on local GPUs (1-docker N-shard) + # ONE container hosts all N pytest shards as parallel subshells. Each + # shard pins ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES to its own + # non-default cuda:N and pulls files from the shared work queue. This + # collapses the previous N-container layout into one container, removing + # cross-container races on the shared workspace mount (``_isaac_sim`` + # symlink, ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker + # init per shard, at the cost of sharing the container's /isaac-sim/ + # writable subtrees. Per-shard HOME (under /tmp) keeps user-level + # caches isolated. + # + # Scaling: shard count is derived inside the container from + # ``nvidia-smi -L`` (truth — torch.cuda.device_count() under-counts MIG + # slices on the same parent GPU), so this works unchanged on any GPU + # count or topology. ``CUDA_VISIBLE_DEVICES`` is set to the + # comma-separated MIG-UUID list only on MIG-mode hosts; discrete-GPU + # runners rely on ``--gpus all``. + env: + IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} + INCLUDE_FILES: ${{ steps.discover.outputs.include }} + PATHS: ${{ steps.discover.outputs.paths }} + run: | + set +e + host_uid="$(id -u)" + host_gid="$(id -g)" + host_user="$(id -un)" + # ``/isaac-sim`` is the Kit install root inside the container; pytest + # invocations under ``./isaaclab.sh -p`` resolve the bundled Python as + # ``./_isaac_sim/python.sh``. Mounting the workspace at + # ``/workspace/isaaclab`` brings the host's CWD into the container, + # so we plant a host-side symlink whose target resolves only inside + # the container. Single creator (this step) — no concurrent creators. + rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + + runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" + mkdir -p "$runtime_dir" + + # Directory-based work queue (atomic os.rename pattern; conftest claims + # entries via os.rename(queue/X, inflight//X) — POSIX rename is + # atomic on the same filesystem, so two shards racing on the same + # entry are kernel-serialized: one wins, the other gets ENOENT and + # moves on). Each pending test path is materialized as an empty file + # in queue/ with '/' slugged to '__' so it's a flat filename. + # + # Subdirs: + # queue/ — pending claims + # inflight// — claimed by this shard, test in flight + # done// — test exited cleanly + # + # At job-end the reconciler step asserts queue/ + inflight/ are empty + # — if either is non-empty the job FAILS (silent-drop signal: never + # claimed OR claimed but crashed mid-test). See the reconciler step + # near the end of this job for the exact assertion. + queue_root="$runtime_dir/queue" + mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" + IFS=',' read -ra _paths <<< "$PATHS" + for path in "${_paths[@]}"; do + [ -n "$path" ] || continue + slug="${path//\//__}" + : > "$queue_root/queue/$slug" + done + echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + + # Per-shard log dir mounted from the container so we can grouped-print + # the per-shard stream after the run completes. + logs_dir="$runtime_dir/logs" + mkdir -p "$logs_dir" + + # MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. + # nvidia-smi -L is authoritative — see memory rule + # ``reference_gpu_enumeration``: torch.cuda.device_count() under-counts + # MIG slices on the same parent GPU. + cvd_args=() + if nvidia-smi -L | grep -q "^ MIG "; then + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" + else + echo "::notice::discrete GPU mode — relying on --gpus all" + fi + + # --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. + docker run --rm --gpus all --network=host \ + --cap-add=SYS_PTRACE \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${{ github.run_id }}-${{ github.run_attempt }}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh + docker_rc=$? + + # Grouped re-print of per-shard logs (preserved across the host). + for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" + done + + # Reconciler — silent-drop guard. Anything left in queue/ was never + # claimed by any shard (work-queue starvation). Anything still in + # inflight// was claimed but never moved to done/ (shard + # crashed mid-test). Either is a silent-drop signal that the docker + # exit alone can't surface. Fail the job and name the orphans. + echo "::group::work-queue reconciler" + unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) + if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' + fi + orphans=0 + for shard_dir in "$queue_root/inflight"/*/; do + [ -d "$shard_dir" ] || continue + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' + orphans=$((orphans + count)) + fi + done + done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) + echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" + echo "::endgroup::" + + # Treat reconciler signals as fatal even if all shards exited 0. + if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 + fi + + # Export the runtime directory so the downstream "Aggregated test + # summary" step can locate the done// work distribution + # and the per-shard logs. + echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" + + exit "$docker_rc" + + - name: Aggregated test summary + # Runs whether or not the test job exit was clean — surfaces what + # happened even on failure. Per-shard stats + per-file pass / total / + # walltime, plus a single combined table. Also writes the same content + # to $GITHUB_STEP_SUMMARY so it renders at the top of the run page. + if: always() + env: + RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} + run: | + python3 - <<'PY' + import os, glob, xml.etree.ElementTree as ET, sys, pathlib + + runtime_dir = os.environ.get("RUNTIME_DIR", "") + done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + + # Map slug -> shard from done//; slug uses "__" for "/". + slug_to_shard = {} + if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-"):] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + # Per-file rows from JUnit XMLs. + rows = [] + for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, + failures + errors, total_time)) + + # Per-shard aggregate. + shards = {} + for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, + "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + # ---------- Output: plain text to stdout ---------- + def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + out = [] + out.append("PER-FILE RESULTS") + out.append("=" * 110) + hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] + widths = [60, 7, 8, 12, 5, 5, 8] + out.append(fmt_row(hdr, widths)) + out.append("-" * 110) + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) + out.append("") + out.append("PER-SHARD AGGREGATE") + out.append("=" * 80) + shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] + shard_widths = [10, 7, 12, 9, 8, 14] + out.append(fmt_row(shard_hdr, shard_widths)) + out.append("-" * 80) + grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + for sh in sorted(shards): + s = shards[sh] + for k in grand: grand[k] += s[k] + out.append(fmt_row( + [sh, str(s["files"]), f"{s['passed']}/{s['tests']}", str(s["skipped"]), + str(s["failed"]), f"{s['time']:.1f}"], shard_widths)) + out.append("-" * 80) + out.append(fmt_row( + ["TOTAL", str(grand["files"]), f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), str(grand["failed"]), f"{grand['time']:.1f}"], + shard_widths)) + print("\n".join(out)) + + # ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + md = ["## Multi-GPU CI: Aggregated Test Summary", "", + "### Per-shard totals", "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|"] + for sh in sorted(shards): + s = shards[sh] + md.append(f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |") + md.append(f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | **{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |") + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = {"PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR"}.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") + PY diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst new file mode 100644 index 000000000000..00b4366e153e --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -0,0 +1,25 @@ +Added +^^^^^ + +* Added :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over + a device set resolved as ``scope ∩ runtime``: ``scope`` is the call-site mask + of devices a test is valid on (default ``"11X"`` ⇒ cpu + cuda:0 + the + non-default GPUs), and the runtime is the ``ISAACLAB_TEST_DEVICES`` env var of + devices a run may use (default ``"110"`` ⇒ cpu + cuda:0). A trailing ``X`` + includes the remaining devices. Single-GPU CI is unchanged; multi-GPU CI sets + the runtime to one non-default GPU per shard. + +* Added ``ISAACLAB_SIM_DEVICE`` env var honored by + :class:`isaaclab.app.AppLauncher` as the implicit-default device when + the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow + boot Kit on a non-default GPU without editing every test's + :class:`~isaaclab.app.AppLauncher` call site. + +* Added ``py-spy`` + ``gdb`` stack capture in ``tools/conftest.py`` on + ``shutdown_hang`` / ``startup_hang`` / ``timeout`` detection. Walks the test + subprocess's process group (cap 8 pids), captures both Python and C++ frames + before ``SIGKILL`` erases them, attaches the output to the JUnit error + report. Makes Kit binary hangs observable in CI logs; safe no-op when + ``py-spy``/``gdb`` are missing. Workflow side adds ``--cap-add=SYS_PTRACE`` + on the per-shard ``docker run`` (required to attach) and adds ``py-spy`` to + the in-container ``pip install`` list. From cef4569956b530ae16d8f0fcc21179b6efc12536 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:56:39 +0000 Subject: [PATCH 08/23] [Tests] Parametrize unit tests over multi-GPU device scope Switches device-parametrized unit tests to test_devices() so they also run on the non-default GPUs in the multi-GPU lane. Mechanical scope change only; no test logic changes. --- .../test/sim/test_newton_model_utils.py | 19 ++-- .../test/sim/test_simulation_context.py | 3 +- .../test/sim/test_views_xform_prim.py | 13 +-- .../isaaclab/test/utils/test_episode_data.py | 11 ++- source/isaaclab/test/utils/test_math.py | 95 +++++++++--------- source/isaaclab/test/utils/test_modifiers.py | 5 +- source/isaaclab/test/utils/test_noise.py | 7 +- .../test/utils/test_wrench_composer.py | 75 +++++++------- .../test/assets/test_articulation.py | 99 ++++++++++--------- .../test/assets/test_rigid_object.py | 41 ++++---- .../assets/test_rigid_object_collection.py | 29 +++--- .../test/sensors/test_contact_sensor.py | 16 +-- .../test/sim/test_views_xform_prim_newton.py | 10 +- .../changelog.d/jichuanh-multi-gpu-ci.skip | 0 .../test/assets/test_articulation.py | 70 ++++++------- .../test/assets/test_rigid_object.py | 40 ++++---- .../assets/test_rigid_object_collection.py | 30 +++--- .../changelog.d/jichuanh-multi-gpu-ci.skip | 0 .../test/assets/test_articulation.py | 91 ++++++++--------- .../test/assets/test_rigid_object.py | 41 ++++---- .../assets/test_rigid_object_collection.py | 29 +++--- .../test/sim/test_views_xform_prim_fabric.py | 26 ++--- 22 files changed, 381 insertions(+), 369 deletions(-) create mode 100644 source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip create mode 100644 source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip diff --git a/source/isaaclab/test/sim/test_newton_model_utils.py b/source/isaaclab/test/sim/test_newton_model_utils.py index cc157074fd42..5cfedaab3354 100644 --- a/source/isaaclab/test/sim/test_newton_model_utils.py +++ b/source/isaaclab/test/sim/test_newton_model_utils.py @@ -27,6 +27,7 @@ _scatter_shape_color_rows_kernel, replace_newton_shape_colors, ) +from isaaclab.test.utils import test_devices _WARNING_MESSAGE = "Newton shape color replacement is enabled; this workaround will be deprecated in a future release." @@ -177,7 +178,7 @@ def _run_scatter_shape_color_rows_kernel( pytest.param((-0.25, 1.75, 0.5), id="oob_clamps_pow"), ], ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_scatter_shape_color_rows_kernel(device: str, linear_rgb: tuple[float, float, float]): """Packed RGB per case; the two parametrized cases jointly cover every ``_linear_channel_to_srgb_warp`` branch.""" after = _run_scatter_shape_color_rows_kernel([linear_rgb], device=device) @@ -284,7 +285,7 @@ def test_replace_newton_shape_colors_warning(): replace_newton_shape_colors(model) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPatch, device: str): """Setting ``ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS`` to ``0`` disables the workaround.""" monkeypatch.setenv("ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS", "0") @@ -315,7 +316,7 @@ def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPa assert not any(issubclass(w.category, FutureWarning) for w in recorded) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_invalid_prim(device: str): """Invalid prim path leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -330,7 +331,7 @@ def test_replace_newton_shape_colors_invalid_prim(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_guide_purpose(device: str): """Guide-purpose mesh leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -349,7 +350,7 @@ def test_replace_newton_shape_colors_guide_purpose(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_no_material_binding(device: str): """No material: ``displayColor`` or unbound gray as linear RGB, then sRGB OETF into ``shape_color``.""" stage = Usd.Stage.CreateInMemory() @@ -381,7 +382,7 @@ def test_replace_newton_shape_colors_no_material_binding(device: str): @pytest.mark.parametrize(("diffuse_color_constant", "diffuse_tint"), _OMNIPBR_ALBEDO_INPUT_CASES) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_omnipbr_binding( device: str, diffuse_color_constant: tuple[float, float, float] | None, @@ -405,7 +406,7 @@ def test_replace_newton_shape_colors_omnipbr_binding( ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_neutral_material(device: str): """Bound ``UsdPreviewSurface`` material leaves ``shape_color`` unchanged.""" stage, mesh_path = _make_preview_surface_bound_mesh_stage() @@ -418,7 +419,7 @@ def test_replace_newton_shape_colors_neutral_material(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_respects_binding_strength(device: str): """Parent stronger-than-descendants binding overrides direct child binding.""" # Scene graph (``ComputeBoundMaterial`` on the mesh yields ParentMat / green, not ChildMat / red): @@ -470,7 +471,7 @@ def test_replace_newton_shape_colors_respects_binding_strength(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_instanced(device: str): """Instance-proxy labels deduplicate via canonical prototype paths in the per-key cache.""" stage = Usd.Stage.CreateInMemory() diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 6ea578a85e30..c96883b839a0 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -6,6 +6,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -43,7 +44,7 @@ def test_setup_teardown(): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_init(device): """Test the simulation context initialization.""" from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 64cd86a7466f..827ed20d3261 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -11,6 +11,7 @@ """ from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices simulation_app = AppLauncher(headless=True).app @@ -101,7 +102,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_toggle(device): """Test toggling visibility multiple times.""" if device == "cuda" and not torch.cuda.is_available(): @@ -129,7 +130,7 @@ def test_visibility_toggle(device): assert vis[0] and not vis[1] and vis[2] -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_parent_inheritance(device): """Making a parent invisible hides all children.""" if device == "cuda" and not torch.cuda.is_available(): @@ -155,7 +156,7 @@ def test_visibility_parent_inheritance(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_prim_ordering_follows_creation_order(device): """Prims are returned in USD creation order (DFS), not alphabetical.""" if device == "cuda" and not torch.cuda.is_available(): @@ -181,7 +182,7 @@ def test_prim_ordering_follows_creation_order(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_standardize_transform_op(device): """FrameView standardizes a prim with xformOp:transform to translate/orient/scale.""" if device == "cuda" and not torch.cuda.is_available(): @@ -209,7 +210,7 @@ def test_standardize_transform_op(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_nested_hierarchy_world_poses(device): """World pose of nested child == sum of parent + child translations.""" if device == "cuda" and not torch.cuda.is_available(): @@ -271,7 +272,7 @@ def test_compare_get_world_poses_with_isaacsim(): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_with_franka_robots(device): """Verify FrameView works with real Franka robot USD assets.""" if device == "cuda" and not torch.cuda.is_available(): diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index a2d570d9d6ef..6062c25d0400 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,10 +5,11 @@ import pytest import torch +from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -18,7 +19,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -55,7 +56,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -102,7 +103,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -114,7 +115,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index c2e5b3550816..4fad6881e56e 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,6 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils +from isaaclab.test.utils import test_devices DECIMAL_PRECISION = 5 """Precision of the test. @@ -22,7 +23,7 @@ """ -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_scale_unscale_transform(device, size): """Test scale_transform and unscale_transform.""" @@ -59,7 +60,7 @@ def test_scale_unscale_transform(device, size): torch.testing.assert_close(output_unscale_offset, inputs) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_saturate(device, size): "Test saturate of a tensor of differed shapes and device." @@ -81,7 +82,7 @@ def test_saturate(device, size): assert torch.all(torch.less_equal(output_per_batch, upper_per_batch)).item() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_normalize(device, size): """Test normalize of a tensor along its last dimension and check the norm of that dimension is close to 1.0.""" @@ -93,7 +94,7 @@ def test_normalize(device, size): torch.testing.assert_close(norm, torch.ones(size[0:-1], device=device)) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_copysign(device): """Test copysign by copying a sign from both a negative and positive value and verify that the new sign is the same. @@ -123,7 +124,7 @@ def test_copysign(device): torch.testing.assert_close(expected_value_neg_dim1_neg, value_neg_dim1_neg) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_is_identity_pose(device): """Test is_identity_pose method.""" # Single row identity pose (xyzw format) @@ -147,7 +148,7 @@ def test_is_identity_pose(device): assert math_utils.is_identity_pose(identity_pos, identity_rot) is False -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat(device): """Test axis_angle_from_quat method.""" # Quaternions of the form (2,4) and (2,2,4) in xyzw format @@ -171,7 +172,7 @@ def test_axis_angle_from_quat(device): torch.testing.assert_close(math_utils.axis_angle_from_quat(quat), angle) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat_approximation(device): """Test the Taylor approximation from axis_angle_from_quat method. @@ -197,7 +198,7 @@ def test_axis_angle_from_quat_approximation(device): torch.testing.assert_close(axis_angle_computed, axis_angle_expected) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_magnitude(device): """Test quat_error_magnitude method.""" # No rotation (xyzw format) @@ -236,7 +237,7 @@ def test_quat_error_magnitude(device): torch.testing.assert_close(q12_diff, expected_diff) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_unique(device): """Test quat_unique method.""" # Define test cases @@ -254,7 +255,7 @@ def test_quat_unique(device): torch.testing.assert_close(pos_real_quats[~non_pos_indices], quats[~non_pos_indices]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_mul_with_quat_unique(device): """Test quat_mul method with different quaternions. @@ -287,7 +288,7 @@ def test_quat_mul_with_quat_unique(device): torch.testing.assert_close(quat_result_3, quat_result_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_mag_with_quat_unique(device): """Test quat_error_magnitude method with positive real quaternions.""" @@ -310,7 +311,7 @@ def test_quat_error_mag_with_quat_unique(device): torch.testing.assert_close(error_4, error_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_convention_converter(device): """Test convert_camera_frame_orientation_convention to and from ros, opengl, and world conventions.""" # Quaternions in xyzw format (converted from original wxyz test values) @@ -348,7 +349,7 @@ def test_convention_converter(device): ) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((10, 4), (5, 3, 4))) def test_convert_quat(device, size): """Test convert_quat from "xyzw" to "wxyz" and back to "xyzw" and verify the correct rolling of the tensor. @@ -383,7 +384,7 @@ def test_convert_quat(device, size): math_utils.convert_quat(quat, to="xwyz") -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_conjugate(device): """Test quat_conjugate by checking the sign of the imaginary part changes but the magnitudes stay the same.""" @@ -397,7 +398,7 @@ def test_quat_conjugate(device): torch.testing.assert_close(expected_real, value[..., 3]) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", (1, 10)) @pytest.mark.parametrize( "euler_angles", @@ -428,7 +429,7 @@ def test_quat_from_euler_xyz(device, num_envs, euler_angles): torch.testing.assert_close(expected_quat, quat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_wrap_to_pi(device): """Test wrap_to_pi method.""" # No wrapping needed @@ -464,7 +465,7 @@ def test_wrap_to_pi(device): torch.testing.assert_close(wrapped_angle, expected_angle) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("shape", ((3,), (1024, 3))) def test_skew_symmetric_matrix(device, shape): """Test skew_symmetric_matrix.""" @@ -494,7 +495,7 @@ def test_skew_symmetric_matrix(device, shape): torch.testing.assert_close(vec_rand_resized[:, 0], mat_value[:, 2, 1]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_orthogonalize_perspective_depth(device): """Test for converting perspective depth to orthogonal depth.""" # Create a sample perspective depth image (N, H, W) @@ -515,7 +516,7 @@ def test_orthogonalize_perspective_depth(device): torch.testing.assert_close(orthogonal_depth, expected_orthogonal_depth) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_combine_frame_transform(device): """Test combine_frame_transforms function.""" # create random poses @@ -540,7 +541,7 @@ def test_combine_frame_transform(device): torch.testing.assert_close(pose01, torch.cat((pos01, quat01), dim=-1)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4]) def test_interpolate_poses(device, seed): """Test interpolate_poses function. @@ -608,7 +609,7 @@ def test_pose_inv(): np.testing.assert_array_almost_equal(result, expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_to_and_from_angle_axis(device): """Test that axis_angle_from_quat against scipy and that quat_from_angle_axis are the inverse of each other.""" n = 1024 @@ -627,7 +628,7 @@ def test_quat_to_and_from_angle_axis(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus(device): """Test quat_box_minus method. @@ -645,7 +646,7 @@ def test_quat_box_minus(device): torch.testing.assert_close(expected_diff, axis_diff, atol=1e-06, rtol=1e-06) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus_and_quat_box_plus(device): """Test consistency of quat_box_plus and quat_box_minus. @@ -687,7 +688,7 @@ def test_quat_box_minus_and_quat_box_plus(device): torch.testing.assert_close(delta_result, delta_angle, atol=1e-04, rtol=1e-04) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t12_inputs", ["True", "False"]) @pytest.mark.parametrize("q12_inputs", ["True", "False"]) def test_combine_frame_transforms(device, t12_inputs, q12_inputs): @@ -726,7 +727,7 @@ def test_combine_frame_transforms(device, t12_inputs, q12_inputs): torch.testing.assert_close(math_utils.quat_unique(expected_quat), math_utils.quat_unique(quat_value)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t02_inputs", ["True", "False"]) @pytest.mark.parametrize("q02_inputs", ["True", "False"]) def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): @@ -766,7 +767,7 @@ def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): torch.testing.assert_close(math_utils.quat_unique(q02_expected), math_utils.quat_unique(q02_compare)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("rot_error_type", ("quat", "axis_angle")) def test_compute_pose_error(device, rot_error_type): """Test compute_pose_error for different rot_error_type.""" @@ -794,7 +795,7 @@ def test_compute_pose_error(device, rot_error_type): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_twist_transform(device): """Test rigid_body_twist_transform method. @@ -822,7 +823,7 @@ def test_rigid_body_twist_transform(device): torch.testing.assert_close(w_AA_, w_AA) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_yaw_quat(device): """ Test for yaw_quat methods. @@ -844,7 +845,7 @@ def test_yaw_quat(device): torch.testing.assert_close(result, expected_output) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_slerp(device): """Test quat_slerp function. @@ -874,7 +875,7 @@ def test_quat_slerp(device): np.testing.assert_array_almost_equal(result.cpu(), expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_matrix_from_quat(device): """test matrix_from_quat against scipy.""" # prepare random quaternions and vectors @@ -893,7 +894,7 @@ def test_matrix_from_quat(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "euler_angles", [ @@ -926,7 +927,7 @@ def test_matrix_from_euler(device, euler_angles, convention): torch.testing.assert_close(expected_mag, mat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply(device): """Test for quat_apply against scipy.""" # prepare random quaternions and vectors @@ -943,7 +944,7 @@ def test_quat_apply(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply_inverse(device): """Test for quat_apply against scipy.""" @@ -963,7 +964,7 @@ def test_quat_apply_inverse(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_inv(device): """Test for quat_inv method. @@ -1040,7 +1041,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1064,7 +1065,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) @@ -1315,7 +1316,7 @@ def test_euler_xyz_from_quat(): torch.testing.assert_close(output, wrapped) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): """Camera above target on +Z axis with Z-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, 5.0]], device=device) @@ -1327,7 +1328,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): """Camera at +Y looking at origin with Y-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 5.0, 0.0]], device=device) @@ -1339,7 +1340,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): """Camera below target looking up (-Z alignment with Z-up) should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, -5.0]], device=device) @@ -1351,7 +1352,7 @@ def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): """When eyes == targets the forward direction is undefined; all entries of the row are NaN.""" eyes = torch.tensor([[1.0, 2.0, 3.0]], device=device) @@ -1360,7 +1361,7 @@ def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_batched_partial_failure(device): """Mixed batch with one degenerate row should produce NaN in that row and a valid rotation in the other.""" eyes = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 5.0]], device=device) @@ -1371,7 +1372,7 @@ def test_create_rotation_matrix_from_view_batched_partial_failure(device): torch.testing.assert_close(torch.linalg.det(R[1]), torch.tensor(1.0, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_unit_norm_on_valid_input(device): """quat_from_matrix should produce unit quaternions for any valid rotation matrix.""" n = 100 @@ -1382,7 +1383,7 @@ def test_quat_from_matrix_unit_norm_on_valid_input(device): torch.testing.assert_close(norms, torch.ones(n, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_singular_matrix_returns_nan(device): """quat_from_matrix on a singular (non-rotation) matrix should signal NaN, not garbage.""" singular = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device) @@ -1390,7 +1391,7 @@ def test_quat_from_matrix_singular_matrix_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_standard(device): """Sanity: off-axis eye produces an orthonormal frame whose z-axis points from target back to eye.""" eyes = torch.tensor([[3.0, 0.0, 4.0]], device=device) @@ -1404,7 +1405,7 @@ def test_create_rotation_matrix_from_view_standard(device): torch.testing.assert_close(R[:, :, 2], expected_z, atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): """Non-finite input (NaN or Inf in eyes/targets) should produce NaN rows.""" eyes = torch.tensor([[float("nan"), 0.0, 0.0]], device=device) @@ -1413,7 +1414,7 @@ def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_reflection_returns_nan(device): """A reflection matrix (det = -1) is not a proper rotation; the safeguard should signal NaN.""" reflection = torch.diag(torch.tensor([1.0, 1.0, -1.0], device=device)).unsqueeze(0) @@ -1421,7 +1422,7 @@ def test_quat_from_matrix_reflection_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_non_orthonormal_returns_nan(device): """A non-orthonormal matrix (1% scale error on one axis) is not a valid rotation; expect NaN.""" R = torch.diag(torch.tensor([1.01, 1.0, 1.0], device=device)).unsqueeze(0) diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index 6e0e39820fdf..b14622e90550 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -9,6 +9,7 @@ import torch import isaaclab.utils.modifiers as modifiers +from isaaclab.test.utils import test_devices from isaaclab.utils.configclass import configclass @@ -142,7 +143,7 @@ def test_torch_relu_modifier(): assert torch.allclose(output, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_digital_filter(device): """Test digital filter modifier.""" # create test data @@ -178,7 +179,7 @@ def test_digital_filter(device): torch.testing.assert_close(processed_data, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_integral(device): """Test integral modifier.""" # create test data diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index 45f9d2d61384..c1035453b5a9 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,9 +7,10 @@ import torch import isaaclab.utils.noise as noise +from isaaclab.test.utils import test_devices -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -42,7 +43,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -77,7 +78,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab/test/utils/test_wrench_composer.py b/source/isaaclab/test/utils/test_wrench_composer.py index b711aaab44a6..2b13cf23a9f2 100644 --- a/source/isaaclab/test/utils/test_wrench_composer.py +++ b/source/isaaclab/test/utils/test_wrench_composer.py @@ -9,6 +9,7 @@ import warp as wp from isaaclab.test.mock_interfaces.assets import MockRigidObjectCollection +from isaaclab.test.utils import test_devices from isaaclab.utils.wrench_composer import WrenchComposer @@ -97,7 +98,7 @@ def random_unit_quaternion_np(rng: np.random.Generator, shape: tuple) -> np.ndar return q -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): @@ -136,7 +137,7 @@ def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_force_np, hand_calculated_composed_force_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int): @@ -175,7 +176,7 @@ def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int) assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): @@ -234,7 +235,7 @@ def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -280,7 +281,7 @@ def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -343,7 +344,7 @@ def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodi assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): @@ -392,7 +393,7 @@ def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -434,7 +435,7 @@ def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -476,7 +477,7 @@ def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: in ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies: int): @@ -540,7 +541,7 @@ def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_local_vs_global_identity_quaternion(device: str): """Test that local and global give same result with identity quaternion and zero position.""" rng = np.random.default_rng(seed=13) @@ -582,7 +583,7 @@ def test_local_vs_global_identity_quaternion(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_90_degree_rotation_global_force(device: str): """Test global force with a known 90-degree rotation for easy verification.""" num_envs, num_bodies = 1, 1 @@ -615,7 +616,7 @@ def test_90_degree_rotation_global_force(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composition_mixed_local_and_global(device: str): """Test that local and global forces can be composed together correctly.""" rng = np.random.default_rng(seed=14) @@ -660,7 +661,7 @@ def test_composition_mixed_local_and_global(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: int): @@ -705,7 +706,7 @@ def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: assert np.allclose(composed_torque_np, expected_torques, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_link_origin_no_torque(device: str): """Test that a global force applied at the link origin produces no torque.""" rng = np.random.default_rng(seed=16) @@ -753,7 +754,7 @@ def test_global_force_at_link_origin_no_torque(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): @@ -819,7 +820,7 @@ def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): ), "add_raw_buffers_from torque mismatch vs direct accumulation" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_add_raw_buffers_from_inactive_is_noop(device: str): """Test that add_raw_buffers_from is a no-op when the source composer is inactive.""" num_envs, num_bodies = 4, 2 @@ -853,7 +854,7 @@ def test_add_raw_buffers_from_inactive_is_noop(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): @@ -908,7 +909,7 @@ def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): ), f"Mask vs index torque mismatch (envs={num_envs}, bodies={num_bodies})" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): @@ -963,7 +964,7 @@ def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_overwrites_previous_add(device: str): """Test that set_forces_and_torques_index clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -992,7 +993,7 @@ def test_set_forces_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_index clears only the targeted environments.""" num_envs, num_bodies = 4, 3 @@ -1063,7 +1064,7 @@ def test_set_forces_clears_targeted_envs_only(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_partial_reset_zeros_only_specified_envs(device: str): """Test that partial reset zeros only the specified environments and leaves others intact.""" num_envs, num_bodies = 8, 3 @@ -1114,7 +1115,7 @@ def test_partial_reset_zeros_only_specified_envs(device: str): assert composer._dirty -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_full_reset_clears_active_flag(device: str): """Test that full reset (no args) clears the _active flag.""" num_envs, num_bodies = 4, 2 @@ -1138,7 +1139,7 @@ def test_full_reset_clears_active_flag(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_force_emits_deprecation_warning(device: str): """Test that accessing composed_force emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1158,7 +1159,7 @@ def test_composed_force_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_force_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_torque_emits_deprecation_warning(device: str): """Test that accessing composed_torque emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1177,7 +1178,7 @@ def test_composed_torque_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_torque_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_add_forces_and_torques_emits_warning(device: str): """Test that the deprecated add_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1202,7 +1203,7 @@ def test_deprecated_add_forces_and_torques_emits_warning(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_overwrites_previous_add(device: str): """Test that set_forces_and_torques_mask clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -1231,7 +1232,7 @@ def test_set_forces_mask_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_mask clears only the masked environments.""" num_envs, num_bodies = 4, 3 @@ -1301,7 +1302,7 @@ def test_set_forces_mask_clears_targeted_envs_only(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_matches_set_forces_index(device: str): """Test that set_forces_and_torques_mask produces the same result as the index variant.""" num_envs, num_bodies = 6, 3 @@ -1351,7 +1352,7 @@ def test_set_forces_mask_matches_set_forces_index(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_force_b_triggers_lazy_composition(device: str): """Test that accessing out_force_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1378,7 +1379,7 @@ def test_out_force_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_torque_b_triggers_lazy_composition(device: str): """Test that accessing out_torque_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1405,7 +1406,7 @@ def test_out_torque_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_lazy_composition_tracks_dirty_flag(device: str): """Test that the dirty flag is correctly managed through add/compose/add cycles.""" num_envs, num_bodies = 2, 1 @@ -1442,7 +1443,7 @@ def test_lazy_composition_tracks_dirty_flag(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), expected, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_compose_is_idempotent(device: str): """Calling compose_to_body_frame twice without intervening writes produces the same result.""" rng = np.random.default_rng(seed=456) @@ -1493,7 +1494,7 @@ def test_compose_is_idempotent(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_with_com_offset(device: str): """Test that torque correction uses CoM position, not link position, when they differ.""" num_envs, num_bodies = 2, 1 @@ -1548,7 +1549,7 @@ def test_global_force_with_com_offset(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_com_no_torque_with_com_offset(device: str): """Test that a global force at CoM position produces zero torque even with CoM offset.""" num_envs, num_bodies = 2, 1 @@ -1594,7 +1595,7 @@ def test_global_force_at_com_no_torque_with_com_offset(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_com_offset_with_rotation(device: str): """Test torque correction with both CoM offset and non-identity rotation.""" num_envs, num_bodies = 1, 1 @@ -1652,7 +1653,7 @@ def test_com_offset_with_rotation(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_emits_warning(device: str): """Test that the deprecated set_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1672,7 +1673,7 @@ def test_deprecated_set_forces_and_torques_emits_warning(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_clears_previous(device: str): """Test that deprecated set_forces_and_torques actually replaces previous values.""" num_envs, num_bodies = 4, 2 diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index e21f520c4324..40540c0ac655 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -479,7 +480,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -529,7 +530,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_articulations", [2, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -569,7 +570,7 @@ def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -619,7 +620,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base(sim, num_articulations, device, articulation_type): """Test initialization for fixed base. @@ -676,7 +677,7 @@ def test_initialization_fixed_base(sim, num_articulations, device, articulation_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -734,7 +735,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) def test_initialization_hand_with_tendons(sim, num_articulations, device, articulation_type): """Test initialization for fixed base articulated hand with tendons. @@ -783,7 +784,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device, articu @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base_made_fixed_base( @@ -837,7 +838,7 @@ def test_initialization_floating_base_made_fixed_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base_made_floating_base( @@ -883,7 +884,7 @@ def test_initialization_fixed_base_made_floating_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -914,7 +915,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_vel(sim, device, articulation_type): """Test that the default joint velocity from configuration is out of range. @@ -939,7 +940,7 @@ def test_out_of_range_default_joint_vel(sim, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1015,7 +1016,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1049,7 +1050,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_buffer(sim, num_articulations, device, articulation_type): """Test if external force buffer correctly updates in the force value is zero case. @@ -1134,7 +1135,7 @@ def test_external_force_buffer(sim, num_articulations, device, articulation_type @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation. @@ -1192,7 +1193,7 @@ def test_external_force_on_single_body(sim, num_articulations, device, articulat @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation at a given position. @@ -1287,7 +1288,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation. @@ -1347,7 +1348,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device, artic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation at a given position. @@ -1441,7 +1442,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_loading_gains_from_usd(sim, num_articulations, device, articulation_type): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1503,7 +1504,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device, articulation_typ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1538,7 +1539,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulation_type): """Test that gains are loaded from the configuration dictionary correctly. @@ -1571,7 +1572,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1643,7 +1644,7 @@ def test_setting_velocity_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1699,7 +1700,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) @@ -1756,7 +1757,7 @@ def test_setting_effort_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1822,7 +1823,7 @@ def test_setting_effort_limit_explicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_reset(sim, num_articulations, device, articulation_type): """Test that reset method works properly.""" @@ -1866,7 +1867,7 @@ def test_reset(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1906,7 +1907,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_body_root_state(sim, num_articulations, device, with_offset, articulation_type): @@ -2031,7 +2032,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset, articulati @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2120,7 +2121,7 @@ def test_write_root_state( torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2139,7 +2140,7 @@ def test_setting_articulation_root_prim_path(sim, device, articulation_type): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2158,7 +2159,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_t @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled, articulation_type): @@ -2264,7 +2265,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) @pytest.mark.skip(reason="Spatial tendons are not supported in Newton yet.") def test_spatial_tendons(sim, num_articulations, device, articulation_type): @@ -2318,7 +2319,7 @@ def test_spatial_tendons(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane, articulation_type): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -2388,7 +2389,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_body_q_consistent_after_root_write(num_articulations, device, articulation_type): """Test that body_q is fresh when collide() runs after a root pose write. @@ -2470,7 +2471,7 @@ def _patched_simulate(cls): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) via view-level APIs.""" @@ -2526,7 +2527,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2550,7 +2551,7 @@ def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_pla @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2583,7 +2584,7 @@ def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail( @@ -2635,7 +2636,7 @@ def test_get_gravity_compensation_forces_not_implemented_on_newton(sim, num_arti @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2661,7 +2662,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2697,7 +2698,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2730,7 +2731,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2756,7 +2757,7 @@ def test_get_mass_matrix_shape_floating_base(sim, num_articulations, device, add assert M.dtype == torch.float32 -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2844,7 +2845,7 @@ def test_heterogeneous_scene_per_view_shapes(sim, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2923,7 +2924,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2974,7 +2975,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3025,7 +3026,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3058,7 +3059,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3135,7 +3136,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 8811de5b9db7..deed30bd5c53 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -121,7 +122,7 @@ def generate_cubes_scene( @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -156,7 +157,7 @@ def test_initialization(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="Newton does not support kinematic rigid bodies") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -194,7 +195,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -212,7 +213,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -229,7 +230,7 @@ def test_initialization_with_articulation_root(num_cubes, device): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -298,7 +299,7 @@ def test_external_force_buffer(device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -373,7 +374,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -462,7 +463,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -530,7 +531,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -573,7 +574,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object via view-level APIs.""" with _newton_sim_context(device, gravity_enabled=True, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -628,7 +629,7 @@ def _set_newton_material_properties(cube_object, friction_val, restitution_val, @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo contact at height=0 does not settle the same as PhysX — cube falls on z-axis") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -668,7 +669,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -681,7 +682,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo uses Coulomb friction (single mu), no static/dynamic distinction") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -761,7 +762,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo restitution model differs from PhysX — inelastic collisions still bounce") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -839,7 +840,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _newton_sim_context(device, gravity_enabled=False, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -877,7 +878,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -953,7 +954,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1066,7 +1067,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1137,7 +1138,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): @@ -1297,7 +1298,7 @@ def test_warmup_attach_stage_not_called_for_cpu(): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_link_pose_w_fresh_after_root_pose_write(device, writer): diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index 18928bedb5d8..9d5355adc31b 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -119,7 +120,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -151,7 +152,7 @@ def test_initialization(num_envs, num_cubes, device): @pytest.mark.skip(reason="Newton doesn't support kinematic rigid bodies yet") @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -187,7 +188,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -202,7 +203,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -258,7 +259,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -320,7 +321,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -403,7 +404,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_object_state(num_envs, num_cubes, device): """Test setting the state of the object. @@ -475,7 +476,7 @@ def test_set_object_state(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -510,7 +511,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object collection via view-level APIs.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -549,7 +550,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -622,7 +623,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset): """Test the object_com_state_w and object_link_state_w properties.""" @@ -717,7 +718,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_object_state(num_envs, num_cubes, device, with_offset, state_location): @@ -795,7 +796,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_object_state_functions_data_consistency(num_envs, num_cubes, device, with_offset, state_location): @@ -932,7 +933,7 @@ def test_write_object_state_functions_data_consistency(num_envs, num_cubes, devi torch.testing.assert_close(body_com_vel_w[..., 3:], link_vel_w[..., 3:]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_pose_write_marks_fk_reset_mask(device, writer): diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 2d92e3274a7e..66452b62d553 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -20,6 +20,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import math @@ -74,7 +76,7 @@ class ContactSensorTestSceneCfg(InteractiveSceneCfg): # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -193,7 +195,7 @@ def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: S assert no_contact_detected[env_idx], f"Env {env_idx}: Contact should stop after lift." -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -298,7 +300,7 @@ def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): """Test that resting object contact force equals weight and points upward. @@ -401,7 +403,7 @@ def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): assert not errs, "\n".join(errs) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contacts: bool): """Test that dropping from higher produces larger peak impact force. @@ -483,7 +485,7 @@ def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contac # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -618,7 +620,7 @@ def test_filter_enables_force_matrix(device: str, use_mujoco_contacts: bool): } -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -815,7 +817,7 @@ def _make_two_box_scene_cfg(num_envs: int) -> ContactSensorTestSceneCfg: return scene_cfg -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_sensor_metadata(device: str): """Verify sensor_names and filter_object_names match the underlying sensing and counterpart configuration across body-mode, body-mode-with-filter, and shape-mode. diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index d114a1da2a80..9ff6c7a8610a 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -13,6 +13,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) @@ -102,7 +104,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_body_path(device): """FrameView rejects prim paths that resolve to a Newton physics body.""" ctx = _sim_context(device, num_envs=2) @@ -116,7 +118,7 @@ def test_reject_body_path(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_shape_path(device): """FrameView rejects prim paths that resolve to a Newton collision shape.""" ctx = _sim_context(device, num_envs=2) @@ -183,7 +185,7 @@ def test_view_can_resolve_from_body_labels_after_reset(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_returns_initial_pose(device): """A world-rooted frame returns its configured position.""" ctx = _sim_context(device, num_envs=2) @@ -201,7 +203,7 @@ def test_world_attached_returns_initial_pose(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_set_world_roundtrip(device): """A world-attached prim can be repositioned via set_world_poses.""" ctx = _sim_context(device, num_envs=2) diff --git a/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index 02f382f3f319..fcb1e2374642 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,6 +56,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -328,7 +330,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): @@ -391,7 +393,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): @@ -455,7 +457,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -526,7 +528,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): @@ -598,7 +600,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -657,7 +659,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_OMNI_PHYSX_SCHEMAS_GAP_REASON, strict=False) @@ -725,7 +727,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): @@ -784,7 +786,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): @@ -815,7 +817,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -840,7 +842,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @@ -916,7 +918,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -949,7 +951,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -1034,7 +1036,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1092,7 +1094,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1187,7 +1189,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1247,7 +1249,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1341,7 +1343,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1403,7 +1405,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): @@ -1438,7 +1440,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1471,7 +1473,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1539,7 +1541,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.isaacsim_ci @@ -1593,7 +1595,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.isaacsim_ci @@ -1646,7 +1648,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.isaacsim_ci @@ -1708,7 +1710,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" @@ -1752,7 +1754,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.isaacsim_ci def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @@ -1792,7 +1794,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci def test_body_root_state(sim, num_articulations, device, with_offset): @@ -1919,7 +1921,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2004,7 +2006,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, device): """Test the data.body_incoming_joint_wrench_b buffer is populated correctly and statically correct for single joint. @@ -2105,7 +2107,7 @@ def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, devic ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2124,7 +2126,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" @@ -2143,7 +2145,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): @@ -2249,7 +2251,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2301,7 +2303,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2394,7 +2396,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 7c9567ade19c..d896a59975ce 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,6 +32,8 @@ import warp as wp from flaky import flaky +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -175,7 +177,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -208,7 +210,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -245,7 +247,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -262,7 +264,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -278,7 +280,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -346,7 +348,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -423,7 +425,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -527,7 +529,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -592,7 +594,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -635,7 +637,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -644,7 +646,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -653,7 +655,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -662,7 +664,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -677,7 +679,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -691,7 +693,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -735,7 +737,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -774,7 +776,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -891,7 +893,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -963,7 +965,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index f17b195e8341..470e97d3cb15 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,6 +28,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, # but the ovphysx wheel is not installed in that environment. Skip gracefully # so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. @@ -171,7 +173,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -200,7 +202,7 @@ def test_initialization(num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_id_conversion(device): """Test environment and object index conversion to physics view indices.""" @@ -240,7 +242,7 @@ def test_id_conversion(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -276,7 +278,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -291,7 +293,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" @@ -347,7 +349,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" @@ -409,7 +411,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -492,7 +494,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @@ -563,7 +565,7 @@ def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -663,7 +665,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravi @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -742,7 +744,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" @@ -778,7 +780,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -787,7 +789,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @@ -822,7 +824,7 @@ def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index af36a365cb66..b13e10d539bf 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -310,7 +311,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -366,7 +367,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -423,7 +424,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -487,7 +488,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -552,7 +553,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -605,7 +606,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base articulation made fixed-base using schema properties. @@ -665,7 +666,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -717,7 +718,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -747,7 +748,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -771,7 +772,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -846,7 +847,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -879,7 +880,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -963,7 +964,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1020,7 +1021,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1114,7 +1115,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1173,7 +1174,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1266,7 +1267,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1327,7 +1328,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1361,7 +1362,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1393,7 +1394,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1460,7 +1461,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1513,7 +1514,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1565,7 +1566,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1627,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1669,7 +1670,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1708,7 +1709,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1831,7 +1832,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1912,7 +1913,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1930,7 +1931,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1948,7 +1949,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2053,7 +2054,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2105,7 +2106,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2198,7 +2199,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) of articulation shapes.""" @@ -2245,7 +2246,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2262,7 +2263,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2289,7 +2290,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2314,7 +2315,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2366,7 +2367,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2402,7 +2403,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2453,7 +2454,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2487,7 +2488,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2576,7 +2577,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2634,7 +2635,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2798,7 +2799,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2832,7 +2833,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index b7f422c4f2f0..f55761b4bdad 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -98,7 +99,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -132,7 +133,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -170,7 +171,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -188,7 +189,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -205,7 +206,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -274,7 +275,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -350,7 +351,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -455,7 +456,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -521,7 +522,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -564,7 +565,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -605,7 +606,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -645,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -694,7 +695,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -705,7 +706,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -791,7 +792,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -874,7 +875,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -918,7 +919,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -958,7 +959,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -1069,7 +1070,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -1139,7 +1140,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index f42f6dfcb085..4ec56828af82 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -110,7 +111,7 @@ def sim(request): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(sim, num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -137,7 +138,7 @@ def test_initialization(sim, num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(sim, device): """Test environment and object index conversion to physics view indices.""" object_collection, _ = generate_cubes_scene(num_envs=2, num_cubes=3, device=device) @@ -175,7 +176,7 @@ def test_id_conversion(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" object_collection, origins = generate_cubes_scene( @@ -209,7 +210,7 @@ def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(sim, num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_cubes=num_cubes, has_api=False, device=device) @@ -222,7 +223,7 @@ def test_initialization_with_no_rigid_body(sim, num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -276,7 +277,7 @@ def test_external_force_buffer(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object.""" object_collection, origins = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -336,7 +337,7 @@ def test_external_force_on_single_body(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -415,7 +416,7 @@ def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, dev @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -480,7 +481,7 @@ def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -576,7 +577,7 @@ def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -656,7 +657,7 @@ def test_write_object_state(sim, num_envs, num_cubes, device, with_offset, state @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(sim, num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -689,7 +690,7 @@ def test_reset_object_collection(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(sim, num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -724,7 +725,7 @@ def test_set_material_properties(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -757,7 +758,7 @@ def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index 3cfe70095fd3..9a1c5e9e34ea 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,7 +10,6 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path @@ -30,6 +29,7 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 +from isaaclab.test.utils import test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) @@ -126,7 +126,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ------------------------------------------------------------------ -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.xfail( reason=( "Issue #5: FabricFrameView.set_world_poses writes to Fabric worldMatrix only. " @@ -155,7 +155,7 @@ def _fill_position(out: wp.array(dtype=wp.float32, ndim=2), x: float, y: float, out[i, 2] = wp.float32(z) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): """Verify that set_world_poses in Fabric mode does NOT sync back to USD. @@ -199,7 +199,7 @@ def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_rebuild_after_topology_change(device, view_factory, monkeypatch): """Forcing the topology-changed branch on a write triggers :meth:`_rebuild_fabric_arrays` and leaves the view in a state where @@ -252,11 +252,7 @@ def force_topology_changed(): # ------------------------------------------------------------------ -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -276,11 +272,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -308,11 +300,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """set_scales -> get_scales roundtrip works on cuda:1. From 5715dcb2a6fa7154b699eabce21ab7d79a1802b4 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:57:18 +0000 Subject: [PATCH 09/23] [App] Honor ISAACLAB_SIM_DEVICE as implicit default device in AppLauncher When the caller does not pass an explicit device, AppLauncher reads ISAACLAB_SIM_DEVICE and uses it as the device. Lets the multi-GPU CI lane boot Kit on a non-default GPU without editing every test's AppLauncher() call site. --- source/isaaclab/isaaclab/app/app_launcher.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 53dd8065aa11..2474d0703047 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1003,6 +1003,17 @@ def _resolve_device_settings(self, launcher_args: dict): device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) + # When the caller didn't pin a device, allow ISAACLAB_SIM_DEVICE to + # override the default. Used by the multi-GPU CI workflow to boot + # Kit with active_gpu=1 without editing every test's AppLauncher() + # call site; Kit's active_gpu is process-global and locked after + # SimulationApp init, so per-test selection cannot retarget it. + if not device_explicitly_passed: + env_device = os.environ.get("ISAACLAB_SIM_DEVICE") + if env_device: + device = env_device + launcher_args["device"] = env_device + if self._xr and not device_explicitly_passed: # If no device is specified, default to the CPU device if we are running in XR device = "cpu" From c514199d41ac37de3e00295b0254cf6f2b5af9d4 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:57:18 +0000 Subject: [PATCH 10/23] TEMP: skip docker/docs/install-ci CI while iterating #5823 Speeds up iteration on the multi-GPU lane: forces run_docker_tests=false in build.yaml and gates docs.yaml / install-ci.yml behind a DO-NOT-MERGE PR-title check. Revert this commit before the PR is merged. --- .github/workflows/build.yaml | 7 ++++++- .github/workflows/docs.yaml | 15 +++++++++++++-- .github/workflows/install-ci.yml | 5 +++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1c599ebcb988..2428799e72d9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,7 +74,12 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} + # TEMP (revert before final review / before landing): force + # run_docker_tests=false while iterating PR #5823. All gated + # test/build jobs skip via their existing if-gate; the + # single-GPU Docker + Tests matrix won't burn the gpu pool on + # every push. Per ~/.claude/skills/pr/ci-iteration-shortcut.md. + run_docker_tests: 'false' steps: - id: detect env: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 013be3a5b126..098d9ba784b0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -25,6 +25,11 @@ jobs: doc-build-type: name: Detect Doc Build Type runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip docs build entirely when the PR title is + # marked DO-NOT-MERGE (scratch / hypothesis-only branches don't need a + # full doc rebuild on every push). Push events on main/develop/release + # still run because they don't carry a PR title. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: trigger-deploy: ${{ steps.trigger-deploy.outputs.defined }} steps: @@ -42,8 +47,14 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # run on non-deploy branches to build current version docs only - if: needs.doc-build-type.outputs.trigger-deploy != 'true' + # Run on non-deploy branches to build current version docs only AND skip + # for DO-NOT-MERGE diagnostic PRs (the needed doc-build-type job already + # skips for those, but GitHub Actions evaluates `needs.X.outputs.Y` as + # empty when X is skipped, which keeps this `!= 'true'` gate true — so + # we re-check the title here to actually cascade the skip). + if: | + needs.doc-build-type.outputs.trigger-deploy != 'true' + && !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 2a2fee50c8ab..df3827bfb870 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -34,6 +34,11 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest + # DIAGNOSTIC PR opt-out: skip install-ci entirely when the PR title is + # marked DO-NOT-MERGE. The downstream install-tests-x86 / install-tests-arm + # jobs gate on `needs.changes.outputs.run_install_tests == 'true'`; with + # this job skipped the output is empty and both dependents skip too. + if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} steps: From 72128e7b46f45b0ab06bdfbcff675537a76bca5f Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 05:01:54 +0000 Subject: [PATCH 11/23] Fix multi-GPU test job to target the multi-GPU runner pool The test job's runs-on carried the `gpu` label, which in the self-hosted fleet tags single-GPU runners. Requiring both `gpu` and `multi-gpu` routed the job onto a single-GPU box, so the shard runner aborted with "Need at least 2 visible devices; found 1"; the over-constrained label also left the job queued for hours. Drop `gpu` so the job targets the multi-GPU pool. --- .github/workflows/test-multi-gpu-pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 55db0e9996c1..b93237d27b25 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -89,7 +89,7 @@ jobs: if: needs.build.result == 'success' # Matches the runner label used by test-multi-gpu.yaml + # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. - runs-on: [self-hosted, linux, x64, gpu, multi-gpu] + runs-on: [self-hosted, linux, x64, multi-gpu] timeout-minutes: 60 steps: - name: Checkout repository From 9d159f39c272d02abb2084119308abf68edf32e6 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 05:05:19 +0000 Subject: [PATCH 12/23] Extract multi-GPU workflow scripts to sibling files Move the host-orchestration bash (symlink, work-queue seed, MIG detection, docker run, reconciler) and the JUnit-XML aggregation Python out of the workflow's inline run: blocks into version-controlled, lint-able scripts under .github/actions/multi-gpu/, alongside the existing multi_gpu_shard_runner.sh. The workflow drops from 456 to 181 lines and each step becomes a one-line call. Behavior-preserving: data still flows via step env, $GITHUB_OUTPUT, $GITHUB_ENV, and $GITHUB_STEP_SUMMARY; the container --name now uses the runner built-ins $GITHUB_RUN_ID/$GITHUB_RUN_ATTEMPT in place of the YAML-only ${{ github.run_id }} templating. Also documents why the test job must not carry the gpu label. --- .../multi-gpu/aggregate_test_summary.py | 176 +++++++++++ .../multi-gpu/multi_gpu_host_launcher.sh | 158 ++++++++++ .github/workflows/test-multi-gpu-pytest.yaml | 298 +----------------- 3 files changed, 347 insertions(+), 285 deletions(-) create mode 100755 .github/actions/multi-gpu/aggregate_test_summary.py create mode 100755 .github/actions/multi-gpu/multi_gpu_host_launcher.sh diff --git a/.github/actions/multi-gpu/aggregate_test_summary.py b/.github/actions/multi-gpu/aggregate_test_summary.py new file mode 100755 index 000000000000..6e1bec25f2f7 --- /dev/null +++ b/.github/actions/multi-gpu/aggregate_test_summary.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate per-shard JUnit XML reports from the multi-GPU pytest lane. + +Reads ``RUNTIME_DIR`` (the host launcher's work-queue root, holding +``queue/done//``) and the ``tests/test-reports-*.xml`` JUnit files, +then prints a per-file + per-shard table to stdout and, when +``GITHUB_STEP_SUMMARY`` is set, writes the same content as markdown so it +renders at the top of the run page. +""" + +import glob +import os +import pathlib +import xml.etree.ElementTree as ET + +runtime_dir = os.environ.get("RUNTIME_DIR", "") +done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + +# Map slug -> shard from done//; slug uses "__" for "/". +slug_to_shard = {} +if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + +def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-") :] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + +# Per-file rows from JUnit XMLs. +rows = [] +for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, failures + errors, total_time)) + +# Per-shard aggregate. +shards = {} +for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + +# ---------- Output: plain text to stdout ---------- +def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + +out = [] +out.append("PER-FILE RESULTS") +out.append("=" * 110) +hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] +widths = [60, 7, 8, 12, 5, 5, 8] +out.append(fmt_row(hdr, widths)) +out.append("-" * 110) +for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) +out.append("") +out.append("PER-SHARD AGGREGATE") +out.append("=" * 80) +shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] +shard_widths = [10, 7, 12, 9, 8, 14] +out.append(fmt_row(shard_hdr, shard_widths)) +out.append("-" * 80) +grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} +for sh in sorted(shards): + s = shards[sh] + for k in grand: + grand[k] += s[k] + out.append( + fmt_row( + [ + sh, + str(s["files"]), + f"{s['passed']}/{s['tests']}", + str(s["skipped"]), + str(s["failed"]), + f"{s['time']:.1f}", + ], + shard_widths, + ) + ) +out.append("-" * 80) +out.append( + fmt_row( + [ + "TOTAL", + str(grand["files"]), + f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), + str(grand["failed"]), + f"{grand['time']:.1f}", + ], + shard_widths, + ) +) +print("\n".join(out)) + +# ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- +summary_path = os.environ.get("GITHUB_STEP_SUMMARY") +if summary_path: + md = [ + "## Multi-GPU CI: Aggregated Test Summary", + "", + "### Per-shard totals", + "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|", + ] + for sh in sorted(shards): + s = shards[sh] + md.append( + f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |" + ) + md.append( + f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | " + f"**{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |" + ) + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = { + "PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR", + }.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh new file mode 100755 index 000000000000..332d5bd73a20 --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Runs ON THE HOST runner — the counterpart to multi_gpu_shard_runner.sh, which +# runs INSIDE the container. Launches ONE container that hosts all N pytest +# shards as parallel subshells, then reconciles the shared work queue. +# +# Invoked by .github/workflows/test-multi-gpu-pytest.yaml's "Run shards in +# parallel" step. Reads from the environment (set by that step): +# IMAGE_TAG — per-commit CI image to ``docker run`` +# INCLUDE_FILES — comma-sep test basenames (conftest include filter) +# PATHS — comma-sep relative test paths (seed the work queue) +# plus the runner built-ins GITHUB_RUN_ID / GITHUB_RUN_ATTEMPT (container name) +# and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). +# +# 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its +# own non-default cuda:N via ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES, pulling +# from the shared queue). Collapsing the previous N-container layout into one +# removes cross-container races on the workspace mount (``_isaac_sim`` symlink, +# ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at +# the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, +# set in the shard runner) keeps user-level caches isolated. +# +# Scaling: shard count is derived inside the container from ``nvidia-smi -L`` +# (truth — torch.cuda.device_count() under-counts MIG slices on the same parent +# GPU), so this works unchanged on any GPU count/topology. CUDA_VISIBLE_DEVICES +# is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete +# runners rely on ``--gpus all``. + +set +e +host_uid="$(id -u)" +host_gid="$(id -g)" +host_user="$(id -un)" +# ``/isaac-sim`` is the Kit install root inside the container; pytest +# invocations under ``./isaaclab.sh -p`` resolve the bundled Python as +# ``./_isaac_sim/python.sh``. Mounting the workspace at ``/workspace/isaaclab`` +# brings the host's CWD into the container, so we plant a host-side symlink +# whose target resolves only inside the container. Single creator (this step) +# — no concurrent creators. +rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + +runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" +mkdir -p "$runtime_dir" + +# Directory-based work queue (atomic os.rename pattern; conftest claims entries +# via os.rename(queue/X, inflight//X) — POSIX rename is atomic on the +# same filesystem, so two shards racing on the same entry are kernel-serialized: +# one wins, the other gets ENOENT and moves on). Each pending test path is +# materialized as an empty file in queue/ with '/' slugged to '__' so it's a +# flat filename. +# +# Subdirs: +# queue/ — pending claims +# inflight// — claimed by this shard, test in flight +# done// — test exited cleanly +# +# At job-end the reconciler below asserts queue/ + inflight/ are empty — if +# either is non-empty the job FAILS (silent-drop signal: never claimed OR +# claimed but crashed mid-test). +queue_root="$runtime_dir/queue" +mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" +IFS=',' read -ra _paths <<< "$PATHS" +for path in "${_paths[@]}"; do + [ -n "$path" ] || continue + slug="${path//\//__}" + : > "$queue_root/queue/$slug" +done +echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + +# Per-shard log dir mounted from the container so we can grouped-print the +# per-shard stream after the run completes. +logs_dir="$runtime_dir/logs" +mkdir -p "$logs_dir" + +# MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. nvidia-smi -L +# is authoritative — torch.cuda.device_count() under-counts MIG slices on the +# same parent GPU. +cvd_args=() +if nvidia-smi -L | grep -q "^ MIG "; then + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" +else + echo "::notice::discrete GPU mode — relying on --gpus all" +fi + +# --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. +docker run --rm --gpus all --network=host \ + --cap-add=SYS_PTRACE \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh +docker_rc=$? + +# Grouped re-print of per-shard logs (preserved across the host). +for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" +done + +# Reconciler — silent-drop guard. Anything left in queue/ was never claimed by +# any shard (work-queue starvation). Anything still in inflight// was +# claimed but never moved to done/ (shard crashed mid-test). Either is a +# silent-drop signal that the docker exit alone can't surface. Fail the job and +# name the orphans. +echo "::group::work-queue reconciler" +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) +if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' +fi +orphans=0 +for shard_dir in "$queue_root/inflight"/*/; do + [ -d "$shard_dir" ] || continue + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' + orphans=$((orphans + count)) + fi +done +done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) +echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" +echo "::endgroup::" + +# Treat reconciler signals as fatal even if all shards exited 0. +if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 +fi + +# Export the runtime directory so the downstream "Aggregated test summary" step +# can locate the done// work distribution and the per-shard logs. +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" + +exit "$docker_rc" diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index b93237d27b25..10229766c421 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -87,8 +87,10 @@ jobs: name: Multi-GPU unit tests needs: [config, build] if: needs.build.result == 'success' - # Matches the runner label used by test-multi-gpu.yaml + - # test-fabric-multi-gpu.yaml — the canonical multi-GPU pool. + # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here: + # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the + # job onto a 1-GPU box and the shard runner aborts with "Need at least 2 + # visible devices; found 1". runs-on: [self-hosted, linux, x64, multi-gpu] timeout-minutes: 60 steps: @@ -161,295 +163,21 @@ jobs: cache-tag: cache-base - name: Run shards in parallel on local GPUs (1-docker N-shard) - # ONE container hosts all N pytest shards as parallel subshells. Each - # shard pins ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES to its own - # non-default cuda:N and pulls files from the shared work queue. This - # collapses the previous N-container layout into one container, removing - # cross-container races on the shared workspace mount (``_isaac_sim`` - # symlink, ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker - # init per shard, at the cost of sharing the container's /isaac-sim/ - # writable subtrees. Per-shard HOME (under /tmp) keeps user-level - # caches isolated. - # - # Scaling: shard count is derived inside the container from - # ``nvidia-smi -L`` (truth — torch.cuda.device_count() under-counts MIG - # slices on the same parent GPU), so this works unchanged on any GPU - # count or topology. ``CUDA_VISIBLE_DEVICES`` is set to the - # comma-separated MIG-UUID list only on MIG-mode hosts; discrete-GPU - # runners rely on ``--gpus all``. + # ONE container hosts all N pytest shards as parallel subshells, each + # pinned to its own non-default cuda:N and pulling from a shared work + # queue; the host reconciles the queue afterward and exports + # MGPU_RUNTIME_DIR for the summary step. Full rationale + the + # 1-docker-N-shard tradeoffs live in the script header. env: IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} INCLUDE_FILES: ${{ steps.discover.outputs.include }} PATHS: ${{ steps.discover.outputs.paths }} - run: | - set +e - host_uid="$(id -u)" - host_gid="$(id -g)" - host_user="$(id -un)" - # ``/isaac-sim`` is the Kit install root inside the container; pytest - # invocations under ``./isaaclab.sh -p`` resolve the bundled Python as - # ``./_isaac_sim/python.sh``. Mounting the workspace at - # ``/workspace/isaaclab`` brings the host's CWD into the container, - # so we plant a host-side symlink whose target resolves only inside - # the container. Single creator (this step) — no concurrent creators. - rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim - - runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" - mkdir -p "$runtime_dir" - - # Directory-based work queue (atomic os.rename pattern; conftest claims - # entries via os.rename(queue/X, inflight//X) — POSIX rename is - # atomic on the same filesystem, so two shards racing on the same - # entry are kernel-serialized: one wins, the other gets ENOENT and - # moves on). Each pending test path is materialized as an empty file - # in queue/ with '/' slugged to '__' so it's a flat filename. - # - # Subdirs: - # queue/ — pending claims - # inflight// — claimed by this shard, test in flight - # done// — test exited cleanly - # - # At job-end the reconciler step asserts queue/ + inflight/ are empty - # — if either is non-empty the job FAILS (silent-drop signal: never - # claimed OR claimed but crashed mid-test). See the reconciler step - # near the end of this job for the exact assertion. - queue_root="$runtime_dir/queue" - mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" - IFS=',' read -ra _paths <<< "$PATHS" - for path in "${_paths[@]}"; do - [ -n "$path" ] || continue - slug="${path//\//__}" - : > "$queue_root/queue/$slug" - done - echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" - - # Per-shard log dir mounted from the container so we can grouped-print - # the per-shard stream after the run completes. - logs_dir="$runtime_dir/logs" - mkdir -p "$logs_dir" - - # MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. - # nvidia-smi -L is authoritative — see memory rule - # ``reference_gpu_enumeration``: torch.cuda.device_count() under-counts - # MIG slices on the same parent GPU. - cvd_args=() - if nvidia-smi -L | grep -q "^ MIG "; then - mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') - MIG_LIST=$(IFS=,; echo "${MIGS[*]}") - cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") - echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" - else - echo "::notice::discrete GPU mode — relying on --gpus all" - fi - - # --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. - docker run --rm --gpus all --network=host \ - --cap-add=SYS_PTRACE \ - --entrypoint bash \ - --user "${host_uid}:${host_gid}" \ - --name "isaac-lab-mgpu-${{ github.run_id }}-${{ github.run_attempt }}" \ - -v "$PWD:/workspace/isaaclab:rw" \ - -v "$queue_root:/mgpu:rw" \ - -v "$logs_dir:/shard-logs:rw" \ - -e USER="${host_user}" \ - -e LOGNAME="${host_user}" \ - -e OMNI_KIT_ACCEPT_EULA=yes \ - -e ACCEPT_EULA=Y \ - -e OMNI_KIT_DISABLE_CUP=1 \ - -e ISAAC_SIM_HEADLESS=1 \ - -e ISAAC_SIM_LOW_MEMORY=1 \ - -e PYTHONUNBUFFERED=1 \ - -e PYTHONIOENCODING=utf-8 \ - -e ISAACLAB_TEST_QUEUE=/mgpu \ - -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ - -e ISAACLAB_PIN_KIT_GPU=1 \ - -e HOME=/tmp/mgpu-base-home \ - -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ - "${cvd_args[@]}" \ - -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ - "$IMAGE_TAG" \ - /multi_gpu_shard_runner.sh - docker_rc=$? - - # Grouped re-print of per-shard logs (preserved across the host). - for log in "$logs_dir"/cuda-*.log; do - cuda=$(basename "$log" .log | sed s/cuda-//) - echo "::group::shard cuda:${cuda} log" - cat "$log" - echo "::endgroup::" - done - - # Reconciler — silent-drop guard. Anything left in queue/ was never - # claimed by any shard (work-queue starvation). Anything still in - # inflight// was claimed but never moved to done/ (shard - # crashed mid-test). Either is a silent-drop signal that the docker - # exit alone can't surface. Fail the job and name the orphans. - echo "::group::work-queue reconciler" - unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) - if [ "$unclaimed" -gt 0 ]; then - echo "::error::${unclaimed} test(s) never claimed by any shard:" - ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' - fi - orphans=0 - for shard_dir in "$queue_root/inflight"/*/; do - [ -d "$shard_dir" ] || continue - count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) - if [ "$count" -gt 0 ]; then - echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" - ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' - orphans=$((orphans + count)) - fi - done - done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) - echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" - echo "::endgroup::" - - # Treat reconciler signals as fatal even if all shards exited 0. - if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then - echo "::error::silent-drop detected — see reconciler group above" - [ "$docker_rc" -eq 0 ] && docker_rc=2 - fi - - # Export the runtime directory so the downstream "Aggregated test - # summary" step can locate the done// work distribution - # and the per-shard logs. - echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" - - exit "$docker_rc" + run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh - name: Aggregated test summary - # Runs whether or not the test job exit was clean — surfaces what - # happened even on failure. Per-shard stats + per-file pass / total / - # walltime, plus a single combined table. Also writes the same content - # to $GITHUB_STEP_SUMMARY so it renders at the top of the run page. + # Per-shard + per-file pass/total/walltime, plus a combined table, also + # written to $GITHUB_STEP_SUMMARY. Runs on success or failure. if: always() env: RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} - run: | - python3 - <<'PY' - import os, glob, xml.etree.ElementTree as ET, sys, pathlib - - runtime_dir = os.environ.get("RUNTIME_DIR", "") - done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None - - # Map slug -> shard from done//; slug uses "__" for "/". - slug_to_shard = {} - if done_root and done_root.is_dir(): - for shard_dir in done_root.iterdir(): - if not shard_dir.is_dir(): - continue - shard = shard_dir.name # e.g. "cuda-1" - for entry in shard_dir.iterdir(): - slug_to_shard[entry.name] = shard - - def unslug(name: str) -> str: - # report file pattern: test-reports-.xml - if name.startswith("test-reports-"): - name = name[len("test-reports-"):] - if name.endswith(".xml"): - name = name[:-4] - return name # this is the slug; the file path is slug.replace("__", "/") - - # Per-file rows from JUnit XMLs. - rows = [] - for path in sorted(glob.glob("tests/test-reports-*.xml")): - slug = unslug(os.path.basename(path)) - file_path = slug.replace("__", "/") - shard = slug_to_shard.get(slug, "?") - try: - tree = ET.parse(path) - root = tree.getroot() - except ET.ParseError: - rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) - continue - tests = failures = errors = skipped = 0 - total_time = 0.0 - for ts in root.iter("testsuite"): - tests += int(ts.get("tests", 0)) - failures += int(ts.get("failures", 0)) - errors += int(ts.get("errors", 0)) - skipped += int(ts.get("skipped", 0)) - total_time += float(ts.get("time", 0) or 0) - passed = tests - failures - errors - skipped - status = "PASS" - if failures + errors > 0: - status = "FAIL" - elif tests == 0: - status = "EMPTY" - rows.append((file_path, shard, status, passed, tests, skipped, - failures + errors, total_time)) - - # Per-shard aggregate. - shards = {} - for fp, sh, st, p, t, sk, f, tm in rows: - if sh not in shards: - shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, - "failed": 0, "time": 0.0} - s = shards[sh] - s["files"] += 1 - s["passed"] += p - s["tests"] += t - s["skipped"] += sk - s["failed"] += f - s["time"] += tm - - # ---------- Output: plain text to stdout ---------- - def fmt_row(cols, widths): - return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) - - out = [] - out.append("PER-FILE RESULTS") - out.append("=" * 110) - hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] - widths = [60, 7, 8, 12, 5, 5, 8] - out.append(fmt_row(hdr, widths)) - out.append("-" * 110) - for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): - out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) - out.append("") - out.append("PER-SHARD AGGREGATE") - out.append("=" * 80) - shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] - shard_widths = [10, 7, 12, 9, 8, 14] - out.append(fmt_row(shard_hdr, shard_widths)) - out.append("-" * 80) - grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} - for sh in sorted(shards): - s = shards[sh] - for k in grand: grand[k] += s[k] - out.append(fmt_row( - [sh, str(s["files"]), f"{s['passed']}/{s['tests']}", str(s["skipped"]), - str(s["failed"]), f"{s['time']:.1f}"], shard_widths)) - out.append("-" * 80) - out.append(fmt_row( - ["TOTAL", str(grand["files"]), f"{grand['passed']}/{grand['tests']}", - str(grand["skipped"]), str(grand["failed"]), f"{grand['time']:.1f}"], - shard_widths)) - print("\n".join(out)) - - # ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- - summary_path = os.environ.get("GITHUB_STEP_SUMMARY") - if summary_path: - md = ["## Multi-GPU CI: Aggregated Test Summary", "", - "### Per-shard totals", "", - "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", - "|---|---:|---:|---:|---:|---:|"] - for sh in sorted(shards): - s = shards[sh] - md.append(f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |") - md.append(f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | **{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |") - md.append("") - md.append("### Per-file results") - md.append("") - md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") - md.append("|---|---|---|---:|---:|---:|---:|") - for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): - status_md = {"PASS": ":white_check_mark: PASS", - "FAIL": ":x: FAIL", - "EMPTY": ":white_circle: EMPTY", - "PARSE-ERROR": ":warning: PARSE-ERROR"}.get(st, st) - md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") - md.append("") - with open(summary_path, "a") as fh: - fh.write("\n".join(md) + "\n") - PY + run: python3 .github/actions/multi-gpu/aggregate_test_summary.py From 437a6d20d410d05fec398e38b35c184010dbec17 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sat, 6 Jun 2026 06:39:07 +0000 Subject: [PATCH 13/23] Build the multi-GPU CI image on the multi-GPU pool The ECR cache repo is resolved per runner pool (single-GPU runners -> gitci-docker-cache; multi-GPU runners -> multigpu-docker-cache). Building on [self-hosted, gpu] pushed the image to gitci-docker-cache, which the multi-GPU test job cannot see, so it rebuilt from scratch (~27 min) on the multi-GPU runner inside its pull step. Build on the multi-GPU pool so the image lands in multigpu-docker-cache and the test job's pull hits. --- .github/workflows/test-multi-gpu-pytest.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 10229766c421..8771c3a69428 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -59,9 +59,14 @@ jobs: build: name: Build / cache image needs: [config] - # Build only needs Docker + ECR access, not multi-GPU. Aligns with the - # build jobs in build.yaml that also publish the per-commit image. - runs-on: [self-hosted, gpu] + # Must run on the SAME pool as the test job. The ECR cache repo is resolved + # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU + # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]` + # the image would land in gitci-docker-cache, which the multi-GPU test job + # cannot see, so it would rebuild from scratch on the scarce multi-GPU + # runner. Building here populates multigpu-docker-cache so the test job's + # pull hits. + runs-on: [self-hosted, linux, x64, multi-gpu] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: From b17679c61c66aed9e853539446c3d693a875ba86 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 14/23] Replace root device-skip conftest with a lane plugin Move non-default-shard device selection out of a repo-root conftest.py into a scoped pytest plugin (mgpu_shard_select) that tools/conftest.py injects per file only on multi-GPU shards. The plugin keys off ISAACLAB_TEST_DEVICES so keep/drop matches each test's device parametrization, deselects out-of-scope variants (cpu, cuda:0, other-index), and maps the all-deselected NO_TESTS_COLLECTED exit to OK. Confines the behavior to the lane instead of a global root conftest. --- .../actions/multi-gpu/mgpu_shard_select.py | 93 +++++++++++++++++++ .github/workflows/test-multi-gpu-pytest.yaml | 8 +- conftest.py | 33 ------- .../isaaclab/isaaclab/test/utils/__init__.py | 4 +- tools/conftest.py | 18 ++++ 5 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 .github/actions/multi-gpu/mgpu_shard_select.py delete mode 100644 conftest.py diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py new file mode 100644 index 000000000000..22f784435bb3 --- /dev/null +++ b/.github/actions/multi-gpu/mgpu_shard_select.py @@ -0,0 +1,93 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""pytest plugin: select which tests a non-default GPU shard runs. + +Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via +``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this +directory to ``PYTHONPATH`` so it is importable). It lives next to the lane +scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. + +Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME +env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's +device parametrization, so this plugin's keep/drop decision agrees with what was +parametrized. (``ISAACLAB_SIM_DEVICE`` is AppLauncher's Kit boot device, a +different concern, and is deliberately NOT used here.) The plugin reads the mask +string directly -- no isaaclab import -- matching the lib-free convention of every +other conftest in the repo. The only shared knowledge is the mask grammar below, +mirrored from ``devices.py`` (position 0 = cpu, position k = cuda:(k-1), trailing +``X`` = include all remaining); keep the two in sync. + +Selection rule, on a shard whose mask excludes cpu + cuda:0: + + keep a test iff it is parametrized over ``device`` AND that variant's device + is active in the mask; deselect everything else. + +That single rule is both criteria at once: candidate = device-parametrized (a +test with no ``device`` param is device-agnostic, covered by single-GPU CI on +cuda:0); scope = only this shard's device variant runs (cpu / cuda:0 / other-index +variants are dropped). Out-of-scope items are deselected (not skipped) so the +shard report shows only what ran. +""" + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +def _position(device): + """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None # not device-parametrized, or an unrecognized device value + + +def _active(position, mask): + """Whether a mask position is included. Trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + +def _shard_mask(): + """Return the runtime mask iff this is a multi-GPU shard run, else None. + + A shard targets non-default GPUs only, so cpu (position 0) and cuda:0 + (position 1) are excluded. Anything else -- unset, or single-GPU CI's default + ``"110"`` -- is not a shard and the plugin is a no-op. + """ + mask = os.environ.get(_RUNTIME_ENV, "") + return mask if mask and not _active(0, mask) and not _active(1, mask) else None + + +def pytest_collection_modifyitems(config, items): + mask = _shard_mask() + if mask is None: + return + keep, drop = [], [] + for item in items: + callspec = getattr(item, "callspec", None) + device = callspec.params.get("device") if callspec is not None else None + if _active(_position(device), mask): + keep.append(item) # device-parametrized AND in this shard's scope + else: + drop.append(item) # non-device, or cpu / cuda:0 / other-index variant + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + +def pytest_sessionfinish(session, exitstatus): + # A file whose tests are all out of scope deselects to zero, so pytest exits + # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file + # exit as a failure (tools/conftest.py), so report "nothing in scope for this + # file" as success rather than a false failure. + if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 8771c3a69428..2ea9a86abace 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -114,10 +114,10 @@ jobs: # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. # # Within a discovered file, tests that are NOT parametrized over the - # ``device`` argument are filtered out at collection time by the - # workspace-root ``conftest.py``: single-GPU CI already covers them on - # ``cuda:0`` and re-running on every non-default shard adds wall-time - # without surfacing any new failure mode. + # ``device`` argument are deselected at collection time by the + # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): + # single-GPU CI already covers them on ``cuda:0`` and re-running on every + # non-default shard adds wall-time without surfacing any new failure mode. id: discover run: | # File-level opt-out: a test file can exclude itself from multi-GPU CI diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 30d6ae4c40dd..000000000000 --- a/conftest.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Workspace-root pytest hooks. - -The only hook today is the multi-GPU shard skip: when a shard pins itself to a -non-default cuda device via :envvar:`ISAACLAB_SIM_DEVICE` (e.g. ``cuda:2``), -skip any test that is not parametrized over a ``device`` argument. Such tests -exercise device-agnostic logic that single-GPU CI already covers on -``cuda:0``, so running them again on every non-default shard duplicates work -without surfacing any new failure mode. - -The hook is a no-op outside the multi-GPU lane (``ISAACLAB_SIM_DEVICE`` unset -or equal to ``cuda:0``), so this file does not change pytest behavior under -single-GPU CI. -""" - -import os - -import pytest - - -def pytest_collection_modifyitems(config, items): - sim_device = os.environ.get("ISAACLAB_SIM_DEVICE", "") - if not sim_device.startswith("cuda:") or sim_device == "cuda:0": - return - skip_marker = pytest.mark.skip(reason=f"non-device-parametrized; covered by single-GPU lane (shard={sim_device})") - for item in items: - callspec = getattr(item, "callspec", None) - if callspec is None or "device" not in callspec.params: - item.add_marker(skip_marker) diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py index e268d22c5c16..9e4bb3d6c875 100644 --- a/source/isaaclab/isaaclab/test/utils/__init__.py +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -6,8 +6,8 @@ """Test-time helpers for Isaac Lab. Exposes :func:`test_devices` for selecting the device list to parametrize tests -over. The set is ``scope ∩ budget``: ``scope`` is the call-site argument (the -devices the test is valid on), ``budget`` is the ``ISAACLAB_TEST_DEVICES`` env +over. The set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the +devices the test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env var (the devices the run may use). """ diff --git a/tools/conftest.py b/tools/conftest.py index ad24c85830b4..7103be2aae5c 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -518,6 +518,19 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" + # Multi-GPU lane only: make the device-selection plugin importable in this + # per-file subprocess (injected via ``-p`` below, not as a repo-root + # conftest). Detect a shard by the runtime device mask excluding cpu + # (position 0) and cuda:0 (position 1) -- the same ISAACLAB_TEST_DEVICES the + # plugin and test_devices() read. The plugin re-checks this; the cheap + # prefix test here leaves single-GPU CI's command (mask unset or "11...") + # unchanged. + _mask = os.environ.get("ISAACLAB_TEST_DEVICES", "") + _inject_shard_select = _mask[:2] == "00" + if _inject_shard_select: + _plugin_dir = os.path.join(workspace_root, ".github", "actions", "multi-gpu") + env["PYTHONPATH"] = _plugin_dir + os.pathsep + env.get("PYTHONPATH", "") + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) # Read the test file once for cold-cache check. @@ -559,6 +572,11 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): f"--junitxml={report_file}", "--tb=short", ] + if _inject_shard_select: + cmd += [ + "-p", + "mgpu_shard_select", + ] # multi-GPU lane test-selection plugin (importable via PYTHONPATH set above) if isaacsim_ci: cmd.append("-m") From 49aadc572f2355687fdfbe96fd1d7b91c0c04255 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 15/23] Document multi-GPU scripts; drop py-spy remnants Annotate the uncommon bash idioms in the two multi-GPU orchestration scripts for non-bash readers. Remove the now-dead py-spy/gdb hang-capture remnants -- the SYS_PTRACE cap-add and the py-spy pip install -- along with the matching changelog bullet, since the conftest-side capture they supported was removed. --- .../multi-gpu/multi_gpu_host_launcher.sh | 39 ++++++++-------- .../multi-gpu/multi_gpu_shard_runner.sh | 44 ++++++++++--------- .../jichuanh-multi-gpu-ci.minor.rst | 9 ---- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh index 332d5bd73a20..ad623c61c649 100755 --- a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -25,7 +25,7 @@ # is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete # runners rely on ``--gpus all``. -set +e +set +e # keep going on errors so the reconciler below always runs and reports host_uid="$(id -u)" host_gid="$(id -g)" host_user="$(id -un)" @@ -37,6 +37,7 @@ host_user="$(id -un)" # — no concurrent creators. rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim +# unique per-run temp dir (RUNNER_TEMP if set, else /tmp; XXXXXX -> random suffix) runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" mkdir -p "$runtime_dir" @@ -57,11 +58,11 @@ mkdir -p "$runtime_dir" # claimed but crashed mid-test). queue_root="$runtime_dir/queue" mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" -IFS=',' read -ra _paths <<< "$PATHS" +IFS=',' read -ra _paths <<< "$PATHS" # split the comma-separated PATHS into the _paths array for path in "${_paths[@]}"; do - [ -n "$path" ] || continue - slug="${path//\//__}" - : > "$queue_root/queue/$slug" + [ -n "$path" ] || continue # skip empty entries (PATHS has a trailing comma) + slug="${path//\//__}" # flatten to a filename: replace every '/' with '__' + : > "$queue_root/queue/$slug" # create an empty marker file (':' = no-op, '>' creates it) done echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" @@ -75,17 +76,17 @@ mkdir -p "$logs_dir" # same parent GPU. cvd_args=() if nvidia-smi -L | grep -q "^ MIG "; then + # read each MIG slice's UUID into the MIGS array (one element per line) mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') - MIG_LIST=$(IFS=,; echo "${MIGS[*]}") - cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") # join the array into a comma-separated string + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") # stays empty on discrete hosts -> no flag added echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" else echo "::notice::discrete GPU mode — relying on --gpus all" fi -# --cap-add=SYS_PTRACE for py-spy/gdb hang capture in conftest. +# "${cvd_args[@]}" expands to the CUDA_VISIBLE_DEVICES flag on MIG hosts, nothing otherwise. docker run --rm --gpus all --network=host \ - --cap-add=SYS_PTRACE \ --entrypoint bash \ --user "${host_uid}:${host_gid}" \ --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ @@ -110,11 +111,11 @@ docker run --rm --gpus all --network=host \ -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ "$IMAGE_TAG" \ /multi_gpu_shard_runner.sh -docker_rc=$? +docker_rc=$? # $? = exit code of the docker run above # Grouped re-print of per-shard logs (preserved across the host). for log in "$logs_dir"/cuda-*.log; do - cuda=$(basename "$log" .log | sed s/cuda-//) + cuda=$(basename "$log" .log | sed s/cuda-//) # extract N from "cuda-N.log" echo "::group::shard cuda:${cuda} log" cat "$log" echo "::endgroup::" @@ -126,19 +127,19 @@ done # silent-drop signal that the docker exit alone can't surface. Fail the job and # name the orphans. echo "::group::work-queue reconciler" -unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) # files still pending (never claimed) if [ "$unclaimed" -gt 0 ]; then echo "::error::${unclaimed} test(s) never claimed by any shard:" - ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent for the log fi orphans=0 -for shard_dir in "$queue_root/inflight"/*/; do - [ -d "$shard_dir" ] || continue +for shard_dir in "$queue_root/inflight"/*/; do # each inflight// dir (the /*/ matches dirs only) + [ -d "$shard_dir" ] || continue # skip if the glob matched nothing count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) if [ "$count" -gt 0 ]; then echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" - ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' - orphans=$((orphans + count)) + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent + orphans=$((orphans + count)) # $(( )) = integer arithmetic fi done done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) @@ -148,11 +149,11 @@ echo "::endgroup::" # Treat reconciler signals as fatal even if all shards exited 0. if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then echo "::error::silent-drop detected — see reconciler group above" - [ "$docker_rc" -eq 0 ] && docker_rc=2 + [ "$docker_rc" -eq 0 ] && docker_rc=2 # clean docker exit but drops found -> force a non-zero code fi # Export the runtime directory so the downstream "Aggregated test summary" step # can locate the done// work distribution and the per-shard logs. -echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" # export to later workflow steps via GitHub's env file exit "$docker_rc" diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh index dc5fcbfb6c1a..902b51f6001c 100755 --- a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash # Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest -# shards. Mounted by both the CI workflow (.github/workflows/test-multi-gpu- -# pytest.yaml) and the local probe under ``data/mgpu-1docker-probe/run.sh``, -# so the logic lives in exactly one place and changes are version-controlled -# + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` so it sits -# next to the workflow that consumes it rather than alongside ``tools/``. +# shards. Mounted by the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) so the logic lives in exactly one place and changes are +# version-controlled + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` +# so it sits next to the workflow that consumes it rather than alongside ``tools/``. # # Container expectations (set by the host launcher): # --gpus all @@ -32,9 +31,9 @@ # 6. Waits on every shard before aggregating exit codes — a fast failure doesn't # tear down still-running siblings -set +e +set +e # keep going on errors; per-shard exit codes are aggregated at the end cd /workspace/isaaclab -unset HUB__ARGS__DETECT_ONLY DISPLAY +unset HUB__ARGS__DETECT_ONLY DISPLAY # clear vars that would force Kit into detect-only / headed (X11) mode # Container-level HOME + PYTHONUSERBASE for pip --user installs. The image # runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so @@ -50,11 +49,11 @@ mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase # Pytest deps (same as run-tests action). junitparser is imported at # tools/conftest.py load time, so it must be present first. -./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky py-spy "coverage>=7.6.1" +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky "coverage>=7.6.1" # Shard count from nvidia-smi -L (truth; torch under-counts MIG). -MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") -GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") # grep -c = count of matching lines (MIG slices) +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") # count of whole GPUs if [ "$MIG_COUNT" -gt 0 ]; then DEV_COUNT=$MIG_COUNT echo "::notice::container: MIG mode, $MIG_COUNT slices" @@ -80,11 +79,11 @@ fi # Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME # (per-shard isolation for .cache, .local/share, etc.) and per-shard # ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. -declare -A pids -for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do +declare -A pids # associative array: shard index -> background PID +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) zeros="" - for ((i = 0; i <= cuda; i++)); do zeros+="0"; done - runtime_devices="${zeros}1" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done # build the leading zeros of the device mask + runtime_devices="${zeros}1" # e.g. cuda:2 -> "0001": only this GPU active in the ISAACLAB_TEST_DEVICES mask shard_home="/tmp/isaaclab-ci-home-${cuda}" mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ @@ -93,6 +92,7 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do shard_log="/shard-logs/cuda-${cuda}.log" + # each shard runs in its own subshell, backgrounded with '&' below, so all shards run in parallel ( export HOME="$shard_home" export XDG_CACHE_HOME="${HOME}/.cache" @@ -106,6 +106,8 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # logs, etc. only end up in the shard logfile, NOT on the live workflow # log. The workflow's grouped re-print at end of run still shows the # full $shard_log under a collapsible ``::group::shard cuda:N log``. + # (tee = full output to the log file; stdbuf -oL = flush per line so the + # filtered grep/sed stream appears live, not in delayed chunks.) ./isaaclab.sh -p -m pytest \ --ignore=tools/conftest.py \ --ignore=source/isaaclab/test/install_ci \ @@ -115,24 +117,24 @@ for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ | stdbuf -oL sed "s/^/[cuda:${cuda}] /" - exit "${PIPESTATUS[0]}" + exit "${PIPESTATUS[0]}" # exit with pytest's code, not tee/grep/sed's (PIPESTATUS[0] = first pipe stage) ) & - pids[$cuda]=$! + pids[$cuda]=$! # $! = PID of the subshell just backgrounded echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" done # Wait for every shard before aggregating exits — a fast failure must not # tear down still-running siblings. -declare -A results -for cuda in "${!pids[@]}"; do - wait "${pids[$cuda]}" - results[$cuda]=$? +declare -A results # associative array: shard index -> exit code +for cuda in "${!pids[@]}"; do # "${!pids[@]}" = the array's keys (the shard indices) + wait "${pids[$cuda]}" # block until that shard's PID exits + results[$cuda]=$? # $? = the waited shard's exit code done fail=0 for cuda in "${!results[@]}"; do rc="${results[$cuda]}" echo "shard cuda:${cuda} exited $rc" - [ "$rc" -eq 0 ] || fail=1 + [ "$rc" -eq 0 ] || fail=1 # any non-zero shard makes the whole run fail done exit $fail diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst index 00b4366e153e..5f66a3c858a0 100644 --- a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -14,12 +14,3 @@ Added the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow boot Kit on a non-default GPU without editing every test's :class:`~isaaclab.app.AppLauncher` call site. - -* Added ``py-spy`` + ``gdb`` stack capture in ``tools/conftest.py`` on - ``shutdown_hang`` / ``startup_hang`` / ``timeout`` detection. Walks the test - subprocess's process group (cap 8 pids), captures both Python and C++ frames - before ``SIGKILL`` erases them, attaches the output to the JUnit error - report. Makes Kit binary hangs observable in CI logs; safe no-op when - ``py-spy``/``gdb`` are missing. Workflow side adds ``--cap-add=SYS_PTRACE`` - on the per-shard ``docker run`` (required to attach) and adds ``py-spy`` to - the in-container ``pip install`` list. From 1e710b153f8847b8d1f189e62afdac6a19b50d41 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 16/23] Revert "[CI] Forward extra env vars through run-tests / run-package-tests actions" This reverts commit 6cef107ab0c20659bbd0d04ab86d6b44da839890. --- .github/actions/run-package-tests/action.yml | 5 ---- .github/actions/run-tests/action.yml | 29 +------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 4bde3eb22228..264e01a158b8 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,10 +69,6 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false - extra-env-vars: - description: 'Extra env vars to forward into the container (one KEY=value per line)' - default: '' - required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -154,7 +150,6 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} - extra-env-vars: ${{ inputs.extra-env-vars }} - name: Check Test Results if: always() diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 4565f5ddcfd4..ceea3a1c4e65 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,14 +69,6 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false - extra-env-vars: - description: >- - Extra environment variables to forward into the container, one per line in - ``KEY=value`` form. Whitespace-only lines and lines starting with ``#`` are - ignored. Used by the multi-GPU workflow to inject - ``ISAACLAB_TEST_DEVICES`` / ``ISAACLAB_SIM_DEVICE``. - default: '' - required: false runs: using: composite @@ -101,7 +93,6 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" - local extra_env_vars="${16}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -213,24 +204,6 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi - # Caller-supplied extra env vars (one KEY=value per line). Skips - # blank lines and full-line comments (line where the first - # non-whitespace char is ``#``). Mid-line ``#`` is preserved so - # values like ``IMAGE_TAG=v1.0#nightly`` survive. - if [ -n "$extra_env_vars" ]; then - while IFS= read -r line; do - # Strip leading whitespace (YAML ``|`` block can leave indent). - line="${line#"${line%%[![:space:]]*}"}" - [ -z "$line" ] && continue - [ "${line:0:1}" = "#" ] && continue - key="${line%%=*}" - value="${line#*=}" - export "$key"="$value" - docker_env_vars="$docker_env_vars -e $key" - echo "Forwarding extra env var: $key" - done <<< "$extra_env_vars" - fi - # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -419,7 +392,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.extra-env-vars }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" - name: Kill container on cancellation if: cancelled() From 4441f68129c0040674fc9ac4d830d7c604379595 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Mon, 8 Jun 2026 08:37:30 +0000 Subject: [PATCH 17/23] Revert "TEMP: skip docker/docs/install-ci CI while iterating #5823" This reverts commit c514199d41ac37de3e00295b0254cf6f2b5af9d4. --- .github/workflows/build.yaml | 7 +------ .github/workflows/docs.yaml | 15 ++------------- .github/workflows/install-ci.yml | 5 ----- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2428799e72d9..1c599ebcb988 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,12 +74,7 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - # TEMP (revert before final review / before landing): force - # run_docker_tests=false while iterating PR #5823. All gated - # test/build jobs skip via their existing if-gate; the - # single-GPU Docker + Tests matrix won't burn the gpu pool on - # every push. Per ~/.claude/skills/pr/ci-iteration-shortcut.md. - run_docker_tests: 'false' + run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} steps: - id: detect env: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 098d9ba784b0..013be3a5b126 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -25,11 +25,6 @@ jobs: doc-build-type: name: Detect Doc Build Type runs-on: ubuntu-latest - # DIAGNOSTIC PR opt-out: skip docs build entirely when the PR title is - # marked DO-NOT-MERGE (scratch / hypothesis-only branches don't need a - # full doc rebuild on every push). Push events on main/develop/release - # still run because they don't carry a PR title. - if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: trigger-deploy: ${{ steps.trigger-deploy.outputs.defined }} steps: @@ -47,14 +42,8 @@ jobs: name: Build Latest Docs runs-on: ubuntu-latest needs: [doc-build-type] - # Run on non-deploy branches to build current version docs only AND skip - # for DO-NOT-MERGE diagnostic PRs (the needed doc-build-type job already - # skips for those, but GitHub Actions evaluates `needs.X.outputs.Y` as - # empty when X is skipped, which keeps this `!= 'true'` gate true — so - # we re-check the title here to actually cascade the skip). - if: | - needs.doc-build-type.outputs.trigger-deploy != 'true' - && !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) + # run on non-deploy branches to build current version docs only + if: needs.doc-build-type.outputs.trigger-deploy != 'true' steps: - name: Checkout code diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index df3827bfb870..2a2fee50c8ab 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -34,11 +34,6 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest - # DIAGNOSTIC PR opt-out: skip install-ci entirely when the PR title is - # marked DO-NOT-MERGE. The downstream install-tests-x86 / install-tests-arm - # jobs gate on `needs.changes.outputs.run_install_tests == 'true'`; with - # this job skipped the output is empty and both dependents skip too. - if: ${{ !(github.event_name == 'pull_request' && contains(github.event.pull_request.title, 'DO-NOT-MERGE')) }} outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} steps: From c54f214fba1fa2d80b47f51a259d3f19eff6fd6d Mon Sep 17 00:00:00 2001 From: jichuanh Date: Thu, 11 Jun 2026 23:55:11 +0000 Subject: [PATCH 18/23] Add pure device-plan planner for the test runner Introduce tools/_device_plan.py: a pure planner that turns a runner's files plus its concrete runtime device mask into (file, mask) work units, splitting a device_isolated file into one single-device unit per device only when the runtime spans more than one device. Includes is_isolated() marker detection by source regex (no import). Covered by 14 unit tests. This is the first extraction of the multi-GPU / device-split runner refactor; nothing is wired into conftest yet, so runner behavior is unchanged. --- tools/_device_plan.py | 104 ++++++++++++++++++++++++++++++++++++++ tools/test_device_plan.py | 100 ++++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 tools/_device_plan.py create mode 100644 tools/test_device_plan.py diff --git a/tools/_device_plan.py b/tools/_device_plan.py new file mode 100644 index 000000000000..8fe3dc1c315c --- /dev/null +++ b/tools/_device_plan.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure planner for the per-file CI test runner. + +Turns a runner's claimed files plus its runtime device mask into a list of +``(file, mask)`` work units, where ``mask`` is the ``ISAACLAB_TEST_DEVICES`` +value the executor will set for that subprocess. A file is one unit unless it +declares the ``device_isolated`` marker and the runtime spans more than one +device, in which case it splits into one single-device unit per device (to work +around backends whose device mode is process-global, e.g. ovphysx). + +This module is pure: no I/O beyond reading a file's source for marker detection, +no subprocess, no pytest collection. It replaces the ``DEVICE_SPLIT_PASSES`` / +``is_device_split_file`` pieces of the former ``_device_split`` module. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path + +_ISOLATED_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_isolated\b", re.MULTILINE) +"""Match a module-level ``pytestmark`` assignment that mentions ``device_isolated``. + +Recognises both the single-mark and single-line list forms: + +* ``pytestmark = pytest.mark.device_isolated`` +* ``pytestmark = [pytest.mark.device_isolated, pytest.mark.slow]`` +""" + + +def is_isolated(path: Path | str, source: str | None = None) -> bool: + """Return whether a test file declares the ``device_isolated`` marker. + + Detection is by source regex, not import, so the planner stays collection-free. + + Args: + path: Filesystem path to the candidate test file. + source: Optional preloaded source text to inspect instead of reading + ``path``. + + Returns: + ``True`` when the file's module-level ``pytestmark`` mentions + ``device_isolated``; ``False`` otherwise (including a missing or + unreadable file). + """ + if source is None: + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return bool(_ISOLATED_MARK_RE.search(source)) + + +def _single_bit_mask(index: int, width: int) -> str: + """Return a width-``width`` device mask with only ``index`` set. + + Args: + index: The single device position to enable. + width: Total mask width. + + Returns: + A mask string such as ``"010"`` (``index=1``, ``width=3``). + """ + return "".join("1" if pos == index else "0" for pos in range(width)) + + +def plan_units( + files: list[str], + runtime_mask: str, + is_isolated: Callable[[Path | str], bool] = is_isolated, +) -> list[tuple[str, str]]: + """Plan the ``(file, mask)`` work units for one runner. + + Args: + files: Test files this runner is responsible for, in run order. + runtime_mask: The runner's concrete device mask (e.g. ``"110"`` on the + single-GPU lane, ``"0001"`` for an mgpu shard). Must not contain the + open-ended ``"X"`` form, which is a scope-only construct. + is_isolated: Predicate deciding whether a file needs one process per + device. Defaults to :func:`is_isolated` (marker detection by source). + + Returns: + Work units in run order. A non-isolated file, or any file on a + single-device runtime, yields one unit at ``runtime_mask``. An isolated + file on a multi-device runtime yields one single-device unit per set bit. + + Raises: + ValueError: When ``runtime_mask`` contains ``"X"``. + """ + if "X" in runtime_mask: + raise ValueError(f"runtime mask {runtime_mask!r} must be concrete (no 'X'); 'X' is a scope-only construct") + set_bits = [pos for pos, char in enumerate(runtime_mask) if char == "1"] + units: list[tuple[str, str]] = [] + for test_file in files: + if is_isolated(test_file) and len(set_bits) > 1: + units.extend((test_file, _single_bit_mask(pos, len(runtime_mask))) for pos in set_bits) + else: + units.append((test_file, runtime_mask)) + return units diff --git a/tools/test_device_plan.py b/tools/test_device_plan.py new file mode 100644 index 000000000000..a047a1769c11 --- /dev/null +++ b/tools/test_device_plan.py @@ -0,0 +1,100 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the pure test-run planner (``tools/_device_plan.py``). + +The planner turns (files, runtime device mask, marker predicate) into a list of +``(file, mask)`` work units. It is pure: no I/O, no subprocess, no collection. +""" + +from __future__ import annotations + +import pytest +from _device_plan import is_isolated, plan_units + +# ---- plan_units: the per-runner planning logic -------------------------------- + + +def test_mix_ok_file_is_a_single_unit(): + # A file that does not need device isolation runs once, covering the whole mask. + assert plan_units(["a.py"], "110", is_isolated=lambda f: False) == [("a.py", "110")] + + +def test_cant_mix_file_splits_into_one_unit_per_set_bit(): + # An isolated file on a multi-device runtime splits into one process per device. + assert plan_units(["a.py"], "110", is_isolated=lambda f: True) == [("a.py", "100"), ("a.py", "010")] + + +def test_cant_mix_on_single_device_runtime_does_not_split(): + # An mgpu shard's runtime is one device, so even an isolated file is one unit. + assert plan_units(["a.py"], "0001", is_isolated=lambda f: True) == [("a.py", "0001")] + + +def test_cant_mix_on_cpu_only_runtime_does_not_split(): + assert plan_units(["a.py"], "100", is_isolated=lambda f: True) == [("a.py", "100")] + + +def test_split_masks_use_each_set_bit_position(): + # Set bits at positions 0 and 2 -> two single-bit masks of the same width. + assert plan_units(["a.py"], "1010", is_isolated=lambda f: True) == [("a.py", "1000"), ("a.py", "0010")] + + +def test_multiple_files_preserve_order_and_decide_per_file(): + def isolated(f): + return f == "b.py" + + assert plan_units(["a.py", "b.py"], "110", is_isolated=isolated) == [ + ("a.py", "110"), + ("b.py", "100"), + ("b.py", "010"), + ] + + +def test_runtime_mask_with_trailing_x_is_rejected(): + # Runtime masks are always concrete; the open-ended ``X`` form is a scope-only + # construct and must never reach the planner. + with pytest.raises(ValueError): + plan_units(["a.py"], "11X", is_isolated=lambda f: True) + + +# ---- is_isolated: the device-isolation marker detector ------------------------ + + +def test_single_mark_is_detected(tmp_path): + f = tmp_path / "t.py" + f.write_text("import pytest\npytestmark = pytest.mark.device_isolated\ndef test_x(): pass\n") + assert is_isolated(f) is True + + +def test_list_form_mark_is_detected(tmp_path): + f = tmp_path / "t.py" + f.write_text("import pytest\npytestmark = [pytest.mark.device_isolated, pytest.mark.slow]\ndef test_x(): pass\n") + assert is_isolated(f) is True + + +def test_preloaded_source_is_used(): + assert is_isolated("does_not_exist.py", source="pytestmark = pytest.mark.device_isolated\n") is True + + +def test_no_mark_is_not_detected(tmp_path): + f = tmp_path / "t.py" + f.write_text("import pytest\ndef test_x(): pass\n") + assert is_isolated(f) is False + + +def test_word_in_comment_does_not_match(tmp_path): + f = tmp_path / "t.py" + f.write_text("# device_isolated explains the lock\ndef test_x(): pass\n") + assert is_isolated(f) is False + + +def test_unrelated_pytestmark_does_not_match(tmp_path): + f = tmp_path / "t.py" + f.write_text("import pytest\npytestmark = pytest.mark.slow\ndef test_x(): pass\n") + assert is_isolated(f) is False + + +def test_missing_file_returns_false(tmp_path): + assert is_isolated(tmp_path / "nope.py") is False From 08aceb0b2a510e527571168bd4a46f5f9a9923ec Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 12 Jun 2026 00:10:48 +0000 Subject: [PATCH 19/23] Drive the test runner from planner + executor; unify device selection Replace the device_split (-k) and mgpu_shard_select mechanisms with one model: the planner emits (file, mask) units and the executor sets the unit's mask as ISAACLAB_TEST_DEVICES so test_devices() does all device-variant selection. A device_isolated file (renamed from device_split) splits into one single-device unit per device only when the runtime spans more than one; an mgpu shard, with a single-device runtime, runs each file once. - Add tools/_device_exec.py: the executor (subprocess lifecycle, timeouts, retries, JUnit parsing) plus run_unit(); conftest's run_individual_tests is now a thin planner+executor driver. - Add tools/_agnostic_select.py: a dumb plugin that drops device-agnostic tests, injected by the executor only for cpu-less masks (shards and the cuda unit of a split). Replaces mgpu_shard_select's agnostic-drop; its variant filtering is now handled by test_devices()+mask. - Remove tools/_device_split.py (DEVICE_SPLIT_PASSES, -k) and .github/actions/multi-gpu/mgpu_shard_select.py. - Rename the device_split marker to device_isolated across the ovphysx tests and the pytest marker registration. Behavior is preserved: the same tests run on the same devices on both lanes; the single-GPU lane still uses no Kit GPU pin and mgpu still pins via the shell. --- .../actions/multi-gpu/mgpu_shard_select.py | 93 -- .github/workflows/test-multi-gpu-pytest.yaml | 7 +- pyproject.toml | 2 +- .../test/assets/test_articulation.py | 2 +- .../test/assets/test_rigid_object.py | 2 +- .../assets/test_rigid_object_collection.py | 2 +- .../test/sensors/test_contact_sensor.py | 2 +- .../test/sim/test_views_xform_prim_ovphysx.py | 2 +- tools/_agnostic_select.py | 31 + tools/_device_exec.py | 754 +++++++++++++++++ tools/_device_plan.py | 30 +- tools/_device_split.py | 65 -- tools/conftest.py | 797 +----------------- tools/test_agnostic_select.py | 91 ++ tools/test_device_plan.py | 25 +- tools/test_device_split.py | 108 --- 16 files changed, 956 insertions(+), 1057 deletions(-) delete mode 100644 .github/actions/multi-gpu/mgpu_shard_select.py create mode 100644 tools/_agnostic_select.py create mode 100644 tools/_device_exec.py delete mode 100644 tools/_device_split.py create mode 100644 tools/test_agnostic_select.py delete mode 100644 tools/test_device_split.py diff --git a/.github/actions/multi-gpu/mgpu_shard_select.py b/.github/actions/multi-gpu/mgpu_shard_select.py deleted file mode 100644 index 22f784435bb3..000000000000 --- a/.github/actions/multi-gpu/mgpu_shard_select.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""pytest plugin: select which tests a non-default GPU shard runs. - -Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via -``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this -directory to ``PYTHONPATH`` so it is importable). It lives next to the lane -scripts rather than as a repo-root ``conftest.py`` so it only affects the lane. - -Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME -env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's -device parametrization, so this plugin's keep/drop decision agrees with what was -parametrized. (``ISAACLAB_SIM_DEVICE`` is AppLauncher's Kit boot device, a -different concern, and is deliberately NOT used here.) The plugin reads the mask -string directly -- no isaaclab import -- matching the lib-free convention of every -other conftest in the repo. The only shared knowledge is the mask grammar below, -mirrored from ``devices.py`` (position 0 = cpu, position k = cuda:(k-1), trailing -``X`` = include all remaining); keep the two in sync. - -Selection rule, on a shard whose mask excludes cpu + cuda:0: - - keep a test iff it is parametrized over ``device`` AND that variant's device - is active in the mask; deselect everything else. - -That single rule is both criteria at once: candidate = device-parametrized (a -test with no ``device`` param is device-agnostic, covered by single-GPU CI on -cuda:0); scope = only this shard's device variant runs (cpu / cuda:0 / other-index -variants are dropped). Out-of-scope items are deselected (not skipped) so the -shard report shows only what ran. -""" - -import os - -import pytest - -_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" - - -def _position(device): - """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" - if device == "cpu": - return 0 - if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): - return int(device[len("cuda:") :]) + 1 - return None # not device-parametrized, or an unrecognized device value - - -def _active(position, mask): - """Whether a mask position is included. Trailing ``X`` includes all remaining.""" - if position is None: - return False - body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) - return body[position] == "1" if 0 <= position < len(body) else fill - - -def _shard_mask(): - """Return the runtime mask iff this is a multi-GPU shard run, else None. - - A shard targets non-default GPUs only, so cpu (position 0) and cuda:0 - (position 1) are excluded. Anything else -- unset, or single-GPU CI's default - ``"110"`` -- is not a shard and the plugin is a no-op. - """ - mask = os.environ.get(_RUNTIME_ENV, "") - return mask if mask and not _active(0, mask) and not _active(1, mask) else None - - -def pytest_collection_modifyitems(config, items): - mask = _shard_mask() - if mask is None: - return - keep, drop = [], [] - for item in items: - callspec = getattr(item, "callspec", None) - device = callspec.params.get("device") if callspec is not None else None - if _active(_position(device), mask): - keep.append(item) # device-parametrized AND in this shard's scope - else: - drop.append(item) # non-device, or cpu / cuda:0 / other-index variant - if drop: - items[:] = keep - config.hook.pytest_deselected(items=drop) - - -def pytest_sessionfinish(session, exitstatus): - # A file whose tests are all out of scope deselects to zero, so pytest exits - # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file - # exit as a failure (tools/conftest.py), so report "nothing in scope for this - # file" as success rather than a false failure. - if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = pytest.ExitCode.OK diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 2ea9a86abace..b1a0973e70bb 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -115,9 +115,10 @@ jobs: # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the - # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``): - # single-GPU CI already covers them on ``cuda:0`` and re-running on every - # non-default shard adds wall-time without surfacing any new failure mode. + # ``_agnostic_select`` plugin (injected by ``tools/conftest.py`` for any + # unit whose mask lacks cpu, i.e. every shard): single-GPU CI already + # covers them on ``cuda:0`` and re-running on every non-default shard adds + # wall-time without surfacing any new failure mode. id: discover run: | # File-level opt-out: a test file can exclude itself from multi-GPU CI diff --git a/pyproject.toml b/pyproject.toml index a4a31197858c..bd4fe1a2f64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty,segway" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", - "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", + "device_isolated: this file's backend holds a process-global device lock (e.g. ovphysx<=0.3.7 gap G5); the CI runner runs it once per device instead of mixing devices in one process", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index 9586ff0a01c5..b14b416cf5bc 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -84,7 +84,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _OMNI_PHYSX_SCHEMAS_GAP_REASON = ( diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 2e8833114000..9e875d08fb69 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -57,7 +57,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _logger = logging.getLogger(__name__) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 138a4cbf18da..0839efa6802f 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -54,7 +54,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _LOCKED_DEVICE: list[str | None] = [None] diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index eae32ddb82c4..72d9c546c71f 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -63,7 +63,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated # --------------------------------------------------------------------------- # Device-lock autouse fixture diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index e606ce7738b9..1b80ace67583 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -24,7 +24,7 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) diff --git a/tools/_agnostic_select.py b/tools/_agnostic_select.py new file mode 100644 index 000000000000..1aef4518b57c --- /dev/null +++ b/tools/_agnostic_select.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""pytest plugin: drop device-agnostic tests from this run. + +Deselects every collected test that is not parametrized over ``device``. It +carries no mask logic of its own. The executor (``tools/conftest.py``) injects +it via ``-p _agnostic_select`` only for a unit whose mask lacks cpu — i.e. an +mgpu shard, or the cuda unit of a can't-mix split — because in those units a +device-agnostic test would either run redundantly on every shard or run a second +time alongside the cpu unit. Device variant selection itself is handled upstream +by ``test_devices()`` reading the unit's ``ISAACLAB_TEST_DEVICES`` mask, so this +plugin only has to remove the paramless remainder. +""" + +from __future__ import annotations + + +def _has_device_param(item) -> bool: + """Return whether a collected item is parametrized over ``device``.""" + callspec = getattr(item, "callspec", None) + return callspec is not None and "device" in callspec.params + + +def pytest_collection_modifyitems(config, items): + drop = [item for item in items if not _has_device_param(item)] + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = [item for item in items if _has_device_param(item)] diff --git a/tools/_device_exec.py b/tools/_device_exec.py new file mode 100644 index 000000000000..4e3a935d6de8 --- /dev/null +++ b/tools/_device_exec.py @@ -0,0 +1,754 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Executor for the per-file CI test runner. + +Runs one :class:`~_device_plan.Unit` as its own pytest subprocess and owns the +subprocess lifecycle: streaming capture, startup-hang / hard-timeout / +shutdown-hang detection, fresh-process retries, and JUnit report parsing. + +Device selection is the planner's job, expressed as the unit's mask: the +executor sets ``ISAACLAB_TEST_DEVICES`` to that mask (``test_devices()`` reads +it to pick the device variants) and, for a unit whose mask lacks cpu — an mgpu +shard or the cuda unit of a can't-mix split — injects the ``_agnostic_select`` +plugin so device-agnostic tests do not run there. The executor never reads a +marker or a ``-k`` selector. +""" + +from __future__ import annotations + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time +from dataclasses import dataclass + +from junitparser import Error, JUnitXml, TestCase, TestSuite + +from _device_plan import Unit # isort: skip + +COLD_CACHE_BUFFER = 700 +"""Extra seconds added to the first camera-enabled test's hard timeout. + +The first test that uses ``enable_cameras=True`` may compile shaders during its +run (~600 s). This buffer prevents that from being misreported as a test +timeout. Only the first such test gets the extension — after it runs, the +on-disk cache is populated. +""" + +STARTUP_DEADLINE = 120 +"""Seconds to wait for AppLauncher init or pytest collection before declaring a +startup hang. + +AppLauncher prints ``[ISAACLAB] AppLauncher initialization complete`` to +``sys.__stderr__`` (never suppressed) when Kit finishes initializing, and pytest +prints ``collected N items`` to stdout after collection. If neither appears +within this deadline the process is treated as hung. Kit startup can exceed +60 s on cold CI workers, so this catches real startup hangs without killing +legitimate slow launches. +""" + +STARTUP_HANG_RETRIES = 2 +"""Number of times to retry a test that hangs during startup before giving up.""" + +TIMEOUT_RETRIES = 0 +"""Number of times to retry a test that reaches its hard timeout before giving up.""" + +PROCESS_FAILURE_RETRIES_BY_FILE = { + "test_visualizer_integration_physx.py": 4, + "test_visualizer_integration_newton.py": 4, + "test_visualizer_tiled_integration_physx.py": 4, + "test_visualizer_tiled_integration_newton.py": 4, +} +"""Extra fresh-process attempts for visualizer tests that can enter stale render states.""" + +SHUTDOWN_GRACE_PERIOD = 30 +"""Seconds to wait for clean exit after the JUnit XML report file appears. + +When a test completes and writes its JUnit report, the subprocess may hang +during ``SimulationApp.close()`` or Kit shutdown. Rather than wasting the +full hard timeout, we give the process a short grace period to exit, then +kill it. The test results are taken from the report file (pass/fail), not +from the kill. +""" + + +def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): + """Run a command with timeout and capture all output while streaming in real-time. + + Args: + cmd: Command to execute. + timeout: Maximum wall-clock seconds before the process is killed. + env: Environment variables for the subprocess. + startup_deadline: If > 0, the process is killed early when neither + ``AppLauncher initialization complete`` (stderr) nor ``collected`` + (stdout) appears within this many seconds. + report_file: Path to the JUnit XML report file. When set, the process + is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after + the file appears on disk. + + Returns: + Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, + wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, + ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process + did not reach pytest collection in time, or ``"shutdown_hang"`` when + the test completed but the process hung during shutdown. + """ + stdout_data = b"" + stderr_data = b"" + process = None + + try: + # Each test gets its own session so orphaned Kit/Isaac Sim child + # processes cannot send SIGHUP to the next test's process group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + + while process.poll() is None: + elapsed = time.time() - start_time + + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + SHUTDOWN_GRACE_PERIOD + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = _capture_system_diagnostics() + + # Kill the entire process group (test + any Kit children). + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=5) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except subprocess.TimeoutExpired: + pass + wall_time = time.time() - start_time + return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + # Drain any output the process wrote before or just after exiting. + try: + remaining_stdout, remaining_stderr = process.communicate(timeout=10) + stdout_data += remaining_stdout + stderr_data += remaining_stderr + except Exception: + pass + + # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. + try: + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + + wall_time = time.time() - start_time + return process.returncode, stdout_data, stderr_data, "", wall_time, "" + + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +_SIGNAL_DESCRIPTIONS = { + 1: "SIGHUP — session leader exit or orphaned process cleanup", + 6: "SIGABRT", + 9: "SIGKILL — likely OOM killed", + 11: "SIGSEGV — segmentation fault", + 15: "SIGTERM", +} + + +def _signal_description(sig): + """Return a human-readable description for a process killed by a signal.""" + base = f"Process killed by signal {sig}" + desc = _SIGNAL_DESCRIPTIONS.get(sig) + return f"{base} ({desc})" if desc else base + + +def _create_error_report(prefix, file_name, message, details): + """Create a JUnit XML error report for a test that failed to produce its own. + + Returns a :class:`JUnitXml` object ready to be written to disk. + """ + suite_name = os.path.splitext(file_name)[0] + suite = TestSuite(name=f"{prefix}_{suite_name}") + case = TestCase(name="test_execution", classname=suite_name) + error = Error(message=message) + error.text = details + case.result = error + suite.add_testcase(case) + report = JUnitXml() + report.add_testsuite(suite) + return report + + +def _get_diagnostics(pre_kill_diag=""): + """Return system diagnostics, truncated to 10 000 chars.""" + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + return diag + + +def _capture_system_diagnostics(): + """Capture system diagnostics (GPU, memory, processes) for crash investigation. + + All errors are caught and reported inline so this never raises. + """ + sections = [] + + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + + return "\n\n".join(sections) + + +def _read_test_report(report_file, file_name): + """Read a pytest JUnit report and return its summary fields.""" + report = JUnitXml.fromfile(report_file) + for suite in report: + if suite.name == "pytest": + suite.name = os.path.splitext(file_name)[0] + report.write(report_file) + + errors = int(report.errors) if report.errors is not None else 0 + failures = int(report.failures) if report.failures is not None else 0 + skipped = int(report.skipped) if report.skipped is not None else 0 + tests = int(report.tests) if report.tests is not None else 0 + time_elapsed = float(report.time) if report.time is not None else 0.0 + return report, errors, failures, skipped, tests, time_elapsed + + +def _retry_failed_test_in_fresh_process( + *, + test_file, + file_name, + cmd, + timeout, + env, + startup_deadline, + report_file, + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, +): + """Retry selected failed test files in a fresh subprocess.""" + has_test_failures = errors > 0 or failures > 0 + process_failure_attempts = 0 + max_process_failure_retries = PROCESS_FAILURE_RETRIES_BY_FILE.get(file_name, 0) + + while has_test_failures and process_failure_attempts < max_process_failure_retries: + process_failure_attempts += 1 + print( + f"⚠️ {test_file}: failed in subprocess" + f" (attempt {process_failure_attempts}/{max_process_failure_retries + 1}), retrying in fresh process..." + ) + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file + ) + if not os.path.exists(report_file): + break + + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) + has_test_failures = errors > 0 or failures > 0 + except Exception as e: + print(f"Error reading retry test report {report_file}: {e}") + has_test_failures = True + errors = 1 + failures = 0 + skipped = 0 + tests = 0 + time_elapsed = 0.0 + break + + return ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) + + +@dataclass +class _UnitContext: + """Per-file inputs shared across the units the runner drives for one file. + + Attributes: + test_file: Path to the test file being driven. + file_name: Basename of ``test_file`` (used for JUnit naming). + workspace_root: Repository root; passed to pytest's ``--config-file`` and + used to locate the ``_agnostic_select`` plugin. + isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the + ``-m isaacsim_ci`` selector. + timeout: Per-unit hard timeout in seconds. + startup_deadline: Per-unit startup-hang deadline in seconds. + env: Base environment for the pytest subprocess; :func:`run_unit` copies + it and adds the per-unit ``ISAACLAB_TEST_DEVICES`` mask. + """ + + test_file: str + file_name: str + workspace_root: str + isaacsim_ci: bool + timeout: int + startup_deadline: int + env: dict + + +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} + + +def merge_unit_status(prev: dict | None, new: dict) -> dict: + """Merge per-unit status dicts into a single per-file entry. + + Counters (``errors``, ``failures``, ``skipped``, ``tests``, + ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more + severe of the two via :data:`_RESULT_PRIORITY`. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + +def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``unit`` and return its results. + + Args: + ctx: Static per-file context (paths, timeouts, base env). + unit: The work unit to run; its mask becomes ``ISAACLAB_TEST_DEVICES``. + + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the unit produced + no report (e.g. startup hang). + * ``status_dict``: per-unit counters compatible with the entries + appended to ``test_status``. + * ``was_failure``: whether the unit should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + # Split units of one file share a slug, so suffix the report with the mask to + # keep both XMLs; a non-split unit (one per file) keeps the suffix-free name + # the mgpu aggregator unslugs back to a path. + report_suffix = f"-{unit.mask}" if unit.split else "" + pass_file_label = f"{ctx.file_name}{report_suffix}" + # Slug the full test path (not just the basename) into the report filename so + # two concurrent shards running same-basename files (e.g. + # ``isaaclab_newton/.../test_articulation.py`` vs + # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path + # inside the shared ``/workspace/isaaclab`` mount and trigger false + # shutdown_hang detections in sibling shards via the report-file existence check. + report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") + report_file = f"tests/test-reports-{report_slug}{report_suffix}.xml" + + # The mask drives device-variant selection through test_devices(); no -k needed. + env = dict(ctx.env) + env["ISAACLAB_TEST_DEVICES"] = unit.mask + + # A mask without cpu (position 0) — an mgpu shard or the cuda unit of a split — + # must not run device-agnostic tests. Inject the dumb drop-paramless plugin and + # put tools/ on PYTHONPATH so the per-file subprocess can import it. + inject_agnostic = unit.mask[:1] == "0" + if inject_agnostic: + tools_dir = os.path.join(ctx.workspace_root, "tools") + env["PYTHONPATH"] = tools_dir + os.pathsep + env.get("PYTHONPATH", "") + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit + "--no-header", + f"--config-file={ctx.workspace_root}/pyproject.toml", + f"--junitxml={report_file}", + "--tb=short", + ] + if inject_agnostic: + cmd += ["-p", "_agnostic_select"] + if ctx.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + cmd.append(str(ctx.test_file)) + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) + + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {ctx.test_file}{report_suffix}: startup hang detected after {ctx.startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {ctx.test_file}{report_suffix}: timeout detected after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) + if stdout_data: + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + + # -- Resolve result from kill_reason and report file ---------------- + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {ctx.test_file}{report_suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + print(diag) + + msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 1, + "result": "STARTUP_HANG", + "time_elapsed": 0.0, + "wall_time": wall_time, + }, + True, + ) + + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {ctx.test_file}{report_suffix} timed out after {ctx.timeout} seconds...") + print(diag) + + msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 1, + "result": "TIMEOUT", + "time_elapsed": ctx.timeout, + "wall_time": wall_time, + }, + True, + ) + + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {ctx.test_file}{report_suffix}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", pass_file_label, reason, details) + error_report.write(report_file) + return ( + error_report, + { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 1, + "result": "CRASHED", + "time_elapsed": 0.0, + "wall_time": wall_time, + }, + True, + ) + + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {ctx.test_file}{report_suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return ( + None, + { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 0, + "result": "FAILED", + "time_elapsed": 0.0, + "wall_time": wall_time, + }, + True, + ) + + ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) = _retry_failed_test_in_fresh_process( + test_file=ctx.test_file, + file_name=ctx.file_name, + cmd=cmd, + timeout=ctx.timeout, + env=env, + startup_deadline=ctx.startup_deadline, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout_data=stdout_data, + stderr_data=stderr_data, + kill_reason=kill_reason, + wall_time=wall_time, + pre_kill_diag=pre_kill_diag, + ) + + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) + + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" + + return ( + report, + { + "errors": errors, + "failures": failures, + "skipped": skipped, + "tests": tests, + "result": result, + "time_elapsed": time_elapsed, + "wall_time": wall_time, + }, + was_failure, + ) diff --git a/tools/_device_plan.py b/tools/_device_plan.py index 8fe3dc1c315c..15086cc67028 100644 --- a/tools/_device_plan.py +++ b/tools/_device_plan.py @@ -21,8 +21,28 @@ import re from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path + +@dataclass(frozen=True) +class Unit: + """One pytest subprocess to run for the test runner. + + Attributes: + file: Test file to run. + mask: Device mask set as ``ISAACLAB_TEST_DEVICES`` for the subprocess; + ``test_devices()`` reads it to select the device variants. + split: Whether this unit is one of several produced by splitting a + ``device_isolated`` file. The executor uses it only to give split + units distinct JUnit report filenames; a non-split unit (one per + file) keeps the suffix-free name the mgpu aggregator expects. + """ + + file: str + mask: str + split: bool = False + _ISOLATED_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_isolated\b", re.MULTILINE) """Match a module-level ``pytestmark`` assignment that mentions ``device_isolated``. @@ -73,8 +93,8 @@ def plan_units( files: list[str], runtime_mask: str, is_isolated: Callable[[Path | str], bool] = is_isolated, -) -> list[tuple[str, str]]: - """Plan the ``(file, mask)`` work units for one runner. +) -> list[Unit]: + """Plan the work units for one runner. Args: files: Test files this runner is responsible for, in run order. @@ -95,10 +115,10 @@ def plan_units( if "X" in runtime_mask: raise ValueError(f"runtime mask {runtime_mask!r} must be concrete (no 'X'); 'X' is a scope-only construct") set_bits = [pos for pos, char in enumerate(runtime_mask) if char == "1"] - units: list[tuple[str, str]] = [] + units: list[Unit] = [] for test_file in files: if is_isolated(test_file) and len(set_bits) > 1: - units.extend((test_file, _single_bit_mask(pos, len(runtime_mask))) for pos in set_bits) + units.extend(Unit(test_file, _single_bit_mask(pos, len(runtime_mask)), split=True) for pos in set_bits) else: - units.append((test_file, runtime_mask)) + units.append(Unit(test_file, runtime_mask)) return units diff --git a/tools/_device_split.py b/tools/_device_split.py deleted file mode 100644 index e8bad4069133..000000000000 --- a/tools/_device_split.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helpers for detecting and driving the ``device_split`` pytest marker. - -Test files that declare ``pytestmark = pytest.mark.device_split`` at module -scope must be re-invoked once per device (CPU and GPU) in separate processes -to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. -The :func:`is_device_split_file` predicate lets the per-file CI runner in -``tools/conftest.py`` detect this without importing the test module. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) -"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. - -Recognises both single-mark and single-line list forms: - -* ``pytestmark = pytest.mark.device_split`` -* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` - -Multi-line list forms are not supported (currently no test file uses one); if -a future test needs that, expand the parsing rule. -""" - -# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file -# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: -# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. -# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps -# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized -# tests only. -DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ - ("-cpu", "cpu or not cuda"), - ("-cuda", "cuda"), -] - - -def is_device_split_file(path: Path | str, source: str | None = None) -> bool: - """Return whether the test file at ``path`` declares the ``device_split`` marker. - - Matches :data:`_DEVICE_SPLIT_MARK_RE` against ``source`` when supplied. - Otherwise, reads the file source from ``path``. A missing or unreadable - file returns ``False`` so the caller falls back to the default single-pass - invocation. - - Args: - path: Filesystem path to a candidate test file. - source: Optional preloaded source text to inspect. - - Returns: - ``True`` when the file's module-level ``pytestmark`` mentions - ``device_split``; ``False`` otherwise. - """ - if source is None: - try: - source = Path(path).read_text(encoding="utf-8", errors="replace") - except OSError: - return False - return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index 57c56c15fe51..49e19bf91f9b 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -6,20 +6,15 @@ import contextlib import os import re -import select -import signal -import subprocess -import sys -import time -from dataclasses import dataclass import pytest -from junitparser import Error, JUnitXml, TestCase, TestSuite +from junitparser import JUnitXml from prettytable import PrettyTable # Local imports import test_settings as test_settings # isort: skip -from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip +from _device_exec import COLD_CACHE_BUFFER, STARTUP_DEADLINE, _UnitContext, merge_unit_status, run_unit # isort: skip +from _device_plan import is_isolated as _is_isolated, plan_units # isort: skip def pytest_ignore_collect(collection_path, config): @@ -27,294 +22,6 @@ def pytest_ignore_collect(collection_path, config): return True -COLD_CACHE_BUFFER = 700 -"""Extra seconds added to the first camera-enabled test's hard timeout. - -The first test that uses ``enable_cameras=True`` may compile shaders during its -run (~600 s). This buffer prevents that from being misreported as a test -timeout. Only the first such test gets the extension — after it runs, the -on-disk cache is populated. -""" - -STARTUP_DEADLINE = 120 -"""Seconds to wait for AppLauncher init or pytest collection before declaring a -startup hang. - -AppLauncher prints ``[ISAACLAB] AppLauncher initialization complete`` to -``sys.__stderr__`` (never suppressed) when Kit finishes initializing, and pytest -prints ``collected N items`` to stdout after collection. If neither appears -within this deadline the process is treated as hung. Kit startup can exceed -60 s on cold CI workers, so this catches real startup hangs without killing -legitimate slow launches. -""" - -STARTUP_HANG_RETRIES = 2 -"""Number of times to retry a test that hangs during startup before giving up.""" - -TIMEOUT_RETRIES = 0 -"""Number of times to retry a test that reaches its hard timeout before giving up.""" - -PROCESS_FAILURE_RETRIES_BY_FILE = { - "test_visualizer_integration_physx.py": 4, - "test_visualizer_integration_newton.py": 4, - "test_visualizer_tiled_integration_physx.py": 4, - "test_visualizer_tiled_integration_newton.py": 4, -} -"""Extra fresh-process attempts for visualizer tests that can enter stale render states.""" - -SHUTDOWN_GRACE_PERIOD = 30 -"""Seconds to wait for clean exit after the JUnit XML report file appears. - -When a test completes and writes its JUnit report, the subprocess may hang -during ``SimulationApp.close()`` or Kit shutdown. Rather than wasting the -full hard timeout, we give the process a short grace period to exit, then -kill it. The test results are taken from the report file (pass/fail), not -from the kill. -""" - - -def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): - """Run a command with timeout and capture all output while streaming in real-time. - - Args: - cmd: Command to execute. - timeout: Maximum wall-clock seconds before the process is killed. - env: Environment variables for the subprocess. - startup_deadline: If > 0, the process is killed early when neither - ``AppLauncher initialization complete`` (stderr) nor ``collected`` - (stdout) appears within this many seconds. - report_file: Path to the JUnit XML report file. When set, the process - is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after - the file appears on disk. - - Returns: - Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, - wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, - ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process - did not reach pytest collection in time, or ``"shutdown_hang"`` when - the test completed but the process hung during shutdown. - """ - stdout_data = b"" - stderr_data = b"" - process = None - - try: - # Each test gets its own session so orphaned Kit/Isaac Sim child - # processes cannot send SIGHUP to the next test's process group. - process = subprocess.Popen( - cmd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0, - universal_newlines=False, - start_new_session=True, - ) - pgid = os.getpgid(process.pid) - - stdout_fd = process.stdout.fileno() - stderr_fd = process.stderr.fileno() - - try: - import fcntl - - for fd in [stdout_fd, stderr_fd]: - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - except ImportError: - pass - - start_time = time.time() - startup_done = startup_deadline <= 0 - shutdown_deadline = 0.0 - - while process.poll() is None: - elapsed = time.time() - start_time - - if not startup_done: - if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: - startup_done = True - - if report_file and not shutdown_deadline and os.path.exists(report_file): - shutdown_deadline = time.time() + SHUTDOWN_GRACE_PERIOD - - kill_reason = None - if not startup_done and elapsed > startup_deadline: - kill_reason = "startup_hang" - elif shutdown_deadline and time.time() > shutdown_deadline: - kill_reason = "shutdown_hang" - elif elapsed > timeout: - kill_reason = "timeout" - - if kill_reason: - pre_kill_diag = _capture_system_diagnostics() - - # Kill the entire process group (test + any Kit children). - try: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - process.kill() - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=5) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except subprocess.TimeoutExpired: - pass - wall_time = time.time() - start_time - return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag - - try: - ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) - - for fd in ready_fds: - with contextlib.suppress(OSError): - if fd == stdout_fd: - chunk = process.stdout.read(1024) - if chunk: - stdout_data += chunk - sys.stdout.buffer.write(chunk) - sys.stdout.buffer.flush() - elif fd == stderr_fd: - chunk = process.stderr.read(1024) - if chunk: - stderr_data += chunk - sys.stderr.buffer.write(chunk) - sys.stderr.buffer.flush() - except OSError: - time.sleep(0.1) - continue - - # Drain any output the process wrote before or just after exiting. - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=10) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except Exception: - pass - - # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. - try: - os.killpg(pgid, signal.SIGKILL) - time.sleep(1) - except (ProcessLookupError, PermissionError, OSError): - pass - - wall_time = time.time() - start_time - return process.returncode, stdout_data, stderr_data, "", wall_time, "" - - except Exception as e: - if process is not None and process.poll() is None: - process.kill() - with contextlib.suppress(Exception): - rem_out, rem_err = process.communicate(timeout=5) - stdout_data += rem_out - stderr_data += rem_err - stdout_data += f"\n[capture error: {e}]\n".encode() - return -1, stdout_data, stderr_data, "", 0.0, "" - - -_SIGNAL_DESCRIPTIONS = { - 1: "SIGHUP — session leader exit or orphaned process cleanup", - 6: "SIGABRT", - 9: "SIGKILL — likely OOM killed", - 11: "SIGSEGV — segmentation fault", - 15: "SIGTERM", -} - - -def _signal_description(sig): - """Return a human-readable description for a process killed by a signal.""" - base = f"Process killed by signal {sig}" - desc = _SIGNAL_DESCRIPTIONS.get(sig) - return f"{base} ({desc})" if desc else base - - -def _create_error_report(prefix, file_name, message, details): - """Create a JUnit XML error report for a test that failed to produce its own. - - Returns a :class:`JUnitXml` object ready to be written to disk. - """ - suite_name = os.path.splitext(file_name)[0] - suite = TestSuite(name=f"{prefix}_{suite_name}") - case = TestCase(name="test_execution", classname=suite_name) - error = Error(message=message) - error.text = details - case.result = error - suite.add_testcase(case) - report = JUnitXml() - report.add_testsuite(suite) - return report - - -def _get_diagnostics(pre_kill_diag=""): - """Return system diagnostics, truncated to 10 000 chars.""" - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - return diag - - -def _capture_system_diagnostics(): - """Capture system diagnostics (GPU, memory, processes) for crash investigation. - - All errors are caught and reported inline so this never raises. - """ - sections = [] - - try: - r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) - if r.stdout: - sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- nvidia-smi --- FAILED: {e}") - - try: - with open("/proc/meminfo") as f: - lines = f.readlines() - keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") - relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] - if relevant: - sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) - except Exception as e: - sections.append(f"--- /proc/meminfo --- FAILED: {e}") - - cgroup_lines = [] - for path in ( - "/sys/fs/cgroup/memory.current", - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory.events", - "/sys/fs/cgroup/memory/memory.usage_in_bytes", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - "/sys/fs/cgroup/memory/memory.oom_control", - ): - try: - with open(path) as f: - cgroup_lines.append(f"{path}: {f.read().strip()}") - except FileNotFoundError: - pass - except Exception as e: - cgroup_lines.append(f"{path}: FAILED ({e})") - if cgroup_lines: - sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) - - try: - r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) - if r.stdout: - sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- process tree --- FAILED: {e}") - - try: - r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) - if r.stdout: - lines = r.stdout.strip().split("\n") - sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) - except Exception: - pass - - return "\n\n".join(sections) - - def _slugify_test_path(test_path): """Encode a test path as a flat queue entry name. @@ -411,442 +118,16 @@ def _queued_files(queue_dir): yield claimed -def _read_test_report(report_file, file_name): - """Read a pytest JUnit report and return its summary fields.""" - report = JUnitXml.fromfile(report_file) - for suite in report: - if suite.name == "pytest": - suite.name = os.path.splitext(file_name)[0] - report.write(report_file) - - errors = int(report.errors) if report.errors is not None else 0 - failures = int(report.failures) if report.failures is not None else 0 - skipped = int(report.skipped) if report.skipped is not None else 0 - tests = int(report.tests) if report.tests is not None else 0 - time_elapsed = float(report.time) if report.time is not None else 0.0 - return report, errors, failures, skipped, tests, time_elapsed - - -def _retry_failed_test_in_fresh_process( - *, - test_file, - file_name, - cmd, - timeout, - env, - startup_deadline, - report_file, - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, -): - """Retry selected failed test files in a fresh subprocess.""" - has_test_failures = errors > 0 or failures > 0 - process_failure_attempts = 0 - max_process_failure_retries = PROCESS_FAILURE_RETRIES_BY_FILE.get(file_name, 0) - - while has_test_failures and process_failure_attempts < max_process_failure_retries: - process_failure_attempts += 1 - print( - f"⚠️ {test_file}: failed in subprocess" - f" (attempt {process_failure_attempts}/{max_process_failure_retries + 1}), retrying in fresh process..." - ) - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - if not os.path.exists(report_file): - break - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - has_test_failures = errors > 0 or failures > 0 - except Exception as e: - print(f"Error reading retry test report {report_file}: {e}") - has_test_failures = True - errors = 1 - failures = 0 - skipped = 0 - tests = 0 - time_elapsed = 0.0 - break - - return ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) - - -@dataclass -class _PassContext: - """Inputs shared across all pytest invocations for a single test file. - - Attributes: - test_file: Absolute path to the test file being driven. - file_name: Basename of ``test_file`` (used for JUnit naming). - workspace_root: Repository root; passed to pytest's ``--config-file``. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. - timeout: Per-pass hard timeout in seconds. - startup_deadline: Per-pass startup-hang deadline in seconds. - env: Environment passed to the pytest subprocess. - inject_shard_select: Whether to load the multi-GPU ``mgpu_shard_select`` - plugin in the pytest subprocess; true only on a non-default-GPU shard. - """ - - test_file: str - file_name: str - workspace_root: str - isaacsim_ci: bool - timeout: int - startup_deadline: int - env: dict - inject_shard_select: bool - - -_RESULT_PRIORITY = { - "STARTUP_HANG": 5, - "CRASHED": 4, - "TIMEOUT": 3, - "FAILED": 2, - "passed (shutdown hanged)": 1, - "passed": 0, -} - - -def _merge_pass_status(prev: dict | None, new: dict) -> dict: - """Merge per-pass status dicts into a single per-file entry. - - Counters (``errors``, ``failures``, ``skipped``, ``tests``, - ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more - severe of the two via :data:`_RESULT_PRIORITY`. - """ - if prev is None: - return new - return { - "errors": prev["errors"] + new["errors"], - "failures": prev["failures"] + new["failures"], - "skipped": prev["skipped"] + new["skipped"], - "tests": prev["tests"] + new["tests"], - "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], - "wall_time": prev["wall_time"] + new["wall_time"], - "result": prev["result"] - if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) - else new["result"], - } - - -def _run_one_pass( - ctx: _PassContext, - k_expr: str | None, - suffix: str, -) -> tuple[JUnitXml | None, dict, bool]: - """Drive one pytest subprocess for ``ctx.test_file`` and return its results. - - Args: - ctx: Static per-file context (paths, timeouts, env). - k_expr: Optional ``-k`` selector. ``None`` means no selector (default - single-pass invocation). - suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` - or ``""`` for the unsplit default. - - Returns: - A 3-tuple ``(xml_report, status_dict, was_failure)``: - * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced - no report (e.g. startup hang). - * ``status_dict``: per-pass counters compatible with the entries - currently appended to ``test_status``. - * ``was_failure``: whether the pass should add ``ctx.test_file`` to - the ``failed_tests`` list. - """ - pass_file_label = f"{ctx.file_name}{suffix}" - # Slug the full test path (not just the basename) into the report filename so - # two concurrent shards running same-basename files (e.g. - # ``isaaclab_newton/.../test_articulation.py`` vs - # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path - # inside the shared ``/workspace/isaaclab`` mount and trigger false - # shutdown_hang detections in sibling shards via the report-file existence check. - report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") - report_file = f"tests/test-reports-{report_slug}{suffix}.xml" - - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit - "--no-header", - f"--config-file={ctx.workspace_root}/pyproject.toml", - f"--junitxml={report_file}", - "--tb=short", - ] - if ctx.inject_shard_select: - # multi-GPU lane test-selection plugin (importable via the PYTHONPATH set in - # run_individual_tests); deselects out-of-scope device variants on a shard. - cmd += ["-p", "mgpu_shard_select"] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] - if k_expr is not None: - cmd += ["-k", k_expr] - cmd.append(str(ctx.test_file)) - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file - ) - - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break - - # -- Resolve result from kill_reason and report file ---------------- - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") - print(diag) - - msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "STARTUP_HANG", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") - print(diag) - - msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "TIMEOUT", - "time_elapsed": ctx.timeout, - "wall_time": wall_time, - }, - True, - ) - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {ctx.test_file}{suffix}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", pass_file_label, reason, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "CRASHED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - return ( - None, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 0, - "result": "FAILED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=ctx.test_file, - file_name=ctx.file_name, - cmd=cmd, - timeout=ctx.timeout, - env=ctx.env, - startup_deadline=ctx.startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, - ) - - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures - was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" - - return ( - report, - { - "errors": errors, - "failures": failures, - "skipped": skipped, - "tests": tests, - "result": result, - "time_elapsed": time_elapsed, - "wall_time": wall_time, - }, - was_failure, - ) - - def run_individual_tests(test_files, workspace_root, isaacsim_ci): """Run each test file separately, ensuring one finishes before starting the next. - When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue file, files are claimed - from it (work-stealing across sibling shard containers) instead of iterating - ``test_files``; each file still runs once, on this container's pinned GPU. + When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue directory, files are + claimed from it (work-stealing across sibling shard containers) instead of + iterating ``test_files``; each file still runs once, on this container's GPU. + + The planner turns each file into one or more units (a device_isolated file + splits into one unit per device when the runtime spans more than one); the + executor runs each unit with the unit's mask set as ``ISAACLAB_TEST_DEVICES``. """ failed_tests = [] test_status = {} @@ -856,28 +137,20 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") file_source = _queued_files(queue_path) if queue_path else test_files + # The runner's device set. Unset (single-GPU CI) is cpu + cuda:0, matching + # test_devices()'s default; an mgpu shard sets a single-device mask. The + # planner splits a device_isolated file only when this spans more than one. + runtime_mask = os.environ.get("ISAACLAB_TEST_DEVICES") or "110" + for test_file in file_source: - print(f"\n\n🚀 Running {test_file} independently...\n") + print(f"\n\n\U0001f680 Running {test_file} independently...\n") file_name = os.path.basename(test_file) env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" - # Multi-GPU lane only: make the device-selection plugin importable in this - # per-file subprocess (injected via ``-p`` in _run_one_pass, not as a - # repo-root conftest). Detect a shard by the runtime device mask excluding cpu - # (position 0) and cuda:0 (position 1) -- the same ISAACLAB_TEST_DEVICES the - # plugin and test_devices() read. The plugin re-checks this; the cheap - # prefix test here leaves single-GPU CI's command (mask unset or "11...") - # unchanged. - _mask = os.environ.get("ISAACLAB_TEST_DEVICES", "") - _inject_shard_select = _mask[:2] == "00" - if _inject_shard_select: - _plugin_dir = os.path.join(workspace_root, ".github", "actions", "multi-gpu") - env["PYTHONPATH"] = _plugin_dir + os.pathsep + env.get("PYTHONPATH", "") - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - # Read the test file once for cold-cache and device-split detection. + # Read the test file once for cold-cache and device-isolation detection. try: with open(test_file) as fh: test_content = fh.read() @@ -890,12 +163,12 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): if is_cold_cache_test: timeout += COLD_CACHE_BUFFER cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + print(f"\u23f1\ufe0f Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - ctx = _PassContext( + ctx = _UnitContext( test_file=test_file, file_name=file_name, workspace_root=workspace_root, @@ -903,39 +176,27 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): timeout=timeout, startup_deadline=startup_deadline, env=env, - inject_shard_select=_inject_shard_select, ) - # On a multi-GPU shard, test_devices() already resolves to this shard's single - # GPU and mgpu_shard_select drops every other variant, so the device_split - # CPU/GPU two-pass (which exists to dodge the process-global device lock when - # CPU and GPU share one container) is unnecessary here — the CPU pass would - # collect zero tests yet still pay full Kit-startup cost. Run once on a shard. - if _inject_shard_select: - passes = [("", None)] - elif is_device_split_file(test_file, source=test_content): - print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") - passes = DEVICE_SPLIT_PASSES - else: - passes = [("", None)] + units = plan_units([test_file], runtime_mask, is_isolated=lambda f: _is_isolated(f, source=test_content)) + if len(units) > 1: + print(f"\u2699\ufe0f device_isolated \u2014 invoking {file_name} once per device ({len(units)} units)") merged_status: dict | None = None - for suffix, k_expr in passes: - report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + for unit in units: + report, status, was_failure = run_unit(ctx, unit) if report is not None: xml_reports.append(report) if was_failure and test_file not in failed_tests: failed_tests.append(test_file) - merged_status = _merge_pass_status(merged_status, status) + merged_status = merge_unit_status(merged_status, status) - assert merged_status is not None # the pass list is never empty + assert merged_status is not None # the unit list is never empty test_status[test_file] = merged_status - # When running under the directory-based work queue (option 2), move the - # claim entry from inflight// to done// so the post-run - # reconciler can distinguish "ran to completion" from "claimed but - # crashed mid-test". A claim that stays in inflight at job-end is a - # silent drop signal. + # In work-queue mode, move the claim from inflight// to done// + # so the post-run reconciler can tell "ran to completion" from "claimed but + # crashed mid-test". A claim left in inflight at job-end is a silent drop. if queue_path: _mark_queued_file_done(queue_path, test_file) diff --git a/tools/test_agnostic_select.py b/tools/test_agnostic_select.py new file mode 100644 index 000000000000..8c323cd74111 --- /dev/null +++ b/tools/test_agnostic_select.py @@ -0,0 +1,91 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the agnostic-drop plugin (``tools/_agnostic_select.py``). + +The plugin deselects every collected test that is not parametrized over +``device``. It carries no mask logic; the executor injects it only for units +whose mask lacks cpu (mgpu shards and the cuda unit of a can't-mix split), where +device-agnostic tests must not run. +""" + +from __future__ import annotations + +from _agnostic_select import pytest_collection_modifyitems + + +class _FakeCallspec: + def __init__(self, params): + self.params = params + + +class _FakeItem: + """Stand-in for a pytest item. Omitting ``params`` means no callspec at all.""" + + def __init__(self, name, params=None): + self.name = name + if params is not None: + self.callspec = _FakeCallspec(params) + + +class _FakeHook: + def __init__(self): + self.deselected = [] + + def pytest_deselected(self, items): + self.deselected.extend(items) + + +class _FakeConfig: + def __init__(self): + self.hook = _FakeHook() + + +def test_drops_items_without_a_device_param(): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + gpu = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _FakeItem("t_logic") # no callspec + items = [cpu, gpu, agnostic] + config = _FakeConfig() + + pytest_collection_modifyitems(config, items) + + assert items == [cpu, gpu] + assert config.hook.deselected == [agnostic] + + +def test_drops_parametrized_item_that_lacks_a_device_key(): + # Parametrized over something else (num_envs) but not device -> still agnostic. + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + other = _FakeItem("t[2]", {"num_envs": 2}) + items = [cpu, other] + config = _FakeConfig() + + pytest_collection_modifyitems(config, items) + + assert items == [cpu] + assert config.hook.deselected == [other] + + +def test_keeps_all_when_every_item_has_a_device_param(): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + gpu = _FakeItem("t[cuda:1]", {"device": "cuda:1"}) + items = [cpu, gpu] + config = _FakeConfig() + + pytest_collection_modifyitems(config, items) + + assert items == [cpu, gpu] + assert config.hook.deselected == [] # nothing deselected + + +def test_empty_item_list_is_a_no_op(): + items = [] + config = _FakeConfig() + + pytest_collection_modifyitems(config, items) + + assert items == [] + assert config.hook.deselected == [] diff --git a/tools/test_device_plan.py b/tools/test_device_plan.py index a047a1769c11..d99d623772b9 100644 --- a/tools/test_device_plan.py +++ b/tools/test_device_plan.py @@ -12,33 +12,40 @@ from __future__ import annotations import pytest -from _device_plan import is_isolated, plan_units +from _device_plan import Unit, is_isolated, plan_units # ---- plan_units: the per-runner planning logic -------------------------------- def test_mix_ok_file_is_a_single_unit(): # A file that does not need device isolation runs once, covering the whole mask. - assert plan_units(["a.py"], "110", is_isolated=lambda f: False) == [("a.py", "110")] + assert plan_units(["a.py"], "110", is_isolated=lambda f: False) == [Unit("a.py", "110")] def test_cant_mix_file_splits_into_one_unit_per_set_bit(): # An isolated file on a multi-device runtime splits into one process per device. - assert plan_units(["a.py"], "110", is_isolated=lambda f: True) == [("a.py", "100"), ("a.py", "010")] + # Split units are flagged so the executor gives them distinct report names. + assert plan_units(["a.py"], "110", is_isolated=lambda f: True) == [ + Unit("a.py", "100", split=True), + Unit("a.py", "010", split=True), + ] def test_cant_mix_on_single_device_runtime_does_not_split(): # An mgpu shard's runtime is one device, so even an isolated file is one unit. - assert plan_units(["a.py"], "0001", is_isolated=lambda f: True) == [("a.py", "0001")] + assert plan_units(["a.py"], "0001", is_isolated=lambda f: True) == [Unit("a.py", "0001")] def test_cant_mix_on_cpu_only_runtime_does_not_split(): - assert plan_units(["a.py"], "100", is_isolated=lambda f: True) == [("a.py", "100")] + assert plan_units(["a.py"], "100", is_isolated=lambda f: True) == [Unit("a.py", "100")] def test_split_masks_use_each_set_bit_position(): # Set bits at positions 0 and 2 -> two single-bit masks of the same width. - assert plan_units(["a.py"], "1010", is_isolated=lambda f: True) == [("a.py", "1000"), ("a.py", "0010")] + assert plan_units(["a.py"], "1010", is_isolated=lambda f: True) == [ + Unit("a.py", "1000", split=True), + Unit("a.py", "0010", split=True), + ] def test_multiple_files_preserve_order_and_decide_per_file(): @@ -46,9 +53,9 @@ def isolated(f): return f == "b.py" assert plan_units(["a.py", "b.py"], "110", is_isolated=isolated) == [ - ("a.py", "110"), - ("b.py", "100"), - ("b.py", "010"), + Unit("a.py", "110"), + Unit("b.py", "100", split=True), + Unit("b.py", "010", split=True), ] diff --git a/tools/test_device_split.py b/tools/test_device_split.py deleted file mode 100644 index fb34ce40c4f1..000000000000 --- a/tools/test_device_split.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for ``tools/_device_split.py``.""" - -from __future__ import annotations - -import textwrap -from pathlib import Path - -from _device_split import is_device_split_file - - -def _write(tmp_path: Path, content: str) -> Path: - p = tmp_path / "test_foo.py" - p.write_text(textwrap.dedent(content)) - return p - - -def test_single_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.device_split - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_list_form_single_line(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = [pytest.mark.device_split, pytest.mark.foo] - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_preloaded_source(tmp_path): - source = textwrap.dedent( - """ - import pytest - - pytestmark = pytest.mark.device_split - """ - ) - assert is_device_split_file(tmp_path / "does_not_exist.py", source=source) is True - - -def test_no_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_word_in_comment_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - # This file mentions device_split in a comment but is not marked. - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_unrelated_pytestmark_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.skipif(False, reason="x") - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_missing_file(tmp_path): - # A path that does not exist must not raise; treat as not-marked. - assert is_device_split_file(tmp_path / "does_not_exist.py") is False From 0ff80a7acddffead94c744b915f28dd36af8fa42 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 12 Jun 2026 00:16:13 +0000 Subject: [PATCH 20/23] Extract _build_unit_cmd and guard the executor's device selection Pull the per-unit command/env construction out of run_unit into a pure _build_unit_cmd() and cover it with deterministic tests (no subprocess, GPU, or Kit): the unit's mask becomes ISAACLAB_TEST_DEVICES, the _agnostic_select plugin is injected only for a cpu-less mask, -k is never used, and split units get a suffixed report name while single units keep the aggregator-friendly one. This is the behavior-equivalence guard for the planner+executor refactor. --- tools/_device_exec.py | 47 +++++++++++++++------- tools/test_device_exec.py | 85 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 15 deletions(-) create mode 100644 tools/test_device_exec.py diff --git a/tools/_device_exec.py b/tools/_device_exec.py index 4e3a935d6de8..50ed8dced737 100644 --- a/tools/_device_exec.py +++ b/tools/_device_exec.py @@ -466,21 +466,21 @@ def merge_unit_status(prev: dict | None, new: dict) -> dict: } -def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool]: - """Drive one pytest subprocess for ``unit`` and return its results. +def _build_unit_cmd(ctx: _UnitContext, unit: Unit) -> tuple[list[str], dict, str, str]: + """Build the pytest invocation for one unit (no subprocess). + + Device-variant selection is the unit's mask, set as ``ISAACLAB_TEST_DEVICES`` + for ``test_devices()`` to read — there is no ``-k``. A mask without cpu + (position 0) additionally injects the ``_agnostic_select`` plugin so + device-agnostic tests do not run there. Args: - ctx: Static per-file context (paths, timeouts, base env). - unit: The work unit to run; its mask becomes ``ISAACLAB_TEST_DEVICES``. + ctx: Static per-file context (paths, isaacsim_ci flag, base env). + unit: The work unit to run. Returns: - A 3-tuple ``(xml_report, status_dict, was_failure)``: - * ``xml_report``: parsed JUnit XML, or ``None`` if the unit produced - no report (e.g. startup hang). - * ``status_dict``: per-unit counters compatible with the entries - appended to ``test_status``. - * ``was_failure``: whether the unit should add ``ctx.test_file`` to - the ``failed_tests`` list. + ``(cmd, env, report_file, pass_file_label)`` — the argv, the per-unit + environment, the JUnit report path, and the label used in error reports. """ # Split units of one file share a slug, so suffix the report with the mask to # keep both XMLs; a non-split unit (one per file) keeps the suffix-free name @@ -496,13 +496,9 @@ def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") report_file = f"tests/test-reports-{report_slug}{report_suffix}.xml" - # The mask drives device-variant selection through test_devices(); no -k needed. env = dict(ctx.env) env["ISAACLAB_TEST_DEVICES"] = unit.mask - # A mask without cpu (position 0) — an mgpu shard or the cuda unit of a split — - # must not run device-agnostic tests. Inject the dumb drop-paramless plugin and - # put tools/ on PYTHONPATH so the per-file subprocess can import it. inject_agnostic = unit.mask[:1] == "0" if inject_agnostic: tools_dir = os.path.join(ctx.workspace_root, "tools") @@ -524,6 +520,27 @@ def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool if ctx.isaacsim_ci: cmd += ["-m", "isaacsim_ci"] cmd.append(str(ctx.test_file)) + return cmd, env, report_file, pass_file_label + + +def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``unit`` and return its results. + + Args: + ctx: Static per-file context (paths, timeouts, base env). + unit: The work unit to run; its mask becomes ``ISAACLAB_TEST_DEVICES``. + + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the unit produced + no report (e.g. startup hang). + * ``status_dict``: per-unit counters compatible with the entries + appended to ``test_status``. + * ``was_failure``: whether the unit should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + cmd, env, report_file, pass_file_label = _build_unit_cmd(ctx, unit) + report_suffix = f"-{unit.mask}" if unit.split else "" # display suffix for this unit's log lines # -- Run with retry on startup hang or hard timeout ----------------- returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" diff --git a/tools/test_device_exec.py b/tools/test_device_exec.py new file mode 100644 index 000000000000..f9a2502c4522 --- /dev/null +++ b/tools/test_device_exec.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Behavior guard for the executor's per-unit command construction. + +These assert the device-selection contract without launching a subprocess: the +unit's mask becomes ``ISAACLAB_TEST_DEVICES`` (so ``test_devices()`` selects the +variants), the ``_agnostic_select`` plugin is injected only for a cpu-less mask +(shards and the cuda unit of a split), ``-k`` is never used, and split units get +a suffixed report name while single units keep the mgpu-aggregator-friendly one. +""" + +from __future__ import annotations + +import os + +from _device_exec import _build_unit_cmd, _UnitContext +from _device_plan import Unit + + +def _ctx(test_file="source/pkg/test_x.py", isaacsim_ci=False): + return _UnitContext( + test_file=test_file, + file_name="test_x.py", + workspace_root="/ws", + isaacsim_ci=isaacsim_ci, + timeout=60, + startup_deadline=30, + env={"BASE": "1"}, + ) + + +def test_mix_ok_unit_sets_mask_with_no_plugin_and_no_k(): + cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "110")) + assert env["ISAACLAB_TEST_DEVICES"] == "110" + assert env["BASE"] == "1" # base env preserved + assert "_agnostic_select" not in cmd + assert "-k" not in cmd + assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" # no suffix for a single unit + assert label == "test_x.py" + assert cmd[-1] == "source/pkg/test_x.py" + + +def test_cpu_split_unit_is_suffixed_and_keeps_agnostic_tests(): + cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "100", split=True)) + assert env["ISAACLAB_TEST_DEVICES"] == "100" + assert "_agnostic_select" not in cmd # cpu in mask -> agnostic tests run here + assert report_file.endswith("test_x.py-100.xml") + assert label == "test_x.py-100" + + +def test_cuda_split_unit_injects_agnostic_plugin_and_tools_pythonpath(): + cmd, env, report_file, _ = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "010", split=True)) + assert env["ISAACLAB_TEST_DEVICES"] == "010" + assert cmd.count("-p") == 1 and "_agnostic_select" in cmd + assert env["PYTHONPATH"].split(os.pathsep)[0] == os.path.join("/ws", "tools") + assert report_file.endswith("test_x.py-010.xml") + + +def test_shard_unit_injects_agnostic_plugin_without_a_suffix(): + cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "0001")) + assert env["ISAACLAB_TEST_DEVICES"] == "0001" + assert "_agnostic_select" in cmd # cpu-less mask -> drop device-agnostic tests + # one unit per file on a shard -> suffix-free name the aggregator unslugs to a path + assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" + assert label == "test_x.py" + + +def test_isaacsim_ci_adds_the_marker_selector(): + # ``-m pytest`` appears first, so look for the ``-m isaacsim_ci`` pair specifically. + cmd, _, _, _ = _build_unit_cmd(_ctx(isaacsim_ci=True), Unit("source/pkg/test_x.py", "110")) + assert any(cmd[i] == "-m" and cmd[i + 1] == "isaacsim_ci" for i in range(len(cmd) - 1)) + + +def test_no_isaacsim_ci_marker_when_disabled(): + cmd, _, _, _ = _build_unit_cmd(_ctx(isaacsim_ci=False), Unit("source/pkg/test_x.py", "110")) + assert "isaacsim_ci" not in cmd + + +def test_no_k_selector_for_any_mask(): + for mask, split in [("110", False), ("100", True), ("010", True), ("0001", False)]: + cmd, *_ = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", mask, split=split)) + assert "-k" not in cmd From 0c6d29069e16047ac3b6cc089d2509c79b07a6d9 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 12 Jun 2026 01:26:57 +0000 Subject: [PATCH 21/23] Fix: filter device variants per unit, not just agnostic tests The first refactor assumed test_devices()+mask did all device-variant selection, so the plugin only dropped device-agnostic tests. But tests with a literal device list (e.g. parametrize("device", ["cuda:0", "cpu"])) ignore the mask, so their out-of-scope variant leaked into the wrong split unit and tripped ovphysx's process-global device lock (RuntimeError: locked to 'gpu', cannot switch to 'cpu'). Restore per-unit device filtering: rename the plugin to _device_select and have it keep only the tests whose device param is in the unit's mask (plus the agnostic rule), reading the structured param so it catches literal lists too. The executor injects it for both halves of a split and for shards. This reproduces the old -k 'cpu or not cuda' / -k 'cuda' selection exactly. --- .github/workflows/test-multi-gpu-pytest.yaml | 7 +- tools/_agnostic_select.py | 31 ----- tools/_device_exec.py | 24 ++-- tools/_device_select.py | 81 ++++++++++++ tools/test_agnostic_select.py | 91 ------------- tools/test_device_exec.py | 20 +-- tools/test_device_select.py | 131 +++++++++++++++++++ 7 files changed, 240 insertions(+), 145 deletions(-) delete mode 100644 tools/_agnostic_select.py create mode 100644 tools/_device_select.py delete mode 100644 tools/test_agnostic_select.py create mode 100644 tools/test_device_select.py diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index b1a0973e70bb..7e0b21e3be0e 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -115,10 +115,9 @@ jobs: # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the - # ``_agnostic_select`` plugin (injected by ``tools/conftest.py`` for any - # unit whose mask lacks cpu, i.e. every shard): single-GPU CI already - # covers them on ``cuda:0`` and re-running on every non-default shard adds - # wall-time without surfacing any new failure mode. + # ``_device_select`` plugin (injected by ``tools/conftest.py`` on every + # shard): single-GPU CI already covers them on ``cuda:0`` and re-running on + # every non-default shard adds wall-time without surfacing any new failure mode. id: discover run: | # File-level opt-out: a test file can exclude itself from multi-GPU CI diff --git a/tools/_agnostic_select.py b/tools/_agnostic_select.py deleted file mode 100644 index 1aef4518b57c..000000000000 --- a/tools/_agnostic_select.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""pytest plugin: drop device-agnostic tests from this run. - -Deselects every collected test that is not parametrized over ``device``. It -carries no mask logic of its own. The executor (``tools/conftest.py``) injects -it via ``-p _agnostic_select`` only for a unit whose mask lacks cpu — i.e. an -mgpu shard, or the cuda unit of a can't-mix split — because in those units a -device-agnostic test would either run redundantly on every shard or run a second -time alongside the cpu unit. Device variant selection itself is handled upstream -by ``test_devices()`` reading the unit's ``ISAACLAB_TEST_DEVICES`` mask, so this -plugin only has to remove the paramless remainder. -""" - -from __future__ import annotations - - -def _has_device_param(item) -> bool: - """Return whether a collected item is parametrized over ``device``.""" - callspec = getattr(item, "callspec", None) - return callspec is not None and "device" in callspec.params - - -def pytest_collection_modifyitems(config, items): - drop = [item for item in items if not _has_device_param(item)] - if drop: - config.hook.pytest_deselected(items=drop) - items[:] = [item for item in items if _has_device_param(item)] diff --git a/tools/_device_exec.py b/tools/_device_exec.py index 50ed8dced737..b0fd5f2f57aa 100644 --- a/tools/_device_exec.py +++ b/tools/_device_exec.py @@ -11,10 +11,10 @@ Device selection is the planner's job, expressed as the unit's mask: the executor sets ``ISAACLAB_TEST_DEVICES`` to that mask (``test_devices()`` reads -it to pick the device variants) and, for a unit whose mask lacks cpu — an mgpu -shard or the cuda unit of a can't-mix split — injects the ``_agnostic_select`` -plugin so device-agnostic tests do not run there. The executor never reads a -marker or a ``-k`` selector. +it to pick the device variants) and, for a split unit (either half) or an mgpu +shard, injects the ``_device_select`` plugin to drop out-of-scope device variants +— including literal device-param variants the mask alone cannot narrow — and +device-agnostic tests. The executor never reads a marker or a ``-k`` selector. """ from __future__ import annotations @@ -416,7 +416,7 @@ class _UnitContext: test_file: Path to the test file being driven. file_name: Basename of ``test_file`` (used for JUnit naming). workspace_root: Repository root; passed to pytest's ``--config-file`` and - used to locate the ``_agnostic_select`` plugin. + used to locate the ``_device_select`` plugin. isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the ``-m isaacsim_ci`` selector. timeout: Per-unit hard timeout in seconds. @@ -471,7 +471,7 @@ def _build_unit_cmd(ctx: _UnitContext, unit: Unit) -> tuple[list[str], dict, str Device-variant selection is the unit's mask, set as ``ISAACLAB_TEST_DEVICES`` for ``test_devices()`` to read — there is no ``-k``. A mask without cpu - (position 0) additionally injects the ``_agnostic_select`` plugin so + (position 0) additionally injects the ``_device_select`` plugin so device-agnostic tests do not run there. Args: @@ -499,8 +499,12 @@ def _build_unit_cmd(ctx: _UnitContext, unit: Unit) -> tuple[list[str], dict, str env = dict(ctx.env) env["ISAACLAB_TEST_DEVICES"] = unit.mask - inject_agnostic = unit.mask[:1] == "0" - if inject_agnostic: + # A split unit (either half) or a cpu-less shard must run exactly its mask's + # devices, so inject the device selector to drop out-of-scope variants — + # including literal device-param variants the mask alone cannot narrow (only + # test_devices() reads the mask; a literal ["cuda:0", "cpu"] list does not). + select_by_device = unit.split or unit.mask[:1] == "0" + if select_by_device: tools_dir = os.path.join(ctx.workspace_root, "tools") env["PYTHONPATH"] = tools_dir + os.pathsep + env.get("PYTHONPATH", "") @@ -515,8 +519,8 @@ def _build_unit_cmd(ctx: _UnitContext, unit: Unit) -> tuple[list[str], dict, str f"--junitxml={report_file}", "--tb=short", ] - if inject_agnostic: - cmd += ["-p", "_agnostic_select"] + if select_by_device: + cmd += ["-p", "_device_select"] if ctx.isaacsim_ci: cmd += ["-m", "isaacsim_ci"] cmd.append(str(ctx.test_file)) diff --git a/tools/_device_select.py b/tools/_device_select.py new file mode 100644 index 000000000000..ed2498eb7eba --- /dev/null +++ b/tools/_device_select.py @@ -0,0 +1,81 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""pytest plugin: keep only the tests a unit's device mask allows. + +The executor (``tools/conftest.py`` via ``_device_exec``) injects this with +``-p _device_select`` for any unit that must run on a single device — both halves +of a ``device_isolated`` split (cpu unit, cuda unit) and every mgpu shard — so a +process that is locked to one device by a backend (e.g. ovphysx<=0.3.7) never +also tries another device. + +Selection reads ``ISAACLAB_TEST_DEVICES`` (the unit's mask). For each collected +test: + +* parametrized over ``device``: keep iff that device is active in the mask. This + is what catches a literal ``["cuda:0", "cpu"]`` variant whose device the mask + alone cannot narrow (only ``test_devices()`` reads the mask at parametrize + time; a literal list does not). +* not parametrized over ``device`` (agnostic): keep only when cpu is in the mask + — i.e. the default unit. On a cuda split unit or a shard it is dropped, since + single-GPU CI already covers it on cpu/cuda:0. + +A unit that deselects to empty (no in-scope tests in this file) exits +``NO_TESTS_COLLECTED``; the runner treats any non-zero per-file exit as failure, +so report "nothing in scope" as success. +""" + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +def _position(device): + """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None # not device-parametrized, or an unrecognized device value + + +def _active(position, mask): + """Whether a mask position is included. A trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + +def _device_of(item): + """The ``device`` parametrize value of a collected item, or ``None`` if agnostic.""" + callspec = getattr(item, "callspec", None) + return callspec.params.get("device") if callspec is not None else None + + +def pytest_collection_modifyitems(config, items): + mask = os.environ.get(_RUNTIME_ENV, "") + cpu_in_mask = _active(0, mask) + keep, drop = [], [] + for item in items: + device = _device_of(item) + if device is None: + (keep if cpu_in_mask else drop).append(item) # agnostic: only in a cpu-inclusive unit + elif _active(_position(device), mask): + keep.append(item) # this device variant is in scope for the unit + else: + drop.append(item) # out-of-scope variant (e.g. a literal cpu variant in a cuda unit) + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + +def pytest_sessionfinish(session, exitstatus): + # A file with nothing in scope for this unit deselects to zero and exits + # NO_TESTS_COLLECTED (5). That is success here, not a failure. + if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK diff --git a/tools/test_agnostic_select.py b/tools/test_agnostic_select.py deleted file mode 100644 index 8c323cd74111..000000000000 --- a/tools/test_agnostic_select.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for the agnostic-drop plugin (``tools/_agnostic_select.py``). - -The plugin deselects every collected test that is not parametrized over -``device``. It carries no mask logic; the executor injects it only for units -whose mask lacks cpu (mgpu shards and the cuda unit of a can't-mix split), where -device-agnostic tests must not run. -""" - -from __future__ import annotations - -from _agnostic_select import pytest_collection_modifyitems - - -class _FakeCallspec: - def __init__(self, params): - self.params = params - - -class _FakeItem: - """Stand-in for a pytest item. Omitting ``params`` means no callspec at all.""" - - def __init__(self, name, params=None): - self.name = name - if params is not None: - self.callspec = _FakeCallspec(params) - - -class _FakeHook: - def __init__(self): - self.deselected = [] - - def pytest_deselected(self, items): - self.deselected.extend(items) - - -class _FakeConfig: - def __init__(self): - self.hook = _FakeHook() - - -def test_drops_items_without_a_device_param(): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - gpu = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) - agnostic = _FakeItem("t_logic") # no callspec - items = [cpu, gpu, agnostic] - config = _FakeConfig() - - pytest_collection_modifyitems(config, items) - - assert items == [cpu, gpu] - assert config.hook.deselected == [agnostic] - - -def test_drops_parametrized_item_that_lacks_a_device_key(): - # Parametrized over something else (num_envs) but not device -> still agnostic. - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - other = _FakeItem("t[2]", {"num_envs": 2}) - items = [cpu, other] - config = _FakeConfig() - - pytest_collection_modifyitems(config, items) - - assert items == [cpu] - assert config.hook.deselected == [other] - - -def test_keeps_all_when_every_item_has_a_device_param(): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - gpu = _FakeItem("t[cuda:1]", {"device": "cuda:1"}) - items = [cpu, gpu] - config = _FakeConfig() - - pytest_collection_modifyitems(config, items) - - assert items == [cpu, gpu] - assert config.hook.deselected == [] # nothing deselected - - -def test_empty_item_list_is_a_no_op(): - items = [] - config = _FakeConfig() - - pytest_collection_modifyitems(config, items) - - assert items == [] - assert config.hook.deselected == [] diff --git a/tools/test_device_exec.py b/tools/test_device_exec.py index f9a2502c4522..66da1c7182fe 100644 --- a/tools/test_device_exec.py +++ b/tools/test_device_exec.py @@ -7,8 +7,8 @@ These assert the device-selection contract without launching a subprocess: the unit's mask becomes ``ISAACLAB_TEST_DEVICES`` (so ``test_devices()`` selects the -variants), the ``_agnostic_select`` plugin is injected only for a cpu-less mask -(shards and the cuda unit of a split), ``-k`` is never used, and split units get +variants), the ``_device_select`` plugin is injected for split units (both +halves) and shards, ``-k`` is never used, and split units get a suffixed report name while single units keep the mgpu-aggregator-friendly one. """ @@ -36,33 +36,35 @@ def test_mix_ok_unit_sets_mask_with_no_plugin_and_no_k(): cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "110")) assert env["ISAACLAB_TEST_DEVICES"] == "110" assert env["BASE"] == "1" # base env preserved - assert "_agnostic_select" not in cmd + assert "_device_select" not in cmd # mix-ok single unit: no isolation needed assert "-k" not in cmd assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" # no suffix for a single unit assert label == "test_x.py" assert cmd[-1] == "source/pkg/test_x.py" -def test_cpu_split_unit_is_suffixed_and_keeps_agnostic_tests(): +def test_cpu_split_unit_is_suffixed_and_injects_device_select(): + # The cpu half of a split must also drop out-of-scope literal cuda variants, + # else they leak into the cpu-locked process. So it injects the selector too. cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "100", split=True)) assert env["ISAACLAB_TEST_DEVICES"] == "100" - assert "_agnostic_select" not in cmd # cpu in mask -> agnostic tests run here + assert "_device_select" in cmd assert report_file.endswith("test_x.py-100.xml") assert label == "test_x.py-100" -def test_cuda_split_unit_injects_agnostic_plugin_and_tools_pythonpath(): +def test_cuda_split_unit_injects_device_select_and_tools_pythonpath(): cmd, env, report_file, _ = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "010", split=True)) assert env["ISAACLAB_TEST_DEVICES"] == "010" - assert cmd.count("-p") == 1 and "_agnostic_select" in cmd + assert cmd.count("-p") == 1 and "_device_select" in cmd assert env["PYTHONPATH"].split(os.pathsep)[0] == os.path.join("/ws", "tools") assert report_file.endswith("test_x.py-010.xml") -def test_shard_unit_injects_agnostic_plugin_without_a_suffix(): +def test_shard_unit_injects_device_select_without_a_suffix(): cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "0001")) assert env["ISAACLAB_TEST_DEVICES"] == "0001" - assert "_agnostic_select" in cmd # cpu-less mask -> drop device-agnostic tests + assert "_device_select" in cmd # cpu-less shard -> drop out-of-scope + agnostic # one unit per file on a shard -> suffix-free name the aggregator unslugs to a path assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" assert label == "test_x.py" diff --git a/tools/test_device_select.py b/tools/test_device_select.py new file mode 100644 index 000000000000..6318df0ada3a --- /dev/null +++ b/tools/test_device_select.py @@ -0,0 +1,131 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the per-unit device selector (``tools/_device_select.py``). + +These pin the rule that broke when device selection was left to the mask alone: +a literal ``["cuda:0", "cpu"]`` variant must still be dropped when it is out of +the unit's mask, or it leaks into a device-locked process. The plugin reads the +structured ``device`` param, so it catches both literal and ``test_devices()`` +variants. +""" + +from __future__ import annotations + +import _device_select +from _device_select import _active, _position, pytest_collection_modifyitems + + +class _FakeCallspec: + def __init__(self, params): + self.params = params + + +class _FakeItem: + """Stand-in for a pytest item. Omitting ``params`` means no callspec (agnostic).""" + + def __init__(self, name, params=None): + self.name = name + if params is not None: + self.callspec = _FakeCallspec(params) + + +class _FakeHook: + def __init__(self): + self.deselected = [] + + def pytest_deselected(self, items): + self.deselected.extend(items) + + +class _FakeConfig: + def __init__(self): + self.hook = _FakeHook() + + +def _select(mask, items, monkeypatch): + monkeypatch.setenv("ISAACLAB_TEST_DEVICES", mask) + config = _FakeConfig() + pytest_collection_modifyitems(config, items) + return items, config.hook.deselected + + +# ---- position / active helpers ------------------------------------------------ + + +def test_position_maps_cpu_and_cuda(): + assert _position("cpu") == 0 + assert _position("cuda:0") == 1 + assert _position("cuda:2") == 3 + assert _position(None) is None + assert _position("weird") is None + + +def test_active_reads_mask_position_and_trailing_x(): + assert _active(0, "110") is True # cpu + assert _active(1, "110") is True # cuda:0 + assert _active(2, "110") is False # cuda:1 not set + assert _active(3, "00X") is True # trailing X includes the rest + assert _active(None, "110") is False + + +# ---- selection: the bug this guards against ----------------------------------- + + +def test_cpu_unit_drops_literal_cuda_variant(monkeypatch): + # The bug: a literal ["cuda:0","cpu"] test's cuda variant leaked into the cpu + # unit and tripped the device lock. The cpu unit (mask 100) must drop it. + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) + items, dropped = _select("100", [cpu, cuda], monkeypatch) + assert items == [cpu] + assert dropped == [cuda] + + +def test_cuda_unit_drops_literal_cpu_variant_and_agnostic(monkeypatch): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _FakeItem("t_logic") + items, dropped = _select("010", [cpu, cuda, agnostic], monkeypatch) + assert items == [cuda] + assert set(dropped) == {cpu, agnostic} + + +def test_shard_keeps_its_gpu_variant_drops_others_and_agnostic(monkeypatch): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + cuda0 = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) + cuda2 = _FakeItem("t[cuda:2]", {"device": "cuda:2"}) + agnostic = _FakeItem("t_logic") + items, dropped = _select("0001", [cpu, cuda0, cuda2, agnostic], monkeypatch) # cuda:2 shard + assert items == [cuda2] + assert set(dropped) == {cpu, cuda0, agnostic} + + +def test_cpu_unit_keeps_agnostic(monkeypatch): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + agnostic = _FakeItem("t_logic") + items, dropped = _select("100", [cpu, agnostic], monkeypatch) + assert items == [cpu, agnostic] # cpu in mask -> agnostic kept + assert dropped == [] + + +def test_default_mask_keeps_cpu_and_cuda0_and_agnostic(monkeypatch): + cpu = _FakeItem("t[cpu]", {"device": "cpu"}) + cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _FakeItem("t_logic") + items, dropped = _select("110", [cpu, cuda, agnostic], monkeypatch) + assert items == [cpu, cuda, agnostic] + assert dropped == [] + + +def test_sessionfinish_maps_no_tests_collected_to_ok(): + import pytest + + class _Session: + exitstatus = pytest.ExitCode.NO_TESTS_COLLECTED + + session = _Session() + _device_select.pytest_sessionfinish(session, pytest.ExitCode.NO_TESTS_COLLECTED) + assert session.exitstatus == pytest.ExitCode.OK From 89001ca7f8430c266f25368564ccbcf26b4d4e30 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 12 Jun 2026 04:25:51 +0000 Subject: [PATCH 22/23] Add test_runner framework package (planning + execution + selector) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Begin rebuilding the per-file CI runner as a tools/test_runner/ package, one module per lifecycle stage, driven by a single RunnerConfig that turns the scattered constants and env reads into configuration: - planning.py: RunnerConfig (with from_env), Unit, Planner (device_isolated split). - process.py: ProcessCapture (subprocess + hang/timeout + diagnostics), JUnitReport. - execution.py: ExecContext, UnitRunner (build cmd, drive capture, classify, retry), merge_statuses. - selector.py: DeviceSelect, the in-process device-mask plugin as a class with documented hooks and a one-line registrar. Additive only — conftest still uses the current modules; the wiring and removal of the old _device_* modules follow. 25 unit tests, ruff clean. --- tools/test_runner/__init__.py | 25 ++ tools/test_runner/execution.py | 355 ++++++++++++++++++++++ tools/test_runner/planning.py | 207 +++++++++++++ tools/test_runner/process.py | 264 ++++++++++++++++ tools/test_runner/selector.py | 121 ++++++++ tools/test_runner/tests/__init__.py | 6 + tools/test_runner/tests/test_execution.py | 94 ++++++ tools/test_runner/tests/test_planning.py | 82 +++++ tools/test_runner/tests/test_selector.py | 112 +++++++ 9 files changed, 1266 insertions(+) create mode 100644 tools/test_runner/__init__.py create mode 100644 tools/test_runner/execution.py create mode 100644 tools/test_runner/planning.py create mode 100644 tools/test_runner/process.py create mode 100644 tools/test_runner/selector.py create mode 100644 tools/test_runner/tests/__init__.py create mode 100644 tools/test_runner/tests/test_execution.py create mode 100644 tools/test_runner/tests/test_planning.py create mode 100644 tools/test_runner/tests/test_selector.py diff --git a/tools/test_runner/__init__.py b/tools/test_runner/__init__.py new file mode 100644 index 000000000000..3ce7c128f7b9 --- /dev/null +++ b/tools/test_runner/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Per-file CI test runner framework. + +``tools/conftest.py`` runs each test file as its own pytest subprocess rather +than collecting everything into one process. This package is that runner, split +into the stages of a test-run lifecycle: + +* :mod:`test_runner.planning` — what to run: the :class:`~test_runner.planning.RunnerConfig` + knobs, the :class:`~test_runner.planning.Unit` work item, and the + :class:`~test_runner.planning.Planner` that turns a file + device mask into units. +* :mod:`test_runner.execution` — how to run one unit: subprocess capture, + timeout/hang handling, report parsing, retries. +* :mod:`test_runner.selector` — an in-process pytest plugin the executor injects + into a unit's subprocess to keep only that unit's device variants. +* :mod:`test_runner.session` — the lifecycle: collect files, plan units, run + them, aggregate and report. + +The whole runner is driven by one :class:`~test_runner.planning.RunnerConfig`, so +its behavior (timeouts, retries, cold-cache budget, device mask, filters) is +configuration rather than scattered constants. +""" diff --git a/tools/test_runner/execution.py b/tools/test_runner/execution.py new file mode 100644 index 000000000000..41f07c1f95f5 --- /dev/null +++ b/tools/test_runner/execution.py @@ -0,0 +1,355 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""How to run one unit: build its command, drive the subprocess, classify the result. + +:class:`UnitRunner` turns a :class:`~test_runner.planning.Unit` into a pytest +subprocess via :class:`~test_runner.process.ProcessCapture`, then resolves the +outcome (startup hang / timeout / crash / pass / fail) into a status dict, with +fresh-process retries for flaky files. The per-file dynamic inputs that vary +within a run live in :class:`ExecContext`; the static knobs come from the +:class:`~test_runner.planning.RunnerConfig`. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +from dataclasses import dataclass + +from test_runner.planning import RunnerConfig, Unit +from test_runner.process import JUnitReport, ProcessCapture + +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} + + +@dataclass +class ExecContext: + """Per-file inputs that vary within a run (static knobs are in RunnerConfig). + + Attributes: + file_name: Basename of the file, used in JUnit naming and messages. + timeout: Per-unit hard timeout [s] (already cold-cache-adjusted). + startup_deadline: Per-unit startup-hang deadline [s]. + env: Base environment; :class:`UnitRunner` copies it and adds the unit's mask. + """ + + file_name: str + timeout: int + startup_deadline: int + env: dict + + +def merge_statuses(prev: dict | None, new: dict) -> dict: + """Combine per-unit status dicts into a per-file entry. + + Counters sum; ``result`` becomes the more severe of the two by + :data:`_RESULT_PRIORITY`. Used by the session to fold a split file's units. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + +class UnitRunner: + """Run one unit as a pytest subprocess and classify its outcome.""" + + def __init__(self, config: RunnerConfig): + self._config = config + self._capture = ProcessCapture(config.shutdown_grace_period) + + @staticmethod + def _status(result, *, errors=0, failures=0, skipped=0, tests=1, time_elapsed=0.0, wall_time=0.0) -> dict: + """A per-unit status dict (the shape the reporter and merge_statuses expect).""" + return { + "errors": errors, + "failures": failures, + "skipped": skipped, + "tests": tests, + "result": result, + "time_elapsed": time_elapsed, + "wall_time": wall_time, + } + + def _error_result( + self, + kind, + label, + suffix, + unit, + ctx, + result, + wall_time, + cap, + pre, + *, + msg, + stdout, + stderr, + report_file, + time_elapsed=0.0, + ): + """Print diagnostics, write a synthetic error report, and return its failure status.""" + diag = cap.diagnostics(pre) + print(f"⚠️ {unit.file}{suffix}: {msg}") + print(diag) + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout: + details += "=== STDOUT (last 5000 chars) ===\n" + stdout.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr: + details += "=== STDERR (last 5000 chars) ===\n" + stderr.decode("utf-8", errors="replace")[-5000:] + "\n" + report = JUnitReport.error(kind, label, msg, details) + report.write(report_file) + return report, self._status(result, errors=1, tests=1, time_elapsed=time_elapsed, wall_time=wall_time), True + + def _build_cmd(self, unit: Unit, ctx: ExecContext): + """Build ``(cmd, env, report_file, label, report_suffix)`` for a unit. + + Device-variant selection is the unit's mask, set as ``ISAACLAB_TEST_DEVICES`` + for ``test_devices()`` — never ``-k``. A split unit (either half) or a + cpu-less shard also injects the selector plugin to drop out-of-scope + variants (including literal device-param ones the mask cannot narrow). + """ + config = self._config + # Split units of one file share a slug, so suffix the report with the mask; + # a lone unit keeps the suffix-free name the mgpu aggregator unslugs to a path. + report_suffix = f"-{unit.mask}" if unit.split else "" + pass_file_label = f"{ctx.file_name}{report_suffix}" + # Slug the full path (not the basename) so concurrent shards running + # same-basename files don't collide on the shared workspace mount. + report_slug = str(unit.file).replace("/", "__").replace("\\", "__") + report_file = os.path.join(config.report_dir, f"test-reports-{report_slug}{report_suffix}.xml") + + env = dict(ctx.env) + env["ISAACLAB_TEST_DEVICES"] = unit.mask + select_by_device = unit.split or unit.mask[:1] == "0" + if select_by_device: + env["PYTHONPATH"] = config.tools_dir + os.pathsep + env.get("PYTHONPATH", "") + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit + "--no-header", + f"--config-file={config.config_file}", + f"--junitxml={report_file}", + "--tb=short", + ] + if select_by_device: + cmd += ["-p", "test_runner.selector"] + if config.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + cmd.append(str(unit.file)) + return cmd, env, report_file, pass_file_label, report_suffix + + def _retry_failed_in_fresh_process( + self, + *, + unit, + ctx, + cmd, + env, + report_file, + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout, + stderr, + kill_reason, + wall_time, + pre, + ): + """Re-run a failed file in fresh processes (for files with stale-state flakiness).""" + has_failures = errors > 0 or failures > 0 + attempts = 0 + max_retries = self._config.process_failure_retries.get(ctx.file_name, 0) + while has_failures and attempts < max_retries: + attempts += 1 + print( + f"⚠️ {unit.file}: failed in subprocess" + f" (attempt {attempts}/{max_retries + 1}), retrying in fresh process..." + ) + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + returncode, stdout, stderr, kill_reason, wall_time, pre = self._capture.run( + cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) + if not os.path.exists(report_file): + break + try: + report, errors, failures, skipped, tests, time_elapsed = JUnitReport.read(report_file, ctx.file_name) + has_failures = errors > 0 or failures > 0 + except Exception as e: + print(f"Error reading retry test report {report_file}: {e}") + report, errors, failures, skipped, tests, time_elapsed = report, 1, 0, 0, 0, 0.0 + has_failures = True + break + return report, errors, failures, skipped, tests, time_elapsed, returncode, kill_reason, wall_time, has_failures + + def run(self, unit: Unit, ctx: ExecContext) -> tuple: + """Run ``unit`` and return ``(xml_report_or_None, status_dict, was_failure)``.""" + cap = self._capture + cmd, env, report_file, label, suffix = self._build_cmd(unit, ctx) + + # -- launch, retrying startup hangs / hard timeouts -- + returncode, stdout, stderr, kill_reason, wall_time, pre = -1, b"", b"", "", 0.0, "" + startup_attempts = timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + returncode, stdout, stderr, kill_reason, wall_time, pre = cap.run( + cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) + has_report = os.path.exists(report_file) + if kill_reason == "startup_hang" and startup_attempts < self._config.startup_hang_retries: + startup_attempts += 1 + print( + f"⚠️ {unit.file}{suffix}: startup hang after {ctx.startup_deadline}s" + f" (attempt {startup_attempts}/{self._config.startup_hang_retries + 1}), retrying..." + ) + if stderr: + print("=== STDERR (last 5000 chars) ===") + print(stderr.decode("utf-8", errors="replace")[-5000:]) + print(cap.diagnostics(pre)) + continue + if kill_reason == "timeout" and not has_report and timeout_attempts < self._config.timeout_retries: + timeout_attempts += 1 + print( + f"⚠️ {unit.file}{suffix}: timeout after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{self._config.timeout_retries + 1}), retrying..." + ) + if stdout: + print("=== STDOUT (last 5000 chars) ===") + print(stdout.decode("utf-8", errors="replace")[-5000:]) + print(cap.diagnostics(pre)) + continue + break + + # -- classify the outcome -- + has_report = os.path.exists(report_file) + if kill_reason == "startup_hang": + return self._error_result( + "startup_hang", + label, + suffix, + unit, + ctx, + "STARTUP_HANG", + wall_time, + cap, + pre, + msg=f"Startup hang after {ctx.startup_deadline}s (retried {self._config.startup_hang_retries} time(s))", + stdout=stdout, + stderr=stderr, + report_file=report_file, + ) + if kill_reason == "timeout" and not has_report: + return self._error_result( + "timeout", + label, + suffix, + unit, + ctx, + "TIMEOUT", + wall_time, + cap, + pre, + msg=f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))", + stdout=stdout, + stderr=stderr, + report_file=report_file, + time_elapsed=ctx.timeout, + ) + if not has_report: + reason = ( + cap.signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + return self._error_result( + "crash", + label, + suffix, + unit, + ctx, + "CRASHED", + wall_time, + cap, + "", + msg=reason, + stdout=stdout, + stderr=stderr, + report_file=report_file, + ) + + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {unit.file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test completed)") + try: + report, errors, failures, skipped, tests, time_elapsed = JUnitReport.read(report_file, label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return None, self._status("FAILED", errors=1, tests=0, wall_time=wall_time), True + + report, errors, failures, skipped, tests, time_elapsed, returncode, kill_reason, wall_time, has_failures = ( + self._retry_failed_in_fresh_process( + unit=unit, + ctx=ctx, + cmd=cmd, + env=env, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout=stdout, + stderr=stderr, + kill_reason=kill_reason, + wall_time=wall_time, + pre=pre, + ) + ) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_failures + was_failure = has_failures or (returncode != 0 and not shutdown_hanged) + result = "passed (shutdown hanged)" if shutdown_hanged else ("FAILED" if has_failures else "passed") + return ( + report, + self._status( + result, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + wall_time=wall_time, + ), + was_failure, + ) diff --git a/tools/test_runner/planning.py b/tools/test_runner/planning.py new file mode 100644 index 000000000000..b8aa2da655b7 --- /dev/null +++ b/tools/test_runner/planning.py @@ -0,0 +1,207 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""What to run: the runner configuration, the work unit, and the planner. + +:class:`RunnerConfig` centralizes every knob the runner used to read ad hoc from +the environment or hardcode as a module constant. :class:`Unit` is the work item +(a file plus the device mask to run it under). :class:`Planner` turns a file into +its units, splitting a ``device_isolated`` file into one single-device unit per +device when the run spans more than one. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field + +# Per-file fresh-process retry counts for tests that can enter stale render states. +# A default so the runner has no hardcoded file list; override via RunnerConfig. +_DEFAULT_PROCESS_FAILURE_RETRIES = { + "test_visualizer_integration_physx.py": 4, + "test_visualizer_integration_newton.py": 4, + "test_visualizer_tiled_integration_physx.py": 4, + "test_visualizer_tiled_integration_newton.py": 4, +} + + +@dataclass +class RunnerConfig: + """All runner behavior as data, so nothing is hardcoded mid-pipeline. + + Build it once with :meth:`from_env` at the pytest entry point and pass it to + every stage. Fields group into: run scope (mask, queue, isaacsim_ci), + collection (filters/sharding), per-unit execution (timeouts, deadlines, + retries, cold-cache), and reporting (paths). + + Attributes: + workspace_root: Repository root; pytest ``--config-file`` and the + ``tools/`` plugin dir derive from it. + runtime_mask: Device mask for the run (e.g. ``"110"`` cpu+cuda:0, + ``"0001"`` a shard). Concrete — never the open-ended ``"X"`` form. + queue_path: Shared work-queue root for mgpu work-stealing; empty when the + runner iterates its own file list. + isaacsim_ci: Whether to pass ``-m isaacsim_ci`` to each subprocess. + filter_pattern: Substring a test path must contain to run; empty = all. + exclude_pattern: Comma-separated substrings; a path matching any is skipped. + include_files: Basenames to restrict the run to; empty = no restriction. + quarantined_only: Run only the quarantined set. + curobo_only: Run only the cuRobo set. + shard_index: This file-shard's index, or ``None`` for no file sharding. + shard_count: Total file shards, or ``None``. + default_timeout: Per-unit hard timeout [s] when a file has no override. + per_file_timeouts: Basename -> hard timeout [s] overrides. + cold_cache_buffer: Extra timeout [s] for the first camera-enabled test + (shader compile on a cold cache). + cold_cache_marker: Source substring that marks a camera-enabled test. + startup_deadline: Seconds to reach AppLauncher init / pytest collection + before declaring a startup hang. + startup_hang_retries: Startup-hang retries before giving up. + timeout_retries: Hard-timeout retries before giving up. + shutdown_grace_period: Seconds to wait for clean exit after the report + file appears before killing a shutdown-hung process. + process_failure_retries: Basename -> extra fresh-process retries on test + failure (for tests with stale-state flakiness). + report_dir: Directory for per-unit and aggregate JUnit XML. + result_file: Aggregate report filename within :attr:`report_dir`. + """ + + workspace_root: str + # -- run scope -- + runtime_mask: str = "110" + queue_path: str = "" + isaacsim_ci: bool = False + # -- collection -- + filter_pattern: str = "" + exclude_pattern: str = "" + include_files: frozenset[str] = frozenset() + quarantined_only: bool = False + curobo_only: bool = False + shard_index: int | None = None + shard_count: int | None = None + # -- per-unit execution -- + default_timeout: int = 600 + per_file_timeouts: dict = field(default_factory=dict) + cold_cache_buffer: int = 700 + cold_cache_marker: str = "enable_cameras=True" + startup_deadline: int = 120 + startup_hang_retries: int = 2 + timeout_retries: int = 0 + shutdown_grace_period: int = 30 + process_failure_retries: dict = field(default_factory=lambda: dict(_DEFAULT_PROCESS_FAILURE_RETRIES)) + # -- reporting -- + report_dir: str = "tests" + result_file: str = "full_report.xml" + + @property + def config_file(self) -> str: + """Path passed to pytest's ``--config-file`` for each subprocess.""" + return os.path.join(self.workspace_root, "pyproject.toml") + + @property + def tools_dir(self) -> str: + """Directory added to a subprocess PYTHONPATH so it can import the selector.""" + return os.path.join(self.workspace_root, "tools") + + @classmethod + def from_env(cls, workspace_root: str) -> RunnerConfig: + """Build a config from the environment and ``test_settings``. + + This is the single place environment variables are read; the rest of the + runner takes the resulting config. + + Args: + workspace_root: Repository root. + + Returns: + A fully-populated config for this run. + """ + import test_settings # tools/ module; imported lazily so unit tests need no env + + shard_index = os.environ.get("TEST_SHARD_INDEX", "") + shard_count = os.environ.get("TEST_SHARD_COUNT", "") + raw_includes = os.environ.get("TEST_INCLUDE_FILES", "").split(",") + include = {os.path.basename(f.strip()) for f in raw_includes if f.strip()} + return cls( + workspace_root=workspace_root, + # An unset mask means single-GPU CI: cpu + cuda:0, matching test_devices()'s default. + runtime_mask=os.environ.get("ISAACLAB_TEST_DEVICES") or "110", + queue_path=os.environ.get("ISAACLAB_TEST_QUEUE", ""), + isaacsim_ci=os.environ.get("ISAACSIM_CI_SHORT", "false") == "true", + filter_pattern=os.environ.get("TEST_FILTER_PATTERN", ""), + exclude_pattern=os.environ.get("TEST_EXCLUDE_PATTERN", ""), + include_files=frozenset(include), + quarantined_only=os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true", + curobo_only=os.environ.get("TEST_CUROBO_ONLY", "false") == "true", + shard_index=int(shard_index) if shard_index else None, + shard_count=int(shard_count) if shard_count else None, + default_timeout=test_settings.DEFAULT_TIMEOUT, + per_file_timeouts=dict(test_settings.PER_TEST_TIMEOUTS), + result_file=os.environ.get("TEST_RESULT_FILE", "full_report.xml"), + ) + + +@dataclass(frozen=True) +class Unit: + """One pytest subprocess to run. + + Attributes: + file: Test file to run. + mask: Device mask set as ``ISAACLAB_TEST_DEVICES`` for the subprocess; + ``test_devices()`` reads it to select the device variants. + split: Whether this is one of several units from splitting a + ``device_isolated`` file. The executor uses it only to give split + units distinct report filenames; a lone unit keeps the suffix-free + name the mgpu aggregator expects. + """ + + file: str + mask: str + split: bool = False + + +class Planner: + """Turns a test file into the units to run, given the run's device mask. + + A ``device_isolated`` file (a backend with a process-global device lock, e.g. + ovphysx) splits into one single-device unit per device, but only when the run + spans more than one device — an mgpu shard's single-device mask yields one + unit either way. + """ + + _ISOLATED_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_isolated\b", re.MULTILINE) + """Match a module-level ``pytestmark`` mentioning ``device_isolated`` (single or list form).""" + + def __init__(self, runtime_mask: str): + if "X" in runtime_mask: + raise ValueError(f"runtime mask {runtime_mask!r} must be concrete; 'X' is a scope-only construct") + self._mask = runtime_mask + + @staticmethod + def _single_bit_mask(index: int, width: int) -> str: + """Return a width-``width`` mask with only ``index`` set, e.g. ``(1, 3) -> "010"``.""" + return "".join("1" if pos == index else "0" for pos in range(width)) + + def is_isolated(self, source: str) -> bool: + """Whether a test file's source declares the ``device_isolated`` marker.""" + return bool(self._ISOLATED_MARK_RE.search(source)) + + def plan(self, test_file: str, source: str) -> list[Unit]: + """Plan the units for one file. + + Args: + test_file: Path to the test file. + source: The file's source text (the caller already reads it for the + cold-cache check), inspected for the isolation marker. + + Returns: + One unit at the run's mask, or — for an isolated file on a + multi-device run — one single-device unit per active device. + """ + set_bits = [pos for pos, char in enumerate(self._mask) if char == "1"] + if self.is_isolated(source) and len(set_bits) > 1: + return [Unit(test_file, self._single_bit_mask(pos, len(self._mask)), split=True) for pos in set_bits] + return [Unit(test_file, self._mask)] diff --git a/tools/test_runner/process.py b/tools/test_runner/process.py new file mode 100644 index 000000000000..d1524494157a --- /dev/null +++ b/tools/test_runner/process.py @@ -0,0 +1,264 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Low-level subprocess execution and JUnit report I/O for one pytest run. + +:class:`ProcessCapture` launches a command, streams its output live, and kills it +on a startup hang, hard timeout, or shutdown hang, returning enough to classify +the outcome. :class:`JUnitReport` reads a produced report or synthesizes an error +report when the process produced none. Both are device- and unit-agnostic; the +unit-level orchestration that uses them is :class:`~test_runner.execution.UnitRunner`. +""" + +from __future__ import annotations + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time + +from junitparser import Error, JUnitXml, TestCase, TestSuite + +_SIGNAL_DESCRIPTIONS = { + 1: "SIGHUP — session leader exit or orphaned process cleanup", + 6: "SIGABRT", + 9: "SIGKILL — likely OOM killed", + 11: "SIGSEGV — segmentation fault", + 15: "SIGTERM", +} +_DIAG_LIMIT = 10000 # chars; truncate diagnostics so a crash dump can't flood the log + + +class ProcessCapture: + """Run a pytest subprocess with hang/timeout detection and live output streaming.""" + + def __init__(self, shutdown_grace_period: int): + # Seconds to allow for clean exit after the report file appears, before + # killing a process that hung in SimulationApp.close() / Kit shutdown. + self._grace = shutdown_grace_period + + @staticmethod + def signal_description(sig: int) -> str: + """Human-readable reason a process was killed by signal ``sig``.""" + base = f"Process killed by signal {sig}" + desc = _SIGNAL_DESCRIPTIONS.get(sig) + return f"{base} ({desc})" if desc else base + + @staticmethod + def _system_diagnostics() -> str: + """GPU/memory/process snapshot for crash investigation. Never raises.""" + sections = [] + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + return "\n\n".join(sections) + + def diagnostics(self, pre_kill: str = "") -> str: + """Return diagnostics (a pre-kill snapshot if given, else a fresh one), truncated.""" + diag = pre_kill or self._system_diagnostics() + return diag[:_DIAG_LIMIT] + "\n... (truncated)" if len(diag) > _DIAG_LIMIT else diag + + def run(self, cmd, timeout, env, startup_deadline=0, report_file=""): + """Run ``cmd``, streaming output, and kill it on a hang or timeout. + + Args: + cmd: argv to execute. + timeout: hard wall-clock limit [s]. + env: subprocess environment. + startup_deadline: if > 0, kill when neither AppLauncher-init nor + pytest-collection markers appear within this many seconds. + report_file: when set, the process gets only the configured grace + period to exit after this file appears on disk. + + Returns: + ``(returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, + pre_kill_diag)``; ``kill_reason`` is ``""`` for a clean exit, else + ``"startup_hang"`` / ``"timeout"`` / ``"shutdown_hang"``. + """ + stdout_data = b"" + stderr_data = b"" + process = None + try: + # Own session so orphaned Kit children can't SIGHUP the next test's group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + while process.poll() is None: + elapsed = time.time() - start_time + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + self._grace + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = self._system_diagnostics() + try: # kill the whole group (test + any Kit children) + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + except subprocess.TimeoutExpired: + pass + return -1, stdout_data, stderr_data, kill_reason, time.time() - start_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + try: # drain output written just before/after exit + rem_out, rem_err = process.communicate(timeout=10) + stdout_data += rem_out + stderr_data += rem_err + except Exception: + pass + try: # reap any orphaned Kit/Isaac Sim children + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + return process.returncode, stdout_data, stderr_data, "", time.time() - start_time, "" + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +class JUnitReport: + """Read a unit's JUnit report, or synthesize one when the process produced none.""" + + @staticmethod + def error(prefix: str, file_name: str, message: str, details: str) -> JUnitXml: + """Build a one-case JUnit report standing in for a run that wrote no report.""" + suite_name = os.path.splitext(file_name)[0] + suite = TestSuite(name=f"{prefix}_{suite_name}") + case = TestCase(name="test_execution", classname=suite_name) + result = Error(message=message) + result.text = details + case.result = result + suite.add_testcase(case) + report = JUnitXml() + report.add_testsuite(suite) + return report + + @staticmethod + def read(report_file: str, file_name: str): + """Parse a JUnit report; rename its synthetic ``pytest`` suite to the file. + + Returns: + ``(report, errors, failures, skipped, tests, time_elapsed)``. + """ + report = JUnitXml.fromfile(report_file) + for suite in report: + if suite.name == "pytest": + suite.name = os.path.splitext(file_name)[0] + report.write(report_file) + return ( + report, + int(report.errors) if report.errors is not None else 0, + int(report.failures) if report.failures is not None else 0, + int(report.skipped) if report.skipped is not None else 0, + int(report.tests) if report.tests is not None else 0, + float(report.time) if report.time is not None else 0.0, + ) diff --git a/tools/test_runner/selector.py b/tools/test_runner/selector.py new file mode 100644 index 000000000000..53710d095c54 --- /dev/null +++ b/tools/test_runner/selector.py @@ -0,0 +1,121 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""In-process pytest plugin: keep only the tests a unit's device mask allows. + +The executor injects this into a unit's subprocess with ``-p test_runner.selector`` +for any unit that must run on a single device — both halves of a ``device_isolated`` +split and every mgpu shard. It runs inside that subprocess, at collection time, so +a process locked to one device by a backend (e.g. ovphysx) never also tries another. + +It is registered as a class instance (see :func:`pytest_configure`): the hooks are +methods of :class:`DeviceSelect`, so the device mask is read once into instance +state instead of from the environment on every call. +""" + +from __future__ import annotations + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +class DeviceSelect: + """Decide, per collected test, whether it belongs to this unit's device mask. + + Device-parametrized tests are kept iff their device is active in the mask + (this catches literal ``["cuda:0", "cpu"]`` variants the mask cannot narrow at + parametrize time); agnostic tests (no ``device`` param) are kept only when cpu + is in the mask, i.e. the default unit — never on a cuda split unit or a shard. + """ + + def __init__(self, mask: str): + self._mask = mask + + @staticmethod + def _position(device) -> int | None: + """Mask position for a device: cpu -> 0, cuda:k -> k + 1, else None (agnostic/unknown).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None + + def _active(self, position: int | None) -> bool: + """Whether a mask position is included; a trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (self._mask[:-1], True) if self._mask.endswith("X") else (self._mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + def keeps(self, device: str | None) -> bool: + """Whether a test on ``device`` (or ``None`` if agnostic) runs in this unit.""" + if device is None: + return self._active(0) # agnostic: only in a cpu-inclusive unit + return self._active(self._position(device)) + + # -- pytest hooks (methods of the registered instance) -- + + def pytest_collection_modifyitems(self, config, items): + """PYTEST HOOK — filter the collected tests in place to this unit's devices. + + Called by pytest after collection, before any test runs. + + Args: + config: the session ``pytest.Config``; ``config.hook.pytest_deselected`` + reports the dropped tests so they count as deselected, not missing. + items: the live, ordered ``list[pytest.Item]`` of collected tests. This + hook mutates it in place (``items[:] = keep``) — that is how pytest + learns what to run; reassigning a new list would have no effect. + + Returns: + None. Its effect is the in-place edit of ``items`` plus the + ``pytest_deselected`` report. + """ + keep, drop = [], [] + for item in items: + callspec = getattr(item, "callspec", None) + device = callspec.params.get("device") if callspec is not None else None + (keep if self.keeps(device) else drop).append(item) + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + def pytest_sessionfinish(self, session, exitstatus): + """PYTEST HOOK — treat a unit that selected nothing as a clean pass. + + Called by pytest once the whole session ends. + + Args: + session: the ``pytest.Session``; reassign ``session.exitstatus`` to + change the process exit code. + exitstatus: the proposed exit code (a ``pytest.ExitCode``). + + Returns: + None. When this unit deselected every test (NO_TESTS_COLLECTED), it + rewrites the exit to OK so the runner does not read a non-zero exit as + a failure — "nothing in scope for this file" is success here. + """ + if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK + + +def pytest_configure(config): + """PYTEST HOOK — the plugin's entry point: register the device selector. + + Loading this module with ``-p test_runner.selector`` makes pytest call this + module-level hook during config init. It builds a :class:`DeviceSelect` from + the unit's mask (the ``ISAACLAB_TEST_DEVICES`` the executor set) and registers + the instance, so the instance's hook methods become active for the session. + + Args: + config: the session ``pytest.Config`` (provides the plugin manager). + + Returns: + None. + """ + config.pluginmanager.register(DeviceSelect(os.environ.get(_RUNTIME_ENV, ""))) diff --git a/tools/test_runner/tests/__init__.py b/tools/test_runner/tests/__init__.py new file mode 100644 index 000000000000..440ba074bf57 --- /dev/null +++ b/tools/test_runner/tests/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the test_runner framework (run locally; see package docstring).""" diff --git a/tools/test_runner/tests/test_execution.py b/tools/test_runner/tests/test_execution.py new file mode 100644 index 000000000000..3feaa2f30b0f --- /dev/null +++ b/tools/test_runner/tests/test_execution.py @@ -0,0 +1,94 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.execution: the unit command contract + status merge.""" + +from __future__ import annotations + +import os + +from test_runner.execution import ExecContext, UnitRunner, merge_statuses +from test_runner.planning import RunnerConfig, Unit + + +def _runner(isaacsim_ci=False): + return UnitRunner(RunnerConfig(workspace_root="/ws", isaacsim_ci=isaacsim_ci)) + + +def _ctx(): + return ExecContext(file_name="test_x.py", timeout=60, startup_deadline=30, env={"BASE": "1"}) + + +def _unit(mask, split=False): + return Unit("source/pkg/test_x.py", mask, split=split) + + +def test_mix_ok_unit_sets_mask_no_selector_no_k(): + cmd, env, report, label, suffix = _runner()._build_cmd(_unit("110"), _ctx()) + assert env["ISAACLAB_TEST_DEVICES"] == "110" + assert env["BASE"] == "1" # base env preserved + assert "test_runner.selector" not in cmd + assert "-k" not in cmd + assert report == "tests/test-reports-source__pkg__test_x.py.xml" + assert label == "test_x.py" + assert suffix == "" + assert cmd[-1] == "source/pkg/test_x.py" + + +def test_cpu_split_unit_injects_selector_and_is_suffixed(): + cmd, _, report, _, suffix = _runner()._build_cmd(_unit("100", split=True), _ctx()) + assert "test_runner.selector" in cmd # cpu half still drops out-of-scope cuda variants + assert suffix == "-100" + assert report.endswith("test_x.py-100.xml") + + +def test_cuda_split_unit_injects_selector_and_tools_pythonpath(): + cmd, env, _, _, _ = _runner()._build_cmd(_unit("010", split=True), _ctx()) + assert cmd.count("-p") == 1 and "test_runner.selector" in cmd + assert env["ISAACLAB_TEST_DEVICES"] == "010" + assert env["PYTHONPATH"].split(os.pathsep)[0] == os.path.join("/ws", "tools") + + +def test_shard_unit_injects_selector_without_a_suffix(): + cmd, env, report, _, suffix = _runner()._build_cmd(_unit("0001"), _ctx()) + assert "test_runner.selector" in cmd # cpu-less shard + assert suffix == "" + assert report == "tests/test-reports-source__pkg__test_x.py.xml" + + +def test_isaacsim_ci_adds_marker_selector(): + cmd, *_ = _runner(isaacsim_ci=True)._build_cmd(_unit("110"), _ctx()) + assert any(cmd[i] == "-m" and cmd[i + 1] == "isaacsim_ci" for i in range(len(cmd) - 1)) + + +def test_no_k_selector_for_any_mask(): + for mask, split in [("110", False), ("100", True), ("010", True), ("0001", False)]: + cmd, *_ = _runner()._build_cmd(_unit(mask, split=split), _ctx()) + assert "-k" not in cmd + + +def test_merge_statuses_sums_counters_and_takes_worst_result(): + a = { + "errors": 0, + "failures": 0, + "skipped": 1, + "tests": 3, + "time_elapsed": 1.0, + "wall_time": 2.0, + "result": "passed", + } + b = { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 2, + "time_elapsed": 0.5, + "wall_time": 1.0, + "result": "FAILED", + } + merged = merge_statuses(a, b) + assert merged["tests"] == 5 and merged["skipped"] == 1 and merged["errors"] == 1 + assert merged["result"] == "FAILED" # more severe wins + assert merge_statuses(None, b) == b # first unit seeds the entry diff --git a/tools/test_runner/tests/test_planning.py b/tools/test_runner/tests/test_planning.py new file mode 100644 index 000000000000..359aeb6c65ff --- /dev/null +++ b/tools/test_runner/tests/test_planning.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.planning (Planner + RunnerConfig).""" + +from __future__ import annotations + +import pytest + +from test_runner.planning import Planner, RunnerConfig, Unit + +_ISOLATED = "import pytest\npytestmark = pytest.mark.device_isolated\n" + + +# ---- Planner.plan ---- + + +def test_mix_ok_file_is_a_single_unit(): + assert Planner("110").plan("a.py", "def test_x(): pass\n") == [Unit("a.py", "110")] + + +def test_isolated_file_splits_on_a_multi_device_run(): + assert Planner("110").plan("a.py", _ISOLATED) == [Unit("a.py", "100", split=True), Unit("a.py", "010", split=True)] + + +def test_isolated_file_does_not_split_on_a_single_device_run(): + assert Planner("0001").plan("a.py", _ISOLATED) == [Unit("a.py", "0001")] + + +def test_split_uses_each_set_bit_position(): + assert Planner("1010").plan("a.py", _ISOLATED) == [ + Unit("a.py", "1000", split=True), + Unit("a.py", "0010", split=True), + ] + + +def test_runtime_mask_with_trailing_x_is_rejected(): + with pytest.raises(ValueError): + Planner("11X") + + +# ---- Planner.is_isolated ---- + + +def test_is_isolated_detects_single_and_list_forms(): + p = Planner("110") + assert p.is_isolated("pytestmark = pytest.mark.device_isolated") is True + assert p.is_isolated("pytestmark = [pytest.mark.device_isolated, pytest.mark.slow]") is True + + +def test_is_isolated_ignores_comments_and_other_marks(): + p = Planner("110") + assert p.is_isolated("# device_isolated explains the lock") is False + assert p.is_isolated("pytestmark = pytest.mark.slow") is False + + +# ---- RunnerConfig ---- + + +def test_config_defaults_and_derived_paths(): + c = RunnerConfig(workspace_root="/ws") + assert c.config_file == "/ws/pyproject.toml" + assert c.tools_dir == "/ws/tools" + assert c.runtime_mask == "110" # single-GPU default + assert c.process_failure_retries["test_visualizer_integration_physx.py"] == 4 + + +def test_from_env_reads_mask_flags_and_includes(monkeypatch): + monkeypatch.setenv("ISAACLAB_TEST_DEVICES", "0001") + monkeypatch.setenv("ISAACSIM_CI_SHORT", "true") + monkeypatch.setenv("TEST_INCLUDE_FILES", "a/b/test_x.py, test_y.py") + c = RunnerConfig.from_env("/ws") + assert c.runtime_mask == "0001" + assert c.isaacsim_ci is True + assert c.include_files == frozenset({"test_x.py", "test_y.py"}) + + +def test_from_env_defaults_mask_when_unset(monkeypatch): + monkeypatch.delenv("ISAACLAB_TEST_DEVICES", raising=False) + assert RunnerConfig.from_env("/ws").runtime_mask == "110" diff --git a/tools/test_runner/tests/test_selector.py b/tools/test_runner/tests/test_selector.py new file mode 100644 index 000000000000..e3011b4e0126 --- /dev/null +++ b/tools/test_runner/tests/test_selector.py @@ -0,0 +1,112 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.selector.DeviceSelect. + +The class form is directly testable: construct DeviceSelect(mask) and call keeps() +or the hooks with fakes — no env var, no monkeypatch. +""" + +from __future__ import annotations + +import pytest + +from test_runner.selector import DeviceSelect + + +class _Callspec: + def __init__(self, params): + self.params = params + + +class _Item: + def __init__(self, name, params=None): + self.name = name + if params is not None: + self.callspec = _Callspec(params) + + +class _Hook: + def __init__(self): + self.deselected = [] + + def pytest_deselected(self, items): + self.deselected.extend(items) + + +class _Config: + def __init__(self): + self.hook = _Hook() + + +# ---- pure mask logic ---- + + +def test_position_maps_cpu_and_cuda(): + assert DeviceSelect._position("cpu") == 0 + assert DeviceSelect._position("cuda:0") == 1 + assert DeviceSelect._position("cuda:2") == 3 + assert DeviceSelect._position(None) is None + + +def test_active_handles_trailing_x(): + assert DeviceSelect("00X")._active(3) is True # cuda:2 via the trailing fill + assert DeviceSelect("110")._active(2) is False + + +def test_keeps_cpu_unit(): + s = DeviceSelect("100") + assert s.keeps("cpu") is True + assert s.keeps("cuda:0") is False # literal cuda variant dropped from the cpu unit + assert s.keeps(None) is True # agnostic kept (cpu in mask) + + +def test_keeps_cuda_unit(): + s = DeviceSelect("010") + assert s.keeps("cuda:0") is True + assert s.keeps("cpu") is False # literal cpu variant dropped from the cuda unit + assert s.keeps(None) is False # agnostic dropped (no cpu) + + +def test_keeps_shard(): + s = DeviceSelect("0001") # cuda:2 shard + assert s.keeps("cuda:2") is True + assert s.keeps("cuda:0") is False + assert s.keeps("cpu") is False + assert s.keeps(None) is False + + +# ---- the collection hook ---- + + +def test_modifyitems_drops_out_of_scope_and_agnostic_in_cuda_unit(): + cpu = _Item("t[cpu]", {"device": "cpu"}) + cuda = _Item("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _Item("t_logic") + items = [cpu, cuda, agnostic] + config = _Config() + DeviceSelect("010").pytest_collection_modifyitems(config, items) + assert items == [cuda] + assert set(config.hook.deselected) == {cpu, agnostic} + + +def test_modifyitems_keeps_everything_in_scope_for_default_mask(): + cpu = _Item("t[cpu]", {"device": "cpu"}) + cuda = _Item("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _Item("t_logic") + items = [cpu, cuda, agnostic] + config = _Config() + DeviceSelect("110").pytest_collection_modifyitems(config, items) + assert items == [cpu, cuda, agnostic] + assert config.hook.deselected == [] + + +def test_sessionfinish_maps_no_tests_collected_to_ok(): + class _Session: + exitstatus = pytest.ExitCode.NO_TESTS_COLLECTED + + session = _Session() + DeviceSelect("010").pytest_sessionfinish(session, pytest.ExitCode.NO_TESTS_COLLECTED) + assert session.exitstatus == pytest.ExitCode.OK From 957cb59941164f29a6b7495754d9a8e005bfaf81 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 12 Jun 2026 04:33:29 +0000 Subject: [PATCH 23/23] Switch the runner to the test_runner framework; remove old modules conftest.py is now two documented hooks that build a RunnerConfig from the environment and run the Session lifecycle (collect -> plan -> execute -> report), exiting with the runner's code. Add session.py (WorkQueue, Collector, Reporter, Session) completing the framework, and delete the flat _device_plan/_device_exec/ _device_select modules and their tests. Behavior is preserved: the same tests run on the same devices on both lanes, verified by a literal-device-param e2e (the cpu unit drops cuda variants, the cuda unit drops cpu + agnostic) and 32 framework unit tests. ruff clean. --- .github/workflows/test-multi-gpu-pytest.yaml | 2 +- tools/_device_exec.py | 775 ------------------- tools/_device_plan.py | 124 --- tools/_device_select.py | 81 -- tools/conftest.py | 507 +----------- tools/test_device_exec.py | 87 --- tools/test_device_plan.py | 107 --- tools/test_device_select.py | 131 ---- tools/test_runner/planning.py | 5 + tools/test_runner/session.py | 350 +++++++++ tools/test_runner/tests/test_session.py | 97 +++ 11 files changed, 486 insertions(+), 1780 deletions(-) delete mode 100644 tools/_device_exec.py delete mode 100644 tools/_device_plan.py delete mode 100644 tools/_device_select.py delete mode 100644 tools/test_device_exec.py delete mode 100644 tools/test_device_plan.py delete mode 100644 tools/test_device_select.py create mode 100644 tools/test_runner/session.py create mode 100644 tools/test_runner/tests/test_session.py diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml index 7e0b21e3be0e..e9bbeb07832b 100644 --- a/.github/workflows/test-multi-gpu-pytest.yaml +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -115,7 +115,7 @@ jobs: # # Within a discovered file, tests that are NOT parametrized over the # ``device`` argument are deselected at collection time by the - # ``_device_select`` plugin (injected by ``tools/conftest.py`` on every + # ``test_runner.selector`` plugin (which the runner injects on every # shard): single-GPU CI already covers them on ``cuda:0`` and re-running on # every non-default shard adds wall-time without surfacing any new failure mode. id: discover diff --git a/tools/_device_exec.py b/tools/_device_exec.py deleted file mode 100644 index b0fd5f2f57aa..000000000000 --- a/tools/_device_exec.py +++ /dev/null @@ -1,775 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Executor for the per-file CI test runner. - -Runs one :class:`~_device_plan.Unit` as its own pytest subprocess and owns the -subprocess lifecycle: streaming capture, startup-hang / hard-timeout / -shutdown-hang detection, fresh-process retries, and JUnit report parsing. - -Device selection is the planner's job, expressed as the unit's mask: the -executor sets ``ISAACLAB_TEST_DEVICES`` to that mask (``test_devices()`` reads -it to pick the device variants) and, for a split unit (either half) or an mgpu -shard, injects the ``_device_select`` plugin to drop out-of-scope device variants -— including literal device-param variants the mask alone cannot narrow — and -device-agnostic tests. The executor never reads a marker or a ``-k`` selector. -""" - -from __future__ import annotations - -import contextlib -import os -import select -import signal -import subprocess -import sys -import time -from dataclasses import dataclass - -from junitparser import Error, JUnitXml, TestCase, TestSuite - -from _device_plan import Unit # isort: skip - -COLD_CACHE_BUFFER = 700 -"""Extra seconds added to the first camera-enabled test's hard timeout. - -The first test that uses ``enable_cameras=True`` may compile shaders during its -run (~600 s). This buffer prevents that from being misreported as a test -timeout. Only the first such test gets the extension — after it runs, the -on-disk cache is populated. -""" - -STARTUP_DEADLINE = 120 -"""Seconds to wait for AppLauncher init or pytest collection before declaring a -startup hang. - -AppLauncher prints ``[ISAACLAB] AppLauncher initialization complete`` to -``sys.__stderr__`` (never suppressed) when Kit finishes initializing, and pytest -prints ``collected N items`` to stdout after collection. If neither appears -within this deadline the process is treated as hung. Kit startup can exceed -60 s on cold CI workers, so this catches real startup hangs without killing -legitimate slow launches. -""" - -STARTUP_HANG_RETRIES = 2 -"""Number of times to retry a test that hangs during startup before giving up.""" - -TIMEOUT_RETRIES = 0 -"""Number of times to retry a test that reaches its hard timeout before giving up.""" - -PROCESS_FAILURE_RETRIES_BY_FILE = { - "test_visualizer_integration_physx.py": 4, - "test_visualizer_integration_newton.py": 4, - "test_visualizer_tiled_integration_physx.py": 4, - "test_visualizer_tiled_integration_newton.py": 4, -} -"""Extra fresh-process attempts for visualizer tests that can enter stale render states.""" - -SHUTDOWN_GRACE_PERIOD = 30 -"""Seconds to wait for clean exit after the JUnit XML report file appears. - -When a test completes and writes its JUnit report, the subprocess may hang -during ``SimulationApp.close()`` or Kit shutdown. Rather than wasting the -full hard timeout, we give the process a short grace period to exit, then -kill it. The test results are taken from the report file (pass/fail), not -from the kill. -""" - - -def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): - """Run a command with timeout and capture all output while streaming in real-time. - - Args: - cmd: Command to execute. - timeout: Maximum wall-clock seconds before the process is killed. - env: Environment variables for the subprocess. - startup_deadline: If > 0, the process is killed early when neither - ``AppLauncher initialization complete`` (stderr) nor ``collected`` - (stdout) appears within this many seconds. - report_file: Path to the JUnit XML report file. When set, the process - is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after - the file appears on disk. - - Returns: - Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, - wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, - ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process - did not reach pytest collection in time, or ``"shutdown_hang"`` when - the test completed but the process hung during shutdown. - """ - stdout_data = b"" - stderr_data = b"" - process = None - - try: - # Each test gets its own session so orphaned Kit/Isaac Sim child - # processes cannot send SIGHUP to the next test's process group. - process = subprocess.Popen( - cmd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0, - universal_newlines=False, - start_new_session=True, - ) - pgid = os.getpgid(process.pid) - - stdout_fd = process.stdout.fileno() - stderr_fd = process.stderr.fileno() - - try: - import fcntl - - for fd in [stdout_fd, stderr_fd]: - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - except ImportError: - pass - - start_time = time.time() - startup_done = startup_deadline <= 0 - shutdown_deadline = 0.0 - - while process.poll() is None: - elapsed = time.time() - start_time - - if not startup_done: - if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: - startup_done = True - - if report_file and not shutdown_deadline and os.path.exists(report_file): - shutdown_deadline = time.time() + SHUTDOWN_GRACE_PERIOD - - kill_reason = None - if not startup_done and elapsed > startup_deadline: - kill_reason = "startup_hang" - elif shutdown_deadline and time.time() > shutdown_deadline: - kill_reason = "shutdown_hang" - elif elapsed > timeout: - kill_reason = "timeout" - - if kill_reason: - pre_kill_diag = _capture_system_diagnostics() - - # Kill the entire process group (test + any Kit children). - try: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - process.kill() - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=5) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except subprocess.TimeoutExpired: - pass - wall_time = time.time() - start_time - return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag - - try: - ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) - - for fd in ready_fds: - with contextlib.suppress(OSError): - if fd == stdout_fd: - chunk = process.stdout.read(1024) - if chunk: - stdout_data += chunk - sys.stdout.buffer.write(chunk) - sys.stdout.buffer.flush() - elif fd == stderr_fd: - chunk = process.stderr.read(1024) - if chunk: - stderr_data += chunk - sys.stderr.buffer.write(chunk) - sys.stderr.buffer.flush() - except OSError: - time.sleep(0.1) - continue - - # Drain any output the process wrote before or just after exiting. - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=10) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except Exception: - pass - - # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. - try: - os.killpg(pgid, signal.SIGKILL) - time.sleep(1) - except (ProcessLookupError, PermissionError, OSError): - pass - - wall_time = time.time() - start_time - return process.returncode, stdout_data, stderr_data, "", wall_time, "" - - except Exception as e: - if process is not None and process.poll() is None: - process.kill() - with contextlib.suppress(Exception): - rem_out, rem_err = process.communicate(timeout=5) - stdout_data += rem_out - stderr_data += rem_err - stdout_data += f"\n[capture error: {e}]\n".encode() - return -1, stdout_data, stderr_data, "", 0.0, "" - - -_SIGNAL_DESCRIPTIONS = { - 1: "SIGHUP — session leader exit or orphaned process cleanup", - 6: "SIGABRT", - 9: "SIGKILL — likely OOM killed", - 11: "SIGSEGV — segmentation fault", - 15: "SIGTERM", -} - - -def _signal_description(sig): - """Return a human-readable description for a process killed by a signal.""" - base = f"Process killed by signal {sig}" - desc = _SIGNAL_DESCRIPTIONS.get(sig) - return f"{base} ({desc})" if desc else base - - -def _create_error_report(prefix, file_name, message, details): - """Create a JUnit XML error report for a test that failed to produce its own. - - Returns a :class:`JUnitXml` object ready to be written to disk. - """ - suite_name = os.path.splitext(file_name)[0] - suite = TestSuite(name=f"{prefix}_{suite_name}") - case = TestCase(name="test_execution", classname=suite_name) - error = Error(message=message) - error.text = details - case.result = error - suite.add_testcase(case) - report = JUnitXml() - report.add_testsuite(suite) - return report - - -def _get_diagnostics(pre_kill_diag=""): - """Return system diagnostics, truncated to 10 000 chars.""" - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - return diag - - -def _capture_system_diagnostics(): - """Capture system diagnostics (GPU, memory, processes) for crash investigation. - - All errors are caught and reported inline so this never raises. - """ - sections = [] - - try: - r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) - if r.stdout: - sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- nvidia-smi --- FAILED: {e}") - - try: - with open("/proc/meminfo") as f: - lines = f.readlines() - keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") - relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] - if relevant: - sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) - except Exception as e: - sections.append(f"--- /proc/meminfo --- FAILED: {e}") - - cgroup_lines = [] - for path in ( - "/sys/fs/cgroup/memory.current", - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory.events", - "/sys/fs/cgroup/memory/memory.usage_in_bytes", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - "/sys/fs/cgroup/memory/memory.oom_control", - ): - try: - with open(path) as f: - cgroup_lines.append(f"{path}: {f.read().strip()}") - except FileNotFoundError: - pass - except Exception as e: - cgroup_lines.append(f"{path}: FAILED ({e})") - if cgroup_lines: - sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) - - try: - r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) - if r.stdout: - sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- process tree --- FAILED: {e}") - - try: - r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) - if r.stdout: - lines = r.stdout.strip().split("\n") - sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) - except Exception: - pass - - return "\n\n".join(sections) - - -def _read_test_report(report_file, file_name): - """Read a pytest JUnit report and return its summary fields.""" - report = JUnitXml.fromfile(report_file) - for suite in report: - if suite.name == "pytest": - suite.name = os.path.splitext(file_name)[0] - report.write(report_file) - - errors = int(report.errors) if report.errors is not None else 0 - failures = int(report.failures) if report.failures is not None else 0 - skipped = int(report.skipped) if report.skipped is not None else 0 - tests = int(report.tests) if report.tests is not None else 0 - time_elapsed = float(report.time) if report.time is not None else 0.0 - return report, errors, failures, skipped, tests, time_elapsed - - -def _retry_failed_test_in_fresh_process( - *, - test_file, - file_name, - cmd, - timeout, - env, - startup_deadline, - report_file, - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, -): - """Retry selected failed test files in a fresh subprocess.""" - has_test_failures = errors > 0 or failures > 0 - process_failure_attempts = 0 - max_process_failure_retries = PROCESS_FAILURE_RETRIES_BY_FILE.get(file_name, 0) - - while has_test_failures and process_failure_attempts < max_process_failure_retries: - process_failure_attempts += 1 - print( - f"⚠️ {test_file}: failed in subprocess" - f" (attempt {process_failure_attempts}/{max_process_failure_retries + 1}), retrying in fresh process..." - ) - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - if not os.path.exists(report_file): - break - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - has_test_failures = errors > 0 or failures > 0 - except Exception as e: - print(f"Error reading retry test report {report_file}: {e}") - has_test_failures = True - errors = 1 - failures = 0 - skipped = 0 - tests = 0 - time_elapsed = 0.0 - break - - return ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) - - -@dataclass -class _UnitContext: - """Per-file inputs shared across the units the runner drives for one file. - - Attributes: - test_file: Path to the test file being driven. - file_name: Basename of ``test_file`` (used for JUnit naming). - workspace_root: Repository root; passed to pytest's ``--config-file`` and - used to locate the ``_device_select`` plugin. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. - timeout: Per-unit hard timeout in seconds. - startup_deadline: Per-unit startup-hang deadline in seconds. - env: Base environment for the pytest subprocess; :func:`run_unit` copies - it and adds the per-unit ``ISAACLAB_TEST_DEVICES`` mask. - """ - - test_file: str - file_name: str - workspace_root: str - isaacsim_ci: bool - timeout: int - startup_deadline: int - env: dict - - -_RESULT_PRIORITY = { - "STARTUP_HANG": 5, - "CRASHED": 4, - "TIMEOUT": 3, - "FAILED": 2, - "passed (shutdown hanged)": 1, - "passed": 0, -} - - -def merge_unit_status(prev: dict | None, new: dict) -> dict: - """Merge per-unit status dicts into a single per-file entry. - - Counters (``errors``, ``failures``, ``skipped``, ``tests``, - ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more - severe of the two via :data:`_RESULT_PRIORITY`. - """ - if prev is None: - return new - return { - "errors": prev["errors"] + new["errors"], - "failures": prev["failures"] + new["failures"], - "skipped": prev["skipped"] + new["skipped"], - "tests": prev["tests"] + new["tests"], - "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], - "wall_time": prev["wall_time"] + new["wall_time"], - "result": prev["result"] - if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) - else new["result"], - } - - -def _build_unit_cmd(ctx: _UnitContext, unit: Unit) -> tuple[list[str], dict, str, str]: - """Build the pytest invocation for one unit (no subprocess). - - Device-variant selection is the unit's mask, set as ``ISAACLAB_TEST_DEVICES`` - for ``test_devices()`` to read — there is no ``-k``. A mask without cpu - (position 0) additionally injects the ``_device_select`` plugin so - device-agnostic tests do not run there. - - Args: - ctx: Static per-file context (paths, isaacsim_ci flag, base env). - unit: The work unit to run. - - Returns: - ``(cmd, env, report_file, pass_file_label)`` — the argv, the per-unit - environment, the JUnit report path, and the label used in error reports. - """ - # Split units of one file share a slug, so suffix the report with the mask to - # keep both XMLs; a non-split unit (one per file) keeps the suffix-free name - # the mgpu aggregator unslugs back to a path. - report_suffix = f"-{unit.mask}" if unit.split else "" - pass_file_label = f"{ctx.file_name}{report_suffix}" - # Slug the full test path (not just the basename) into the report filename so - # two concurrent shards running same-basename files (e.g. - # ``isaaclab_newton/.../test_articulation.py`` vs - # ``isaaclab_physx/.../test_articulation.py``) don't write to the same path - # inside the shared ``/workspace/isaaclab`` mount and trigger false - # shutdown_hang detections in sibling shards via the report-file existence check. - report_slug = str(ctx.test_file).replace("/", "__").replace("\\", "__") - report_file = f"tests/test-reports-{report_slug}{report_suffix}.xml" - - env = dict(ctx.env) - env["ISAACLAB_TEST_DEVICES"] = unit.mask - - # A split unit (either half) or a cpu-less shard must run exactly its mask's - # devices, so inject the device selector to drop out-of-scope variants — - # including literal device-param variants the mask alone cannot narrow (only - # test_devices() reads the mask; a literal ["cuda:0", "cpu"] list does not). - select_by_device = unit.split or unit.mask[:1] == "0" - if select_by_device: - tools_dir = os.path.join(ctx.workspace_root, "tools") - env["PYTHONPATH"] = tools_dir + os.pathsep + env.get("PYTHONPATH", "") - - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit - "--no-header", - f"--config-file={ctx.workspace_root}/pyproject.toml", - f"--junitxml={report_file}", - "--tb=short", - ] - if select_by_device: - cmd += ["-p", "_device_select"] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] - cmd.append(str(ctx.test_file)) - return cmd, env, report_file, pass_file_label - - -def run_unit(ctx: _UnitContext, unit: Unit) -> tuple[JUnitXml | None, dict, bool]: - """Drive one pytest subprocess for ``unit`` and return its results. - - Args: - ctx: Static per-file context (paths, timeouts, base env). - unit: The work unit to run; its mask becomes ``ISAACLAB_TEST_DEVICES``. - - Returns: - A 3-tuple ``(xml_report, status_dict, was_failure)``: - * ``xml_report``: parsed JUnit XML, or ``None`` if the unit produced - no report (e.g. startup hang). - * ``status_dict``: per-unit counters compatible with the entries - appended to ``test_status``. - * ``was_failure``: whether the unit should add ``ctx.test_file`` to - the ``failed_tests`` list. - """ - cmd, env, report_file, pass_file_label = _build_unit_cmd(ctx, unit) - report_suffix = f"-{unit.mask}" if unit.split else "" # display suffix for this unit's log lines - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file - ) - - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {ctx.test_file}{report_suffix}: startup hang detected after {ctx.startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {ctx.test_file}{report_suffix}: timeout detected after {ctx.timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break - - # -- Resolve result from kill_reason and report file ---------------- - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {ctx.test_file}{report_suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") - print(diag) - - msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "STARTUP_HANG", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {ctx.test_file}{report_suffix} timed out after {ctx.timeout} seconds...") - print(diag) - - msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "TIMEOUT", - "time_elapsed": ctx.timeout, - "wall_time": wall_time, - }, - True, - ) - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {ctx.test_file}{report_suffix}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", pass_file_label, reason, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "CRASHED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {ctx.test_file}{report_suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - return ( - None, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 0, - "result": "FAILED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=ctx.test_file, - file_name=ctx.file_name, - cmd=cmd, - timeout=ctx.timeout, - env=env, - startup_deadline=ctx.startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, - ) - - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures - was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" - - return ( - report, - { - "errors": errors, - "failures": failures, - "skipped": skipped, - "tests": tests, - "result": result, - "time_elapsed": time_elapsed, - "wall_time": wall_time, - }, - was_failure, - ) diff --git a/tools/_device_plan.py b/tools/_device_plan.py deleted file mode 100644 index 15086cc67028..000000000000 --- a/tools/_device_plan.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Pure planner for the per-file CI test runner. - -Turns a runner's claimed files plus its runtime device mask into a list of -``(file, mask)`` work units, where ``mask`` is the ``ISAACLAB_TEST_DEVICES`` -value the executor will set for that subprocess. A file is one unit unless it -declares the ``device_isolated`` marker and the runtime spans more than one -device, in which case it splits into one single-device unit per device (to work -around backends whose device mode is process-global, e.g. ovphysx). - -This module is pure: no I/O beyond reading a file's source for marker detection, -no subprocess, no pytest collection. It replaces the ``DEVICE_SPLIT_PASSES`` / -``is_device_split_file`` pieces of the former ``_device_split`` module. -""" - -from __future__ import annotations - -import re -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class Unit: - """One pytest subprocess to run for the test runner. - - Attributes: - file: Test file to run. - mask: Device mask set as ``ISAACLAB_TEST_DEVICES`` for the subprocess; - ``test_devices()`` reads it to select the device variants. - split: Whether this unit is one of several produced by splitting a - ``device_isolated`` file. The executor uses it only to give split - units distinct JUnit report filenames; a non-split unit (one per - file) keeps the suffix-free name the mgpu aggregator expects. - """ - - file: str - mask: str - split: bool = False - -_ISOLATED_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_isolated\b", re.MULTILINE) -"""Match a module-level ``pytestmark`` assignment that mentions ``device_isolated``. - -Recognises both the single-mark and single-line list forms: - -* ``pytestmark = pytest.mark.device_isolated`` -* ``pytestmark = [pytest.mark.device_isolated, pytest.mark.slow]`` -""" - - -def is_isolated(path: Path | str, source: str | None = None) -> bool: - """Return whether a test file declares the ``device_isolated`` marker. - - Detection is by source regex, not import, so the planner stays collection-free. - - Args: - path: Filesystem path to the candidate test file. - source: Optional preloaded source text to inspect instead of reading - ``path``. - - Returns: - ``True`` when the file's module-level ``pytestmark`` mentions - ``device_isolated``; ``False`` otherwise (including a missing or - unreadable file). - """ - if source is None: - try: - source = Path(path).read_text(encoding="utf-8", errors="replace") - except OSError: - return False - return bool(_ISOLATED_MARK_RE.search(source)) - - -def _single_bit_mask(index: int, width: int) -> str: - """Return a width-``width`` device mask with only ``index`` set. - - Args: - index: The single device position to enable. - width: Total mask width. - - Returns: - A mask string such as ``"010"`` (``index=1``, ``width=3``). - """ - return "".join("1" if pos == index else "0" for pos in range(width)) - - -def plan_units( - files: list[str], - runtime_mask: str, - is_isolated: Callable[[Path | str], bool] = is_isolated, -) -> list[Unit]: - """Plan the work units for one runner. - - Args: - files: Test files this runner is responsible for, in run order. - runtime_mask: The runner's concrete device mask (e.g. ``"110"`` on the - single-GPU lane, ``"0001"`` for an mgpu shard). Must not contain the - open-ended ``"X"`` form, which is a scope-only construct. - is_isolated: Predicate deciding whether a file needs one process per - device. Defaults to :func:`is_isolated` (marker detection by source). - - Returns: - Work units in run order. A non-isolated file, or any file on a - single-device runtime, yields one unit at ``runtime_mask``. An isolated - file on a multi-device runtime yields one single-device unit per set bit. - - Raises: - ValueError: When ``runtime_mask`` contains ``"X"``. - """ - if "X" in runtime_mask: - raise ValueError(f"runtime mask {runtime_mask!r} must be concrete (no 'X'); 'X' is a scope-only construct") - set_bits = [pos for pos, char in enumerate(runtime_mask) if char == "1"] - units: list[Unit] = [] - for test_file in files: - if is_isolated(test_file) and len(set_bits) > 1: - units.extend(Unit(test_file, _single_bit_mask(pos, len(runtime_mask)), split=True) for pos in set_bits) - else: - units.append(Unit(test_file, runtime_mask)) - return units diff --git a/tools/_device_select.py b/tools/_device_select.py deleted file mode 100644 index ed2498eb7eba..000000000000 --- a/tools/_device_select.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""pytest plugin: keep only the tests a unit's device mask allows. - -The executor (``tools/conftest.py`` via ``_device_exec``) injects this with -``-p _device_select`` for any unit that must run on a single device — both halves -of a ``device_isolated`` split (cpu unit, cuda unit) and every mgpu shard — so a -process that is locked to one device by a backend (e.g. ovphysx<=0.3.7) never -also tries another device. - -Selection reads ``ISAACLAB_TEST_DEVICES`` (the unit's mask). For each collected -test: - -* parametrized over ``device``: keep iff that device is active in the mask. This - is what catches a literal ``["cuda:0", "cpu"]`` variant whose device the mask - alone cannot narrow (only ``test_devices()`` reads the mask at parametrize - time; a literal list does not). -* not parametrized over ``device`` (agnostic): keep only when cpu is in the mask - — i.e. the default unit. On a cuda split unit or a shard it is dropped, since - single-GPU CI already covers it on cpu/cuda:0. - -A unit that deselects to empty (no in-scope tests in this file) exits -``NO_TESTS_COLLECTED``; the runner treats any non-zero per-file exit as failure, -so report "nothing in scope" as success. -""" - -import os - -import pytest - -_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" - - -def _position(device): - """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None).""" - if device == "cpu": - return 0 - if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): - return int(device[len("cuda:") :]) + 1 - return None # not device-parametrized, or an unrecognized device value - - -def _active(position, mask): - """Whether a mask position is included. A trailing ``X`` includes all remaining.""" - if position is None: - return False - body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) - return body[position] == "1" if 0 <= position < len(body) else fill - - -def _device_of(item): - """The ``device`` parametrize value of a collected item, or ``None`` if agnostic.""" - callspec = getattr(item, "callspec", None) - return callspec.params.get("device") if callspec is not None else None - - -def pytest_collection_modifyitems(config, items): - mask = os.environ.get(_RUNTIME_ENV, "") - cpu_in_mask = _active(0, mask) - keep, drop = [], [] - for item in items: - device = _device_of(item) - if device is None: - (keep if cpu_in_mask else drop).append(item) # agnostic: only in a cpu-inclusive unit - elif _active(_position(device), mask): - keep.append(item) # this device variant is in scope for the unit - else: - drop.append(item) # out-of-scope variant (e.g. a literal cpu variant in a cuda unit) - if drop: - items[:] = keep - config.hook.pytest_deselected(items=drop) - - -def pytest_sessionfinish(session, exitstatus): - # A file with nothing in scope for this unit deselects to zero and exits - # NO_TESTS_COLLECTED (5). That is success here, not a failure. - if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = pytest.ExitCode.OK diff --git a/tools/conftest.py b/tools/conftest.py index 49e19bf91f9b..dedc9f8b3087 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -3,496 +3,55 @@ # # SPDX-License-Identifier: BSD-3-Clause -import contextlib +"""CI entry point for the per-file test runner. + +This conftest does not run tests in-process. It hands the whole run to the +:mod:`test_runner` framework, which executes each test file in its own pytest +subprocess (so a crash or device lock in one file cannot affect the next). All +behavior is in the framework; this file is just the two pytest hooks that wire +it in. +""" + import os -import re import pytest -from junitparser import JUnitXml -from prettytable import PrettyTable - -# Local imports -import test_settings as test_settings # isort: skip -from _device_exec import COLD_CACHE_BUFFER, STARTUP_DEADLINE, _UnitContext, merge_unit_status, run_unit # isort: skip -from _device_plan import is_isolated as _is_isolated, plan_units # isort: skip +from test_runner.planning import RunnerConfig +from test_runner.session import Session def pytest_ignore_collect(collection_path, config): - # Skip collection and run each test script individually - return True - - -def _slugify_test_path(test_path): - """Encode a test path as a flat queue entry name. - - The queue uses one file per pending test. Slashes are not legal inside a - filename, so we encode the relative path by replacing ``/`` with ``__``. - The decoder is :func:`_unslugify_queue_entry`. - """ - return test_path.replace("/", "__") - - -def _unslugify_queue_entry(entry_name): - """Reverse of :func:`_slugify_test_path`.""" - return entry_name.replace("__", "/") - - -def _claim_queued_file(queue_dir): - """Atomically claim one pending test from the work-queue directory. - - The queue is a directory of files (one per pending test); the shard claims - one by renaming it from ``queue/`` into its private ``inflight/cuda-N/``. - POSIX rename is atomic on the same filesystem, so two shards racing on the - same source file are serialized by the kernel: exactly one rename succeeds, - the other gets ``FileNotFoundError`` and tries the next entry. - - On success, the test's queue entry is now sitting in ``inflight/cuda-N/``; - the caller is expected to move it to ``done/cuda-N/`` after the per-test - pytest invocation exits with a clean result, leaving anything still in - ``inflight/`` at job-end as recoverable evidence of a crashed test. + """PYTEST HOOK — skip pytest's own collection; the runner drives files itself. Args: - queue_dir: Path to the shared work-queue root. Must contain a - ``queue/`` subdir (pending entries) and an ``inflight//`` - subdir for this shard (claim destination). + collection_path: candidate path pytest is about to collect (unused). + config: the session ``pytest.Config`` (unused). Returns: - The decoded test path for the claimed file, or ``None`` when the - queue is empty. + ``True`` always, so nothing is collected in this process; the per-file + subprocesses launched by :class:`~test_runner.session.Session` do the + real collection. """ - shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") - pending_dir = os.path.join(queue_dir, "queue") - inflight_dir = os.path.join(queue_dir, "inflight", shard) - os.makedirs(inflight_dir, exist_ok=True) - - # Listdir is intentionally not cached: another shard may have just removed - # an entry we'd otherwise try. We pay one listdir per claim attempt; with - # N≤20 entries this is microseconds. - try: - entries = sorted(os.listdir(pending_dir)) - except FileNotFoundError: - return None - - for entry in entries: - src = os.path.join(pending_dir, entry) - dst = os.path.join(inflight_dir, entry) - try: - os.rename(src, dst) - except FileNotFoundError: - # Lost the race for this entry; another shard claimed it first. - # Continue to the next entry in our (potentially stale) listing. - continue - except OSError: - # Any other rename failure (e.g. permission) is a hard error. - raise - return _unslugify_queue_entry(entry) - - return None - - -def _mark_queued_file_done(queue_dir, test_path): - """Move a successfully-completed claim from ``inflight/cuda-N/`` to ``done/cuda-N/``. - - Called by the test runner after a per-file pytest invocation exits cleanly. - The inflight residual is what the post-run reconciler uses to detect - crashed shards: anything still in ``inflight/`` at job-end is an orphan. - """ - shard = os.environ.get("ISAACLAB_SIM_DEVICE", "cuda").replace(":", "-") - entry = _slugify_test_path(test_path) - src = os.path.join(queue_dir, "inflight", shard, entry) - dst_dir = os.path.join(queue_dir, "done", shard) - os.makedirs(dst_dir, exist_ok=True) - dst = os.path.join(dst_dir, entry) - # Suppress: already moved (idempotent) or the runner crashed before we - # could mark done — the reconciler catches the second case. - with contextlib.suppress(FileNotFoundError): - os.rename(src, dst) - + return True -def _queued_files(queue_dir): - """Yield files claimed from the shared work queue until it is empty.""" - while True: - claimed = _claim_queued_file(queue_dir) - if claimed is None: - return - yield claimed +def pytest_sessionstart(session): + """PYTEST HOOK — build the runner from the environment and run the lifecycle. -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next. + Called by pytest at session start. It exits pytest with the runner's code so + normal pytest does not run (and overwrite the aggregate report) afterward. - When ``ISAACLAB_TEST_QUEUE`` names a shared work-queue directory, files are - claimed from it (work-stealing across sibling shard containers) instead of - iterating ``test_files``; each file still runs once, on this container's GPU. + Args: + session: the ``pytest.Session`` (only its startup timing is used). - The planner turns each file into one or more units (a device_isolated file - splits into one unit per device when the runtime spans more than one); the - executor runs each unit with the unit's mask set as ``ISAACLAB_TEST_DEVICES``. + Returns: + None — terminates the process via :func:`pytest.exit` with 0 when every + file passed, else 1. """ - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False - - queue_path = os.environ.get("ISAACLAB_TEST_QUEUE", "") - file_source = _queued_files(queue_path) if queue_path else test_files - - # The runner's device set. Unset (single-GPU CI) is cpu + cuda:0, matching - # test_devices()'s default; an mgpu shard sets a single-device mask. The - # planner splits a device_isolated file only when this spans more than one. - runtime_mask = os.environ.get("ISAACLAB_TEST_DEVICES") or "110" - - for test_file in file_source: - print(f"\n\n\U0001f680 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" - - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - - # Read the test file once for cold-cache and device-isolation detection. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" - - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"\u23f1\ufe0f Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") - - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - - ctx = _UnitContext( - test_file=test_file, - file_name=file_name, - workspace_root=workspace_root, - isaacsim_ci=isaacsim_ci, - timeout=timeout, - startup_deadline=startup_deadline, - env=env, - ) - - units = plan_units([test_file], runtime_mask, is_isolated=lambda f: _is_isolated(f, source=test_content)) - if len(units) > 1: - print(f"\u2699\ufe0f device_isolated \u2014 invoking {file_name} once per device ({len(units)} units)") - - merged_status: dict | None = None - for unit in units: - report, status, was_failure = run_unit(ctx, unit) - if report is not None: - xml_reports.append(report) - if was_failure and test_file not in failed_tests: - failed_tests.append(test_file) - merged_status = merge_unit_status(merged_status, status) - - assert merged_status is not None # the unit list is never empty - test_status[test_file] = merged_status - - # In work-queue mode, move the claim from inflight// to done// - # so the post-run reconciler can tell "ran to completion" from "claimed but - # crashed mid-test". A claim left in inflight at job-end is a silent drop. - if queue_path: - _mark_queued_file_done(queue_path, test_file) - - print("~~~~~~~~~~~~ Finished running all tests") - - return failed_tests, test_status, xml_reports - - -def _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, -): - """Collect test files from source directories, applying all active filters.""" - test_files = [] - for source_dir in source_dirs: - if not os.path.exists(source_dir): - print(f"Error: source directory not found at {source_dir}") - pytest.exit("Source directory not found", returncode=1) - - for root, _, files in os.walk(source_dir): - # source/isaaclab/test/install_ci/ has its own pytest config and conftest. - # It is run via .github/actions/install-ci-run, never via this collector, - # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. - if "install_ci" in root.replace("\\", "/").split("/"): - continue - - for file in files: - if not (file.startswith("test_") and file.endswith(".py")): - continue - - # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) - if quarantined_only: - if file not in test_settings.QUARANTINED_TESTS: - continue - elif curobo_only: - if file not in test_settings.CUROBO_TESTS: - continue - else: - # An explicit include_files entry overrides TESTS_TO_SKIP, allowing - # dedicated jobs (e.g. test-environments-training) to run tests that - # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: - print(f"Skipping {file} as it's in the skip list") - continue - - full_path = os.path.join(root, file) - - if filter_pattern and filter_pattern not in full_path: - print(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") - continue - if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): - print(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") - continue - if include_files and file not in include_files: - print(f"Skipping {full_path} (not in include files list)") - continue - - test_files.append(full_path) - - # Apply file-level sharding: sort deterministically, then select every Nth file. - # Skip when include_files is set — in that case the test's own conftest handles - # sharding at the test-item level (e.g. parametrized test cases). - shard_index = os.environ.get("TEST_SHARD_INDEX", "") - shard_count = os.environ.get("TEST_SHARD_COUNT", "") - if shard_index and shard_count and not include_files: - shard_index = int(shard_index) - shard_count = int(shard_count) - test_files.sort() - test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] - print(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") - - return test_files - - -def _write_empty_report(): - """Write an empty JUnit XML report so downstream CI steps find a valid file.""" - os.makedirs("tests", exist_ok=True) - result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") - report = JUnitXml() - report.write(f"tests/{result_file}") - print(f"Wrote empty report to tests/{result_file}") - - -def pytest_sessionstart(session): - """Intercept pytest startup to execute tests in the correct order.""" - # Get the workspace root directory (one level up from tools) workspace_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - source_dirs = [ - os.path.join(workspace_root, "scripts"), - os.path.join(workspace_root, "source"), - ] - - # Get filter pattern from environment variable or command line - filter_pattern = os.environ.get("TEST_FILTER_PATTERN", "") - exclude_pattern = os.environ.get("TEST_EXCLUDE_PATTERN", "") - include_files_str = os.environ.get("TEST_INCLUDE_FILES", "") - quarantined_only = os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true" - curobo_only = os.environ.get("TEST_CUROBO_ONLY", "false") == "true" - - isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" - - # Parse include files list (comma-separated paths) - include_files = set() - if include_files_str: - for f in include_files_str.split(","): - f = f.strip() - if f: - include_files.add(os.path.basename(f)) - - # Also try to get from pytest config - if hasattr(session.config, "option") and hasattr(session.config.option, "filter_pattern"): - filter_pattern = filter_pattern or getattr(session.config.option, "filter_pattern", "") - if hasattr(session.config, "option") and hasattr(session.config.option, "exclude_pattern"): - exclude_pattern = exclude_pattern or getattr(session.config.option, "exclude_pattern", "") - - print("=" * 50) - print("CONFTEST.PY DEBUG INFO") - print("=" * 50) - print(f"Filter pattern: '{filter_pattern}'") - print(f"Exclude pattern: '{exclude_pattern}'") - print(f"Include files: {include_files if include_files else 'none'}") - print(f"Quarantined-only mode: {quarantined_only}") - print(f"Curobo-only mode: {curobo_only}") - print(f"TEST_FILTER_PATTERN env var: '{os.environ.get('TEST_FILTER_PATTERN', 'NOT_SET')}'") - print(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") - print(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") - print(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") - print(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") - print("=" * 50) - - # Get all test files in the source directories - test_files = _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, - ) - - if isaacsim_ci: - new_test_files = [] - for test_file in test_files: - with open(test_file) as f: - if "@pytest.mark.isaacsim_ci" in f.read(): - new_test_files.append(test_file) - test_files = new_test_files - - if not test_files: - if quarantined_only: - print("No quarantined tests configured — nothing to run.") - _write_empty_report() - pytest.exit("No quarantined tests configured", returncode=0) - if filter_pattern: - print(f"No test files found matching filter pattern '{filter_pattern}' — nothing to run.") - _write_empty_report() - pytest.exit("No test files found for filter", returncode=0) - print("No test files found in source directory") - pytest.exit("No test files found", returncode=1) - - print(f"Found {len(test_files)} test files after filtering:") - for test_file in test_files: - print(f" - {test_file}") - - # Run all tests individually - failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) - - # In work-queue mode this container ran only the files it claimed; report on those. - if os.environ.get("ISAACLAB_TEST_QUEUE"): - test_files = list(test_status) - - print("failed tests:", failed_tests) - - # Collect reports - print("~~~~~~~~~ Collecting final report...") - - # Merge in-memory report objects collected during the test run. Reading the - # on-disk files again risks losing elements if the junitparser - # read/write round-trip does not preserve them faithfully. - full_report = JUnitXml() - for xml_report in xml_reports: - print(xml_report) - full_report += xml_report - print("~~~~~~~~~~~~ Writing final report...") - # write content to full report - result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") - full_report_path = f"tests/{result_file}" - # Ensure the directory exists even when this shard claimed zero files - # from the work queue (per-test JUnit XMLs are what normally create - # ``tests/``; with no tests run there is nothing to create it). - os.makedirs("tests", exist_ok=True) - print(f"Using result file: {result_file}") - full_report.write(full_report_path) - print("~~~~~~~~~~~~ Report written to", full_report_path) - - # print test status in a nice table - # Calculate the number and percentage of passing tests - num_tests = len(test_status) - num_passing = len([p for p in test_files if test_status[p]["result"].startswith("passed")]) - num_failing = len([p for p in test_files if test_status[p]["result"] == "FAILED"]) - num_timeout = len([p for p in test_files if test_status[p]["result"] == "TIMEOUT"]) - num_crashed = len([p for p in test_files if test_status[p]["result"] == "CRASHED"]) - num_startup_hang = len([p for p in test_files if test_status[p]["result"] == "STARTUP_HANG"]) - - if num_tests == 0: - passing_percentage = 100 - else: - passing_percentage = num_passing / num_tests * 100 - - # Print summaries of test results - summary_str = "\n\n" - summary_str += "===================\n" - summary_str += "Test Result Summary\n" - summary_str += "===================\n" - - summary_str += f"Total: {num_tests}\n" - summary_str += f"Passing: {num_passing}\n" - summary_str += f"Failing: {num_failing}\n" - summary_str += f"Crashed: {num_crashed}\n" - summary_str += f"Startup Hang: {num_startup_hang}\n" - summary_str += f"Timeout: {num_timeout}\n" - summary_str += f"Passing Percentage: {passing_percentage:.2f}%\n" - - total_wall = sum(test_status[test_path]["wall_time"] for test_path in test_files) - total_test = sum(test_status[test_path]["time_elapsed"] for test_path in test_files) - - summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" - summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" - - # GPU this run used (the shard's boot device); ``cuda:0`` when unset. - run_device = os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0" - - summary_str += "\n\n=======================\n" - summary_str += "Per File Result Summary\n" - summary_str += "=======================\n" - - per_file_result_table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_file_result_table.align["Test Path"] = "l" - per_file_result_table.align["Test (s)"] = "r" - per_file_result_table.align["Wall (s)"] = "r" - for test_path in test_files: - num_tests_passed = ( - test_status[test_path]["tests"] - - test_status[test_path]["failures"] - - test_status[test_path]["errors"] - - test_status[test_path]["skipped"] - ) - per_file_result_table.add_row( - [ - test_path, - run_device, - test_status[test_path]["result"], - f"{test_status[test_path]['time_elapsed']:0.2f}", - f"{test_status[test_path]['wall_time']:0.2f}", - f"{num_tests_passed}/{test_status[test_path]['tests']}", - ] - ) - - summary_str += per_file_result_table.get_string() - - # Per-test run times, slowest first, from the merged JUnit report. The - # device is read from the test id params (e.g. ``...[size0-cuda:1]``), - # falling back to the run's boot device. - summary_str += "\n\n=================\n" - summary_str += "Per Test Run Time\n" - summary_str += "=================\n" - - per_test_time_table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) - per_test_time_table.align["Test"] = "l" - per_test_time_table.align["Time (s)"] = "r" - test_times = [] - for suite in full_report: - for case in suite: - full_name = f"{case.classname}::{case.name}" if case.classname else case.name - device = run_device - bracket = re.search(r"\[(.*)\]", full_name) - if bracket: - dev_match = re.search(r"cuda:\d+|\bcpu\b", bracket.group(1)) - if dev_match: - device = dev_match.group(0) - elapsed = float(case.time) if case.time is not None else 0.0 - test_times.append((full_name, device, elapsed)) - for full_name, device, elapsed in sorted(test_times, key=lambda row: row[2], reverse=True): - per_test_time_table.add_row([full_name, device, f"{elapsed:0.3f}"]) - - summary_str += per_test_time_table.get_string() - - # Print summary to console and log file - print(summary_str) - - # Exit pytest after custom execution to prevent normal pytest from overwriting our report - pytest.exit( - "Custom test execution completed", - returncode=0 if (num_failing == 0 and num_timeout == 0 and num_crashed == 0 and num_startup_hang == 0) else 1, + config = RunnerConfig.from_env(workspace_root) + print( + f"test_runner config: mask={config.runtime_mask} queue={'on' if config.queue_path else 'off'}" + f" isaacsim_ci={config.isaacsim_ci} filter={config.filter_pattern!r}" + f" exclude={config.exclude_pattern!r} include={sorted(config.include_files) or 'none'}" ) + pytest.exit("Custom test execution completed", returncode=Session(config).run()) diff --git a/tools/test_device_exec.py b/tools/test_device_exec.py deleted file mode 100644 index 66da1c7182fe..000000000000 --- a/tools/test_device_exec.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Behavior guard for the executor's per-unit command construction. - -These assert the device-selection contract without launching a subprocess: the -unit's mask becomes ``ISAACLAB_TEST_DEVICES`` (so ``test_devices()`` selects the -variants), the ``_device_select`` plugin is injected for split units (both -halves) and shards, ``-k`` is never used, and split units get -a suffixed report name while single units keep the mgpu-aggregator-friendly one. -""" - -from __future__ import annotations - -import os - -from _device_exec import _build_unit_cmd, _UnitContext -from _device_plan import Unit - - -def _ctx(test_file="source/pkg/test_x.py", isaacsim_ci=False): - return _UnitContext( - test_file=test_file, - file_name="test_x.py", - workspace_root="/ws", - isaacsim_ci=isaacsim_ci, - timeout=60, - startup_deadline=30, - env={"BASE": "1"}, - ) - - -def test_mix_ok_unit_sets_mask_with_no_plugin_and_no_k(): - cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "110")) - assert env["ISAACLAB_TEST_DEVICES"] == "110" - assert env["BASE"] == "1" # base env preserved - assert "_device_select" not in cmd # mix-ok single unit: no isolation needed - assert "-k" not in cmd - assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" # no suffix for a single unit - assert label == "test_x.py" - assert cmd[-1] == "source/pkg/test_x.py" - - -def test_cpu_split_unit_is_suffixed_and_injects_device_select(): - # The cpu half of a split must also drop out-of-scope literal cuda variants, - # else they leak into the cpu-locked process. So it injects the selector too. - cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "100", split=True)) - assert env["ISAACLAB_TEST_DEVICES"] == "100" - assert "_device_select" in cmd - assert report_file.endswith("test_x.py-100.xml") - assert label == "test_x.py-100" - - -def test_cuda_split_unit_injects_device_select_and_tools_pythonpath(): - cmd, env, report_file, _ = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "010", split=True)) - assert env["ISAACLAB_TEST_DEVICES"] == "010" - assert cmd.count("-p") == 1 and "_device_select" in cmd - assert env["PYTHONPATH"].split(os.pathsep)[0] == os.path.join("/ws", "tools") - assert report_file.endswith("test_x.py-010.xml") - - -def test_shard_unit_injects_device_select_without_a_suffix(): - cmd, env, report_file, label = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", "0001")) - assert env["ISAACLAB_TEST_DEVICES"] == "0001" - assert "_device_select" in cmd # cpu-less shard -> drop out-of-scope + agnostic - # one unit per file on a shard -> suffix-free name the aggregator unslugs to a path - assert report_file == "tests/test-reports-source__pkg__test_x.py.xml" - assert label == "test_x.py" - - -def test_isaacsim_ci_adds_the_marker_selector(): - # ``-m pytest`` appears first, so look for the ``-m isaacsim_ci`` pair specifically. - cmd, _, _, _ = _build_unit_cmd(_ctx(isaacsim_ci=True), Unit("source/pkg/test_x.py", "110")) - assert any(cmd[i] == "-m" and cmd[i + 1] == "isaacsim_ci" for i in range(len(cmd) - 1)) - - -def test_no_isaacsim_ci_marker_when_disabled(): - cmd, _, _, _ = _build_unit_cmd(_ctx(isaacsim_ci=False), Unit("source/pkg/test_x.py", "110")) - assert "isaacsim_ci" not in cmd - - -def test_no_k_selector_for_any_mask(): - for mask, split in [("110", False), ("100", True), ("010", True), ("0001", False)]: - cmd, *_ = _build_unit_cmd(_ctx(), Unit("source/pkg/test_x.py", mask, split=split)) - assert "-k" not in cmd diff --git a/tools/test_device_plan.py b/tools/test_device_plan.py deleted file mode 100644 index d99d623772b9..000000000000 --- a/tools/test_device_plan.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for the pure test-run planner (``tools/_device_plan.py``). - -The planner turns (files, runtime device mask, marker predicate) into a list of -``(file, mask)`` work units. It is pure: no I/O, no subprocess, no collection. -""" - -from __future__ import annotations - -import pytest -from _device_plan import Unit, is_isolated, plan_units - -# ---- plan_units: the per-runner planning logic -------------------------------- - - -def test_mix_ok_file_is_a_single_unit(): - # A file that does not need device isolation runs once, covering the whole mask. - assert plan_units(["a.py"], "110", is_isolated=lambda f: False) == [Unit("a.py", "110")] - - -def test_cant_mix_file_splits_into_one_unit_per_set_bit(): - # An isolated file on a multi-device runtime splits into one process per device. - # Split units are flagged so the executor gives them distinct report names. - assert plan_units(["a.py"], "110", is_isolated=lambda f: True) == [ - Unit("a.py", "100", split=True), - Unit("a.py", "010", split=True), - ] - - -def test_cant_mix_on_single_device_runtime_does_not_split(): - # An mgpu shard's runtime is one device, so even an isolated file is one unit. - assert plan_units(["a.py"], "0001", is_isolated=lambda f: True) == [Unit("a.py", "0001")] - - -def test_cant_mix_on_cpu_only_runtime_does_not_split(): - assert plan_units(["a.py"], "100", is_isolated=lambda f: True) == [Unit("a.py", "100")] - - -def test_split_masks_use_each_set_bit_position(): - # Set bits at positions 0 and 2 -> two single-bit masks of the same width. - assert plan_units(["a.py"], "1010", is_isolated=lambda f: True) == [ - Unit("a.py", "1000", split=True), - Unit("a.py", "0010", split=True), - ] - - -def test_multiple_files_preserve_order_and_decide_per_file(): - def isolated(f): - return f == "b.py" - - assert plan_units(["a.py", "b.py"], "110", is_isolated=isolated) == [ - Unit("a.py", "110"), - Unit("b.py", "100", split=True), - Unit("b.py", "010", split=True), - ] - - -def test_runtime_mask_with_trailing_x_is_rejected(): - # Runtime masks are always concrete; the open-ended ``X`` form is a scope-only - # construct and must never reach the planner. - with pytest.raises(ValueError): - plan_units(["a.py"], "11X", is_isolated=lambda f: True) - - -# ---- is_isolated: the device-isolation marker detector ------------------------ - - -def test_single_mark_is_detected(tmp_path): - f = tmp_path / "t.py" - f.write_text("import pytest\npytestmark = pytest.mark.device_isolated\ndef test_x(): pass\n") - assert is_isolated(f) is True - - -def test_list_form_mark_is_detected(tmp_path): - f = tmp_path / "t.py" - f.write_text("import pytest\npytestmark = [pytest.mark.device_isolated, pytest.mark.slow]\ndef test_x(): pass\n") - assert is_isolated(f) is True - - -def test_preloaded_source_is_used(): - assert is_isolated("does_not_exist.py", source="pytestmark = pytest.mark.device_isolated\n") is True - - -def test_no_mark_is_not_detected(tmp_path): - f = tmp_path / "t.py" - f.write_text("import pytest\ndef test_x(): pass\n") - assert is_isolated(f) is False - - -def test_word_in_comment_does_not_match(tmp_path): - f = tmp_path / "t.py" - f.write_text("# device_isolated explains the lock\ndef test_x(): pass\n") - assert is_isolated(f) is False - - -def test_unrelated_pytestmark_does_not_match(tmp_path): - f = tmp_path / "t.py" - f.write_text("import pytest\npytestmark = pytest.mark.slow\ndef test_x(): pass\n") - assert is_isolated(f) is False - - -def test_missing_file_returns_false(tmp_path): - assert is_isolated(tmp_path / "nope.py") is False diff --git a/tools/test_device_select.py b/tools/test_device_select.py deleted file mode 100644 index 6318df0ada3a..000000000000 --- a/tools/test_device_select.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for the per-unit device selector (``tools/_device_select.py``). - -These pin the rule that broke when device selection was left to the mask alone: -a literal ``["cuda:0", "cpu"]`` variant must still be dropped when it is out of -the unit's mask, or it leaks into a device-locked process. The plugin reads the -structured ``device`` param, so it catches both literal and ``test_devices()`` -variants. -""" - -from __future__ import annotations - -import _device_select -from _device_select import _active, _position, pytest_collection_modifyitems - - -class _FakeCallspec: - def __init__(self, params): - self.params = params - - -class _FakeItem: - """Stand-in for a pytest item. Omitting ``params`` means no callspec (agnostic).""" - - def __init__(self, name, params=None): - self.name = name - if params is not None: - self.callspec = _FakeCallspec(params) - - -class _FakeHook: - def __init__(self): - self.deselected = [] - - def pytest_deselected(self, items): - self.deselected.extend(items) - - -class _FakeConfig: - def __init__(self): - self.hook = _FakeHook() - - -def _select(mask, items, monkeypatch): - monkeypatch.setenv("ISAACLAB_TEST_DEVICES", mask) - config = _FakeConfig() - pytest_collection_modifyitems(config, items) - return items, config.hook.deselected - - -# ---- position / active helpers ------------------------------------------------ - - -def test_position_maps_cpu_and_cuda(): - assert _position("cpu") == 0 - assert _position("cuda:0") == 1 - assert _position("cuda:2") == 3 - assert _position(None) is None - assert _position("weird") is None - - -def test_active_reads_mask_position_and_trailing_x(): - assert _active(0, "110") is True # cpu - assert _active(1, "110") is True # cuda:0 - assert _active(2, "110") is False # cuda:1 not set - assert _active(3, "00X") is True # trailing X includes the rest - assert _active(None, "110") is False - - -# ---- selection: the bug this guards against ----------------------------------- - - -def test_cpu_unit_drops_literal_cuda_variant(monkeypatch): - # The bug: a literal ["cuda:0","cpu"] test's cuda variant leaked into the cpu - # unit and tripped the device lock. The cpu unit (mask 100) must drop it. - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) - items, dropped = _select("100", [cpu, cuda], monkeypatch) - assert items == [cpu] - assert dropped == [cuda] - - -def test_cuda_unit_drops_literal_cpu_variant_and_agnostic(monkeypatch): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) - agnostic = _FakeItem("t_logic") - items, dropped = _select("010", [cpu, cuda, agnostic], monkeypatch) - assert items == [cuda] - assert set(dropped) == {cpu, agnostic} - - -def test_shard_keeps_its_gpu_variant_drops_others_and_agnostic(monkeypatch): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - cuda0 = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) - cuda2 = _FakeItem("t[cuda:2]", {"device": "cuda:2"}) - agnostic = _FakeItem("t_logic") - items, dropped = _select("0001", [cpu, cuda0, cuda2, agnostic], monkeypatch) # cuda:2 shard - assert items == [cuda2] - assert set(dropped) == {cpu, cuda0, agnostic} - - -def test_cpu_unit_keeps_agnostic(monkeypatch): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - agnostic = _FakeItem("t_logic") - items, dropped = _select("100", [cpu, agnostic], monkeypatch) - assert items == [cpu, agnostic] # cpu in mask -> agnostic kept - assert dropped == [] - - -def test_default_mask_keeps_cpu_and_cuda0_and_agnostic(monkeypatch): - cpu = _FakeItem("t[cpu]", {"device": "cpu"}) - cuda = _FakeItem("t[cuda:0]", {"device": "cuda:0"}) - agnostic = _FakeItem("t_logic") - items, dropped = _select("110", [cpu, cuda, agnostic], monkeypatch) - assert items == [cpu, cuda, agnostic] - assert dropped == [] - - -def test_sessionfinish_maps_no_tests_collected_to_ok(): - import pytest - - class _Session: - exitstatus = pytest.ExitCode.NO_TESTS_COLLECTED - - session = _Session() - _device_select.pytest_sessionfinish(session, pytest.ExitCode.NO_TESTS_COLLECTED) - assert session.exitstatus == pytest.ExitCode.OK diff --git a/tools/test_runner/planning.py b/tools/test_runner/planning.py index b8aa2da655b7..831c14a21cfb 100644 --- a/tools/test_runner/planning.py +++ b/tools/test_runner/planning.py @@ -44,6 +44,8 @@ class RunnerConfig: ``"0001"`` a shard). Concrete — never the open-ended ``"X"`` form. queue_path: Shared work-queue root for mgpu work-stealing; empty when the runner iterates its own file list. + sim_device: Kit boot device; names the per-shard work-queue subdir and + labels rows in the report. isaacsim_ci: Whether to pass ``-m isaacsim_ci`` to each subprocess. filter_pattern: Substring a test path must contain to run; empty = all. exclude_pattern: Comma-separated substrings; a path matching any is skipped. @@ -73,6 +75,7 @@ class RunnerConfig: # -- run scope -- runtime_mask: str = "110" queue_path: str = "" + sim_device: str = "cuda:0" isaacsim_ci: bool = False # -- collection -- filter_pattern: str = "" @@ -130,6 +133,8 @@ def from_env(cls, workspace_root: str) -> RunnerConfig: # An unset mask means single-GPU CI: cpu + cuda:0, matching test_devices()'s default. runtime_mask=os.environ.get("ISAACLAB_TEST_DEVICES") or "110", queue_path=os.environ.get("ISAACLAB_TEST_QUEUE", ""), + # Kit boot device; names the per-shard work-queue subdir and labels the report. + sim_device=os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0", isaacsim_ci=os.environ.get("ISAACSIM_CI_SHORT", "false") == "true", filter_pattern=os.environ.get("TEST_FILTER_PATTERN", ""), exclude_pattern=os.environ.get("TEST_EXCLUDE_PATTERN", ""), diff --git a/tools/test_runner/session.py b/tools/test_runner/session.py new file mode 100644 index 000000000000..79a333887dbf --- /dev/null +++ b/tools/test_runner/session.py @@ -0,0 +1,350 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The test-run lifecycle: collect files, plan units, run them, report. + +:class:`Session` is the orchestrator that ``conftest.pytest_sessionstart`` +drives. It composes the stages: :class:`Collector` (which files), :class:`WorkQueue` +(how files are handed out across mgpu shards), the planner and unit runner from +the sibling modules (how each file runs), and :class:`Reporter` (merge + summary). +""" + +from __future__ import annotations + +import contextlib +import os +import re + +import pytest +import test_settings # tools/ module: QUARANTINED_TESTS / CUROBO_TESTS / TESTS_TO_SKIP +from junitparser import JUnitXml +from prettytable import PrettyTable + +from test_runner.execution import ExecContext, UnitRunner, merge_statuses +from test_runner.planning import Planner, RunnerConfig + + +class WorkQueue: + """Hand test files out across mgpu shards via an atomic-rename work queue. + + The queue is a directory of one file per pending test. A shard claims a test + by renaming it ``queue/`` -> ``inflight//`` (POSIX rename is + atomic, so racing shards are kernel-serialized), then ``-> done//`` + when it finishes. Anything left in ``inflight`` at job-end is a crashed claim. + When inactive (no queue configured) the runner just iterates its own list. + """ + + def __init__(self, config: RunnerConfig): + self._dir = config.queue_path + self._shard = config.sim_device.replace(":", "-") + + @property + def active(self) -> bool: + """Whether a shared queue is configured (mgpu) versus plain iteration.""" + return bool(self._dir) + + @staticmethod + def _slug(test_path: str) -> str: + """Encode a path as a flat queue filename (``/`` -> ``__``).""" + return test_path.replace("/", "__") + + def _claim(self) -> str | None: + """Atomically claim one pending test, or return None when the queue is empty.""" + pending_dir = os.path.join(self._dir, "queue") + inflight_dir = os.path.join(self._dir, "inflight", self._shard) + os.makedirs(inflight_dir, exist_ok=True) + try: + entries = sorted(os.listdir(pending_dir)) # uncached: another shard may have just taken one + except FileNotFoundError: + return None + for entry in entries: + try: + os.rename(os.path.join(pending_dir, entry), os.path.join(inflight_dir, entry)) + except FileNotFoundError: + continue # lost the race for this entry; try the next + return entry.replace("__", "/") + return None + + def mark_done(self, test_path: str) -> None: + """Move a finished claim from ``inflight//`` to ``done//``.""" + entry = self._slug(test_path) + done_dir = os.path.join(self._dir, "done", self._shard) + os.makedirs(done_dir, exist_ok=True) + # Suppress: already moved, or the runner died before marking — the reconciler catches that. + with contextlib.suppress(FileNotFoundError): + os.rename(os.path.join(self._dir, "inflight", self._shard, entry), os.path.join(done_dir, entry)) + + def iter(self, files: list[str]): + """Yield files to run: claimed from the queue when active, else ``files`` as-is.""" + if not self.active: + yield from files + return + while (claimed := self._claim()) is not None: + yield claimed + + +class Collector: + """Discover the test files to run, applying filters and file-level sharding.""" + + def __init__(self, config: RunnerConfig): + self._config = config + + def _source_dirs(self) -> list[str]: + """The roots walked for ``test_*.py`` files.""" + return [os.path.join(self._config.workspace_root, d) for d in ("scripts", "source")] + + def _selected(self, file_name: str, full_path: str) -> bool: + """Whether one ``test_*.py`` passes the active mode/skip/pattern filters.""" + config = self._config + if config.quarantined_only: + return file_name in test_settings.QUARANTINED_TESTS + if config.curobo_only: + return file_name in test_settings.CUROBO_TESTS + # An explicit include overrides the skip list (dedicated jobs run otherwise-skipped tests). + if file_name in test_settings.TESTS_TO_SKIP and file_name not in config.include_files: + return False + if config.filter_pattern and config.filter_pattern not in full_path: + return False + if config.exclude_pattern and any(p.strip() in full_path for p in config.exclude_pattern.split(",")): + return False + if config.include_files and file_name not in config.include_files: + return False + return True + + def _sharded(self, files: list[str]) -> list[str]: + """Select every Nth file for this file-shard; a no-op unless both shard env are set.""" + config = self._config + if config.shard_index is None or config.shard_count is None or config.include_files: + return files + files = sorted(files) + selected = [f for i, f in enumerate(files) if i % config.shard_count == config.shard_index] + print(f"Shard {config.shard_index}/{config.shard_count}: selected {len(selected)} test files") + return selected + + def _isaacsim_ci_only(self, files: list[str]) -> list[str]: + """Keep only files containing an ``@pytest.mark.isaacsim_ci`` test.""" + kept = [] + for test_file in files: + with open(test_file) as f: + if "@pytest.mark.isaacsim_ci" in f.read(): + kept.append(test_file) + return kept + + def discover(self) -> list[str]: + """Walk the source roots and return the filtered, sharded list of test files.""" + files = [] + for source_dir in self._source_dirs(): + if not os.path.exists(source_dir): + print(f"Error: source directory not found at {source_dir}") + pytest.exit("Source directory not found", returncode=1) + for root, _, names in os.walk(source_dir): + # install_ci has its own config/conftest and runs via its own action. + if "install_ci" in root.replace("\\", "/").split("/"): + continue + for name in names: + if name.startswith("test_") and name.endswith(".py"): + full_path = os.path.join(root, name) + if self._selected(name, full_path): + files.append(full_path) + files = self._sharded(files) + if self._config.isaacsim_ci: + files = self._isaacsim_ci_only(files) + return files + + +class Reporter: + """Merge per-unit JUnit reports and render the run summary.""" + + def __init__(self, config: RunnerConfig): + self._config = config + + @property + def _report_path(self) -> str: + return os.path.join(self._config.report_dir, self._config.result_file) + + def write_empty(self) -> None: + """Write an empty aggregate report so downstream CI steps find a valid file.""" + os.makedirs(self._config.report_dir, exist_ok=True) + JUnitXml().write(self._report_path) + print(f"Wrote empty report to {self._report_path}") + + @staticmethod + def _counts(files: list[str], status: dict) -> dict: + """Tally per-result-class file counts plus total wall/test time.""" + + def n(result): + return len([f for f in files if status[f]["result"] == result]) + + return { + "total": len(status), + "passing": len([f for f in files if status[f]["result"].startswith("passed")]), + "failing": n("FAILED"), + "timeout": n("TIMEOUT"), + "crashed": n("CRASHED"), + "startup_hang": n("STARTUP_HANG"), + "wall": sum(status[f]["wall_time"] for f in files), + "test": sum(status[f]["time_elapsed"] for f in files), + } + + def _per_file_table(self, files: list[str], status: dict) -> str: + """One row per file: path, GPU, result, times, pass/total.""" + table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + table.align["Test Path"] = "l" + table.align["Test (s)"] = "r" + table.align["Wall (s)"] = "r" + for f in files: + s = status[f] + passed = s["tests"] - s["failures"] - s["errors"] - s["skipped"] + table.add_row( + [ + f, + self._config.sim_device, + s["result"], + f"{s['time_elapsed']:0.2f}", + f"{s['wall_time']:0.2f}", + f"{passed}/{s['tests']}", + ] + ) + return table.get_string() + + def _per_test_time_table(self, full_report: JUnitXml) -> str: + """Per-test run times, slowest first; device read from the test id, else the boot device.""" + table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + table.align["Test"] = "l" + table.align["Time (s)"] = "r" + rows = [] + for suite in full_report: + for case in suite: + name = f"{case.classname}::{case.name}" if case.classname else case.name + device = self._config.sim_device + bracket = re.search(r"\[(.*)\]", name) + if bracket and (dev := re.search(r"cuda:\d+|\bcpu\b", bracket.group(1))): + device = dev.group(0) + rows.append((name, device, float(case.time) if case.time is not None else 0.0)) + for name, device, elapsed in sorted(rows, key=lambda r: r[2], reverse=True): + table.add_row([name, device, f"{elapsed:0.3f}"]) + return table.get_string() + + def _summary(self, files: list[str], status: dict, counts: dict, full_report: JUnitXml) -> str: + """Assemble the printed summary: result tally, per-file table, per-test times.""" + pct = 100 if counts["total"] == 0 else counts["passing"] / counts["total"] * 100 + wall, test = counts["wall"], counts["test"] + lines = [ + "\n\n===================\nTest Result Summary\n===================", + f"Total: {counts['total']}", + f"Passing: {counts['passing']}", + f"Failing: {counts['failing']}", + f"Crashed: {counts['crashed']}", + f"Startup Hang: {counts['startup_hang']}", + f"Timeout: {counts['timeout']}", + f"Passing Percentage: {pct:.2f}%", + f"Total Wall Time: {wall // 3600:.0f}h{wall // 60 % 60:.0f}m{wall % 60:.2f}s", + f"Total Test Time: {test // 3600:.0f}h{test // 60 % 60:.0f}m{test % 60:.2f}s", + "\n=======================\nPer File Result Summary\n=======================", + self._per_file_table(files, status), + "\n=================\nPer Test Run Time\n=================", + self._per_test_time_table(full_report), + ] + return "\n".join(lines) + + def finalize(self, files: list[str], status: dict, xml_reports: list) -> int: + """Merge reports, write the aggregate, print the summary, return the exit code. + + Returns 0 when every file passed, else 1. + """ + print("~~~~~~~~~ Collecting final report...") + # Merge the in-memory report objects (re-reading the files risks dropping + # elements through the junitparser round-trip). + full_report = JUnitXml() + for report in xml_reports: + full_report += report + os.makedirs(self._config.report_dir, exist_ok=True) + full_report.write(self._report_path) + print("~~~~~~~~~~~~ Report written to", self._report_path) + + counts = self._counts(files, status) + print(self._summary(files, status, counts, full_report)) + clean = counts["failing"] == counts["timeout"] == counts["crashed"] == counts["startup_hang"] == 0 + return 0 if clean else 1 + + +class Session: + """The per-file test-run lifecycle, driven by ``conftest.pytest_sessionstart``.""" + + def __init__(self, config: RunnerConfig): + self._config = config + self._collector = Collector(config) + self._queue = WorkQueue(config) + self._planner = Planner(config.runtime_mask) + self._runner = UnitRunner(config) + self._reporter = Reporter(config) + self._cold_cache_applied = False + + def _exec_context(self, file_name: str, source: str) -> ExecContext: + """Per-file execution context, applying the one-time cold-cache timeout bump.""" + config = self._config + timeout = config.per_file_timeouts.get(file_name, config.default_timeout) + # The first camera-enabled test compiles shaders on a cold cache (~600 s); + # give it (and only it) extra time so that isn't misread as a hang. + is_cold = not self._cold_cache_applied and config.cold_cache_marker in source + if is_cold: + timeout += config.cold_cache_buffer + self._cold_cache_applied = True + print(f"⏱️ Adding {config.cold_cache_buffer}s cold-cache buffer (timeout now {timeout}s)") + startup_deadline = min(timeout, config.startup_deadline + (config.cold_cache_buffer if is_cold else 0)) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + return ExecContext(file_name=file_name, timeout=timeout, startup_deadline=startup_deadline, env=env) + + def _run_file(self, test_file: str, status: dict, failed: list, reports: list) -> None: + """Plan ``test_file`` into units, run each, and fold their statuses into ``status``.""" + print(f"\n\n🚀 Running {test_file} independently...\n") + try: + with open(test_file) as fh: + source = fh.read() + except OSError: + source = "" + ctx = self._exec_context(os.path.basename(test_file), source) + units = self._planner.plan(test_file, source) + if len(units) > 1: + print(f"⚙️ device_isolated — invoking {ctx.file_name} once per device ({len(units)} units)") + merged = None + for unit in units: + report, unit_status, was_failure = self._runner.run(unit, ctx) + if report is not None: + reports.append(report) + if was_failure and test_file not in failed: + failed.append(test_file) + merged = merge_statuses(merged, unit_status) + status[test_file] = merged + + def run(self) -> int: + """Run the lifecycle and return the process exit code. + + Returns: + 0 when nothing failed (including a clean "nothing to run"), else 1. + """ + files = self._collector.discover() + if not files: + # A configured-but-empty scope is success; a bare empty run is an error. + if self._config.quarantined_only or self._config.filter_pattern: + print("No tests in scope — nothing to run.") + self._reporter.write_empty() + return 0 + print("No test files found in source directory") + return 1 + + print(f"Found {len(files)} test files after filtering") + status, failed, reports = {}, [], [] + for test_file in self._queue.iter(files): + self._run_file(test_file, status, failed, reports) + if self._queue.active: + self._queue.mark_done(test_file) + print("~~~~~~~~~~~~ Finished running all tests") + print("failed tests:", failed) + + # In work-queue mode this container ran only the files it claimed; report on those. + reported_files = list(status) if self._queue.active else files + return self._reporter.finalize(reported_files, status, reports) diff --git a/tools/test_runner/tests/test_session.py b/tools/test_runner/tests/test_session.py new file mode 100644 index 000000000000..9133b3239f1d --- /dev/null +++ b/tools/test_runner/tests/test_session.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.session: WorkQueue, Collector filters, Reporter, cold-cache.""" + +from __future__ import annotations + +from test_runner.planning import RunnerConfig +from test_runner.session import Collector, Reporter, Session, WorkQueue + + +def _status(result, *, tests=1, failures=0, errors=0, skipped=0, wall=1.0, elapsed=0.5): + return { + "result": result, + "tests": tests, + "failures": failures, + "errors": errors, + "skipped": skipped, + "wall_time": wall, + "time_elapsed": elapsed, + } + + +# ---- WorkQueue ---- + + +def test_workqueue_inactive_iterates_the_given_files(): + wq = WorkQueue(RunnerConfig(workspace_root="/ws")) # no queue_path + assert wq.active is False + assert list(wq.iter(["a.py", "b.py"])) == ["a.py", "b.py"] + + +def test_workqueue_claim_then_mark_done_roundtrip(tmp_path): + config = RunnerConfig(workspace_root="/ws", queue_path=str(tmp_path), sim_device="cuda:1") + (tmp_path / "queue").mkdir() + (tmp_path / "queue" / "src__pkg__test_x.py").write_text("") + wq = WorkQueue(config) + assert wq.active is True + claimed = list(wq.iter([])) + assert claimed == ["src/pkg/test_x.py"] # decoded path + assert (tmp_path / "inflight" / "cuda-1" / "src__pkg__test_x.py").exists() + wq.mark_done("src/pkg/test_x.py") + assert (tmp_path / "done" / "cuda-1" / "src__pkg__test_x.py").exists() + assert not (tmp_path / "inflight" / "cuda-1" / "src__pkg__test_x.py").exists() + + +# ---- Collector filters ---- + + +def test_collector_selects_by_filter_and_exclude_pattern(): + c = Collector(RunnerConfig(workspace_root="/ws", filter_pattern="physx")) + assert c._selected("test_zzz_synthetic.py", "/a/physx/test_zzz_synthetic.py") is True + assert c._selected("test_zzz_synthetic.py", "/a/newton/test_zzz_synthetic.py") is False + + c = Collector(RunnerConfig(workspace_root="/ws", exclude_pattern="newton")) + assert c._selected("test_zzz_synthetic.py", "/a/newton/test_zzz_synthetic.py") is False + + +def test_collector_respects_include_files(): + c = Collector(RunnerConfig(workspace_root="/ws", include_files=frozenset({"test_keep.py"}))) + assert c._selected("test_keep.py", "/a/test_keep.py") is True + assert c._selected("test_drop.py", "/a/test_drop.py") is False + + +# ---- Reporter counts ---- + + +def test_reporter_counts_tally_results_and_time(): + files = ["a.py", "b.py", "c.py"] + status = {"a.py": _status("passed"), "b.py": _status("FAILED"), "c.py": _status("passed (shutdown hanged)")} + counts = Reporter._counts(files, status) + assert counts["total"] == 3 + assert counts["passing"] == 2 # both "passed*" results + assert counts["failing"] == 1 + assert counts["wall"] == 3.0 + + +# ---- Session cold-cache timeout policy ---- + + +def test_cold_cache_buffer_applies_once_to_the_first_camera_file(): + config = RunnerConfig(workspace_root="/ws", default_timeout=600, cold_cache_buffer=700, startup_deadline=120) + session = Session(config) + first = session._exec_context("test_cam.py", "enable_cameras=True") + assert first.timeout == 1300 # 600 + 700 + assert first.startup_deadline == 820 # min(1300, 120 + 700) + second = session._exec_context("test_cam2.py", "enable_cameras=True") + assert second.timeout == 600 # buffer already spent + assert second.startup_deadline == 120 + + +def test_per_file_timeout_override(): + config = RunnerConfig(workspace_root="/ws", default_timeout=600, per_file_timeouts={"test_slow.py": 1200}) + ctx = Session(config)._exec_context("test_slow.py", "") + assert ctx.timeout == 1200