From 22bf04026ac99e11ba255796142d131488d3ce5d Mon Sep 17 00:00:00 2001 From: jichuanh Date: Fri, 5 Jun 2026 08:48:06 +0000 Subject: [PATCH 1/4] Add ISAACLAB_PIN_KIT_GPU to pin Kit renderer to one GPU Add an ISAACLAB_PIN_KIT_GPU env var to AppLauncher. When truthy, it appends Kit command-line overrides that pin the renderer to a single GPU (renderer.multiGpu.enabled=False, autoEnable=False, maxGpuCount=1) and disable the fabric GPU-interop path (physics.fabricUseGPUInterop= false), so each Kit process touches only its assigned GPU instead of enumerating every visible GPU at startup. Used by the multi-GPU CI workflow to avoid a shared GPU-interop context across concurrent sibling shards, which otherwise surfaces as "Stage X already attached" errors and SimulationApp.close hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). Off by default; single-GPU and user-facing rendering paths are unchanged. --- .../jichuanh-mgpu-pin-kit-resources.rst | 13 ++++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..a78313ab14d9 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,13 @@ +Added +^^^^^ + +* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. + When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, + ``--/renderer/multiGpu/autoEnable=False``, ``--/renderer/multiGpu/maxGpuCount=1`` + and ``--/physics/fabricUseGPUInterop=false`` to the Kit command line so each + Kit process touches only its assigned GPU (rather than enumerating every + visible GPU at startup). Used by the multi-GPU CI workflow to prevent the + shared GPU-interop context across sibling shards that surfaces as + ``[Error] [omni.physx.plugin] Stage X already attached`` and + ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). + Off by default; single-GPU and user-facing rendering paths are unchanged. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e8ff526496a1..53dd8065aa11 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1060,6 +1060,32 @@ def _resolve_device_settings(self, launcher_args: dict): launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id + # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is + # truthy. The default ``apps/isaaclab.python.headless.kit`` sets + # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable + # = true``, so each Kit process enumerates every visible GPU at + # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per + # container, one Kit per non-default cuda device), that produces a + # shared GPU-interop context across sibling processes -- surfacing as + # ``[Error] [omni.physx.plugin] Stage X already attached`` mid-test and + # ``SimulationApp.close`` hanging in teardown (see + # https://github.com/isaac-sim/IsaacLab/issues/3475). The mitigation is + # to set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount = 1`` so + # each Kit only touches its assigned GPU. + if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: + sys.argv.append("--/renderer/multiGpu/enabled=False") + sys.argv.append("--/renderer/multiGpu/autoEnable=False") + sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") + # Also disable the fabric GPU-interop path. The renderer multiGpu + # flags above mitigate the startup-time enumeration race; this + # mitigates the runtime GPU-interop race on top. Safe for the + # multi-GPU CI lane: it covers physics / scene / utility tests, + # not rendering. + sys.argv.append("--/physics/fabricUseGPUInterop=false") + logger.info( + "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" + ) + # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash # Kit's platform-info fork during startup. From b0526168b04a2641132504a1ef6dd13286733cb7 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 30 Jun 2026 19:15:17 +0000 Subject: [PATCH 2/4] Refine Kit multi-GPU controls Disable single-process renderer multi-GPU by default and provide an explicit opt-in. Keep the Fabric interop mitigation independent so multi-GPU CI can apply it temporarily without changing user-facing renderer behavior. --- .../jichuanh-mgpu-pin-kit-resources.rst | 21 ++--- source/isaaclab/isaaclab/app/app_launcher.py | 77 +++++++++++-------- .../test/app/test_argparser_launch.py | 21 +++++ .../isaaclab/test/app/test_env_var_launch.py | 70 +++++++++++++++++ 4 files changed, 147 insertions(+), 42 deletions(-) diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst index a78313ab14d9..80c4b76170ef 100644 --- a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -1,13 +1,14 @@ +Changed +^^^^^^^ + +* Changed :class:`~isaaclab.app.AppLauncher` to disable single-process multi-GPU + rendering by default. Set ``multi_gpu=True`` or pass ``--multi_gpu`` to restore + the previous rendering behavior. + Added ^^^^^ -* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. - When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, - ``--/renderer/multiGpu/autoEnable=False``, ``--/renderer/multiGpu/maxGpuCount=1`` - and ``--/physics/fabricUseGPUInterop=false`` to the Kit command line so each - Kit process touches only its assigned GPU (rather than enumerating every - visible GPU at startup). Used by the multi-GPU CI workflow to prevent the - shared GPU-interop context across sibling shards that surfaces as - ``[Error] [omni.physx.plugin] Stage X already attached`` and - ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). - Off by default; single-GPU and user-facing rendering paths are unchanged. +* Added ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` to override the corresponding PhysX + Fabric Kit setting without changing renderer multi-GPU behavior. The multi-GPU + CI override is a temporary workaround to remove after the underlying Kit/PhysX + problem is fixed. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index ec6f2875a23e..7bff50c6f4cd 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -33,6 +33,8 @@ # import logger logger = logging.getLogger(__name__) +_FABRIC_GPU_INTEROP_ENV = "ISAACLAB_FABRIC_USE_GPU_INTEROP" + # Suppress noisy debug-level logs from third-party libraries logging.getLogger("websockets").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING) @@ -80,6 +82,10 @@ class AppLauncher: value >-1. In other words, if ``livestream=-1``, then the value from the environment variable ``LIVESTREAM`` is used. + The ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` environment variable optionally overrides the + ``/physics/fabricUseGPUInterop`` Kit setting. Set it to ``1`` or ``0`` to enable or disable the setting. + When unset, Kit's configured default is preserved. + """ @staticmethod @@ -411,6 +417,9 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: - ``cuda``: Use GPU with device ID ``0``. - ``cuda:N``: Use GPU, where N is the device ID. For example, "cuda:0". + * ``multi_gpu`` (bool): Enable single-process multi-GPU rendering. Disabled by default so each Kit + process uses only its assigned GPU. Distributed execution always disables this setting per process. + * ``experience`` (str): The experience file to load when launching the SimulationApp. If a relative path is provided, it is resolved relative to the ``apps`` folder in Isaac Sim and Isaac Lab (in that order). @@ -523,6 +532,12 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: default=AppLauncher._APPLAUNCHER_CFG_INFO["device"][1], help='The device to run the simulation on. Can be "cpu", "cuda", "cuda:N", where N is the device ID', ) + arg_group.add_argument( + "--multi_gpu", + action="store_true", + default=AppLauncher._APPLAUNCHER_CFG_INFO["multi_gpu"][1], + help="Enable single-process multi-GPU rendering.", + ) arg_group.add_argument( "--visualizer", "--viz", @@ -627,6 +642,7 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: "enable_cameras": ([bool], False), "xr": ([bool], False), "device": ([str], "cuda:0"), + "multi_gpu": ([bool], False), "experience": ([str], ""), "deterministic": ([bool], False), "rendering_mode": ([str], "balanced"), @@ -1002,6 +1018,7 @@ def _resolve_device_settings(self, launcher_args: dict): """Resolve simulation GPU device related settings.""" self.device_id = 0 device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) + multi_gpu = launcher_args.get("multi_gpu", AppLauncher._APPLAUNCHER_CFG_INFO["multi_gpu"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) if self._xr and not device_explicitly_passed: @@ -1045,7 +1062,7 @@ def _resolve_device_settings(self, launcher_args: dict): self.device_id = 0 device = "cuda:" + str(self.device_id) - launcher_args["multi_gpu"] = False + multi_gpu = False # limit CPU threads to minimize thread context switching # this ensures processes do not take up all available threads and fight for resources num_cpu_cores = os.cpu_count() @@ -1060,32 +1077,7 @@ def _resolve_device_settings(self, launcher_args: dict): # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used. launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id - - # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is - # truthy. The default ``apps/isaaclab.python.headless.kit`` sets - # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable - # = true``, so each Kit process enumerates every visible GPU at - # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per - # container, one Kit per non-default cuda device), that produces a - # shared GPU-interop context across sibling processes -- surfacing as - # ``[Error] [omni.physx.plugin] Stage X already attached`` mid-test and - # ``SimulationApp.close`` hanging in teardown (see - # https://github.com/isaac-sim/IsaacLab/issues/3475). The mitigation is - # to set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount = 1`` so - # each Kit only touches its assigned GPU. - if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: - sys.argv.append("--/renderer/multiGpu/enabled=False") - sys.argv.append("--/renderer/multiGpu/autoEnable=False") - sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") - # Also disable the fabric GPU-interop path. The renderer multiGpu - # flags above mitigate the startup-time enumeration race; this - # mitigates the runtime GPU-interop race on top. Safe for the - # multi-GPU CI lane: it covers physics / scene / utility tests, - # not rendering. - sys.argv.append("--/physics/fabricUseGPUInterop=false") - logger.info( - "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" - ) + launcher_args["multi_gpu"] = multi_gpu # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash @@ -1195,11 +1187,32 @@ def _resolve_anim_recording_settings(self, launcher_args: dict): def _resolve_kit_args(self, launcher_args: dict): """Resolve additional arguments passed to Kit.""" - # Resolve additional arguments passed to Kit - self._kit_args = [] - if "kit_args" in launcher_args: - self._kit_args = [arg for arg in launcher_args["kit_args"].split()] - sys.argv += self._kit_args + self._kit_args = launcher_args.get("kit_args", "").split() + + default_args = [] + if not launcher_args["multi_gpu"]: + default_args = [ + "--/renderer/multiGpu/enabled=false", + "--/renderer/multiGpu/autoEnable=false", + "--/renderer/multiGpu/maxGpuCount=1", + ] + + fabric_gpu_interop = os.environ.get(_FABRIC_GPU_INTEROP_ENV) + if fabric_gpu_interop is not None: + if fabric_gpu_interop not in {"0", "1"}: + raise ValueError( + f"Invalid value for environment variable `{_FABRIC_GPU_INTEROP_ENV}`: {fabric_gpu_interop}." + " Expected: 0 or 1." + ) + default_args.append(f"--/physics/fabricUseGPUInterop={'true' if fabric_gpu_interop == '1' else 'false'}") + + # Preserve explicit Kit settings from sys.argv or kit_args. + for argument in default_args: + setting = argument.partition("=")[0] + if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): + self._kit_args.append(argument) + + sys.argv += self._kit_args def _create_app(self): """Launch and create the SimulationApp based on the parsed simulation config.""" diff --git a/source/isaaclab/test/app/test_argparser_launch.py b/source/isaaclab/test/app/test_argparser_launch.py index 6bce001522b7..7ed5ed977599 100644 --- a/source/isaaclab/test/app/test_argparser_launch.py +++ b/source/isaaclab/test/app/test_argparser_launch.py @@ -54,6 +54,27 @@ def test_headless_deprecated_arg_parsing(): assert args.headless_explicit is True +def test_multi_gpu_arg_parsing(): + """Test that renderer multi-GPU is disabled by default and can be enabled explicitly.""" + parser = argparse.ArgumentParser() + AppLauncher.add_app_launcher_args(parser) + + assert parser.parse_args([]).multi_gpu is False + assert parser.parse_args(["--multi_gpu"]).multi_gpu is True + + +def test_multi_gpu_defaults_to_disabled_without_argparser(): + """Test that direct AppLauncher construction resolves the same default as the argument parser.""" + launcher = AppLauncher.__new__(AppLauncher) + launcher._xr = False + launcher._deferred_cuda_device_id = None + launcher_args = {} + + launcher._resolve_device_settings(launcher_args) + + assert launcher_args["multi_gpu"] is False + + @pytest.mark.parametrize("value", ["none", "None"]) def test_visualizer_none_parsing(value: str): parser = argparse.ArgumentParser() diff --git a/source/isaaclab/test/app/test_env_var_launch.py b/source/isaaclab/test/app/test_env_var_launch.py index 6a58b220692d..de057fbff6ea 100644 --- a/source/isaaclab/test/app/test_env_var_launch.py +++ b/source/isaaclab/test/app/test_env_var_launch.py @@ -4,9 +4,11 @@ # SPDX-License-Identifier: BSD-3-Clause import os +import sys import pytest +import isaaclab.app.app_launcher as app_launcher_module from isaaclab.app import AppLauncher @@ -33,3 +35,71 @@ def test_livestream_launch_with_env_vars(mocker): # close the app on exit app.close() + + +def _resolve_kit_args( + monkeypatch: pytest.MonkeyPatch, launcher_args: dict, argv: list[str] | None = None +) -> tuple[list[str], list[str]]: + monkeypatch.setattr(sys, "argv", ["pytest", *(argv or [])]) + launcher = AppLauncher.__new__(AppLauncher) + launcher._resolve_kit_args({"multi_gpu": False, **launcher_args}) + return sys.argv[1:], launcher._kit_args + + +def test_renderer_multi_gpu_is_disabled_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {}) + + assert kit_args == [ + "--/renderer/multiGpu/enabled=false", + "--/renderer/multiGpu/autoEnable=false", + "--/renderer/multiGpu/maxGpuCount=1", + ] + assert resolved_args == kit_args + + +def test_renderer_multi_gpu_can_be_enabled(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"multi_gpu": True}) + + assert kit_args == [] + assert resolved_args == [] + + +@pytest.mark.parametrize("env_value, expected", [("0", "false"), ("1", "true")]) +def test_fabric_gpu_interop_env_adds_independent_override( + monkeypatch: pytest.MonkeyPatch, env_value: str, expected: str +): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, env_value) + + kit_args, _ = _resolve_kit_args(monkeypatch, {"multi_gpu": True}) + + assert kit_args == [f"--/physics/fabricUseGPUInterop={expected}"] + + +def test_fabric_gpu_interop_env_rejects_invalid_value(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "false") + + with pytest.raises(ValueError, match="Expected: 0 or 1"): + _resolve_kit_args(monkeypatch, {"multi_gpu": True}) + + +def test_explicit_kit_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0") + explicit_arg = "--/physics/fabricUseGPUInterop=true" + + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"multi_gpu": True}, [explicit_arg]) + + assert kit_args == [explicit_arg] + assert resolved_args == [] + + +def test_explicit_renderer_kit_arg_is_not_duplicated(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) + explicit_arg = "--/renderer/multiGpu/enabled=true" + + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"kit_args": explicit_arg}) + + assert kit_args.count(explicit_arg) == 1 + assert resolved_args[0] == explicit_arg + assert "--/renderer/multiGpu/enabled=false" not in resolved_args From 92cfa435ba9492f5b96fba941fc4c4d5ebb8a46a Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 1 Jul 2026 17:36:21 +0000 Subject: [PATCH 3/4] Move renderer multi-GPU defaults to Kit Default the Isaac Lab Kit experiences to one renderer GPU while keeping XR explicitly multi-GPU. Keep Fabric GPU interop independent through a temporary environment override so CI can disable it without changing renderer behavior. --- apps/isaaclab.python.headless.kit | 10 ++--- apps/isaaclab.python.kit | 6 +++ apps/isaaclab.python.xr.openxr.kit | 2 + .../jichuanh-mgpu-pin-kit-resources.rst | 6 +-- source/isaaclab/isaaclab/app/app_launcher.py | 29 ++----------- .../test/app/test_argparser_launch.py | 21 --------- .../isaaclab/test/app/test_env_var_launch.py | 43 +++++-------------- 7 files changed, 30 insertions(+), 87 deletions(-) diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 2d0858512ede..8dfef00323ed 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -62,11 +62,11 @@ interceptSysStdOutput = false logSysStdOutput = false [settings] -# MGPU is always on, you can turn it from the settings, and force this off to save even more resource if you -# only want to use a single GPU on your MGPU system -# False for Isaac Sim -renderer.multiGpu.enabled = true -renderer.multiGpu.autoEnable = true +# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these +# settings through AppLauncher's ``kit_args``. +renderer.multiGpu.enabled = false +renderer.multiGpu.autoEnable = false +renderer.multiGpu.maxGpuCount = 1 'rtx-transient'.resourcemanager.enableTextureStreaming = true app.asyncRendering = false app.asyncRenderingLowLatency = false diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index d9dbfd942a6c..000eae89a497 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -249,6 +249,12 @@ omni.replicator.asyncRendering = false app.asyncRendering = false app.asyncRenderingLowLatency = false +# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these +# settings through AppLauncher's ``kit_args``. +renderer.multiGpu.enabled = false +renderer.multiGpu.autoEnable = false +renderer.multiGpu.maxGpuCount = 1 + ### FSD app.useFabricSceneDelegate = true rtx.hydra.readTransformsFromFabricInRenderDelegate = true diff --git a/apps/isaaclab.python.xr.openxr.kit b/apps/isaaclab.python.xr.openxr.kit index 6248484e58b0..2a4385e0942d 100644 --- a/apps/isaaclab.python.xr.openxr.kit +++ b/apps/isaaclab.python.xr.openxr.kit @@ -24,6 +24,8 @@ app.asyncRendering = true app.asyncRenderingLowLatency = true # For XR, set this back to default "#define OMNI_MAX_DEVICE_GROUP_DEVICE_COUNT 16" +renderer.multiGpu.enabled = true +renderer.multiGpu.autoEnable = true renderer.multiGpu.maxGpuCount = 16 renderer.gpuEnumeration.glInterop.enabled = true # Allow Kit XR OpenXR to render headless diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst index 80c4b76170ef..d2f13c6efca0 100644 --- a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -1,9 +1,9 @@ Changed ^^^^^^^ -* Changed :class:`~isaaclab.app.AppLauncher` to disable single-process multi-GPU - rendering by default. Set ``multi_gpu=True`` or pass ``--multi_gpu`` to restore - the previous rendering behavior. +* Changed the Isaac Lab Kit experiences to use one renderer GPU by default. To + enable single-process multi-GPU rendering, pass the ``renderer.multiGpu`` + settings explicitly through :class:`~isaaclab.app.AppLauncher`'s ``kit_args``. Added ^^^^^ diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 7bff50c6f4cd..1384b4a0a026 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -417,9 +417,6 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: - ``cuda``: Use GPU with device ID ``0``. - ``cuda:N``: Use GPU, where N is the device ID. For example, "cuda:0". - * ``multi_gpu`` (bool): Enable single-process multi-GPU rendering. Disabled by default so each Kit - process uses only its assigned GPU. Distributed execution always disables this setting per process. - * ``experience`` (str): The experience file to load when launching the SimulationApp. If a relative path is provided, it is resolved relative to the ``apps`` folder in Isaac Sim and Isaac Lab (in that order). @@ -440,6 +437,8 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: * ``kit_args`` (str): Optional command line arguments to be passed to Omniverse Kit directly. Arguments should be combined into a single string separated by space. Example usage: --kit_args "--ext-folder=/path/to/ext1 --ext-folder=/path/to/ext2" + Isaac Lab experiences use one renderer GPU by default. Applications that need single-process + multi-GPU rendering can override the ``renderer.multiGpu`` settings through this argument. * ``visualizer`` (str): Visualizer backends to enable. Valid options are: @@ -532,12 +531,6 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: default=AppLauncher._APPLAUNCHER_CFG_INFO["device"][1], help='The device to run the simulation on. Can be "cpu", "cuda", "cuda:N", where N is the device ID', ) - arg_group.add_argument( - "--multi_gpu", - action="store_true", - default=AppLauncher._APPLAUNCHER_CFG_INFO["multi_gpu"][1], - help="Enable single-process multi-GPU rendering.", - ) arg_group.add_argument( "--visualizer", "--viz", @@ -642,7 +635,6 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: "enable_cameras": ([bool], False), "xr": ([bool], False), "device": ([str], "cuda:0"), - "multi_gpu": ([bool], False), "experience": ([str], ""), "deterministic": ([bool], False), "rendering_mode": ([str], "balanced"), @@ -1018,7 +1010,6 @@ def _resolve_device_settings(self, launcher_args: dict): """Resolve simulation GPU device related settings.""" self.device_id = 0 device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) - multi_gpu = launcher_args.get("multi_gpu", AppLauncher._APPLAUNCHER_CFG_INFO["multi_gpu"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) if self._xr and not device_explicitly_passed: @@ -1062,7 +1053,7 @@ def _resolve_device_settings(self, launcher_args: dict): self.device_id = 0 device = "cuda:" + str(self.device_id) - multi_gpu = False + launcher_args["multi_gpu"] = False # limit CPU threads to minimize thread context switching # this ensures processes do not take up all available threads and fight for resources num_cpu_cores = os.cpu_count() @@ -1077,7 +1068,6 @@ def _resolve_device_settings(self, launcher_args: dict): # as the active_gpu device. Setting physics_gpu explicitly may result in a different device to be used. launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id - launcher_args["multi_gpu"] = multi_gpu # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash @@ -1189,14 +1179,6 @@ def _resolve_kit_args(self, launcher_args: dict): """Resolve additional arguments passed to Kit.""" self._kit_args = launcher_args.get("kit_args", "").split() - default_args = [] - if not launcher_args["multi_gpu"]: - default_args = [ - "--/renderer/multiGpu/enabled=false", - "--/renderer/multiGpu/autoEnable=false", - "--/renderer/multiGpu/maxGpuCount=1", - ] - fabric_gpu_interop = os.environ.get(_FABRIC_GPU_INTEROP_ENV) if fabric_gpu_interop is not None: if fabric_gpu_interop not in {"0", "1"}: @@ -1204,10 +1186,7 @@ def _resolve_kit_args(self, launcher_args: dict): f"Invalid value for environment variable `{_FABRIC_GPU_INTEROP_ENV}`: {fabric_gpu_interop}." " Expected: 0 or 1." ) - default_args.append(f"--/physics/fabricUseGPUInterop={'true' if fabric_gpu_interop == '1' else 'false'}") - - # Preserve explicit Kit settings from sys.argv or kit_args. - for argument in default_args: + argument = f"--/physics/fabricUseGPUInterop={'true' if fabric_gpu_interop == '1' else 'false'}" setting = argument.partition("=")[0] if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): self._kit_args.append(argument) diff --git a/source/isaaclab/test/app/test_argparser_launch.py b/source/isaaclab/test/app/test_argparser_launch.py index 7ed5ed977599..6bce001522b7 100644 --- a/source/isaaclab/test/app/test_argparser_launch.py +++ b/source/isaaclab/test/app/test_argparser_launch.py @@ -54,27 +54,6 @@ def test_headless_deprecated_arg_parsing(): assert args.headless_explicit is True -def test_multi_gpu_arg_parsing(): - """Test that renderer multi-GPU is disabled by default and can be enabled explicitly.""" - parser = argparse.ArgumentParser() - AppLauncher.add_app_launcher_args(parser) - - assert parser.parse_args([]).multi_gpu is False - assert parser.parse_args(["--multi_gpu"]).multi_gpu is True - - -def test_multi_gpu_defaults_to_disabled_without_argparser(): - """Test that direct AppLauncher construction resolves the same default as the argument parser.""" - launcher = AppLauncher.__new__(AppLauncher) - launcher._xr = False - launcher._deferred_cuda_device_id = None - launcher_args = {} - - launcher._resolve_device_settings(launcher_args) - - assert launcher_args["multi_gpu"] is False - - @pytest.mark.parametrize("value", ["none", "None"]) def test_visualizer_none_parsing(value: str): parser = argparse.ArgumentParser() diff --git a/source/isaaclab/test/app/test_env_var_launch.py b/source/isaaclab/test/app/test_env_var_launch.py index de057fbff6ea..7e751f3bea3a 100644 --- a/source/isaaclab/test/app/test_env_var_launch.py +++ b/source/isaaclab/test/app/test_env_var_launch.py @@ -42,37 +42,15 @@ def _resolve_kit_args( ) -> tuple[list[str], list[str]]: monkeypatch.setattr(sys, "argv", ["pytest", *(argv or [])]) launcher = AppLauncher.__new__(AppLauncher) - launcher._resolve_kit_args({"multi_gpu": False, **launcher_args}) + launcher._resolve_kit_args(launcher_args) return sys.argv[1:], launcher._kit_args -def test_renderer_multi_gpu_is_disabled_by_default(monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) - kit_args, resolved_args = _resolve_kit_args(monkeypatch, {}) - - assert kit_args == [ - "--/renderer/multiGpu/enabled=false", - "--/renderer/multiGpu/autoEnable=false", - "--/renderer/multiGpu/maxGpuCount=1", - ] - assert resolved_args == kit_args - - -def test_renderer_multi_gpu_can_be_enabled(monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) - kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"multi_gpu": True}) - - assert kit_args == [] - assert resolved_args == [] - - @pytest.mark.parametrize("env_value, expected", [("0", "false"), ("1", "true")]) -def test_fabric_gpu_interop_env_adds_independent_override( - monkeypatch: pytest.MonkeyPatch, env_value: str, expected: str -): +def test_fabric_gpu_interop_env_adds_override(monkeypatch: pytest.MonkeyPatch, env_value: str, expected: str): monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, env_value) - kit_args, _ = _resolve_kit_args(monkeypatch, {"multi_gpu": True}) + kit_args, _ = _resolve_kit_args(monkeypatch, {}) assert kit_args == [f"--/physics/fabricUseGPUInterop={expected}"] @@ -81,25 +59,24 @@ def test_fabric_gpu_interop_env_rejects_invalid_value(monkeypatch: pytest.Monkey monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "false") with pytest.raises(ValueError, match="Expected: 0 or 1"): - _resolve_kit_args(monkeypatch, {"multi_gpu": True}) + _resolve_kit_args(monkeypatch, {}) def test_explicit_kit_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0") explicit_arg = "--/physics/fabricUseGPUInterop=true" - kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"multi_gpu": True}, [explicit_arg]) + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {}, [explicit_arg]) assert kit_args == [explicit_arg] assert resolved_args == [] -def test_explicit_renderer_kit_arg_is_not_duplicated(monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, raising=False) - explicit_arg = "--/renderer/multiGpu/enabled=true" +def test_explicit_kit_args_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0") + explicit_arg = "--/physics/fabricUseGPUInterop=true" kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"kit_args": explicit_arg}) - assert kit_args.count(explicit_arg) == 1 - assert resolved_args[0] == explicit_arg - assert "--/renderer/multiGpu/enabled=false" not in resolved_args + assert kit_args == [explicit_arg] + assert resolved_args == [explicit_arg] From b608df6e121f34b39aa7294f26a68b82bb2c28ff Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 1 Jul 2026 18:20:22 +0000 Subject: [PATCH 4/4] Deduplicate renderer GPU limits Keep the single-GPU renderer limit in the base Kit experiences and let the rendering variants inherit it alongside the other renderer multi-GPU defaults. --- apps/isaaclab.python.headless.rendering.kit | 3 --- apps/isaaclab.python.rendering.kit | 3 --- 2 files changed, 6 deletions(-) diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 9bb0a3692432..d2910f338221 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -76,9 +76,6 @@ rtx.ambientOcclusion.enabled = false # Set the DLSS model rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto) -# Avoids unnecessary GPU context initialization -renderer.multiGpu.maxGpuCount=1 - # Force synchronous rendering to improve training results omni.replicator.asyncRendering = false diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index 08982f9b4958..035d09ab96ad 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -71,9 +71,6 @@ rtx.ambientOcclusion.enabled = false # Set the DLSS model rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto) -# Avoids unnecessary GPU context initialization -renderer.multiGpu.maxGpuCount=1 - # Force synchronous rendering to improve training results omni.replicator.asyncRendering = false