diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 2d0858512ede..8dfef00323ed 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -62,11 +62,11 @@ interceptSysStdOutput = false logSysStdOutput = false [settings] -# MGPU is always on, you can turn it from the settings, and force this off to save even more resource if you -# only want to use a single GPU on your MGPU system -# False for Isaac Sim -renderer.multiGpu.enabled = true -renderer.multiGpu.autoEnable = true +# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these +# settings through AppLauncher's ``kit_args``. +renderer.multiGpu.enabled = false +renderer.multiGpu.autoEnable = false +renderer.multiGpu.maxGpuCount = 1 'rtx-transient'.resourcemanager.enableTextureStreaming = true app.asyncRendering = false app.asyncRenderingLowLatency = false diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 9bb0a3692432..d2910f338221 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -76,9 +76,6 @@ rtx.ambientOcclusion.enabled = false # Set the DLSS model rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto) -# Avoids unnecessary GPU context initialization -renderer.multiGpu.maxGpuCount=1 - # Force synchronous rendering to improve training results omni.replicator.asyncRendering = false diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index d9dbfd942a6c..000eae89a497 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -249,6 +249,12 @@ omni.replicator.asyncRendering = false app.asyncRendering = false app.asyncRenderingLowLatency = false +# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these +# settings through AppLauncher's ``kit_args``. +renderer.multiGpu.enabled = false +renderer.multiGpu.autoEnable = false +renderer.multiGpu.maxGpuCount = 1 + ### FSD app.useFabricSceneDelegate = true rtx.hydra.readTransformsFromFabricInRenderDelegate = true diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index 08982f9b4958..035d09ab96ad 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -71,9 +71,6 @@ rtx.ambientOcclusion.enabled = false # Set the DLSS model rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto) -# Avoids unnecessary GPU context initialization -renderer.multiGpu.maxGpuCount=1 - # Force synchronous rendering to improve training results omni.replicator.asyncRendering = false diff --git a/apps/isaaclab.python.xr.openxr.kit b/apps/isaaclab.python.xr.openxr.kit index 6248484e58b0..2a4385e0942d 100644 --- a/apps/isaaclab.python.xr.openxr.kit +++ b/apps/isaaclab.python.xr.openxr.kit @@ -24,6 +24,8 @@ app.asyncRendering = true app.asyncRenderingLowLatency = true # For XR, set this back to default "#define OMNI_MAX_DEVICE_GROUP_DEVICE_COUNT 16" +renderer.multiGpu.enabled = true +renderer.multiGpu.autoEnable = true renderer.multiGpu.maxGpuCount = 16 renderer.gpuEnumeration.glInterop.enabled = true # Allow Kit XR OpenXR to render headless diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..d2f13c6efca0 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,14 @@ +Changed +^^^^^^^ + +* Changed the Isaac Lab Kit experiences to use one renderer GPU by default. To + enable single-process multi-GPU rendering, pass the ``renderer.multiGpu`` + settings explicitly through :class:`~isaaclab.app.AppLauncher`'s ``kit_args``. + +Added +^^^^^ + +* Added ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` to override the corresponding PhysX + Fabric Kit setting without changing renderer multi-GPU behavior. The multi-GPU + CI override is a temporary workaround to remove after the underlying Kit/PhysX + problem is fixed. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 99215b696afd..1384b4a0a026 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -33,6 +33,8 @@ # import logger logger = logging.getLogger(__name__) +_FABRIC_GPU_INTEROP_ENV = "ISAACLAB_FABRIC_USE_GPU_INTEROP" + # Suppress noisy debug-level logs from third-party libraries logging.getLogger("websockets").setLevel(logging.WARNING) logging.getLogger("matplotlib").setLevel(logging.WARNING) @@ -80,6 +82,10 @@ class AppLauncher: value >-1. In other words, if ``livestream=-1``, then the value from the environment variable ``LIVESTREAM`` is used. + The ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` environment variable optionally overrides the + ``/physics/fabricUseGPUInterop`` Kit setting. Set it to ``1`` or ``0`` to enable or disable the setting. + When unset, Kit's configured default is preserved. + """ @staticmethod @@ -431,6 +437,8 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: * ``kit_args`` (str): Optional command line arguments to be passed to Omniverse Kit directly. Arguments should be combined into a single string separated by space. Example usage: --kit_args "--ext-folder=/path/to/ext1 --ext-folder=/path/to/ext2" + Isaac Lab experiences use one renderer GPU by default. Applications that need single-process + multi-GPU rendering can override the ``renderer.multiGpu`` settings through this argument. * ``visualizer`` (str): Visualizer backends to enable. Valid options are: @@ -1169,11 +1177,21 @@ def _resolve_anim_recording_settings(self, launcher_args: dict): def _resolve_kit_args(self, launcher_args: dict): """Resolve additional arguments passed to Kit.""" - # Resolve additional arguments passed to Kit - self._kit_args = [] - if "kit_args" in launcher_args: - self._kit_args = [arg for arg in launcher_args["kit_args"].split()] - sys.argv += self._kit_args + self._kit_args = launcher_args.get("kit_args", "").split() + + fabric_gpu_interop = os.environ.get(_FABRIC_GPU_INTEROP_ENV) + if fabric_gpu_interop is not None: + if fabric_gpu_interop not in {"0", "1"}: + raise ValueError( + f"Invalid value for environment variable `{_FABRIC_GPU_INTEROP_ENV}`: {fabric_gpu_interop}." + " Expected: 0 or 1." + ) + argument = f"--/physics/fabricUseGPUInterop={'true' if fabric_gpu_interop == '1' else 'false'}" + setting = argument.partition("=")[0] + if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args): + self._kit_args.append(argument) + + sys.argv += self._kit_args def _create_app(self): """Launch and create the SimulationApp based on the parsed simulation config.""" diff --git a/source/isaaclab/test/app/test_env_var_launch.py b/source/isaaclab/test/app/test_env_var_launch.py index 6a58b220692d..7e751f3bea3a 100644 --- a/source/isaaclab/test/app/test_env_var_launch.py +++ b/source/isaaclab/test/app/test_env_var_launch.py @@ -4,9 +4,11 @@ # SPDX-License-Identifier: BSD-3-Clause import os +import sys import pytest +import isaaclab.app.app_launcher as app_launcher_module from isaaclab.app import AppLauncher @@ -33,3 +35,48 @@ def test_livestream_launch_with_env_vars(mocker): # close the app on exit app.close() + + +def _resolve_kit_args( + monkeypatch: pytest.MonkeyPatch, launcher_args: dict, argv: list[str] | None = None +) -> tuple[list[str], list[str]]: + monkeypatch.setattr(sys, "argv", ["pytest", *(argv or [])]) + launcher = AppLauncher.__new__(AppLauncher) + launcher._resolve_kit_args(launcher_args) + return sys.argv[1:], launcher._kit_args + + +@pytest.mark.parametrize("env_value, expected", [("0", "false"), ("1", "true")]) +def test_fabric_gpu_interop_env_adds_override(monkeypatch: pytest.MonkeyPatch, env_value: str, expected: str): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, env_value) + + kit_args, _ = _resolve_kit_args(monkeypatch, {}) + + assert kit_args == [f"--/physics/fabricUseGPUInterop={expected}"] + + +def test_fabric_gpu_interop_env_rejects_invalid_value(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "false") + + with pytest.raises(ValueError, match="Expected: 0 or 1"): + _resolve_kit_args(monkeypatch, {}) + + +def test_explicit_kit_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0") + explicit_arg = "--/physics/fabricUseGPUInterop=true" + + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {}, [explicit_arg]) + + assert kit_args == [explicit_arg] + assert resolved_args == [] + + +def test_explicit_kit_args_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0") + explicit_arg = "--/physics/fabricUseGPUInterop=true" + + kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"kit_args": explicit_arg}) + + assert kit_args == [explicit_arg] + assert resolved_args == [explicit_arg]