[MGPU] App: default Kit renderer to one GPU

apps/isaaclab.python.headless.kit

@@ -62,11 +62,11 @@ interceptSysStdOutput = false
 logSysStdOutput = false
 
 [settings]
-# MGPU is always on, you can turn it from the settings, and force this off to save even more resource if you
-# only want to use a single GPU on your MGPU system
-# False for Isaac Sim
-renderer.multiGpu.enabled = true
-renderer.multiGpu.autoEnable = true
+# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these
+# settings through AppLauncher's ``kit_args``.
+renderer.multiGpu.enabled = false
+renderer.multiGpu.autoEnable = false
+renderer.multiGpu.maxGpuCount = 1
 'rtx-transient'.resourcemanager.enableTextureStreaming = true
 app.asyncRendering = false
 app.asyncRenderingLowLatency = false

apps/isaaclab.python.headless.rendering.kit

@@ -76,9 +76,6 @@ rtx.ambientOcclusion.enabled = false
 # Set the DLSS model
 rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto)
 
-# Avoids unnecessary GPU context initialization
-renderer.multiGpu.maxGpuCount=1
-
 # Force synchronous rendering to improve training results
 omni.replicator.asyncRendering = false
 

apps/isaaclab.python.kit

@@ -249,6 +249,12 @@ omni.replicator.asyncRendering = false
 app.asyncRendering = false
 app.asyncRenderingLowLatency = false
 
+# Use one renderer GPU by default. Applications that need single-process multi-GPU rendering can override these
+# settings through AppLauncher's ``kit_args``.
+renderer.multiGpu.enabled = false
+renderer.multiGpu.autoEnable = false
+renderer.multiGpu.maxGpuCount = 1
+
 ### FSD
 app.useFabricSceneDelegate = true
 rtx.hydra.readTransformsFromFabricInRenderDelegate = true

apps/isaaclab.python.rendering.kit

@@ -71,9 +71,6 @@ rtx.ambientOcclusion.enabled = false
 # Set the DLSS model
 rtx.post.dlss.execMode = 0 # can be 0 (Performance), 1 (Balanced), 2 (Quality), or 3 (Auto)
 
-# Avoids unnecessary GPU context initialization
-renderer.multiGpu.maxGpuCount=1
-
 # Force synchronous rendering to improve training results
 omni.replicator.asyncRendering = false
 

apps/isaaclab.python.xr.openxr.kit

@@ -24,6 +24,8 @@ app.asyncRendering = true
 app.asyncRenderingLowLatency = true
 
 # For XR, set this back to default "#define OMNI_MAX_DEVICE_GROUP_DEVICE_COUNT 16"
+renderer.multiGpu.enabled = true
+renderer.multiGpu.autoEnable = true
 renderer.multiGpu.maxGpuCount = 16
 renderer.gpuEnumeration.glInterop.enabled = true # Allow Kit XR OpenXR to render headless
 

source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst

@@ -0,0 +1,14 @@
+Changed
+^^^^^^^
+
+* Changed the Isaac Lab Kit experiences to use one renderer GPU by default. To
+  enable single-process multi-GPU rendering, pass the ``renderer.multiGpu``
+  settings explicitly through :class:`~isaaclab.app.AppLauncher`'s ``kit_args``.
+
+Added
+^^^^^
+
+* Added ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` to override the corresponding PhysX
+  Fabric Kit setting without changing renderer multi-GPU behavior. The multi-GPU
+  CI override is a temporary workaround to remove after the underlying Kit/PhysX
+  problem is fixed.

source/isaaclab/isaaclab/app/app_launcher.py

@@ -33,6 +33,8 @@
 # import logger
 logger = logging.getLogger(__name__)
 
+_FABRIC_GPU_INTEROP_ENV = "ISAACLAB_FABRIC_USE_GPU_INTEROP"
+
 # Suppress noisy debug-level logs from third-party libraries
 logging.getLogger("websockets").setLevel(logging.WARNING)
 logging.getLogger("matplotlib").setLevel(logging.WARNING)
@@ -80,6 +82,10 @@ class AppLauncher:
         value >-1. In other words, if ``livestream=-1``, then the value from the environment variable
         ``LIVESTREAM`` is used.
 
+    The ``ISAACLAB_FABRIC_USE_GPU_INTEROP`` environment variable optionally overrides the
+    ``/physics/fabricUseGPUInterop`` Kit setting. Set it to ``1`` or ``0`` to enable or disable the setting.
+    When unset, Kit's configured default is preserved.
+
     """
 
     @staticmethod
@@ -431,6 +437,8 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None:
         * ``kit_args`` (str): Optional command line arguments to be passed to Omniverse Kit directly.
           Arguments should be combined into a single string separated by space.
           Example usage: --kit_args "--ext-folder=/path/to/ext1 --ext-folder=/path/to/ext2"
+          Isaac Lab experiences use one renderer GPU by default. Applications that need single-process
+          multi-GPU rendering can override the ``renderer.multiGpu`` settings through this argument.
 
         * ``visualizer`` (str): Visualizer backends to enable.
           Valid options are:
@@ -1169,11 +1177,21 @@ def _resolve_anim_recording_settings(self, launcher_args: dict):
 
     def _resolve_kit_args(self, launcher_args: dict):
         """Resolve additional arguments passed to Kit."""
-        # Resolve additional arguments passed to Kit
-        self._kit_args = []
-        if "kit_args" in launcher_args:
-            self._kit_args = [arg for arg in launcher_args["kit_args"].split()]
-            sys.argv += self._kit_args
+        self._kit_args = launcher_args.get("kit_args", "").split()
+
+        fabric_gpu_interop = os.environ.get(_FABRIC_GPU_INTEROP_ENV)
+        if fabric_gpu_interop is not None:
+            if fabric_gpu_interop not in {"0", "1"}:
+                raise ValueError(
+                    f"Invalid value for environment variable `{_FABRIC_GPU_INTEROP_ENV}`: {fabric_gpu_interop}."
+                    " Expected: 0 or 1."
+                )
+            argument = f"--/physics/fabricUseGPUInterop={'true' if fabric_gpu_interop == '1' else 'false'}"
+            setting = argument.partition("=")[0]
+            if not any(arg.partition("=")[0] == setting for arg in sys.argv + self._kit_args):
+                self._kit_args.append(argument)
+
+        sys.argv += self._kit_args
 
     def _create_app(self):
         """Launch and create the SimulationApp based on the parsed simulation config."""

source/isaaclab/test/app/test_env_var_launch.py

@@ -4,9 +4,11 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import os
+import sys
 
 import pytest
 
+import isaaclab.app.app_launcher as app_launcher_module
 from isaaclab.app import AppLauncher
 
 
@@ -33,3 +35,48 @@ def test_livestream_launch_with_env_vars(mocker):
 
     # close the app on exit
     app.close()
+
+
+def _resolve_kit_args(
+    monkeypatch: pytest.MonkeyPatch, launcher_args: dict, argv: list[str] | None = None
+) -> tuple[list[str], list[str]]:
+    monkeypatch.setattr(sys, "argv", ["pytest", *(argv or [])])
+    launcher = AppLauncher.__new__(AppLauncher)
+    launcher._resolve_kit_args(launcher_args)
+    return sys.argv[1:], launcher._kit_args
+
+
+@pytest.mark.parametrize("env_value, expected", [("0", "false"), ("1", "true")])
+def test_fabric_gpu_interop_env_adds_override(monkeypatch: pytest.MonkeyPatch, env_value: str, expected: str):
+    monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, env_value)
+
+    kit_args, _ = _resolve_kit_args(monkeypatch, {})
+
+    assert kit_args == [f"--/physics/fabricUseGPUInterop={expected}"]
+
+
+def test_fabric_gpu_interop_env_rejects_invalid_value(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "false")
+
+    with pytest.raises(ValueError, match="Expected: 0 or 1"):
+        _resolve_kit_args(monkeypatch, {})
+
+
+def test_explicit_kit_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0")
+    explicit_arg = "--/physics/fabricUseGPUInterop=true"
+
+    kit_args, resolved_args = _resolve_kit_args(monkeypatch, {}, [explicit_arg])
+
+    assert kit_args == [explicit_arg]
+    assert resolved_args == []
+
+
+def test_explicit_kit_args_setting_takes_precedence(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv(app_launcher_module._FABRIC_GPU_INTEROP_ENV, "0")
+    explicit_arg = "--/physics/fabricUseGPUInterop=true"
+
+    kit_args, resolved_args = _resolve_kit_args(monkeypatch, {"kit_args": explicit_arg})
+
+    assert kit_args == [explicit_arg]
+    assert resolved_args == [explicit_arg]