From e87545db0d725bdd6b211f0772a1b8f6e687d275 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Sun, 31 May 2026 00:09:53 +0000 Subject: [PATCH 1/3] Handle SIGHUP and force exit in AppLauncher abort handler Two coupled bugs in :class:`isaaclab.app.AppLauncher`: 1. SIGHUP was unhandled. Kit launches with ``--/app/installSignalHandlers=0``, so when a controlling session leader exits (e.g. the parent shell that supervises sibling shards in multi-GPU CI), child Kit processes receive SIGHUP with default disposition: terminate. ``_atexit_close`` does not run, so ``SimulationApp.close`` is skipped and USD/PhysX state is left attached. The next sibling shard then trips ``[Error] [omni.physx.plugin] Stage X already attached`` and Kit shutdown subsequently hangs on the orphan's state. Register the same handler used for SIGTERM/SIGABRT/SIGSEGV. 2. ``_abort_signal_handle_callback`` swallowed the signal's terminate semantics. After calling ``self._app.close()`` it returned, so Python resumed execution past the signal as if nothing happened. The replaced OS-default disposition would have killed the process; the Python handler did not. Wrap ``_app.close()`` in ``contextlib.suppress(Exception)`` and call ``sys.exit(128 + signum)`` to preserve the conventional signal-exit encoding and actually terminate. --- .../jichuanh-applauncher-sighup-handler.rst | 14 +++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 29 +++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst diff --git a/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst new file mode 100644 index 000000000000..a5b8d4cd3c90 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst @@ -0,0 +1,14 @@ +Fixed +^^^^^ + +* Handled ``SIGHUP`` in :class:`~isaaclab.app.AppLauncher` so the + simulation app shuts down cleanly when the controlling session leader + exits (e.g. parent shell supervising sibling shards in multi-GPU CI). + Previously SIGHUP terminated the process with default disposition, + bypassing :meth:`SimulationApp.close` and leaving USD/PhysX state + attached for the next sibling shard ("Stage X already attached" + log line and downstream shutdown hangs). +* Made :meth:`~isaaclab.app.AppLauncher._abort_signal_handle_callback` + exit the process after closing the app. The previous implementation + swallowed the signal's terminate semantics, allowing Python to + resume past a SIGTERM/SIGABRT/SIGSEGV and leaving Kit half-torn-down. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index d9330c2e5eeb..039852420708 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -350,13 +350,22 @@ def _atexit_close(app=self._app): atexit.register(_atexit_close) - # Set up signal handlers for graceful shutdown - # -- during explicit `kill` commands + # Set up signal handlers for graceful shutdown. Kit launches with + # ``--/app/installSignalHandlers=0``, so it's on us to drive ``app.close()`` + # before the default disposition terminates the process and leaves + # USD / PhysX state attached (the next sibling shard then trips + # "Stage X already attached" or hangs on physx detach). + # -- during explicit ``kill`` commands signal.signal(signal.SIGTERM, self._abort_signal_handle_callback) # -- during aborts signal.signal(signal.SIGABRT, self._abort_signal_handle_callback) # -- during segfaults signal.signal(signal.SIGSEGV, self._abort_signal_handle_callback) + # -- when the controlling session leader (e.g. a parent shell that + # supervises sibling shards) exits: SIGHUP cascades to children, and + # without a handler the default action would terminate before + # ``_atexit_close`` could run. + signal.signal(signal.SIGHUP, self._abort_signal_handle_callback) """ Properties. @@ -1374,7 +1383,15 @@ def _hide_play_button(self, flag): play_button_group._play_button.visible = not flag # type: ignore play_button_group._play_button.enabled = not flag # type: ignore - def _abort_signal_handle_callback(self, signal, frame): - """Handle the abort/segmentation/kill signals.""" - # close the app - self._app.close() + def _abort_signal_handle_callback(self, signum, frame): + """Handle the abort/segmentation/kill/hangup signals. + + Closes :class:`SimulationApp` so Kit detaches USD/PhysX state, then + exits with ``128 + signum`` to preserve the conventional signal-exit + encoding. Without the explicit exit, Python would resume execution + after the handler returns (since we replaced the OS-default + disposition), and Kit would be left half-torn-down. + """ + with contextlib.suppress(Exception): + self._app.close() + sys.exit(128 + signum) From 726439c7b254b89faffcebd7c68492349dd1792a Mon Sep 17 00:00:00 2001 From: jichuanh Date: Tue, 2 Jun 2026 23:58:59 +0000 Subject: [PATCH 2/3] Bound Kit shutdown hang with ISAACLAB_FORCE_EXIT_TIMEOUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds opt-in ``ISAACLAB_FORCE_EXIT_TIMEOUT`` (integer seconds) env var. When set, AppLauncher arms a daemon thread that calls ``os._exit(0)`` after the deadline, from both the ``atexit`` hook and the abort signal handler. Pairs with the SIGHUP handler in the prior commit: SIGHUP brings the shutdown sequence to a clean entry point, the timer bounds how long that sequence can stall inside Kit's binary ``shutdown_and_release_framework`` (the only step proven non-trivial by a step-by-step ``close()`` trace on Isaac Sim 6.0.0-rc.22). The hang is tracked upstream at https://github.com/isaac-sim/IsaacLab/issues/3475 and held by Kit binary code, so neither ``skip_cleanup=True`` nor Python signal handlers can interrupt it — only ``os._exit`` actually returns control. Off by default. CI workflows that need bounded shutdown set ``ISAACLAB_FORCE_EXIT_TIMEOUT=10`` on the runner; interactive user code still gets the graceful teardown path. Local validation: with ``ISAACLAB_FORCE_EXIT_TIMEOUT=3`` and an ``app.close()`` monkeypatched to sleep 30s, the process exits at ~3s after close starts, returncode=0, log shows ``ISAACLAB_FORCE_EXIT_TIMEOUT=3s expired during atexit_close; os._exit(0)``. --- .../jichuanh-applauncher-sighup-handler.rst | 19 ++++- source/isaaclab/isaaclab/app/app_launcher.py | 70 +++++++++++++++++-- 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst index a5b8d4cd3c90..62097a2c41ee 100644 --- a/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst +++ b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst @@ -6,9 +6,24 @@ Fixed exits (e.g. parent shell supervising sibling shards in multi-GPU CI). Previously SIGHUP terminated the process with default disposition, bypassing :meth:`SimulationApp.close` and leaving USD/PhysX state - attached for the next sibling shard ("Stage X already attached" - log line and downstream shutdown hangs). + attached for the next sibling shard ("Stage X already attached" log + line and downstream shutdown hangs). * Made :meth:`~isaaclab.app.AppLauncher._abort_signal_handle_callback` exit the process after closing the app. The previous implementation swallowed the signal's terminate semantics, allowing Python to resume past a SIGTERM/SIGABRT/SIGSEGV and leaving Kit half-torn-down. + +Added +^^^^^ + +* Added ``ISAACLAB_FORCE_EXIT_TIMEOUT`` env var (integer seconds) for + :class:`~isaaclab.app.AppLauncher`. When set, arms a daemon thread that + calls ``os._exit(0)`` after the deadline, from both the ``atexit`` hook + and the abort signal handler. Used by CI to bound the upstream Kit + shutdown hang at + ``shutdown_and_release_framework`` (see + https://github.com/isaac-sim/IsaacLab/issues/3475). The hang sits + inside Kit binary code; ``skip_cleanup=True`` enters the same code + path and Python signal handlers cannot interrupt it, so ``os._exit`` + is the only python-side escape. Off by default — interactive user + code still gets the graceful teardown. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 039852420708..0fb63b1efd72 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -39,6 +39,45 @@ logging.getLogger("h5py").setLevel(logging.WARNING) +def _parse_force_exit_seconds(value: str) -> int: + """Parse the ``ISAACLAB_FORCE_EXIT_TIMEOUT`` env var. + + Returns 0 (no timer) for unset/empty/falsy values, otherwise a positive + integer of seconds. Invalid integers also return 0 with a warning so a + typo cannot accidentally arm a force-exit. + """ + raw = (value or "").strip() + if not raw or raw.lower() in {"0", "false", "no", "off"}: + return 0 + try: + n = int(raw) + except ValueError: + logger.warning("ISAACLAB_FORCE_EXIT_TIMEOUT=%r is not an integer; ignoring", raw) + return 0 + return max(0, n) + + +def _arm_force_exit_timer(seconds: int, label: str) -> None: + """Spawn a daemon thread that calls ``os._exit(0)`` after ``seconds``. + + Bounds the Kit ``shutdown_and_release_framework`` hang in CI environments + (https://github.com/isaac-sim/IsaacLab/issues/3475). The hang sits in + Kit binary code; the only python-side escape is ``os._exit``, which + bypasses any threads still holding the GIL. + """ + import threading + + def _timeout_kill() -> None: + import time as _time + + _time.sleep(seconds) + sys.stderr.write(f"[isaaclab.app] ISAACLAB_FORCE_EXIT_TIMEOUT={seconds}s expired during {label}; os._exit(0)\n") + sys.stderr.flush() + os._exit(0) + + threading.Thread(target=_timeout_kill, name="isaaclab-force-exit", daemon=True).start() + + class ExplicitAction(argparse.Action): """Custom action to track if an argument was explicitly passed by the user.""" @@ -344,7 +383,22 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa # Ensure SimulationApp.close() is called on normal process exit so Kit # shuts down cleanly instead of relying on __del__ (which logs a warning # and can leave GPU resources in a bad state for the next test). - def _atexit_close(app=self._app): + # + # ``ISAACLAB_FORCE_EXIT_TIMEOUT`` (env var, integer seconds) arms a + # daemon thread that calls ``os._exit(0)`` after the deadline. Used by + # CI to bound the Kit shutdown hang + # (https://github.com/isaac-sim/IsaacLab/issues/3475). The hang sits + # inside ``shutdown_and_release_framework`` (Kit binary path), which + # ``skip_cleanup=True`` also enters and Python signal handlers cannot + # interrupt — only ``os._exit`` actually returns control. Unset/empty/0 + # means "wait indefinitely", the current upstream behavior and the + # right default for interactive user code that wants graceful teardown. + force_exit_seconds = _parse_force_exit_seconds(os.environ.get("ISAACLAB_FORCE_EXIT_TIMEOUT", "")) + self._force_exit_seconds = force_exit_seconds + + def _atexit_close(app=self._app, force_exit_seconds=force_exit_seconds): + if force_exit_seconds > 0: + _arm_force_exit_timer(force_exit_seconds, "atexit_close") with contextlib.suppress(Exception): app.close() @@ -1386,12 +1440,16 @@ def _hide_play_button(self, flag): def _abort_signal_handle_callback(self, signum, frame): """Handle the abort/segmentation/kill/hangup signals. - Closes :class:`SimulationApp` so Kit detaches USD/PhysX state, then - exits with ``128 + signum`` to preserve the conventional signal-exit - encoding. Without the explicit exit, Python would resume execution - after the handler returns (since we replaced the OS-default - disposition), and Kit would be left half-torn-down. + Closes :class:`SimulationApp` (honoring ``ISAACLAB_FORCE_EXIT_TIMEOUT`` + if set) so Kit detaches USD/PhysX state, then exits with + ``128 + signum`` to preserve the conventional signal-exit encoding. + Without the explicit exit, Python would resume execution after the + handler returns (since we replaced the OS-default disposition), and + Kit would be left half-torn-down. """ + force_exit_seconds = getattr(self, "_force_exit_seconds", 0) + if force_exit_seconds > 0: + _arm_force_exit_timer(force_exit_seconds, f"abort_signal_{signum}") with contextlib.suppress(Exception): self._app.close() sys.exit(128 + signum) From 5a169abc8c65371e6a56d7e2ea6cb83e20b261f9 Mon Sep 17 00:00:00 2001 From: jichuanh Date: Wed, 3 Jun 2026 00:42:08 +0000 Subject: [PATCH 3/3] Use ctypes libc.kill instead of os._exit for force-exit timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Kit shutdown hang is a GIL deadlock (upstream NVBug 5948099 / OMPE-75416, confirmed by deep-research workflow against current Isaac Sim develop): the C++ teardown frames hold the Python GIL while joining ``carb.tasking`` worker threads that themselves wait for the GIL. Under that worst case, ``os._exit(0)`` from a daemon thread is NOT sound: the daemon needs the bytecode dispatcher (which needs the GIL) to even reach the ``os._exit`` line. The thread blocks on ``PyEval_AcquireThread`` and the timer never fires. Switch to ``ctypes.CDLL("libc.so.6").kill(os.getpid(), 9)``. ``ctypes`` releases the GIL around C calls by default, so the raw ``kill(2)`` syscall fires even when the interpreter is GIL-blocked. Smoke test on a synthetic busy loop: daemon armed for 3s, process exits with returncode 137 (SIGKILL = 128+9) at exactly t+3s. The daemon thread parks in ``time.sleep`` (also a GIL-released wait) BEFORE the close() call starts, so it is already in the GIL-released state when the deadlock begins. Documents the limitation in the docstring and changelog. Caveat: SIGKILL means no atexit / no JUnit cleanup — callers must have written their JUnit XML before triggering close(). For absolutely guaranteed bounding under all hang shapes, pair with an external wrapper-script watchdog at the CI runner level. --- .../jichuanh-applauncher-sighup-handler.rst | 19 +++++----- source/isaaclab/isaaclab/app/app_launcher.py | 36 +++++++++++++++---- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst index 62097a2c41ee..87e2068f56a4 100644 --- a/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst +++ b/source/isaaclab/changelog.d/jichuanh-applauncher-sighup-handler.rst @@ -18,12 +18,15 @@ Added * Added ``ISAACLAB_FORCE_EXIT_TIMEOUT`` env var (integer seconds) for :class:`~isaaclab.app.AppLauncher`. When set, arms a daemon thread that - calls ``os._exit(0)`` after the deadline, from both the ``atexit`` hook + fires SIGKILL on the current process via a raw libc ``kill(2)`` syscall + through ``ctypes`` after the deadline, from both the ``atexit`` hook and the abort signal handler. Used by CI to bound the upstream Kit - shutdown hang at - ``shutdown_and_release_framework`` (see - https://github.com/isaac-sim/IsaacLab/issues/3475). The hang sits - inside Kit binary code; ``skip_cleanup=True`` enters the same code - path and Python signal handlers cannot interrupt it, so ``os._exit`` - is the only python-side escape. Off by default — interactive user - code still gets the graceful teardown. + shutdown hang inside ``quickReleaseFrameworkAndTerminate`` (see + https://github.com/isaac-sim/IsaacLab/issues/3475, NVBug 5948099 / + OMPE-75416). The hang is a GIL deadlock — Python's bytecode dispatcher + cannot run while Kit's teardown C++ frames hold the GIL — so + ``os._exit`` (which needs the dispatcher) cannot reliably fire from a + daemon thread; the ctypes-issued libc syscall releases the GIL across + the C call and is the only python-side approach that fires under the + worst case. Off by default — interactive user code still gets the + graceful teardown. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 0fb63b1efd72..e4bfde005370 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -60,20 +60,44 @@ def _parse_force_exit_seconds(value: str) -> int: def _arm_force_exit_timer(seconds: int, label: str) -> None: """Spawn a daemon thread that calls ``os._exit(0)`` after ``seconds``. - Bounds the Kit ``shutdown_and_release_framework`` hang in CI environments - (https://github.com/isaac-sim/IsaacLab/issues/3475). The hang sits in - Kit binary code; the only python-side escape is ``os._exit``, which - bypasses any threads still holding the GIL. + Bounds the Kit ``quickReleaseFrameworkAndTerminate`` hang in CI + environments (https://github.com/isaac-sim/IsaacLab/issues/3475 / NVBug + 5948099 / OMPE-75416). + + Uses ``ctypes.CDLL("libc.so.6").kill(os.getpid(), SIGKILL)`` rather than + ``os._exit(0)`` because the hang in question is a GIL deadlock: Kit's + teardown C++ frames hold the GIL while joining ``carb.tasking`` worker + threads that themselves wait for the GIL. ``os._exit`` is a Python-level + syscall wrapper that requires the bytecode dispatcher (which needs the + GIL); if the daemon thread cannot acquire the GIL, the wake never runs. + ``ctypes`` releases the GIL around C calls by default, so the raw libc + ``kill(2)`` syscall fires even when the interpreter is GIL-blocked. + + The daemon thread is created (and parks on ``time.sleep``, which releases + the GIL) BEFORE the close() call so it is already in the GIL-released + sleep state when the deadlock starts. SIGKILL ends the process; no + atexit, no JUnit cleanup runs — callers must have already written their + JUnit XML before triggering close(). """ + import ctypes import threading def _timeout_kill() -> None: import time as _time _time.sleep(seconds) - sys.stderr.write(f"[isaaclab.app] ISAACLAB_FORCE_EXIT_TIMEOUT={seconds}s expired during {label}; os._exit(0)\n") + sys.stderr.write( + f"[isaaclab.app] ISAACLAB_FORCE_EXIT_TIMEOUT={seconds}s expired during {label}; libc.kill(self, SIGKILL)\n" + ) sys.stderr.flush() - os._exit(0) + try: + libc = ctypes.CDLL("libc.so.6", use_errno=True) + # SIGKILL is 9 on Linux. Raw syscall — does not require the GIL + # to be dispatchable through the interpreter. + libc.kill(os.getpid(), 9) + except Exception: + # Last-resort fallback: only works when GIL is reachable. + os._exit(0) threading.Thread(target=_timeout_kill, name="isaaclab-force-exit", daemon=True).start()