From 3c1f41de54635bb6289a9f7fcc9edac30e8e1c13 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:23:54 +0200 Subject: [PATCH 01/17] Register device_split pytest marker --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 117311ef207c..51abdcda6479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5; see scripts/run_ovphysx.sh)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. From 0b18415e3fd2124e959f898d969c89be4b0d7447 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:35:33 +0200 Subject: [PATCH 02/17] Add is_device_split_file predicate with unit tests --- tools/_device_split.py | 63 ++++++++++++++++++++++++ tools/test_device_split.py | 99 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 tools/_device_split.py create mode 100644 tools/test_device_split.py diff --git a/tools/_device_split.py b/tools/_device_split.py new file mode 100644 index 000000000000..acc6bf4b1ae9 --- /dev/null +++ b/tools/_device_split.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for detecting and driving the ``device_split`` pytest marker. + +Test files that declare ``pytestmark = pytest.mark.device_split`` at module +scope must be re-invoked once per device (CPU and GPU) in separate processes +to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. +The :func:`is_device_split_file` predicate lets the per-file CI runner in +``tools/conftest.py`` detect this without importing the test module. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) +"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. + +Recognises both single-mark and single-line list forms: + +* ``pytestmark = pytest.mark.device_split`` +* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` + +Multi-line list forms are not supported (currently no test file uses one); if +a future test needs that, expand the parsing rule alongside the marker +contract documented in ``scripts/run_ovphysx.sh``. +""" + +# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: +# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. +# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps +# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized +# tests only. +DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ + ("-cpu", "cpu or not cuda"), + ("-cuda", "cuda"), +] + + +def is_device_split_file(path: Path | str) -> bool: + """Return whether the test file at ``path`` declares the ``device_split`` marker. + + Reads the file source once and matches :data:`_DEVICE_SPLIT_MARK_RE`. A + missing or unreadable file returns ``False`` so the caller falls back to + the default single-pass invocation. + + Args: + path: Filesystem path to a candidate test file. + + Returns: + ``True`` when the file's module-level ``pytestmark`` mentions + ``device_split``; ``False`` otherwise. + """ + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/test_device_split.py b/tools/test_device_split.py new file mode 100644 index 000000000000..d16f1de55d67 --- /dev/null +++ b/tools/test_device_split.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for ``tools/_device_split.py``.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from _device_split import is_device_split_file + + +def _write(tmp_path: Path, content: str) -> Path: + p = tmp_path / "test_foo.py" + p.write_text(textwrap.dedent(content)) + return p + + +def test_single_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.device_split + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_list_form_single_line(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = [pytest.mark.device_split, pytest.mark.foo] + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_no_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_word_in_comment_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + # This file mentions device_split in a comment but is not marked. + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_unrelated_pytestmark_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.skipif(False, reason="x") + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_missing_file(tmp_path): + # A path that does not exist must not raise; treat as not-marked. + assert is_device_split_file(tmp_path / "does_not_exist.py") is False From d185c50356b764d891b0aee66bf2682f6dfa03d0 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:41:03 +0200 Subject: [PATCH 03/17] Drop dead scripts/run_ovphysx.sh references from device_split docs --- pyproject.toml | 2 +- tools/_device_split.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 51abdcda6479..a64fc84af215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", - "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5; see scripts/run_ovphysx.sh)", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/tools/_device_split.py b/tools/_device_split.py index acc6bf4b1ae9..1c802eca489c 100644 --- a/tools/_device_split.py +++ b/tools/_device_split.py @@ -26,8 +26,7 @@ * ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` Multi-line list forms are not supported (currently no test file uses one); if -a future test needs that, expand the parsing rule alongside the marker -contract documented in ``scripts/run_ovphysx.sh``. +a future test needs that, expand the parsing rule. """ # Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file From f0a12eb7703e20a48fdd0932af05daf3c019c618 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 10:47:24 +0200 Subject: [PATCH 04/17] Factor per-file pytest invocation in conftest into _run_one_pass helper Pure refactor; behavior unchanged. Enables the follow-up commit to invoke the helper twice per file for the device_split marker without duplicating the ~200-line retry/timeout/parse pipeline. --- tools/conftest.py | 516 ++++++++++++++++++++++++++-------------------- 1 file changed, 294 insertions(+), 222 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..7c55b39fec7a 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -10,6 +10,7 @@ import subprocess import sys import time +from dataclasses import dataclass import pytest from junitparser import Error, JUnitXml, TestCase, TestSuite @@ -312,6 +313,7 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) + def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -400,131 +402,145 @@ def _retry_failed_test_in_fresh_process( ) -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False +@dataclass +class _PassContext: + """Inputs shared across all pytest invocations for a single test file. + + Attributes: + test_file: Absolute path to the test file being driven. + file_name: Basename of ``test_file`` (used for JUnit naming). + workspace_root: Repository root; passed to pytest's ``--config-file``. + isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the + ``-m isaacsim_ci`` selector. + timeout: Per-pass hard timeout in seconds. + startup_deadline: Per-pass startup-hang deadline in seconds. + env: Environment passed to the pytest subprocess. + is_cold_cache_test: Used by callers to log the cold-cache buffer once. + """ - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" + test_file: str + file_name: str + workspace_root: str + isaacsim_ci: bool + timeout: int + startup_deadline: int + env: dict + is_cold_cache_test: bool - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - # Read the test file once for cold-cache check. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" +def _run_one_pass( + ctx: _PassContext, + k_expr: str | None, + suffix: str, +) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``ctx.test_file`` and return its results. - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") - - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", - "--tb=short", - ] - - if isaacsim_ci: - cmd.append("-m") - cmd.append("isaacsim_ci") - - cmd.append(str(test_file)) - - report_file = f"tests/test-reports-{str(file_name)}.xml" - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = ( - capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - ) + Args: + ctx: Static per-file context (paths, timeouts, env). + k_expr: Optional ``-k`` selector. ``None`` means no selector (default + single-pass invocation). + suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` + or ``""`` for the unsplit default. - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {test_file}: startup hang detected after {startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced + no report (e.g. startup hang). + * ``status_dict``: per-pass counters compatible with the entries + currently appended to ``test_status``. + * ``was_failure``: whether the pass should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + pass_file_label = f"{ctx.file_name}{suffix}" + report_file = f"tests/test-reports-{pass_file_label}.xml" + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "--no-header", + f"--config-file={ctx.workspace_root}/pyproject.toml", + f"--junitxml={report_file}", + "--tb=short", + ] + if ctx.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + if k_expr is not None: + cmd += ["-k", k_expr] + cmd.append(str(ctx.test_file)) + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {test_file}: timeout detected after {timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) - # -- Resolve result from kill_reason and report file ---------------- has_report = os.path.exists(report_file) - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {test_file}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" print(diag) + continue - msg = f"Startup hang after {startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + + # -- Resolve result from kill_reason and report file ---------------- + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + print(diag) + + msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -532,61 +548,63 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "STARTUP_HANG", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {test_file} timed out after {timeout} seconds...") - print(diag) - - msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + }, + True, + ) - error_report = _create_error_report("timeout", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") + print(diag) + + msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, "tests": 1, "result": "TIMEOUT", - "time_elapsed": timeout, + "time_elapsed": ctx.timeout, "wall_time": wall_time, - } - continue - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {test_file}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + }, + True, + ) - error_report = _create_error_report("crash", file_name, reason, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {ctx.test_file}{suffix}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", pass_file_label, reason, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -594,19 +612,21 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "CRASHED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue + }, + True, + ) - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {test_file}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - failed_tests.append(test_file) - test_status[test_file] = { + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return ( + None, + { "errors": 1, "failures": 0, "skipped": 0, @@ -614,60 +634,59 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "FAILED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - has_test_failures = errors > 0 or failures > 0 - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=test_file, - file_name=file_name, - cmd=cmd, - timeout=timeout, - env=env, - startup_deadline=startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, + }, + True, ) - xml_reports.append(report) - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) = _retry_failed_test_in_fresh_process( + test_file=ctx.test_file, + file_name=ctx.file_name, + cmd=cmd, + timeout=ctx.timeout, + env=ctx.env, + startup_deadline=ctx.startup_deadline, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout_data=stdout_data, + stderr_data=stderr_data, + kill_reason=kill_reason, + wall_time=wall_time, + pre_kill_diag=pre_kill_diag, + ) - if has_test_failures or (returncode != 0 and not shutdown_hanged): - failed_tests.append(test_file) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" - test_status[test_file] = { + return ( + report, + { "errors": errors, "failures": failures, "skipped": skipped, @@ -675,13 +694,66 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": result, "time_elapsed": time_elapsed, "wall_time": wall_time, - } + }, + was_failure, + ) + + +def run_individual_tests(test_files, workspace_root, isaacsim_ci): + """Run each test file separately, ensuring one finishes before starting the next.""" + failed_tests = [] + test_status = {} + xml_reports = [] + cold_cache_applied = False + + for test_file in test_files: + print(f"\n\n🚀 Running {test_file} independently...\n") + file_name = os.path.basename(test_file) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + + # Read the test file once for cold-cache and device-split detection. + try: + with open(test_file) as fh: + test_content = fh.read() + except OSError: + test_content = "" + + # The first camera-enabled test in a fresh container compiles shaders + # (~600 s). Give it extra time so that doesn't look like a test timeout. + is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + if is_cold_cache_test: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + + extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 + startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + + ctx = _PassContext( + test_file=test_file, + file_name=file_name, + workspace_root=workspace_root, + isaacsim_ci=isaacsim_ci, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + is_cold_cache_test=is_cold_cache_test, + ) + + report, status, was_failure = _run_one_pass(ctx, k_expr=None, suffix="") + if report is not None: + xml_reports.append(report) + if was_failure: + failed_tests.append(test_file) + test_status[test_file] = status print("~~~~~~~~~~~~ Finished running all tests") return failed_tests, test_status, xml_reports - def _collect_test_files( source_dirs, filter_pattern, From 37ee8e16249aa270bb2eb16d16188b7c1ca6f0e1 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:32:01 +0200 Subject: [PATCH 05/17] Run device_split-marked test files twice in conftest (CPU then GPU) Resolves the ovphysx<=0.3.7 gap G5 device lock for any CI test file that opts in via 'pytestmark = pytest.mark.device_split'. Each pass gets its own subprocess, its own JUnit XML, and the merged counts are written under the original file key so the summary table is unchanged. --- tools/conftest.py | 56 ++++++++++++++++++++++++++++++++++---- tools/test_device_split.py | 2 -- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 7c55b39fec7a..77700500b20b 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -18,6 +18,7 @@ # Local imports import test_settings as test_settings # isort: skip +from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip def pytest_ignore_collect(collection_path, config): @@ -428,6 +429,38 @@ class _PassContext: is_cold_cache_test: bool +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} + + +def _merge_pass_status(prev: dict | None, new: dict) -> dict: + """Merge per-pass status dicts into a single per-file entry. + + Counters (``errors``, ``failures``, ``skipped``, ``tests``, + ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more + severe of the two via :data:`_RESULT_PRIORITY`. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + def _run_one_pass( ctx: _PassContext, k_expr: str | None, @@ -743,12 +776,23 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): is_cold_cache_test=is_cold_cache_test, ) - report, status, was_failure = _run_one_pass(ctx, k_expr=None, suffix="") - if report is not None: - xml_reports.append(report) - if was_failure: - failed_tests.append(test_file) - test_status[test_file] = status + if is_device_split_file(test_file): + print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") + passes = DEVICE_SPLIT_PASSES + else: + passes = [("", None)] + + merged_status: dict | None = None + for suffix, k_expr in passes: + report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + if report is not None: + xml_reports.append(report) + if was_failure: + failed_tests.append(test_file) + merged_status = _merge_pass_status(merged_status, status) + + assert merged_status is not None # the pass list is never empty + test_status[test_file] = merged_status print("~~~~~~~~~~~~ Finished running all tests") diff --git a/tools/test_device_split.py b/tools/test_device_split.py index d16f1de55d67..c3f7ea67a72e 100644 --- a/tools/test_device_split.py +++ b/tools/test_device_split.py @@ -10,8 +10,6 @@ import textwrap from pathlib import Path -import pytest - from _device_split import is_device_split_file From 55b3f7327c03030cf95695460a3315546692d6cf Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:36:45 +0200 Subject: [PATCH 06/17] Mark test_views_xform_prim_ovphysx as device_split Avoids the ovphysx<=0.3.7 gap G5 device-lock failure surfaced by PR 5780 in the isaaclab_ov CI job. The conftest device_split runner splits this file into separate CPU and GPU subprocesses. --- .../isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index bebd8aaa460d..797b311d5515 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -25,6 +25,8 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) +pytestmark = pytest.mark.device_split + @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) def test_factory_dispatches_to_ovphysx_frame_view(device): From 40fc8d5b3c7300709dc7e9414eaf3ee365047aef Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:39:13 +0200 Subject: [PATCH 07/17] Mark test_contact_sensor (ovphysx) as device_split Avoids the ovphysx<=0.3.7 gap G5 device-lock failure on the test_no_contact_reporting case that runs after GPU-parametrized cases in the isaaclab_ov CI job. --- source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index 38b3bd9e579d..abf37ab06cb9 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -64,6 +64,8 @@ wp.init() +pytestmark = pytest.mark.device_split + # --------------------------------------------------------------------------- # Device-lock autouse fixture # --------------------------------------------------------------------------- From 48d5682d280bf4d8be3bc868a5571cc248bd334d Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 13:41:12 +0200 Subject: [PATCH 08/17] Add changelog fragment for ovphysx device_split CI fix --- .../changelog.d/antoiner-ovphysx-device-split-ci.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst new file mode 100644 index 000000000000..8f8b515af6e9 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Re-enabled both CPU and GPU coverage in CI for + :file:`test/sim/test_views_xform_prim_ovphysx.py` and + :file:`test/sensors/test_contact_sensor.py` by tagging them with the new + ``device_split`` pytest marker, which causes the CI driver to invoke each + file once per device in separate subprocesses. Works around the + ``ovphysx<=0.3.7`` process-global device lock (gap G5). From 349608c14ad3d2170ba7cc77c635bd45995de0df Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Wed, 27 May 2026 17:31:27 +0200 Subject: [PATCH 09/17] Drop dead is_cold_cache_test field; dedup failed_tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tiny cleanups on the device_split CI machinery added in this PR: * Remove _PassContext.is_cold_cache_test — set by the caller but never read by _run_one_pass; the cold-cache buffer is applied to timeout and startup_deadline before the dataclass is built, so the field was purely informational. * Add a presence check before appending to failed_tests so a device_split file failing on both CPU and GPU passes only appears once in the 'failed tests:' debug print (exit code is unaffected; it derives from test_status keys). --- tools/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 77700500b20b..7cbff087384e 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -416,7 +416,6 @@ class _PassContext: timeout: Per-pass hard timeout in seconds. startup_deadline: Per-pass startup-hang deadline in seconds. env: Environment passed to the pytest subprocess. - is_cold_cache_test: Used by callers to log the cold-cache buffer once. """ test_file: str @@ -426,7 +425,6 @@ class _PassContext: timeout: int startup_deadline: int env: dict - is_cold_cache_test: bool _RESULT_PRIORITY = { @@ -773,7 +771,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): timeout=timeout, startup_deadline=startup_deadline, env=env, - is_cold_cache_test=is_cold_cache_test, ) if is_device_split_file(test_file): @@ -787,7 +784,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) if report is not None: xml_reports.append(report) - if was_failure: + if was_failure and test_file not in failed_tests: failed_tests.append(test_file) merged_status = _merge_pass_status(merged_status, status) From ec6f667ceea71521853c64696720ef760a88f90e Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Mon, 1 Jun 2026 10:44:40 +0200 Subject: [PATCH 10/17] Keep ovphysx optional in installs Install ovphysx explicitly in CI jobs that exercise OV coverage instead of adding it to the default user install path. Restore optional OVPhysX test guards so local and partial installs skip when the runtime wheel is absent. --- .github/workflows/build.yaml | 1 + source/isaaclab_ovphysx/test/assets/test_articulation.py | 5 ++--- .../test/assets/test_articulation_helpers.py | 5 ++--- source/isaaclab_ovphysx/test/assets/test_rigid_object.py | 5 ++--- .../test/assets/test_rigid_object_collection.py | 5 ++--- .../test/assets/test_rigid_object_helpers.py | 5 ++--- .../test/physics/test_ovphysx_scene_data_backend.py | 5 ++--- source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py | 5 ++--- .../test/sim/test_views_xform_prim_ovphysx.py | 5 ++--- 9 files changed, 17 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e09ec651fe3f..eff91c7c61c7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -543,6 +543,7 @@ jobs: isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} filter-pattern: "isaaclab_ov" + extra-pip-packages: "ovrtx ovphysx" container-name: isaac-lab-ov-test # Folded from the former standalone verify-base-non-root job: reuses the diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index e4b052065127..ba930ed1ad3b 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,9 +56,8 @@ import torch import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx import tensor_types as TT # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py index db0a2cc89403..da85e53facbc 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py @@ -19,9 +19,8 @@ from pxr import Sdf, Usd, UsdPhysics -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets.articulation.articulation import Articulation # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 407cf4b41e22..ae54bc51b9c7 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,9 +32,8 @@ import warp as wp from flaky import flaky -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObject # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 07ec860d6ec6..d23086ae2867 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,9 +28,8 @@ import torch import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObjectCollection # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py index e57d8d2651d7..a8162765a79b 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py @@ -15,9 +15,8 @@ import pytest import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx import tensor_types as TT # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py index 013ef8bdc092..37487b5dd85e 100644 --- a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py +++ b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py @@ -11,9 +11,8 @@ import pytest -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index abf37ab06cb9..ccec0c7eb28d 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -45,9 +45,8 @@ import warp as wp from flaky import flaky -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObject # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index 797b311d5515..e606ce7738b9 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -12,9 +12,8 @@ import pytest -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 From a47511b465a8ba4be1af1142c9ead51613e1a90b Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Mon, 1 Jun 2026 14:09:38 +0200 Subject: [PATCH 11/17] Format test runner conftest --- tools/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/conftest.py b/tools/conftest.py index 7cbff087384e..eea9f2e427e0 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -314,7 +314,6 @@ def _capture_system_diagnostics(): return "\n\n".join(sections) - def _read_test_report(report_file, file_name): """Read a pytest JUnit report and return its summary fields.""" report = JUnitXml.fromfile(report_file) @@ -795,6 +794,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): return failed_tests, test_status, xml_reports + def _collect_test_files( source_dirs, filter_pattern, From 98bfa090b698b09d576e833fa49925b1d1fdf260 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Mon, 1 Jun 2026 14:17:50 +0200 Subject: [PATCH 12/17] Revert broad device split runner changes --- pyproject.toml | 1 - .../antoiner-ovphysx-device-split-ci.rst | 9 - .../test/sensors/test_contact_sensor.py | 2 - .../test/sim/test_views_xform_prim_ovphysx.py | 2 - tools/_device_split.py | 62 -- tools/conftest.py | 551 +++++++----------- tools/test_device_split.py | 97 --- 7 files changed, 219 insertions(+), 505 deletions(-) delete mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst delete mode 100644 tools/_device_split.py delete mode 100644 tools/test_device_split.py diff --git a/pyproject.toml b/pyproject.toml index a64fc84af215..117311ef207c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,6 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", - "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst deleted file mode 100644 index 8f8b515af6e9..000000000000 --- a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst +++ /dev/null @@ -1,9 +0,0 @@ -Fixed -^^^^^ - -* Re-enabled both CPU and GPU coverage in CI for - :file:`test/sim/test_views_xform_prim_ovphysx.py` and - :file:`test/sensors/test_contact_sensor.py` by tagging them with the new - ``device_split`` pytest marker, which causes the CI driver to invoke each - file once per device in separate subprocesses. Works around the - ``ovphysx<=0.3.7`` process-global device lock (gap G5). diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index ccec0c7eb28d..e7d9f509720b 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -63,8 +63,6 @@ wp.init() -pytestmark = pytest.mark.device_split - # --------------------------------------------------------------------------- # Device-lock autouse fixture # --------------------------------------------------------------------------- diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index e606ce7738b9..927a01cc979c 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -24,8 +24,6 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) -pytestmark = pytest.mark.device_split - @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) def test_factory_dispatches_to_ovphysx_frame_view(device): diff --git a/tools/_device_split.py b/tools/_device_split.py deleted file mode 100644 index 1c802eca489c..000000000000 --- a/tools/_device_split.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helpers for detecting and driving the ``device_split`` pytest marker. - -Test files that declare ``pytestmark = pytest.mark.device_split`` at module -scope must be re-invoked once per device (CPU and GPU) in separate processes -to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. -The :func:`is_device_split_file` predicate lets the per-file CI runner in -``tools/conftest.py`` detect this without importing the test module. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) -"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. - -Recognises both single-mark and single-line list forms: - -* ``pytestmark = pytest.mark.device_split`` -* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` - -Multi-line list forms are not supported (currently no test file uses one); if -a future test needs that, expand the parsing rule. -""" - -# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file -# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: -# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. -# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps -# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized -# tests only. -DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ - ("-cpu", "cpu or not cuda"), - ("-cuda", "cuda"), -] - - -def is_device_split_file(path: Path | str) -> bool: - """Return whether the test file at ``path`` declares the ``device_split`` marker. - - Reads the file source once and matches :data:`_DEVICE_SPLIT_MARK_RE`. A - missing or unreadable file returns ``False`` so the caller falls back to - the default single-pass invocation. - - Args: - path: Filesystem path to a candidate test file. - - Returns: - ``True`` when the file's module-level ``pytestmark`` mentions - ``device_split``; ``False`` otherwise. - """ - try: - source = Path(path).read_text(encoding="utf-8", errors="replace") - except OSError: - return False - return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index eea9f2e427e0..9187099ee92a 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -10,7 +10,6 @@ import subprocess import sys import time -from dataclasses import dataclass import pytest from junitparser import Error, JUnitXml, TestCase, TestSuite @@ -18,7 +17,6 @@ # Local imports import test_settings as test_settings # isort: skip -from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip def pytest_ignore_collect(collection_path, config): @@ -402,175 +400,131 @@ def _retry_failed_test_in_fresh_process( ) -@dataclass -class _PassContext: - """Inputs shared across all pytest invocations for a single test file. +def run_individual_tests(test_files, workspace_root, isaacsim_ci): + """Run each test file separately, ensuring one finishes before starting the next.""" + failed_tests = [] + test_status = {} + xml_reports = [] + cold_cache_applied = False - Attributes: - test_file: Absolute path to the test file being driven. - file_name: Basename of ``test_file`` (used for JUnit naming). - workspace_root: Repository root; passed to pytest's ``--config-file``. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. - timeout: Per-pass hard timeout in seconds. - startup_deadline: Per-pass startup-hang deadline in seconds. - env: Environment passed to the pytest subprocess. - """ + for test_file in test_files: + print(f"\n\n🚀 Running {test_file} independently...\n") + file_name = os.path.basename(test_file) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" - test_file: str - file_name: str - workspace_root: str - isaacsim_ci: bool - timeout: int - startup_deadline: int - env: dict - - -_RESULT_PRIORITY = { - "STARTUP_HANG": 5, - "CRASHED": 4, - "TIMEOUT": 3, - "FAILED": 2, - "passed (shutdown hanged)": 1, - "passed": 0, -} + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + # Read the test file once for cold-cache check. + try: + with open(test_file) as fh: + test_content = fh.read() + except OSError: + test_content = "" -def _merge_pass_status(prev: dict | None, new: dict) -> dict: - """Merge per-pass status dicts into a single per-file entry. + # The first camera-enabled test in a fresh container compiles shaders + # (~600 s). Give it extra time so that doesn't look like a test timeout. + is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + if is_cold_cache_test: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") - Counters (``errors``, ``failures``, ``skipped``, ``tests``, - ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more - severe of the two via :data:`_RESULT_PRIORITY`. - """ - if prev is None: - return new - return { - "errors": prev["errors"] + new["errors"], - "failures": prev["failures"] + new["failures"], - "skipped": prev["skipped"] + new["skipped"], - "tests": prev["tests"] + new["tests"], - "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], - "wall_time": prev["wall_time"] + new["wall_time"], - "result": prev["result"] - if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) - else new["result"], - } - - -def _run_one_pass( - ctx: _PassContext, - k_expr: str | None, - suffix: str, -) -> tuple[JUnitXml | None, dict, bool]: - """Drive one pytest subprocess for ``ctx.test_file`` and return its results. + extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 + startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - Args: - ctx: Static per-file context (paths, timeouts, env). - k_expr: Optional ``-k`` selector. ``None`` means no selector (default - single-pass invocation). - suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` - or ``""`` for the unsplit default. + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "--no-header", + f"--config-file={workspace_root}/pyproject.toml", + f"--junitxml=tests/test-reports-{str(file_name)}.xml", + "--tb=short", + ] + + if isaacsim_ci: + cmd.append("-m") + cmd.append("isaacsim_ci") + + cmd.append(str(test_file)) + + report_file = f"tests/test-reports-{str(file_name)}.xml" + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = ( + capture_test_output_with_timeout( + cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file + ) + ) - Returns: - A 3-tuple ``(xml_report, status_dict, was_failure)``: - * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced - no report (e.g. startup hang). - * ``status_dict``: per-pass counters compatible with the entries - currently appended to ``test_status``. - * ``was_failure``: whether the pass should add ``ctx.test_file`` to - the ``failed_tests`` list. - """ - pass_file_label = f"{ctx.file_name}{suffix}" - report_file = f"tests/test-reports-{pass_file_label}.xml" - - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={ctx.workspace_root}/pyproject.toml", - f"--junitxml={report_file}", - "--tb=short", - ] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] - if k_expr is not None: - cmd += ["-k", k_expr] - cmd.append(str(ctx.test_file)) - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {test_file}: startup hang detected after {startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file - ) + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {test_file}: timeout detected after {timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) + if stdout_data: + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + # -- Resolve result from kill_reason and report file ---------------- has_report = os.path.exists(report_file) - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {test_file}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") print(diag) - continue - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + msg = f"Startup hang after {startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break - - # -- Resolve result from kill_reason and report file ---------------- - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") - print(diag) - - msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", file_name, msg, details) + error_report.write(report_file) + xml_reports.append(error_report) + failed_tests.append(test_file) + test_status[test_file] = { "errors": 1, "failures": 0, "skipped": 0, @@ -578,63 +532,61 @@ def _run_one_pass( "result": "STARTUP_HANG", "time_elapsed": 0.0, "wall_time": wall_time, - }, - True, - ) + } + continue + + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {test_file} timed out after {timeout} seconds...") + print(diag) - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") - print(diag) - - msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { + msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", file_name, msg, details) + error_report.write(report_file) + xml_reports.append(error_report) + failed_tests.append(test_file) + test_status[test_file] = { "errors": 1, "failures": 0, "skipped": 0, "tests": 1, "result": "TIMEOUT", - "time_elapsed": ctx.timeout, + "time_elapsed": timeout, "wall_time": wall_time, - }, - True, - ) + } + continue - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {ctx.test_file}{suffix}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", pass_file_label, reason, details) - error_report.write(report_file) - return ( - error_report, - { + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {test_file}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", file_name, reason, details) + error_report.write(report_file) + xml_reports.append(error_report) + failed_tests.append(test_file) + test_status[test_file] = { "errors": 1, "failures": 0, "skipped": 0, @@ -642,21 +594,19 @@ def _run_one_pass( "result": "CRASHED", "time_elapsed": 0.0, "wall_time": wall_time, - }, - True, - ) + } + continue - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {test_file}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - return ( - None, - { + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + failed_tests.append(test_file) + test_status[test_file] = { "errors": 1, "failures": 0, "skipped": 0, @@ -664,59 +614,60 @@ def _run_one_pass( "result": "FAILED", "time_elapsed": 0.0, "wall_time": wall_time, - }, - True, + } + continue + + has_test_failures = errors > 0 or failures > 0 + ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) = _retry_failed_test_in_fresh_process( + test_file=test_file, + file_name=file_name, + cmd=cmd, + timeout=timeout, + env=env, + startup_deadline=startup_deadline, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout_data=stdout_data, + stderr_data=stderr_data, + kill_reason=kill_reason, + wall_time=wall_time, + pre_kill_diag=pre_kill_diag, ) - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=ctx.test_file, - file_name=ctx.file_name, - cmd=cmd, - timeout=ctx.timeout, - env=ctx.env, - startup_deadline=ctx.startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, - ) + xml_reports.append(report) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures - was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) + if has_test_failures or (returncode != 0 and not shutdown_hanged): + failed_tests.append(test_file) - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" - return ( - report, - { + test_status[test_file] = { "errors": errors, "failures": failures, "skipped": skipped, @@ -724,71 +675,7 @@ def _run_one_pass( "result": result, "time_elapsed": time_elapsed, "wall_time": wall_time, - }, - was_failure, - ) - - -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False - - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" - - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - - # Read the test file once for cold-cache and device-split detection. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" - - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") - - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - - ctx = _PassContext( - test_file=test_file, - file_name=file_name, - workspace_root=workspace_root, - isaacsim_ci=isaacsim_ci, - timeout=timeout, - startup_deadline=startup_deadline, - env=env, - ) - - if is_device_split_file(test_file): - print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") - passes = DEVICE_SPLIT_PASSES - else: - passes = [("", None)] - - merged_status: dict | None = None - for suffix, k_expr in passes: - report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) - if report is not None: - xml_reports.append(report) - if was_failure and test_file not in failed_tests: - failed_tests.append(test_file) - merged_status = _merge_pass_status(merged_status, status) - - assert merged_status is not None # the pass list is never empty - test_status[test_file] = merged_status + } print("~~~~~~~~~~~~ Finished running all tests") diff --git a/tools/test_device_split.py b/tools/test_device_split.py deleted file mode 100644 index c3f7ea67a72e..000000000000 --- a/tools/test_device_split.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for ``tools/_device_split.py``.""" - -from __future__ import annotations - -import textwrap -from pathlib import Path - -from _device_split import is_device_split_file - - -def _write(tmp_path: Path, content: str) -> Path: - p = tmp_path / "test_foo.py" - p.write_text(textwrap.dedent(content)) - return p - - -def test_single_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.device_split - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_list_form_single_line(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = [pytest.mark.device_split, pytest.mark.foo] - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_no_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_word_in_comment_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - # This file mentions device_split in a comment but is not marked. - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_unrelated_pytestmark_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.skipif(False, reason="x") - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_missing_file(tmp_path): - # A path that does not exist must not raise; treat as not-marked. - assert is_device_split_file(tmp_path / "does_not_exist.py") is False From 1fce150a7f0d11de51a6e7f94373c5faa8ebe93d Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Mon, 1 Jun 2026 14:18:57 +0200 Subject: [PATCH 13/17] Add ovphysx CI changelog skip --- .../isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip new file mode 100644 index 000000000000..e69de29bb2d1 From 2dba76da2f00bfb83f5b7459591c924f9106ee2e Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Mon, 1 Jun 2026 14:24:56 +0200 Subject: [PATCH 14/17] Restore automatic ovphysx device split --- pyproject.toml | 1 + .../antoiner-ovphysx-ci-install.skip | 0 .../antoiner-ovphysx-device-split-ci.rst | 9 + .../test/sensors/test_contact_sensor.py | 2 + .../test/sim/test_views_xform_prim_ovphysx.py | 2 + tools/_device_split.py | 62 ++ tools/conftest.py | 551 +++++++++++------- tools/test_device_split.py | 97 +++ 8 files changed, 505 insertions(+), 219 deletions(-) delete mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip create mode 100644 source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst create mode 100644 tools/_device_split.py create mode 100644 tools/test_device_split.py diff --git a/pyproject.toml b/pyproject.toml index 117311ef207c..a64fc84af215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-ci-install.skip deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst new file mode 100644 index 000000000000..8f8b515af6e9 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst @@ -0,0 +1,9 @@ +Fixed +^^^^^ + +* Re-enabled both CPU and GPU coverage in CI for + :file:`test/sim/test_views_xform_prim_ovphysx.py` and + :file:`test/sensors/test_contact_sensor.py` by tagging them with the new + ``device_split`` pytest marker, which causes the CI driver to invoke each + file once per device in separate subprocesses. Works around the + ``ovphysx<=0.3.7`` process-global device lock (gap G5). diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index e7d9f509720b..ccec0c7eb28d 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -63,6 +63,8 @@ wp.init() +pytestmark = pytest.mark.device_split + # --------------------------------------------------------------------------- # Device-lock autouse fixture # --------------------------------------------------------------------------- diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index 927a01cc979c..e606ce7738b9 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -24,6 +24,8 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) +pytestmark = pytest.mark.device_split + @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) def test_factory_dispatches_to_ovphysx_frame_view(device): diff --git a/tools/_device_split.py b/tools/_device_split.py new file mode 100644 index 000000000000..1c802eca489c --- /dev/null +++ b/tools/_device_split.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for detecting and driving the ``device_split`` pytest marker. + +Test files that declare ``pytestmark = pytest.mark.device_split`` at module +scope must be re-invoked once per device (CPU and GPU) in separate processes +to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. +The :func:`is_device_split_file` predicate lets the per-file CI runner in +``tools/conftest.py`` detect this without importing the test module. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) +"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. + +Recognises both single-mark and single-line list forms: + +* ``pytestmark = pytest.mark.device_split`` +* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` + +Multi-line list forms are not supported (currently no test file uses one); if +a future test needs that, expand the parsing rule. +""" + +# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: +# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. +# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps +# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized +# tests only. +DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ + ("-cpu", "cpu or not cuda"), + ("-cuda", "cuda"), +] + + +def is_device_split_file(path: Path | str) -> bool: + """Return whether the test file at ``path`` declares the ``device_split`` marker. + + Reads the file source once and matches :data:`_DEVICE_SPLIT_MARK_RE`. A + missing or unreadable file returns ``False`` so the caller falls back to + the default single-pass invocation. + + Args: + path: Filesystem path to a candidate test file. + + Returns: + ``True`` when the file's module-level ``pytestmark`` mentions + ``device_split``; ``False`` otherwise. + """ + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..eea9f2e427e0 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -10,6 +10,7 @@ import subprocess import sys import time +from dataclasses import dataclass import pytest from junitparser import Error, JUnitXml, TestCase, TestSuite @@ -17,6 +18,7 @@ # Local imports import test_settings as test_settings # isort: skip +from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip def pytest_ignore_collect(collection_path, config): @@ -400,131 +402,175 @@ def _retry_failed_test_in_fresh_process( ) -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False +@dataclass +class _PassContext: + """Inputs shared across all pytest invocations for a single test file. - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" + Attributes: + test_file: Absolute path to the test file being driven. + file_name: Basename of ``test_file`` (used for JUnit naming). + workspace_root: Repository root; passed to pytest's ``--config-file``. + isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the + ``-m isaacsim_ci`` selector. + timeout: Per-pass hard timeout in seconds. + startup_deadline: Per-pass startup-hang deadline in seconds. + env: Environment passed to the pytest subprocess. + """ - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + test_file: str + file_name: str + workspace_root: str + isaacsim_ci: bool + timeout: int + startup_deadline: int + env: dict + + +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} - # Read the test file once for cold-cache check. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") +def _merge_pass_status(prev: dict | None, new: dict) -> dict: + """Merge per-pass status dicts into a single per-file entry. - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + Counters (``errors``, ``failures``, ``skipped``, ``tests``, + ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more + severe of the two via :data:`_RESULT_PRIORITY`. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + +def _run_one_pass( + ctx: _PassContext, + k_expr: str | None, + suffix: str, +) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``ctx.test_file`` and return its results. - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", - "--tb=short", - ] - - if isaacsim_ci: - cmd.append("-m") - cmd.append("isaacsim_ci") - - cmd.append(str(test_file)) - - report_file = f"tests/test-reports-{str(file_name)}.xml" - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = ( - capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - ) + Args: + ctx: Static per-file context (paths, timeouts, env). + k_expr: Optional ``-k`` selector. ``None`` means no selector (default + single-pass invocation). + suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` + or ``""`` for the unsplit default. - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {test_file}: startup hang detected after {startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced + no report (e.g. startup hang). + * ``status_dict``: per-pass counters compatible with the entries + currently appended to ``test_status``. + * ``was_failure``: whether the pass should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + pass_file_label = f"{ctx.file_name}{suffix}" + report_file = f"tests/test-reports-{pass_file_label}.xml" + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "--no-header", + f"--config-file={ctx.workspace_root}/pyproject.toml", + f"--junitxml={report_file}", + "--tb=short", + ] + if ctx.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + if k_expr is not None: + cmd += ["-k", k_expr] + cmd.append(str(ctx.test_file)) + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {test_file}: timeout detected after {timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) - # -- Resolve result from kill_reason and report file ---------------- has_report = os.path.exists(report_file) - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {test_file}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" print(diag) + continue - msg = f"Startup hang after {startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + + # -- Resolve result from kill_reason and report file ---------------- + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + print(diag) + + msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -532,61 +578,63 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "STARTUP_HANG", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {test_file} timed out after {timeout} seconds...") - print(diag) + }, + True, + ) - msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") + print(diag) + + msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, "tests": 1, "result": "TIMEOUT", - "time_elapsed": timeout, + "time_elapsed": ctx.timeout, "wall_time": wall_time, - } - continue - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {test_file}: {reason}") - print(diag) + }, + True, + ) - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", file_name, reason, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {ctx.test_file}{suffix}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", pass_file_label, reason, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -594,19 +642,21 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "CRASHED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue + }, + True, + ) - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {test_file}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - failed_tests.append(test_file) - test_status[test_file] = { + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return ( + None, + { "errors": 1, "failures": 0, "skipped": 0, @@ -614,60 +664,59 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "FAILED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - has_test_failures = errors > 0 or failures > 0 - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=test_file, - file_name=file_name, - cmd=cmd, - timeout=timeout, - env=env, - startup_deadline=startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, + }, + True, ) - xml_reports.append(report) - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) = _retry_failed_test_in_fresh_process( + test_file=ctx.test_file, + file_name=ctx.file_name, + cmd=cmd, + timeout=ctx.timeout, + env=ctx.env, + startup_deadline=ctx.startup_deadline, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout_data=stdout_data, + stderr_data=stderr_data, + kill_reason=kill_reason, + wall_time=wall_time, + pre_kill_diag=pre_kill_diag, + ) - if has_test_failures or (returncode != 0 and not shutdown_hanged): - failed_tests.append(test_file) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" - test_status[test_file] = { + return ( + report, + { "errors": errors, "failures": failures, "skipped": skipped, @@ -675,7 +724,71 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": result, "time_elapsed": time_elapsed, "wall_time": wall_time, - } + }, + was_failure, + ) + + +def run_individual_tests(test_files, workspace_root, isaacsim_ci): + """Run each test file separately, ensuring one finishes before starting the next.""" + failed_tests = [] + test_status = {} + xml_reports = [] + cold_cache_applied = False + + for test_file in test_files: + print(f"\n\n🚀 Running {test_file} independently...\n") + file_name = os.path.basename(test_file) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + + # Read the test file once for cold-cache and device-split detection. + try: + with open(test_file) as fh: + test_content = fh.read() + except OSError: + test_content = "" + + # The first camera-enabled test in a fresh container compiles shaders + # (~600 s). Give it extra time so that doesn't look like a test timeout. + is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + if is_cold_cache_test: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + + extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 + startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + + ctx = _PassContext( + test_file=test_file, + file_name=file_name, + workspace_root=workspace_root, + isaacsim_ci=isaacsim_ci, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + ) + + if is_device_split_file(test_file): + print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") + passes = DEVICE_SPLIT_PASSES + else: + passes = [("", None)] + + merged_status: dict | None = None + for suffix, k_expr in passes: + report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + if report is not None: + xml_reports.append(report) + if was_failure and test_file not in failed_tests: + failed_tests.append(test_file) + merged_status = _merge_pass_status(merged_status, status) + + assert merged_status is not None # the pass list is never empty + test_status[test_file] = merged_status print("~~~~~~~~~~~~ Finished running all tests") diff --git a/tools/test_device_split.py b/tools/test_device_split.py new file mode 100644 index 000000000000..c3f7ea67a72e --- /dev/null +++ b/tools/test_device_split.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for ``tools/_device_split.py``.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +from _device_split import is_device_split_file + + +def _write(tmp_path: Path, content: str) -> Path: + p = tmp_path / "test_foo.py" + p.write_text(textwrap.dedent(content)) + return p + + +def test_single_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.device_split + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_list_form_single_line(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = [pytest.mark.device_split, pytest.mark.foo] + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_no_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_word_in_comment_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + # This file mentions device_split in a comment but is not marked. + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_unrelated_pytestmark_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.skipif(False, reason="x") + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_missing_file(tmp_path): + # A path that does not exist must not raise; treat as not-marked. + assert is_device_split_file(tmp_path / "does_not_exist.py") is False From dbd83a4177a3978cd641399ea4d32fefd87a205c Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Tue, 2 Jun 2026 10:48:42 +0200 Subject: [PATCH 15/17] Show device split runs in test summary --- .../isaaclab_ovphysx/test/assets/test_articulation.py | 2 ++ .../isaaclab_ovphysx/test/assets/test_rigid_object.py | 2 ++ .../test/assets/test_rigid_object_collection.py | 2 ++ tools/conftest.py | 10 +++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index ba930ed1ad3b..4060da7a326a 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -82,6 +82,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _OMNI_PHYSX_SCHEMAS_GAP_REASON = ( "Schema-level fixed-joint creation in :mod:`isaaclab.sim.schemas` imports " diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index ae54bc51b9c7..a7db008f46b4 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -55,6 +55,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _logger = logging.getLogger(__name__) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index d23086ae2867..a86baab0a172 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -52,6 +52,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _LOCKED_DEVICE: list[str | None] = [None] """Device the session pins to on the first parametrized test that runs.""" diff --git a/tools/conftest.py b/tools/conftest.py index eea9f2e427e0..53c7dfb21b97 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -445,6 +445,11 @@ def _merge_pass_status(prev: dict | None, new: dict) -> dict: """ if prev is None: return new + passes = [] + for status in (prev, new): + for pass_name in status.get("passes", []): + if pass_name not in passes: + passes.append(pass_name) return { "errors": prev["errors"] + new["errors"], "failures": prev["failures"] + new["failures"], @@ -452,6 +457,7 @@ def _merge_pass_status(prev: dict | None, new: dict) -> dict: "tests": prev["tests"] + new["tests"], "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], "wall_time": prev["wall_time"] + new["wall_time"], + "passes": passes, "result": prev["result"] if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) else new["result"], @@ -781,6 +787,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): merged_status: dict | None = None for suffix, k_expr in passes: report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + status["passes"] = [suffix.lstrip("-") or "single"] if report is not None: xml_reports.append(report) if was_failure and test_file not in failed_tests: @@ -1016,7 +1023,7 @@ def pytest_sessionstart(session): summary_str += "Per Test Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) + per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Runs", "Test (s)", "Wall (s)", "# Tests"]) per_test_result_table.align["Test Path"] = "l" per_test_result_table.align["Test (s)"] = "r" per_test_result_table.align["Wall (s)"] = "r" @@ -1031,6 +1038,7 @@ def pytest_sessionstart(session): [ test_path, test_status[test_path]["result"], + "+".join(test_status[test_path].get("passes", ["single"])), f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", f"{num_tests_passed}/{test_status[test_path]['tests']}", From 5f232fbd4d7bdeb2dd717da4a7ecca15f6d142d3 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Tue, 2 Jun 2026 13:58:50 +0200 Subject: [PATCH 16/17] Remove device split runs summary column --- tools/conftest.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/conftest.py b/tools/conftest.py index 53c7dfb21b97..eea9f2e427e0 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -445,11 +445,6 @@ def _merge_pass_status(prev: dict | None, new: dict) -> dict: """ if prev is None: return new - passes = [] - for status in (prev, new): - for pass_name in status.get("passes", []): - if pass_name not in passes: - passes.append(pass_name) return { "errors": prev["errors"] + new["errors"], "failures": prev["failures"] + new["failures"], @@ -457,7 +452,6 @@ def _merge_pass_status(prev: dict | None, new: dict) -> dict: "tests": prev["tests"] + new["tests"], "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], "wall_time": prev["wall_time"] + new["wall_time"], - "passes": passes, "result": prev["result"] if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) else new["result"], @@ -787,7 +781,6 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): merged_status: dict | None = None for suffix, k_expr in passes: report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) - status["passes"] = [suffix.lstrip("-") or "single"] if report is not None: xml_reports.append(report) if was_failure and test_file not in failed_tests: @@ -1023,7 +1016,7 @@ def pytest_sessionstart(session): summary_str += "Per Test Result Summary\n" summary_str += "=======================\n" - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Runs", "Test (s)", "Wall (s)", "# Tests"]) + per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) per_test_result_table.align["Test Path"] = "l" per_test_result_table.align["Test (s)"] = "r" per_test_result_table.align["Wall (s)"] = "r" @@ -1038,7 +1031,6 @@ def pytest_sessionstart(session): [ test_path, test_status[test_path]["result"], - "+".join(test_status[test_path].get("passes", ["single"])), f"{test_status[test_path]['time_elapsed']:0.2f}", f"{test_status[test_path]['wall_time']:0.2f}", f"{num_tests_passed}/{test_status[test_path]['tests']}", From c746eaa35041599b55070b6c58ef6abcc98633c1 Mon Sep 17 00:00:00 2001 From: Antoine Richard Date: Fri, 5 Jun 2026 20:37:10 +0200 Subject: [PATCH 17/17] Address ovphysx device split review Pass preloaded test source into the device_split detector so the per-file runner does not read the same test twice. Update the OVPhysX changelog fragment to list every test file now tagged for split CPU and GPU coverage. --- .../antoiner-ovphysx-device-split-ci.rst | 9 ++++++--- tools/_device_split.py | 19 +++++++++++-------- tools/conftest.py | 2 +- tools/test_device_split.py | 11 +++++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst index 8f8b515af6e9..7588fd36b241 100644 --- a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst @@ -1,9 +1,12 @@ Fixed ^^^^^ -* Re-enabled both CPU and GPU coverage in CI for - :file:`test/sim/test_views_xform_prim_ovphysx.py` and - :file:`test/sensors/test_contact_sensor.py` by tagging them with the new +* Re-enabled both CPU and GPU coverage in CI for OVPhysX tests by tagging + :file:`test/assets/test_articulation.py`, + :file:`test/assets/test_rigid_object.py`, + :file:`test/assets/test_rigid_object_collection.py`, + :file:`test/sensors/test_contact_sensor.py`, and + :file:`test/sim/test_views_xform_prim_ovphysx.py` with the new ``device_split`` pytest marker, which causes the CI driver to invoke each file once per device in separate subprocesses. Works around the ``ovphysx<=0.3.7`` process-global device lock (gap G5). diff --git a/tools/_device_split.py b/tools/_device_split.py index 1c802eca489c..e8bad4069133 100644 --- a/tools/_device_split.py +++ b/tools/_device_split.py @@ -41,22 +41,25 @@ ] -def is_device_split_file(path: Path | str) -> bool: +def is_device_split_file(path: Path | str, source: str | None = None) -> bool: """Return whether the test file at ``path`` declares the ``device_split`` marker. - Reads the file source once and matches :data:`_DEVICE_SPLIT_MARK_RE`. A - missing or unreadable file returns ``False`` so the caller falls back to - the default single-pass invocation. + Matches :data:`_DEVICE_SPLIT_MARK_RE` against ``source`` when supplied. + Otherwise, reads the file source from ``path``. A missing or unreadable + file returns ``False`` so the caller falls back to the default single-pass + invocation. Args: path: Filesystem path to a candidate test file. + source: Optional preloaded source text to inspect. Returns: ``True`` when the file's module-level ``pytestmark`` mentions ``device_split``; ``False`` otherwise. """ - try: - source = Path(path).read_text(encoding="utf-8", errors="replace") - except OSError: - return False + if source is None: + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index eea9f2e427e0..a45f8cd9183c 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -772,7 +772,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): env=env, ) - if is_device_split_file(test_file): + if is_device_split_file(test_file, source=test_content): print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") passes = DEVICE_SPLIT_PASSES else: diff --git a/tools/test_device_split.py b/tools/test_device_split.py index c3f7ea67a72e..fb34ce40c4f1 100644 --- a/tools/test_device_split.py +++ b/tools/test_device_split.py @@ -49,6 +49,17 @@ def test_x(): assert is_device_split_file(f) is True +def test_preloaded_source(tmp_path): + source = textwrap.dedent( + """ + import pytest + + pytestmark = pytest.mark.device_split + """ + ) + assert is_device_split_file(tmp_path / "does_not_exist.py", source=source) is True + + def test_no_mark(tmp_path): f = _write( tmp_path,