diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e09ec651fe3f..eff91c7c61c7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -543,6 +543,7 @@ jobs: isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} filter-pattern: "isaaclab_ov" + extra-pip-packages: "ovrtx ovphysx" container-name: isaac-lab-ov-test # Folded from the former standalone verify-base-non-root job: reuses the diff --git a/pyproject.toml b/pyproject.toml index 117311ef207c..a64fc84af215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", + "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst new file mode 100644 index 000000000000..7588fd36b241 --- /dev/null +++ b/source/isaaclab_ovphysx/changelog.d/antoiner-ovphysx-device-split-ci.rst @@ -0,0 +1,12 @@ +Fixed +^^^^^ + +* Re-enabled both CPU and GPU coverage in CI for OVPhysX tests by tagging + :file:`test/assets/test_articulation.py`, + :file:`test/assets/test_rigid_object.py`, + :file:`test/assets/test_rigid_object_collection.py`, + :file:`test/sensors/test_contact_sensor.py`, and + :file:`test/sim/test_views_xform_prim_ovphysx.py` with the new + ``device_split`` pytest marker, which causes the CI driver to invoke each + file once per device in separate subprocesses. Works around the + ``ovphysx<=0.3.7`` process-global device lock (gap G5). diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index e4b052065127..4060da7a326a 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,9 +56,8 @@ import torch import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx import tensor_types as TT # noqa: E402 @@ -83,6 +82,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _OMNI_PHYSX_SCHEMAS_GAP_REASON = ( "Schema-level fixed-joint creation in :mod:`isaaclab.sim.schemas` imports " diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py index db0a2cc89403..da85e53facbc 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation_helpers.py @@ -19,9 +19,8 @@ from pxr import Sdf, Usd, UsdPhysics -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets.articulation.articulation import Articulation # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 407cf4b41e22..a7db008f46b4 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,9 +32,8 @@ import warp as wp from flaky import flaky -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObject # noqa: E402 @@ -56,6 +55,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _logger = logging.getLogger(__name__) diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 07ec860d6ec6..a86baab0a172 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,9 +28,8 @@ import torch import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObjectCollection # noqa: E402 @@ -53,6 +52,8 @@ wp.init() +pytestmark = pytest.mark.device_split + _LOCKED_DEVICE: list[str | None] = [None] """Device the session pins to on the first parametrized test that runs.""" diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py index e57d8d2651d7..a8162765a79b 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_helpers.py @@ -15,9 +15,8 @@ import pytest import warp as wp -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx import tensor_types as TT # noqa: E402 diff --git a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py index 013ef8bdc092..37487b5dd85e 100644 --- a/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py +++ b/source/isaaclab_ovphysx/test/physics/test_ovphysx_scene_data_backend.py @@ -11,9 +11,8 @@ import pytest -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index 38b3bd9e579d..ccec0c7eb28d 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -45,9 +45,8 @@ import warp as wp from flaky import flaky -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.assets import RigidObject # noqa: E402 @@ -64,6 +63,8 @@ wp.init() +pytestmark = pytest.mark.device_split + # --------------------------------------------------------------------------- # Device-lock autouse fixture # --------------------------------------------------------------------------- diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index bebd8aaa460d..e606ce7738b9 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -12,9 +12,8 @@ import pytest -# The CI isaaclab_ov* pattern unintentionally collects isaaclab_ovphysx tests, -# but the ovphysx wheel is not installed in that environment. Skip gracefully -# so the isaaclab_ov CI pipeline is not blocked by an unrelated dependency. +# The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; +# CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") from isaaclab_ovphysx.physics import OvPhysxCfg # noqa: E402 @@ -25,6 +24,8 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) +pytestmark = pytest.mark.device_split + @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) def test_factory_dispatches_to_ovphysx_frame_view(device): diff --git a/tools/_device_split.py b/tools/_device_split.py new file mode 100644 index 000000000000..e8bad4069133 --- /dev/null +++ b/tools/_device_split.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Helpers for detecting and driving the ``device_split`` pytest marker. + +Test files that declare ``pytestmark = pytest.mark.device_split`` at module +scope must be re-invoked once per device (CPU and GPU) in separate processes +to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. +The :func:`is_device_split_file` predicate lets the per-file CI runner in +``tools/conftest.py`` detect this without importing the test module. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) +"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. + +Recognises both single-mark and single-line list forms: + +* ``pytestmark = pytest.mark.device_split`` +* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` + +Multi-line list forms are not supported (currently no test file uses one); if +a future test needs that, expand the parsing rule. +""" + +# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file +# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: +# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. +# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps +# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized +# tests only. +DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ + ("-cpu", "cpu or not cuda"), + ("-cuda", "cuda"), +] + + +def is_device_split_file(path: Path | str, source: str | None = None) -> bool: + """Return whether the test file at ``path`` declares the ``device_split`` marker. + + Matches :data:`_DEVICE_SPLIT_MARK_RE` against ``source`` when supplied. + Otherwise, reads the file source from ``path``. A missing or unreadable + file returns ``False`` so the caller falls back to the default single-pass + invocation. + + Args: + path: Filesystem path to a candidate test file. + source: Optional preloaded source text to inspect. + + Returns: + ``True`` when the file's module-level ``pytestmark`` mentions + ``device_split``; ``False`` otherwise. + """ + if source is None: + try: + source = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index 9187099ee92a..a45f8cd9183c 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -10,6 +10,7 @@ import subprocess import sys import time +from dataclasses import dataclass import pytest from junitparser import Error, JUnitXml, TestCase, TestSuite @@ -17,6 +18,7 @@ # Local imports import test_settings as test_settings # isort: skip +from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip def pytest_ignore_collect(collection_path, config): @@ -400,131 +402,175 @@ def _retry_failed_test_in_fresh_process( ) -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False +@dataclass +class _PassContext: + """Inputs shared across all pytest invocations for a single test file. - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" + Attributes: + test_file: Absolute path to the test file being driven. + file_name: Basename of ``test_file`` (used for JUnit naming). + workspace_root: Repository root; passed to pytest's ``--config-file``. + isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the + ``-m isaacsim_ci`` selector. + timeout: Per-pass hard timeout in seconds. + startup_deadline: Per-pass startup-hang deadline in seconds. + env: Environment passed to the pytest subprocess. + """ - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + test_file: str + file_name: str + workspace_root: str + isaacsim_ci: bool + timeout: int + startup_deadline: int + env: dict + + +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} - # Read the test file once for cold-cache check. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") +def _merge_pass_status(prev: dict | None, new: dict) -> dict: + """Merge per-pass status dicts into a single per-file entry. - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + Counters (``errors``, ``failures``, ``skipped``, ``tests``, + ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more + severe of the two via :data:`_RESULT_PRIORITY`. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + +def _run_one_pass( + ctx: _PassContext, + k_expr: str | None, + suffix: str, +) -> tuple[JUnitXml | None, dict, bool]: + """Drive one pytest subprocess for ``ctx.test_file`` and return its results. - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={workspace_root}/pyproject.toml", - f"--junitxml=tests/test-reports-{str(file_name)}.xml", - "--tb=short", - ] - - if isaacsim_ci: - cmd.append("-m") - cmd.append("isaacsim_ci") - - cmd.append(str(test_file)) - - report_file = f"tests/test-reports-{str(file_name)}.xml" - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = ( - capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - ) + Args: + ctx: Static per-file context (paths, timeouts, env). + k_expr: Optional ``-k`` selector. ``None`` means no selector (default + single-pass invocation). + suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` + or ``""`` for the unsplit default. - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {test_file}: startup hang detected after {startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue + Returns: + A 3-tuple ``(xml_report, status_dict, was_failure)``: + * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced + no report (e.g. startup hang). + * ``status_dict``: per-pass counters compatible with the entries + currently appended to ``test_status``. + * ``was_failure``: whether the pass should add ``ctx.test_file`` to + the ``failed_tests`` list. + """ + pass_file_label = f"{ctx.file_name}{suffix}" + report_file = f"tests/test-reports-{pass_file_label}.xml" + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "--no-header", + f"--config-file={ctx.workspace_root}/pyproject.toml", + f"--junitxml={report_file}", + "--tb=short", + ] + if ctx.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + if k_expr is not None: + cmd += ["-k", k_expr] + cmd.append(str(ctx.test_file)) + + # -- Run with retry on startup hang or hard timeout ----------------- + returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" + wall_time, pre_kill_diag = 0.0, "" + startup_hang_attempts = 0 + timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {test_file}: timeout detected after {timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break + returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( + cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) - # -- Resolve result from kill_reason and report file ---------------- has_report = os.path.exists(report_file) - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {test_file}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: + startup_hang_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" + f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." + ) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" print(diag) + continue - msg = f"Startup hang after {startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: + timeout_attempts += 1 + print( + f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." + ) if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + print("=== STDOUT (last 5000 chars) ===") + print(stdout_data.decode("utf-8", errors="replace")[-5000:]) + if stderr_data: + print("=== STDERR (last 5000 chars) ===") + print(stderr_data.decode("utf-8", errors="replace")[-5000:]) + diag = pre_kill_diag or _capture_system_diagnostics() + if len(diag) > 10000: + diag = diag[:10000] + "\n... (truncated)" + print(diag) + continue + break + + # -- Resolve result from kill_reason and report file ---------------- + has_report = os.path.exists(report_file) + + if kill_reason == "startup_hang": + diag = _get_diagnostics(pre_kill_diag) + print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") + print(diag) + + msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("startup_hang", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -532,61 +578,63 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "STARTUP_HANG", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {test_file} timed out after {timeout} seconds...") - print(diag) + }, + True, + ) - msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", file_name, msg, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if kill_reason == "timeout" and not has_report: + diag = _get_diagnostics(pre_kill_diag) + print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") + print(diag) + + msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 5000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr_data: + details += "=== STDERR (last 5000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" + + error_report = _create_error_report("timeout", pass_file_label, msg, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, "tests": 1, "result": "TIMEOUT", - "time_elapsed": timeout, + "time_elapsed": ctx.timeout, "wall_time": wall_time, - } - continue - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {test_file}: {reason}") - print(diag) + }, + True, + ) - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", file_name, reason, details) - error_report.write(report_file) - xml_reports.append(error_report) - failed_tests.append(test_file) - test_status[test_file] = { + if not has_report: + reason = ( + _signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + diag = _get_diagnostics() + print(f"⚠️ {ctx.test_file}{suffix}: {reason}") + print(diag) + + details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout_data: + details += "=== STDOUT (last 2000 chars) ===\n" + details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" + if stderr_data: + details += "=== STDERR (last 2000 chars) ===\n" + details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" + + error_report = _create_error_report("crash", pass_file_label, reason, details) + error_report.write(report_file) + return ( + error_report, + { "errors": 1, "failures": 0, "skipped": 0, @@ -594,19 +642,21 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "CRASHED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue + }, + True, + ) - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {test_file}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") + # -- Report file exists: parse actual test results ----------------- + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - failed_tests.append(test_file) - test_status[test_file] = { + try: + report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return ( + None, + { "errors": 1, "failures": 0, "skipped": 0, @@ -614,60 +664,59 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": "FAILED", "time_elapsed": 0.0, "wall_time": wall_time, - } - continue - - has_test_failures = errors > 0 or failures > 0 - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=test_file, - file_name=file_name, - cmd=cmd, - timeout=timeout, - env=env, - startup_deadline=startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, + }, + True, ) - xml_reports.append(report) - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + ( + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout_data, + stderr_data, + kill_reason, + wall_time, + pre_kill_diag, + has_test_failures, + ) = _retry_failed_test_in_fresh_process( + test_file=ctx.test_file, + file_name=ctx.file_name, + cmd=cmd, + timeout=ctx.timeout, + env=ctx.env, + startup_deadline=ctx.startup_deadline, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout_data=stdout_data, + stderr_data=stderr_data, + kill_reason=kill_reason, + wall_time=wall_time, + pre_kill_diag=pre_kill_diag, + ) - if has_test_failures or (returncode != 0 and not shutdown_hanged): - failed_tests.append(test_file) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures + was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" + if shutdown_hanged: + result = "passed (shutdown hanged)" + elif has_test_failures: + result = "FAILED" + else: + result = "passed" - test_status[test_file] = { + return ( + report, + { "errors": errors, "failures": failures, "skipped": skipped, @@ -675,7 +724,71 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): "result": result, "time_elapsed": time_elapsed, "wall_time": wall_time, - } + }, + was_failure, + ) + + +def run_individual_tests(test_files, workspace_root, isaacsim_ci): + """Run each test file separately, ensuring one finishes before starting the next.""" + failed_tests = [] + test_status = {} + xml_reports = [] + cold_cache_applied = False + + for test_file in test_files: + print(f"\n\n🚀 Running {test_file} independently...\n") + file_name = os.path.basename(test_file) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + + timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) + + # Read the test file once for cold-cache and device-split detection. + try: + with open(test_file) as fh: + test_content = fh.read() + except OSError: + test_content = "" + + # The first camera-enabled test in a fresh container compiles shaders + # (~600 s). Give it extra time so that doesn't look like a test timeout. + is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content + if is_cold_cache_test: + timeout += COLD_CACHE_BUFFER + cold_cache_applied = True + print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") + + extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 + startup_deadline = min(timeout, STARTUP_DEADLINE + extra) + + ctx = _PassContext( + test_file=test_file, + file_name=file_name, + workspace_root=workspace_root, + isaacsim_ci=isaacsim_ci, + timeout=timeout, + startup_deadline=startup_deadline, + env=env, + ) + + if is_device_split_file(test_file, source=test_content): + print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") + passes = DEVICE_SPLIT_PASSES + else: + passes = [("", None)] + + merged_status: dict | None = None + for suffix, k_expr in passes: + report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) + if report is not None: + xml_reports.append(report) + if was_failure and test_file not in failed_tests: + failed_tests.append(test_file) + merged_status = _merge_pass_status(merged_status, status) + + assert merged_status is not None # the pass list is never empty + test_status[test_file] = merged_status print("~~~~~~~~~~~~ Finished running all tests") diff --git a/tools/test_device_split.py b/tools/test_device_split.py new file mode 100644 index 000000000000..fb34ce40c4f1 --- /dev/null +++ b/tools/test_device_split.py @@ -0,0 +1,108 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for ``tools/_device_split.py``.""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +from _device_split import is_device_split_file + + +def _write(tmp_path: Path, content: str) -> Path: + p = tmp_path / "test_foo.py" + p.write_text(textwrap.dedent(content)) + return p + + +def test_single_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.device_split + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_list_form_single_line(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = [pytest.mark.device_split, pytest.mark.foo] + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is True + + +def test_preloaded_source(tmp_path): + source = textwrap.dedent( + """ + import pytest + + pytestmark = pytest.mark.device_split + """ + ) + assert is_device_split_file(tmp_path / "does_not_exist.py", source=source) is True + + +def test_no_mark(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_word_in_comment_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + # This file mentions device_split in a comment but is not marked. + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_unrelated_pytestmark_does_not_match(tmp_path): + f = _write( + tmp_path, + """ + import pytest + + pytestmark = pytest.mark.skipif(False, reason="x") + + def test_x(): + pass + """, + ) + assert is_device_split_file(f) is False + + +def test_missing_file(tmp_path): + # A path that does not exist must not raise; treat as not-marked. + assert is_device_split_file(tmp_path / "does_not_exist.py") is False