From 14e674d275f7f10fdc7e90a36e84fd3dfde871a1 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 3 Jul 2026 19:51:40 +0000 Subject: [PATCH 01/20] Add Testmon subprocess coverage tracking pytest-testmon 2.x only records coverage inside the pytest process, so tests that exercise code by spawning child Python processes never have those dependencies tracked and are not reselected when the child-only code changes. Add tools/testmon_subprocess_coverage.py and its testmon_coverage_context coverage plugin, which install a process_startup .pth shim, tag child coverage with the active test's node id, and merge child coverage back into Testmon before each report. Wire the hooks in via a repository-root conftest.py; they are no-ops unless Testmon is collecting. Include a regression test that verifies a subprocess-only dependency edit reselects the test. --- .gitignore | 3 + conftest.py | 26 ++ .../mataylor-testmon-subprocess-coverage.skip | 3 + .../test/test_testmon_subprocess_coverage.py | 151 +++++++++++ tools/testmon_coverage_context.py | 24 ++ tools/testmon_subprocess_coverage.py | 240 ++++++++++++++++++ 6 files changed, 447 insertions(+) create mode 100644 conftest.py create mode 100644 source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip create mode 100644 source/isaaclab/test/test_testmon_subprocess_coverage.py create mode 100644 tools/testmon_coverage_context.py create mode 100644 tools/testmon_subprocess_coverage.py diff --git a/.gitignore b/.gitignore index d78a978c60a2..34b836740b17 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ **/*.egg-info/ **/__pycache__/ **/.pytest_cache/ +**/.testmon/ +**/.testmondata +**/.tmontmp/ **/*.pyc **/*.pb diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000000..29029a944518 --- /dev/null +++ b/conftest.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared pytest configuration for repository tests. + +Wires the Testmon subprocess-coverage hooks (see :mod:`tools.testmon_subprocess_coverage`) +into the session so that code executed in child Python processes is attributed to the +active test. The hooks are no-ops unless pytest-testmon is collecting coverage. +""" + +import sys +from pathlib import Path + +_TOOLS_DIR = Path(__file__).resolve().parent / "tools" +if _TOOLS_DIR.is_dir() and str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + +from testmon_subprocess_coverage import ( # noqa: E402, F401 + pytest_runtest_makereport, + pytest_runtest_setup, + pytest_runtest_teardown, + pytest_sessionfinish, + pytest_sessionstart, +) diff --git a/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip b/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip new file mode 100644 index 000000000000..71f945ce9f49 --- /dev/null +++ b/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip @@ -0,0 +1,3 @@ +Test/tooling only: add Testmon subprocess coverage tracking so tests that +spawn child Python processes get reselected when their subprocess-only +dependencies change. diff --git a/source/isaaclab/test/test_testmon_subprocess_coverage.py b/source/isaaclab/test/test_testmon_subprocess_coverage.py new file mode 100644 index 000000000000..65852d8de04a --- /dev/null +++ b/source/isaaclab/test/test_testmon_subprocess_coverage.py @@ -0,0 +1,151 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression tests for Testmon subprocess dependency tracking. + +Spins up an isolated pytest-in-pytest project whose only link to a helper module +is a child ``subprocess``. Verifies that :mod:`testmon_subprocess_coverage` +records the helper as a dependency, so editing it reselects the test. +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _find_tools_dir() -> Path: + """Locate the repository ``tools/`` directory that holds the Testmon helpers.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / "tools" / "testmon_subprocess_coverage.py" + if candidate.is_file(): + return candidate.parent + raise RuntimeError("could not locate tools/testmon_subprocess_coverage.py") + + +_TOOLS_DIR = _find_tools_dir() + +_HELPER_MODULE = "subprocess_only_helper" + +# A test whose sole link to the helper is a child ``python -c "..."`` process. +# The helper is never imported in the pytest process, so only subprocess +# coverage can register it as a Testmon dependency. +_TEST_BODY = """\ +import os +import subprocess +import sys + + +def test_calls_helper_in_subprocess(): + # Record that this test actually executed (the harness counts these lines). + with open(os.environ["SUBPROC_DEP_RUNS"], "a", encoding="utf-8") as handle: + handle.write("run\\n") + + result = subprocess.run( + [sys.executable, "-c", "import {module}; assert {module}.contribution() >= 0"], + env=os.environ.copy(), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr +""" + +_CONFTEST_BODY = """\ +from testmon_subprocess_coverage import ( # noqa: F401 + pytest_runtest_makereport, + pytest_runtest_setup, + pytest_runtest_teardown, + pytest_sessionfinish, + pytest_sessionstart, +) +""" + + +def _dependencies_available() -> bool: + return all(importlib.util.find_spec(name) is not None for name in ("testmon", "coverage")) + + +def _write_helper(project: Path, value: int) -> None: + (project / f"{_HELPER_MODULE}.py").write_text( + f"def contribution():\n return {value}\n", + encoding="utf-8", + ) + + +def _clean_env(project: Path, datafile: Path, runs_file: Path) -> dict[str, str]: + """Environment for a nested pytest run, scrubbed of inherited coverage state.""" + env = os.environ.copy() + # Drop any coverage/testmon state leaking in from the outer pytest session so + # the child session manages its own coverage lifecycle. + for key in ("COVERAGE_PROCESS_START", "COVERAGE_CONTEXT", "COVERAGE_FILE"): + env.pop(key, None) + env["PYTHONPATH"] = os.pathsep.join([str(_TOOLS_DIR), str(project), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + env["TESTMON_DATAFILE"] = str(datafile) + env["SUBPROC_DEP_RUNS"] = str(runs_file) + return env + + +def _run_pytest(project: Path, env: dict[str, str], *testmon_args: str) -> subprocess.CompletedProcess[str]: + cmd = [ + sys.executable, + "-m", + "pytest", + "-o", + "addopts=", + "--testmon", + *testmon_args, + "-q", + str(project), + ] + return subprocess.run(cmd, cwd=project, env=env, capture_output=True, text=True, timeout=300) + + +def _run_count(runs_file: Path) -> int: + if not runs_file.is_file(): + return 0 + return len([line for line in runs_file.read_text(encoding="utf-8").splitlines() if line.strip()]) + + +@pytest.mark.skipif(not _dependencies_available(), reason="pytest-testmon and coverage are required") +def test_editing_subprocess_only_dependency_reselects_test(tmp_path: Path) -> None: + """Testmon reselects a test when a dependency reached only via a subprocess changes.""" + project = tmp_path / "proj" + project.mkdir() + (project / f"test_{_HELPER_MODULE}.py").write_text(_TEST_BODY.format(module=_HELPER_MODULE), encoding="utf-8") + (project / "conftest.py").write_text(_CONFTEST_BODY, encoding="utf-8") + _write_helper(project, value=1) + + datafile = project / ".testmondata" + runs_file = project / "runs.txt" + env = _clean_env(project, datafile, runs_file) + + # 1. Prime Testmon: run everything and record dependencies (incl. subprocess). + collect = _run_pytest(project, env, "--testmon-noselect") + assert collect.returncode == 0, f"collect run failed:\n{collect.stdout}\n{collect.stderr}" + assert datafile.is_file(), "Testmon did not create its data file during collection" + assert _run_count(runs_file) == 1, f"expected the test to run once during collection:\n{collect.stdout}" + + # 2. Nothing changed -> Testmon must deselect the test (it does not run again). + unchanged = _run_pytest(project, env) + assert _run_count(runs_file) == 1, ( + "Testmon reran the test even though nothing changed; deselection is not working.\n" + f"{unchanged.stdout}\n{unchanged.stderr}" + ) + + # 3. Edit the helper that is only reachable through the subprocess. + _write_helper(project, value=2) + + # 4. Testmon must notice the subprocess-only dependency changed and reselect. + changed = _run_pytest(project, env) + assert _run_count(runs_file) == 2, ( + "Testmon did not reselect the test after its subprocess-only dependency changed; " + "subprocess coverage is not being attributed to the test.\n" + f"{changed.stdout}\n{changed.stderr}" + ) diff --git a/tools/testmon_coverage_context.py b/tools/testmon_coverage_context.py new file mode 100644 index 000000000000..1ddc27225360 --- /dev/null +++ b/tools/testmon_coverage_context.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Coverage plugin that tags child-process lines with the active pytest node id.""" + +from __future__ import annotations + +import os +from types import FrameType + +from coverage import CoveragePlugin + + +class NodeIdContextPlugin(CoveragePlugin): + """Report the active pytest node id as coverage's dynamic context.""" + + def dynamic_context(self, frame: FrameType) -> str | None: + return os.environ.get("COVERAGE_CONTEXT") or None + + +def coverage_init(reg, options) -> None: + reg.add_dynamic_context(NodeIdContextPlugin()) diff --git a/tools/testmon_subprocess_coverage.py b/tools/testmon_subprocess_coverage.py new file mode 100644 index 000000000000..97ac77a459b2 --- /dev/null +++ b/tools/testmon_subprocess_coverage.py @@ -0,0 +1,240 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Testmon subprocess coverage helpers. + +pytest-testmon 2.x only tracks coverage in the pytest process. Tests that spawn +child Python processes (for example ``./isaaclab.sh -p scripts/.../train.py``) +need: + +1. A ``.pth`` shim in ``site-packages`` so child interpreters call + :func:`coverage.process_startup`. +2. ``COVERAGE_PROCESS_START`` pointing at a config with ``parallel = true`` and + the :mod:`testmon_coverage_context` plugin, so lines are tagged with the + active test's node id. +3. Child data merged into Testmon's in-memory coverage via + :meth:`coverage.CoverageData.update` before each batch is read. + +Import the pytest hook functions into ``conftest.py``. +""" + +from __future__ import annotations + +import os +import sysconfig +import warnings +from dataclasses import dataclass +from pathlib import Path + +import pytest + +_TMONTMP_DIR = ".tmontmp" +_COVERAGE_PROCESS_START = "COVERAGE_PROCESS_START" +_COVERAGE_CONTEXT = "COVERAGE_CONTEXT" +_STATE_KEY = pytest.StashKey["SubprocessCoverageState"]() + +_SHIM_FILENAME = "testmon_subprocess.pth" +_SHIM_CONTENT = "import coverage; coverage.process_startup()\n" + + +@dataclass(frozen=True) +class SubprocessCoverageState: + """Session state for subprocess coverage collection.""" + + rc_path: Path + data_prefix: Path + shim_path: Path | None + created_shim: bool + enabled: bool + + +def _python_lib_omit_patterns() -> list[str]: + return [os.path.join(value, "*") for key, value in sysconfig.get_paths().items() if key.endswith("lib")] + + +def _is_testmon_collecting(config: pytest.Config) -> bool: + tm_conf = getattr(config, "testmon_config", None) + return tm_conf is not None and tm_conf.collect + + +def _site_packages_dir() -> Path | None: + purelib = sysconfig.get_paths().get("purelib") + if not purelib: + return None + path = Path(purelib) + return path if path.is_dir() else None + + +def _subprocess_shim_available(site_dir: Path) -> Path | None: + """Return an existing ``.pth`` that calls ``process_startup``, if any.""" + for pth_file in site_dir.glob("*.pth"): + try: + if "process_startup" in pth_file.read_text(encoding="utf-8", errors="replace"): + return pth_file + except OSError: + continue + return None + + +def _install_subprocess_shim() -> tuple[Path | None, bool]: + """Ensure a ``process_startup`` ``.pth`` shim is on the interpreter path.""" + try: + import coverage # noqa: F401 + except ImportError: + return None, False + + site_dir = _site_packages_dir() + if site_dir is None: + return None, False + + existing = _subprocess_shim_available(site_dir) + if existing is not None: + return existing, False + + target = site_dir / _SHIM_FILENAME + try: + target.write_text(_SHIM_CONTENT, encoding="utf-8") + except OSError: + return None, False + return target, True + + +def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState | None: + """Configure ``COVERAGE_PROCESS_START`` for the pytest session.""" + if not _is_testmon_collecting(config): + return None + + shim_path, created_shim = _install_subprocess_shim() + if shim_path is None: + warnings.warn( + "testmon subprocess coverage: could not install a process_startup .pth shim into " + "site-packages, so code executed in child Python processes will not be tracked.", + stacklevel=1, + ) + state = SubprocessCoverageState( + rc_path=Path(), data_prefix=Path(), shim_path=None, created_shim=False, enabled=False + ) + config.stash[_STATE_KEY] = state + return state + + rootdir = Path(config.rootpath) + tmontmp = rootdir / _TMONTMP_DIR + tmontmp.mkdir(exist_ok=True) + + tools_dir = rootdir / "tools" + pythonpath_entries = [str(tools_dir)] + if existing := os.environ.get("PYTHONPATH"): + pythonpath_entries.append(existing) + os.environ["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) + + data_prefix = tmontmp / f"subprocess-{os.getpid()}" + rc_path = tmontmp / f"subprocess-{os.getpid()}.coveragerc" + rc_path.write_text( + "\n".join( + [ + "[run]", + f"data_file = {data_prefix}", + "parallel = true", + "plugins = testmon_coverage_context", + "include =", + f" {os.path.join(str(rootdir), '*')}", + "omit =", + *(f" {pattern}" for pattern in _python_lib_omit_patterns()), + "", + ] + ), + encoding="utf-8", + ) + os.environ[_COVERAGE_PROCESS_START] = str(rc_path.resolve()) + state = SubprocessCoverageState( + rc_path=rc_path, data_prefix=data_prefix, shim_path=shim_path, created_shim=created_shim, enabled=True + ) + config.stash[_STATE_KEY] = state + return state + + +def _child_data_files(data_prefix: Path) -> list[Path]: + if not data_prefix.parent.is_dir(): + return [] + return sorted(p for p in data_prefix.parent.glob(f"{data_prefix.name}.*") if p.is_file()) + + +def combine_subprocess_coverage(config: pytest.Config) -> None: + """Merge child-process coverage into Testmon's in-memory coverage data.""" + state = config.stash.get(_STATE_KEY, None) + if state is None or not state.enabled: + return + + child_files = _child_data_files(state.data_prefix) + if not child_files: + return + + collect_plugin = config.pluginmanager.get_plugin("TestmonCollect") + if collect_plugin is None: + return + + cov = collect_plugin.testmon.cov + if cov is None: + return + + from coverage import CoverageData + + was_started = cov._started + if was_started: + cov.stop() + target = cov.get_data() + for child_file in child_files: + child = CoverageData(basename=str(child_file)) + try: + child.read() + except Exception: + continue + target.update(child) + child_file.unlink(missing_ok=True) + if was_started: + cov.start() + + +def teardown_subprocess_coverage(config: pytest.Config) -> None: + """Remove temporary subprocess coverage configuration.""" + state = config.stash.get(_STATE_KEY, None) + if state is None: + return + + os.environ.pop(_COVERAGE_PROCESS_START, None) + os.environ.pop(_COVERAGE_CONTEXT, None) + if state.enabled and state.rc_path.is_file(): + state.rc_path.unlink(missing_ok=True) + if state.enabled: + for leftover in _child_data_files(state.data_prefix): + leftover.unlink(missing_ok=True) + if state.created_shim and state.shim_path is not None and state.shim_path.is_file(): + state.shim_path.unlink(missing_ok=True) + if _STATE_KEY in config.stash: + del config.stash[_STATE_KEY] + + +def pytest_sessionstart(session: pytest.Session) -> None: + setup_subprocess_coverage(session.config) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + if os.environ.get(_COVERAGE_PROCESS_START): + os.environ[_COVERAGE_CONTEXT] = item.nodeid + + +def pytest_runtest_teardown(item: pytest.Item) -> None: + os.environ.pop(_COVERAGE_CONTEXT, None) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]): + if call.when == "teardown": + combine_subprocess_coverage(item.config) + yield + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + teardown_subprocess_coverage(session.config) From 712626582c3174436c70c4a3b933dd4ad842088f Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 3 Jul 2026 20:00:16 +0000 Subject: [PATCH 02/20] add testmon to docker for tests --- source/isaaclab/test/install_ci/Dockerfile.installci | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/install_ci/Dockerfile.installci b/source/isaaclab/test/install_ci/Dockerfile.installci index 662af204ce33..8a79841932a9 100644 --- a/source/isaaclab/test/install_ci/Dockerfile.installci +++ b/source/isaaclab/test/install_ci/Dockerfile.installci @@ -52,7 +52,9 @@ RUN ln -sf /usr/bin/python3 /usr/bin/python # Install test runner dependencies into the system Python RUN pip install --break-system-packages \ pytest>=8.0 \ - pytest-timeout>=2.0 + "pytest-testmon>=2.2,<3" \ + pytest-timeout>=2.0 \ + "coverage>=7.6.1" # Install uv system-wide so non-root users can invoke it without PATH trickery RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh From e73da4310f23a83c4416040fa8455c8f9090dfd4 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 3 Jul 2026 16:14:06 -0400 Subject: [PATCH 03/20] Address greptile review feedback on subprocess coverage Restart coverage in a finally block so a merge failure cannot leave it permanently stopped, guard the private _started attribute with a getattr default, migrate the deprecated hookwrapper hook to wrapper, and restore PYTHONPATH on session teardown. --- tools/testmon_subprocess_coverage.py | 58 ++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/tools/testmon_subprocess_coverage.py b/tools/testmon_subprocess_coverage.py index 97ac77a459b2..921861d0abc2 100644 --- a/tools/testmon_subprocess_coverage.py +++ b/tools/testmon_subprocess_coverage.py @@ -48,6 +48,7 @@ class SubprocessCoverageState: shim_path: Path | None created_shim: bool enabled: bool + prev_pythonpath: str | None = None def _python_lib_omit_patterns() -> list[str]: @@ -124,9 +125,10 @@ def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState tmontmp.mkdir(exist_ok=True) tools_dir = rootdir / "tools" + prev_pythonpath = os.environ.get("PYTHONPATH") pythonpath_entries = [str(tools_dir)] - if existing := os.environ.get("PYTHONPATH"): - pythonpath_entries.append(existing) + if prev_pythonpath: + pythonpath_entries.append(prev_pythonpath) os.environ["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) data_prefix = tmontmp / f"subprocess-{os.getpid()}" @@ -149,7 +151,12 @@ def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState ) os.environ[_COVERAGE_PROCESS_START] = str(rc_path.resolve()) state = SubprocessCoverageState( - rc_path=rc_path, data_prefix=data_prefix, shim_path=shim_path, created_shim=created_shim, enabled=True + rc_path=rc_path, + data_prefix=data_prefix, + shim_path=shim_path, + created_shim=created_shim, + enabled=True, + prev_pythonpath=prev_pythonpath, ) config.stash[_STATE_KEY] = state return state @@ -181,20 +188,32 @@ def combine_subprocess_coverage(config: pytest.Config) -> None: from coverage import CoverageData - was_started = cov._started + # ``Coverage`` exposes no public "is running" flag, so fall back to assuming + # it was running (the normal case while testmon collects) if the private + # attribute is unavailable, avoiding an ``AttributeError`` that would + # silently disable merging on a future coverage release. + was_started = getattr(cov, "_started", True) if was_started: cov.stop() - target = cov.get_data() - for child_file in child_files: - child = CoverageData(basename=str(child_file)) - try: - child.read() - except Exception: - continue - target.update(child) - child_file.unlink(missing_ok=True) - if was_started: - cov.start() + # Always restart coverage in ``finally`` so a merge failure cannot leave it + # stopped, which would silently record no dependencies for the rest of the + # session. + try: + target = cov.get_data() + for child_file in child_files: + child = CoverageData(basename=str(child_file)) + try: + child.read() + except Exception: + continue + try: + target.update(child) + except Exception: + continue + child_file.unlink(missing_ok=True) + finally: + if was_started: + cov.start() def teardown_subprocess_coverage(config: pytest.Config) -> None: @@ -205,6 +224,11 @@ def teardown_subprocess_coverage(config: pytest.Config) -> None: os.environ.pop(_COVERAGE_PROCESS_START, None) os.environ.pop(_COVERAGE_CONTEXT, None) + if state.enabled: + if state.prev_pythonpath is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = state.prev_pythonpath if state.enabled and state.rc_path.is_file(): state.rc_path.unlink(missing_ok=True) if state.enabled: @@ -229,11 +253,11 @@ def pytest_runtest_teardown(item: pytest.Item) -> None: os.environ.pop(_COVERAGE_CONTEXT, None) -@pytest.hookimpl(tryfirst=True, hookwrapper=True) +@pytest.hookimpl(tryfirst=True, wrapper=True) def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]): if call.when == "teardown": combine_subprocess_coverage(item.config) - yield + return (yield) def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: From 1e431bd7e031f9f06530765f564919e907ec75b9 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Fri, 3 Jul 2026 16:21:55 -0400 Subject: [PATCH 04/20] Guard testmon conftest import against missing tools dir Wrap the subprocess-coverage hook import in contextlib.suppress so a partial checkout without tools/ leaves the hooks as unregistered no-ops instead of aborting all test collection with ModuleNotFoundError. --- conftest.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/conftest.py b/conftest.py index 29029a944518..0deda6bd32f4 100644 --- a/conftest.py +++ b/conftest.py @@ -10,17 +10,23 @@ active test. The hooks are no-ops unless pytest-testmon is collecting coverage. """ +import contextlib import sys from pathlib import Path _TOOLS_DIR = Path(__file__).resolve().parent / "tools" -if _TOOLS_DIR.is_dir() and str(_TOOLS_DIR) not in sys.path: - sys.path.insert(0, str(_TOOLS_DIR)) +if _TOOLS_DIR.is_dir(): + if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) -from testmon_subprocess_coverage import ( # noqa: E402, F401 - pytest_runtest_makereport, - pytest_runtest_setup, - pytest_runtest_teardown, - pytest_sessionfinish, - pytest_sessionstart, -) + # Guard the import too: when ``tools/`` is absent (partial checkout, artefact-only + # CI image) the hooks should stay unregistered no-ops rather than aborting the whole + # test collection with a ``ModuleNotFoundError``. + with contextlib.suppress(ImportError): + from testmon_subprocess_coverage import ( # noqa: F401 + pytest_runtest_makereport, + pytest_runtest_setup, + pytest_runtest_teardown, + pytest_sessionfinish, + pytest_sessionstart, + ) From 179cd1d6fa5994bd54e0b7ec6c4065d22f4667ca Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Sat, 4 Jul 2026 12:34:36 -0400 Subject: [PATCH 05/20] update --- .github/actions/install-ci-run/action.yml | 24 ++++++++++++ .github/actions/run-package-tests/action.yml | 34 +++++++++++++++++ .../run-package-tests/select_testmon_mode.py | 29 ++++++++++++++ .../test_select_testmon_mode.py | 32 ++++++++++++++++ .github/actions/run-tests/action.yml | 38 +++++++++++++++++-- .github/workflows/build.yaml | 2 + .github/workflows/install-ci.yml | 28 +++++++++++++- .../changelog.d/test-selection-rules.skip | 1 + tools/run_install_ci.py | 15 ++++++++ 9 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 .github/actions/run-package-tests/select_testmon_mode.py create mode 100644 .github/actions/run-package-tests/test_select_testmon_mode.py create mode 100644 source/isaaclab/changelog.d/test-selection-rules.skip diff --git a/.github/actions/install-ci-run/action.yml b/.github/actions/install-ci-run/action.yml index e099be41e561..c5befe340fe1 100644 --- a/.github/actions/install-ci-run/action.yml +++ b/.github/actions/install-ci-run/action.yml @@ -14,6 +14,14 @@ inputs: description: 'Docker base image for the test container' required: false default: 'ubuntu:24.04' + testmon-data-dir: + description: 'Host directory for the persisted Testmon dependency database' + required: false + default: '' + testmon-mode: + description: 'Testmon mode: select affected tests, collect a full baseline, or off' + required: false + default: 'off' test-filter: description: 'pytest -k expression (empty = run everything)' required: false @@ -33,12 +41,28 @@ runs: shell: bash env: BASE_IMAGE: ${{ inputs.base-image }} + TESTMON_DATA_DIR: ${{ inputs.testmon-data-dir }} + TESTMON_MODE: ${{ inputs.testmon-mode }} TEST_FILTER: ${{ inputs.test-filter }} run: | args=(docker --gpu --base-image "$BASE_IMAGE" --build-wheel + --testmon-data-dir "$TESTMON_DATA_DIR" --results-dir "${{ github.workspace }}/results" -- --tb=short -sv) + case "$TESTMON_MODE" in + select) + if [ -s "$TESTMON_DATA_DIR/.testmondata" ]; then + args+=(--testmon --testmon-forceselect) + else + echo "::warning::Testmon data is missing; collecting a full baseline" + args+=(--testmon --testmon-noselect) + fi + ;; + collect) args+=(--testmon --testmon-noselect) ;; + off) ;; + *) echo "Unknown Testmon mode: $TESTMON_MODE"; exit 1 ;; + esac [ -n "$TEST_FILTER" ] && args+=(-k "$TEST_FILTER") # Make `python3` resolve to the uv-managed 3.12 for build.sh's gen_pyproject.py. # --seed installs pip/setuptools/wheel into the venv so build.sh's `python3 -m pip install` diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 264e01a158b8..a224c29a121a 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -69,6 +69,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + testmon-mode: + description: 'Override Testmon mode with select, collect, or off' + default: '' + required: false container-name: description: 'Docker container name prefix (run-id is appended automatically)' required: true @@ -133,6 +137,34 @@ runs: printf "🔵 Image pull took %dm %ds\n" $((elapsed/60)) $((elapsed%60)) echo "🔵 Docker Image Pulled in ${elapsed}s" >> "$GITHUB_STEP_SUMMARY" + - name: Select Testmon Mode + id: testmon-mode + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + REQUESTED_MODE: ${{ inputs.testmon-mode }} + run: | + mode="${REQUESTED_MODE:-collect}" + if [ -z "$REQUESTED_MODE" ] && [ "${{ github.event_name }}" = "pull_request" ]; then + if changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then + mode="$(printf '%s\n' "$changed_files" | python3 .github/actions/run-package-tests/select_testmon_mode.py)" + else + echo "::warning::Could not list changed files; running the full test suite" + fi + fi + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Testmon mode: $mode" >> "$GITHUB_STEP_SUMMARY" + + - name: Restore Testmon Data + if: steps.testmon-mode.outputs.mode != 'off' + uses: actions/cache@v4 + with: + path: .testmon/${{ inputs.container-name }} + key: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('tools/conftest.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}-${{ github.sha }} + restore-keys: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('tools/conftest.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}- + - name: Run Tests uses: ./.github/actions/run-tests with: @@ -150,6 +182,8 @@ runs: include-files: ${{ inputs.include-files }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} + testmon-data-dir: .testmon/${{ inputs.container-name }} + testmon-mode: ${{ steps.testmon-mode.outputs.mode }} - name: Check Test Results if: always() diff --git a/.github/actions/run-package-tests/select_testmon_mode.py b/.github/actions/run-package-tests/select_testmon_mode.py new file mode 100644 index 000000000000..70dd1f2620a1 --- /dev/null +++ b/.github/actions/run-package-tests/select_testmon_mode.py @@ -0,0 +1,29 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Choose whether Testmon can safely select affected tests for a change.""" + +from __future__ import annotations + +import re +import sys + +_RELEVANT = re.compile( + r"^(?:source|docker|tools|apps|scripts)/" + r"|^\.github/(?:workflows/(?:build|config)\.yaml|actions/)" + r"|^\.gitmodules$" + r"|^[^/]+\.(?:toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$" +) +_IGNORED_SUFFIXES = (".md", ".rst", ".skip") + + +def select_testmon_mode(paths: list[str]) -> str: + """Return ``select`` for tracked Python-only changes, otherwise ``collect``.""" + relevant = [path for path in paths if _RELEVANT.search(path) and not path.endswith(_IGNORED_SUFFIXES)] + return "select" if not relevant or all(path.endswith(".py") for path in relevant) else "collect" + + +if __name__ == "__main__": + print(select_testmon_mode([line.strip() for line in sys.stdin if line.strip()])) diff --git a/.github/actions/run-package-tests/test_select_testmon_mode.py b/.github/actions/run-package-tests/test_select_testmon_mode.py new file mode 100644 index 000000000000..1b340d3803cc --- /dev/null +++ b/.github/actions/run-package-tests/test_select_testmon_mode.py @@ -0,0 +1,32 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for changed-file Testmon mode selection.""" + +import importlib.util +from pathlib import Path + +_MODULE_PATH = Path(__file__).with_name("select_testmon_mode.py") +_SPEC = importlib.util.spec_from_file_location("select_testmon_mode", _MODULE_PATH) +assert _SPEC and _SPEC.loader +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +select_testmon_mode = _MODULE.select_testmon_mode + + +def test_python_only_relevant_changes_select_affected_tests() -> None: + assert select_testmon_mode(["source/isaaclab/isaaclab/app.py", "tools/helper.py"]) == "select" + + +def test_irrelevant_changes_select_no_affected_tests() -> None: + assert select_testmon_mode(["docs/guide.md", "source/isaaclab/changelog.d/change.skip"]) == "select" + + +def test_static_relevant_change_collects_full_suite() -> None: + assert select_testmon_mode(["source/isaaclab/config/extension.toml"]) == "collect" + + +def test_mixed_change_collects_full_suite() -> None: + assert select_testmon_mode(["source/isaaclab/code.py", "docker/Dockerfile.base"]) == "collect" diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index ceea3a1c4e65..ec92da016381 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -69,6 +69,14 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + testmon-data-dir: + description: 'Workspace-relative directory for the Testmon dependency database' + default: '' + required: false + testmon-mode: + description: 'Testmon mode: select affected tests, collect a full baseline, or off' + default: 'off' + required: false runs: using: composite @@ -93,6 +101,8 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" + local testmon_data_dir="${16}" + local testmon_mode="${17}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -118,6 +128,9 @@ runs: if [ -n "$extra_pip_packages" ]; then echo "With extra pip packages: $extra_pip_packages" fi + if [ "$testmon_mode" != "off" ]; then + echo "Testmon mode: $testmon_mode" + fi if [ -n "$filter_pattern" ]; then echo "With filter pattern: $filter_pattern" fi @@ -204,6 +217,25 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + testmon_options="" + if [ -n "$testmon_data_dir" ]; then + mkdir -p "$testmon_data_dir" + docker_env_vars="$docker_env_vars -e TESTMON_DATAFILE=/workspace/isaaclab/$testmon_data_dir/.testmondata" + case "$testmon_mode" in + select) + if [ -s "$testmon_data_dir/.testmondata" ]; then + testmon_options="--testmon --testmon-forceselect" + else + echo "::warning::Testmon data is missing; collecting a full baseline" + testmon_options="--testmon --testmon-noselect" + fi + ;; + collect) testmon_options="--testmon --testmon-noselect" ;; + off) ;; + *) echo "Unknown Testmon mode: $testmon_mode"; return 1 ;; + esac + fi + # Volume mount for deps-cache-hit mode: bind-mount the checked-out # source code over /workspace/isaaclab instead of baking it into the image. docker_volume_args="" @@ -283,7 +315,7 @@ runs: # set this detect-only flag, which makes cold asset downloads # fall back to slow repeated retries. unset HUB__ARGS__DETECT_ONLY - ./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky \"coverage>=7.6.1\" + ./isaaclab.sh -p -m pip install pytest pytest-mock \"pytest-testmon>=2.2,<3\" junitparser flatdict flaky \"coverage>=7.6.1\" if [ -n \"\${TEST_EXTRA_PIP_PACKAGES:-}\" ]; then echo \"Installing extra pip packages: \${TEST_EXTRA_PIP_PACKAGES}\" ./isaaclab.sh -p -m pip install \${TEST_EXTRA_PIP_PACKAGES} @@ -295,7 +327,7 @@ runs: esac fi echo 'Starting pytest with path: $test_path' - ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file + ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options $testmon_options -v --junitxml=tests/$result_file " # Stream container logs in background. @@ -392,7 +424,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.testmon-data-dir }}" "${{ inputs.testmon-mode }}" - name: Kill container on cancellation if: cancelled() diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5d715d4e2945..8a43770ad752 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -100,6 +100,8 @@ jobs: $'^\\.github/workflows/build\\.yaml$\tThis workflow file' $'^\\.github/workflows/config\\.yaml$\tBase image config' $'^\\.github/actions/\tCI actions' + $'^\\.gitmodules$\tGit submodule config' + $'^[^/]+\\.(toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$\tRoot configuration and scripts' ) triggered_jobs="Docker build jobs + all test-* matrix jobs" diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 8d86eec15fca..412e29476cc2 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -36,7 +36,9 @@ jobs: runs-on: ubuntu-latest outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} + testmon_mode: ${{ steps.detect.outputs.testmon_mode }} steps: + - uses: actions/checkout@v6 - id: detect env: GH_TOKEN: ${{ github.token }} @@ -52,14 +54,18 @@ jobs: # filter (a not-triggered required check would block the PR forever). patterns=( $'^apps/\tStandalone apps' + $'^docker/\tContainer build inputs' + $'^scripts/\tStandalone scripts' $'^tools/\tBuild tooling' $'^source/\tLibrary source code' $'^\\.github/actions/run-package-tests/\tTest action' $'^\\.github/actions/install-ci-(collect|run)/\tInstall-ci composite actions' $'^\\.github/workflows/install-ci\\.yml$\tThis workflow file' + $'^\\.gitmodules$\tGit submodule config' $'^VERSION$\tVersion file' $'(^|/)pyproject\\.toml$\tPython project metadata' $'(^|/)environment\\.ya?ml$\tConda environment file' + $'^[^/]+\\.(toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$\tRoot configuration and scripts' ) render_table() { @@ -93,9 +99,10 @@ jobs: } decide() { - local decision="$1" reason="$2" files="${3:-}" + local decision="$1" reason="$2" files="${3:-}" testmon_mode="${4:-collect}" echo "Decision: run_install_tests=$decision ($reason)" echo "run_install_tests=$decision" >> "$GITHUB_OUTPUT" + echo "testmon_mode=$testmon_mode" >> "$GITHUB_OUTPUT" { if [ -n "$files" ]; then render_table "$files" @@ -118,7 +125,8 @@ jobs: printf '%s\n' "$changed_files" if any_match "$changed_files"; then - decide true "relevant paths changed" "$changed_files" + testmon_mode="$(printf '%s\n' "$changed_files" | python3 .github/actions/run-package-tests/select_testmon_mode.py)" + decide true "relevant paths changed" "$changed_files" "$testmon_mode" else decide false "no relevant paths changed" "$changed_files" fi @@ -132,6 +140,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Restore Testmon Data + uses: actions/cache@v4 + with: + path: .testmon/${{ runner.arch }} + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: @@ -140,6 +154,8 @@ jobs: uses: ./.github/actions/install-ci-run with: base-image: ${{ inputs.base_image || 'ubuntu:24.04' }} + testmon-data-dir: ${{ github.workspace }}/.testmon/${{ runner.arch }} + testmon-mode: ${{ needs.changes.outputs.testmon_mode }} test-filter: ${{ inputs.test_filter }} install-tests-arm: @@ -151,6 +167,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Restore Testmon Data + uses: actions/cache@v4 + with: + path: .testmon/${{ runner.arch }} + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: @@ -159,4 +181,6 @@ jobs: uses: ./.github/actions/install-ci-run with: base-image: ${{ inputs.base_image || 'ubuntu:24.04' }} + testmon-data-dir: ${{ github.workspace }}/.testmon/${{ runner.arch }} + testmon-mode: ${{ needs.changes.outputs.testmon_mode }} test-filter: ${{ inputs.test_filter }} diff --git a/source/isaaclab/changelog.d/test-selection-rules.skip b/source/isaaclab/changelog.d/test-selection-rules.skip new file mode 100644 index 000000000000..9f59b1e1f3f3 --- /dev/null +++ b/source/isaaclab/changelog.d/test-selection-rules.skip @@ -0,0 +1 @@ +CI and test-selection changes only. diff --git a/tools/run_install_ci.py b/tools/run_install_ci.py index 5b48dc90ab45..60213d7a8423 100755 --- a/tools/run_install_ci.py +++ b/tools/run_install_ci.py @@ -317,6 +317,16 @@ def _cmd_docker(args: argparse.Namespace) -> int: if not args.no_uv_cache: docker_run_cmd.extend(["-v", "isaaclab-install-ci-uv-cache:/home/isaaclab/.cache/uv"]) + if args.testmon_data_dir: + testmon_data_dir = Path(args.testmon_data_dir).resolve() + testmon_data_dir.mkdir(parents=True, exist_ok=True) + testmon_data_file = testmon_data_dir / ".testmondata" + testmon_data_dir.chmod(0o777) + if testmon_data_file.exists(): + testmon_data_file.chmod(0o666) + docker_run_cmd.extend(["-v", f"{testmon_data_dir}:/tmp/testmon"]) + docker_run_cmd.extend(["-e", "TESTMON_DATAFILE=/tmp/testmon/.testmondata"]) + # Pass environment variables docker_run_cmd.extend(["-e", "OMNI_KIT_ACCEPT_EULA=Y"]) docker_run_cmd.extend(["-e", "ACCEPT_EULA=Y"]) @@ -408,6 +418,8 @@ def main() -> int: --no-pip-cache Disable persistent pip cache volume --no-uv-cache Disable persistent uv cache volume --results-dir DIR Host directory for test results (auto-adds --junitxml) + --testmon-data-dir DIR + Host directory persisted as the Testmon dependency database --wheel PATH Path to pre-built isaaclab wheel file --build-wheel Build the isaaclab wheel and pass it to tests (mutually exclusive with --wheel) --debug Stream wheel-build output live (default: quiet, only dumped on failure) @@ -453,6 +465,9 @@ def main() -> int: docker_p.add_argument( "--results-dir", type=str, default=None, help="Host directory for test results (auto-adds --junitxml)" ) + docker_p.add_argument( + "--testmon-data-dir", type=str, default=None, help="Host directory for the Testmon dependency database" + ) docker_p.add_argument("--wheel", type=str, default=None, help="Path to pre-built isaaclab wheel file") docker_p.add_argument( "--build-wheel", From 97346f06205199a77d227faa68d23aa3e49008ef Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 13:38:22 -0400 Subject: [PATCH 06/20] update --- .github/workflows/install-ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 412e29476cc2..234f2c2c1e2b 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -38,7 +38,6 @@ jobs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} testmon_mode: ${{ steps.detect.outputs.testmon_mode }} steps: - - uses: actions/checkout@v6 - id: detect env: GH_TOKEN: ${{ github.token }} @@ -54,18 +53,14 @@ jobs: # filter (a not-triggered required check would block the PR forever). patterns=( $'^apps/\tStandalone apps' - $'^docker/\tContainer build inputs' - $'^scripts/\tStandalone scripts' $'^tools/\tBuild tooling' $'^source/\tLibrary source code' $'^\\.github/actions/run-package-tests/\tTest action' $'^\\.github/actions/install-ci-(collect|run)/\tInstall-ci composite actions' $'^\\.github/workflows/install-ci\\.yml$\tThis workflow file' - $'^\\.gitmodules$\tGit submodule config' $'^VERSION$\tVersion file' $'(^|/)pyproject\\.toml$\tPython project metadata' $'(^|/)environment\\.ya?ml$\tConda environment file' - $'^[^/]+\\.(toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$\tRoot configuration and scripts' ) render_table() { From 7e987448a02d0fe217572636430b8ea419cb9fe7 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 13:59:22 -0400 Subject: [PATCH 07/20] run all post-merge --- .github/actions/run-package-tests/action.yml | 10 ------ .github/actions/run-tests/action.yml | 31 ++++------------- .github/test-subsets/postmerge-rendering.toml | 34 ------------------- .github/workflows/build.yaml | 27 ++------------- 4 files changed, 8 insertions(+), 94 deletions(-) delete mode 100644 .github/test-subsets/postmerge-rendering.toml diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 61980e71831e..0009f26c635b 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -67,14 +67,6 @@ inputs: description: 'Comma-separated list of specific test files to include' default: '' required: false - test-node-ids-file: - description: 'TOML file containing exact pytest node IDs to run' - default: '' - required: false - test-node-ids-key: - description: 'Top-level key in test-node-ids-file containing the node IDs for this job' - default: '' - required: false pytest-options: description: 'Additional pytest options' default: '' @@ -329,8 +321,6 @@ runs: curobo-only: ${{ inputs.curobo-only }} quarantined-only: ${{ inputs.quarantined-only }} include-files: ${{ inputs.include-files }} - test-node-ids-file: ${{ inputs.test-node-ids-file }} - test-node-ids-key: ${{ inputs.test-node-ids-key }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} testmon-data-dir: .testmon/${{ inputs.container-name }} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 5f32be5e4b26..6bad5f5dd90e 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -61,14 +61,6 @@ inputs: description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)' default: '' required: false - test-node-ids-file: - description: 'TOML file containing exact pytest node IDs to run' - default: '' - required: false - test-node-ids-key: - description: 'Top-level key in test-node-ids-file containing the node IDs for this job' - default: '' - required: false shard-index: description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)' default: '' @@ -133,13 +125,11 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" - local test_node_ids_file="${16}" - local test_node_ids_key="${17}" - local wheelhouse_host_dir="${18}" - local wheelhouse_packages="${19}" - local testmon_data_dir="${20}" - local testmon_mode="${21}" - local test_k_expr="${22}" + local wheelhouse_host_dir="${16}" + local wheelhouse_packages="${17}" + local test_k_expr="${18}" + local testmon_data_dir="${19}" + local testmon_mode="${20}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -195,15 +185,6 @@ runs: echo "Shard: $shard_index of $shard_count" fi - if [ -n "$test_node_ids_file" ] || [ -n "$test_node_ids_key" ]; then - if [ -z "$test_node_ids_file" ] || [ -z "$test_node_ids_key" ]; then - echo "Both test-node-ids-file and test-node-ids-key must be set together" - return 1 - fi - export TEST_NODE_IDS_FILE="$test_node_ids_file" - export TEST_NODE_IDS_KEY="$test_node_ids_key" - fi - # Create reports directory mkdir -p "$reports_dir" @@ -540,7 +521,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "${{ inputs.testmon-data-dir }}" "${{ inputs.testmon-mode }}" "$TEST_K_EXPR_INPUT" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "${{ inputs.testmon-data-dir }}" "${{ inputs.testmon-mode }}" "$TEST_K_EXPR_INPUT" - name: Kill container on cancellation if: cancelled() diff --git a/.github/test-subsets/postmerge-rendering.toml b/.github/test-subsets/postmerge-rendering.toml deleted file mode 100644 index ee52d8f1f548..000000000000 --- a/.github/test-subsets/postmerge-rendering.toml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Stable rendering correctness tests used by post-merge CI. -rendering-correctness = [ - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-semantic_segmentation]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-semantic_segmentation]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-albedo-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-depth-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-None-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-rgb-cartpole]", -] - -rendering-correctness-kitless = [ - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-ovrtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-ovrtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-semantic_segmentation]", -] diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 39f9ffab1b2c..7198ab3dcee4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -108,9 +108,9 @@ jobs: $'^\\.github/test-subsets/\tCI test subset config' ) if [ "$EVENT_NAME" = "push" ]; then - triggered_jobs="Docker base build job + rendering-correctness + rendering-correctness-kitless" + triggered_jobs="Docker build jobs + all test-* matrix jobs (full suite via Testmon collect)" else - triggered_jobs="Docker build jobs + all test-* matrix jobs" + triggered_jobs="Docker build jobs + all test-* matrix jobs (Testmon-selected subset)" fi render_table() { @@ -255,7 +255,6 @@ jobs: runs-on: [self-hosted, gpu] needs: [changes, config] if: >- - github.event_name != 'push' && needs.changes.outputs.run_docker_tests == 'true' steps: - name: Checkout Code @@ -283,7 +282,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -308,7 +306,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -333,7 +330,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -357,7 +353,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -380,7 +375,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -403,7 +397,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -426,7 +419,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -448,7 +440,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -469,7 +460,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -490,7 +480,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -511,7 +500,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -532,7 +520,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -553,7 +540,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -574,7 +560,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -595,7 +580,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_resource != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} @@ -652,7 +636,6 @@ jobs: continue-on-error: true needs: [build-curobo, config] if: >- - github.event_name != 'push' && needs.build-curobo.result == 'success' steps: - uses: actions/checkout@v6 @@ -702,7 +685,6 @@ jobs: continue-on-error: true needs: [build-curobo, config] if: >- - github.event_name != 'push' && needs.build-curobo.result == 'success' steps: - uses: actions/checkout@v6 @@ -726,7 +708,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -767,8 +748,6 @@ jobs: test_rendering_dexsuite_kuka_hetero.py, test_rendering_registered_tasks.py, test_rendering_shadow_hand.py - test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} - test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness' || '' }} container-name: isaac-lab-rendering-correctness-test omni-github-test-type: rendering-correctness @@ -804,8 +783,6 @@ jobs: test_rendering_dexsuite_kuka_homo_kitless.py, test_rendering_dexsuite_kuka_hetero_kitless.py, test_rendering_shadow_hand_kitless.py - test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} - test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless' || '' }} container-name: isaac-lab-rendering-correctness-kitless-test omni-github-test-type: rendering-correctness-kitless #endregion From a5f9bb0789a60d8f52fa33bc39af24f5e8c6657c Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:02:52 -0400 Subject: [PATCH 08/20] Check out repo in install-ci Detect Changes job The detect step invokes .github/actions/run-package-tests/select_testmon_mode.py, but the job never checked out the repository, so the script was missing and the step failed with exit code 2. Add an actions/checkout step so the file is present. --- .github/workflows/install-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 9db6470a07d1..bf90149c1cfe 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -39,6 +39,7 @@ jobs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} testmon_mode: ${{ steps.detect.outputs.testmon_mode }} steps: + - uses: actions/checkout@v6 - id: detect env: GH_TOKEN: ${{ github.token }} From 4fe7d1bbc3fe0421c1af3c414129cdd418f05766 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:10:50 -0400 Subject: [PATCH 09/20] Fix testmon arg wiring in run-tests action Close two unclosed if blocks (the testmon-mode echo and the testmon-data-dir case) that produced bash parse errors, and align the run_tests local parameter order with the call site so testmon-data-dir, testmon-mode, and test-k-expr receive the correct positional arguments. --- .github/actions/run-tests/action.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 6bad5f5dd90e..f00ef85bc54c 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -127,9 +127,9 @@ runs: local extra_pip_packages="${15}" local wheelhouse_host_dir="${16}" local wheelhouse_packages="${17}" - local test_k_expr="${18}" - local testmon_data_dir="${19}" - local testmon_mode="${20}" + local testmon_data_dir="${18}" + local testmon_mode="${19}" + local test_k_expr="${20}" local logs_pid="" local wait_pid="" local docker_wait_file="/tmp/.docker_exit_${container_name}" @@ -157,6 +157,7 @@ runs: fi if [ "$testmon_mode" != "off" ]; then echo "Testmon mode: $testmon_mode" + fi if [ -n "$wheelhouse_host_dir" ]; then echo "With wheelhouse host directory: $wheelhouse_host_dir" fi @@ -282,6 +283,7 @@ runs: off) ;; *) echo "Unknown Testmon mode: $testmon_mode"; return 1 ;; esac + fi if [ -n "$test_k_expr" ]; then export TEST_K_EXPR="$test_k_expr" docker_env_vars="$docker_env_vars -e TEST_K_EXPR" From f603ddd95054804e055884a219373f0b98d89229 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:23:17 -0400 Subject: [PATCH 10/20] Treat all workflow YAML changes as testmon collect Broaden the _RELEVANT regex so any .github/workflows/*.yml change (not just build.yaml/config.yaml) forces collect mode. Otherwise a PR that only touches a workflow such as install-ci.yml would trigger its test job while testmon deselected every test, silently running nothing. --- .github/actions/run-package-tests/select_testmon_mode.py | 2 +- .../actions/run-package-tests/test_select_testmon_mode.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-package-tests/select_testmon_mode.py b/.github/actions/run-package-tests/select_testmon_mode.py index 70dd1f2620a1..6c723c639adf 100644 --- a/.github/actions/run-package-tests/select_testmon_mode.py +++ b/.github/actions/run-package-tests/select_testmon_mode.py @@ -12,7 +12,7 @@ _RELEVANT = re.compile( r"^(?:source|docker|tools|apps|scripts)/" - r"|^\.github/(?:workflows/(?:build|config)\.yaml|actions/)" + r"|^\.github/(?:workflows/[^/]+\.ya?ml|actions/)" r"|^\.gitmodules$" r"|^[^/]+\.(?:toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$" ) diff --git a/.github/actions/run-package-tests/test_select_testmon_mode.py b/.github/actions/run-package-tests/test_select_testmon_mode.py index 1b340d3803cc..7f59628f64d7 100644 --- a/.github/actions/run-package-tests/test_select_testmon_mode.py +++ b/.github/actions/run-package-tests/test_select_testmon_mode.py @@ -30,3 +30,9 @@ def test_static_relevant_change_collects_full_suite() -> None: def test_mixed_change_collects_full_suite() -> None: assert select_testmon_mode(["source/isaaclab/code.py", "docker/Dockerfile.base"]) == "collect" + + +def test_workflow_yaml_change_collects_full_suite() -> None: + # A workflow change that testmon cannot reason about must run the full suite, + # otherwise a workflow-only PR would trigger a job that deselects every test. + assert select_testmon_mode([".github/workflows/install-ci.yml"]) == "collect" From 5209535422b388a39d28573a482508b17d8a9972 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:35:36 -0400 Subject: [PATCH 11/20] Fix readonly testmon DB in install CI container The install CI container runs as the non-root 'isaaclab' user (uid 1000), while the testmon data directory is bind-mounted from the host. When the Testmon cache is restored, SQLite opens the database in WAL mode and must write its sibling files (.testmondata-wal / .testmondata-shm). Only the main .testmondata file was made world-writable, so a restored sibling with default 0644 permissions caused SQLite to report 'attempt to write a readonly database' during pytest_configure, aborting the run. World-write every file in the testmon data directory (not just .testmondata) and tolerate chmod failures, matching the handling used for the results directory. --- tools/run_install_ci.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/run_install_ci.py b/tools/run_install_ci.py index 60213d7a8423..6b67011b70f8 100755 --- a/tools/run_install_ci.py +++ b/tools/run_install_ci.py @@ -320,10 +320,19 @@ def _cmd_docker(args: argparse.Namespace) -> int: if args.testmon_data_dir: testmon_data_dir = Path(args.testmon_data_dir).resolve() testmon_data_dir.mkdir(parents=True, exist_ok=True) - testmon_data_file = testmon_data_dir / ".testmondata" - testmon_data_dir.chmod(0o777) - if testmon_data_file.exists(): - testmon_data_file.chmod(0o666) + # The container runs as the non-root 'isaaclab' user (uid 1000). SQLite + # opens the database in WAL mode, so it must also write the sibling + # ``.testmondata-wal``/``.testmondata-shm`` files restored from the cache. + # World-write the directory and every file inside it (not just + # ``.testmondata``); otherwise a restored sibling with default 0644 + # perms makes SQLite report "attempt to write a readonly database". + try: + testmon_data_dir.chmod(0o777) + for entry in testmon_data_dir.iterdir(): + if entry.is_file(): + entry.chmod(0o666) + except OSError as exc: + print(f"Warning: could not chmod testmon data dir {testmon_data_dir}: {exc}", file=sys.stderr) docker_run_cmd.extend(["-v", f"{testmon_data_dir}:/tmp/testmon"]) docker_run_cmd.extend(["-e", "TESTMON_DATAFILE=/tmp/testmon/.testmondata"]) From 37362c2758dcf5c06995e7f16346b4161da5f5ee Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:45:38 -0400 Subject: [PATCH 12/20] upload testmon data --- .github/actions/install-ci-run/action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-ci-run/action.yml b/.github/actions/install-ci-run/action.yml index 1c8ef1fc881f..85a1c4fdc358 100644 --- a/.github/actions/install-ci-run/action.yml +++ b/.github/actions/install-ci-run/action.yml @@ -7,7 +7,9 @@ name: 'Run Installation CI Tests' description: > Runs the install_ci suite inside the project's Docker harness via tools/run_install_ci.py and uploads the JUnit XML report as an artifact - named install-ci-junit-. + named install-ci-junit-. When a Testmon data dir is provided, + the .testmondata database is also uploaded as install-ci-testmon- + for triage. inputs: base-image: @@ -70,6 +72,14 @@ runs: uv venv --seed --python 3.12 "${RUNNER_TEMP}/venv-runner" source "${RUNNER_TEMP}/venv-runner/bin/activate" tools/run_install_ci.py "${args[@]}" + - name: Upload Testmon data + if: always() && inputs.testmon-data-dir != '' + uses: actions/upload-artifact@v7 + with: + name: install-ci-testmon-${{ runner.arch }} + path: ${{ inputs.testmon-data-dir }}/.testmondata + if-no-files-found: warn + retention-days: 7 - name: Upload JUnit XML report if: always() id: upload-junit-report From 63c07cb919a23108ee3c1fbdeefc257e9739b830 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:48:45 -0400 Subject: [PATCH 13/20] Remove dead node-ID diagnostic prints in run-tests action The test_node_ids_file/test_node_ids_key local variables and the TEST_NODE_IDS_FILE/TEST_NODE_IDS_KEY docker env wiring were left over from the node-ID selection mechanism replaced by testmon. Their guards are always false now, so the blocks are unreachable dead code. --- .github/actions/run-tests/action.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index f00ef85bc54c..9f5a51dc8029 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -176,12 +176,6 @@ runs: if [ -n "$include_files" ]; then echo "Include files: $include_files" fi - if [ -n "$test_node_ids_file" ]; then - echo "Test node IDs file: $test_node_ids_file" - fi - if [ -n "$test_node_ids_key" ]; then - echo "Test node IDs key: $test_node_ids_key" - fi if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then echo "Shard: $shard_index of $shard_count" fi @@ -227,11 +221,6 @@ runs: echo "Setting TEST_NODE_IDS" fi - if [ -n "${TEST_NODE_IDS_FILE:-}" ]; then - docker_env_vars="$docker_env_vars -e TEST_NODE_IDS_FILE -e TEST_NODE_IDS_KEY" - echo "Setting TEST_NODE_IDS_FILE=$TEST_NODE_IDS_FILE TEST_NODE_IDS_KEY=$TEST_NODE_IDS_KEY" - fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count" echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count" From c79b57c68149544238cca40daf0eb3aa238b00e4 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 14:58:23 -0400 Subject: [PATCH 14/20] Harden testmon coverage lookup against private-API changes Use getattr fallbacks for the collect_plugin.testmon.cov chain so a future testmon rename disables subprocess-coverage merging gracefully instead of raising AttributeError and breaking the test session. --- tools/testmon_subprocess_coverage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/testmon_subprocess_coverage.py b/tools/testmon_subprocess_coverage.py index 921861d0abc2..cfc6e5d83787 100644 --- a/tools/testmon_subprocess_coverage.py +++ b/tools/testmon_subprocess_coverage.py @@ -182,7 +182,12 @@ def combine_subprocess_coverage(config: pytest.Config) -> None: if collect_plugin is None: return - cov = collect_plugin.testmon.cov + # ``collect_plugin.testmon.cov`` is a private chain that could change on a + # future testmon release; fall back to ``None`` rather than raising + # ``AttributeError`` so a rename simply disables merging instead of breaking + # the whole test session. + testmon = getattr(collect_plugin, "testmon", None) + cov = getattr(testmon, "cov", None) if cov is None: return From 63e963b49c339859a2775f6ce019d3af888b221d Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 15:55:59 -0400 Subject: [PATCH 15/20] always run smoke tests --- conftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/conftest.py b/conftest.py index 0deda6bd32f4..df668e71d427 100644 --- a/conftest.py +++ b/conftest.py @@ -12,8 +12,23 @@ import contextlib import sys +from collections.abc import Generator from pathlib import Path +import pytest + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> Generator[None, None, None]: + """Keep tests marked ``smoke`` selected when Testmon filters the collection.""" + always_items = [item for item in items if item.get_closest_marker("smoke") is not None] + yield + + if config.getoption("testmon_forceselect", default=False): + selected = set(items) + items.extend(item for item in always_items if item not in selected) + + _TOOLS_DIR = Path(__file__).resolve().parent / "tools" if _TOOLS_DIR.is_dir(): if str(_TOOLS_DIR) not in sys.path: From bd1ad49b06de701772dd48f0c968c16284dcf4db Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 15:59:40 -0400 Subject: [PATCH 16/20] Force full collect for .github/actions Python changes Testmon has no dependency data for Python helpers under .github/actions/ because they run outside the pytest process. The mode selector's Python-only fast path incorrectly returned 'select' for changes to those scripts, so a PR touching only such a file would deselect every test and leave CI green without verifying anything. Exclude .github/ Python files from the tracked-python fast path so they trigger a full collection, add a regression test, and include the root conftest.py and subprocess-coverage plugin in the testmon cache key. --- .github/actions/run-package-tests/action.yml | 4 ++-- .../actions/run-package-tests/select_testmon_mode.py | 12 +++++++++++- .../run-package-tests/test_select_testmon_mode.py | 7 +++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 0009f26c635b..23891e32daab 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -179,8 +179,8 @@ runs: uses: actions/cache@v4 with: path: .testmon/${{ inputs.container-name }} - key: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('tools/conftest.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}-${{ github.sha }} - restore-keys: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('tools/conftest.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}- + key: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('conftest.py', 'tools/conftest.py', 'tools/testmon_subprocess_coverage.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}-${{ github.sha }} + restore-keys: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('conftest.py', 'tools/conftest.py', 'tools/testmon_subprocess_coverage.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}- - name: Restore NGC CLI cache if: inputs.wheelhouse-resource != '' && env.NGC_API_KEY != '' uses: actions/cache@v4 diff --git a/.github/actions/run-package-tests/select_testmon_mode.py b/.github/actions/run-package-tests/select_testmon_mode.py index 6c723c639adf..c7b575b75984 100644 --- a/.github/actions/run-package-tests/select_testmon_mode.py +++ b/.github/actions/run-package-tests/select_testmon_mode.py @@ -19,10 +19,20 @@ _IGNORED_SUFFIXES = (".md", ".rst", ".skip") +def _is_tracked_python(path: str) -> bool: + """Return whether ``path`` is a Python source file whose dependencies Testmon tracks. + + Python files under ``.github/`` (e.g. action helper scripts) run outside the pytest + process, so Testmon has no dependency data for them; changes to those files must fall + back to a full collection rather than the Python-only fast path. + """ + return path.endswith(".py") and not path.startswith(".github/") + + def select_testmon_mode(paths: list[str]) -> str: """Return ``select`` for tracked Python-only changes, otherwise ``collect``.""" relevant = [path for path in paths if _RELEVANT.search(path) and not path.endswith(_IGNORED_SUFFIXES)] - return "select" if not relevant or all(path.endswith(".py") for path in relevant) else "collect" + return "select" if not relevant or all(_is_tracked_python(path) for path in relevant) else "collect" if __name__ == "__main__": diff --git a/.github/actions/run-package-tests/test_select_testmon_mode.py b/.github/actions/run-package-tests/test_select_testmon_mode.py index 7f59628f64d7..2d33b4d2eb87 100644 --- a/.github/actions/run-package-tests/test_select_testmon_mode.py +++ b/.github/actions/run-package-tests/test_select_testmon_mode.py @@ -36,3 +36,10 @@ def test_workflow_yaml_change_collects_full_suite() -> None: # A workflow change that testmon cannot reason about must run the full suite, # otherwise a workflow-only PR would trigger a job that deselects every test. assert select_testmon_mode([".github/workflows/install-ci.yml"]) == "collect" + + +def test_action_python_change_collects_full_suite() -> None: + # Python helpers under .github/actions/ run outside pytest, so testmon has no + # dependency data for them; a change to one must run the full suite rather than + # taking the Python-only fast path and deselecting every test. + assert select_testmon_mode([".github/actions/run-package-tests/select_testmon_mode.py"]) == "collect" From 2bef0067b4dd80ea353a254fa027891d0cc61b82 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 18:07:18 -0400 Subject: [PATCH 17/20] fix test --- tools/testmon_subprocess_coverage.py | 107 +++++++++++++-------------- 1 file changed, 52 insertions(+), 55 deletions(-) diff --git a/tools/testmon_subprocess_coverage.py b/tools/testmon_subprocess_coverage.py index cfc6e5d83787..0d4c91486b61 100644 --- a/tools/testmon_subprocess_coverage.py +++ b/tools/testmon_subprocess_coverage.py @@ -9,8 +9,11 @@ child Python processes (for example ``./isaaclab.sh -p scripts/.../train.py``) need: -1. A ``.pth`` shim in ``site-packages`` so child interpreters call - :func:`coverage.process_startup`. +1. A ``sitecustomize.py`` shim on ``PYTHONPATH`` so child interpreters call + :func:`coverage.process_startup` at startup. This is written into a + repository-local temporary directory rather than ``site-packages`` so it + works even when ``site-packages`` is read-only (for example inside CI + containers running as a non-root user). 2. ``COVERAGE_PROCESS_START`` pointing at a config with ``parallel = true`` and the :mod:`testmon_coverage_context` plugin, so lines are tagged with the active test's node id. @@ -23,6 +26,7 @@ from __future__ import annotations import os +import shutil import sysconfig import warnings from dataclasses import dataclass @@ -35,8 +39,21 @@ _COVERAGE_CONTEXT = "COVERAGE_CONTEXT" _STATE_KEY = pytest.StashKey["SubprocessCoverageState"]() -_SHIM_FILENAME = "testmon_subprocess.pth" -_SHIM_CONTENT = "import coverage; coverage.process_startup()\n" +# ``sitecustomize`` is imported unconditionally by :mod:`site` at interpreter +# startup (unlike ``usercustomize``, which is gated on ``ENABLE_USER_SITE``), so +# a ``sitecustomize.py`` placed first on ``PYTHONPATH`` reliably runs in every +# child interpreter. ``coverage.process_startup`` is a no-op unless +# ``COVERAGE_PROCESS_START`` is set, so the shim is harmless outside collection. +_SHIM_FILENAME = "sitecustomize.py" +_SHIM_CONTENT = ( + "# Auto-generated by testmon_subprocess_coverage to attribute child-process\n" + "# coverage back to the active test. No-op unless COVERAGE_PROCESS_START is set.\n" + "try:\n" + " import coverage\n" + " coverage.process_startup()\n" + "except Exception:\n" + " pass\n" +) @dataclass(frozen=True) @@ -45,8 +62,7 @@ class SubprocessCoverageState: rc_path: Path data_prefix: Path - shim_path: Path | None - created_shim: bool + shim_dir: Path | None enabled: bool prev_pythonpath: str | None = None @@ -60,46 +76,23 @@ def _is_testmon_collecting(config: pytest.Config) -> bool: return tm_conf is not None and tm_conf.collect -def _site_packages_dir() -> Path | None: - purelib = sysconfig.get_paths().get("purelib") - if not purelib: - return None - path = Path(purelib) - return path if path.is_dir() else None - - -def _subprocess_shim_available(site_dir: Path) -> Path | None: - """Return an existing ``.pth`` that calls ``process_startup``, if any.""" - for pth_file in site_dir.glob("*.pth"): - try: - if "process_startup" in pth_file.read_text(encoding="utf-8", errors="replace"): - return pth_file - except OSError: - continue - return None +def _install_subprocess_shim(shim_dir: Path) -> Path | None: + """Write a ``sitecustomize.py`` ``process_startup`` shim into ``shim_dir``. - -def _install_subprocess_shim() -> tuple[Path | None, bool]: - """Ensure a ``process_startup`` ``.pth`` shim is on the interpreter path.""" + Returns the shim directory (to be prepended to ``PYTHONPATH``) or ``None`` + if :mod:`coverage` is unavailable or the directory cannot be written. + """ try: import coverage # noqa: F401 except ImportError: - return None, False - - site_dir = _site_packages_dir() - if site_dir is None: - return None, False - - existing = _subprocess_shim_available(site_dir) - if existing is not None: - return existing, False + return None - target = site_dir / _SHIM_FILENAME try: - target.write_text(_SHIM_CONTENT, encoding="utf-8") + shim_dir.mkdir(parents=True, exist_ok=True) + (shim_dir / _SHIM_FILENAME).write_text(_SHIM_CONTENT, encoding="utf-8") except OSError: - return None, False - return target, True + return None + return shim_dir def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState | None: @@ -107,26 +100,27 @@ def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState if not _is_testmon_collecting(config): return None - shim_path, created_shim = _install_subprocess_shim() - if shim_path is None: + rootdir = Path(config.rootpath) + tmontmp = rootdir / _TMONTMP_DIR + tmontmp.mkdir(exist_ok=True) + + shim_dir = _install_subprocess_shim(tmontmp / f"shim-{os.getpid()}") + if shim_dir is None: warnings.warn( - "testmon subprocess coverage: could not install a process_startup .pth shim into " - "site-packages, so code executed in child Python processes will not be tracked.", + "testmon subprocess coverage: could not install a process_startup sitecustomize shim, " + "so code executed in child Python processes will not be tracked.", stacklevel=1, ) - state = SubprocessCoverageState( - rc_path=Path(), data_prefix=Path(), shim_path=None, created_shim=False, enabled=False - ) + state = SubprocessCoverageState(rc_path=Path(), data_prefix=Path(), shim_dir=None, enabled=False) config.stash[_STATE_KEY] = state return state - rootdir = Path(config.rootpath) - tmontmp = rootdir / _TMONTMP_DIR - tmontmp.mkdir(exist_ok=True) - + # Prepend the shim directory (so child interpreters import our + # ``sitecustomize``) and ``tools/`` (so the coveragerc can load + # :mod:`testmon_coverage_context`). tools_dir = rootdir / "tools" prev_pythonpath = os.environ.get("PYTHONPATH") - pythonpath_entries = [str(tools_dir)] + pythonpath_entries = [str(shim_dir), str(tools_dir)] if prev_pythonpath: pythonpath_entries.append(prev_pythonpath) os.environ["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) @@ -143,6 +137,10 @@ def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState "include =", f" {os.path.join(str(rootdir), '*')}", "omit =", + # The Testmon scratch dir (including our sitecustomize shim) lives + # under ``rootdir`` but must not be tracked as a test dependency; + # its per-PID path changes every run and would force reselection. + f" {os.path.join(str(tmontmp), '*')}", *(f" {pattern}" for pattern in _python_lib_omit_patterns()), "", ] @@ -153,8 +151,7 @@ def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState state = SubprocessCoverageState( rc_path=rc_path, data_prefix=data_prefix, - shim_path=shim_path, - created_shim=created_shim, + shim_dir=shim_dir, enabled=True, prev_pythonpath=prev_pythonpath, ) @@ -239,8 +236,8 @@ def teardown_subprocess_coverage(config: pytest.Config) -> None: if state.enabled: for leftover in _child_data_files(state.data_prefix): leftover.unlink(missing_ok=True) - if state.created_shim and state.shim_path is not None and state.shim_path.is_file(): - state.shim_path.unlink(missing_ok=True) + if state.shim_dir is not None and state.shim_dir.is_dir(): + shutil.rmtree(state.shim_dir, ignore_errors=True) if _STATE_KEY in config.stash: del config.stash[_STATE_KEY] From 85758e1dc87a6e3ad1148171498c827944a37ff2 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 18:42:11 -0400 Subject: [PATCH 18/20] remove leftover --- .github/workflows/build.yaml | 1 - tools/conftest.py | 31 ------------------------------- 2 files changed, 32 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7198ab3dcee4..2750e8851195 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -105,7 +105,6 @@ jobs: $'^\\.github/workflows/build\\.yaml$\tThis workflow file' $'^\\.github/workflows/config\\.yaml$\tBase image config' $'^\\.github/actions/\tCI actions' - $'^\\.github/test-subsets/\tCI test subset config' ) if [ "$EVENT_NAME" = "push" ]; then triggered_jobs="Docker build jobs + all test-* matrix jobs (full suite via Testmon collect)" diff --git a/tools/conftest.py b/tools/conftest.py index 963853194f3d..ea5ff614ca82 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -13,7 +13,6 @@ from dataclasses import dataclass import pytest -import tomllib from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable @@ -878,37 +877,9 @@ def _collect_test_files( return test_files -def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: - """Load exact pytest node IDs from a TOML file configured in the environment.""" - node_ids_file = os.environ.get("TEST_NODE_IDS_FILE") - node_ids_key = os.environ.get("TEST_NODE_IDS_KEY") - if not (node_ids_file or node_ids_key): - return [] - if not (node_ids_file and node_ids_key): - pytest.exit("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", returncode=1) - - path = node_ids_file if os.path.isabs(node_ids_file) else os.path.join(workspace_root, node_ids_file) - - try: - with open(os.path.normpath(path), "rb") as stream: - node_ids = tomllib.load(stream).get(node_ids_key) - except OSError as exc: - pytest.exit(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", returncode=1) - except tomllib.TOMLDecodeError as exc: - pytest.exit(f"{node_ids_file}: invalid TOML: {exc}", returncode=1) - - if not node_ids: - pytest.exit(f"{node_ids_key!r} not found or empty in {node_ids_file}", returncode=1) - if not isinstance(node_ids, list) or not all(isinstance(node_id, str) for node_id in node_ids): - pytest.exit(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", returncode=1) - - return node_ids - - def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: """Group exact pytest node IDs by absolute test file path.""" node_ids = [line.strip() for line in os.environ.get("TEST_NODE_IDS", "").splitlines() if line.strip()] - node_ids.extend(_load_test_node_ids_from_toml(workspace_root)) if len(node_ids) != len(set(node_ids)): pytest.exit("Configured test node IDs contain duplicates", returncode=1) @@ -989,8 +960,6 @@ def pytest_sessionstart(session): print(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") print(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") print(f"TEST_NODE_IDS env var: '{'SET' if os.environ.get('TEST_NODE_IDS') else 'NOT_SET'}'") - print(f"TEST_NODE_IDS_FILE env var: '{os.environ.get('TEST_NODE_IDS_FILE', 'NOT_SET')}'") - print(f"TEST_NODE_IDS_KEY env var: '{os.environ.get('TEST_NODE_IDS_KEY', 'NOT_SET')}'") print(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") print(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") print("=" * 50) From ea523dcd99a054464689ccbbe7183815cc3d3308 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 18:52:05 -0400 Subject: [PATCH 19/20] update --- .github/workflows/install-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index bf90149c1cfe..c21d3fcc4e27 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -141,8 +141,8 @@ jobs: uses: actions/cache@v4 with: path: .testmon/${{ runner.arch }} - key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} - restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: @@ -168,8 +168,8 @@ jobs: uses: actions/cache@v4 with: path: .testmon/${{ runner.arch }} - key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} - restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: From c67a3180e4ffe76390b760dba8b0fb1bfb68a39a Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Mon, 6 Jul 2026 19:02:08 -0400 Subject: [PATCH 20/20] Test multiple module selection --- source/isaaclab/isaaclab/utils/array.py | 2 +- source/isaaclab/isaaclab/utils/string.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/isaaclab/utils/array.py b/source/isaaclab/isaaclab/utils/array.py index d15fbc275dc7..7d514e4ee49e 100644 --- a/source/isaaclab/isaaclab/utils/array.py +++ b/source/isaaclab/isaaclab/utils/array.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for working with different array backends.""" +"""Utilities for working consistently with different array backends.""" # needed to import for allowing type-hinting: torch.device | str | None from __future__ import annotations diff --git a/source/isaaclab/isaaclab/utils/string.py b/source/isaaclab/isaaclab/utils/string.py index fa83688154d6..dba6104c916f 100644 --- a/source/isaaclab/isaaclab/utils/string.py +++ b/source/isaaclab/isaaclab/utils/string.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for transforming strings and regular expressions.""" +"""Utilities for transforming strings and regular-expression patterns.""" import ast import functools