diff --git a/.github/actions/install-ci-run/action.yml b/.github/actions/install-ci-run/action.yml index f86ef7105a33..85a1c4fdc358 100644 --- a/.github/actions/install-ci-run/action.yml +++ b/.github/actions/install-ci-run/action.yml @@ -7,13 +7,23 @@ name: 'Run Installation CI Tests' description: > Runs the install_ci suite inside the project's Docker harness via tools/run_install_ci.py and uploads the JUnit XML report as an artifact - named install-ci-junit-. + named install-ci-junit-. When a Testmon data dir is provided, + the .testmondata database is also uploaded as install-ci-testmon- + for triage. inputs: base-image: description: 'Docker base image for the test container' required: false default: 'ubuntu:24.04' + testmon-data-dir: + description: 'Host directory for the persisted Testmon dependency database' + required: false + default: '' + testmon-mode: + description: 'Testmon mode: select affected tests, collect a full baseline, or off' + required: false + default: 'off' test-filter: description: 'pytest -k expression (empty = run everything)' required: false @@ -33,12 +43,28 @@ runs: shell: bash env: BASE_IMAGE: ${{ inputs.base-image }} + TESTMON_DATA_DIR: ${{ inputs.testmon-data-dir }} + TESTMON_MODE: ${{ inputs.testmon-mode }} TEST_FILTER: ${{ inputs.test-filter }} run: | args=(docker --gpu --base-image "$BASE_IMAGE" --build-wheel + --testmon-data-dir "$TESTMON_DATA_DIR" --results-dir "${{ github.workspace }}/results" -- --tb=short -sv) + case "$TESTMON_MODE" in + select) + if [ -s "$TESTMON_DATA_DIR/.testmondata" ]; then + args+=(--testmon --testmon-forceselect) + else + echo "::warning::Testmon data is missing; collecting a full baseline" + args+=(--testmon --testmon-noselect) + fi + ;; + collect) args+=(--testmon --testmon-noselect) ;; + off) ;; + *) echo "Unknown Testmon mode: $TESTMON_MODE"; exit 1 ;; + esac [ -n "$TEST_FILTER" ] && args+=(-k "$TEST_FILTER") # Make `python3` resolve to the uv-managed 3.12 for build.sh's gen_pyproject.py. # --seed installs pip/setuptools/wheel into the venv so build.sh's `python3 -m pip install` @@ -46,6 +72,14 @@ runs: uv venv --seed --python 3.12 "${RUNNER_TEMP}/venv-runner" source "${RUNNER_TEMP}/venv-runner/bin/activate" tools/run_install_ci.py "${args[@]}" + - name: Upload Testmon data + if: always() && inputs.testmon-data-dir != '' + uses: actions/upload-artifact@v7 + with: + name: install-ci-testmon-${{ runner.arch }} + path: ${{ inputs.testmon-data-dir }}/.testmondata + if-no-files-found: warn + retention-days: 7 - name: Upload JUnit XML report if: always() id: upload-junit-report diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml index 9ee6dcddd9b8..23891e32daab 100644 --- a/.github/actions/run-package-tests/action.yml +++ b/.github/actions/run-package-tests/action.yml @@ -67,14 +67,6 @@ inputs: description: 'Comma-separated list of specific test files to include' default: '' required: false - test-node-ids-file: - description: 'TOML file containing exact pytest node IDs to run' - default: '' - required: false - test-node-ids-key: - description: 'Top-level key in test-node-ids-file containing the node IDs for this job' - default: '' - required: false pytest-options: description: 'Additional pytest options' default: '' @@ -83,6 +75,10 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + testmon-mode: + description: 'Override Testmon mode with select, collect, or off' + default: '' + required: false wheelhouse-resource: description: 'Optional NGC resource containing wheelhouse/ and manifest.json for offline pip installs' default: '' @@ -158,6 +154,33 @@ runs: printf "🔵 Image pull took %dm %ds\n" $((elapsed/60)) $((elapsed%60)) echo "🔵 Docker Image Pulled in ${elapsed}s" >> "$GITHUB_STEP_SUMMARY" + - name: Select Testmon Mode + id: testmon-mode + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + REQUESTED_MODE: ${{ inputs.testmon-mode }} + run: | + mode="${REQUESTED_MODE:-collect}" + if [ -z "$REQUESTED_MODE" ] && [ "${{ github.event_name }}" = "pull_request" ]; then + if changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then + mode="$(printf '%s\n' "$changed_files" | python3 .github/actions/run-package-tests/select_testmon_mode.py)" + else + echo "::warning::Could not list changed files; running the full test suite" + fi + fi + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "Testmon mode: $mode" >> "$GITHUB_STEP_SUMMARY" + + - name: Restore Testmon Data + if: steps.testmon-mode.outputs.mode != 'off' + uses: actions/cache@v4 + with: + path: .testmon/${{ inputs.container-name }} + key: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('conftest.py', 'tools/conftest.py', 'tools/testmon_subprocess_coverage.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}-${{ github.sha }} + restore-keys: testmon-${{ runner.arch }}-${{ inputs.container-name }}-${{ hashFiles('conftest.py', 'tools/conftest.py', 'tools/testmon_subprocess_coverage.py', 'tools/test_settings.py', '.github/actions/run-tests/action.yml', 'docker/Dockerfile.base', 'docker/Dockerfile.curobo', '.github/workflows/config.yaml') }}- - name: Restore NGC CLI cache if: inputs.wheelhouse-resource != '' && env.NGC_API_KEY != '' uses: actions/cache@v4 @@ -298,10 +321,10 @@ runs: curobo-only: ${{ inputs.curobo-only }} quarantined-only: ${{ inputs.quarantined-only }} include-files: ${{ inputs.include-files }} - test-node-ids-file: ${{ inputs.test-node-ids-file }} - test-node-ids-key: ${{ inputs.test-node-ids-key }} volume-mount-source: ${{ github.workspace }} extra-pip-packages: ${{ inputs.extra-pip-packages }} + testmon-data-dir: .testmon/${{ inputs.container-name }} + testmon-mode: ${{ steps.testmon-mode.outputs.mode }} wheelhouse-host-dir: ${{ steps.extract-wheelhouse.outputs.wheelhouse_host_dir }} wheelhouse-packages: ${{ inputs.wheelhouse-packages }} omni-github-test-type: ${{ inputs.omni-github-test-type }} diff --git a/.github/actions/run-package-tests/select_testmon_mode.py b/.github/actions/run-package-tests/select_testmon_mode.py new file mode 100644 index 000000000000..c7b575b75984 --- /dev/null +++ b/.github/actions/run-package-tests/select_testmon_mode.py @@ -0,0 +1,39 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Choose whether Testmon can safely select affected tests for a change.""" + +from __future__ import annotations + +import re +import sys + +_RELEVANT = re.compile( + r"^(?:source|docker|tools|apps|scripts)/" + r"|^\.github/(?:workflows/[^/]+\.ya?ml|actions/)" + r"|^\.gitmodules$" + r"|^[^/]+\.(?:toml|yaml|yml|json|ini|cfg|conf|lock|sh|bat|ps1)$" +) +_IGNORED_SUFFIXES = (".md", ".rst", ".skip") + + +def _is_tracked_python(path: str) -> bool: + """Return whether ``path`` is a Python source file whose dependencies Testmon tracks. + + Python files under ``.github/`` (e.g. action helper scripts) run outside the pytest + process, so Testmon has no dependency data for them; changes to those files must fall + back to a full collection rather than the Python-only fast path. + """ + return path.endswith(".py") and not path.startswith(".github/") + + +def select_testmon_mode(paths: list[str]) -> str: + """Return ``select`` for tracked Python-only changes, otherwise ``collect``.""" + relevant = [path for path in paths if _RELEVANT.search(path) and not path.endswith(_IGNORED_SUFFIXES)] + return "select" if not relevant or all(_is_tracked_python(path) for path in relevant) else "collect" + + +if __name__ == "__main__": + print(select_testmon_mode([line.strip() for line in sys.stdin if line.strip()])) diff --git a/.github/actions/run-package-tests/test_select_testmon_mode.py b/.github/actions/run-package-tests/test_select_testmon_mode.py new file mode 100644 index 000000000000..2d33b4d2eb87 --- /dev/null +++ b/.github/actions/run-package-tests/test_select_testmon_mode.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for changed-file Testmon mode selection.""" + +import importlib.util +from pathlib import Path + +_MODULE_PATH = Path(__file__).with_name("select_testmon_mode.py") +_SPEC = importlib.util.spec_from_file_location("select_testmon_mode", _MODULE_PATH) +assert _SPEC and _SPEC.loader +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +select_testmon_mode = _MODULE.select_testmon_mode + + +def test_python_only_relevant_changes_select_affected_tests() -> None: + assert select_testmon_mode(["source/isaaclab/isaaclab/app.py", "tools/helper.py"]) == "select" + + +def test_irrelevant_changes_select_no_affected_tests() -> None: + assert select_testmon_mode(["docs/guide.md", "source/isaaclab/changelog.d/change.skip"]) == "select" + + +def test_static_relevant_change_collects_full_suite() -> None: + assert select_testmon_mode(["source/isaaclab/config/extension.toml"]) == "collect" + + +def test_mixed_change_collects_full_suite() -> None: + assert select_testmon_mode(["source/isaaclab/code.py", "docker/Dockerfile.base"]) == "collect" + + +def test_workflow_yaml_change_collects_full_suite() -> None: + # A workflow change that testmon cannot reason about must run the full suite, + # otherwise a workflow-only PR would trigger a job that deselects every test. + assert select_testmon_mode([".github/workflows/install-ci.yml"]) == "collect" + + +def test_action_python_change_collects_full_suite() -> None: + # Python helpers under .github/actions/ run outside pytest, so testmon has no + # dependency data for them; a change to one must run the full suite rather than + # taking the Python-only fast path and deselecting every test. + assert select_testmon_mode([".github/actions/run-package-tests/select_testmon_mode.py"]) == "collect" diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index fbc7043cdf8f..9f5a51dc8029 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -61,14 +61,6 @@ inputs: description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)' default: '' required: false - test-node-ids-file: - description: 'TOML file containing exact pytest node IDs to run' - default: '' - required: false - test-node-ids-key: - description: 'Top-level key in test-node-ids-file containing the node IDs for this job' - default: '' - required: false shard-index: description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)' default: '' @@ -85,6 +77,13 @@ inputs: description: 'Space-separated pip packages to install inside the Docker container before pytest starts' default: '' required: false + testmon-data-dir: + description: 'Workspace-relative directory for the Testmon dependency database' + default: '' + required: false + testmon-mode: + description: 'Testmon mode: select affected tests, collect a full baseline, or off' + default: 'off' wheelhouse-host-dir: description: 'Host directory containing wheelhouse/ and manifest.json for offline pip installs' default: '' @@ -126,10 +125,10 @@ runs: local shard_count="${13}" local volume_mount_source="${14}" local extra_pip_packages="${15}" - local test_node_ids_file="${16}" - local test_node_ids_key="${17}" - local wheelhouse_host_dir="${18}" - local wheelhouse_packages="${19}" + local wheelhouse_host_dir="${16}" + local wheelhouse_packages="${17}" + local testmon_data_dir="${18}" + local testmon_mode="${19}" local test_k_expr="${20}" local logs_pid="" local wait_pid="" @@ -156,6 +155,9 @@ runs: if [ -n "$extra_pip_packages" ]; then echo "With extra pip packages: $extra_pip_packages" fi + if [ "$testmon_mode" != "off" ]; then + echo "Testmon mode: $testmon_mode" + fi if [ -n "$wheelhouse_host_dir" ]; then echo "With wheelhouse host directory: $wheelhouse_host_dir" fi @@ -174,25 +176,10 @@ runs: if [ -n "$include_files" ]; then echo "Include files: $include_files" fi - if [ -n "$test_node_ids_file" ]; then - echo "Test node IDs file: $test_node_ids_file" - fi - if [ -n "$test_node_ids_key" ]; then - echo "Test node IDs key: $test_node_ids_key" - fi if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then echo "Shard: $shard_index of $shard_count" fi - if [ -n "$test_node_ids_file" ] || [ -n "$test_node_ids_key" ]; then - if [ -z "$test_node_ids_file" ] || [ -z "$test_node_ids_key" ]; then - echo "Both test-node-ids-file and test-node-ids-key must be set together" - return 1 - fi - export TEST_NODE_IDS_FILE="$test_node_ids_file" - export TEST_NODE_IDS_KEY="$test_node_ids_key" - fi - # Create reports directory mkdir -p "$reports_dir" @@ -234,11 +221,6 @@ runs: echo "Setting TEST_NODE_IDS" fi - if [ -n "${TEST_NODE_IDS_FILE:-}" ]; then - docker_env_vars="$docker_env_vars -e TEST_NODE_IDS_FILE -e TEST_NODE_IDS_KEY" - echo "Setting TEST_NODE_IDS_FILE=$TEST_NODE_IDS_FILE TEST_NODE_IDS_KEY=$TEST_NODE_IDS_KEY" - fi - if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count" echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count" @@ -273,6 +255,24 @@ runs: docker_env_vars="$docker_env_vars -e TEST_EXTRA_PIP_PACKAGES" fi + testmon_options="" + if [ -n "$testmon_data_dir" ]; then + mkdir -p "$testmon_data_dir" + docker_env_vars="$docker_env_vars -e TESTMON_DATAFILE=/workspace/isaaclab/$testmon_data_dir/.testmondata" + case "$testmon_mode" in + select) + if [ -s "$testmon_data_dir/.testmondata" ]; then + testmon_options="--testmon --testmon-forceselect" + else + echo "::warning::Testmon data is missing; collecting a full baseline" + testmon_options="--testmon --testmon-noselect" + fi + ;; + collect) testmon_options="--testmon --testmon-noselect" ;; + off) ;; + *) echo "Unknown Testmon mode: $testmon_mode"; return 1 ;; + esac + fi if [ -n "$test_k_expr" ]; then export TEST_K_EXPR="$test_k_expr" docker_env_vars="$docker_env_vars -e TEST_K_EXPR" @@ -383,7 +383,7 @@ runs: # set this detect-only flag, which makes cold asset downloads # fall back to slow repeated retries. unset HUB__ARGS__DETECT_ONLY - ./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky \"coverage>=7.6.1\" + ./isaaclab.sh -p -m pip install pytest pytest-mock \"pytest-testmon>=2.2,<3\" junitparser flatdict flaky \"coverage>=7.6.1\" if [ -n \"\${TEST_WHEELHOUSE_PACKAGES:-}\" ]; then if [ ! -d \"\${TEST_WHEELHOUSE_PATH:-}\" ]; then echo \"Wheelhouse path is missing: \${TEST_WHEELHOUSE_PATH:-}\" @@ -415,7 +415,7 @@ runs: esac fi echo 'Starting pytest with path: $test_path' - ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file + ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options $testmon_options -v --junitxml=tests/$result_file " # Stream container logs in background. @@ -512,7 +512,7 @@ runs: } # Call the function with provided parameters - run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" + run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "${{ inputs.testmon-data-dir }}" "${{ inputs.testmon-mode }}" "$TEST_K_EXPR_INPUT" - name: Kill container on cancellation if: cancelled() diff --git a/.github/test-subsets/postmerge-rendering.toml b/.github/test-subsets/postmerge-rendering.toml deleted file mode 100644 index ee52d8f1f548..000000000000 --- a/.github/test-subsets/postmerge-rendering.toml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Stable rendering correctness tests used by post-merge CI. -rendering-correctness = [ - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[newton-isaacsim_rtx-semantic_segmentation]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-isaacsim_rtx-semantic_segmentation]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole.py::test_rendering_cartpole[physx-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-albedo-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-depth-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-None-cartpole]", - "source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py::test_rendering_registered_tasks[Isaac-Cartpole-Camera-Direct-rgb-cartpole]", -] - -rendering-correctness-kitless = [ - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-ovrtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[newton-ovrtx-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-newton_warp-depth]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-newton_warp-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-albedo]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-rgb]", - "source/isaaclab_tasks/test/core/test_rendering_cartpole_kitless.py::test_rendering_cartpole_kitless[ovphysx-ovrtx-semantic_segmentation]", -] diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 39f9ffab1b2c..2750e8851195 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -105,12 +105,11 @@ jobs: $'^\\.github/workflows/build\\.yaml$\tThis workflow file' $'^\\.github/workflows/config\\.yaml$\tBase image config' $'^\\.github/actions/\tCI actions' - $'^\\.github/test-subsets/\tCI test subset config' ) if [ "$EVENT_NAME" = "push" ]; then - triggered_jobs="Docker base build job + rendering-correctness + rendering-correctness-kitless" + triggered_jobs="Docker build jobs + all test-* matrix jobs (full suite via Testmon collect)" else - triggered_jobs="Docker build jobs + all test-* matrix jobs" + triggered_jobs="Docker build jobs + all test-* matrix jobs (Testmon-selected subset)" fi render_table() { @@ -255,7 +254,6 @@ jobs: runs-on: [self-hosted, gpu] needs: [changes, config] if: >- - github.event_name != 'push' && needs.changes.outputs.run_docker_tests == 'true' steps: - name: Checkout Code @@ -283,7 +281,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -308,7 +305,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -333,7 +329,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -357,7 +352,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -380,7 +374,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -403,7 +396,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -426,7 +418,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -448,7 +439,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -469,7 +459,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -490,7 +479,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -511,7 +499,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -532,7 +519,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -553,7 +539,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -574,7 +559,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -595,7 +579,6 @@ jobs: timeout-minutes: 180 needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' env: USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_resource != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }} @@ -652,7 +635,6 @@ jobs: continue-on-error: true needs: [build-curobo, config] if: >- - github.event_name != 'push' && needs.build-curobo.result == 'success' steps: - uses: actions/checkout@v6 @@ -702,7 +684,6 @@ jobs: continue-on-error: true needs: [build-curobo, config] if: >- - github.event_name != 'push' && needs.build-curobo.result == 'success' steps: - uses: actions/checkout@v6 @@ -726,7 +707,6 @@ jobs: continue-on-error: true needs: [build, config] if: >- - github.event_name != 'push' && needs.build.result == 'success' steps: - uses: actions/checkout@v6 @@ -767,8 +747,6 @@ jobs: test_rendering_dexsuite_kuka_hetero.py, test_rendering_registered_tasks.py, test_rendering_shadow_hand.py - test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} - test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness' || '' }} container-name: isaac-lab-rendering-correctness-test omni-github-test-type: rendering-correctness @@ -804,8 +782,6 @@ jobs: test_rendering_dexsuite_kuka_homo_kitless.py, test_rendering_dexsuite_kuka_hetero_kitless.py, test_rendering_shadow_hand_kitless.py - test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }} - test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless' || '' }} container-name: isaac-lab-rendering-correctness-kitless-test omni-github-test-type: rendering-correctness-kitless #endregion diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 226222d425bd..c21d3fcc4e27 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -37,7 +37,9 @@ jobs: runs-on: ubuntu-latest outputs: run_install_tests: ${{ steps.detect.outputs.run_install_tests }} + testmon_mode: ${{ steps.detect.outputs.testmon_mode }} steps: + - uses: actions/checkout@v6 - id: detect env: GH_TOKEN: ${{ github.token }} @@ -94,9 +96,10 @@ jobs: } decide() { - local decision="$1" reason="$2" files="${3:-}" + local decision="$1" reason="$2" files="${3:-}" testmon_mode="${4:-collect}" echo "Decision: run_install_tests=$decision ($reason)" echo "run_install_tests=$decision" >> "$GITHUB_OUTPUT" + echo "testmon_mode=$testmon_mode" >> "$GITHUB_OUTPUT" { if [ -n "$files" ]; then render_table "$files" @@ -119,7 +122,8 @@ jobs: printf '%s\n' "$changed_files" if any_match "$changed_files"; then - decide true "relevant paths changed" "$changed_files" + testmon_mode="$(printf '%s\n' "$changed_files" | python3 .github/actions/run-package-tests/select_testmon_mode.py)" + decide true "relevant paths changed" "$changed_files" "$testmon_mode" else decide false "no relevant paths changed" "$changed_files" fi @@ -133,6 +137,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Restore Testmon Data + uses: actions/cache@v4 + with: + path: .testmon/${{ runner.arch }} + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: @@ -141,6 +151,8 @@ jobs: uses: ./.github/actions/install-ci-run with: base-image: ${{ inputs.base_image || 'ubuntu:24.04' }} + testmon-data-dir: ${{ github.workspace }}/.testmon/${{ runner.arch }} + testmon-mode: ${{ needs.changes.outputs.testmon_mode }} test-filter: ${{ inputs.test_filter }} install-tests-arm: @@ -152,6 +164,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Restore Testmon Data + uses: actions/cache@v4 + with: + path: .testmon/${{ runner.arch }} + key: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}-${{ github.sha }} + restore-keys: install-ci-testmon-${{ runner.arch }}-${{ hashFiles('conftest.py', 'tools/run_install_ci.py', 'tools/testmon_subprocess_coverage.py', 'tools/testmon_coverage_context.py', 'source/isaaclab/test/install_ci/conftest.py', 'source/isaaclab/test/install_ci/Dockerfile.installci', 'source/isaaclab/test/install_ci/pytest.ini') }}- - name: Show Collected Tests uses: ./.github/actions/install-ci-collect with: @@ -160,4 +178,6 @@ jobs: uses: ./.github/actions/install-ci-run with: base-image: ${{ inputs.base_image || 'ubuntu:24.04' }} + testmon-data-dir: ${{ github.workspace }}/.testmon/${{ runner.arch }} + testmon-mode: ${{ needs.changes.outputs.testmon_mode }} test-filter: ${{ inputs.test_filter }} diff --git a/.gitignore b/.gitignore index d78a978c60a2..34b836740b17 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ **/*.egg-info/ **/__pycache__/ **/.pytest_cache/ +**/.testmon/ +**/.testmondata +**/.tmontmp/ **/*.pyc **/*.pb diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000000..df668e71d427 --- /dev/null +++ b/conftest.py @@ -0,0 +1,47 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared pytest configuration for repository tests. + +Wires the Testmon subprocess-coverage hooks (see :mod:`tools.testmon_subprocess_coverage`) +into the session so that code executed in child Python processes is attributed to the +active test. The hooks are no-ops unless pytest-testmon is collecting coverage. +""" + +import contextlib +import sys +from collections.abc import Generator +from pathlib import Path + +import pytest + + +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> Generator[None, None, None]: + """Keep tests marked ``smoke`` selected when Testmon filters the collection.""" + always_items = [item for item in items if item.get_closest_marker("smoke") is not None] + yield + + if config.getoption("testmon_forceselect", default=False): + selected = set(items) + items.extend(item for item in always_items if item not in selected) + + +_TOOLS_DIR = Path(__file__).resolve().parent / "tools" +if _TOOLS_DIR.is_dir(): + if str(_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(_TOOLS_DIR)) + + # Guard the import too: when ``tools/`` is absent (partial checkout, artefact-only + # CI image) the hooks should stay unregistered no-ops rather than aborting the whole + # test collection with a ``ModuleNotFoundError``. + with contextlib.suppress(ImportError): + from testmon_subprocess_coverage import ( # noqa: F401 + pytest_runtest_makereport, + pytest_runtest_setup, + pytest_runtest_teardown, + pytest_sessionfinish, + pytest_sessionstart, + ) diff --git a/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip b/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip new file mode 100644 index 000000000000..71f945ce9f49 --- /dev/null +++ b/source/isaaclab/changelog.d/mataylor-testmon-subprocess-coverage.skip @@ -0,0 +1,3 @@ +Test/tooling only: add Testmon subprocess coverage tracking so tests that +spawn child Python processes get reselected when their subprocess-only +dependencies change. diff --git a/source/isaaclab/changelog.d/test-selection-rules.skip b/source/isaaclab/changelog.d/test-selection-rules.skip new file mode 100644 index 000000000000..9f59b1e1f3f3 --- /dev/null +++ b/source/isaaclab/changelog.d/test-selection-rules.skip @@ -0,0 +1 @@ +CI and test-selection changes only. diff --git a/source/isaaclab/isaaclab/utils/array.py b/source/isaaclab/isaaclab/utils/array.py index d15fbc275dc7..7d514e4ee49e 100644 --- a/source/isaaclab/isaaclab/utils/array.py +++ b/source/isaaclab/isaaclab/utils/array.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for working with different array backends.""" +"""Utilities for working consistently with different array backends.""" # needed to import for allowing type-hinting: torch.device | str | None from __future__ import annotations diff --git a/source/isaaclab/isaaclab/utils/string.py b/source/isaaclab/isaaclab/utils/string.py index fa83688154d6..dba6104c916f 100644 --- a/source/isaaclab/isaaclab/utils/string.py +++ b/source/isaaclab/isaaclab/utils/string.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Sub-module containing utilities for transforming strings and regular expressions.""" +"""Utilities for transforming strings and regular-expression patterns.""" import ast import functools diff --git a/source/isaaclab/test/install_ci/Dockerfile.installci b/source/isaaclab/test/install_ci/Dockerfile.installci index 68b127df7b34..8178b94621a7 100644 --- a/source/isaaclab/test/install_ci/Dockerfile.installci +++ b/source/isaaclab/test/install_ci/Dockerfile.installci @@ -52,7 +52,9 @@ RUN ln -sf /usr/bin/python3 /usr/bin/python # Install test runner dependencies into the system Python RUN pip install --break-system-packages \ pytest>=8.0 \ - pytest-timeout>=2.0 + "pytest-testmon>=2.2,<3" \ + pytest-timeout>=2.0 \ + "coverage>=7.6.1" # Install uv system-wide so non-root users can invoke it without PATH trickery RUN curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh diff --git a/source/isaaclab/test/test_testmon_subprocess_coverage.py b/source/isaaclab/test/test_testmon_subprocess_coverage.py new file mode 100644 index 000000000000..65852d8de04a --- /dev/null +++ b/source/isaaclab/test/test_testmon_subprocess_coverage.py @@ -0,0 +1,151 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Regression tests for Testmon subprocess dependency tracking. + +Spins up an isolated pytest-in-pytest project whose only link to a helper module +is a child ``subprocess``. Verifies that :mod:`testmon_subprocess_coverage` +records the helper as a dependency, so editing it reselects the test. +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _find_tools_dir() -> Path: + """Locate the repository ``tools/`` directory that holds the Testmon helpers.""" + for parent in Path(__file__).resolve().parents: + candidate = parent / "tools" / "testmon_subprocess_coverage.py" + if candidate.is_file(): + return candidate.parent + raise RuntimeError("could not locate tools/testmon_subprocess_coverage.py") + + +_TOOLS_DIR = _find_tools_dir() + +_HELPER_MODULE = "subprocess_only_helper" + +# A test whose sole link to the helper is a child ``python -c "..."`` process. +# The helper is never imported in the pytest process, so only subprocess +# coverage can register it as a Testmon dependency. +_TEST_BODY = """\ +import os +import subprocess +import sys + + +def test_calls_helper_in_subprocess(): + # Record that this test actually executed (the harness counts these lines). + with open(os.environ["SUBPROC_DEP_RUNS"], "a", encoding="utf-8") as handle: + handle.write("run\\n") + + result = subprocess.run( + [sys.executable, "-c", "import {module}; assert {module}.contribution() >= 0"], + env=os.environ.copy(), + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr +""" + +_CONFTEST_BODY = """\ +from testmon_subprocess_coverage import ( # noqa: F401 + pytest_runtest_makereport, + pytest_runtest_setup, + pytest_runtest_teardown, + pytest_sessionfinish, + pytest_sessionstart, +) +""" + + +def _dependencies_available() -> bool: + return all(importlib.util.find_spec(name) is not None for name in ("testmon", "coverage")) + + +def _write_helper(project: Path, value: int) -> None: + (project / f"{_HELPER_MODULE}.py").write_text( + f"def contribution():\n return {value}\n", + encoding="utf-8", + ) + + +def _clean_env(project: Path, datafile: Path, runs_file: Path) -> dict[str, str]: + """Environment for a nested pytest run, scrubbed of inherited coverage state.""" + env = os.environ.copy() + # Drop any coverage/testmon state leaking in from the outer pytest session so + # the child session manages its own coverage lifecycle. + for key in ("COVERAGE_PROCESS_START", "COVERAGE_CONTEXT", "COVERAGE_FILE"): + env.pop(key, None) + env["PYTHONPATH"] = os.pathsep.join([str(_TOOLS_DIR), str(project), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + env["TESTMON_DATAFILE"] = str(datafile) + env["SUBPROC_DEP_RUNS"] = str(runs_file) + return env + + +def _run_pytest(project: Path, env: dict[str, str], *testmon_args: str) -> subprocess.CompletedProcess[str]: + cmd = [ + sys.executable, + "-m", + "pytest", + "-o", + "addopts=", + "--testmon", + *testmon_args, + "-q", + str(project), + ] + return subprocess.run(cmd, cwd=project, env=env, capture_output=True, text=True, timeout=300) + + +def _run_count(runs_file: Path) -> int: + if not runs_file.is_file(): + return 0 + return len([line for line in runs_file.read_text(encoding="utf-8").splitlines() if line.strip()]) + + +@pytest.mark.skipif(not _dependencies_available(), reason="pytest-testmon and coverage are required") +def test_editing_subprocess_only_dependency_reselects_test(tmp_path: Path) -> None: + """Testmon reselects a test when a dependency reached only via a subprocess changes.""" + project = tmp_path / "proj" + project.mkdir() + (project / f"test_{_HELPER_MODULE}.py").write_text(_TEST_BODY.format(module=_HELPER_MODULE), encoding="utf-8") + (project / "conftest.py").write_text(_CONFTEST_BODY, encoding="utf-8") + _write_helper(project, value=1) + + datafile = project / ".testmondata" + runs_file = project / "runs.txt" + env = _clean_env(project, datafile, runs_file) + + # 1. Prime Testmon: run everything and record dependencies (incl. subprocess). + collect = _run_pytest(project, env, "--testmon-noselect") + assert collect.returncode == 0, f"collect run failed:\n{collect.stdout}\n{collect.stderr}" + assert datafile.is_file(), "Testmon did not create its data file during collection" + assert _run_count(runs_file) == 1, f"expected the test to run once during collection:\n{collect.stdout}" + + # 2. Nothing changed -> Testmon must deselect the test (it does not run again). + unchanged = _run_pytest(project, env) + assert _run_count(runs_file) == 1, ( + "Testmon reran the test even though nothing changed; deselection is not working.\n" + f"{unchanged.stdout}\n{unchanged.stderr}" + ) + + # 3. Edit the helper that is only reachable through the subprocess. + _write_helper(project, value=2) + + # 4. Testmon must notice the subprocess-only dependency changed and reselect. + changed = _run_pytest(project, env) + assert _run_count(runs_file) == 2, ( + "Testmon did not reselect the test after its subprocess-only dependency changed; " + "subprocess coverage is not being attributed to the test.\n" + f"{changed.stdout}\n{changed.stderr}" + ) diff --git a/tools/conftest.py b/tools/conftest.py index 963853194f3d..ea5ff614ca82 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -13,7 +13,6 @@ from dataclasses import dataclass import pytest -import tomllib from junitparser import Error, JUnitXml, TestCase, TestSuite from prettytable import PrettyTable @@ -878,37 +877,9 @@ def _collect_test_files( return test_files -def _load_test_node_ids_from_toml(workspace_root: str) -> list[str]: - """Load exact pytest node IDs from a TOML file configured in the environment.""" - node_ids_file = os.environ.get("TEST_NODE_IDS_FILE") - node_ids_key = os.environ.get("TEST_NODE_IDS_KEY") - if not (node_ids_file or node_ids_key): - return [] - if not (node_ids_file and node_ids_key): - pytest.exit("Both TEST_NODE_IDS_FILE and TEST_NODE_IDS_KEY must be set together", returncode=1) - - path = node_ids_file if os.path.isabs(node_ids_file) else os.path.join(workspace_root, node_ids_file) - - try: - with open(os.path.normpath(path), "rb") as stream: - node_ids = tomllib.load(stream).get(node_ids_key) - except OSError as exc: - pytest.exit(f"Could not read TEST_NODE_IDS_FILE {node_ids_file!r}: {exc}", returncode=1) - except tomllib.TOMLDecodeError as exc: - pytest.exit(f"{node_ids_file}: invalid TOML: {exc}", returncode=1) - - if not node_ids: - pytest.exit(f"{node_ids_key!r} not found or empty in {node_ids_file}", returncode=1) - if not isinstance(node_ids, list) or not all(isinstance(node_id, str) for node_id in node_ids): - pytest.exit(f"{node_ids_key!r} must be a TOML array of strings in {node_ids_file}", returncode=1) - - return node_ids - - def _collect_test_node_ids_by_file(workspace_root: str) -> dict[str, list[str]]: """Group exact pytest node IDs by absolute test file path.""" node_ids = [line.strip() for line in os.environ.get("TEST_NODE_IDS", "").splitlines() if line.strip()] - node_ids.extend(_load_test_node_ids_from_toml(workspace_root)) if len(node_ids) != len(set(node_ids)): pytest.exit("Configured test node IDs contain duplicates", returncode=1) @@ -989,8 +960,6 @@ def pytest_sessionstart(session): print(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") print(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") print(f"TEST_NODE_IDS env var: '{'SET' if os.environ.get('TEST_NODE_IDS') else 'NOT_SET'}'") - print(f"TEST_NODE_IDS_FILE env var: '{os.environ.get('TEST_NODE_IDS_FILE', 'NOT_SET')}'") - print(f"TEST_NODE_IDS_KEY env var: '{os.environ.get('TEST_NODE_IDS_KEY', 'NOT_SET')}'") print(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") print(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") print("=" * 50) diff --git a/tools/run_install_ci.py b/tools/run_install_ci.py index 5b48dc90ab45..6b67011b70f8 100755 --- a/tools/run_install_ci.py +++ b/tools/run_install_ci.py @@ -317,6 +317,25 @@ def _cmd_docker(args: argparse.Namespace) -> int: if not args.no_uv_cache: docker_run_cmd.extend(["-v", "isaaclab-install-ci-uv-cache:/home/isaaclab/.cache/uv"]) + if args.testmon_data_dir: + testmon_data_dir = Path(args.testmon_data_dir).resolve() + testmon_data_dir.mkdir(parents=True, exist_ok=True) + # The container runs as the non-root 'isaaclab' user (uid 1000). SQLite + # opens the database in WAL mode, so it must also write the sibling + # ``.testmondata-wal``/``.testmondata-shm`` files restored from the cache. + # World-write the directory and every file inside it (not just + # ``.testmondata``); otherwise a restored sibling with default 0644 + # perms makes SQLite report "attempt to write a readonly database". + try: + testmon_data_dir.chmod(0o777) + for entry in testmon_data_dir.iterdir(): + if entry.is_file(): + entry.chmod(0o666) + except OSError as exc: + print(f"Warning: could not chmod testmon data dir {testmon_data_dir}: {exc}", file=sys.stderr) + docker_run_cmd.extend(["-v", f"{testmon_data_dir}:/tmp/testmon"]) + docker_run_cmd.extend(["-e", "TESTMON_DATAFILE=/tmp/testmon/.testmondata"]) + # Pass environment variables docker_run_cmd.extend(["-e", "OMNI_KIT_ACCEPT_EULA=Y"]) docker_run_cmd.extend(["-e", "ACCEPT_EULA=Y"]) @@ -408,6 +427,8 @@ def main() -> int: --no-pip-cache Disable persistent pip cache volume --no-uv-cache Disable persistent uv cache volume --results-dir DIR Host directory for test results (auto-adds --junitxml) + --testmon-data-dir DIR + Host directory persisted as the Testmon dependency database --wheel PATH Path to pre-built isaaclab wheel file --build-wheel Build the isaaclab wheel and pass it to tests (mutually exclusive with --wheel) --debug Stream wheel-build output live (default: quiet, only dumped on failure) @@ -453,6 +474,9 @@ def main() -> int: docker_p.add_argument( "--results-dir", type=str, default=None, help="Host directory for test results (auto-adds --junitxml)" ) + docker_p.add_argument( + "--testmon-data-dir", type=str, default=None, help="Host directory for the Testmon dependency database" + ) docker_p.add_argument("--wheel", type=str, default=None, help="Path to pre-built isaaclab wheel file") docker_p.add_argument( "--build-wheel", diff --git a/tools/testmon_coverage_context.py b/tools/testmon_coverage_context.py new file mode 100644 index 000000000000..1ddc27225360 --- /dev/null +++ b/tools/testmon_coverage_context.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Coverage plugin that tags child-process lines with the active pytest node id.""" + +from __future__ import annotations + +import os +from types import FrameType + +from coverage import CoveragePlugin + + +class NodeIdContextPlugin(CoveragePlugin): + """Report the active pytest node id as coverage's dynamic context.""" + + def dynamic_context(self, frame: FrameType) -> str | None: + return os.environ.get("COVERAGE_CONTEXT") or None + + +def coverage_init(reg, options) -> None: + reg.add_dynamic_context(NodeIdContextPlugin()) diff --git a/tools/testmon_subprocess_coverage.py b/tools/testmon_subprocess_coverage.py new file mode 100644 index 000000000000..0d4c91486b61 --- /dev/null +++ b/tools/testmon_subprocess_coverage.py @@ -0,0 +1,266 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Testmon subprocess coverage helpers. + +pytest-testmon 2.x only tracks coverage in the pytest process. Tests that spawn +child Python processes (for example ``./isaaclab.sh -p scripts/.../train.py``) +need: + +1. A ``sitecustomize.py`` shim on ``PYTHONPATH`` so child interpreters call + :func:`coverage.process_startup` at startup. This is written into a + repository-local temporary directory rather than ``site-packages`` so it + works even when ``site-packages`` is read-only (for example inside CI + containers running as a non-root user). +2. ``COVERAGE_PROCESS_START`` pointing at a config with ``parallel = true`` and + the :mod:`testmon_coverage_context` plugin, so lines are tagged with the + active test's node id. +3. Child data merged into Testmon's in-memory coverage via + :meth:`coverage.CoverageData.update` before each batch is read. + +Import the pytest hook functions into ``conftest.py``. +""" + +from __future__ import annotations + +import os +import shutil +import sysconfig +import warnings +from dataclasses import dataclass +from pathlib import Path + +import pytest + +_TMONTMP_DIR = ".tmontmp" +_COVERAGE_PROCESS_START = "COVERAGE_PROCESS_START" +_COVERAGE_CONTEXT = "COVERAGE_CONTEXT" +_STATE_KEY = pytest.StashKey["SubprocessCoverageState"]() + +# ``sitecustomize`` is imported unconditionally by :mod:`site` at interpreter +# startup (unlike ``usercustomize``, which is gated on ``ENABLE_USER_SITE``), so +# a ``sitecustomize.py`` placed first on ``PYTHONPATH`` reliably runs in every +# child interpreter. ``coverage.process_startup`` is a no-op unless +# ``COVERAGE_PROCESS_START`` is set, so the shim is harmless outside collection. +_SHIM_FILENAME = "sitecustomize.py" +_SHIM_CONTENT = ( + "# Auto-generated by testmon_subprocess_coverage to attribute child-process\n" + "# coverage back to the active test. No-op unless COVERAGE_PROCESS_START is set.\n" + "try:\n" + " import coverage\n" + " coverage.process_startup()\n" + "except Exception:\n" + " pass\n" +) + + +@dataclass(frozen=True) +class SubprocessCoverageState: + """Session state for subprocess coverage collection.""" + + rc_path: Path + data_prefix: Path + shim_dir: Path | None + enabled: bool + prev_pythonpath: str | None = None + + +def _python_lib_omit_patterns() -> list[str]: + return [os.path.join(value, "*") for key, value in sysconfig.get_paths().items() if key.endswith("lib")] + + +def _is_testmon_collecting(config: pytest.Config) -> bool: + tm_conf = getattr(config, "testmon_config", None) + return tm_conf is not None and tm_conf.collect + + +def _install_subprocess_shim(shim_dir: Path) -> Path | None: + """Write a ``sitecustomize.py`` ``process_startup`` shim into ``shim_dir``. + + Returns the shim directory (to be prepended to ``PYTHONPATH``) or ``None`` + if :mod:`coverage` is unavailable or the directory cannot be written. + """ + try: + import coverage # noqa: F401 + except ImportError: + return None + + try: + shim_dir.mkdir(parents=True, exist_ok=True) + (shim_dir / _SHIM_FILENAME).write_text(_SHIM_CONTENT, encoding="utf-8") + except OSError: + return None + return shim_dir + + +def setup_subprocess_coverage(config: pytest.Config) -> SubprocessCoverageState | None: + """Configure ``COVERAGE_PROCESS_START`` for the pytest session.""" + if not _is_testmon_collecting(config): + return None + + rootdir = Path(config.rootpath) + tmontmp = rootdir / _TMONTMP_DIR + tmontmp.mkdir(exist_ok=True) + + shim_dir = _install_subprocess_shim(tmontmp / f"shim-{os.getpid()}") + if shim_dir is None: + warnings.warn( + "testmon subprocess coverage: could not install a process_startup sitecustomize shim, " + "so code executed in child Python processes will not be tracked.", + stacklevel=1, + ) + state = SubprocessCoverageState(rc_path=Path(), data_prefix=Path(), shim_dir=None, enabled=False) + config.stash[_STATE_KEY] = state + return state + + # Prepend the shim directory (so child interpreters import our + # ``sitecustomize``) and ``tools/`` (so the coveragerc can load + # :mod:`testmon_coverage_context`). + tools_dir = rootdir / "tools" + prev_pythonpath = os.environ.get("PYTHONPATH") + pythonpath_entries = [str(shim_dir), str(tools_dir)] + if prev_pythonpath: + pythonpath_entries.append(prev_pythonpath) + os.environ["PYTHONPATH"] = os.pathsep.join(pythonpath_entries) + + data_prefix = tmontmp / f"subprocess-{os.getpid()}" + rc_path = tmontmp / f"subprocess-{os.getpid()}.coveragerc" + rc_path.write_text( + "\n".join( + [ + "[run]", + f"data_file = {data_prefix}", + "parallel = true", + "plugins = testmon_coverage_context", + "include =", + f" {os.path.join(str(rootdir), '*')}", + "omit =", + # The Testmon scratch dir (including our sitecustomize shim) lives + # under ``rootdir`` but must not be tracked as a test dependency; + # its per-PID path changes every run and would force reselection. + f" {os.path.join(str(tmontmp), '*')}", + *(f" {pattern}" for pattern in _python_lib_omit_patterns()), + "", + ] + ), + encoding="utf-8", + ) + os.environ[_COVERAGE_PROCESS_START] = str(rc_path.resolve()) + state = SubprocessCoverageState( + rc_path=rc_path, + data_prefix=data_prefix, + shim_dir=shim_dir, + enabled=True, + prev_pythonpath=prev_pythonpath, + ) + config.stash[_STATE_KEY] = state + return state + + +def _child_data_files(data_prefix: Path) -> list[Path]: + if not data_prefix.parent.is_dir(): + return [] + return sorted(p for p in data_prefix.parent.glob(f"{data_prefix.name}.*") if p.is_file()) + + +def combine_subprocess_coverage(config: pytest.Config) -> None: + """Merge child-process coverage into Testmon's in-memory coverage data.""" + state = config.stash.get(_STATE_KEY, None) + if state is None or not state.enabled: + return + + child_files = _child_data_files(state.data_prefix) + if not child_files: + return + + collect_plugin = config.pluginmanager.get_plugin("TestmonCollect") + if collect_plugin is None: + return + + # ``collect_plugin.testmon.cov`` is a private chain that could change on a + # future testmon release; fall back to ``None`` rather than raising + # ``AttributeError`` so a rename simply disables merging instead of breaking + # the whole test session. + testmon = getattr(collect_plugin, "testmon", None) + cov = getattr(testmon, "cov", None) + if cov is None: + return + + from coverage import CoverageData + + # ``Coverage`` exposes no public "is running" flag, so fall back to assuming + # it was running (the normal case while testmon collects) if the private + # attribute is unavailable, avoiding an ``AttributeError`` that would + # silently disable merging on a future coverage release. + was_started = getattr(cov, "_started", True) + if was_started: + cov.stop() + # Always restart coverage in ``finally`` so a merge failure cannot leave it + # stopped, which would silently record no dependencies for the rest of the + # session. + try: + target = cov.get_data() + for child_file in child_files: + child = CoverageData(basename=str(child_file)) + try: + child.read() + except Exception: + continue + try: + target.update(child) + except Exception: + continue + child_file.unlink(missing_ok=True) + finally: + if was_started: + cov.start() + + +def teardown_subprocess_coverage(config: pytest.Config) -> None: + """Remove temporary subprocess coverage configuration.""" + state = config.stash.get(_STATE_KEY, None) + if state is None: + return + + os.environ.pop(_COVERAGE_PROCESS_START, None) + os.environ.pop(_COVERAGE_CONTEXT, None) + if state.enabled: + if state.prev_pythonpath is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = state.prev_pythonpath + if state.enabled and state.rc_path.is_file(): + state.rc_path.unlink(missing_ok=True) + if state.enabled: + for leftover in _child_data_files(state.data_prefix): + leftover.unlink(missing_ok=True) + if state.shim_dir is not None and state.shim_dir.is_dir(): + shutil.rmtree(state.shim_dir, ignore_errors=True) + if _STATE_KEY in config.stash: + del config.stash[_STATE_KEY] + + +def pytest_sessionstart(session: pytest.Session) -> None: + setup_subprocess_coverage(session.config) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + if os.environ.get(_COVERAGE_PROCESS_START): + os.environ[_COVERAGE_CONTEXT] = item.nodeid + + +def pytest_runtest_teardown(item: pytest.Item) -> None: + os.environ.pop(_COVERAGE_CONTEXT, None) + + +@pytest.hookimpl(tryfirst=True, wrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]): + if call.when == "teardown": + combine_subprocess_coverage(item.config) + return (yield) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + teardown_subprocess_coverage(session.config)